From 476014bfe2ba9babef33bae8fd1feeb56c887928 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 001/133] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- Makefile | 1 + app/demo_adapter.py | 210 ++++++++++++++++++++++- app/main.py | 33 +++- app/routers/account/account.py | 8 + app/routers/compute/compute.py | 4 +- app/routers/error_handlers.py | 6 + app/routers/facility/__init__.py | 0 app/routers/facility/facility.py | 77 +++++++++ app/routers/facility/facility_adapter.py | 65 +++++++ app/routers/facility/models.py | 58 +++++++ app/routers/iri_router.py | 36 +++- 11 files changed, 484 insertions(+), 14 deletions(-) create mode 100644 app/routers/facility/__init__.py create mode 100644 app/routers/facility/facility.py create mode 100644 app/routers/facility/facility_adapter.py create mode 100644 app/routers/facility/models.py diff --git a/Makefile b/Makefile index 5bd90346..ba7552ab 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ dev : .venv @source ./.venv/bin/activate && \ + IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 9cbc7961..daf266d8 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -13,6 +13,7 @@ from pydantic import BaseModel from typing import Any, Tuple from fastapi import HTTPException +from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter from .routers.compute import models as compute_models, facility_adapter as compute_adapter @@ -40,9 +41,13 @@ def get_base_temp_dir(cls): return cls._base_temp_dir +def demo_uuid(kind: str, name: str) -> str: + return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, - task_adapter.FacilityAdapter): + task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): def __init__(self): self.resources = [] self.incidents = [] @@ -52,11 +57,83 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - + self.locations = [] + self.facility = {} + self.sites = [] self._init_state() def _init_state(self): + now = datetime.datetime.now(datetime.timezone.utc) + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + + facility = facility_models.Facility( + id=demo_uuid("facility", "demo_facility"), + name="Demo Facility", + description="A demo facility for testing the IRI Facility API", + last_modified=now, + short_name="DEMO", + organization_name="Demo Organization", + support_uri="https://support.demo.example", + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], + ) + + self.facility = facility + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + + day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -182,6 +259,135 @@ def _init_state(self): d += datetime.timedelta(minutes=int(random.random() * 15 + 1)) + # ---------------------------- + # Facility API + # ---------------------------- + + async def get_facility( + self: "DemoAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility: + return self.facility + + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + # ---------------------------- + # Status API + # ---------------------------- async def get_resources( self : "DemoAdapter", diff --git a/app/main.py b/app/main.py index be5feaaf..080645ef 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,11 @@ """Main API application""" import logging from fastapi import FastAPI +from fastapi import Request +from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers +from app.routers.facility import facility from app.routers.status import status from app.routers.account import account from app.routers.compute import compute @@ -19,11 +22,39 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" +@APP.get(api_prefix) +async def api_discovery(request: Request): + base = str(request.base_url).rstrip("/") + items = [] + for route in APP.router.routes: + if not isinstance(route, APIRoute): + continue + # skip docs & openapi + if route.path.startswith("/docs") or route.path.startswith("/openapi"): + continue + for method in route.methods: + if method == "HEAD" or method == "OPTIONS": + continue + items.append({ + "id": route.name or f"{method}_{route.path}", + "method": method, + "path": route.path, + "_links": [ + { + "rel": "self", + "href": f"{base.rstrip('/')}{route.path}", + "type": "application/json" + } + ] + }) + return items + # Attach routers under the prefix +APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) APP.include_router(account.router, prefix=api_prefix) APP.include_router(compute.router, prefix=api_prefix) APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") +logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 951fc9b7..508d556b 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -21,6 +21,7 @@ ) async def get_capabilities( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -35,6 +36,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -53,6 +55,7 @@ async def get_capability( ) async def get_projects( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -71,6 +74,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -93,6 +97,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -116,6 +121,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -141,6 +147,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -169,6 +176,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index e7481acc..72a8169c 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -154,8 +154,8 @@ async def get_job_status( async def get_job_statuses( resource_id : str, request : Request, - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=10000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, historical : bool = False, include_spec: bool = False, diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 09769ec2..337b5fc2 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -65,6 +65,12 @@ async def validation_error_handler(request: Request, exc: RequestValidationError @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException): + if exc.status_code == 304: + return JSONResponse( + status_code=304, + content=None, + headers=exc.headers or {}) + if exc.status_code == 401: return problem_response( request=request, diff --git a/app/routers/facility/__init__.py b/app/routers/facility/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py new file mode 100644 index 00000000..99649256 --- /dev/null +++ b/app/routers/facility/facility.py @@ -0,0 +1,77 @@ +from fastapi import Request, HTTPException, Depends, Query +from .. import iri_router +from ..error_handlers import DEFAULT_RESPONSES +from .import models, facility_adapter + + +router = iri_router.IriRouter( + facility_adapter.FacilityAdapter, + prefix="/facility", + tags=["facility"], +) + +@router.get("", responses=DEFAULT_RESPONSES) +async def get_facility( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + ) -> models.Facility: + """Get facility information""" + return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES) +async def list_sites( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +async def get_site( + request: Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +async def get_site_location( + request : Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES) +async def list_locations( + request : Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +async def get_location( + request : Request, + location_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py new file mode 100644 index 00000000..d316674d --- /dev/null +++ b/app/routers/facility/facility_adapter.py @@ -0,0 +1,65 @@ +from abc import abstractmethod +from . import models as facility_models +from ..iri_router import AuthenticatedAdapter + + +class FacilityAdapter(AuthenticatedAdapter): + """ + Facility-specific code is handled by the implementation of this interface. + Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) + to install your facility adapter before the API starts. + """ + + @abstractmethod + async def get_facility( + self: "FacilityAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility | None: + pass + + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py new file mode 100644 index 00000000..0c04cc42 --- /dev/null +++ b/app/routers/facility/models.py @@ -0,0 +1,58 @@ +from datetime import datetime +from uuid import UUID +from typing import List, Optional +from pydantic import BaseModel, Field, HttpUrl, computed_field +from .. import iri_router +from ... import config + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") + country_name: Optional[str] = Field(None, description="Country name of the Location.") + locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") + state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") + street_address: Optional[str] = Field(None, description="Street address of the Location.") + unlocode: Optional[str] = Field(None, description="United Nations trade and transport location code.") + altitude: Optional[float] = Field(None, description="Altitude of the Location.") + latitude: Optional[float] = Field(None, description="Latitude of the Location.") + longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") + +class Facility(NamedObject): + def _self_path(self) -> str: + return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index dafb9702..2de7e693 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -3,11 +3,13 @@ import logging import importlib import datetime +from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from pydantic_core import core_schema from .account.models import User + bearer_token = APIKeyHeader(name="Authorization") @@ -128,17 +130,33 @@ async def get_user( def forbidExtraQueryParams(*allowedParams: str): - """Dependency to forbid extra query parameters not in allowedParams.""" - - async def checker(_req: Request): + async def checker(req: Request): if "*" in allowedParams: - return # Permit anything - incoming = set(_req.query_params.keys()) + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + allowed = set(allowedParams) - unknown = incoming - allowed - if unknown: - raise HTTPException(status_code=422, - detail=[{"type": "extra_forbidden", "loc": ["query", param], "msg": f"Unexpected query parameter: {param}"} for param in unknown]) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) return checker class StrictDateTime: From 9985b7d23538b8b6bd179c45c1a8e452a79fe716 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 07:50:47 -0600 Subject: [PATCH 002/133] Make NamedObject reusable --- app/routers/facility/models.py | 22 ++------- app/routers/models.py | 46 +++++++++++++++++++ app/routers/status/models.py | 82 +++++++++------------------------- 3 files changed, 69 insertions(+), 81 deletions(-) create mode 100644 app/routers/models.py diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 0c04cc42..2bea62f8 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,23 +1,7 @@ -from datetime import datetime -from uuid import UUID +"""Facility-related models.""" from typing import List, Optional -from pydantic import BaseModel, Field, HttpUrl, computed_field -from .. import iri_router -from ... import config - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - +from pydantic import Field, HttpUrl +from ..models import NamedObject class Site(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/models.py b/app/routers/models.py new file mode 100644 index 00000000..f9a1c007 --- /dev/null +++ b/app/routers/models.py @@ -0,0 +1,46 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from pydantic import BaseModel, Field, computed_field +from . import iri_router +from .. import config + + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index b1f3a8bb..2c7dc765 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,6 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config +from ..models import NamedObject class Link(BaseModel): rel : str @@ -15,38 +16,6 @@ class Status(enum.Enum): unknown = "unknown" -class NamedResource(BaseModel): - id : str - name : str - description : str - last_modified : datetime.datetime - - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a - - class ResourceType(enum.Enum): website = "website" service = "service" @@ -57,28 +26,24 @@ class ResourceType(enum.Enum): unknown = "unknown" -class Resource(NamedResource): +class Resource(NamedObject): + + def _self_path(self) -> str: + return f"/status/resources/{self.id}" + capability_ids: list[str] = Field(exclude=True) group: str | None current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] - @staticmethod def find(resources, name, description, group, modified_since, resource_type): - a = NamedResource.find(resources, name, description, modified_since) + a = NamedObject.find(resources, name, description, modified_since) if group: a = [aa for aa in a if aa.group == group] if resource_type: @@ -86,25 +51,21 @@ def find(resources, name, description, group, modified_since, resource_type): return a -class Event(NamedResource): +class Event(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.incident_id}/events/{self.id}" + occurred_at : datetime.datetime status : Status resource_id : str = Field(exclude=True) incident_id : str | None = Field(exclude=True, default=None) - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}/events/{self.id}" - - @computed_field(description="The resource belonging to this event") @property def resource_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.resource_id}" - @computed_field(description="The event's incident") @property def incident_uri(self) -> str|None: @@ -123,7 +84,7 @@ def find( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, ) -> list: - events = NamedResource.find(events, name, description, modified_since) + events = NamedObject.find(events, name, description, modified_since) if resource_id: events = [e for e in events if e.resource_id == resource_id] if status: @@ -150,7 +111,11 @@ class Resolution(enum.Enum): pending = "pending" -class Incident(NamedResource): +class Incident(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.id}" + status : Status resource_ids : list[str] = Field(exclude=True) event_ids : list[str] = Field(exclude=True) @@ -159,25 +124,18 @@ class Incident(NamedResource): type : IncidentType resolution : Resolution - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def event_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}/events/{e}" for e in self.event_ids] - @computed_field(description="The list of resources that may be impacted by this incident") @property def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] def find( + self, incidents : list, name : str | None = None, description : str | None = None, @@ -189,7 +147,7 @@ def find( modified_since : datetime.datetime | None = None, resource_id : str | None = None, ) -> list: - incidents = NamedResource.find(incidents, name, description, modified_since) + incidents = NamedObject.find(incidents, name, description, modified_since) if resource_id: incidents = [e for e in incidents if resource_id in e.resource_ids] if status: From bf82bccf71a696df5bd8513e58c44662ef4a0eaa Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 08:16:21 -0600 Subject: [PATCH 003/133] Include operation_id for facility (similar to pull request #21) --- app/routers/facility/facility.py | 12 ++++++------ app/routers/status/models.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 99649256..0e6f9609 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -10,7 +10,7 @@ tags=["facility"], ) -@router.get("", responses=DEFAULT_RESPONSES) +@router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -19,7 +19,7 @@ async def get_facility( """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) -@router.get("/sites", responses=DEFAULT_RESPONSES) +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") async def list_sites( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -32,7 +32,7 @@ async def list_sites( """List sites""" return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") async def get_site( request: Request, site_id: str, @@ -42,7 +42,7 @@ async def get_site( """Get site by ID""" return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") async def get_site_location( request : Request, site_id: str, @@ -52,7 +52,7 @@ async def get_site_location( """Get site location by site ID""" return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) -@router.get("/locations", responses=DEFAULT_RESPONSES) +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") async def list_locations( request : Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -66,7 +66,7 @@ async def list_locations( """List locations""" return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") async def get_location( request : Request, location_id: str, diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 2c7dc765..bb1f87b0 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -134,8 +134,8 @@ def event_uris(self) -> list[str]: def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] + @staticmethod def find( - self, incidents : list, name : str | None = None, description : str | None = None, From 75945e0ba66794cd0eeb9806f296a391239f9901 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Thu, 15 Jan 2026 11:48:45 -0800 Subject: [PATCH 004/133] simplified facility endpoint proposal --- app/demo_adapter.py | 184 +---------------------- app/routers/facility/facility.py | 57 ------- app/routers/facility/facility_adapter.py | 47 ------ app/routers/facility/models.py | 33 +--- 4 files changed, 14 insertions(+), 307 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index daf266d8..2d2ca30c 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -65,50 +65,7 @@ def __init__(self): def _init_state(self): now = datetime.datetime.now(datetime.timezone.utc) - loc1 = facility_models.Location( - id=demo_uuid("location", "demo_location_1"), - name="Demo Location 1", - description="The first demo location", - last_modified=now, - short_name="DL1", - country_name="USA", - locality_name="Demo City", - state_or_province_name="DC", - latitude=36.173357, - longitude=-234.51452) - - loc2 = facility_models.Location( - id=demo_uuid("location", "demo_location_2"), - name="Demo Location 2", - description="The second demo location", - last_modified=now, - short_name="DL2", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - latitude=38.410558, - longitude=-286.36999) - - site1 = facility_models.Site( - id=demo_uuid("site", "demo_site_1"), - name="Demo Site 1", - description="The first demo site", - last_modified=now, - short_name="DS1", - operating_organization="Demo Org", - location_uri=loc1.self_uri, - resource_uris=[]) - site2 = facility_models.Site( - id=demo_uuid("site", "demo_site_2"), - name="Demo Site 2", - description="The second demo site", - last_modified=now, - short_name="DS2", - operating_organization="Demo Org", - location_uri=loc2.self_uri, - resource_uris=[]) - - facility = facility_models.Facility( + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", description="A demo facility for testing the IRI Facility API", @@ -116,24 +73,15 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - site_uris=[site1.self_uri, site2.self_uri], - location_uris=[loc1.self_uri, loc2.self_uri], - resource_uris=[], - event_uris=[], - incident_uris=[], - capability_uris=[], - project_uris=[], - project_allocation_uris=[], - user_allocation_uris=[], + facility_uri="https://www.demo.example", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + street_address="1 main st", + latitude=38.410558, + longitude=-286.36999 ) - self.facility = facility - loc1.site_uris.append(site1.self_uri) - loc2.site_uris.append(site2.self_uri) - self.locations = [loc1, loc2] - self.sites = [site1, site2] - - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -269,122 +217,6 @@ async def get_facility( ) -> facility_models.Facility: return self.facility - - async def list_sites( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - - sites = self.sites - - if name: - sites = [s for s in sites if name.lower() in s.name.lower()] - - if short_name: - sites = [s for s in sites if s.short_name == short_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - sites = [s for s in sites if s.last_modified > ms] - - o = offset or 0 - l = limit or len(sites) - return sites[o:o+l] - - - async def get_site( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site: - - site = next((s for s in self.sites if s.id == site_id), None) - if not site: - raise HTTPException(status_code=404, detail="Site not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if site.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) - - return site - - - async def get_site_location( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - site = await self.get_site(site_id) - - if not site.location_uri: - raise HTTPException(status_code=404, detail="Site has no location") - - location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - - return location - - - async def list_locations( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - - locs = self.locations - - if name: - locs = [l for l in locs if name.lower() in l.name.lower()] - - if short_name: - locs = [l for l in locs if l.short_name == short_name] - - if country_name: - locs = [l for l in locs if l.country_name == country_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - locs = [l for l in locs if l.last_modified > ms] - - o = offset or 0 - l = limit or len(locs) - return locs[o:o+l] - - - async def get_location( - self: "DemoAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - location = next((l for l in self.locations if l.id == location_id), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - return location - # ---------------------------- # Status API # ---------------------------- diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 0e6f9609..9b7545b2 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,60 +18,3 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - -@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") -async def list_sites( - request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), - )-> list[models.Site]: - """List sites""" - return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) - -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") -async def get_site( - request: Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Site: - """Get site by ID""" - return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) - -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") -async def get_site_location( - request : Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get site location by site ID""" - return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) - -@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") -async def list_locations( - request : Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - country_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), - )-> list[models.Location]: - """List locations""" - return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) - -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") -async def get_location( - request : Request, - location_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get location by ID""" - return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index d316674d..bccdcfe5 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,50 +16,3 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass - - @abstractmethod - async def list_sites( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - pass - - @abstractmethod - async def get_site( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site | None: - pass - - @abstractmethod - async def get_site_location( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass - - @abstractmethod - async def list_locations( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - pass - - @abstractmethod - async def get_location( - self: "FacilityAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 2bea62f8..efd93dd0 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,20 +1,13 @@ """Facility-related models.""" -from typing import List, Optional +from typing import Optional from pydantic import Field, HttpUrl from ..models import NamedObject -class Site(NamedObject): - def _self_path(self) -> str: - return f"/facility/sites/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Site.") - operating_organization: str = Field(..., description="Organization operating the Site.") - location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") - -class Location(NamedObject): - def _self_path(self) -> str: - return f"/facility/locations/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Location.") +class Facility(NamedObject): + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization's name.") + facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -23,20 +16,6 @@ def _self_path(self) -> str: altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") -class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization’s name.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") - location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") - event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") - incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") - capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") - project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") - project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") - user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") From 52567f26d8abae17400f382824e4f9c5a016c740 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 005/133] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- app/demo_adapter.py | 2 -- app/routers/facility/facility.py | 1 + app/routers/facility/facility_adapter.py | 1 + app/routers/facility/models.py | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 2d2ca30c..0d581038 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -57,9 +57,7 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - self.locations = [] self.facility = {} - self.sites = [] self._init_state() diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 9b7545b2..fdba1241 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,3 +18,4 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index bccdcfe5..cbee9515 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,3 +16,4 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass + diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index efd93dd0..5b21d8a3 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -19,3 +19,4 @@ class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + From 0dbb6d8eaad88ae80a40e02bdb23c47c52c6e7bc Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 15:27:00 -0600 Subject: [PATCH 006/133] Remove /api/v1 --- app/main.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/app/main.py b/app/main.py index 080645ef..78641430 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,6 @@ """Main API application""" import logging from fastapi import FastAPI -from fastapi import Request -from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -22,33 +20,6 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" -@APP.get(api_prefix) -async def api_discovery(request: Request): - base = str(request.base_url).rstrip("/") - items = [] - for route in APP.router.routes: - if not isinstance(route, APIRoute): - continue - # skip docs & openapi - if route.path.startswith("/docs") or route.path.startswith("/openapi"): - continue - for method in route.methods: - if method == "HEAD" or method == "OPTIONS": - continue - items.append({ - "id": route.name or f"{method}_{route.path}", - "method": method, - "path": route.path, - "_links": [ - { - "rel": "self", - "href": f"{base.rstrip('/')}{route.path}", - "type": "application/json" - } - ] - }) - return items - # Attach routers under the prefix APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) @@ -57,4 +28,4 @@ async def api_discovery(request: Request): APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file +logging.getLogger().info(f"API path: {api_prefix}") From 65a4e46849160eea8e2b7e415e81d1eec170b9c0 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:14:23 -0600 Subject: [PATCH 007/133] Refactor shared validators & models to fix import loading issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move common validators and base models into routers/dependencies and update imports across routers and schemas to use the new shared location. This keeps API behavior unchanged. No functional API changes — purely structural and validation hygiene: --- app/routers/account/account.py | 17 +- app/routers/account/models.py | 15 +- app/routers/compute/compute.py | 88 ++++++----- app/routers/compute/models.py | 60 ++++--- app/routers/dependencies.py | 176 +++++++++++++++++++++ app/routers/facility/facility.py | 8 +- app/routers/facility/models.py | 2 +- app/routers/filesystem/facility_adapter.py | 4 +- app/routers/filesystem/filesystem.py | 42 +---- app/routers/filesystem/models.py | 9 +- app/routers/iri_router.py | 77 --------- app/routers/models.py | 46 ------ app/routers/status/models.py | 2 +- app/routers/status/status.py | 25 +-- app/routers/task/models.py | 19 ++- 15 files changed, 311 insertions(+), 279 deletions(-) create mode 100644 app/routers/dependencies.py delete mode 100644 app/routers/models.py diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 508d556b..b2c41f44 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import forbidExtraQueryParams router = iri_router.IriRouter( @@ -21,7 +22,7 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -36,7 +37,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -55,7 +56,7 @@ async def get_capability( ) async def get_projects( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -74,7 +75,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -97,7 +98,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -121,7 +122,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -147,7 +148,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -176,7 +177,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 6ed69ea0..c012ee32 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel, computed_field, Field +from pydantic import computed_field, Field import enum from ... import config +from ..dependencies import IRIBaseModel class AllocationUnit(enum.Enum): @@ -9,7 +10,7 @@ class AllocationUnit(enum.Enum): inodes = "inodes" -class Capability(BaseModel): +class Capability(IRIBaseModel): """ An aspect of a resource that can have an allocation. For example, Perlmutter nodes with GPUs @@ -22,7 +23,7 @@ class Capability(BaseModel): units: list[AllocationUnit] -class User(BaseModel): +class User(IRIBaseModel): """A user of the facility""" id: str name: str @@ -31,7 +32,7 @@ class User(BaseModel): # we could expose more fields here (eg. email) but it might be against policy -class Project(BaseModel): +class Project(IRIBaseModel): """A project and its users at a facility""" id: str name: str @@ -39,14 +40,14 @@ class Project(BaseModel): user_ids: list[str] -class AllocationEntry(BaseModel): +class AllocationEntry(IRIBaseModel): """Base class for allocations.""" allocation: float # how much this allocation can spend usage: float # how much this allocation has spent unit: AllocationUnit -class ProjectAllocation(BaseModel): +class ProjectAllocation(IRIBaseModel): """ A project's allocation for a capability. (aka. repo) This allocation is a piece of the total allocation for the capability. (eg. 5% of the total node hours of Perlmutter GPU nodes) @@ -71,7 +72,7 @@ def capability_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{self.capability_id}" -class UserAllocation(BaseModel): +class UserAllocation(IRIBaseModel): """ A user's allcation in a project. This allocation is a piece of the project's allocation. diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 72a8169c..7a2db23c 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,9 +1,10 @@ -from typing import List, Annotated -from fastapi import HTTPException, Request, Depends, status, Form, Query +from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router + from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router +from ..dependencies import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -24,6 +25,7 @@ async def submit_job( resource_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Submit a job on a compute resource @@ -45,39 +47,41 @@ async def submit_job( return await router.adapter.submit_job(resource, user, job_spec) -@router.post( - "/job/script/{resource_id:str}", - dependencies=[Depends(router.current_user)], - response_model=models.Job, - response_model_exclude_unset=True, - responses=DEFAULT_RESPONSES, - operation_id="launchJobScript", -) -async def submit_job_path( - resource_id: str, - job_script_path : str, - request : Request, - args : Annotated[List[str], Form()] = [], - ): - """ - Submit a job on a compute resource - - - **resource**: the name of the compute resource to use - - **job_script_path**: path to the job script on the compute resource - - **args**: optional arguments to the job script - - This command will attempt to submit a job and return its id. - """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - - # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) - - # the handler can use whatever means it wants to submit the job and then fill in its id - # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.submit_job_script(resource, user, job_script_path, args) +# TODO: this conflicts with PUT commented out while we finalize the API design +#@router.post( +# "/job/script/{resource_id:str}", +# dependencies=[Depends(router.current_user)], +# response_model=models.Job, +# response_model_exclude_unset=True, +# responses=DEFAULT_RESPONSES, +# operation_id="launchJobScript", +#) +#async def submit_job_path( +# resource_id: str, +# job_script_path : str, +# request : Request, +# args : Annotated[List[str], Form()] = [], +# _forbid = Depends(iri_router.forbidExtraQueryParams("job_script_path")), +# ): +# """ +# Submit a job on a compute resource +# +# - **resource**: the name of the compute resource to use +# - **job_script_path**: path to the job script on the compute resource +# - **args**: optional arguments to the job script +# +# This command will attempt to submit a job and return its id. +# """ +# user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) +# if not user: +# raise HTTPException(status_code=404, detail="User not found") +# +# # look up the resource (todo: maybe ensure it's available) +# resource = await status_router.adapter.get_resource(resource_id) +# +# # the handler can use whatever means it wants to submit the job and then fill in its id +# # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs +# return await router.adapter.submit_job_script(resource, user, job_script_path, args) @router.put( @@ -93,6 +97,7 @@ async def update_job( job_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Update a previously submitted job for a resource. @@ -126,8 +131,9 @@ async def get_job_status( resource_id : str, job_id : str, request : Request, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -149,7 +155,7 @@ async def get_job_status( response_model=list[models.Job], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, - operation_id="getJobs", + operation_id="getAllJobs", ) async def get_job_statuses( resource_id : str, @@ -157,8 +163,9 @@ async def get_job_statuses( offset : int = Query(default=0, ge=0, le=1000), limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -187,6 +194,7 @@ async def cancel_job( resource_id : str, job_id : str, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """Cancel a job""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 35d34ef9..006aabc5 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,46 +1,45 @@ -from typing import Annotated -from pydantic import BaseModel, field_serializer, ConfigDict, Field -import datetime from enum import IntEnum +from pydantic import field_serializer, ConfigDict, StrictBool, Field +from ..dependencies import IRIBaseModel -class ResourceSpec(BaseModel): - node_count: int | None = None - process_count: int | None = None - processes_per_node: int | None = None - cpu_cores_per_process: int | None = None - gpu_cores_per_process: int | None = None - exclusive_node_use: bool = True - memory: int | None = None +class ResourceSpec(IRIBaseModel): + node_count: int = Field(default=None, ge=1, description="Number of nodes") + process_count: int = Field(default=None, ge=1, description="Number of processes") + processes_per_node: int = Field(default=None, ge=1, description="Number of processes per node") + cpu_cores_per_process: int = Field(default=None, ge=1, description="Number of CPU cores per process") + gpu_cores_per_process: int = Field(default=None, ge=1, description="Number of GPU cores per process") + exclusive_node_use: StrictBool = True + memory: int = Field(default=None, ge=1, description="Amount of memory in megabytes") -class JobAttributes(BaseModel): - duration: Annotated[int | None, Field(description="Duration in seconds", ge=0, examples=[30, 60, 120])] = None - queue_name: str | None = None - account: str | None = None - reservation_id: str | None = None +class JobAttributes(IRIBaseModel): + duration: int = Field(default=None, ge=1, description="Duration in seconds", examples=[30, 60, 120]) + queue_name: str = Field(default=None, min_length=1, description="Name of the queue/partition to use") + account: str = Field(default=None, min_length=1, description="Account/Project name to charge") + reservation_id: str = Field(default=None, min_length=1, description="Reservation ID to use") custom_attributes: dict[str, str] = {} -class JobSpec(BaseModel): +class JobSpec(IRIBaseModel): model_config = ConfigDict(extra="forbid") - executable : str | None = None + executable : str = Field(min_length=1, description="The executable to run") arguments: list[str] = [] - directory: str | None = None - name: str | None = None - inherit_environment: bool = True + directory: str = Field(default=None, min_length=1, description="The working directory for the job") + name: str = Field(default=None, min_length=1, description="The name of the job") + inherit_environment: StrictBool = Field(default=True, description="Whether to inherit the environment") environment: dict[str, str] = {} - stdin_path: str | None = None - stdout_path: str | None = None - stderr_path: str | None = None + stdin_path: str = Field(default=None, min_length=1, description="Path to the standard input file") + stdout_path: str = Field(default=None, min_length=1, description="Path to the standard output file") + stderr_path: str = Field(default=None, min_length=1, description="Path to the standard error file") resources: ResourceSpec | None = None attributes: JobAttributes | None = None - pre_launch: str | None = None - post_launch: str | None = None - launcher: str | None = None + pre_launch: str = Field(default=None, min_length=1, description="Command to run before launching the job") + post_launch: str = Field(default=None, min_length=1, description="Command to run after launching the job") + launcher: str = Field(default=None, min_length=1, description="Launcher to use for the job") -class CommandResult(BaseModel): +class CommandResult(IRIBaseModel): status : str result : str | None = None @@ -80,20 +79,19 @@ class JobState(IntEnum): """Represents a job that was canceled by a call to :func:`~psij.Job.cancel()`.""" -class JobStatus(BaseModel): +class JobStatus(IRIBaseModel): state : JobState time : float | None = None message : str | None = None exit_code : int | None = None meta_data : dict[str, object] | None = None - @field_serializer('state') def serialize_state(self, state: JobState): return state.name -class Job(BaseModel): +class Job(IRIBaseModel): id : str status : JobStatus | None = None job_spec : JobSpec | None = None diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py new file mode 100644 index 00000000..17b4f096 --- /dev/null +++ b/app/routers/dependencies.py @@ -0,0 +1,176 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from urllib.parse import parse_qs + +from pydantic_core import core_schema +from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer +from fastapi import Request, HTTPException + +from .. import config + + +# These are Pydantic custom types for strict validation +# that are not implmented in Pydantic by default. +# ----------------------------------------------------------------------- +# StrictBool: a strict boolean type +class StrictBool: + """Strict boolean: + - Accepts: real booleans, 'true', 'false' + - Rejects everything else. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + """Validate the input value as a strict boolean.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + v = value.strip().lower() + if v == "true": + return True + if v == "false": + return False + raise ValueError("Invalid boolean value. Expected 'true' or 'false'.") + raise ValueError("Invalid boolean value. Expected true/false or 'true'/'false'.") + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "boolean", + "description": "Strict boolean. Only true/false allowed (bool or string)." + } + +# ----------------------------------------------------------------------- +# StrictDateTime: a strict ISO8601 datetime type + +class StrictDateTime: + """ + Strict ISO8601 datetime: + - Accepts datetime objects + - Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 + - Converts 'Z' → UTC + - Converts naive datetimes → UTC + - Rejects integers ("0"), null, garbage strings, etc. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + if isinstance(value, datetime.datetime): + return StrictDateTime._normalize(value) + if not isinstance(value, str): + raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") + v = value.strip() + if v.endswith("Z"): + v = v[:-1] + "+00:00" + try: + dt = datetime.datetime.fromisoformat(v) + except Exception as ex: + raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex + + return StrictDateTime._normalize(dt) + + @staticmethod + def _normalize(dt: datetime.datetime) -> datetime.datetime: + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "string", + "format": "date-time", + "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." + } + + +def forbidExtraQueryParams(*allowedParams: str): + async def checker(req: Request): + if "*" in allowedParams: + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + + allowed = set(allowedParams) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) + return checker + + +class IRIBaseModel(BaseModel): + """Base model for IRI models.""" + model_config = ConfigDict(extra="allow") + + @model_serializer(mode="wrap") + def _hide_extra(self, handler): + data = handler(self) + extra = getattr(self, "__pydantic_extra__", {}) or {} + for k in extra: + data.pop(k, None) + return data + +class NamedObject(IRIBaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index fdba1241..1c389186 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -1,7 +1,8 @@ -from fastapi import Request, HTTPException, Depends, Query +from fastapi import Request, Depends, Query from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( @@ -13,9 +14,8 @@ @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5b21d8a3..5db7a18b 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..models import NamedObject +from ..dependencies import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index 2c08a3cd..a70efb0d 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -1,15 +1,15 @@ import os from abc import abstractmethod +from typing import Any, Tuple from ..status import models as status_models from ..account import models as account_models from . import models as filesystem_models from ..iri_router import AuthenticatedAdapter -from typing import Any, Tuple def to_int(name, default_value): try: - return os.environ.get(name) or default_value + return int(os.environ.get(name) or default_value) except: return default_value diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index a111484a..d583c642 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -354,40 +354,13 @@ async def get_view( resource_id: str, request : Request, path: Annotated[str, Query(description="File path")], - size: Annotated[ - int, - Query( - alias="size", - description="Value, in bytes, of the size of data to be retrieved from the file.", - ), - ] = facility_adapter.OPS_SIZE_LIMIT, - offset: Annotated[ - int, - Query( - alias="offset", - description="Value in bytes of the offset.", - ), - ] = 0, -) -> str: - user, resource = await _user_resource(resource_id, request) - if offset < 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`offset` value must be an integer value equal or greater than 0", - ) - - if size <= 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`size` value must be an integer value greater than 0", - ) + size: Annotated[int, Query(description="Value, in bytes, of the size of data to be retrieved from the file.", + ge=1, le=facility_adapter.OPS_SIZE_LIMIT)] = facility_adapter.OPS_SIZE_LIMIT, - if size > facility_adapter.OPS_SIZE_LIMIT: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"`size` value must be less than {facility_adapter.OPS_SIZE_LIMIT} bytes", - ) + offset: Annotated[int, Query( description="Value in bytes of the offset.", ge=0)] = 0 + ) -> str: + user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user, @@ -397,9 +370,8 @@ async def get_view( command="view", args={ "path": path, - "size": size or facility_adapter.OPS_SIZE_LIMIT, - "offset": offset or 0, - + "size": size, + "offset": offset, } ) ) diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index b24c908a..d32ad943 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -12,11 +12,10 @@ class CompressionType(str, Enum): - none = "none" - bzip2 = "bzip2" - gzip = "gzip" - xz = "xz" - + none = "none" + bzip2 = "bzip2" + gzip = "gzip" + xz = "xz" class ContentUnit(str, Enum): lines = "lines" diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 2de7e693..d3fc9e16 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -2,11 +2,9 @@ import os import logging import importlib -import datetime from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader -from pydantic_core import core_schema from .account.models import User @@ -127,78 +125,3 @@ async def get_user( Retrieve additional user information (name, email, etc.) for the given user_id. """ pass - - -def forbidExtraQueryParams(*allowedParams: str): - async def checker(req: Request): - if "*" in allowedParams: - return - - raw_qs = req.scope.get("query_string", b"") - parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) - - allowed = set(allowedParams) - - for key, values in parsed.items(): - if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) - - if len(values) > 1: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) - return checker - -class StrictDateTime: - """ - Strict ISO8601 datetime: - ✔ Accepts datetime objects - ✔ Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 - ✔ Converts 'Z' → UTC - ✔ Converts naive datetimes → UTC - ✘ Rejects integers ("0"), null, garbage strings, etc. - """ - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - return core_schema.no_info_plain_validator_function(cls.validate) - - @staticmethod - def validate(value): - if isinstance(value, datetime.datetime): - return StrictDateTime._normalize(value) - if not isinstance(value, str): - raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") - v = value.strip() - if v.endswith("Z"): - v = v[:-1] + "+00:00" - try: - dt = datetime.datetime.fromisoformat(v) - except Exception as ex: - raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex - - return StrictDateTime._normalize(dt) - - @staticmethod - def _normalize(dt: datetime.datetime) -> datetime.datetime: - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - - @classmethod - def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "string", - "format": "date-time", - "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." - } diff --git a/app/routers/models.py b/app/routers/models.py deleted file mode 100644 index f9a1c007..00000000 --- a/app/routers/models.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Default models used by multiple routers.""" -import datetime -from typing import Optional -from pydantic import BaseModel, Field, computed_field -from . import iri_router -from .. import config - - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - """Computed self URI property.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index bb1f87b0..ff5d4e48 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..models import NamedObject +from ..dependencies import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 7bb0fd0b..2daf30e7 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -23,9 +24,9 @@ async def get_resources( group : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - modified_since: iri_router.StrictDateTime = Query(default=None), + modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), ) -> list[models.Resource]: return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) @@ -60,14 +61,14 @@ async def get_incidents( description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), type_: models.IncidentType = Query(alias="type", default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), ) -> list[models.Incident]: return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) @@ -104,13 +105,13 @@ async def get_events( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) diff --git a/app/routers/task/models.py b/app/routers/task/models.py index da3c5ccc..cea97873 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,19 +1,18 @@ -from pydantic import BaseModel import enum +from pydantic import BaseModel class TaskStatus(str, enum.Enum): - pending = "pending" - active = "active" - completed = "completed" - failed = "failed" - canceled = "canceled" - + pending = "pending" + active = "active" + completed = "completed" + failed = "failed" + canceled = "canceled" class TaskCommand(BaseModel): - router: str - command: str - args: dict + router: str + command: str + args: dict class Task(BaseModel): From f4adcb66f90f4ff99af0c7f8bbc25cd6cea6ad8a Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:17:00 -0600 Subject: [PATCH 008/133] Github Action to validate api --- .github/workflows/api-validation.yml | 133 +++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 .github/workflows/api-validation.yml diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml new file mode 100644 index 00000000..624e84e2 --- /dev/null +++ b/.github/workflows/api-validation.yml @@ -0,0 +1,133 @@ +name: API Validation with Schemathesis + +on: + pull_request: + push: + branches: [ main ] + +jobs: + schemathesis: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged + - name: Checkout schema validator repository + uses: actions/checkout@v4 + with: + repository: juztas/iri-facility-api-docs + ref: schemavalidator + path: schema-validator + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + run: pip install uv + + - name: Build an image + run: docker build --platform=linux/amd64 -t iri-facility-api-base . + + - name: Run Facility API container + run: | + docker run -d \ + -p 8000:8000 \ + --platform=linux/amd64 \ + --name iri-facility-api-base \ + -e IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + -e API_URL_ROOT=http://127.0.0.1:8000 \ + -e IRI_API_TOKEN=12345 \ + iri-facility-api-base + + - name: Wait for API to be ready + run: | + for i in {1..60}; do + if curl -fs http://127.0.0.1:8000/openapi.json; then + echo "API ready" + exit 0 + fi + sleep 2 + done + echo "API did not start" + exit 1 + + - name: Create venv & install validator dependencies + run: | + uv venv + source .venv/bin/activate + uv pip install -r schema-validator/verification/requirements.txt + + - name: Run Schemathesis validation (local spec) + id: schemathesis_local + env: + IRI_API_TOKEN: "12345" # This is dummy token for testing (mock adapter) + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://127.0.0.1:8000 \ + --report-name schemathesis-local + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Run Schemathesis validation (official spec) + id: schemathesis_official + env: + IRI_API_TOKEN: "12345" + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://localhost:8000 \ + --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --report-name schemathesis-official + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Fail if any Schemathesis run failed + if: always() + run: | + if [ "${{ steps.schemathesis_local.outputs.exitcode }}" != "0" ] || \ + [ "${{ steps.schemathesis_official.outputs.exitcode }}" != "0" ]; then + echo "One or more Schemathesis validations failed" + exit 1 + else + echo "Both Schemathesis validations passed" + fi + + - name: Upload Schemathesis report # This only works on git actions + if: always() && env.ACT != 'true' + uses: actions/upload-artifact@v4 + with: + if-no-files-found: warn + name: schemathesis-report + path: | + schemathesis-local.html + schemathesis-local.xml + schemathesis-official.html + schemathesis-official.xml + + - name: Save Schemathesis reports locally # This only works if run locally with act + if: always() && env.ACT == 'true' + run: | + mkdir -p artifacts + cp schemathesis-local.html schemathesis-local.xml artifacts/ || true + cp schemathesis-official.html schemathesis-official.xml artifacts/ || true + + - name: Dump API logs + if: always() + run: docker logs iri-facility-api-base || true + + - name: Stop container + if: always() + run: docker stop iri-facility-api-base || true From b3e96f7fb52720c514d1f1f4e646cc1e9ed99bb8 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:41:35 -0600 Subject: [PATCH 009/133] Add custom get extra function --- app/routers/dependencies.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py index 17b4f096..7bfaf0b8 100644 --- a/app/routers/dependencies.py +++ b/app/routers/dependencies.py @@ -136,6 +136,10 @@ def _hide_extra(self, handler): data.pop(k, None) return data + def get_extra(self, key, default=None): + return getattr(self, "__pydantic_extra__", {}).get(key, default) + + class NamedObject(IRIBaseModel): id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") def _self_path(self) -> str: From 5b4d7e2cb7e56c367c24d94f64bc5c287e8cb7da Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 13:06:36 -0600 Subject: [PATCH 010/133] Implement opentelemetry. Use UTC in demo adapter --- Makefile | 1 + README.md | 2 ++ VALIDATION.MD | 23 +++++++++++++++++++++++ app/config.py | 5 +++++ app/demo_adapter.py | 31 +++++++++++++++++++++---------- app/main.py | 32 ++++++++++++++++++++++++++++++++ pyproject.toml | 8 ++++++-- 7 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 VALIDATION.MD diff --git a/Makefile b/Makefile index ba7552ab..abd87e4a 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ dev : .venv IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + OPENTELEMETRY_ENABLED=true \ API_URL_ROOT='http://127.0.0.1:8000' fastapi dev diff --git a/README.md b/README.md index 8160cc43..e87a916f 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ If using docker (see next section), your dockerfile could extend this reference - `API_URL_ROOT`: the base url when constructing links returned by the api (eg.: https://iri.myfacility.com) - `API_PREFIX`: the path prefix where the api is hosted. Defaults to `/`. (eg.: `/api`) - `API_URL`: the path to the api itself. Defaults to `api/v1`. +- `OPENTELEMETRY_ENABLED`: Enables OpenTelemetry. If enabled, the application will use OpenTelemetry SDKs and emit traces, metrics, and logs. Default to false +- `OTLP_ENDPOINT`: OpenTelemetry Protocol collector endpoint to export telemetry data. If empty or not set, telemetry data is logged locally to log file. Default: "" Links to data, created by this api, will concatenate these values producing links, eg: `https://iri.myfacility.com/my_api_prefix/my_api_url/projects/123` diff --git a/VALIDATION.MD b/VALIDATION.MD new file mode 100644 index 00000000..26f77ef6 --- /dev/null +++ b/VALIDATION.MD @@ -0,0 +1,23 @@ +# API Validation with Schemathesis + +On every pull request or push to `main` branch, Github Actions run the following steps below that validates an IRI Facility API implementation against OpenAPI spec using Schemathesis. + +1. Builds the Facility API Docker image from Dockerfile. +2. Runs the API container with demo adapter. +3. Waits for `/openapi.json` to become available on localhost:8000. +4. Runs Schemathesis validation twice: + - Against Facilities API’s OpenAPI spec. (http://localhost:8000/openapi.json) + - Against the official IRI Facility API OpenAPI spec. (https://github.com/doe-iri/iri-facility-api-docs/blob/main/specification/openapi/openapi_iri_facility_api_v1.json) +5. Fails the workflow if either validation fails. +6. Saves Schemathesis HTML/XML reports as artifacts (or saves it locally when run with `act`). +7. Dumps API container logs and do clean up to stop container. + +## Running locally + +```bash +act -W .github/workflows/api-validator.yml -s GITHUB_TOKEN= +``` + +## Known issues + +Python implementation not fully aligns with the official Specification. Running against Official Spec will continue to fail, until Spec or Py implementation is fixed. diff --git a/app/config.py b/app/config.py index 078b6a52..71a7c20d 100644 --- a/app/config.py +++ b/app/config.py @@ -40,3 +40,8 @@ API_URL_ROOT = os.environ.get("API_URL_ROOT", "https://api.iri.nersc.gov") API_PREFIX = os.environ.get("API_PREFIX", "/") API_URL = os.environ.get("API_URL", "api/v1") + +OPENTELEMETRY_ENABLED = os.environ.get("OPENTELEMETRY_ENABLED", "false").lower() == "true" +OPENTELEMETRY_DEBUG = os.environ.get("OPENTELEMETRY_DEBUG", "false").lower() == "true" +OTLP_ENDPOINT = os.environ.get("OTLP_ENDPOINT", "") +OTEL_SAMPLE_RATE = float(os.environ.get("OTEL_SAMPLE_RATE", "0.2")) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 0d581038..e63c7201 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -1,7 +1,6 @@ import datetime import random import uuid -import time import os import stat import pwd @@ -45,6 +44,16 @@ def demo_uuid(kind: str, name: str) -> str: return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) +def utc_now() -> datetime.datetime: + """Return current UTC datetime timestamp""" + return datetime.datetime.now(datetime.timezone.utc) + + +def utc_timestamp() -> int: + """Return current UTC datetime timestamp as integer""" + return int(utc_now().timestamp()) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): @@ -62,7 +71,8 @@ def __init__(self): def _init_state(self): - now = datetime.datetime.now(datetime.timezone.utc) + now = utc_now() + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -80,7 +90,8 @@ def _init_state(self): longitude=-286.36999 ) - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) + + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -352,7 +363,7 @@ async def submit_job( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -371,7 +382,7 @@ async def submit_job_script( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -390,7 +401,7 @@ async def update_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.ACTIVE, - time=time.time(), + time=utc_timestamp(), message="job updated", exit_code=None, meta_data={ "account": "account1" }, @@ -410,7 +421,7 @@ async def get_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.COMPLETED, - time=time.time(), + time=utc_timestamp(), message="job completed successfully", exit_code=0, meta_data={ "account": "account1" }, @@ -432,7 +443,7 @@ async def get_jobs( id=f"job_{i}", status=compute_models.JobStatus( state=random.choice([s for s in compute_models.JobState]), - time=time.time() - (random.random() * 100), + time=utc_timestamp() - int(random.random() * 100), message="", exit_code=random.choice([0, 0, 0, 0, 0, 1, 1, 128, 127]), meta_data={ "account": "account1" }, @@ -900,7 +911,7 @@ class DemoTaskQueue: @staticmethod async def _process_tasks(da: DemoAdapter): - now = time.time() + now = utc_timestamp() _tasks = [] for t in DemoTaskQueue.tasks: if now - t.start > 5 * 60 and t.status in [task_models.TaskStatus.completed, task_models.TaskStatus.canceled, task_models.TaskStatus.failed]: @@ -921,5 +932,5 @@ async def _process_tasks(da: DemoAdapter): @staticmethod def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> str: task_id = f"task_{len(DemoTaskQueue.tasks)}" - DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=time.time())) + DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) return task_id diff --git a/app/main.py b/app/main.py index 78641430..fa3f1eda 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,13 @@ """Main API application""" import logging from fastapi import FastAPI +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, BatchSpanProcessor, SimpleSpanProcessor +from opentelemetry.sdk.trace.sampling import TraceIdRatioBased, ParentBased +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -13,9 +20,34 @@ from . import config +# ------------------------------------------------------------------ +# OpenTelemetry Tracing Configuration +# ------------------------------------------------------------------ +if config.OPENTELEMETRY_ENABLED: + resource = Resource.create({ + "service.name": "iri-facility-api", + "service.version": config.API_VERSION, + "service.endpoint": config.API_URL_ROOT}) + + samplerate = "1.0" if config.OPENTELEMETRY_DEBUG else config.OTEL_SAMPLE_RATE + provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(samplerate))) + trace.set_tracer_provider(provider) + + if config.OTLP_ENDPOINT: + exporter = OTLPSpanExporter(endpoint=config.OTLP_ENDPOINT, insecure=True) + span_processor = BatchSpanProcessor(exporter) + else: + exporter = ConsoleSpanExporter() + span_processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(span_processor) + tracer = trace.get_tracer(__name__) +# ------------------------------------------------------------------ APP = FastAPI(**config.API_CONFIG) +if config.OPENTELEMETRY_ENABLED: + FastAPIInstrumentor.instrument_app(APP) + install_error_handlers(APP) api_prefix = f"{config.API_PREFIX}{config.API_URL}" diff --git a/pyproject.toml b/pyproject.toml index 03858a43..1f5c0d64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,5 +5,9 @@ requires-python = ">=3.12" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", - "humps>=0.2.2" -] + "humps>=0.2.2", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation-fastapi", + "opentelemetry-exporter-otlp" +] \ No newline at end of file From b697d186629e6eadd62436fdffedb536c5a1953f Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 14:28:41 -0600 Subject: [PATCH 011/133] Rename dependencies to common --- app/routers/account/account.py | 2 +- app/routers/account/models.py | 2 +- app/routers/{dependencies.py => common.py} | 0 app/routers/compute/compute.py | 2 +- app/routers/compute/models.py | 2 +- app/routers/facility/facility.py | 2 +- app/routers/facility/models.py | 2 +- app/routers/status/models.py | 2 +- app/routers/status/status.py | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename app/routers/{dependencies.py => common.py} (100%) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index b2c41f44..fe16b8bf 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import forbidExtraQueryParams +from ..common import forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/account/models.py b/app/routers/account/models.py index c012ee32..2158575e 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,7 +1,7 @@ from pydantic import computed_field, Field import enum from ... import config -from ..dependencies import IRIBaseModel +from ..common import IRIBaseModel class AllocationUnit(enum.Enum): diff --git a/app/routers/dependencies.py b/app/routers/common.py similarity index 100% rename from app/routers/dependencies.py rename to app/routers/common.py diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 7a2db23c..20081ee6 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -4,7 +4,7 @@ from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router -from ..dependencies import forbidExtraQueryParams, StrictBool +from ..common import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 006aabc5..1b876caf 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,6 +1,6 @@ from enum import IntEnum from pydantic import field_serializer, ConfigDict, StrictBool, Field -from ..dependencies import IRIBaseModel +from ..common import IRIBaseModel class ResourceSpec(IRIBaseModel): diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 1c389186..c0d3e80d 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -2,7 +2,7 @@ from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5db7a18b..508dbcf3 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..dependencies import NamedObject +from ..common import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/status/models.py b/app/routers/status/models.py index ff5d4e48..d338e2cf 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..dependencies import NamedObject +from ..common import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 2daf30e7..d599866c 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, From db5878cec3d4062ae257719d2e7d40001b95b626 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 09:26:32 -0600 Subject: [PATCH 012/133] Do not swallow exceptions --- app/routers/compute/compute.py | 9 ++++----- app/routers/iri_router.py | 6 ++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 20081ee6..cca45be5 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,3 +1,4 @@ +"""Compute resource API router""" from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router @@ -6,13 +7,13 @@ from ..status.status import router as status_router from ..common import forbidExtraQueryParams, StrictBool + router = iri_router.IriRouter( facility_adapter.FacilityAdapter, prefix="/compute", tags=["compute"], ) - @router.post( "/job/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -204,8 +205,6 @@ async def cancel_job( # look up the resource (todo: maybe ensure it's available) resource = await status_router.adapter.get_resource(resource_id) - try: - await router.adapter.cancel_job(resource, user, job_id) - except Exception as exc: - raise HTTPException(status_code=400, detail=f"Unable to cancel job: {str(exc)}") from exc + await router.adapter.cancel_job(resource, user, job_id) + return None diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index d3fc9e16..f0b5b49c 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod +import traceback import os import logging import importlib -from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from .account.models import User @@ -12,9 +12,6 @@ def get_client_ip(request : Request) -> str|None: - # logging.debug("Request headers=%s" % request.headers) - # logging.debug("client=%s" % request.client.host) - forwarded_for = request.headers.get("X-Forwarded-For") if forwarded_for: return forwarded_for.split(",")[0].strip() @@ -91,6 +88,7 @@ async def current_user( user_id = await self.adapter.get_current_user(api_key, get_client_ip(request)) except Exception as exc: logging.getLogger().error(f"Error parsing IRI_API_PARAMS: {exc}") + traceback.print_exc() raise HTTPException(status_code=401, detail="Invalid or malformed Authorization parameters") from exc if not user_id: raise HTTPException(status_code=403, detail="Unauthorized access") From df38b39fad33e5c9e37bc87fdc64c339b2c6567c Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 18:58:36 -0600 Subject: [PATCH 013/133] Ensure that computed fields are included in output --- app/routers/common.py | 14 ++++++++++++-- app/routers/status/models.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index 7bfaf0b8..f46c888f 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -129,13 +129,23 @@ class IRIBaseModel(BaseModel): model_config = ConfigDict(extra="allow") @model_serializer(mode="wrap") - def _hide_extra(self, handler): + def _hide_extra(self, handler, info): data = handler(self) + + model_fields = set(self.model_fields or {}) + computed_fields = set(self.model_computed_fields or {}) + print(model_fields) + print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: - data.pop(k, None) + if k not in model_fields and k not in computed_fields: + data.pop(k, None) + return data + + + def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) diff --git a/app/routers/status/models.py b/app/routers/status/models.py index d338e2cf..448a7e21 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -36,7 +36,7 @@ def _self_path(self) -> str: current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - @computed_field(description="The list of past events in this incident") + @computed_field(description="The list of capabilities in this resource") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] From a634c63d25c3a3cd415cbbfb718d8904b2b2fe92 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 09:57:21 -0600 Subject: [PATCH 014/133] Code base compliant with the official Spec --- app/demo_adapter.py | 203 +++++++++++++++++++++-- app/routers/account/account.py | 17 +- app/routers/account/facility_adapter.py | 3 +- app/routers/account/models.py | 22 +-- app/routers/common.py | 48 ++++-- app/routers/facility/facility.py | 57 +++++++ app/routers/facility/facility_adapter.py | 46 +++++ app/routers/facility/models.py | 35 +++- app/routers/status/facility_adapter.py | 6 +- app/routers/status/models.py | 2 + app/routers/status/status.py | 32 ++-- 11 files changed, 398 insertions(+), 73 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index e63c7201..e68635e6 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -9,9 +9,10 @@ import subprocess import pathlib import base64 -from pydantic import BaseModel from typing import Any, Tuple +from pydantic import BaseModel from fastapi import HTTPException +from .routers.common import AllocationUnit, Capability from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter @@ -35,7 +36,7 @@ def get_base_temp_dir(cls): os.makedirs(cls._base_temp_dir, exist_ok=True) # create a test file - with open(f"{cls._base_temp_dir}/test.txt", "w") as f: + with open(f"{cls._base_temp_dir}/test.txt", encoding="utf-8", mode="w") as f: f.write("hello world") return cls._base_temp_dir @@ -67,12 +68,57 @@ def __init__(self): self.project_allocations = [] self.user_allocations = [] self.facility = {} + self.locations = [] + self.sites = [] self._init_state() def _init_state(self): now = utc_now() + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -81,22 +127,29 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - facility_uri="https://www.demo.example", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - street_address="1 main st", - latitude=38.410558, - longitude=-286.36999 + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], ) + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { - "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "hpss": account_models.Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), - "gpfs": account_models.Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), + "cpu": Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[AllocationUnit.node_hours]), + "gpu": Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[AllocationUnit.node_hours]), + "hpss": Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), + "gpfs": Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), } pm = status_models.Resource(id=str(uuid.uuid4()), group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", capability_ids=[ @@ -226,6 +279,125 @@ async def get_facility( ) -> facility_models.Facility: return self.facility + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + + + # ---------------------------- # Status API # ---------------------------- @@ -239,6 +411,8 @@ async def get_resources( group : str | None = None, modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, + current_status : status_models.Status | None = None, + capability: Capability | None = None ) -> list[status_models.Resource]: return status_models.Resource.find(self.resources, name, description, group, modified_since, resource_type)[offset:offset + limit] @@ -288,6 +462,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: return status_models.Incident.find(self.incidents, name, description, status, type, from_, to, time_, modified_since, resource_id)[offset:offset + limit] @@ -301,7 +476,7 @@ async def get_incident( async def get_capabilities( self : "DemoAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: return self.capabilities.values() diff --git a/app/routers/account/account.py b/app/routers/account/account.py index fe16b8bf..f856312e 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -1,8 +1,8 @@ -from fastapi import HTTPException, Request, Depends +from fastapi import HTTPException, Request, Depends, Query from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import forbidExtraQueryParams +from ..common import forbidExtraQueryParams, StrictDateTime, Capability router = iri_router.IriRouter( @@ -22,8 +22,12 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> list[models.Capability]: + name : str = Query(default=None, min_length=1), + modified_since: StrictDateTime = Query(default=None), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + _forbid = Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), + ) -> list[Capability]: return await router.adapter.get_capabilities() @@ -37,8 +41,9 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> models.Capability: + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + ) -> Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) if not cc: diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 78b622fd..235a2f73 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -1,5 +1,6 @@ from abc import abstractmethod from . import models as account_models +from ..common import Capability from ..iri_router import AuthenticatedAdapter @@ -13,7 +14,7 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def get_capabilities( self : "FacilityAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: pass diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 2158575e..1a9333d8 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,26 +1,6 @@ from pydantic import computed_field, Field -import enum from ... import config -from ..common import IRIBaseModel - - -class AllocationUnit(enum.Enum): - node_hours = "node_hours" - bytes = "bytes" - inodes = "inodes" - - -class Capability(IRIBaseModel): - """ - An aspect of a resource that can have an allocation. - For example, Perlmutter nodes with GPUs - For some resources at a facility, this will be 1 to 1 with the resource. - It is a way to further subdivide a resource into allocatable sub-resources. - The word "capability" is also known to users as something they need for a job to run. (eg. gpu) - """ - id: str - name: str - units: list[AllocationUnit] +from ..common import IRIBaseModel, AllocationUnit class User(IRIBaseModel): diff --git a/app/routers/common.py b/app/routers/common.py index f46c888f..d5805656 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -1,5 +1,6 @@ """Default models used by multiple routers.""" import datetime +import enum from typing import Optional from urllib.parse import parse_qs @@ -93,7 +94,11 @@ def __get_pydantic_json_schema__(cls, schema, handler): } -def forbidExtraQueryParams(*allowedParams: str): +def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): + multiParams = multiParams or set() + + print(allowedParams, multiParams) + async def checker(req: Request): if "*" in allowedParams: return @@ -110,20 +115,26 @@ async def checker(req: Request): detail=[{ "type": "extra_forbidden", "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) + "msg": f"Unexpected query parameter: {key}", + }], + ) - if len(values) > 1: + if len(values) > 1 and key not in multiParams: raise HTTPException( status_code=422, detail=[{ "type": "duplicate_forbidden", "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) + "msg": f"Duplicate query parameter: {key}", + }], + ) + return checker + + + class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") @@ -134,18 +145,12 @@ def _hide_extra(self, handler, info): model_fields = set(self.model_fields or {}) computed_fields = set(self.model_computed_fields or {}) - print(model_fields) - print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: if k not in model_fields and k not in computed_fields: data.pop(k, None) - return data - - - def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) @@ -188,3 +193,22 @@ def normalize(dt: datetime) -> datetime: modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] return a + + +class AllocationUnit(enum.Enum): + node_hours = "node_hours" + bytes = "bytes" + inodes = "inodes" + + +class Capability(IRIBaseModel): + """ + An aspect of a resource that can have an allocation. + For example, Perlmutter nodes with GPUs + For some resources at a facility, this will be 1 to 1 with the resource. + It is a way to further subdivide a resource into allocatable sub-resources. + The word "capability" is also known to users as something they need for a job to run. (eg. gpu) + """ + id: str + name: str + units: list[AllocationUnit] \ No newline at end of file diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index c0d3e80d..7c685e15 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -19,3 +19,60 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") +async def list_sites( + request: Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") +async def get_site( + request: Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") +async def get_site_location( + request : Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") +async def list_locations( + request : Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") +async def get_location( + request : Request, + location_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index cbee9515..d316674d 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -17,3 +17,49 @@ async def get_facility( ) -> facility_models.Facility | None: pass + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 508dbcf3..fd164fad 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,13 +1,22 @@ """Facility-related models.""" -from typing import Optional +from typing import Optional, List from pydantic import Field, HttpUrl from ..common import NamedObject -class Facility(NamedObject): - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization's name.") - facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -16,7 +25,21 @@ class Facility(NamedObject): altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") +class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 6753a470..d7358c50 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -2,6 +2,7 @@ import datetime from fastapi import Query from . import models as status_models +from ..common import Capability class FacilityAdapter(ABC): @@ -21,7 +22,9 @@ async def get_resources( description : str | None = None, group : str | None = None, modified_since : datetime.datetime | None = None, - resource_type: status_models.ResourceType = Query(default=None) + resource_type: status_models.ResourceType = Query(default=None), + current_status: status_models.Status = Query(default=None), + capability: Capability | None = None, ) -> list[status_models.Resource]: pass @@ -75,6 +78,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 448a7e21..3650451a 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -101,6 +101,7 @@ def find( class IncidentType(enum.Enum): planned = "planned" unplanned = "unplanned" + reservation = "reservation" class Resolution(enum.Enum): @@ -111,6 +112,7 @@ class Resolution(enum.Enum): pending = "pending" + class Incident(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/status/status.py b/app/routers/status/status.py index d599866c..5290a8a9 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -1,8 +1,9 @@ +from typing import Optional, List, Annotated from fastapi import HTTPException, Request, Query, Depends from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams, AllocationUnit router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -22,13 +23,16 @@ async def get_resources( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), group : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + current_status: models.Status = Query(default=None), + #event_uris: Optional[List[str]] = Query(default=None, min_length=1), + capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) @router.get( @@ -48,7 +52,7 @@ async def get_resource( return item -@router.get( +@router.get( "/incidents", summary="Get all incidents without their events", description="Get a list of all incidents. Each incident will be returned without its events. You can optionally filter the returned list by specifying attributes.", @@ -66,11 +70,15 @@ async def get_incidents( to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), - _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + resolution : models.Resolution = Query(default=None), + resource_uris: Optional[List[str]] = Query(default=None, min_length=1), + event_uris: Optional[List[str]] = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", + "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), ) -> list[models.Incident]: - return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) + return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id, resolution) @router.get( @@ -109,8 +117,8 @@ async def get_events( time_ : StrictDateTime = Query(alias="time", default=None), to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) From bca4945eafa8bcdda363559501aea12c20aac2f7 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:08:21 -0600 Subject: [PATCH 015/133] Fully compliant with official spec --- app/routers/common.py | 28 +++++++++------------------- app/routers/status/status.py | 5 ++--- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index d5805656..fd2882fe 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -97,8 +97,6 @@ def __get_pydantic_json_schema__(cls, schema, handler): def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): multiParams = multiParams or set() - print(allowedParams, multiParams) - async def checker(req: Request): if "*" in allowedParams: return @@ -110,31 +108,23 @@ async def checker(req: Request): for key, values in parsed.items(): if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}"}]) + if len(values) > 1 and key not in multiParams: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}"}]) return checker - class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 5290a8a9..6a0e9489 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -28,11 +28,10 @@ async def get_resources( modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), current_status: models.Status = Query(default=None), - #event_uris: Optional[List[str]] = Query(default=None, min_length=1), - capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + capability: List[AllocationUnit] = Query(default=None, min_length=1), _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status, capability) @router.get( From a6bc9af87fa386dd9677d9508037dea0e8f99d62 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:45:23 -0600 Subject: [PATCH 016/133] Enforce Py 3.14 as used release for everything --- .github/workflows/api-validation.yml | 11 ++++++----- Dockerfile | 2 +- pyproject.toml | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml index 624e84e2..e4d688b4 100644 --- a/.github/workflows/api-validation.yml +++ b/.github/workflows/api-validation.yml @@ -15,19 +15,18 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged - name: Checkout schema validator repository uses: actions/checkout@v4 with: - repository: juztas/iri-facility-api-docs - ref: schemavalidator + repository: doe-iri/iri-facility-api-docs + ref: main path: schema-validator token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.14" - name: Install uv run: pip install uv @@ -81,6 +80,8 @@ jobs: --report-name schemathesis-local echo "exitcode=$?" >> $GITHUB_OUTPUT + # TODO: Change back to https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json + # Once https://github.com/doe-iri/iri-facility-api-docs/pull/12 merged. - name: Run Schemathesis validation (official spec) id: schemathesis_official env: @@ -90,7 +91,7 @@ jobs: source .venv/bin/activate python schema-validator/verification/api-validator.py \ --baseurl http://localhost:8000 \ - --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --schema-url https://raw.githubusercontent.com/juztas/iri-facility-api-docs/refs/heads/newspec/specification/openapi/openapi_iri_facility_api_v1.json \ --report-name schemathesis-official echo "exitcode=$?" >> $GITHUB_OUTPUT diff --git a/Dockerfile b/Dockerfile index f3ad071d..93c80d90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.14 RUN mkdir /app COPY . /app diff --git a/pyproject.toml b/pyproject.toml index 1f5c0d64..6dbf14b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12" +requires-python = ">=3.12,<3.13" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", @@ -10,4 +10,4 @@ dependencies = [ "opentelemetry-sdk", "opentelemetry-instrumentation-fastapi", "opentelemetry-exporter-otlp" -] \ No newline at end of file +] From 8d830e0110e7f7a1c5ac58778fb56d52fee2c408 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 17:10:38 -0600 Subject: [PATCH 017/133] Enable deepsource scanning --- .deepsource.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .deepsource.toml diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 00000000..01a10663 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,9 @@ +version = 1 + +[[analyzers]] +name = "python" +enabled = true + + [analyzers.meta] + runtime_version = "3.x.x" + max_line_length = 200 From f4914e87ec34380479a9534581d3c0db21606a16 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Sun, 25 Jan 2026 08:08:41 -0600 Subject: [PATCH 018/133] Enforce consistent package versions (pin major/minor version) --- pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6dbf14b1..63f6c020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12,<3.13" +requires-python = ">=3.14,<3.15" dependencies = [ - "fastapi[standard]>=0.100.0", - "uvicorn[standard]>=0.22.0", - "humps>=0.2.2", - "opentelemetry-api", - "opentelemetry-sdk", - "opentelemetry-instrumentation-fastapi", - "opentelemetry-exporter-otlp" -] + "fastapi[standard]>=0.128.0,<0.129.0", + "uvicorn[standard]>=0.40.0,<0.41.0", + "humps>=0.2.2,<0.3.0", + "opentelemetry-api>=1.39.1,<1.40.0", + "opentelemetry-sdk>=1.39.1,<1.40.0", + "opentelemetry-instrumentation-fastapi>=0.60b1,<0.61b0", + "opentelemetry-exporter-otlp>=1.39.1,<1.40.0" +] \ No newline at end of file From 6a765803d751aff779d39bbdff15be7402ffdcc2 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Tue, 25 Nov 2025 14:55:21 +0000 Subject: [PATCH 019/133] Add minimum changes to support containers --- app/routers/compute/models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 35d34ef9..10eb6c44 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -22,9 +22,20 @@ class JobAttributes(BaseModel): custom_attributes: dict[str, str] = {} +class VolumeMount(BaseModel): + source: str + target: str + read_only: bool = True + +class Container(BaseModel): + image: str + volume_mounts: list[VolumeMount] = [] + + class JobSpec(BaseModel): model_config = ConfigDict(extra="forbid") executable : str | None = None + container: Container | None = None arguments: list[str] = [] directory: str | None = None name: str | None = None From 0f50a06909f3df0931b8cec6ed6e3b2d2027a056 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Thu, 4 Dec 2025 19:00:48 +0000 Subject: [PATCH 020/133] Add documentation --- app/routers/compute/models.py | 82 ++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 10eb6c44..e3f4c808 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -5,50 +5,70 @@ class ResourceSpec(BaseModel): - node_count: int | None = None - process_count: int | None = None - processes_per_node: int | None = None - cpu_cores_per_process: int | None = None - gpu_cores_per_process: int | None = None - exclusive_node_use: bool = True - memory: int | None = None + """ + Specification of computational resources required for a job. + """ + node_count: Annotated[int | None, Field(description="Number of compute nodes to allocate")] = None + process_count: Annotated[int | None, Field(description="Total number of processes to launch")] = None + processes_per_node: Annotated[int | None, Field(description="Number of processes to launch per node")] = None + cpu_cores_per_process: Annotated[int | None, Field(description="Number of CPU cores to allocate per process")] = None + gpu_cores_per_process: Annotated[int | None, Field(description="Number of GPU cores to allocate per process")] = None + exclusive_node_use: Annotated[bool, Field(description="Whether to request exclusive use of allocated nodes")] = True + memory: Annotated[int | None, Field(description="Amount of memory to allocate in bytes")] = None class JobAttributes(BaseModel): + """ + Additional attributes and scheduling parameters for a job. + """ duration: Annotated[int | None, Field(description="Duration in seconds", ge=0, examples=[30, 60, 120])] = None - queue_name: str | None = None - account: str | None = None - reservation_id: str | None = None - custom_attributes: dict[str, str] = {} + queue_name: Annotated[str | None, Field(description="Name of the queue or partition to submit the job to")] = None + account: Annotated[str | None, Field(description="Account or project to charge for resource usage")] = None + reservation_id: Annotated[str | None, Field(description="ID of a reservation to use for the job")] = None + custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs")] = {} class VolumeMount(BaseModel): - source: str - target: str - read_only: bool = True + """ + Represents a volume mount for a container. + """ + source: Annotated[str, Field(description="The source path on the host system to mount")] + target: Annotated[str, Field(description="The target path inside the container where the volume will be mounted")] + read_only: Annotated[bool, Field(description="Whether the mount should be read-only")] = True class Container(BaseModel): - image: str - volume_mounts: list[VolumeMount] = [] + """ + Represents a container specification for job execution. + + Implementation notes: The value of gpu_cores_per_process in ResourceSpec should be used to determine + if the container should be run with GPU support. Likewise, the value of launcher in JobSpec should be used + to determine if the container should be run with MPI support. The container should by default. be run with + host networking. + """ + image: Annotated[str, Field(description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] + volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] class JobSpec(BaseModel): + """ + Specification for job. + """ model_config = ConfigDict(extra="forbid") - executable : str | None = None - container: Container | None = None - arguments: list[str] = [] - directory: str | None = None - name: str | None = None - inherit_environment: bool = True - environment: dict[str, str] = {} - stdin_path: str | None = None - stdout_path: str | None = None - stderr_path: str | None = None - resources: ResourceSpec | None = None - attributes: JobAttributes | None = None - pre_launch: str | None = None - post_launch: str | None = None - launcher: str | None = None + executable: Annotated[str | None, Field(description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.")] = None + container: Annotated[Container | None, Field(description="Container specification for containerized execution")] = None + arguments: Annotated[list[str], Field(description="Command-line arguments to pass to the executable or container")] = [] + directory: Annotated[str | None, Field(description="Working directory for the job")] = None + name: Annotated[str | None, Field(description="Name of the job")] = None + inherit_environment: Annotated[bool, Field(description="Whether to inherit the environment variables from the submission environment")] = True + environment: Annotated[dict[str, str], Field(description="Environment variables to set for the job. If container is specified, these will be set inside the container.")] = {} + stdin_path: Annotated[str | None, Field(description="Path to file to use as standard input")] = None + stdout_path: Annotated[str | None, Field(description="Path to file to write standard output")] = None + stderr_path: Annotated[str | None, Field(description="Path to file to write standard error")] = None + resources: Annotated[ResourceSpec | None, Field(description="Resource requirements for the job")] = None + attributes: Annotated[JobAttributes | None, Field(description="Additional job attributes such as duration, queue, and account")] = None + pre_launch: Annotated[str | None, Field(description="Script or commands to run before launching the job")] = None + post_launch: Annotated[str | None, Field(description="Script or commands to run after the job completes")] = None + launcher: Annotated[str | None, Field(description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None class CommandResult(BaseModel): From ec3b448f9b6efdaf1cd46f352d2a3523af645c5e Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Tue, 3 Feb 2026 13:47:55 +0000 Subject: [PATCH 021/133] Remove used import --- app/routers/compute/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index e3f4c808..167643ec 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,6 +1,5 @@ from typing import Annotated from pydantic import BaseModel, field_serializer, ConfigDict, Field -import datetime from enum import IntEnum From 09b409d408b6d94aae350973794ae9040833b3b4 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 022/133] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- Makefile | 1 + app/demo_adapter.py | 210 ++++++++++++++++++++++- app/main.py | 33 +++- app/routers/account/account.py | 8 + app/routers/compute/compute.py | 4 +- app/routers/error_handlers.py | 6 + app/routers/facility/__init__.py | 0 app/routers/facility/facility.py | 77 +++++++++ app/routers/facility/facility_adapter.py | 65 +++++++ app/routers/facility/models.py | 58 +++++++ app/routers/iri_router.py | 36 +++- 11 files changed, 484 insertions(+), 14 deletions(-) create mode 100644 app/routers/facility/__init__.py create mode 100644 app/routers/facility/facility.py create mode 100644 app/routers/facility/facility_adapter.py create mode 100644 app/routers/facility/models.py diff --git a/Makefile b/Makefile index 5bd90346..ba7552ab 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ dev : .venv @source ./.venv/bin/activate && \ + IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 9cbc7961..daf266d8 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -13,6 +13,7 @@ from pydantic import BaseModel from typing import Any, Tuple from fastapi import HTTPException +from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter from .routers.compute import models as compute_models, facility_adapter as compute_adapter @@ -40,9 +41,13 @@ def get_base_temp_dir(cls): return cls._base_temp_dir +def demo_uuid(kind: str, name: str) -> str: + return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, - task_adapter.FacilityAdapter): + task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): def __init__(self): self.resources = [] self.incidents = [] @@ -52,11 +57,83 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - + self.locations = [] + self.facility = {} + self.sites = [] self._init_state() def _init_state(self): + now = datetime.datetime.now(datetime.timezone.utc) + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + + facility = facility_models.Facility( + id=demo_uuid("facility", "demo_facility"), + name="Demo Facility", + description="A demo facility for testing the IRI Facility API", + last_modified=now, + short_name="DEMO", + organization_name="Demo Organization", + support_uri="https://support.demo.example", + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], + ) + + self.facility = facility + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + + day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -182,6 +259,135 @@ def _init_state(self): d += datetime.timedelta(minutes=int(random.random() * 15 + 1)) + # ---------------------------- + # Facility API + # ---------------------------- + + async def get_facility( + self: "DemoAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility: + return self.facility + + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + # ---------------------------- + # Status API + # ---------------------------- async def get_resources( self : "DemoAdapter", diff --git a/app/main.py b/app/main.py index be5feaaf..080645ef 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,11 @@ """Main API application""" import logging from fastapi import FastAPI +from fastapi import Request +from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers +from app.routers.facility import facility from app.routers.status import status from app.routers.account import account from app.routers.compute import compute @@ -19,11 +22,39 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" +@APP.get(api_prefix) +async def api_discovery(request: Request): + base = str(request.base_url).rstrip("/") + items = [] + for route in APP.router.routes: + if not isinstance(route, APIRoute): + continue + # skip docs & openapi + if route.path.startswith("/docs") or route.path.startswith("/openapi"): + continue + for method in route.methods: + if method == "HEAD" or method == "OPTIONS": + continue + items.append({ + "id": route.name or f"{method}_{route.path}", + "method": method, + "path": route.path, + "_links": [ + { + "rel": "self", + "href": f"{base.rstrip('/')}{route.path}", + "type": "application/json" + } + ] + }) + return items + # Attach routers under the prefix +APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) APP.include_router(account.router, prefix=api_prefix) APP.include_router(compute.router, prefix=api_prefix) APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") +logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 951fc9b7..508d556b 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -21,6 +21,7 @@ ) async def get_capabilities( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -35,6 +36,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -53,6 +55,7 @@ async def get_capability( ) async def get_projects( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -71,6 +74,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -93,6 +97,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -116,6 +121,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -141,6 +147,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -169,6 +176,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index e7481acc..72a8169c 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -154,8 +154,8 @@ async def get_job_status( async def get_job_statuses( resource_id : str, request : Request, - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=10000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, historical : bool = False, include_spec: bool = False, diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 09769ec2..337b5fc2 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -65,6 +65,12 @@ async def validation_error_handler(request: Request, exc: RequestValidationError @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException): + if exc.status_code == 304: + return JSONResponse( + status_code=304, + content=None, + headers=exc.headers or {}) + if exc.status_code == 401: return problem_response( request=request, diff --git a/app/routers/facility/__init__.py b/app/routers/facility/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py new file mode 100644 index 00000000..99649256 --- /dev/null +++ b/app/routers/facility/facility.py @@ -0,0 +1,77 @@ +from fastapi import Request, HTTPException, Depends, Query +from .. import iri_router +from ..error_handlers import DEFAULT_RESPONSES +from .import models, facility_adapter + + +router = iri_router.IriRouter( + facility_adapter.FacilityAdapter, + prefix="/facility", + tags=["facility"], +) + +@router.get("", responses=DEFAULT_RESPONSES) +async def get_facility( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + ) -> models.Facility: + """Get facility information""" + return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES) +async def list_sites( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +async def get_site( + request: Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +async def get_site_location( + request : Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES) +async def list_locations( + request : Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +async def get_location( + request : Request, + location_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py new file mode 100644 index 00000000..d316674d --- /dev/null +++ b/app/routers/facility/facility_adapter.py @@ -0,0 +1,65 @@ +from abc import abstractmethod +from . import models as facility_models +from ..iri_router import AuthenticatedAdapter + + +class FacilityAdapter(AuthenticatedAdapter): + """ + Facility-specific code is handled by the implementation of this interface. + Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) + to install your facility adapter before the API starts. + """ + + @abstractmethod + async def get_facility( + self: "FacilityAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility | None: + pass + + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py new file mode 100644 index 00000000..0c04cc42 --- /dev/null +++ b/app/routers/facility/models.py @@ -0,0 +1,58 @@ +from datetime import datetime +from uuid import UUID +from typing import List, Optional +from pydantic import BaseModel, Field, HttpUrl, computed_field +from .. import iri_router +from ... import config + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") + country_name: Optional[str] = Field(None, description="Country name of the Location.") + locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") + state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") + street_address: Optional[str] = Field(None, description="Street address of the Location.") + unlocode: Optional[str] = Field(None, description="United Nations trade and transport location code.") + altitude: Optional[float] = Field(None, description="Altitude of the Location.") + latitude: Optional[float] = Field(None, description="Latitude of the Location.") + longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") + +class Facility(NamedObject): + def _self_path(self) -> str: + return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index dafb9702..2de7e693 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -3,11 +3,13 @@ import logging import importlib import datetime +from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from pydantic_core import core_schema from .account.models import User + bearer_token = APIKeyHeader(name="Authorization") @@ -128,17 +130,33 @@ async def get_user( def forbidExtraQueryParams(*allowedParams: str): - """Dependency to forbid extra query parameters not in allowedParams.""" - - async def checker(_req: Request): + async def checker(req: Request): if "*" in allowedParams: - return # Permit anything - incoming = set(_req.query_params.keys()) + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + allowed = set(allowedParams) - unknown = incoming - allowed - if unknown: - raise HTTPException(status_code=422, - detail=[{"type": "extra_forbidden", "loc": ["query", param], "msg": f"Unexpected query parameter: {param}"} for param in unknown]) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) return checker class StrictDateTime: From 3e8b0b287594b23ebc07357af511a7f1c9e8dd06 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 07:50:47 -0600 Subject: [PATCH 023/133] Make NamedObject reusable --- app/routers/facility/models.py | 22 ++------- app/routers/models.py | 46 +++++++++++++++++++ app/routers/status/models.py | 82 +++++++++------------------------- 3 files changed, 69 insertions(+), 81 deletions(-) create mode 100644 app/routers/models.py diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 0c04cc42..2bea62f8 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,23 +1,7 @@ -from datetime import datetime -from uuid import UUID +"""Facility-related models.""" from typing import List, Optional -from pydantic import BaseModel, Field, HttpUrl, computed_field -from .. import iri_router -from ... import config - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - +from pydantic import Field, HttpUrl +from ..models import NamedObject class Site(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/models.py b/app/routers/models.py new file mode 100644 index 00000000..f9a1c007 --- /dev/null +++ b/app/routers/models.py @@ -0,0 +1,46 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from pydantic import BaseModel, Field, computed_field +from . import iri_router +from .. import config + + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index b1f3a8bb..2c7dc765 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,6 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config +from ..models import NamedObject class Link(BaseModel): rel : str @@ -15,38 +16,6 @@ class Status(enum.Enum): unknown = "unknown" -class NamedResource(BaseModel): - id : str - name : str - description : str - last_modified : datetime.datetime - - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a - - class ResourceType(enum.Enum): website = "website" service = "service" @@ -57,28 +26,24 @@ class ResourceType(enum.Enum): unknown = "unknown" -class Resource(NamedResource): +class Resource(NamedObject): + + def _self_path(self) -> str: + return f"/status/resources/{self.id}" + capability_ids: list[str] = Field(exclude=True) group: str | None current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] - @staticmethod def find(resources, name, description, group, modified_since, resource_type): - a = NamedResource.find(resources, name, description, modified_since) + a = NamedObject.find(resources, name, description, modified_since) if group: a = [aa for aa in a if aa.group == group] if resource_type: @@ -86,25 +51,21 @@ def find(resources, name, description, group, modified_since, resource_type): return a -class Event(NamedResource): +class Event(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.incident_id}/events/{self.id}" + occurred_at : datetime.datetime status : Status resource_id : str = Field(exclude=True) incident_id : str | None = Field(exclude=True, default=None) - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}/events/{self.id}" - - @computed_field(description="The resource belonging to this event") @property def resource_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.resource_id}" - @computed_field(description="The event's incident") @property def incident_uri(self) -> str|None: @@ -123,7 +84,7 @@ def find( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, ) -> list: - events = NamedResource.find(events, name, description, modified_since) + events = NamedObject.find(events, name, description, modified_since) if resource_id: events = [e for e in events if e.resource_id == resource_id] if status: @@ -150,7 +111,11 @@ class Resolution(enum.Enum): pending = "pending" -class Incident(NamedResource): +class Incident(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.id}" + status : Status resource_ids : list[str] = Field(exclude=True) event_ids : list[str] = Field(exclude=True) @@ -159,25 +124,18 @@ class Incident(NamedResource): type : IncidentType resolution : Resolution - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def event_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}/events/{e}" for e in self.event_ids] - @computed_field(description="The list of resources that may be impacted by this incident") @property def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] def find( + self, incidents : list, name : str | None = None, description : str | None = None, @@ -189,7 +147,7 @@ def find( modified_since : datetime.datetime | None = None, resource_id : str | None = None, ) -> list: - incidents = NamedResource.find(incidents, name, description, modified_since) + incidents = NamedObject.find(incidents, name, description, modified_since) if resource_id: incidents = [e for e in incidents if resource_id in e.resource_ids] if status: From cf79917c7994ba62c0624ed8f03961acdc5736f1 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 08:16:21 -0600 Subject: [PATCH 024/133] Include operation_id for facility (similar to pull request #21) --- app/routers/facility/facility.py | 12 ++++++------ app/routers/status/models.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 99649256..0e6f9609 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -10,7 +10,7 @@ tags=["facility"], ) -@router.get("", responses=DEFAULT_RESPONSES) +@router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -19,7 +19,7 @@ async def get_facility( """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) -@router.get("/sites", responses=DEFAULT_RESPONSES) +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") async def list_sites( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -32,7 +32,7 @@ async def list_sites( """List sites""" return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") async def get_site( request: Request, site_id: str, @@ -42,7 +42,7 @@ async def get_site( """Get site by ID""" return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") async def get_site_location( request : Request, site_id: str, @@ -52,7 +52,7 @@ async def get_site_location( """Get site location by site ID""" return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) -@router.get("/locations", responses=DEFAULT_RESPONSES) +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") async def list_locations( request : Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -66,7 +66,7 @@ async def list_locations( """List locations""" return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") async def get_location( request : Request, location_id: str, diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 2c7dc765..bb1f87b0 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -134,8 +134,8 @@ def event_uris(self) -> list[str]: def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] + @staticmethod def find( - self, incidents : list, name : str | None = None, description : str | None = None, From 0d68a1ca0e5e9ff9b0bc590c85d316009d7c4039 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Thu, 15 Jan 2026 11:48:45 -0800 Subject: [PATCH 025/133] simplified facility endpoint proposal --- app/demo_adapter.py | 184 +---------------------- app/routers/facility/facility.py | 57 ------- app/routers/facility/facility_adapter.py | 47 ------ app/routers/facility/models.py | 33 +--- 4 files changed, 14 insertions(+), 307 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index daf266d8..2d2ca30c 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -65,50 +65,7 @@ def __init__(self): def _init_state(self): now = datetime.datetime.now(datetime.timezone.utc) - loc1 = facility_models.Location( - id=demo_uuid("location", "demo_location_1"), - name="Demo Location 1", - description="The first demo location", - last_modified=now, - short_name="DL1", - country_name="USA", - locality_name="Demo City", - state_or_province_name="DC", - latitude=36.173357, - longitude=-234.51452) - - loc2 = facility_models.Location( - id=demo_uuid("location", "demo_location_2"), - name="Demo Location 2", - description="The second demo location", - last_modified=now, - short_name="DL2", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - latitude=38.410558, - longitude=-286.36999) - - site1 = facility_models.Site( - id=demo_uuid("site", "demo_site_1"), - name="Demo Site 1", - description="The first demo site", - last_modified=now, - short_name="DS1", - operating_organization="Demo Org", - location_uri=loc1.self_uri, - resource_uris=[]) - site2 = facility_models.Site( - id=demo_uuid("site", "demo_site_2"), - name="Demo Site 2", - description="The second demo site", - last_modified=now, - short_name="DS2", - operating_organization="Demo Org", - location_uri=loc2.self_uri, - resource_uris=[]) - - facility = facility_models.Facility( + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", description="A demo facility for testing the IRI Facility API", @@ -116,24 +73,15 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - site_uris=[site1.self_uri, site2.self_uri], - location_uris=[loc1.self_uri, loc2.self_uri], - resource_uris=[], - event_uris=[], - incident_uris=[], - capability_uris=[], - project_uris=[], - project_allocation_uris=[], - user_allocation_uris=[], + facility_uri="https://www.demo.example", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + street_address="1 main st", + latitude=38.410558, + longitude=-286.36999 ) - self.facility = facility - loc1.site_uris.append(site1.self_uri) - loc2.site_uris.append(site2.self_uri) - self.locations = [loc1, loc2] - self.sites = [site1, site2] - - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -269,122 +217,6 @@ async def get_facility( ) -> facility_models.Facility: return self.facility - - async def list_sites( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - - sites = self.sites - - if name: - sites = [s for s in sites if name.lower() in s.name.lower()] - - if short_name: - sites = [s for s in sites if s.short_name == short_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - sites = [s for s in sites if s.last_modified > ms] - - o = offset or 0 - l = limit or len(sites) - return sites[o:o+l] - - - async def get_site( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site: - - site = next((s for s in self.sites if s.id == site_id), None) - if not site: - raise HTTPException(status_code=404, detail="Site not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if site.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) - - return site - - - async def get_site_location( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - site = await self.get_site(site_id) - - if not site.location_uri: - raise HTTPException(status_code=404, detail="Site has no location") - - location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - - return location - - - async def list_locations( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - - locs = self.locations - - if name: - locs = [l for l in locs if name.lower() in l.name.lower()] - - if short_name: - locs = [l for l in locs if l.short_name == short_name] - - if country_name: - locs = [l for l in locs if l.country_name == country_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - locs = [l for l in locs if l.last_modified > ms] - - o = offset or 0 - l = limit or len(locs) - return locs[o:o+l] - - - async def get_location( - self: "DemoAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - location = next((l for l in self.locations if l.id == location_id), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - return location - # ---------------------------- # Status API # ---------------------------- diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 0e6f9609..9b7545b2 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,60 +18,3 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - -@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") -async def list_sites( - request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), - )-> list[models.Site]: - """List sites""" - return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) - -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") -async def get_site( - request: Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Site: - """Get site by ID""" - return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) - -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") -async def get_site_location( - request : Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get site location by site ID""" - return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) - -@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") -async def list_locations( - request : Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - country_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), - )-> list[models.Location]: - """List locations""" - return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) - -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") -async def get_location( - request : Request, - location_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get location by ID""" - return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index d316674d..bccdcfe5 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,50 +16,3 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass - - @abstractmethod - async def list_sites( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - pass - - @abstractmethod - async def get_site( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site | None: - pass - - @abstractmethod - async def get_site_location( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass - - @abstractmethod - async def list_locations( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - pass - - @abstractmethod - async def get_location( - self: "FacilityAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 2bea62f8..efd93dd0 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,20 +1,13 @@ """Facility-related models.""" -from typing import List, Optional +from typing import Optional from pydantic import Field, HttpUrl from ..models import NamedObject -class Site(NamedObject): - def _self_path(self) -> str: - return f"/facility/sites/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Site.") - operating_organization: str = Field(..., description="Organization operating the Site.") - location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") - -class Location(NamedObject): - def _self_path(self) -> str: - return f"/facility/locations/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Location.") +class Facility(NamedObject): + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization's name.") + facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -23,20 +16,6 @@ def _self_path(self) -> str: altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") -class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization’s name.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") - location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") - event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") - incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") - capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") - project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") - project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") - user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") From ed6befa032e9b8eb47662cb8cab653ddab52a991 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 026/133] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- app/demo_adapter.py | 2 -- app/routers/facility/facility.py | 1 + app/routers/facility/facility_adapter.py | 1 + app/routers/facility/models.py | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 2d2ca30c..0d581038 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -57,9 +57,7 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - self.locations = [] self.facility = {} - self.sites = [] self._init_state() diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 9b7545b2..fdba1241 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,3 +18,4 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index bccdcfe5..cbee9515 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,3 +16,4 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass + diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index efd93dd0..5b21d8a3 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -19,3 +19,4 @@ class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + From 28e5ba75413b14b433f68bf4c2fc2d9cda850991 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 15:27:00 -0600 Subject: [PATCH 027/133] Remove /api/v1 --- app/main.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/app/main.py b/app/main.py index 080645ef..78641430 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,6 @@ """Main API application""" import logging from fastapi import FastAPI -from fastapi import Request -from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -22,33 +20,6 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" -@APP.get(api_prefix) -async def api_discovery(request: Request): - base = str(request.base_url).rstrip("/") - items = [] - for route in APP.router.routes: - if not isinstance(route, APIRoute): - continue - # skip docs & openapi - if route.path.startswith("/docs") or route.path.startswith("/openapi"): - continue - for method in route.methods: - if method == "HEAD" or method == "OPTIONS": - continue - items.append({ - "id": route.name or f"{method}_{route.path}", - "method": method, - "path": route.path, - "_links": [ - { - "rel": "self", - "href": f"{base.rstrip('/')}{route.path}", - "type": "application/json" - } - ] - }) - return items - # Attach routers under the prefix APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) @@ -57,4 +28,4 @@ async def api_discovery(request: Request): APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file +logging.getLogger().info(f"API path: {api_prefix}") From e8d256b4f630c525f90c4097743c61a8daca805f Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:14:23 -0600 Subject: [PATCH 028/133] Refactor shared validators & models to fix import loading issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move common validators and base models into routers/dependencies and update imports across routers and schemas to use the new shared location. This keeps API behavior unchanged. No functional API changes — purely structural and validation hygiene: --- app/routers/account/account.py | 17 +- app/routers/account/models.py | 15 +- app/routers/compute/compute.py | 88 ++++++----- app/routers/compute/models.py | 70 ++++---- app/routers/dependencies.py | 176 +++++++++++++++++++++ app/routers/facility/facility.py | 8 +- app/routers/facility/models.py | 2 +- app/routers/filesystem/facility_adapter.py | 4 +- app/routers/filesystem/filesystem.py | 42 +---- app/routers/filesystem/models.py | 9 +- app/routers/iri_router.py | 77 --------- app/routers/models.py | 46 ------ app/routers/status/models.py | 2 +- app/routers/status/status.py | 25 +-- app/routers/task/models.py | 19 ++- 15 files changed, 317 insertions(+), 283 deletions(-) create mode 100644 app/routers/dependencies.py delete mode 100644 app/routers/models.py diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 508d556b..b2c41f44 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import forbidExtraQueryParams router = iri_router.IriRouter( @@ -21,7 +22,7 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -36,7 +37,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -55,7 +56,7 @@ async def get_capability( ) async def get_projects( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -74,7 +75,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -97,7 +98,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -121,7 +122,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -147,7 +148,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -176,7 +177,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 6ed69ea0..c012ee32 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel, computed_field, Field +from pydantic import computed_field, Field import enum from ... import config +from ..dependencies import IRIBaseModel class AllocationUnit(enum.Enum): @@ -9,7 +10,7 @@ class AllocationUnit(enum.Enum): inodes = "inodes" -class Capability(BaseModel): +class Capability(IRIBaseModel): """ An aspect of a resource that can have an allocation. For example, Perlmutter nodes with GPUs @@ -22,7 +23,7 @@ class Capability(BaseModel): units: list[AllocationUnit] -class User(BaseModel): +class User(IRIBaseModel): """A user of the facility""" id: str name: str @@ -31,7 +32,7 @@ class User(BaseModel): # we could expose more fields here (eg. email) but it might be against policy -class Project(BaseModel): +class Project(IRIBaseModel): """A project and its users at a facility""" id: str name: str @@ -39,14 +40,14 @@ class Project(BaseModel): user_ids: list[str] -class AllocationEntry(BaseModel): +class AllocationEntry(IRIBaseModel): """Base class for allocations.""" allocation: float # how much this allocation can spend usage: float # how much this allocation has spent unit: AllocationUnit -class ProjectAllocation(BaseModel): +class ProjectAllocation(IRIBaseModel): """ A project's allocation for a capability. (aka. repo) This allocation is a piece of the total allocation for the capability. (eg. 5% of the total node hours of Perlmutter GPU nodes) @@ -71,7 +72,7 @@ def capability_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{self.capability_id}" -class UserAllocation(BaseModel): +class UserAllocation(IRIBaseModel): """ A user's allcation in a project. This allocation is a piece of the project's allocation. diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 72a8169c..7a2db23c 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,9 +1,10 @@ -from typing import List, Annotated -from fastapi import HTTPException, Request, Depends, status, Form, Query +from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router + from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router +from ..dependencies import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -24,6 +25,7 @@ async def submit_job( resource_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Submit a job on a compute resource @@ -45,39 +47,41 @@ async def submit_job( return await router.adapter.submit_job(resource, user, job_spec) -@router.post( - "/job/script/{resource_id:str}", - dependencies=[Depends(router.current_user)], - response_model=models.Job, - response_model_exclude_unset=True, - responses=DEFAULT_RESPONSES, - operation_id="launchJobScript", -) -async def submit_job_path( - resource_id: str, - job_script_path : str, - request : Request, - args : Annotated[List[str], Form()] = [], - ): - """ - Submit a job on a compute resource - - - **resource**: the name of the compute resource to use - - **job_script_path**: path to the job script on the compute resource - - **args**: optional arguments to the job script - - This command will attempt to submit a job and return its id. - """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - - # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) - - # the handler can use whatever means it wants to submit the job and then fill in its id - # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.submit_job_script(resource, user, job_script_path, args) +# TODO: this conflicts with PUT commented out while we finalize the API design +#@router.post( +# "/job/script/{resource_id:str}", +# dependencies=[Depends(router.current_user)], +# response_model=models.Job, +# response_model_exclude_unset=True, +# responses=DEFAULT_RESPONSES, +# operation_id="launchJobScript", +#) +#async def submit_job_path( +# resource_id: str, +# job_script_path : str, +# request : Request, +# args : Annotated[List[str], Form()] = [], +# _forbid = Depends(iri_router.forbidExtraQueryParams("job_script_path")), +# ): +# """ +# Submit a job on a compute resource +# +# - **resource**: the name of the compute resource to use +# - **job_script_path**: path to the job script on the compute resource +# - **args**: optional arguments to the job script +# +# This command will attempt to submit a job and return its id. +# """ +# user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) +# if not user: +# raise HTTPException(status_code=404, detail="User not found") +# +# # look up the resource (todo: maybe ensure it's available) +# resource = await status_router.adapter.get_resource(resource_id) +# +# # the handler can use whatever means it wants to submit the job and then fill in its id +# # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs +# return await router.adapter.submit_job_script(resource, user, job_script_path, args) @router.put( @@ -93,6 +97,7 @@ async def update_job( job_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Update a previously submitted job for a resource. @@ -126,8 +131,9 @@ async def get_job_status( resource_id : str, job_id : str, request : Request, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -149,7 +155,7 @@ async def get_job_status( response_model=list[models.Job], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, - operation_id="getJobs", + operation_id="getAllJobs", ) async def get_job_statuses( resource_id : str, @@ -157,8 +163,9 @@ async def get_job_statuses( offset : int = Query(default=0, ge=0, le=1000), limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -187,6 +194,7 @@ async def cancel_job( resource_id : str, job_id : str, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """Cancel a job""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 167643ec..7bc4ab2c 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,41 +1,42 @@ from typing import Annotated -from pydantic import BaseModel, field_serializer, ConfigDict, Field from enum import IntEnum +from pydantic import field_serializer, ConfigDict, StrictBool, Field +from ..dependencies import IRIBaseModel -class ResourceSpec(BaseModel): +class ResourceSpec(IRIBaseModel): """ Specification of computational resources required for a job. """ - node_count: Annotated[int | None, Field(description="Number of compute nodes to allocate")] = None - process_count: Annotated[int | None, Field(description="Total number of processes to launch")] = None - processes_per_node: Annotated[int | None, Field(description="Number of processes to launch per node")] = None - cpu_cores_per_process: Annotated[int | None, Field(description="Number of CPU cores to allocate per process")] = None - gpu_cores_per_process: Annotated[int | None, Field(description="Number of GPU cores to allocate per process")] = None - exclusive_node_use: Annotated[bool, Field(description="Whether to request exclusive use of allocated nodes")] = True - memory: Annotated[int | None, Field(description="Amount of memory to allocate in bytes")] = None + node_count: Annotated[int | None, Field(ge=1, description="Number of compute nodes to allocate")] = None + process_count: Annotated[int | None, Field(ge=1, description="Total number of processes to launch")] = None + processes_per_node: Annotated[int | None, Field(ge=1, description="Number of processes to launch per node")] = None + cpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of CPU cores to allocate per process")] = None + gpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of GPU cores to allocate per process")] = None + exclusive_node_use: Annotated[StrictBool, Field(description="Whether to request exclusive use of allocated nodes")] = True + memory: Annotated[int | None, Field(ge=1,description="Amount of memory to allocate in bytes")] = None -class JobAttributes(BaseModel): +class JobAttributes(IRIBaseModel): """ Additional attributes and scheduling parameters for a job. """ - duration: Annotated[int | None, Field(description="Duration in seconds", ge=0, examples=[30, 60, 120])] = None - queue_name: Annotated[str | None, Field(description="Name of the queue or partition to submit the job to")] = None - account: Annotated[str | None, Field(description="Account or project to charge for resource usage")] = None - reservation_id: Annotated[str | None, Field(description="ID of a reservation to use for the job")] = None + duration: Annotated[int | None, Field(description="Duration in seconds", ge=1, examples=[30, 60, 120])] = None + queue_name: Annotated[str | None, Field(min_length=1, description="Name of the queue or partition to submit the job to")] = None + account: Annotated[str | None, Field(min_length=1, description="Account or project to charge for resource usage")] = None + reservation_id: Annotated[str | None, Field(min_length=1, description="ID of a reservation to use for the job")] = None custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs")] = {} -class VolumeMount(BaseModel): +class VolumeMount(IRIBaseModel): """ Represents a volume mount for a container. """ - source: Annotated[str, Field(description="The source path on the host system to mount")] - target: Annotated[str, Field(description="The target path inside the container where the volume will be mounted")] - read_only: Annotated[bool, Field(description="Whether the mount should be read-only")] = True + source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount")] + target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted")] + read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only")] = True -class Container(BaseModel): +class Container(IRIBaseModel): """ Represents a container specification for job execution. @@ -44,33 +45,33 @@ class Container(BaseModel): to determine if the container should be run with MPI support. The container should by default. be run with host networking. """ - image: Annotated[str, Field(description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] + image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] -class JobSpec(BaseModel): +class JobSpec(IRIBaseModel): """ Specification for job. """ model_config = ConfigDict(extra="forbid") - executable: Annotated[str | None, Field(description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.")] = None + executable: Annotated[str | None, Field(min_length=1, description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.")] = None container: Annotated[Container | None, Field(description="Container specification for containerized execution")] = None arguments: Annotated[list[str], Field(description="Command-line arguments to pass to the executable or container")] = [] - directory: Annotated[str | None, Field(description="Working directory for the job")] = None - name: Annotated[str | None, Field(description="Name of the job")] = None - inherit_environment: Annotated[bool, Field(description="Whether to inherit the environment variables from the submission environment")] = True + directory: Annotated[str | None, Field(min_length=1, description="Working directory for the job")] = None + name: Annotated[str | None, Field(min_length=1, description="Name of the job")] = None + inherit_environment: Annotated[StrictBool, Field(description="Whether to inherit the environment variables from the submission environment")] = True environment: Annotated[dict[str, str], Field(description="Environment variables to set for the job. If container is specified, these will be set inside the container.")] = {} - stdin_path: Annotated[str | None, Field(description="Path to file to use as standard input")] = None - stdout_path: Annotated[str | None, Field(description="Path to file to write standard output")] = None - stderr_path: Annotated[str | None, Field(description="Path to file to write standard error")] = None + stdin_path: Annotated[str | None, Field(min_length=1, description="Path to file to use as standard input")] = None + stdout_path: Annotated[str | None, Field(min_length=1, description="Path to file to write standard output")] = None + stderr_path: Annotated[str | None, Field(min_length=1, description="Path to file to write standard error")] = None resources: Annotated[ResourceSpec | None, Field(description="Resource requirements for the job")] = None attributes: Annotated[JobAttributes | None, Field(description="Additional job attributes such as duration, queue, and account")] = None - pre_launch: Annotated[str | None, Field(description="Script or commands to run before launching the job")] = None - post_launch: Annotated[str | None, Field(description="Script or commands to run after the job completes")] = None - launcher: Annotated[str | None, Field(description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None + pre_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run before launching the job")] = None + post_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run after the job completes")] = None + launcher: Annotated[str | None, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None -class CommandResult(BaseModel): +class CommandResult(IRIBaseModel): status : str result : str | None = None @@ -110,20 +111,19 @@ class JobState(IntEnum): """Represents a job that was canceled by a call to :func:`~psij.Job.cancel()`.""" -class JobStatus(BaseModel): +class JobStatus(IRIBaseModel): state : JobState time : float | None = None message : str | None = None exit_code : int | None = None meta_data : dict[str, object] | None = None - @field_serializer('state') def serialize_state(self, state: JobState): return state.name -class Job(BaseModel): +class Job(IRIBaseModel): id : str status : JobStatus | None = None job_spec : JobSpec | None = None diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py new file mode 100644 index 00000000..17b4f096 --- /dev/null +++ b/app/routers/dependencies.py @@ -0,0 +1,176 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from urllib.parse import parse_qs + +from pydantic_core import core_schema +from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer +from fastapi import Request, HTTPException + +from .. import config + + +# These are Pydantic custom types for strict validation +# that are not implmented in Pydantic by default. +# ----------------------------------------------------------------------- +# StrictBool: a strict boolean type +class StrictBool: + """Strict boolean: + - Accepts: real booleans, 'true', 'false' + - Rejects everything else. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + """Validate the input value as a strict boolean.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + v = value.strip().lower() + if v == "true": + return True + if v == "false": + return False + raise ValueError("Invalid boolean value. Expected 'true' or 'false'.") + raise ValueError("Invalid boolean value. Expected true/false or 'true'/'false'.") + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "boolean", + "description": "Strict boolean. Only true/false allowed (bool or string)." + } + +# ----------------------------------------------------------------------- +# StrictDateTime: a strict ISO8601 datetime type + +class StrictDateTime: + """ + Strict ISO8601 datetime: + - Accepts datetime objects + - Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 + - Converts 'Z' → UTC + - Converts naive datetimes → UTC + - Rejects integers ("0"), null, garbage strings, etc. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + if isinstance(value, datetime.datetime): + return StrictDateTime._normalize(value) + if not isinstance(value, str): + raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") + v = value.strip() + if v.endswith("Z"): + v = v[:-1] + "+00:00" + try: + dt = datetime.datetime.fromisoformat(v) + except Exception as ex: + raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex + + return StrictDateTime._normalize(dt) + + @staticmethod + def _normalize(dt: datetime.datetime) -> datetime.datetime: + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "string", + "format": "date-time", + "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." + } + + +def forbidExtraQueryParams(*allowedParams: str): + async def checker(req: Request): + if "*" in allowedParams: + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + + allowed = set(allowedParams) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) + return checker + + +class IRIBaseModel(BaseModel): + """Base model for IRI models.""" + model_config = ConfigDict(extra="allow") + + @model_serializer(mode="wrap") + def _hide_extra(self, handler): + data = handler(self) + extra = getattr(self, "__pydantic_extra__", {}) or {} + for k in extra: + data.pop(k, None) + return data + +class NamedObject(IRIBaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index fdba1241..1c389186 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -1,7 +1,8 @@ -from fastapi import Request, HTTPException, Depends, Query +from fastapi import Request, Depends, Query from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( @@ -13,9 +14,8 @@ @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5b21d8a3..5db7a18b 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..models import NamedObject +from ..dependencies import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index 2c08a3cd..a70efb0d 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -1,15 +1,15 @@ import os from abc import abstractmethod +from typing import Any, Tuple from ..status import models as status_models from ..account import models as account_models from . import models as filesystem_models from ..iri_router import AuthenticatedAdapter -from typing import Any, Tuple def to_int(name, default_value): try: - return os.environ.get(name) or default_value + return int(os.environ.get(name) or default_value) except: return default_value diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index a111484a..d583c642 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -354,40 +354,13 @@ async def get_view( resource_id: str, request : Request, path: Annotated[str, Query(description="File path")], - size: Annotated[ - int, - Query( - alias="size", - description="Value, in bytes, of the size of data to be retrieved from the file.", - ), - ] = facility_adapter.OPS_SIZE_LIMIT, - offset: Annotated[ - int, - Query( - alias="offset", - description="Value in bytes of the offset.", - ), - ] = 0, -) -> str: - user, resource = await _user_resource(resource_id, request) - if offset < 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`offset` value must be an integer value equal or greater than 0", - ) - - if size <= 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`size` value must be an integer value greater than 0", - ) + size: Annotated[int, Query(description="Value, in bytes, of the size of data to be retrieved from the file.", + ge=1, le=facility_adapter.OPS_SIZE_LIMIT)] = facility_adapter.OPS_SIZE_LIMIT, - if size > facility_adapter.OPS_SIZE_LIMIT: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"`size` value must be less than {facility_adapter.OPS_SIZE_LIMIT} bytes", - ) + offset: Annotated[int, Query( description="Value in bytes of the offset.", ge=0)] = 0 + ) -> str: + user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user, @@ -397,9 +370,8 @@ async def get_view( command="view", args={ "path": path, - "size": size or facility_adapter.OPS_SIZE_LIMIT, - "offset": offset or 0, - + "size": size, + "offset": offset, } ) ) diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index b24c908a..d32ad943 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -12,11 +12,10 @@ class CompressionType(str, Enum): - none = "none" - bzip2 = "bzip2" - gzip = "gzip" - xz = "xz" - + none = "none" + bzip2 = "bzip2" + gzip = "gzip" + xz = "xz" class ContentUnit(str, Enum): lines = "lines" diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 2de7e693..d3fc9e16 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -2,11 +2,9 @@ import os import logging import importlib -import datetime from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader -from pydantic_core import core_schema from .account.models import User @@ -127,78 +125,3 @@ async def get_user( Retrieve additional user information (name, email, etc.) for the given user_id. """ pass - - -def forbidExtraQueryParams(*allowedParams: str): - async def checker(req: Request): - if "*" in allowedParams: - return - - raw_qs = req.scope.get("query_string", b"") - parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) - - allowed = set(allowedParams) - - for key, values in parsed.items(): - if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) - - if len(values) > 1: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) - return checker - -class StrictDateTime: - """ - Strict ISO8601 datetime: - ✔ Accepts datetime objects - ✔ Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 - ✔ Converts 'Z' → UTC - ✔ Converts naive datetimes → UTC - ✘ Rejects integers ("0"), null, garbage strings, etc. - """ - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - return core_schema.no_info_plain_validator_function(cls.validate) - - @staticmethod - def validate(value): - if isinstance(value, datetime.datetime): - return StrictDateTime._normalize(value) - if not isinstance(value, str): - raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") - v = value.strip() - if v.endswith("Z"): - v = v[:-1] + "+00:00" - try: - dt = datetime.datetime.fromisoformat(v) - except Exception as ex: - raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex - - return StrictDateTime._normalize(dt) - - @staticmethod - def _normalize(dt: datetime.datetime) -> datetime.datetime: - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - - @classmethod - def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "string", - "format": "date-time", - "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." - } diff --git a/app/routers/models.py b/app/routers/models.py deleted file mode 100644 index f9a1c007..00000000 --- a/app/routers/models.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Default models used by multiple routers.""" -import datetime -from typing import Optional -from pydantic import BaseModel, Field, computed_field -from . import iri_router -from .. import config - - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - """Computed self URI property.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index bb1f87b0..ff5d4e48 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..models import NamedObject +from ..dependencies import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 7bb0fd0b..2daf30e7 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -23,9 +24,9 @@ async def get_resources( group : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - modified_since: iri_router.StrictDateTime = Query(default=None), + modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), ) -> list[models.Resource]: return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) @@ -60,14 +61,14 @@ async def get_incidents( description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), type_: models.IncidentType = Query(alias="type", default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), ) -> list[models.Incident]: return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) @@ -104,13 +105,13 @@ async def get_events( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) diff --git a/app/routers/task/models.py b/app/routers/task/models.py index da3c5ccc..cea97873 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,19 +1,18 @@ -from pydantic import BaseModel import enum +from pydantic import BaseModel class TaskStatus(str, enum.Enum): - pending = "pending" - active = "active" - completed = "completed" - failed = "failed" - canceled = "canceled" - + pending = "pending" + active = "active" + completed = "completed" + failed = "failed" + canceled = "canceled" class TaskCommand(BaseModel): - router: str - command: str - args: dict + router: str + command: str + args: dict class Task(BaseModel): From 58b9ef8c0dc6c55163d50fea89dd3c19c34edfb1 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:17:00 -0600 Subject: [PATCH 029/133] Github Action to validate api --- .github/workflows/api-validation.yml | 133 +++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 .github/workflows/api-validation.yml diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml new file mode 100644 index 00000000..624e84e2 --- /dev/null +++ b/.github/workflows/api-validation.yml @@ -0,0 +1,133 @@ +name: API Validation with Schemathesis + +on: + pull_request: + push: + branches: [ main ] + +jobs: + schemathesis: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged + - name: Checkout schema validator repository + uses: actions/checkout@v4 + with: + repository: juztas/iri-facility-api-docs + ref: schemavalidator + path: schema-validator + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + run: pip install uv + + - name: Build an image + run: docker build --platform=linux/amd64 -t iri-facility-api-base . + + - name: Run Facility API container + run: | + docker run -d \ + -p 8000:8000 \ + --platform=linux/amd64 \ + --name iri-facility-api-base \ + -e IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + -e API_URL_ROOT=http://127.0.0.1:8000 \ + -e IRI_API_TOKEN=12345 \ + iri-facility-api-base + + - name: Wait for API to be ready + run: | + for i in {1..60}; do + if curl -fs http://127.0.0.1:8000/openapi.json; then + echo "API ready" + exit 0 + fi + sleep 2 + done + echo "API did not start" + exit 1 + + - name: Create venv & install validator dependencies + run: | + uv venv + source .venv/bin/activate + uv pip install -r schema-validator/verification/requirements.txt + + - name: Run Schemathesis validation (local spec) + id: schemathesis_local + env: + IRI_API_TOKEN: "12345" # This is dummy token for testing (mock adapter) + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://127.0.0.1:8000 \ + --report-name schemathesis-local + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Run Schemathesis validation (official spec) + id: schemathesis_official + env: + IRI_API_TOKEN: "12345" + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://localhost:8000 \ + --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --report-name schemathesis-official + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Fail if any Schemathesis run failed + if: always() + run: | + if [ "${{ steps.schemathesis_local.outputs.exitcode }}" != "0" ] || \ + [ "${{ steps.schemathesis_official.outputs.exitcode }}" != "0" ]; then + echo "One or more Schemathesis validations failed" + exit 1 + else + echo "Both Schemathesis validations passed" + fi + + - name: Upload Schemathesis report # This only works on git actions + if: always() && env.ACT != 'true' + uses: actions/upload-artifact@v4 + with: + if-no-files-found: warn + name: schemathesis-report + path: | + schemathesis-local.html + schemathesis-local.xml + schemathesis-official.html + schemathesis-official.xml + + - name: Save Schemathesis reports locally # This only works if run locally with act + if: always() && env.ACT == 'true' + run: | + mkdir -p artifacts + cp schemathesis-local.html schemathesis-local.xml artifacts/ || true + cp schemathesis-official.html schemathesis-official.xml artifacts/ || true + + - name: Dump API logs + if: always() + run: docker logs iri-facility-api-base || true + + - name: Stop container + if: always() + run: docker stop iri-facility-api-base || true From bc5584359fbd0152cf350abd3bbed7d69939ba6c Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:41:35 -0600 Subject: [PATCH 030/133] Add custom get extra function --- app/routers/dependencies.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py index 17b4f096..7bfaf0b8 100644 --- a/app/routers/dependencies.py +++ b/app/routers/dependencies.py @@ -136,6 +136,10 @@ def _hide_extra(self, handler): data.pop(k, None) return data + def get_extra(self, key, default=None): + return getattr(self, "__pydantic_extra__", {}).get(key, default) + + class NamedObject(IRIBaseModel): id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") def _self_path(self) -> str: From 75a5cb6e1502c3423f8a1fbf0669661402e5420e Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 13:06:36 -0600 Subject: [PATCH 031/133] Implement opentelemetry. Use UTC in demo adapter --- Makefile | 1 + README.md | 2 ++ VALIDATION.MD | 23 +++++++++++++++++++++++ app/config.py | 5 +++++ app/demo_adapter.py | 31 +++++++++++++++++++++---------- app/main.py | 32 ++++++++++++++++++++++++++++++++ pyproject.toml | 8 ++++++-- 7 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 VALIDATION.MD diff --git a/Makefile b/Makefile index ba7552ab..abd87e4a 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ dev : .venv IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + OPENTELEMETRY_ENABLED=true \ API_URL_ROOT='http://127.0.0.1:8000' fastapi dev diff --git a/README.md b/README.md index 5ef41a6a..02b796c9 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ If using docker (see next section), your dockerfile could extend this reference - `API_URL_ROOT`: the base url when constructing links returned by the api (eg.: https://iri.myfacility.com) - `API_PREFIX`: the path prefix where the api is hosted. Defaults to `/`. (eg.: `/api`) - `API_URL`: the path to the api itself. Defaults to `api/v1`. +- `OPENTELEMETRY_ENABLED`: Enables OpenTelemetry. If enabled, the application will use OpenTelemetry SDKs and emit traces, metrics, and logs. Default to false +- `OTLP_ENDPOINT`: OpenTelemetry Protocol collector endpoint to export telemetry data. If empty or not set, telemetry data is logged locally to log file. Default: "" Links to data, created by this api, will concatenate these values producing links, eg: `https://iri.myfacility.com/my_api_prefix/my_api_url/projects/123` diff --git a/VALIDATION.MD b/VALIDATION.MD new file mode 100644 index 00000000..26f77ef6 --- /dev/null +++ b/VALIDATION.MD @@ -0,0 +1,23 @@ +# API Validation with Schemathesis + +On every pull request or push to `main` branch, Github Actions run the following steps below that validates an IRI Facility API implementation against OpenAPI spec using Schemathesis. + +1. Builds the Facility API Docker image from Dockerfile. +2. Runs the API container with demo adapter. +3. Waits for `/openapi.json` to become available on localhost:8000. +4. Runs Schemathesis validation twice: + - Against Facilities API’s OpenAPI spec. (http://localhost:8000/openapi.json) + - Against the official IRI Facility API OpenAPI spec. (https://github.com/doe-iri/iri-facility-api-docs/blob/main/specification/openapi/openapi_iri_facility_api_v1.json) +5. Fails the workflow if either validation fails. +6. Saves Schemathesis HTML/XML reports as artifacts (or saves it locally when run with `act`). +7. Dumps API container logs and do clean up to stop container. + +## Running locally + +```bash +act -W .github/workflows/api-validator.yml -s GITHUB_TOKEN= +``` + +## Known issues + +Python implementation not fully aligns with the official Specification. Running against Official Spec will continue to fail, until Spec or Py implementation is fixed. diff --git a/app/config.py b/app/config.py index 078b6a52..71a7c20d 100644 --- a/app/config.py +++ b/app/config.py @@ -40,3 +40,8 @@ API_URL_ROOT = os.environ.get("API_URL_ROOT", "https://api.iri.nersc.gov") API_PREFIX = os.environ.get("API_PREFIX", "/") API_URL = os.environ.get("API_URL", "api/v1") + +OPENTELEMETRY_ENABLED = os.environ.get("OPENTELEMETRY_ENABLED", "false").lower() == "true" +OPENTELEMETRY_DEBUG = os.environ.get("OPENTELEMETRY_DEBUG", "false").lower() == "true" +OTLP_ENDPOINT = os.environ.get("OTLP_ENDPOINT", "") +OTEL_SAMPLE_RATE = float(os.environ.get("OTEL_SAMPLE_RATE", "0.2")) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 0d581038..e63c7201 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -1,7 +1,6 @@ import datetime import random import uuid -import time import os import stat import pwd @@ -45,6 +44,16 @@ def demo_uuid(kind: str, name: str) -> str: return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) +def utc_now() -> datetime.datetime: + """Return current UTC datetime timestamp""" + return datetime.datetime.now(datetime.timezone.utc) + + +def utc_timestamp() -> int: + """Return current UTC datetime timestamp as integer""" + return int(utc_now().timestamp()) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): @@ -62,7 +71,8 @@ def __init__(self): def _init_state(self): - now = datetime.datetime.now(datetime.timezone.utc) + now = utc_now() + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -80,7 +90,8 @@ def _init_state(self): longitude=-286.36999 ) - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) + + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -352,7 +363,7 @@ async def submit_job( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -371,7 +382,7 @@ async def submit_job_script( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -390,7 +401,7 @@ async def update_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.ACTIVE, - time=time.time(), + time=utc_timestamp(), message="job updated", exit_code=None, meta_data={ "account": "account1" }, @@ -410,7 +421,7 @@ async def get_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.COMPLETED, - time=time.time(), + time=utc_timestamp(), message="job completed successfully", exit_code=0, meta_data={ "account": "account1" }, @@ -432,7 +443,7 @@ async def get_jobs( id=f"job_{i}", status=compute_models.JobStatus( state=random.choice([s for s in compute_models.JobState]), - time=time.time() - (random.random() * 100), + time=utc_timestamp() - int(random.random() * 100), message="", exit_code=random.choice([0, 0, 0, 0, 0, 1, 1, 128, 127]), meta_data={ "account": "account1" }, @@ -900,7 +911,7 @@ class DemoTaskQueue: @staticmethod async def _process_tasks(da: DemoAdapter): - now = time.time() + now = utc_timestamp() _tasks = [] for t in DemoTaskQueue.tasks: if now - t.start > 5 * 60 and t.status in [task_models.TaskStatus.completed, task_models.TaskStatus.canceled, task_models.TaskStatus.failed]: @@ -921,5 +932,5 @@ async def _process_tasks(da: DemoAdapter): @staticmethod def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> str: task_id = f"task_{len(DemoTaskQueue.tasks)}" - DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=time.time())) + DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) return task_id diff --git a/app/main.py b/app/main.py index 78641430..fa3f1eda 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,13 @@ """Main API application""" import logging from fastapi import FastAPI +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, BatchSpanProcessor, SimpleSpanProcessor +from opentelemetry.sdk.trace.sampling import TraceIdRatioBased, ParentBased +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -13,9 +20,34 @@ from . import config +# ------------------------------------------------------------------ +# OpenTelemetry Tracing Configuration +# ------------------------------------------------------------------ +if config.OPENTELEMETRY_ENABLED: + resource = Resource.create({ + "service.name": "iri-facility-api", + "service.version": config.API_VERSION, + "service.endpoint": config.API_URL_ROOT}) + + samplerate = "1.0" if config.OPENTELEMETRY_DEBUG else config.OTEL_SAMPLE_RATE + provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(samplerate))) + trace.set_tracer_provider(provider) + + if config.OTLP_ENDPOINT: + exporter = OTLPSpanExporter(endpoint=config.OTLP_ENDPOINT, insecure=True) + span_processor = BatchSpanProcessor(exporter) + else: + exporter = ConsoleSpanExporter() + span_processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(span_processor) + tracer = trace.get_tracer(__name__) +# ------------------------------------------------------------------ APP = FastAPI(**config.API_CONFIG) +if config.OPENTELEMETRY_ENABLED: + FastAPIInstrumentor.instrument_app(APP) + install_error_handlers(APP) api_prefix = f"{config.API_PREFIX}{config.API_URL}" diff --git a/pyproject.toml b/pyproject.toml index 03858a43..1f5c0d64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,5 +5,9 @@ requires-python = ">=3.12" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", - "humps>=0.2.2" -] + "humps>=0.2.2", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation-fastapi", + "opentelemetry-exporter-otlp" +] \ No newline at end of file From be3057016ef83861f7fd6d166bedbde8bafc91fe Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 14:28:41 -0600 Subject: [PATCH 032/133] Rename dependencies to common --- app/routers/account/account.py | 2 +- app/routers/account/models.py | 2 +- app/routers/{dependencies.py => common.py} | 0 app/routers/compute/compute.py | 2 +- app/routers/compute/models.py | 2 +- app/routers/facility/facility.py | 2 +- app/routers/facility/models.py | 2 +- app/routers/status/models.py | 2 +- app/routers/status/status.py | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename app/routers/{dependencies.py => common.py} (100%) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index b2c41f44..fe16b8bf 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import forbidExtraQueryParams +from ..common import forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/account/models.py b/app/routers/account/models.py index c012ee32..2158575e 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,7 +1,7 @@ from pydantic import computed_field, Field import enum from ... import config -from ..dependencies import IRIBaseModel +from ..common import IRIBaseModel class AllocationUnit(enum.Enum): diff --git a/app/routers/dependencies.py b/app/routers/common.py similarity index 100% rename from app/routers/dependencies.py rename to app/routers/common.py diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 7a2db23c..20081ee6 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -4,7 +4,7 @@ from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router -from ..dependencies import forbidExtraQueryParams, StrictBool +from ..common import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 7bc4ab2c..a56d4fe4 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,7 +1,7 @@ from typing import Annotated from enum import IntEnum from pydantic import field_serializer, ConfigDict, StrictBool, Field -from ..dependencies import IRIBaseModel +from ..common import IRIBaseModel class ResourceSpec(IRIBaseModel): diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 1c389186..c0d3e80d 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -2,7 +2,7 @@ from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5db7a18b..508dbcf3 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..dependencies import NamedObject +from ..common import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/status/models.py b/app/routers/status/models.py index ff5d4e48..d338e2cf 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..dependencies import NamedObject +from ..common import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 2daf30e7..d599866c 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, From 6b85fbfcf66e341523ec61ff4dc6d245bdd21f42 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 09:26:32 -0600 Subject: [PATCH 033/133] Do not swallow exceptions --- app/routers/compute/compute.py | 9 ++++----- app/routers/iri_router.py | 6 ++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 20081ee6..cca45be5 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,3 +1,4 @@ +"""Compute resource API router""" from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router @@ -6,13 +7,13 @@ from ..status.status import router as status_router from ..common import forbidExtraQueryParams, StrictBool + router = iri_router.IriRouter( facility_adapter.FacilityAdapter, prefix="/compute", tags=["compute"], ) - @router.post( "/job/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -204,8 +205,6 @@ async def cancel_job( # look up the resource (todo: maybe ensure it's available) resource = await status_router.adapter.get_resource(resource_id) - try: - await router.adapter.cancel_job(resource, user, job_id) - except Exception as exc: - raise HTTPException(status_code=400, detail=f"Unable to cancel job: {str(exc)}") from exc + await router.adapter.cancel_job(resource, user, job_id) + return None diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index d3fc9e16..f0b5b49c 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod +import traceback import os import logging import importlib -from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from .account.models import User @@ -12,9 +12,6 @@ def get_client_ip(request : Request) -> str|None: - # logging.debug("Request headers=%s" % request.headers) - # logging.debug("client=%s" % request.client.host) - forwarded_for = request.headers.get("X-Forwarded-For") if forwarded_for: return forwarded_for.split(",")[0].strip() @@ -91,6 +88,7 @@ async def current_user( user_id = await self.adapter.get_current_user(api_key, get_client_ip(request)) except Exception as exc: logging.getLogger().error(f"Error parsing IRI_API_PARAMS: {exc}") + traceback.print_exc() raise HTTPException(status_code=401, detail="Invalid or malformed Authorization parameters") from exc if not user_id: raise HTTPException(status_code=403, detail="Unauthorized access") From 952aea04bb7e8427fe5426b6890dea1d1c47afc6 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 18:58:36 -0600 Subject: [PATCH 034/133] Ensure that computed fields are included in output --- app/routers/common.py | 14 ++++++++++++-- app/routers/status/models.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index 7bfaf0b8..f46c888f 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -129,13 +129,23 @@ class IRIBaseModel(BaseModel): model_config = ConfigDict(extra="allow") @model_serializer(mode="wrap") - def _hide_extra(self, handler): + def _hide_extra(self, handler, info): data = handler(self) + + model_fields = set(self.model_fields or {}) + computed_fields = set(self.model_computed_fields or {}) + print(model_fields) + print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: - data.pop(k, None) + if k not in model_fields and k not in computed_fields: + data.pop(k, None) + return data + + + def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) diff --git a/app/routers/status/models.py b/app/routers/status/models.py index d338e2cf..448a7e21 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -36,7 +36,7 @@ def _self_path(self) -> str: current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - @computed_field(description="The list of past events in this incident") + @computed_field(description="The list of capabilities in this resource") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] From a1c4e5336be12ece7ff97cae020707be3a4ee380 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 09:57:21 -0600 Subject: [PATCH 035/133] Code base compliant with the official Spec --- app/demo_adapter.py | 203 +++++++++++++++++++++-- app/routers/account/account.py | 17 +- app/routers/account/facility_adapter.py | 3 +- app/routers/account/models.py | 22 +-- app/routers/common.py | 48 ++++-- app/routers/facility/facility.py | 57 +++++++ app/routers/facility/facility_adapter.py | 46 +++++ app/routers/facility/models.py | 35 +++- app/routers/status/facility_adapter.py | 6 +- app/routers/status/models.py | 2 + app/routers/status/status.py | 32 ++-- 11 files changed, 398 insertions(+), 73 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index e63c7201..e68635e6 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -9,9 +9,10 @@ import subprocess import pathlib import base64 -from pydantic import BaseModel from typing import Any, Tuple +from pydantic import BaseModel from fastapi import HTTPException +from .routers.common import AllocationUnit, Capability from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter @@ -35,7 +36,7 @@ def get_base_temp_dir(cls): os.makedirs(cls._base_temp_dir, exist_ok=True) # create a test file - with open(f"{cls._base_temp_dir}/test.txt", "w") as f: + with open(f"{cls._base_temp_dir}/test.txt", encoding="utf-8", mode="w") as f: f.write("hello world") return cls._base_temp_dir @@ -67,12 +68,57 @@ def __init__(self): self.project_allocations = [] self.user_allocations = [] self.facility = {} + self.locations = [] + self.sites = [] self._init_state() def _init_state(self): now = utc_now() + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -81,22 +127,29 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - facility_uri="https://www.demo.example", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - street_address="1 main st", - latitude=38.410558, - longitude=-286.36999 + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], ) + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { - "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "hpss": account_models.Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), - "gpfs": account_models.Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), + "cpu": Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[AllocationUnit.node_hours]), + "gpu": Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[AllocationUnit.node_hours]), + "hpss": Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), + "gpfs": Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), } pm = status_models.Resource(id=str(uuid.uuid4()), group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", capability_ids=[ @@ -226,6 +279,125 @@ async def get_facility( ) -> facility_models.Facility: return self.facility + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + + + # ---------------------------- # Status API # ---------------------------- @@ -239,6 +411,8 @@ async def get_resources( group : str | None = None, modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, + current_status : status_models.Status | None = None, + capability: Capability | None = None ) -> list[status_models.Resource]: return status_models.Resource.find(self.resources, name, description, group, modified_since, resource_type)[offset:offset + limit] @@ -288,6 +462,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: return status_models.Incident.find(self.incidents, name, description, status, type, from_, to, time_, modified_since, resource_id)[offset:offset + limit] @@ -301,7 +476,7 @@ async def get_incident( async def get_capabilities( self : "DemoAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: return self.capabilities.values() diff --git a/app/routers/account/account.py b/app/routers/account/account.py index fe16b8bf..f856312e 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -1,8 +1,8 @@ -from fastapi import HTTPException, Request, Depends +from fastapi import HTTPException, Request, Depends, Query from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import forbidExtraQueryParams +from ..common import forbidExtraQueryParams, StrictDateTime, Capability router = iri_router.IriRouter( @@ -22,8 +22,12 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> list[models.Capability]: + name : str = Query(default=None, min_length=1), + modified_since: StrictDateTime = Query(default=None), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + _forbid = Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), + ) -> list[Capability]: return await router.adapter.get_capabilities() @@ -37,8 +41,9 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> models.Capability: + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + ) -> Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) if not cc: diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 78b622fd..235a2f73 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -1,5 +1,6 @@ from abc import abstractmethod from . import models as account_models +from ..common import Capability from ..iri_router import AuthenticatedAdapter @@ -13,7 +14,7 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def get_capabilities( self : "FacilityAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: pass diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 2158575e..1a9333d8 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,26 +1,6 @@ from pydantic import computed_field, Field -import enum from ... import config -from ..common import IRIBaseModel - - -class AllocationUnit(enum.Enum): - node_hours = "node_hours" - bytes = "bytes" - inodes = "inodes" - - -class Capability(IRIBaseModel): - """ - An aspect of a resource that can have an allocation. - For example, Perlmutter nodes with GPUs - For some resources at a facility, this will be 1 to 1 with the resource. - It is a way to further subdivide a resource into allocatable sub-resources. - The word "capability" is also known to users as something they need for a job to run. (eg. gpu) - """ - id: str - name: str - units: list[AllocationUnit] +from ..common import IRIBaseModel, AllocationUnit class User(IRIBaseModel): diff --git a/app/routers/common.py b/app/routers/common.py index f46c888f..d5805656 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -1,5 +1,6 @@ """Default models used by multiple routers.""" import datetime +import enum from typing import Optional from urllib.parse import parse_qs @@ -93,7 +94,11 @@ def __get_pydantic_json_schema__(cls, schema, handler): } -def forbidExtraQueryParams(*allowedParams: str): +def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): + multiParams = multiParams or set() + + print(allowedParams, multiParams) + async def checker(req: Request): if "*" in allowedParams: return @@ -110,20 +115,26 @@ async def checker(req: Request): detail=[{ "type": "extra_forbidden", "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) + "msg": f"Unexpected query parameter: {key}", + }], + ) - if len(values) > 1: + if len(values) > 1 and key not in multiParams: raise HTTPException( status_code=422, detail=[{ "type": "duplicate_forbidden", "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) + "msg": f"Duplicate query parameter: {key}", + }], + ) + return checker + + + class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") @@ -134,18 +145,12 @@ def _hide_extra(self, handler, info): model_fields = set(self.model_fields or {}) computed_fields = set(self.model_computed_fields or {}) - print(model_fields) - print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: if k not in model_fields and k not in computed_fields: data.pop(k, None) - return data - - - def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) @@ -188,3 +193,22 @@ def normalize(dt: datetime) -> datetime: modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] return a + + +class AllocationUnit(enum.Enum): + node_hours = "node_hours" + bytes = "bytes" + inodes = "inodes" + + +class Capability(IRIBaseModel): + """ + An aspect of a resource that can have an allocation. + For example, Perlmutter nodes with GPUs + For some resources at a facility, this will be 1 to 1 with the resource. + It is a way to further subdivide a resource into allocatable sub-resources. + The word "capability" is also known to users as something they need for a job to run. (eg. gpu) + """ + id: str + name: str + units: list[AllocationUnit] \ No newline at end of file diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index c0d3e80d..7c685e15 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -19,3 +19,60 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") +async def list_sites( + request: Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") +async def get_site( + request: Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") +async def get_site_location( + request : Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") +async def list_locations( + request : Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") +async def get_location( + request : Request, + location_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index cbee9515..d316674d 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -17,3 +17,49 @@ async def get_facility( ) -> facility_models.Facility | None: pass + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 508dbcf3..fd164fad 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,13 +1,22 @@ """Facility-related models.""" -from typing import Optional +from typing import Optional, List from pydantic import Field, HttpUrl from ..common import NamedObject -class Facility(NamedObject): - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization's name.") - facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -16,7 +25,21 @@ class Facility(NamedObject): altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") +class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 6753a470..d7358c50 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -2,6 +2,7 @@ import datetime from fastapi import Query from . import models as status_models +from ..common import Capability class FacilityAdapter(ABC): @@ -21,7 +22,9 @@ async def get_resources( description : str | None = None, group : str | None = None, modified_since : datetime.datetime | None = None, - resource_type: status_models.ResourceType = Query(default=None) + resource_type: status_models.ResourceType = Query(default=None), + current_status: status_models.Status = Query(default=None), + capability: Capability | None = None, ) -> list[status_models.Resource]: pass @@ -75,6 +78,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 448a7e21..3650451a 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -101,6 +101,7 @@ def find( class IncidentType(enum.Enum): planned = "planned" unplanned = "unplanned" + reservation = "reservation" class Resolution(enum.Enum): @@ -111,6 +112,7 @@ class Resolution(enum.Enum): pending = "pending" + class Incident(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/status/status.py b/app/routers/status/status.py index d599866c..5290a8a9 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -1,8 +1,9 @@ +from typing import Optional, List, Annotated from fastapi import HTTPException, Request, Query, Depends from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams, AllocationUnit router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -22,13 +23,16 @@ async def get_resources( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), group : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + current_status: models.Status = Query(default=None), + #event_uris: Optional[List[str]] = Query(default=None, min_length=1), + capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) @router.get( @@ -48,7 +52,7 @@ async def get_resource( return item -@router.get( +@router.get( "/incidents", summary="Get all incidents without their events", description="Get a list of all incidents. Each incident will be returned without its events. You can optionally filter the returned list by specifying attributes.", @@ -66,11 +70,15 @@ async def get_incidents( to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), - _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + resolution : models.Resolution = Query(default=None), + resource_uris: Optional[List[str]] = Query(default=None, min_length=1), + event_uris: Optional[List[str]] = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", + "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), ) -> list[models.Incident]: - return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) + return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id, resolution) @router.get( @@ -109,8 +117,8 @@ async def get_events( time_ : StrictDateTime = Query(alias="time", default=None), to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) From 6fb8e8b054ebb5bffec8801bc707e337c7cc752e Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:08:21 -0600 Subject: [PATCH 036/133] Fully compliant with official spec --- app/routers/common.py | 28 +++++++++------------------- app/routers/status/status.py | 5 ++--- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index d5805656..fd2882fe 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -97,8 +97,6 @@ def __get_pydantic_json_schema__(cls, schema, handler): def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): multiParams = multiParams or set() - print(allowedParams, multiParams) - async def checker(req: Request): if "*" in allowedParams: return @@ -110,31 +108,23 @@ async def checker(req: Request): for key, values in parsed.items(): if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}"}]) + if len(values) > 1 and key not in multiParams: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}"}]) return checker - class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 5290a8a9..6a0e9489 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -28,11 +28,10 @@ async def get_resources( modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), current_status: models.Status = Query(default=None), - #event_uris: Optional[List[str]] = Query(default=None, min_length=1), - capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + capability: List[AllocationUnit] = Query(default=None, min_length=1), _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status, capability) @router.get( From 53bc4c39791ea550b04a484df9e97027744e7894 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:45:23 -0600 Subject: [PATCH 037/133] Enforce Py 3.14 as used release for everything --- .github/workflows/api-validation.yml | 11 ++++++----- Dockerfile | 2 +- pyproject.toml | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml index 624e84e2..e4d688b4 100644 --- a/.github/workflows/api-validation.yml +++ b/.github/workflows/api-validation.yml @@ -15,19 +15,18 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged - name: Checkout schema validator repository uses: actions/checkout@v4 with: - repository: juztas/iri-facility-api-docs - ref: schemavalidator + repository: doe-iri/iri-facility-api-docs + ref: main path: schema-validator token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.14" - name: Install uv run: pip install uv @@ -81,6 +80,8 @@ jobs: --report-name schemathesis-local echo "exitcode=$?" >> $GITHUB_OUTPUT + # TODO: Change back to https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json + # Once https://github.com/doe-iri/iri-facility-api-docs/pull/12 merged. - name: Run Schemathesis validation (official spec) id: schemathesis_official env: @@ -90,7 +91,7 @@ jobs: source .venv/bin/activate python schema-validator/verification/api-validator.py \ --baseurl http://localhost:8000 \ - --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --schema-url https://raw.githubusercontent.com/juztas/iri-facility-api-docs/refs/heads/newspec/specification/openapi/openapi_iri_facility_api_v1.json \ --report-name schemathesis-official echo "exitcode=$?" >> $GITHUB_OUTPUT diff --git a/Dockerfile b/Dockerfile index f3ad071d..93c80d90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.14 RUN mkdir /app COPY . /app diff --git a/pyproject.toml b/pyproject.toml index 1f5c0d64..6dbf14b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12" +requires-python = ">=3.12,<3.13" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", @@ -10,4 +10,4 @@ dependencies = [ "opentelemetry-sdk", "opentelemetry-instrumentation-fastapi", "opentelemetry-exporter-otlp" -] \ No newline at end of file +] From 846c1f8df388ac2dbc110417d5562c167cc0aa46 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 17:10:38 -0600 Subject: [PATCH 038/133] Enable deepsource scanning --- .deepsource.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .deepsource.toml diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 00000000..01a10663 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,9 @@ +version = 1 + +[[analyzers]] +name = "python" +enabled = true + + [analyzers.meta] + runtime_version = "3.x.x" + max_line_length = 200 From 2374d341d34ae4aa6344c4d6170e9e137f94e102 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Sun, 25 Jan 2026 08:08:41 -0600 Subject: [PATCH 039/133] Enforce consistent package versions (pin major/minor version) --- pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6dbf14b1..63f6c020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12,<3.13" +requires-python = ">=3.14,<3.15" dependencies = [ - "fastapi[standard]>=0.100.0", - "uvicorn[standard]>=0.22.0", - "humps>=0.2.2", - "opentelemetry-api", - "opentelemetry-sdk", - "opentelemetry-instrumentation-fastapi", - "opentelemetry-exporter-otlp" -] + "fastapi[standard]>=0.128.0,<0.129.0", + "uvicorn[standard]>=0.40.0,<0.41.0", + "humps>=0.2.2,<0.3.0", + "opentelemetry-api>=1.39.1,<1.40.0", + "opentelemetry-sdk>=1.39.1,<1.40.0", + "opentelemetry-instrumentation-fastapi>=0.60b1,<0.61b0", + "opentelemetry-exporter-otlp>=1.39.1,<1.40.0" +] \ No newline at end of file From cba88c2c672f505f9e759253ae36afbb0c79a09b Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 040/133] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- Makefile | 1 + app/demo_adapter.py | 210 ++++++++++++++++++++++- app/main.py | 33 +++- app/routers/account/account.py | 8 + app/routers/compute/compute.py | 4 +- app/routers/error_handlers.py | 6 + app/routers/facility/__init__.py | 0 app/routers/facility/facility.py | 77 +++++++++ app/routers/facility/facility_adapter.py | 65 +++++++ app/routers/facility/models.py | 58 +++++++ app/routers/iri_router.py | 36 +++- 11 files changed, 484 insertions(+), 14 deletions(-) create mode 100644 app/routers/facility/__init__.py create mode 100644 app/routers/facility/facility.py create mode 100644 app/routers/facility/facility_adapter.py create mode 100644 app/routers/facility/models.py diff --git a/Makefile b/Makefile index 5bd90346..ba7552ab 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ dev : .venv @source ./.venv/bin/activate && \ + IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 9cbc7961..daf266d8 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -13,6 +13,7 @@ from pydantic import BaseModel from typing import Any, Tuple from fastapi import HTTPException +from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter from .routers.compute import models as compute_models, facility_adapter as compute_adapter @@ -40,9 +41,13 @@ def get_base_temp_dir(cls): return cls._base_temp_dir +def demo_uuid(kind: str, name: str) -> str: + return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, - task_adapter.FacilityAdapter): + task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): def __init__(self): self.resources = [] self.incidents = [] @@ -52,11 +57,83 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - + self.locations = [] + self.facility = {} + self.sites = [] self._init_state() def _init_state(self): + now = datetime.datetime.now(datetime.timezone.utc) + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + + facility = facility_models.Facility( + id=demo_uuid("facility", "demo_facility"), + name="Demo Facility", + description="A demo facility for testing the IRI Facility API", + last_modified=now, + short_name="DEMO", + organization_name="Demo Organization", + support_uri="https://support.demo.example", + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], + ) + + self.facility = facility + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + + day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -182,6 +259,135 @@ def _init_state(self): d += datetime.timedelta(minutes=int(random.random() * 15 + 1)) + # ---------------------------- + # Facility API + # ---------------------------- + + async def get_facility( + self: "DemoAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility: + return self.facility + + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + # ---------------------------- + # Status API + # ---------------------------- async def get_resources( self : "DemoAdapter", diff --git a/app/main.py b/app/main.py index be5feaaf..080645ef 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,11 @@ """Main API application""" import logging from fastapi import FastAPI +from fastapi import Request +from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers +from app.routers.facility import facility from app.routers.status import status from app.routers.account import account from app.routers.compute import compute @@ -19,11 +22,39 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" +@APP.get(api_prefix) +async def api_discovery(request: Request): + base = str(request.base_url).rstrip("/") + items = [] + for route in APP.router.routes: + if not isinstance(route, APIRoute): + continue + # skip docs & openapi + if route.path.startswith("/docs") or route.path.startswith("/openapi"): + continue + for method in route.methods: + if method == "HEAD" or method == "OPTIONS": + continue + items.append({ + "id": route.name or f"{method}_{route.path}", + "method": method, + "path": route.path, + "_links": [ + { + "rel": "self", + "href": f"{base.rstrip('/')}{route.path}", + "type": "application/json" + } + ] + }) + return items + # Attach routers under the prefix +APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) APP.include_router(account.router, prefix=api_prefix) APP.include_router(compute.router, prefix=api_prefix) APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") +logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 951fc9b7..508d556b 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -21,6 +21,7 @@ ) async def get_capabilities( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -35,6 +36,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -53,6 +55,7 @@ async def get_capability( ) async def get_projects( request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -71,6 +74,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -93,6 +97,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -116,6 +121,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -141,6 +147,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -169,6 +176,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, + _forbid = Depends(iri_router.forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index e7481acc..72a8169c 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -154,8 +154,8 @@ async def get_job_status( async def get_job_statuses( resource_id : str, request : Request, - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=10000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, historical : bool = False, include_spec: bool = False, diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 09769ec2..337b5fc2 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -65,6 +65,12 @@ async def validation_error_handler(request: Request, exc: RequestValidationError @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException): + if exc.status_code == 304: + return JSONResponse( + status_code=304, + content=None, + headers=exc.headers or {}) + if exc.status_code == 401: return problem_response( request=request, diff --git a/app/routers/facility/__init__.py b/app/routers/facility/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py new file mode 100644 index 00000000..99649256 --- /dev/null +++ b/app/routers/facility/facility.py @@ -0,0 +1,77 @@ +from fastapi import Request, HTTPException, Depends, Query +from .. import iri_router +from ..error_handlers import DEFAULT_RESPONSES +from .import models, facility_adapter + + +router = iri_router.IriRouter( + facility_adapter.FacilityAdapter, + prefix="/facility", + tags=["facility"], +) + +@router.get("", responses=DEFAULT_RESPONSES) +async def get_facility( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + ) -> models.Facility: + """Get facility information""" + return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES) +async def list_sites( + request: Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +async def get_site( + request: Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +async def get_site_location( + request : Request, + site_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES) +async def list_locations( + request : Request, + modified_since: iri_router.StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +async def get_location( + request : Request, + location_id: str, + modified_since: iri_router.StrictDateTime = Query(default=None), + _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py new file mode 100644 index 00000000..d316674d --- /dev/null +++ b/app/routers/facility/facility_adapter.py @@ -0,0 +1,65 @@ +from abc import abstractmethod +from . import models as facility_models +from ..iri_router import AuthenticatedAdapter + + +class FacilityAdapter(AuthenticatedAdapter): + """ + Facility-specific code is handled by the implementation of this interface. + Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) + to install your facility adapter before the API starts. + """ + + @abstractmethod + async def get_facility( + self: "FacilityAdapter", + modified_since: str | None = None, + ) -> facility_models.Facility | None: + pass + + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py new file mode 100644 index 00000000..0c04cc42 --- /dev/null +++ b/app/routers/facility/models.py @@ -0,0 +1,58 @@ +from datetime import datetime +from uuid import UUID +from typing import List, Optional +from pydantic import BaseModel, Field, HttpUrl, computed_field +from .. import iri_router +from ... import config + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") + country_name: Optional[str] = Field(None, description="Country name of the Location.") + locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") + state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") + street_address: Optional[str] = Field(None, description="Street address of the Location.") + unlocode: Optional[str] = Field(None, description="United Nations trade and transport location code.") + altitude: Optional[float] = Field(None, description="Altitude of the Location.") + latitude: Optional[float] = Field(None, description="Latitude of the Location.") + longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") + +class Facility(NamedObject): + def _self_path(self) -> str: + return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index dafb9702..2de7e693 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -3,11 +3,13 @@ import logging import importlib import datetime +from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from pydantic_core import core_schema from .account.models import User + bearer_token = APIKeyHeader(name="Authorization") @@ -128,17 +130,33 @@ async def get_user( def forbidExtraQueryParams(*allowedParams: str): - """Dependency to forbid extra query parameters not in allowedParams.""" - - async def checker(_req: Request): + async def checker(req: Request): if "*" in allowedParams: - return # Permit anything - incoming = set(_req.query_params.keys()) + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + allowed = set(allowedParams) - unknown = incoming - allowed - if unknown: - raise HTTPException(status_code=422, - detail=[{"type": "extra_forbidden", "loc": ["query", param], "msg": f"Unexpected query parameter: {param}"} for param in unknown]) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) return checker class StrictDateTime: From 6f44f6742409c85de32acfac714145f070c39db0 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 07:50:47 -0600 Subject: [PATCH 041/133] Make NamedObject reusable --- app/routers/facility/models.py | 22 ++------- app/routers/models.py | 46 +++++++++++++++++++ app/routers/status/models.py | 82 +++++++++------------------------- 3 files changed, 69 insertions(+), 81 deletions(-) create mode 100644 app/routers/models.py diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 0c04cc42..2bea62f8 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,23 +1,7 @@ -from datetime import datetime -from uuid import UUID +"""Facility-related models.""" from typing import List, Optional -from pydantic import BaseModel, Field, HttpUrl, computed_field -from .. import iri_router -from ... import config - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - +from pydantic import Field, HttpUrl +from ..models import NamedObject class Site(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/models.py b/app/routers/models.py new file mode 100644 index 00000000..f9a1c007 --- /dev/null +++ b/app/routers/models.py @@ -0,0 +1,46 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from pydantic import BaseModel, Field, computed_field +from . import iri_router +from .. import config + + +class NamedObject(BaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index b1f3a8bb..2c7dc765 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,6 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config +from ..models import NamedObject class Link(BaseModel): rel : str @@ -15,38 +16,6 @@ class Status(enum.Enum): unknown = "unknown" -class NamedResource(BaseModel): - id : str - name : str - description : str - last_modified : datetime.datetime - - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a - - class ResourceType(enum.Enum): website = "website" service = "service" @@ -57,28 +26,24 @@ class ResourceType(enum.Enum): unknown = "unknown" -class Resource(NamedResource): +class Resource(NamedObject): + + def _self_path(self) -> str: + return f"/status/resources/{self.id}" + capability_ids: list[str] = Field(exclude=True) group: str | None current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] - @staticmethod def find(resources, name, description, group, modified_since, resource_type): - a = NamedResource.find(resources, name, description, modified_since) + a = NamedObject.find(resources, name, description, modified_since) if group: a = [aa for aa in a if aa.group == group] if resource_type: @@ -86,25 +51,21 @@ def find(resources, name, description, group, modified_since, resource_type): return a -class Event(NamedResource): +class Event(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.incident_id}/events/{self.id}" + occurred_at : datetime.datetime status : Status resource_id : str = Field(exclude=True) incident_id : str | None = Field(exclude=True, default=None) - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}/events/{self.id}" - - @computed_field(description="The resource belonging to this event") @property def resource_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.resource_id}" - @computed_field(description="The event's incident") @property def incident_uri(self) -> str|None: @@ -123,7 +84,7 @@ def find( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, ) -> list: - events = NamedResource.find(events, name, description, modified_since) + events = NamedObject.find(events, name, description, modified_since) if resource_id: events = [e for e in events if e.resource_id == resource_id] if status: @@ -150,7 +111,11 @@ class Resolution(enum.Enum): pending = "pending" -class Incident(NamedResource): +class Incident(NamedObject): + + def _self_path(self) -> str: + return f"/status/incidents/{self.id}" + status : Status resource_ids : list[str] = Field(exclude=True) event_ids : list[str] = Field(exclude=True) @@ -159,25 +124,18 @@ class Incident(NamedResource): type : IncidentType resolution : Resolution - - @computed_field(description="The url of this object") - @property - def self_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}" - - @computed_field(description="The list of past events in this incident") @property def event_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}/events/{e}" for e in self.event_ids] - @computed_field(description="The list of resources that may be impacted by this incident") @property def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] def find( + self, incidents : list, name : str | None = None, description : str | None = None, @@ -189,7 +147,7 @@ def find( modified_since : datetime.datetime | None = None, resource_id : str | None = None, ) -> list: - incidents = NamedResource.find(incidents, name, description, modified_since) + incidents = NamedObject.find(incidents, name, description, modified_since) if resource_id: incidents = [e for e in incidents if resource_id in e.resource_ids] if status: From fbab525d7fa52cfdf617c3a83a49a6e29a01ca4e Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 08:16:21 -0600 Subject: [PATCH 042/133] Include operation_id for facility (similar to pull request #21) --- app/routers/facility/facility.py | 12 ++++++------ app/routers/status/models.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 99649256..0e6f9609 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -10,7 +10,7 @@ tags=["facility"], ) -@router.get("", responses=DEFAULT_RESPONSES) +@router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -19,7 +19,7 @@ async def get_facility( """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) -@router.get("/sites", responses=DEFAULT_RESPONSES) +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") async def list_sites( request: Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -32,7 +32,7 @@ async def list_sites( """List sites""" return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") async def get_site( request: Request, site_id: str, @@ -42,7 +42,7 @@ async def get_site( """Get site by ID""" return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES) +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") async def get_site_location( request : Request, site_id: str, @@ -52,7 +52,7 @@ async def get_site_location( """Get site location by site ID""" return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) -@router.get("/locations", responses=DEFAULT_RESPONSES) +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") async def list_locations( request : Request, modified_since: iri_router.StrictDateTime = Query(default=None), @@ -66,7 +66,7 @@ async def list_locations( """List locations""" return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES) +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") async def get_location( request : Request, location_id: str, diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 2c7dc765..bb1f87b0 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -134,8 +134,8 @@ def event_uris(self) -> list[str]: def resource_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] + @staticmethod def find( - self, incidents : list, name : str | None = None, description : str | None = None, From 7ca9b7565c7fc84a7f310fe126d7133523f60a11 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Thu, 15 Jan 2026 11:48:45 -0800 Subject: [PATCH 043/133] simplified facility endpoint proposal --- app/demo_adapter.py | 184 +---------------------- app/routers/facility/facility.py | 57 ------- app/routers/facility/facility_adapter.py | 47 ------ app/routers/facility/models.py | 33 +--- 4 files changed, 14 insertions(+), 307 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index daf266d8..2d2ca30c 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -65,50 +65,7 @@ def __init__(self): def _init_state(self): now = datetime.datetime.now(datetime.timezone.utc) - loc1 = facility_models.Location( - id=demo_uuid("location", "demo_location_1"), - name="Demo Location 1", - description="The first demo location", - last_modified=now, - short_name="DL1", - country_name="USA", - locality_name="Demo City", - state_or_province_name="DC", - latitude=36.173357, - longitude=-234.51452) - - loc2 = facility_models.Location( - id=demo_uuid("location", "demo_location_2"), - name="Demo Location 2", - description="The second demo location", - last_modified=now, - short_name="DL2", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - latitude=38.410558, - longitude=-286.36999) - - site1 = facility_models.Site( - id=demo_uuid("site", "demo_site_1"), - name="Demo Site 1", - description="The first demo site", - last_modified=now, - short_name="DS1", - operating_organization="Demo Org", - location_uri=loc1.self_uri, - resource_uris=[]) - site2 = facility_models.Site( - id=demo_uuid("site", "demo_site_2"), - name="Demo Site 2", - description="The second demo site", - last_modified=now, - short_name="DS2", - operating_organization="Demo Org", - location_uri=loc2.self_uri, - resource_uris=[]) - - facility = facility_models.Facility( + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", description="A demo facility for testing the IRI Facility API", @@ -116,24 +73,15 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - site_uris=[site1.self_uri, site2.self_uri], - location_uris=[loc1.self_uri, loc2.self_uri], - resource_uris=[], - event_uris=[], - incident_uris=[], - capability_uris=[], - project_uris=[], - project_allocation_uris=[], - user_allocation_uris=[], + facility_uri="https://www.demo.example", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + street_address="1 main st", + latitude=38.410558, + longitude=-286.36999 ) - self.facility = facility - loc1.site_uris.append(site1.self_uri) - loc2.site_uris.append(site2.self_uri) - self.locations = [loc1, loc2] - self.sites = [site1, site2] - - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -269,122 +217,6 @@ async def get_facility( ) -> facility_models.Facility: return self.facility - - async def list_sites( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - - sites = self.sites - - if name: - sites = [s for s in sites if name.lower() in s.name.lower()] - - if short_name: - sites = [s for s in sites if s.short_name == short_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - sites = [s for s in sites if s.last_modified > ms] - - o = offset or 0 - l = limit or len(sites) - return sites[o:o+l] - - - async def get_site( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site: - - site = next((s for s in self.sites if s.id == site_id), None) - if not site: - raise HTTPException(status_code=404, detail="Site not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if site.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) - - return site - - - async def get_site_location( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - site = await self.get_site(site_id) - - if not site.location_uri: - raise HTTPException(status_code=404, detail="Site has no location") - - location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - - return location - - - async def list_locations( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - - locs = self.locations - - if name: - locs = [l for l in locs if name.lower() in l.name.lower()] - - if short_name: - locs = [l for l in locs if l.short_name == short_name] - - if country_name: - locs = [l for l in locs if l.country_name == country_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - locs = [l for l in locs if l.last_modified > ms] - - o = offset or 0 - l = limit or len(locs) - return locs[o:o+l] - - - async def get_location( - self: "DemoAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - location = next((l for l in self.locations if l.id == location_id), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - return location - # ---------------------------- # Status API # ---------------------------- diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 0e6f9609..9b7545b2 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,60 +18,3 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - -@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") -async def list_sites( - request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), - )-> list[models.Site]: - """List sites""" - return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) - -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") -async def get_site( - request: Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Site: - """Get site by ID""" - return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) - -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") -async def get_site_location( - request : Request, - site_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get site location by site ID""" - return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) - -@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") -async def list_locations( - request : Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - country_name: str = Query(default=None, min_length=1), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), - )-> list[models.Location]: - """List locations""" - return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) - -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") -async def get_location( - request : Request, - location_id: str, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get location by ID""" - return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index d316674d..bccdcfe5 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,50 +16,3 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass - - @abstractmethod - async def list_sites( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - ) -> list[facility_models.Site]: - pass - - @abstractmethod - async def get_site( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Site | None: - pass - - @abstractmethod - async def get_site_location( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass - - @abstractmethod - async def list_locations( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - pass - - @abstractmethod - async def get_location( - self: "FacilityAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 2bea62f8..efd93dd0 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,20 +1,13 @@ """Facility-related models.""" -from typing import List, Optional +from typing import Optional from pydantic import Field, HttpUrl from ..models import NamedObject -class Site(NamedObject): - def _self_path(self) -> str: - return f"/facility/sites/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Site.") - operating_organization: str = Field(..., description="Organization operating the Site.") - location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") - -class Location(NamedObject): - def _self_path(self) -> str: - return f"/facility/locations/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Location.") +class Facility(NamedObject): + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization's name.") + facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -23,20 +16,6 @@ def _self_path(self) -> str: altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") -class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization’s name.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") - location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") - event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") - incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") - capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") - project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") - project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") - user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") From 936cf356ced3327ad858c33092eadd635ad66542 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 12 Jan 2026 13:27:56 -0600 Subject: [PATCH 044/133] Add Facility API, demo data, stricter query validation and /api/v1 discovery endpoint This pull requests includes: - Implement /api/v1 to list metadata; - Implement /facility api (most fields are optional, and implemented based on the specification) - Capabilities, project include forbidExtraQueryParams to make validation happy. - Parse Raw query_string (catch duplicate keys) - Add HTTP 304 handling and return correct header --- app/demo_adapter.py | 2 -- app/routers/facility/facility.py | 1 + app/routers/facility/facility_adapter.py | 1 + app/routers/facility/models.py | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 2d2ca30c..0d581038 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -57,9 +57,7 @@ def __init__(self): self.projects = [] self.project_allocations = [] self.user_allocations = [] - self.locations = [] self.facility = {} - self.sites = [] self._init_state() diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 9b7545b2..fdba1241 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -18,3 +18,4 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index bccdcfe5..cbee9515 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -16,3 +16,4 @@ async def get_facility( modified_since: str | None = None, ) -> facility_models.Facility | None: pass + diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index efd93dd0..5b21d8a3 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -19,3 +19,4 @@ class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + From 58d560298850998473658a24cf972ff4f9b57eb9 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 15 Jan 2026 15:27:00 -0600 Subject: [PATCH 045/133] Remove /api/v1 --- app/main.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/app/main.py b/app/main.py index 080645ef..78641430 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,6 @@ """Main API application""" import logging from fastapi import FastAPI -from fastapi import Request -from fastapi.routing import APIRoute from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -22,33 +20,6 @@ api_prefix = f"{config.API_PREFIX}{config.API_URL}" -@APP.get(api_prefix) -async def api_discovery(request: Request): - base = str(request.base_url).rstrip("/") - items = [] - for route in APP.router.routes: - if not isinstance(route, APIRoute): - continue - # skip docs & openapi - if route.path.startswith("/docs") or route.path.startswith("/openapi"): - continue - for method in route.methods: - if method == "HEAD" or method == "OPTIONS": - continue - items.append({ - "id": route.name or f"{method}_{route.path}", - "method": method, - "path": route.path, - "_links": [ - { - "rel": "self", - "href": f"{base.rstrip('/')}{route.path}", - "type": "application/json" - } - ] - }) - return items - # Attach routers under the prefix APP.include_router(facility.router, prefix=api_prefix) APP.include_router(status.router, prefix=api_prefix) @@ -57,4 +28,4 @@ async def api_discovery(request: Request): APP.include_router(filesystem.router, prefix=api_prefix) APP.include_router(task.router, prefix=api_prefix) -logging.getLogger().info(f"API path: {api_prefix}") \ No newline at end of file +logging.getLogger().info(f"API path: {api_prefix}") From 630cd619cb64709e2490bba33f1c07f909a010c5 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:14:23 -0600 Subject: [PATCH 046/133] Refactor shared validators & models to fix import loading issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move common validators and base models into routers/dependencies and update imports across routers and schemas to use the new shared location. This keeps API behavior unchanged. No functional API changes — purely structural and validation hygiene: --- app/routers/account/account.py | 17 +- app/routers/account/models.py | 15 +- app/routers/compute/compute.py | 88 ++++++----- app/routers/compute/models.py | 70 ++++---- app/routers/dependencies.py | 176 +++++++++++++++++++++ app/routers/facility/facility.py | 8 +- app/routers/facility/models.py | 2 +- app/routers/filesystem/facility_adapter.py | 4 +- app/routers/filesystem/filesystem.py | 42 +---- app/routers/filesystem/models.py | 9 +- app/routers/iri_router.py | 77 --------- app/routers/models.py | 46 ------ app/routers/status/models.py | 2 +- app/routers/status/status.py | 25 +-- app/routers/task/models.py | 19 ++- 15 files changed, 317 insertions(+), 283 deletions(-) create mode 100644 app/routers/dependencies.py delete mode 100644 app/routers/models.py diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 508d556b..b2c41f44 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import forbidExtraQueryParams router = iri_router.IriRouter( @@ -21,7 +22,7 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Capability]: return await router.adapter.get_capabilities() @@ -36,7 +37,7 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) @@ -55,7 +56,7 @@ async def get_capability( ) async def get_projects( request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Project]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -74,7 +75,7 @@ async def get_projects( async def get_project( project_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.Project: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -97,7 +98,7 @@ async def get_project( async def get_project_allocations( project_id: str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -121,7 +122,7 @@ async def get_project_allocation( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.ProjectAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -147,7 +148,7 @@ async def get_user_allocations( project_id: str, project_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.UserAllocation]: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: @@ -176,7 +177,7 @@ async def get_user_allocation( project_allocation_id : str, user_allocation_id : str, request : Request, - _forbid = Depends(iri_router.forbidExtraQueryParams()), + _forbid = Depends(forbidExtraQueryParams()), ) -> models.UserAllocation: user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 6ed69ea0..c012ee32 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel, computed_field, Field +from pydantic import computed_field, Field import enum from ... import config +from ..dependencies import IRIBaseModel class AllocationUnit(enum.Enum): @@ -9,7 +10,7 @@ class AllocationUnit(enum.Enum): inodes = "inodes" -class Capability(BaseModel): +class Capability(IRIBaseModel): """ An aspect of a resource that can have an allocation. For example, Perlmutter nodes with GPUs @@ -22,7 +23,7 @@ class Capability(BaseModel): units: list[AllocationUnit] -class User(BaseModel): +class User(IRIBaseModel): """A user of the facility""" id: str name: str @@ -31,7 +32,7 @@ class User(BaseModel): # we could expose more fields here (eg. email) but it might be against policy -class Project(BaseModel): +class Project(IRIBaseModel): """A project and its users at a facility""" id: str name: str @@ -39,14 +40,14 @@ class Project(BaseModel): user_ids: list[str] -class AllocationEntry(BaseModel): +class AllocationEntry(IRIBaseModel): """Base class for allocations.""" allocation: float # how much this allocation can spend usage: float # how much this allocation has spent unit: AllocationUnit -class ProjectAllocation(BaseModel): +class ProjectAllocation(IRIBaseModel): """ A project's allocation for a capability. (aka. repo) This allocation is a piece of the total allocation for the capability. (eg. 5% of the total node hours of Perlmutter GPU nodes) @@ -71,7 +72,7 @@ def capability_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{self.capability_id}" -class UserAllocation(BaseModel): +class UserAllocation(IRIBaseModel): """ A user's allcation in a project. This allocation is a piece of the project's allocation. diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 72a8169c..7a2db23c 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,9 +1,10 @@ -from typing import List, Annotated -from fastapi import HTTPException, Request, Depends, status, Form, Query +from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router + from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router +from ..dependencies import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -24,6 +25,7 @@ async def submit_job( resource_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Submit a job on a compute resource @@ -45,39 +47,41 @@ async def submit_job( return await router.adapter.submit_job(resource, user, job_spec) -@router.post( - "/job/script/{resource_id:str}", - dependencies=[Depends(router.current_user)], - response_model=models.Job, - response_model_exclude_unset=True, - responses=DEFAULT_RESPONSES, - operation_id="launchJobScript", -) -async def submit_job_path( - resource_id: str, - job_script_path : str, - request : Request, - args : Annotated[List[str], Form()] = [], - ): - """ - Submit a job on a compute resource - - - **resource**: the name of the compute resource to use - - **job_script_path**: path to the job script on the compute resource - - **args**: optional arguments to the job script - - This command will attempt to submit a job and return its id. - """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - - # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) - - # the handler can use whatever means it wants to submit the job and then fill in its id - # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.submit_job_script(resource, user, job_script_path, args) +# TODO: this conflicts with PUT commented out while we finalize the API design +#@router.post( +# "/job/script/{resource_id:str}", +# dependencies=[Depends(router.current_user)], +# response_model=models.Job, +# response_model_exclude_unset=True, +# responses=DEFAULT_RESPONSES, +# operation_id="launchJobScript", +#) +#async def submit_job_path( +# resource_id: str, +# job_script_path : str, +# request : Request, +# args : Annotated[List[str], Form()] = [], +# _forbid = Depends(iri_router.forbidExtraQueryParams("job_script_path")), +# ): +# """ +# Submit a job on a compute resource +# +# - **resource**: the name of the compute resource to use +# - **job_script_path**: path to the job script on the compute resource +# - **args**: optional arguments to the job script +# +# This command will attempt to submit a job and return its id. +# """ +# user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) +# if not user: +# raise HTTPException(status_code=404, detail="User not found") +# +# # look up the resource (todo: maybe ensure it's available) +# resource = await status_router.adapter.get_resource(resource_id) +# +# # the handler can use whatever means it wants to submit the job and then fill in its id +# # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs +# return await router.adapter.submit_job_script(resource, user, job_script_path, args) @router.put( @@ -93,6 +97,7 @@ async def update_job( job_id: str, job_spec : models.JobSpec, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """ Update a previously submitted job for a resource. @@ -126,8 +131,9 @@ async def get_job_status( resource_id : str, job_id : str, request : Request, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -149,7 +155,7 @@ async def get_job_status( response_model=list[models.Job], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, - operation_id="getJobs", + operation_id="getAllJobs", ) async def get_job_statuses( resource_id : str, @@ -157,8 +163,9 @@ async def get_job_statuses( offset : int = Query(default=0, ge=0, le=1000), limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, - historical : bool = False, - include_spec: bool = False, + historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) @@ -187,6 +194,7 @@ async def cancel_job( resource_id : str, job_id : str, request : Request, + _forbid = Depends(forbidExtraQueryParams()), ): """Cancel a job""" user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 167643ec..a56d4fe4 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,41 +1,42 @@ from typing import Annotated -from pydantic import BaseModel, field_serializer, ConfigDict, Field from enum import IntEnum +from pydantic import field_serializer, ConfigDict, StrictBool, Field +from ..common import IRIBaseModel -class ResourceSpec(BaseModel): +class ResourceSpec(IRIBaseModel): """ Specification of computational resources required for a job. """ - node_count: Annotated[int | None, Field(description="Number of compute nodes to allocate")] = None - process_count: Annotated[int | None, Field(description="Total number of processes to launch")] = None - processes_per_node: Annotated[int | None, Field(description="Number of processes to launch per node")] = None - cpu_cores_per_process: Annotated[int | None, Field(description="Number of CPU cores to allocate per process")] = None - gpu_cores_per_process: Annotated[int | None, Field(description="Number of GPU cores to allocate per process")] = None - exclusive_node_use: Annotated[bool, Field(description="Whether to request exclusive use of allocated nodes")] = True - memory: Annotated[int | None, Field(description="Amount of memory to allocate in bytes")] = None + node_count: Annotated[int | None, Field(ge=1, description="Number of compute nodes to allocate")] = None + process_count: Annotated[int | None, Field(ge=1, description="Total number of processes to launch")] = None + processes_per_node: Annotated[int | None, Field(ge=1, description="Number of processes to launch per node")] = None + cpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of CPU cores to allocate per process")] = None + gpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of GPU cores to allocate per process")] = None + exclusive_node_use: Annotated[StrictBool, Field(description="Whether to request exclusive use of allocated nodes")] = True + memory: Annotated[int | None, Field(ge=1,description="Amount of memory to allocate in bytes")] = None -class JobAttributes(BaseModel): +class JobAttributes(IRIBaseModel): """ Additional attributes and scheduling parameters for a job. """ - duration: Annotated[int | None, Field(description="Duration in seconds", ge=0, examples=[30, 60, 120])] = None - queue_name: Annotated[str | None, Field(description="Name of the queue or partition to submit the job to")] = None - account: Annotated[str | None, Field(description="Account or project to charge for resource usage")] = None - reservation_id: Annotated[str | None, Field(description="ID of a reservation to use for the job")] = None + duration: Annotated[int | None, Field(description="Duration in seconds", ge=1, examples=[30, 60, 120])] = None + queue_name: Annotated[str | None, Field(min_length=1, description="Name of the queue or partition to submit the job to")] = None + account: Annotated[str | None, Field(min_length=1, description="Account or project to charge for resource usage")] = None + reservation_id: Annotated[str | None, Field(min_length=1, description="ID of a reservation to use for the job")] = None custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs")] = {} -class VolumeMount(BaseModel): +class VolumeMount(IRIBaseModel): """ Represents a volume mount for a container. """ - source: Annotated[str, Field(description="The source path on the host system to mount")] - target: Annotated[str, Field(description="The target path inside the container where the volume will be mounted")] - read_only: Annotated[bool, Field(description="Whether the mount should be read-only")] = True + source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount")] + target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted")] + read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only")] = True -class Container(BaseModel): +class Container(IRIBaseModel): """ Represents a container specification for job execution. @@ -44,33 +45,33 @@ class Container(BaseModel): to determine if the container should be run with MPI support. The container should by default. be run with host networking. """ - image: Annotated[str, Field(description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] + image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] -class JobSpec(BaseModel): +class JobSpec(IRIBaseModel): """ Specification for job. """ model_config = ConfigDict(extra="forbid") - executable: Annotated[str | None, Field(description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.")] = None + executable: Annotated[str | None, Field(min_length=1, description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.")] = None container: Annotated[Container | None, Field(description="Container specification for containerized execution")] = None arguments: Annotated[list[str], Field(description="Command-line arguments to pass to the executable or container")] = [] - directory: Annotated[str | None, Field(description="Working directory for the job")] = None - name: Annotated[str | None, Field(description="Name of the job")] = None - inherit_environment: Annotated[bool, Field(description="Whether to inherit the environment variables from the submission environment")] = True + directory: Annotated[str | None, Field(min_length=1, description="Working directory for the job")] = None + name: Annotated[str | None, Field(min_length=1, description="Name of the job")] = None + inherit_environment: Annotated[StrictBool, Field(description="Whether to inherit the environment variables from the submission environment")] = True environment: Annotated[dict[str, str], Field(description="Environment variables to set for the job. If container is specified, these will be set inside the container.")] = {} - stdin_path: Annotated[str | None, Field(description="Path to file to use as standard input")] = None - stdout_path: Annotated[str | None, Field(description="Path to file to write standard output")] = None - stderr_path: Annotated[str | None, Field(description="Path to file to write standard error")] = None + stdin_path: Annotated[str | None, Field(min_length=1, description="Path to file to use as standard input")] = None + stdout_path: Annotated[str | None, Field(min_length=1, description="Path to file to write standard output")] = None + stderr_path: Annotated[str | None, Field(min_length=1, description="Path to file to write standard error")] = None resources: Annotated[ResourceSpec | None, Field(description="Resource requirements for the job")] = None attributes: Annotated[JobAttributes | None, Field(description="Additional job attributes such as duration, queue, and account")] = None - pre_launch: Annotated[str | None, Field(description="Script or commands to run before launching the job")] = None - post_launch: Annotated[str | None, Field(description="Script or commands to run after the job completes")] = None - launcher: Annotated[str | None, Field(description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None + pre_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run before launching the job")] = None + post_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run after the job completes")] = None + launcher: Annotated[str | None, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None -class CommandResult(BaseModel): +class CommandResult(IRIBaseModel): status : str result : str | None = None @@ -110,20 +111,19 @@ class JobState(IntEnum): """Represents a job that was canceled by a call to :func:`~psij.Job.cancel()`.""" -class JobStatus(BaseModel): +class JobStatus(IRIBaseModel): state : JobState time : float | None = None message : str | None = None exit_code : int | None = None meta_data : dict[str, object] | None = None - @field_serializer('state') def serialize_state(self, state: JobState): return state.name -class Job(BaseModel): +class Job(IRIBaseModel): id : str status : JobStatus | None = None job_spec : JobSpec | None = None diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py new file mode 100644 index 00000000..17b4f096 --- /dev/null +++ b/app/routers/dependencies.py @@ -0,0 +1,176 @@ +"""Default models used by multiple routers.""" +import datetime +from typing import Optional +from urllib.parse import parse_qs + +from pydantic_core import core_schema +from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer +from fastapi import Request, HTTPException + +from .. import config + + +# These are Pydantic custom types for strict validation +# that are not implmented in Pydantic by default. +# ----------------------------------------------------------------------- +# StrictBool: a strict boolean type +class StrictBool: + """Strict boolean: + - Accepts: real booleans, 'true', 'false' + - Rejects everything else. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + """Validate the input value as a strict boolean.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + v = value.strip().lower() + if v == "true": + return True + if v == "false": + return False + raise ValueError("Invalid boolean value. Expected 'true' or 'false'.") + raise ValueError("Invalid boolean value. Expected true/false or 'true'/'false'.") + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "boolean", + "description": "Strict boolean. Only true/false allowed (bool or string)." + } + +# ----------------------------------------------------------------------- +# StrictDateTime: a strict ISO8601 datetime type + +class StrictDateTime: + """ + Strict ISO8601 datetime: + - Accepts datetime objects + - Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 + - Converts 'Z' → UTC + - Converts naive datetimes → UTC + - Rejects integers ("0"), null, garbage strings, etc. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + if isinstance(value, datetime.datetime): + return StrictDateTime._normalize(value) + if not isinstance(value, str): + raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") + v = value.strip() + if v.endswith("Z"): + v = v[:-1] + "+00:00" + try: + dt = datetime.datetime.fromisoformat(v) + except Exception as ex: + raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex + + return StrictDateTime._normalize(dt) + + @staticmethod + def _normalize(dt: datetime.datetime) -> datetime.datetime: + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "string", + "format": "date-time", + "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." + } + + +def forbidExtraQueryParams(*allowedParams: str): + async def checker(req: Request): + if "*" in allowedParams: + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + + allowed = set(allowedParams) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException( + status_code=422, + detail=[{ + "type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}" + }]) + + if len(values) > 1: + raise HTTPException( + status_code=422, + detail=[{ + "type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}" + }]) + return checker + + +class IRIBaseModel(BaseModel): + """Base model for IRI models.""" + model_config = ConfigDict(extra="allow") + + @model_serializer(mode="wrap") + def _hide_extra(self, handler): + data = handler(self) + extra = getattr(self, "__pydantic_extra__", {}) or {} + for k in extra: + data.pop(k, None) + return data + +class NamedObject(IRIBaseModel): + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @staticmethod + def find_by_id(a, id, allow_name: bool|None=False): + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + + + @staticmethod + def find(a, name, description, modified_since): + def normalize(dt: datetime) -> datetime: + # Convert naive datetimes into UTC-aware versions + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + if name: + a = [aa for aa in a if aa.name == name] + if description: + a = [aa for aa in a if description in aa.description] + if modified_since: + if modified_since.tzinfo is None: + modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) + a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] + return a diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index fdba1241..1c389186 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -1,7 +1,8 @@ -from fastapi import Request, HTTPException, Depends, Query +from fastapi import Request, Depends, Query from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( @@ -13,9 +14,8 @@ @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, - modified_since: iri_router.StrictDateTime = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("modified_since")), + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) - diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5b21d8a3..5db7a18b 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..models import NamedObject +from ..dependencies import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index 2c08a3cd..a70efb0d 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -1,15 +1,15 @@ import os from abc import abstractmethod +from typing import Any, Tuple from ..status import models as status_models from ..account import models as account_models from . import models as filesystem_models from ..iri_router import AuthenticatedAdapter -from typing import Any, Tuple def to_int(name, default_value): try: - return os.environ.get(name) or default_value + return int(os.environ.get(name) or default_value) except: return default_value diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index a111484a..d583c642 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -354,40 +354,13 @@ async def get_view( resource_id: str, request : Request, path: Annotated[str, Query(description="File path")], - size: Annotated[ - int, - Query( - alias="size", - description="Value, in bytes, of the size of data to be retrieved from the file.", - ), - ] = facility_adapter.OPS_SIZE_LIMIT, - offset: Annotated[ - int, - Query( - alias="offset", - description="Value in bytes of the offset.", - ), - ] = 0, -) -> str: - user, resource = await _user_resource(resource_id, request) - if offset < 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`offset` value must be an integer value equal or greater than 0", - ) - - if size <= 0: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="`size` value must be an integer value greater than 0", - ) + size: Annotated[int, Query(description="Value, in bytes, of the size of data to be retrieved from the file.", + ge=1, le=facility_adapter.OPS_SIZE_LIMIT)] = facility_adapter.OPS_SIZE_LIMIT, - if size > facility_adapter.OPS_SIZE_LIMIT: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"`size` value must be less than {facility_adapter.OPS_SIZE_LIMIT} bytes", - ) + offset: Annotated[int, Query( description="Value in bytes of the offset.", ge=0)] = 0 + ) -> str: + user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user, @@ -397,9 +370,8 @@ async def get_view( command="view", args={ "path": path, - "size": size or facility_adapter.OPS_SIZE_LIMIT, - "offset": offset or 0, - + "size": size, + "offset": offset, } ) ) diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index b24c908a..d32ad943 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -12,11 +12,10 @@ class CompressionType(str, Enum): - none = "none" - bzip2 = "bzip2" - gzip = "gzip" - xz = "xz" - + none = "none" + bzip2 = "bzip2" + gzip = "gzip" + xz = "xz" class ContentUnit(str, Enum): lines = "lines" diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 2de7e693..d3fc9e16 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -2,11 +2,9 @@ import os import logging import importlib -import datetime from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader -from pydantic_core import core_schema from .account.models import User @@ -127,78 +125,3 @@ async def get_user( Retrieve additional user information (name, email, etc.) for the given user_id. """ pass - - -def forbidExtraQueryParams(*allowedParams: str): - async def checker(req: Request): - if "*" in allowedParams: - return - - raw_qs = req.scope.get("query_string", b"") - parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) - - allowed = set(allowedParams) - - for key, values in parsed.items(): - if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) - - if len(values) > 1: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) - return checker - -class StrictDateTime: - """ - Strict ISO8601 datetime: - ✔ Accepts datetime objects - ✔ Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 - ✔ Converts 'Z' → UTC - ✔ Converts naive datetimes → UTC - ✘ Rejects integers ("0"), null, garbage strings, etc. - """ - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - return core_schema.no_info_plain_validator_function(cls.validate) - - @staticmethod - def validate(value): - if isinstance(value, datetime.datetime): - return StrictDateTime._normalize(value) - if not isinstance(value, str): - raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") - v = value.strip() - if v.endswith("Z"): - v = v[:-1] + "+00:00" - try: - dt = datetime.datetime.fromisoformat(v) - except Exception as ex: - raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex - - return StrictDateTime._normalize(dt) - - @staticmethod - def _normalize(dt: datetime.datetime) -> datetime.datetime: - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - - @classmethod - def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "string", - "format": "date-time", - "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." - } diff --git a/app/routers/models.py b/app/routers/models.py deleted file mode 100644 index f9a1c007..00000000 --- a/app/routers/models.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Default models used by multiple routers.""" -import datetime -from typing import Optional -from pydantic import BaseModel, Field, computed_field -from . import iri_router -from .. import config - - -class NamedObject(BaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - """Computed self URI property.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: iri_router.StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) - - - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - if name: - a = [aa for aa in a if aa.name == name] - if description: - a = [aa for aa in a if description in aa.description] - if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a diff --git a/app/routers/status/models.py b/app/routers/status/models.py index bb1f87b0..ff5d4e48 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..models import NamedObject +from ..dependencies import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 7bb0fd0b..2daf30e7 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,6 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..dependencies import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -23,9 +24,9 @@ async def get_resources( group : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - modified_since: iri_router.StrictDateTime = Query(default=None), + modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), ) -> list[models.Resource]: return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) @@ -60,14 +61,14 @@ async def get_incidents( description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), type_: models.IncidentType = Query(alias="type", default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), ) -> list[models.Incident]: return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) @@ -104,13 +105,13 @@ async def get_events( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), status : models.Status = Query(default=None), - from_: iri_router.StrictDateTime = Query(alias="from", default=None), - time_ : iri_router.StrictDateTime = Query(alias="time", default=None), - to : iri_router.StrictDateTime = Query(default=None), - modified_since : iri_router.StrictDateTime = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_ : StrictDateTime = Query(alias="time", default=None), + to : StrictDateTime = Query(default=None), + modified_since : StrictDateTime = Query(default=None), offset : int = Query(default=0, ge=0), limit : int = Query(default=100, le=1000), - _forbid = Depends(iri_router.forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), + _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) diff --git a/app/routers/task/models.py b/app/routers/task/models.py index da3c5ccc..cea97873 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,19 +1,18 @@ -from pydantic import BaseModel import enum +from pydantic import BaseModel class TaskStatus(str, enum.Enum): - pending = "pending" - active = "active" - completed = "completed" - failed = "failed" - canceled = "canceled" - + pending = "pending" + active = "active" + completed = "completed" + failed = "failed" + canceled = "canceled" class TaskCommand(BaseModel): - router: str - command: str - args: dict + router: str + command: str + args: dict class Task(BaseModel): From 2cf089b78105abfb2d8b185e3df46cf0769e7b3f Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:17:00 -0600 Subject: [PATCH 047/133] Github Action to validate api --- .github/workflows/api-validation.yml | 133 +++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 .github/workflows/api-validation.yml diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml new file mode 100644 index 00000000..624e84e2 --- /dev/null +++ b/.github/workflows/api-validation.yml @@ -0,0 +1,133 @@ +name: API Validation with Schemathesis + +on: + pull_request: + push: + branches: [ main ] + +jobs: + schemathesis: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged + - name: Checkout schema validator repository + uses: actions/checkout@v4 + with: + repository: juztas/iri-facility-api-docs + ref: schemavalidator + path: schema-validator + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + run: pip install uv + + - name: Build an image + run: docker build --platform=linux/amd64 -t iri-facility-api-base . + + - name: Run Facility API container + run: | + docker run -d \ + -p 8000:8000 \ + --platform=linux/amd64 \ + --name iri-facility-api-base \ + -e IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ + -e IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + -e API_URL_ROOT=http://127.0.0.1:8000 \ + -e IRI_API_TOKEN=12345 \ + iri-facility-api-base + + - name: Wait for API to be ready + run: | + for i in {1..60}; do + if curl -fs http://127.0.0.1:8000/openapi.json; then + echo "API ready" + exit 0 + fi + sleep 2 + done + echo "API did not start" + exit 1 + + - name: Create venv & install validator dependencies + run: | + uv venv + source .venv/bin/activate + uv pip install -r schema-validator/verification/requirements.txt + + - name: Run Schemathesis validation (local spec) + id: schemathesis_local + env: + IRI_API_TOKEN: "12345" # This is dummy token for testing (mock adapter) + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://127.0.0.1:8000 \ + --report-name schemathesis-local + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Run Schemathesis validation (official spec) + id: schemathesis_official + env: + IRI_API_TOKEN: "12345" + run: | + set +e + source .venv/bin/activate + python schema-validator/verification/api-validator.py \ + --baseurl http://localhost:8000 \ + --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --report-name schemathesis-official + echo "exitcode=$?" >> $GITHUB_OUTPUT + + - name: Fail if any Schemathesis run failed + if: always() + run: | + if [ "${{ steps.schemathesis_local.outputs.exitcode }}" != "0" ] || \ + [ "${{ steps.schemathesis_official.outputs.exitcode }}" != "0" ]; then + echo "One or more Schemathesis validations failed" + exit 1 + else + echo "Both Schemathesis validations passed" + fi + + - name: Upload Schemathesis report # This only works on git actions + if: always() && env.ACT != 'true' + uses: actions/upload-artifact@v4 + with: + if-no-files-found: warn + name: schemathesis-report + path: | + schemathesis-local.html + schemathesis-local.xml + schemathesis-official.html + schemathesis-official.xml + + - name: Save Schemathesis reports locally # This only works if run locally with act + if: always() && env.ACT == 'true' + run: | + mkdir -p artifacts + cp schemathesis-local.html schemathesis-local.xml artifacts/ || true + cp schemathesis-official.html schemathesis-official.xml artifacts/ || true + + - name: Dump API logs + if: always() + run: docker logs iri-facility-api-base || true + + - name: Stop container + if: always() + run: docker stop iri-facility-api-base || true From ea013e2c9fdb5e8961ce225cd1607dfc2ce50d13 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 10:41:35 -0600 Subject: [PATCH 048/133] Add custom get extra function --- app/routers/dependencies.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/routers/dependencies.py b/app/routers/dependencies.py index 17b4f096..7bfaf0b8 100644 --- a/app/routers/dependencies.py +++ b/app/routers/dependencies.py @@ -136,6 +136,10 @@ def _hide_extra(self, handler): data.pop(k, None) return data + def get_extra(self, key, default=None): + return getattr(self, "__pydantic_extra__", {}).get(key, default) + + class NamedObject(IRIBaseModel): id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") def _self_path(self) -> str: From 9eb22d92638ad2115bdde4dfe48962de073c5fcd Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 13:06:36 -0600 Subject: [PATCH 049/133] Implement opentelemetry. Use UTC in demo adapter --- Makefile | 1 + README.md | 2 ++ VALIDATION.MD | 23 +++++++++++++++++++++++ app/config.py | 5 +++++ app/demo_adapter.py | 31 +++++++++++++++++++++---------- app/main.py | 32 ++++++++++++++++++++++++++++++++ pyproject.toml | 8 ++++++-- 7 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 VALIDATION.MD diff --git a/Makefile b/Makefile index ba7552ab..abd87e4a 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ dev : .venv IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + OPENTELEMETRY_ENABLED=true \ API_URL_ROOT='http://127.0.0.1:8000' fastapi dev diff --git a/README.md b/README.md index 5ef41a6a..02b796c9 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ If using docker (see next section), your dockerfile could extend this reference - `API_URL_ROOT`: the base url when constructing links returned by the api (eg.: https://iri.myfacility.com) - `API_PREFIX`: the path prefix where the api is hosted. Defaults to `/`. (eg.: `/api`) - `API_URL`: the path to the api itself. Defaults to `api/v1`. +- `OPENTELEMETRY_ENABLED`: Enables OpenTelemetry. If enabled, the application will use OpenTelemetry SDKs and emit traces, metrics, and logs. Default to false +- `OTLP_ENDPOINT`: OpenTelemetry Protocol collector endpoint to export telemetry data. If empty or not set, telemetry data is logged locally to log file. Default: "" Links to data, created by this api, will concatenate these values producing links, eg: `https://iri.myfacility.com/my_api_prefix/my_api_url/projects/123` diff --git a/VALIDATION.MD b/VALIDATION.MD new file mode 100644 index 00000000..26f77ef6 --- /dev/null +++ b/VALIDATION.MD @@ -0,0 +1,23 @@ +# API Validation with Schemathesis + +On every pull request or push to `main` branch, Github Actions run the following steps below that validates an IRI Facility API implementation against OpenAPI spec using Schemathesis. + +1. Builds the Facility API Docker image from Dockerfile. +2. Runs the API container with demo adapter. +3. Waits for `/openapi.json` to become available on localhost:8000. +4. Runs Schemathesis validation twice: + - Against Facilities API’s OpenAPI spec. (http://localhost:8000/openapi.json) + - Against the official IRI Facility API OpenAPI spec. (https://github.com/doe-iri/iri-facility-api-docs/blob/main/specification/openapi/openapi_iri_facility_api_v1.json) +5. Fails the workflow if either validation fails. +6. Saves Schemathesis HTML/XML reports as artifacts (or saves it locally when run with `act`). +7. Dumps API container logs and do clean up to stop container. + +## Running locally + +```bash +act -W .github/workflows/api-validator.yml -s GITHUB_TOKEN= +``` + +## Known issues + +Python implementation not fully aligns with the official Specification. Running against Official Spec will continue to fail, until Spec or Py implementation is fixed. diff --git a/app/config.py b/app/config.py index 078b6a52..71a7c20d 100644 --- a/app/config.py +++ b/app/config.py @@ -40,3 +40,8 @@ API_URL_ROOT = os.environ.get("API_URL_ROOT", "https://api.iri.nersc.gov") API_PREFIX = os.environ.get("API_PREFIX", "/") API_URL = os.environ.get("API_URL", "api/v1") + +OPENTELEMETRY_ENABLED = os.environ.get("OPENTELEMETRY_ENABLED", "false").lower() == "true" +OPENTELEMETRY_DEBUG = os.environ.get("OPENTELEMETRY_DEBUG", "false").lower() == "true" +OTLP_ENDPOINT = os.environ.get("OTLP_ENDPOINT", "") +OTEL_SAMPLE_RATE = float(os.environ.get("OTEL_SAMPLE_RATE", "0.2")) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 0d581038..e63c7201 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -1,7 +1,6 @@ import datetime import random import uuid -import time import os import stat import pwd @@ -45,6 +44,16 @@ def demo_uuid(kind: str, name: str) -> str: return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) +def utc_now() -> datetime.datetime: + """Return current UTC datetime timestamp""" + return datetime.datetime.now(datetime.timezone.utc) + + +def utc_timestamp() -> int: + """Return current UTC datetime timestamp as integer""" + return int(utc_now().timestamp()) + + class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): @@ -62,7 +71,8 @@ def __init__(self): def _init_state(self): - now = datetime.datetime.now(datetime.timezone.utc) + now = utc_now() + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -80,7 +90,8 @@ def _init_state(self): longitude=-286.36999 ) - day_ago = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1) + + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), @@ -352,7 +363,7 @@ async def submit_job( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -371,7 +382,7 @@ async def submit_job_script( id="job_123", status=compute_models.JobStatus( state=compute_models.JobState.NEW, - time=time.time(), + time=utc_timestamp(), message="job submitted", exit_code=None, meta_data={ "account": "account1" }, @@ -390,7 +401,7 @@ async def update_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.ACTIVE, - time=time.time(), + time=utc_timestamp(), message="job updated", exit_code=None, meta_data={ "account": "account1" }, @@ -410,7 +421,7 @@ async def get_job( id=job_id, status=compute_models.JobStatus( state=compute_models.JobState.COMPLETED, - time=time.time(), + time=utc_timestamp(), message="job completed successfully", exit_code=0, meta_data={ "account": "account1" }, @@ -432,7 +443,7 @@ async def get_jobs( id=f"job_{i}", status=compute_models.JobStatus( state=random.choice([s for s in compute_models.JobState]), - time=time.time() - (random.random() * 100), + time=utc_timestamp() - int(random.random() * 100), message="", exit_code=random.choice([0, 0, 0, 0, 0, 1, 1, 128, 127]), meta_data={ "account": "account1" }, @@ -900,7 +911,7 @@ class DemoTaskQueue: @staticmethod async def _process_tasks(da: DemoAdapter): - now = time.time() + now = utc_timestamp() _tasks = [] for t in DemoTaskQueue.tasks: if now - t.start > 5 * 60 and t.status in [task_models.TaskStatus.completed, task_models.TaskStatus.canceled, task_models.TaskStatus.failed]: @@ -921,5 +932,5 @@ async def _process_tasks(da: DemoAdapter): @staticmethod def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> str: task_id = f"task_{len(DemoTaskQueue.tasks)}" - DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=time.time())) + DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) return task_id diff --git a/app/main.py b/app/main.py index 78641430..fa3f1eda 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,13 @@ """Main API application""" import logging from fastapi import FastAPI +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, BatchSpanProcessor, SimpleSpanProcessor +from opentelemetry.sdk.trace.sampling import TraceIdRatioBased, ParentBased +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from app.routers.error_handlers import install_error_handlers from app.routers.facility import facility @@ -13,9 +20,34 @@ from . import config +# ------------------------------------------------------------------ +# OpenTelemetry Tracing Configuration +# ------------------------------------------------------------------ +if config.OPENTELEMETRY_ENABLED: + resource = Resource.create({ + "service.name": "iri-facility-api", + "service.version": config.API_VERSION, + "service.endpoint": config.API_URL_ROOT}) + + samplerate = "1.0" if config.OPENTELEMETRY_DEBUG else config.OTEL_SAMPLE_RATE + provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(samplerate))) + trace.set_tracer_provider(provider) + + if config.OTLP_ENDPOINT: + exporter = OTLPSpanExporter(endpoint=config.OTLP_ENDPOINT, insecure=True) + span_processor = BatchSpanProcessor(exporter) + else: + exporter = ConsoleSpanExporter() + span_processor = SimpleSpanProcessor(exporter) + provider.add_span_processor(span_processor) + tracer = trace.get_tracer(__name__) +# ------------------------------------------------------------------ APP = FastAPI(**config.API_CONFIG) +if config.OPENTELEMETRY_ENABLED: + FastAPIInstrumentor.instrument_app(APP) + install_error_handlers(APP) api_prefix = f"{config.API_PREFIX}{config.API_URL}" diff --git a/pyproject.toml b/pyproject.toml index 03858a43..1f5c0d64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,5 +5,9 @@ requires-python = ">=3.12" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", - "humps>=0.2.2" -] + "humps>=0.2.2", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation-fastapi", + "opentelemetry-exporter-otlp" +] \ No newline at end of file From a23407e3a0c9a56f31382c35c320966c944dcdc1 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 20 Jan 2026 14:28:41 -0600 Subject: [PATCH 050/133] Rename dependencies to common --- app/routers/account/account.py | 2 +- app/routers/account/models.py | 2 +- app/routers/{dependencies.py => common.py} | 0 app/routers/compute/compute.py | 2 +- app/routers/facility/facility.py | 2 +- app/routers/facility/models.py | 2 +- app/routers/status/models.py | 2 +- app/routers/status/status.py | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-) rename app/routers/{dependencies.py => common.py} (100%) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index b2c41f44..fe16b8bf 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import forbidExtraQueryParams +from ..common import forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/account/models.py b/app/routers/account/models.py index c012ee32..2158575e 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,7 +1,7 @@ from pydantic import computed_field, Field import enum from ... import config -from ..dependencies import IRIBaseModel +from ..common import IRIBaseModel class AllocationUnit(enum.Enum): diff --git a/app/routers/dependencies.py b/app/routers/common.py similarity index 100% rename from app/routers/dependencies.py rename to app/routers/common.py diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 7a2db23c..20081ee6 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -4,7 +4,7 @@ from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router -from ..dependencies import forbidExtraQueryParams, StrictBool +from ..common import forbidExtraQueryParams, StrictBool router = iri_router.IriRouter( facility_adapter.FacilityAdapter, diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 1c389186..c0d3e80d 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -2,7 +2,7 @@ from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from .import models, facility_adapter -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5db7a18b..508dbcf3 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from typing import Optional from pydantic import Field, HttpUrl -from ..dependencies import NamedObject +from ..common import NamedObject class Facility(NamedObject): short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") diff --git a/app/routers/status/models.py b/app/routers/status/models.py index ff5d4e48..d338e2cf 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, computed_field, Field from ... import config -from ..dependencies import NamedObject +from ..common import NamedObject class Link(BaseModel): rel : str diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 2daf30e7..d599866c 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -2,7 +2,7 @@ from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..dependencies import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams router = iri_router.IriRouter( facility_adapter.FacilityAdapter, From 85c8e766550c6a0783d781e798c610dbd4217b01 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 09:26:32 -0600 Subject: [PATCH 051/133] Do not swallow exceptions --- app/routers/compute/compute.py | 9 ++++----- app/routers/iri_router.py | 6 ++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 20081ee6..cca45be5 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,3 +1,4 @@ +"""Compute resource API router""" from fastapi import HTTPException, Request, Depends, status, Query from . import models, facility_adapter from .. import iri_router @@ -6,13 +7,13 @@ from ..status.status import router as status_router from ..common import forbidExtraQueryParams, StrictBool + router = iri_router.IriRouter( facility_adapter.FacilityAdapter, prefix="/compute", tags=["compute"], ) - @router.post( "/job/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -204,8 +205,6 @@ async def cancel_job( # look up the resource (todo: maybe ensure it's available) resource = await status_router.adapter.get_resource(resource_id) - try: - await router.adapter.cancel_job(resource, user, job_id) - except Exception as exc: - raise HTTPException(status_code=400, detail=f"Unable to cancel job: {str(exc)}") from exc + await router.adapter.cancel_job(resource, user, job_id) + return None diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index d3fc9e16..f0b5b49c 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod +import traceback import os import logging import importlib -from urllib.parse import parse_qs from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import APIKeyHeader from .account.models import User @@ -12,9 +12,6 @@ def get_client_ip(request : Request) -> str|None: - # logging.debug("Request headers=%s" % request.headers) - # logging.debug("client=%s" % request.client.host) - forwarded_for = request.headers.get("X-Forwarded-For") if forwarded_for: return forwarded_for.split(",")[0].strip() @@ -91,6 +88,7 @@ async def current_user( user_id = await self.adapter.get_current_user(api_key, get_client_ip(request)) except Exception as exc: logging.getLogger().error(f"Error parsing IRI_API_PARAMS: {exc}") + traceback.print_exc() raise HTTPException(status_code=401, detail="Invalid or malformed Authorization parameters") from exc if not user_id: raise HTTPException(status_code=403, detail="Unauthorized access") From 29b6ff0abef5d057bb3442226ffc4e3d9fcd41c8 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 21 Jan 2026 18:58:36 -0600 Subject: [PATCH 052/133] Ensure that computed fields are included in output --- app/routers/common.py | 14 ++++++++++++-- app/routers/status/models.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index 7bfaf0b8..f46c888f 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -129,13 +129,23 @@ class IRIBaseModel(BaseModel): model_config = ConfigDict(extra="allow") @model_serializer(mode="wrap") - def _hide_extra(self, handler): + def _hide_extra(self, handler, info): data = handler(self) + + model_fields = set(self.model_fields or {}) + computed_fields = set(self.model_computed_fields or {}) + print(model_fields) + print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: - data.pop(k, None) + if k not in model_fields and k not in computed_fields: + data.pop(k, None) + return data + + + def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) diff --git a/app/routers/status/models.py b/app/routers/status/models.py index d338e2cf..448a7e21 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -36,7 +36,7 @@ def _self_path(self) -> str: current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType - @computed_field(description="The list of past events in this incident") + @computed_field(description="The list of capabilities in this resource") @property def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] From 94094de9cdd86861e61fa82d262adb56e0a5b3af Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 09:57:21 -0600 Subject: [PATCH 053/133] Code base compliant with the official Spec --- app/demo_adapter.py | 203 +++++++++++++++++++++-- app/routers/account/account.py | 17 +- app/routers/account/facility_adapter.py | 3 +- app/routers/account/models.py | 22 +-- app/routers/common.py | 48 ++++-- app/routers/facility/facility.py | 57 +++++++ app/routers/facility/facility_adapter.py | 46 +++++ app/routers/facility/models.py | 35 +++- app/routers/status/facility_adapter.py | 6 +- app/routers/status/models.py | 2 + app/routers/status/status.py | 32 ++-- 11 files changed, 398 insertions(+), 73 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index e63c7201..e68635e6 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -9,9 +9,10 @@ import subprocess import pathlib import base64 -from pydantic import BaseModel from typing import Any, Tuple +from pydantic import BaseModel from fastapi import HTTPException +from .routers.common import AllocationUnit, Capability from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter @@ -35,7 +36,7 @@ def get_base_temp_dir(cls): os.makedirs(cls._base_temp_dir, exist_ok=True) # create a test file - with open(f"{cls._base_temp_dir}/test.txt", "w") as f: + with open(f"{cls._base_temp_dir}/test.txt", encoding="utf-8", mode="w") as f: f.write("hello world") return cls._base_temp_dir @@ -67,12 +68,57 @@ def __init__(self): self.project_allocations = [] self.user_allocations = [] self.facility = {} + self.locations = [] + self.sites = [] self._init_state() def _init_state(self): now = utc_now() + loc1 = facility_models.Location( + id=demo_uuid("location", "demo_location_1"), + name="Demo Location 1", + description="The first demo location", + last_modified=now, + short_name="DL1", + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452) + + loc2 = facility_models.Location( + id=demo_uuid("location", "demo_location_2"), + name="Demo Location 2", + description="The second demo location", + last_modified=now, + short_name="DL2", + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999) + + site1 = facility_models.Site( + id=demo_uuid("site", "demo_site_1"), + name="Demo Site 1", + description="The first demo site", + last_modified=now, + short_name="DS1", + operating_organization="Demo Org", + location_uri=loc1.self_uri, + resource_uris=[]) + site2 = facility_models.Site( + id=demo_uuid("site", "demo_site_2"), + name="Demo Site 2", + description="The second demo site", + last_modified=now, + short_name="DS2", + operating_organization="Demo Org", + location_uri=loc2.self_uri, + resource_uris=[]) + self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), name="Demo Facility", @@ -81,22 +127,29 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - facility_uri="https://www.demo.example", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - street_address="1 main st", - latitude=38.410558, - longitude=-286.36999 + site_uris=[site1.self_uri, site2.self_uri], + location_uris=[loc1.self_uri, loc2.self_uri], + resource_uris=[], + event_uris=[], + incident_uris=[], + capability_uris=[], + project_uris=[], + project_allocation_uris=[], + user_allocation_uris=[], ) + loc1.site_uris.append(site1.self_uri) + loc2.site_uris.append(site2.self_uri) + self.locations = [loc1, loc2] + self.sites = [site1, site2] + day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { - "cpu": account_models.Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "gpu": account_models.Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[account_models.AllocationUnit.node_hours]), - "hpss": account_models.Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), - "gpfs": account_models.Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes]), + "cpu": Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[AllocationUnit.node_hours]), + "gpu": Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[AllocationUnit.node_hours]), + "hpss": Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), + "gpfs": Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), } pm = status_models.Resource(id=str(uuid.uuid4()), group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", capability_ids=[ @@ -226,6 +279,125 @@ async def get_facility( ) -> facility_models.Facility: return self.facility + + async def list_sites( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + + sites = self.sites + + if name: + sites = [s for s in sites if name.lower() in s.name.lower()] + + if short_name: + sites = [s for s in sites if s.short_name == short_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + sites = [s for s in sites if s.last_modified > ms] + + o = offset or 0 + l = limit or len(sites) + return sites[o:o+l] + + + async def get_site( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site: + + site = next((s for s in self.sites if s.id == site_id), None) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if site.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": site.last_modified.isoformat()}) + + return site + + + async def get_site_location( + self: "DemoAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + site = await self.get_site(site_id) + + if not site.location_uri: + raise HTTPException(status_code=404, detail="Site has no location") + + location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + + return location + + + async def list_locations( + self: "DemoAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + + locs = self.locations + + if name: + locs = [l for l in locs if name.lower() in l.name.lower()] + + if short_name: + locs = [l for l in locs if l.short_name == short_name] + + if country_name: + locs = [l for l in locs if l.country_name == country_name] + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + locs = [l for l in locs if l.last_modified > ms] + + o = offset or 0 + l = limit or len(locs) + return locs[o:o+l] + + + async def get_location( + self: "DemoAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location: + + location = next((l for l in self.locations if l.id == location_id), None) + + if not location: + raise HTTPException(status_code=404, detail="Location not found") + + if modified_since: + ms = datetime.datetime.fromisoformat(str(modified_since)) + if location.last_modified <= ms: + raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) + return location + + + + # ---------------------------- # Status API # ---------------------------- @@ -239,6 +411,8 @@ async def get_resources( group : str | None = None, modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, + current_status : status_models.Status | None = None, + capability: Capability | None = None ) -> list[status_models.Resource]: return status_models.Resource.find(self.resources, name, description, group, modified_since, resource_type)[offset:offset + limit] @@ -288,6 +462,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: return status_models.Incident.find(self.incidents, name, description, status, type, from_, to, time_, modified_since, resource_id)[offset:offset + limit] @@ -301,7 +476,7 @@ async def get_incident( async def get_capabilities( self : "DemoAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: return self.capabilities.values() diff --git a/app/routers/account/account.py b/app/routers/account/account.py index fe16b8bf..f856312e 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -1,8 +1,8 @@ -from fastapi import HTTPException, Request, Depends +from fastapi import HTTPException, Request, Depends, Query from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import forbidExtraQueryParams +from ..common import forbidExtraQueryParams, StrictDateTime, Capability router = iri_router.IriRouter( @@ -22,8 +22,12 @@ ) async def get_capabilities( request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> list[models.Capability]: + name : str = Query(default=None, min_length=1), + modified_since: StrictDateTime = Query(default=None), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + _forbid = Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), + ) -> list[Capability]: return await router.adapter.get_capabilities() @@ -37,8 +41,9 @@ async def get_capabilities( async def get_capability( capability_id : str, request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> models.Capability: + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + ) -> Capability: caps = await router.adapter.get_capabilities() cc = next((c for c in caps if c.id == capability_id), None) if not cc: diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 78b622fd..235a2f73 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -1,5 +1,6 @@ from abc import abstractmethod from . import models as account_models +from ..common import Capability from ..iri_router import AuthenticatedAdapter @@ -13,7 +14,7 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def get_capabilities( self : "FacilityAdapter", - ) -> list[account_models.Capability]: + ) -> list[Capability]: pass diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 2158575e..1a9333d8 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,26 +1,6 @@ from pydantic import computed_field, Field -import enum from ... import config -from ..common import IRIBaseModel - - -class AllocationUnit(enum.Enum): - node_hours = "node_hours" - bytes = "bytes" - inodes = "inodes" - - -class Capability(IRIBaseModel): - """ - An aspect of a resource that can have an allocation. - For example, Perlmutter nodes with GPUs - For some resources at a facility, this will be 1 to 1 with the resource. - It is a way to further subdivide a resource into allocatable sub-resources. - The word "capability" is also known to users as something they need for a job to run. (eg. gpu) - """ - id: str - name: str - units: list[AllocationUnit] +from ..common import IRIBaseModel, AllocationUnit class User(IRIBaseModel): diff --git a/app/routers/common.py b/app/routers/common.py index f46c888f..d5805656 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -1,5 +1,6 @@ """Default models used by multiple routers.""" import datetime +import enum from typing import Optional from urllib.parse import parse_qs @@ -93,7 +94,11 @@ def __get_pydantic_json_schema__(cls, schema, handler): } -def forbidExtraQueryParams(*allowedParams: str): +def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): + multiParams = multiParams or set() + + print(allowedParams, multiParams) + async def checker(req: Request): if "*" in allowedParams: return @@ -110,20 +115,26 @@ async def checker(req: Request): detail=[{ "type": "extra_forbidden", "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}" - }]) + "msg": f"Unexpected query parameter: {key}", + }], + ) - if len(values) > 1: + if len(values) > 1 and key not in multiParams: raise HTTPException( status_code=422, detail=[{ "type": "duplicate_forbidden", "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}" - }]) + "msg": f"Duplicate query parameter: {key}", + }], + ) + return checker + + + class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") @@ -134,18 +145,12 @@ def _hide_extra(self, handler, info): model_fields = set(self.model_fields or {}) computed_fields = set(self.model_computed_fields or {}) - print(model_fields) - print(computed_fields) extra = getattr(self, "__pydantic_extra__", {}) or {} for k in extra: if k not in model_fields and k not in computed_fields: data.pop(k, None) - return data - - - def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) @@ -188,3 +193,22 @@ def normalize(dt: datetime) -> datetime: modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] return a + + +class AllocationUnit(enum.Enum): + node_hours = "node_hours" + bytes = "bytes" + inodes = "inodes" + + +class Capability(IRIBaseModel): + """ + An aspect of a resource that can have an allocation. + For example, Perlmutter nodes with GPUs + For some resources at a facility, this will be 1 to 1 with the resource. + It is a way to further subdivide a resource into allocatable sub-resources. + The word "capability" is also known to users as something they need for a job to run. (eg. gpu) + """ + id: str + name: str + units: list[AllocationUnit] \ No newline at end of file diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index c0d3e80d..7c685e15 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -19,3 +19,60 @@ async def get_facility( ) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") +async def list_sites( + request: Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), + )-> list[models.Site]: + """List sites""" + return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") +async def get_site( + request: Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Site: + """Get site by ID""" + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + +@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") +async def get_site_location( + request : Request, + site_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get site location by site ID""" + return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) + +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") +async def list_locations( + request : Request, + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + short_name: str = Query(default=None, min_length=1), + country_name: str = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), + )-> list[models.Location]: + """List locations""" + return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) + +@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") +async def get_location( + request : Request, + location_id: str, + modified_since: StrictDateTime = Query(default=None), + _forbid = Depends(forbidExtraQueryParams("modified_since")), + )-> models.Location: + """Get location by ID""" + return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index cbee9515..d316674d 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -17,3 +17,49 @@ async def get_facility( ) -> facility_models.Facility | None: pass + @abstractmethod + async def list_sites( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + ) -> list[facility_models.Site]: + pass + + @abstractmethod + async def get_site( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Site | None: + pass + + @abstractmethod + async def get_site_location( + self: "FacilityAdapter", + site_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass + + @abstractmethod + async def list_locations( + self: "FacilityAdapter", + modified_since: str | None = None, + name: str | None = None, + offset: int | None = None, + limit: int | None = None, + short_name: str | None = None, + country_name: str | None = None, + ) -> list[facility_models.Location]: + pass + + @abstractmethod + async def get_location( + self: "FacilityAdapter", + location_id: str, + modified_since: str | None = None, + ) -> facility_models.Location | None: + pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 508dbcf3..fd164fad 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,13 +1,22 @@ """Facility-related models.""" -from typing import Optional +from typing import Optional, List from pydantic import Field, HttpUrl from ..common import NamedObject -class Facility(NamedObject): - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization's name.") - facility_uri: Optional[HttpUrl] = Field(None, description="URI of this facility.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + + +class Site(NamedObject): + def _self_path(self) -> str: + return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") + operating_organization: str = Field(..., description="Organization operating the Site.") + location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + +class Location(NamedObject): + def _self_path(self) -> str: + return f"/facility/locations/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Location.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -16,7 +25,21 @@ class Facility(NamedObject): altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") +class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") + organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") + event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") + incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") + capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") + project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") + project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") + user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 6753a470..d7358c50 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -2,6 +2,7 @@ import datetime from fastapi import Query from . import models as status_models +from ..common import Capability class FacilityAdapter(ABC): @@ -21,7 +22,9 @@ async def get_resources( description : str | None = None, group : str | None = None, modified_since : datetime.datetime | None = None, - resource_type: status_models.ResourceType = Query(default=None) + resource_type: status_models.ResourceType = Query(default=None), + current_status: status_models.Status = Query(default=None), + capability: Capability | None = None, ) -> list[status_models.Resource]: pass @@ -75,6 +78,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, + resolution: status_models.Resolution | None = None, ) -> list[status_models.Incident]: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 448a7e21..3650451a 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -101,6 +101,7 @@ def find( class IncidentType(enum.Enum): planned = "planned" unplanned = "unplanned" + reservation = "reservation" class Resolution(enum.Enum): @@ -111,6 +112,7 @@ class Resolution(enum.Enum): pending = "pending" + class Incident(NamedObject): def _self_path(self) -> str: diff --git a/app/routers/status/status.py b/app/routers/status/status.py index d599866c..5290a8a9 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -1,8 +1,9 @@ +from typing import Optional, List, Annotated from fastapi import HTTPException, Request, Query, Depends from . import models, facility_adapter from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import StrictDateTime, forbidExtraQueryParams +from ..common import StrictDateTime, forbidExtraQueryParams, AllocationUnit router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -22,13 +23,16 @@ async def get_resources( name : str = Query(default=None, min_length=1), description : str = Query(default=None, min_length=1), group : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type")), + current_status: models.Status = Query(default=None), + #event_uris: Optional[List[str]] = Query(default=None, min_length=1), + capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) @router.get( @@ -48,7 +52,7 @@ async def get_resource( return item -@router.get( +@router.get( "/incidents", summary="Get all incidents without their events", description="Get a list of all incidents. Each incident will be returned without its events. You can optionally filter the returned list by specifying attributes.", @@ -66,11 +70,15 @@ async def get_incidents( to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), resource_id : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), - _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit")), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), + resolution : models.Resolution = Query(default=None), + resource_uris: Optional[List[str]] = Query(default=None, min_length=1), + event_uris: Optional[List[str]] = Query(default=None, min_length=1), + _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", + "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), ) -> list[models.Incident]: - return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id) + return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id, resolution) @router.get( @@ -109,8 +117,8 @@ async def get_events( time_ : StrictDateTime = Query(alias="time", default=None), to : StrictDateTime = Query(default=None), modified_since : StrictDateTime = Query(default=None), - offset : int = Query(default=0, ge=0), - limit : int = Query(default=100, le=1000), + offset : int = Query(default=0, ge=0, le=1000), + limit : int = Query(default=100, ge=0, le=1000), _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) From 9c2eb9eb0e90ad5dc70d2bfb63a9f44616dbba06 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:08:21 -0600 Subject: [PATCH 054/133] Fully compliant with official spec --- app/routers/common.py | 28 +++++++++------------------- app/routers/status/status.py | 5 ++--- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/app/routers/common.py b/app/routers/common.py index d5805656..fd2882fe 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -97,8 +97,6 @@ def __get_pydantic_json_schema__(cls, schema, handler): def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): multiParams = multiParams or set() - print(allowedParams, multiParams) - async def checker(req: Request): if "*" in allowedParams: return @@ -110,31 +108,23 @@ async def checker(req: Request): for key, values in parsed.items(): if key not in allowed: - raise HTTPException( - status_code=422, - detail=[{ - "type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}"}]) + if len(values) > 1 and key not in multiParams: - raise HTTPException( - status_code=422, - detail=[{ - "type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}", - }], - ) + raise HTTPException(status_code=422, + detail=[{"type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}"}]) return checker - class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 5290a8a9..6a0e9489 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -28,11 +28,10 @@ async def get_resources( modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), current_status: models.Status = Query(default=None), - #event_uris: Optional[List[str]] = Query(default=None, min_length=1), - capability: Annotated[Optional[List[AllocationUnit]], Query()] = None, + capability: List[AllocationUnit] = Query(default=None, min_length=1), _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status) + return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status, capability) @router.get( From 2bd894c0651b9cf02eae815aab91fe3775e9c8f5 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 13:45:23 -0600 Subject: [PATCH 055/133] Enforce Py 3.14 as used release for everything --- .github/workflows/api-validation.yml | 11 ++++++----- Dockerfile | 2 +- pyproject.toml | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml index 624e84e2..e4d688b4 100644 --- a/.github/workflows/api-validation.yml +++ b/.github/workflows/api-validation.yml @@ -15,19 +15,18 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - # TODO: Change to official iri-facility-api-docs repo once https://github.com/doe-iri/iri-facility-api-docs/pull/11 is merged - name: Checkout schema validator repository uses: actions/checkout@v4 with: - repository: juztas/iri-facility-api-docs - ref: schemavalidator + repository: doe-iri/iri-facility-api-docs + ref: main path: schema-validator token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.14" - name: Install uv run: pip install uv @@ -81,6 +80,8 @@ jobs: --report-name schemathesis-local echo "exitcode=$?" >> $GITHUB_OUTPUT + # TODO: Change back to https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json + # Once https://github.com/doe-iri/iri-facility-api-docs/pull/12 merged. - name: Run Schemathesis validation (official spec) id: schemathesis_official env: @@ -90,7 +91,7 @@ jobs: source .venv/bin/activate python schema-validator/verification/api-validator.py \ --baseurl http://localhost:8000 \ - --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --schema-url https://raw.githubusercontent.com/juztas/iri-facility-api-docs/refs/heads/newspec/specification/openapi/openapi_iri_facility_api_v1.json \ --report-name schemathesis-official echo "exitcode=$?" >> $GITHUB_OUTPUT diff --git a/Dockerfile b/Dockerfile index f3ad071d..93c80d90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.14 RUN mkdir /app COPY . /app diff --git a/pyproject.toml b/pyproject.toml index 1f5c0d64..6dbf14b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12" +requires-python = ">=3.12,<3.13" dependencies = [ "fastapi[standard]>=0.100.0", "uvicorn[standard]>=0.22.0", @@ -10,4 +10,4 @@ dependencies = [ "opentelemetry-sdk", "opentelemetry-instrumentation-fastapi", "opentelemetry-exporter-otlp" -] \ No newline at end of file +] From 21c03caa0171bbd0ea72e5f60c8339fd7f8bb73e Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 23 Jan 2026 17:10:38 -0600 Subject: [PATCH 056/133] Enable deepsource scanning --- .deepsource.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .deepsource.toml diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 00000000..01a10663 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,9 @@ +version = 1 + +[[analyzers]] +name = "python" +enabled = true + + [analyzers.meta] + runtime_version = "3.x.x" + max_line_length = 200 From 6cb85bf29142148079968df011425f96561f8f0f Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Sun, 25 Jan 2026 08:08:41 -0600 Subject: [PATCH 057/133] Enforce consistent package versions (pin major/minor version) --- pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6dbf14b1..63f6c020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.12,<3.13" +requires-python = ">=3.14,<3.15" dependencies = [ - "fastapi[standard]>=0.100.0", - "uvicorn[standard]>=0.22.0", - "humps>=0.2.2", - "opentelemetry-api", - "opentelemetry-sdk", - "opentelemetry-instrumentation-fastapi", - "opentelemetry-exporter-otlp" -] + "fastapi[standard]>=0.128.0,<0.129.0", + "uvicorn[standard]>=0.40.0,<0.41.0", + "humps>=0.2.2,<0.3.0", + "opentelemetry-api>=1.39.1,<1.40.0", + "opentelemetry-sdk>=1.39.1,<1.40.0", + "opentelemetry-instrumentation-fastapi>=0.60b1,<0.61b0", + "opentelemetry-exporter-otlp>=1.39.1,<1.40.0" +] \ No newline at end of file From 6ccddc78883a11615dda7de6eaffb3c41b309924 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 3 Feb 2026 10:59:23 -0600 Subject: [PATCH 058/133] Make adapter forward compatible. add filtering, pagination, datetime normalization This pull request introduces: - Use and add **kwargs to all adapter interfaces (with a shared warning function for unused parameters). This enables facilities to implement on their own pace, while continue to be compatible. - Use explicit keyword args - Add common pagination helper and expand filtering based on input values; - Unify datetime handling, normalize to UTC and use it in find; --- app/demo_adapter.py | 157 +++++++++++++++++---- app/routers/account/account.py | 38 ++--- app/routers/account/facility_adapter.py | 12 +- app/routers/common.py | 114 ++++++++++++--- app/routers/compute/compute.py | 36 ++--- app/routers/compute/facility_adapter.py | 5 + app/routers/compute/models.py | 49 ++++++- app/routers/facility/facility.py | 8 +- app/routers/facility/facility_adapter.py | 6 + app/routers/facility/models.py | 21 ++- app/routers/filesystem/facility_adapter.py | 22 ++- app/routers/filesystem/filesystem.py | 112 +++++++-------- app/routers/iri_router.py | 8 ++ app/routers/status/facility_adapter.py | 14 +- app/routers/status/models.py | 127 ++++++++++------- app/routers/status/status.py | 9 +- app/routers/task/facility_adapter.py | 87 ++++++------ app/routers/task/task.py | 8 +- 18 files changed, 572 insertions(+), 261 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index e68635e6..1a520e8e 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -12,7 +12,7 @@ from typing import Any, Tuple from pydantic import BaseModel from fastapi import HTTPException -from .routers.common import AllocationUnit, Capability +from .routers.common import AllocationUnit, Capability, paginate_list from .routers.facility import models as facility_models, facility_adapter as facility_adapter from .routers.status import models as status_models, facility_adapter as status_adapter from .routers.account import models as account_models, facility_adapter as account_adapter @@ -276,7 +276,9 @@ def _init_state(self): async def get_facility( self: "DemoAdapter", modified_since: str | None = None, + **kwargs ) -> facility_models.Facility: + self._warn_on_unused_kwargs("get_facility", kwargs) return self.facility @@ -287,8 +289,9 @@ async def list_sites( offset: int | None = None, limit: int | None = None, short_name: str | None = None, + **kwargs ) -> list[facility_models.Site]: - + self._warn_on_unused_kwargs("list_sites", kwargs) sites = self.sites if name: @@ -310,8 +313,9 @@ async def get_site( self: "DemoAdapter", site_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Site: - + self._warn_on_unused_kwargs("get_site", kwargs) site = next((s for s in self.sites if s.id == site_id), None) if not site: raise HTTPException(status_code=404, detail="Site not found") @@ -328,8 +332,9 @@ async def get_site_location( self: "DemoAdapter", site_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Location: - + self._warn_on_unused_kwargs("get_site_location", kwargs) site = await self.get_site(site_id) if not site.location_uri: @@ -356,8 +361,9 @@ async def list_locations( limit: int | None = None, short_name: str | None = None, country_name: str | None = None, + **kwargs ) -> list[facility_models.Location]: - + self._warn_on_unused_kwargs("list_locations", kwargs) locs = self.locations if name: @@ -382,8 +388,9 @@ async def get_location( self: "DemoAdapter", location_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Location: - + self._warn_on_unused_kwargs("get_location", kwargs) location = next((l for l in self.locations if l.id == location_id), None) if not location: @@ -395,9 +402,6 @@ async def get_location( raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) return location - - - # ---------------------------- # Status API # ---------------------------- @@ -412,17 +416,22 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, current_status : status_models.Status | None = None, - capability: Capability | None = None + capability: Capability | None = None, + **kwargs ) -> list[status_models.Resource]: - return status_models.Resource.find(self.resources, name, description, group, modified_since, resource_type)[offset:offset + limit] + self._warn_on_unused_kwargs("get_resources", kwargs) + resources = status_models.Resource.find(self.resources, name=name, description=description, group=group, modified_since=modified_since, + resource_type=resource_type, current_status=current_status, capability=capability) + return paginate_list(resources, offset, limit) async def get_resource( self : "DemoAdapter", - id : str + id_ : str, + **kwargs ) -> status_models.Resource: - return status_models.Resource.find_by_id(self.resources, id) - + self._warn_on_unused_kwargs("get_resource", kwargs) + return status_models.Resource.find_by_id(self.resources, id_) async def get_events( self : "DemoAdapter", @@ -437,16 +446,22 @@ async def get_events( to : datetime.datetime | None = None, time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, + **kwargs ) -> list[status_models.Event]: - return status_models.Event.find([e for e in self.events if e.incident_id == incident_id], resource_id, name, description, status, from_, to, time_, modified_since)[offset:offset + limit] + self._warn_on_unused_kwargs("get_events", kwargs) + events = status_models.Event.find([e for e in self.events if e.incident_id == incident_id], resource_id=resource_id, name=name, description=description, + status=status, from_=from_, to=to, time_=time_, modified_since=modified_since) + return paginate_list(events, offset, limit) async def get_event( self : "DemoAdapter", incident_id : str, - id : str + id_ : str, + **kwargs ) -> status_models.Event: - return status_models.Event.find_by_id(self.events, id) + self._warn_on_unused_kwargs("get_event", kwargs) + return status_models.Event.find_by_id(self.events, id_) async def get_incidents( @@ -456,39 +471,57 @@ async def get_incidents( name : str | None = None, description : str | None = None, status : status_models.Status | None = None, - type : status_models.IncidentType | None = None, + type_ : status_models.IncidentType | None = None, from_ : datetime.datetime | None = None, to : datetime.datetime | None = None, time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, resolution: status_models.Resolution | None = None, + **kwargs ) -> list[status_models.Incident]: - return status_models.Incident.find(self.incidents, name, description, status, type, from_, to, time_, modified_since, resource_id)[offset:offset + limit] + self._warn_on_unused_kwargs("get_incidents", kwargs) + incidents = status_models.Incident.find(self.incidents, name=name, description=description, status=status, type_=type_,from_=from_, to=to, + time_=time_, modified_since=modified_since, resource_id=resource_id, resolution=resolution) + return paginate_list(incidents, offset, limit) async def get_incident( self : "DemoAdapter", - id : str + id_ : str, + **kwargs ) -> status_models.Incident: - return status_models.Incident.find_by_id(self.incidents, id) + self._warn_on_unused_kwargs("get_incident", kwargs) + return status_models.Incident.find_by_id(self.incidents, id_) async def get_capabilities( self : "DemoAdapter", + name : str | None = None, + modified_since : str | None = None, + offset : int = 0, + limit : int = 1000, + **kwargs ) -> list[Capability]: - return self.capabilities.values() + self._warn_on_unused_kwargs("get_capabilities", kwargs) + caps = list(self.capabilities.values()) + if name: + caps = [c for c in caps if name.lower() in c.name.lower()] + + return paginate_list(caps, offset, limit) async def get_current_user( self : "DemoAdapter", api_key: str, client_ip: str, + **kwargs ) -> str: """ In a real deployment, this would decode the api_key jwt and return the current user's id. This method is not async. """ + self._warn_on_unused_kwargs("get_current_user", kwargs) return "gtorok" @@ -497,7 +530,9 @@ async def get_user( user_id: str, api_key: str, client_ip: str|None, + **kwargs ) -> account_models.User: + self._warn_on_unused_kwargs("get_user", kwargs) if user_id != self.user.id: raise HTTPException(status_code=401, detail="User not found") if api_key != self.user.api_key: @@ -507,16 +542,20 @@ async def get_user( async def get_projects( self : "DemoAdapter", - user: account_models.User + user: account_models.User, + **kwargs ) -> list[account_models.Project]: + self._warn_on_unused_kwargs("get_projects", kwargs) return self.projects async def get_project_allocations( self : "DemoAdapter", project: account_models.Project, - user: account_models.User + user: account_models.User, + **kwargs ) -> list[account_models.ProjectAllocation]: + self._warn_on_unused_kwargs("get_project_allocations", kwargs) return [pa for pa in self.project_allocations if pa.project_id == project.id] @@ -524,7 +563,9 @@ async def get_user_allocations( self : "DemoAdapter", user: account_models.User, project_allocation: account_models.ProjectAllocation, + **kwargs ) -> list[account_models.UserAllocation]: + self._warn_on_unused_kwargs("get_user_allocations", kwargs) return [ua for ua in self.user_allocations if ua.project_allocation_id == project_allocation.id] @@ -533,7 +574,9 @@ async def submit_job( resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, + **kwargs, ) -> compute_models.Job: + self._warn_on_unused_kwargs("submit_job", kwargs) return compute_models.Job( id="job_123", status=compute_models.JobStatus( @@ -552,7 +595,9 @@ async def submit_job_script( user: account_models.User, job_script_path: str, args: list[str] = [], + **kwargs ) -> compute_models.Job: + self._warn_on_unused_kwargs("submit_job_script", kwargs) return compute_models.Job( id="job_123", status=compute_models.JobStatus( @@ -571,7 +616,9 @@ async def update_job( user: account_models.User, job_spec: compute_models.JobSpec, job_id: str, + **kwargs, ) -> compute_models.Job: + self._warn_on_unused_kwargs("update_job", kwargs) return compute_models.Job( id=job_id, status=compute_models.JobStatus( @@ -591,7 +638,9 @@ async def get_job( job_id: str, historical: bool = False, include_spec: bool = False, + **kwargs, ) -> compute_models.Job: + self._warn_on_unused_kwargs("get_job", kwargs) return compute_models.Job( id=job_id, status=compute_models.JobStatus( @@ -613,7 +662,9 @@ async def get_jobs( filters: dict[str, object] | None = None, historical: bool = False, include_spec: bool = False, + **kwargs, ) -> list[compute_models.Job]: + self._warn_on_unused_kwargs("get_jobs", kwargs) return [compute_models.Job( id=f"job_{i}", status=compute_models.JobStatus( @@ -631,7 +682,9 @@ async def cancel_job( resource: status_models.Resource, user: account_models.User, job_id: str, + **kwargs, ) -> bool: + self._warn_on_unused_kwargs("cancel_job", kwargs) # call slurm/etc. to cancel job return True @@ -701,8 +754,10 @@ async def chmod( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest + request_model: filesystem_models.PutFileChmodRequest, + **kwargs, ) -> filesystem_models.PutFileChmodResponse: + self._warn_on_unused_kwargs("chmod", kwargs) rp = self.validate_path(request_model.path) os.chmod(rp, int(request_model.mode, 8)) return filesystem_models.PutFileChmodResponse( @@ -714,8 +769,10 @@ async def chown( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChownRequest + request_model: filesystem_models.PutFileChownRequest, + **kwargs, ) -> filesystem_models.PutFileChownResponse: + self._warn_on_unused_kwargs("chown", kwargs) rp = self.validate_path(request_model.path) os.chown(rp, request_model.owner, request_model.group) return filesystem_models.PutFileChmodResponse( @@ -732,7 +789,9 @@ async def ls( numeric_uid: bool, recursive: bool, dereference: bool, + **kwargs, ) -> filesystem_models.GetDirectoryLsResponse: + self._warn_on_unused_kwargs("ls", kwargs) rp = self.validate_path(path) files = glob.glob(rp, recursive=recursive) return filesystem_models.GetDirectoryLsResponse( @@ -773,7 +832,9 @@ async def head( file_bytes: int | None, lines: int | None, skip_trailing: bool, + **kwargs, ) -> Tuple[Any, int]: + self._warn_on_unused_kwargs("head", kwargs) return self._headtail("head", path, file_bytes, lines) @@ -785,7 +846,9 @@ async def tail( file_bytes: int | None, lines: int | None, skip_trailing: bool, + **kwargs ) -> Tuple[Any, int]: + self._warn_on_unused_kwargs("tail", kwargs) return self._headtail("tail", path, file_bytes, lines) @@ -796,7 +859,9 @@ async def view( path: str, size: int, offset: int, + **kwargs ) -> filesystem_models.GetViewFileResponse: + self._warn_on_unused_kwargs("view", kwargs) rp = self.validate_path(path) result = subprocess.run( f"tail -c +{offset+1} {rp} | head -c {size}", @@ -815,7 +880,9 @@ async def checksum( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> filesystem_models.GetFileChecksumResponse: + self._warn_on_unused_kwargs("checksum", kwargs) rp = self.validate_path(path) result = subprocess.run( ["sha256sum", rp], @@ -835,7 +902,9 @@ async def file( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> filesystem_models.GetFileTypeResponse: + self._warn_on_unused_kwargs("file", kwargs) rp = self.validate_path(path) result = subprocess.run( ["file", "-b", rp], @@ -853,7 +922,9 @@ async def stat( user: account_models.User, path: str, dereference: bool, + **kwargs ) -> filesystem_models.GetFileStatResponse: + self._warn_on_unused_kwargs("stat", kwargs) rp = self.validate_path(path) if dereference: stat_info = os.stat(rp) @@ -880,7 +951,9 @@ async def rm( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ): + self._warn_on_unused_kwargs("rm", kwargs) rp = self.validate_path(path) if rp == PathSandbox.get_base_temp_dir(): raise HTTPException(status_code=400, detail="Cannot delete sandbox") @@ -893,7 +966,9 @@ async def mkdir( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest, + **kwargs ) -> filesystem_models.PostMkdirResponse: + self._warn_on_unused_kwargs("mkdir", kwargs) rp = self.validate_path(request_model.path) args = ["mkdir"] if request_model.parent: @@ -910,7 +985,9 @@ async def symlink( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostFileSymlinkRequest, + **kwargs ) -> filesystem_models.PostFileSymlinkResponse: + self._warn_on_unused_kwargs("symlink", kwargs) rp_src = self.validate_path(request_model.path) rp_dst = self.validate_path(request_model.link_path) subprocess.run(["ln", "-s", rp_src, rp_dst], check=True) @@ -924,7 +1001,9 @@ async def download( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> Any: + self._warn_on_unused_kwargs("download", kwargs) rp = self.validate_path(path) raw_content = pathlib.Path(rp).read_bytes() @@ -940,7 +1019,9 @@ async def upload( user: account_models.User, path: str, content: str, + **kwargs ) -> None: + self._warn_on_unused_kwargs("upload", kwargs) rp = self.validate_path(path) if isinstance(content, bytes): pathlib.Path(rp).write_bytes(content) @@ -955,7 +1036,9 @@ async def compress( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCompressRequest, + **kwargs ) -> filesystem_models.PostCompressResponse: + self._warn_on_unused_kwargs("compress", kwargs) src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -988,7 +1071,9 @@ async def extract( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostExtractRequest, + **kwargs ) -> filesystem_models.PostExtractResponse: + self._warn_on_unused_kwargs("extract", kwargs) src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -1016,7 +1101,9 @@ async def mv( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMoveRequest, + **kwargs ) -> filesystem_models.PostMoveResponse: + self._warn_on_unused_kwargs("mv", kwargs) src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) subprocess.run(["mv", src_rp, dst_rp], check=True) @@ -1030,7 +1117,9 @@ async def cp( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCopyRequest, + **kwargs ) -> filesystem_models.PostCopyResponse: + self._warn_on_unused_kwargs("cp", kwargs) src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) args = ["cp"] @@ -1048,7 +1137,9 @@ async def get_task( self : "DemoAdapter", user: account_models.User, task_id: str, + **kwargs ) -> task_models.Task|None: + self._warn_on_unused_kwargs("get_task", kwargs) await DemoTaskQueue._process_tasks(self) return next((t for t in DemoTaskQueue.tasks if t.user.name == user.name and t.id == task_id), None) @@ -1056,7 +1147,9 @@ async def get_task( async def get_tasks( self : "DemoAdapter", user: account_models.User, + **kwargs ) -> list[task_models.Task]: + self._warn_on_unused_kwargs("get_tasks", kwargs) await DemoTaskQueue._process_tasks(self) return [t for t in DemoTaskQueue.tasks if t.user.name == user.name] @@ -1065,15 +1158,17 @@ async def put_task( self: "DemoAdapter", user: account_models.User, resource: status_models.Resource, - body: str + task: str, + **kwargs, ) -> str: + self._warn_on_unused_kwargs("put_task", kwargs) await DemoTaskQueue._process_tasks(self) - return DemoTaskQueue._create_task(user, resource, body) + return DemoTaskQueue._create_task(user, resource, task) class DemoTask(BaseModel): id: str - body: str + task: str resource: status_models.Resource user: account_models.User start: float @@ -1096,7 +1191,7 @@ async def _process_tasks(da: DemoAdapter): t.status = task_models.TaskStatus.active t.start = now elif t.status == task_models.TaskStatus.active and now - t.start > DEMO_QUEUE_UPDATE_SECS: - cmd = task_models.TaskCommand.model_validate_json(t.body) + cmd = task_models.TaskCommand.model_validate_json(t.task) (result, status) = await DemoAdapter.on_task(t.resource, t.user, cmd) t.result = result t.status = status @@ -1107,5 +1202,5 @@ async def _process_tasks(da: DemoAdapter): @staticmethod def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> str: task_id = f"task_{len(DemoTaskQueue.tasks)}" - DemoTaskQueue.tasks.append(DemoTask(id=task_id, body=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) + DemoTaskQueue.tasks.append(DemoTask(id=task_id, task=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) return task_id diff --git a/app/routers/account/account.py b/app/routers/account/account.py index f856312e..e13cbde2 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -28,7 +28,7 @@ async def get_capabilities( limit : int = Query(default=100, ge=0, le=1000), _forbid = Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), ) -> list[Capability]: - return await router.adapter.get_capabilities() + return await router.adapter.get_capabilities(name=name, modified_since=modified_since, offset=offset, limit=limit) @router.get( @@ -44,7 +44,7 @@ async def get_capability( modified_since: StrictDateTime = Query(default=None), _forbid = Depends(forbidExtraQueryParams("modified_since")), ) -> Capability: - caps = await router.adapter.get_capabilities() + caps = await router.adapter.get_capabilities(name=None, modified_since=modified_since, offset=0, limit=100) cc = next((c for c in caps if c.id == capability_id), None) if not cc: raise HTTPException(status_code=404, detail="Capability not found") @@ -63,7 +63,7 @@ async def get_projects( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.Project]: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") return await router.adapter.get_projects(user) @@ -82,10 +82,10 @@ async def get_project( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> models.Project: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) pp = next((p for p in projects if p.id == project_id), None) if not pp: raise HTTPException(status_code=404, detail="Project not found") @@ -105,14 +105,14 @@ async def get_project_allocations( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: raise HTTPException(status_code=404, detail="Project not found") - return await router.adapter.get_project_allocations(project, user) + return await router.adapter.get_project_allocations(project=project, user=user) @router.get( @@ -129,12 +129,12 @@ async def get_project_allocation( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> models.ProjectAllocation: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) - pas = await router.adapter.get_project_allocations(project, user) + pas = await router.adapter.get_project_allocations(project=project, user=user) pa = next((pa for pa in pas if pa.id == project_allocation_id), None) if not pa: raise HTTPException(status_code=404, detail="Project allocation not found") @@ -155,18 +155,18 @@ async def get_user_allocations( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> list[models.UserAllocation]: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: raise HTTPException(status_code=404, detail="Project not found") - pas = await router.adapter.get_project_allocations(project, user) + pas = await router.adapter.get_project_allocations(project=project, user=user) pa = next((pa for pa in pas if pa.id == project_allocation_id), None) if not pa: raise HTTPException(status_code=404, detail="Project allocation not found") - return await router.adapter.get_user_allocations(user, pa) + return await router.adapter.get_user_allocations(user=user, project_allocation=pa) @router.get( @@ -184,18 +184,18 @@ async def get_user_allocation( request : Request, _forbid = Depends(forbidExtraQueryParams()), ) -> models.UserAllocation: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - projects = await router.adapter.get_projects(user) + projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: raise HTTPException(status_code=404, detail="Project not found") - pas = await router.adapter.get_project_allocations(project, user) + pas = await router.adapter.get_project_allocations(project=project, user=user) pa = next((pa for pa in pas if pa.id == project_allocation_id), None) if not pa: raise HTTPException(status_code=404, detail="Project allocation not found") - uas = await router.adapter.get_user_allocations(user, pa) + uas = await router.adapter.get_user_allocations(user=user, project_allocation=pa) ua = next((ua for ua in uas if ua.id == user_allocation_id), None) if not ua: raise HTTPException(status_code=404, detail="User allocation not found") diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 235a2f73..3abca875 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -14,6 +14,11 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def get_capabilities( self : "FacilityAdapter", + name : str | None = None, + modified_since : str | None = None, + offset : int = 0, + limit : int = 1000, + **kwargs ) -> list[Capability]: pass @@ -21,7 +26,8 @@ async def get_capabilities( @abstractmethod async def get_projects( self : "FacilityAdapter", - user: account_models.User + user: account_models.User, + **kwargs ) -> list[account_models.Project]: pass @@ -30,7 +36,8 @@ async def get_projects( async def get_project_allocations( self : "FacilityAdapter", project: account_models.Project, - user: account_models.User + user: account_models.User, + **kwargs ) -> list[account_models.ProjectAllocation]: pass @@ -40,5 +47,6 @@ async def get_user_allocations( self : "FacilityAdapter", user: account_models.User, project_allocation: account_models.ProjectAllocation, + **kwargs ) -> list[account_models.UserAllocation]: pass diff --git a/app/routers/common.py b/app/routers/common.py index fd2882fe..2826dab7 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -1,16 +1,25 @@ """Default models used by multiple routers.""" import datetime +from email.utils import parsedate_to_datetime import enum from typing import Optional from urllib.parse import parse_qs from pydantic_core import core_schema -from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer -from fastapi import Request, HTTPException +from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer, field_validator +from fastapi import Request, HTTPException, status from .. import config +def paginate_list(items, offset: int | None, limit: int | None): + """Return a sliced items using offset and limit.""" + if offset is not None and offset > 0: + items = items[offset:] + if limit is not None and limit >= 0: + items = items[:limit] + return items + # These are Pydantic custom types for strict validation # that are not implmented in Pydantic by default. # ----------------------------------------------------------------------- @@ -79,6 +88,51 @@ def validate(value): return StrictDateTime._normalize(dt) + + @staticmethod + def modifiedSinceDatetime( + modified_since: str | None, + header_modified_since: str | None + ) -> datetime.datetime | None: + """ + Combine modified_since (ISO8601) and If-Modified-Since (RFC1123). + If both are provided, the most recent timestamp is used. + """ + + parsed_times: list[datetime.datetime] = [] + + # Query param (ISO 8601) + if modified_since is not None: + try: + dt = StrictDateTime.validate(modified_since) + parsed_times.append(dt) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid modified_since query param: {exc}", + ) from exc + + # Header (RFC 1123) + if header_modified_since is not None: + try: + dt = parsedate_to_datetime(header_modified_since) + if dt is None: + raise ValueError("Invalid RFC1123 date") + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + parsed_times.append(dt.astimezone(datetime.timezone.utc)) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid If-Modified-Since header format (must be RFC1123)", + ) from exc + + if not parsed_times: + return None + + # Stricter constraint wins + return max(parsed_times) + @staticmethod def _normalize(dt: datetime.datetime) -> datetime.datetime: if dt.tzinfo is None: @@ -123,8 +177,6 @@ async def checker(req: Request): return checker - - class IRIBaseModel(BaseModel): """Base model for IRI models.""" model_config = ConfigDict(extra="allow") @@ -150,6 +202,23 @@ class NamedObject(IRIBaseModel): def _self_path(self) -> str: raise NotImplementedError + @classmethod + def normalize_dt(cls, dt: datetime | None) -> datetime | None: + """Normalize datetime to UTC-aware.""" + # Convert naive datetimes into UTC-aware versions + if dt is None: + return None + if isinstance(dt, str): + dt = StrictDateTime.validate(dt) + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @field_validator("last_modified", mode="before") + @classmethod + def _norm_dt_field(cls, v): + return cls.normalize_dt(v) + @computed_field(description="The canonical URL of this object") @property def self_uri(self) -> str: @@ -160,29 +229,32 @@ def self_uri(self) -> str: description: Optional[str] = Field(None, description="Human-readable description of the object.") last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - @staticmethod - def find_by_id(a, id, allow_name: bool|None=False): + @classmethod + def find_by_id(cls, items, id_, allow_name: bool = False): + """ Find an object by its id or name == id. """ # Find a resource by its id. # If allow_name is True, the id parameter can also match the resource's name. - return next((r for r in a if r.id == id or (allow_name and r.name == id)), None) + matches = [r for r in items if r.id == id_ or (allow_name and r.name == id_)] + if not matches: + return None + if len(matches) > 1: + raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id}'") + return matches[0] - @staticmethod - def find(a, name, description, modified_since): - def normalize(dt: datetime) -> datetime: - # Convert naive datetimes into UTC-aware versions - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt + @classmethod + def find(cls, items, name=None, description=None, modified_since=None): + """ Find objects matching the given criteria. """ + if not any((name, description, modified_since)): + return items if name: - a = [aa for aa in a if aa.name == name] + items = [item for item in items if item.name == name] if description: - a = [aa for aa in a if description in aa.description] + items = [item for item in items if item.description and description in item.description] if modified_since: - if modified_since.tzinfo is None: - modified_since = modified_since.replace(tzinfo=datetime.timezone.utc) - a = [aa for aa in a if normalize(aa.last_modified) >= modified_since] - return a + modified_since = cls.normalize_dt(modified_since) + items = [item for item in items if item.last_modified >= modified_since] + return items class AllocationUnit(enum.Enum): @@ -201,4 +273,4 @@ class Capability(IRIBaseModel): """ id: str name: str - units: list[AllocationUnit] \ No newline at end of file + units: list[AllocationUnit] diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index cca45be5..804f9a5f 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -36,16 +36,16 @@ async def submit_job( This command will attempt to submit a job and return its id. """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.submit_job(resource, user, job_spec) + return await router.adapter.submit_job(resource=resource, user=user, job_spec=job_spec) # TODO: this conflicts with PUT commented out while we finalize the API design @@ -73,16 +73,16 @@ async def submit_job( # # This command will attempt to submit a job and return its id. # """ -# user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) +# user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) # if not user: # raise HTTPException(status_code=404, detail="User not found") # # # look up the resource (todo: maybe ensure it's available) -# resource = await status_router.adapter.get_resource(resource_id) +# resource = await status_router.adapter.get_resource(resource_id=resource_id) # # # the handler can use whatever means it wants to submit the job and then fill in its id # # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs -# return await router.adapter.submit_job_script(resource, user, job_script_path, args) +# return await router.adapter.submit_job_script(resource=resource, user=user, job_script_path=job_script_path, args=args) @router.put( @@ -108,16 +108,16 @@ async def update_job( - **job_request**: a PSIJ job spec as defined here """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.update_job(resource, user, job_spec, job_id) + return await router.adapter.update_job(resource=resource, user=user, job_spec=job_spec, job_id=job_id) @router.get( @@ -137,15 +137,15 @@ async def get_job_status( _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) - job = await router.adapter.get_job(resource, user, job_id, historical, include_spec) + job = await router.adapter.get_job(resource=resource, user=user, job_id=job_id, historical=historical, include_spec=include_spec) return job @@ -169,15 +169,15 @@ async def get_job_statuses( _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) - jobs = await router.adapter.get_jobs(resource, user, offset, limit, filters, historical, include_spec) + jobs = await router.adapter.get_jobs(resource=resource, user=user, offset=offset, limit=limit, filters=filters, historical=historical, include_spec=include_spec) return jobs @@ -198,13 +198,13 @@ async def cancel_job( _forbid = Depends(forbidExtraQueryParams()), ): """Cancel a job""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) - await router.adapter.cancel_job(resource, user, job_id) + await router.adapter.cancel_job(resource=resource, user=user, job_id=job_id) return None diff --git a/app/routers/compute/facility_adapter.py b/app/routers/compute/facility_adapter.py index 6cf0bb2f..d70ec517 100644 --- a/app/routers/compute/facility_adapter.py +++ b/app/routers/compute/facility_adapter.py @@ -19,6 +19,7 @@ async def submit_job( resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, + **kwargs ) -> compute_models.Job: pass @@ -30,6 +31,7 @@ async def submit_job_script( user: account_models.User, job_script_path: str, args: list[str] = [], + **kwargs ) -> compute_models.Job: pass @@ -41,6 +43,7 @@ async def update_job( user: account_models.User, job_spec: compute_models.JobSpec, job_id: str, + **kwargs ) -> compute_models.Job: pass @@ -67,6 +70,7 @@ async def get_jobs( filters: dict[str, object] | None = None, historical: bool = False, include_spec: bool = False, + **kwargs ) -> list[compute_models.Job]: pass @@ -77,5 +81,6 @@ async def cancel_job( resource: status_models.Resource, user: account_models.User, job_id: str, + **kwargs ) -> bool: pass diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index a56d4fe4..3aa4121b 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,10 +1,11 @@ from typing import Annotated from enum import IntEnum -from pydantic import field_serializer, ConfigDict, StrictBool, Field +from pydantic import field_serializer, StrictBool, Field from ..common import IRIBaseModel class ResourceSpec(IRIBaseModel): +<<<<<<< HEAD """ Specification of computational resources required for a job. """ @@ -27,14 +28,37 @@ class JobAttributes(IRIBaseModel): reservation_id: Annotated[str | None, Field(min_length=1, description="ID of a reservation to use for the job")] = None custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs")] = {} +======= + node_count: int | None = Field(default=None, description="Number of compute nodes to allocate") + process_count: int | None = Field(default=None, description="Total number of processes to launch") + processes_per_node: int | None = Field(default=None, description="Number of processes to launch per node") + cpu_cores_per_process: int | None = Field(default=None, description="Number of CPU cores to allocate per process") + gpu_cores_per_process: int | None = Field(default=None, description="Number of GPU cores to allocate per process") + exclusive_node_use: StrictBool = Field(default=True, description="Whether to request exclusive use of allocated nodes") + memory: int | None = Field(default=None, description="Amount of memory to allocate in bytes") + + +class JobAttributes(IRIBaseModel): + duration: int = Field(default=None, ge=1, description="Duration in seconds", examples=[30, 60, 120]) + queue_name: str | None = Field(default=None, min_length=1, description="Name of the queue or partition to submit the job to") + account: str | None = Field(default=None, min_length=1, description="Account or project to charge for resource usage") + reservation_id: str | None = Field(default=None, min_length=1, description="ID of a reservation to use for the job") + custom_attributes: dict[str, str] = Field(default={}, description="Custom scheduler-specific attributes as key-value pairs") +>>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class VolumeMount(IRIBaseModel): """ Represents a volume mount for a container. """ +<<<<<<< HEAD source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount")] target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted")] read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only")] = True +======= + source: str = Field(description="The source path on the host system to mount") + target: str = Field(description="The target path inside the container where the volume will be mounted") + read_only: StrictBool = Field(default=True, description="Whether the mount should be read-only") +>>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class Container(IRIBaseModel): """ @@ -45,6 +69,7 @@ class Container(IRIBaseModel): to determine if the container should be run with MPI support. The container should by default. be run with host networking. """ +<<<<<<< HEAD image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] @@ -69,6 +94,28 @@ class JobSpec(IRIBaseModel): pre_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run before launching the job")] = None post_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run after the job completes")] = None launcher: Annotated[str | None, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None +======= + image: str = Field(description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')") + volume_mounts: list[VolumeMount] = Field(default=[], description="List of volume mounts for the container") + + +class JobSpec(IRIBaseModel): + executable: str | None = Field(default=None, description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.") + container: Container | None = Field(default=None, description="Container specification for containerized execution") + arguments: list[str] = Field(default=[], description="Command-line arguments to pass to the executable or container") + directory: str | None = Field(default=None, description="Working directory for the job") + name: str | None = Field(default=None, description="Name of the job") + inherit_environment: StrictBool = Field(default=True, description="Whether to inherit the environment variables from the submission environment") + environment: dict[str, str] = Field(default={}, description="Environment variables to set for the job. If container is specified, these will be set inside the container.") + stdin_path: str | None = Field(default=None, description="Path to file to use as standard input") + stdout_path: str | None = Field(default=None, description="Path to file to write standard output") + stderr_path: str | None = Field(default=None, description="Path to file to write standard error") + resources: ResourceSpec | None = Field(default=None, description="Resource requirements for the job") + attributes: JobAttributes | None = Field(default=None, description="Additional job attributes such as duration, queue, and account") + pre_launch: str | None = Field(default=None, description="Script or commands to run before launching the job") + post_launch: str | None = Field(default=None, description="Script or commands to run after the job completes") + launcher: str | None = Field(default=None, description="Job launcher to use (e.g., 'mpirun', 'srun')") +>>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class CommandResult(IRIBaseModel): diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 7c685e15..7d264af3 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -5,11 +5,9 @@ from ..common import StrictDateTime, forbidExtraQueryParams -router = iri_router.IriRouter( - facility_adapter.FacilityAdapter, - prefix="/facility", - tags=["facility"], -) +router = iri_router.IriRouter(facility_adapter.FacilityAdapter, + prefix="/facility", + tags=["facility"]) @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index d316674d..fc6777fb 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -14,6 +14,7 @@ class FacilityAdapter(AuthenticatedAdapter): async def get_facility( self: "FacilityAdapter", modified_since: str | None = None, + **kwargs ) -> facility_models.Facility | None: pass @@ -25,6 +26,7 @@ async def list_sites( offset: int | None = None, limit: int | None = None, short_name: str | None = None, + **kwargs ) -> list[facility_models.Site]: pass @@ -33,6 +35,7 @@ async def get_site( self: "FacilityAdapter", site_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Site | None: pass @@ -41,6 +44,7 @@ async def get_site_location( self: "FacilityAdapter", site_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Location | None: pass @@ -53,6 +57,7 @@ async def list_locations( limit: int | None = None, short_name: str | None = None, country_name: str | None = None, + **kwargs ) -> list[facility_models.Location]: pass @@ -61,5 +66,6 @@ async def get_location( self: "FacilityAdapter", location_id: str, modified_since: str | None = None, + **kwargs ) -> facility_models.Location | None: pass diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index fd164fad..bee399fc 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -13,6 +13,14 @@ def _self_path(self) -> str: location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, short_name=None): + """ Find Sites matching the given criteria. """ + items = super().find(items, name=name, description=description, modified_since=modified_since) + if short_name: + items = [item for item in items if item.short_name == short_name] + return items + class Location(NamedObject): def _self_path(self) -> str: return f"/facility/locations/{self.id}" @@ -27,6 +35,18 @@ def _self_path(self) -> str: longitude: Optional[float] = Field(None, description="Longitude of the Location.") site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, short_name=None, country_name=None): + """ Find Locations matching the given criteria. """ + items = super().find(items, name=name, description=description, modified_since=modified_since) + if short_name: + items = [item for item in items if item.short_name == short_name] + if country_name: + items = [item for item in items if item.country_name == country_name] + return items + + + class Facility(NamedObject): def _self_path(self) -> str: return f"/facility/facilities/{self.id}" @@ -42,4 +62,3 @@ def _self_path(self) -> str: project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") - diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index a70efb0d..8506f1c7 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -29,7 +29,8 @@ async def chmod( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest + request_model: filesystem_models.PutFileChmodRequest, + **kwargs ) -> filesystem_models.PutFileChmodResponse: pass @@ -39,7 +40,8 @@ async def chown( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChownRequest + request_model: filesystem_models.PutFileChownRequest, + **kwargs ) -> filesystem_models.PutFileChownResponse: pass @@ -54,6 +56,7 @@ async def ls( numeric_uid: bool, recursive: bool, dereference: bool, + **kwargs ) -> filesystem_models.GetDirectoryLsResponse: pass @@ -67,6 +70,7 @@ async def head( file_bytes: int, lines: int, skip_trailing: bool, + **kwargs ) -> Tuple[Any, int]: pass @@ -80,6 +84,7 @@ async def tail( file_bytes: int | None, lines: int | None, skip_trailing: bool, + **kwargs ) -> Tuple[Any, int]: pass @@ -92,6 +97,7 @@ async def view( path: str, size: int, offset: int, + **kwargs ) -> filesystem_models.GetViewFileResponse: pass @@ -102,6 +108,7 @@ async def checksum( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> filesystem_models.GetFileChecksumResponse: pass @@ -112,6 +119,7 @@ async def file( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> filesystem_models.GetFileTypeResponse: pass @@ -123,6 +131,7 @@ async def stat( user: account_models.User, path: str, dereference: bool, + **kwargs ) -> filesystem_models.GetFileStatResponse: pass @@ -133,6 +142,7 @@ async def rm( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ): pass @@ -143,6 +153,7 @@ async def mkdir( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest, + **kwargs ) -> filesystem_models.PostMkdirResponse: pass @@ -153,6 +164,7 @@ async def symlink( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostFileSymlinkRequest, + **kwargs ) -> filesystem_models.PostFileSymlinkResponse: pass @@ -163,6 +175,7 @@ async def download( resource: status_models.Resource, user: account_models.User, path: str, + **kwargs ) -> Any: pass @@ -174,6 +187,7 @@ async def upload( user: account_models.User, path: str, content: str, + **kwargs ) -> None: pass @@ -184,6 +198,7 @@ async def compress( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCompressRequest, + **kwargs ) -> filesystem_models.PostCompressResponse: pass @@ -194,6 +209,7 @@ async def extract( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostExtractRequest, + **kwargs ) -> filesystem_models.PostExtractResponse: pass @@ -204,6 +220,7 @@ async def mv( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMoveRequest, + **kwargs ) -> filesystem_models.PostMoveResponse: pass @@ -214,5 +231,6 @@ async def cp( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCopyRequest, + **kwargs ) -> filesystem_models.PostCopyResponse: pass diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index d583c642..fb27427e 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -34,12 +34,12 @@ async def _user_resource( resource_id: str, request: Request, ) -> tuple[account_models.User, status_models.Resource]: - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id=resource_id) if not resource: raise HTTPException(status_code=404, detail="Resource not found") return (user, resource) @@ -62,9 +62,9 @@ async def put_chmod( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="chmod", args={ @@ -91,9 +91,9 @@ async def put_chown( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="chown", args={ @@ -121,9 +121,9 @@ async def get_file( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="file", args={ @@ -151,9 +151,9 @@ async def get_stat( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="stat", args={ @@ -181,9 +181,9 @@ async def post_mkdir( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="mkdir", args={ @@ -211,9 +211,9 @@ async def post_symlink( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="symlink", args={ @@ -257,9 +257,9 @@ async def get_ls_async( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="ls", args={ @@ -325,9 +325,9 @@ async def get_head( detail="Exactly one of `bytes` or `lines` must be specified." ) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="head", args={ @@ -363,9 +363,9 @@ async def get_view( user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="view", args={ @@ -422,9 +422,9 @@ async def get_tail( detail="Exactly one of `bytes` or `lines` must be specified." ) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="tail", args={ @@ -455,9 +455,9 @@ async def get_checksum( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="checksum", args={ @@ -481,9 +481,9 @@ async def delete_rm( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="rm", args={ @@ -510,9 +510,9 @@ async def post_compress( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="compress", args={ @@ -539,9 +539,9 @@ async def post_extract( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="extract", args={ @@ -568,9 +568,9 @@ async def move_mv( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="mv", args={ @@ -597,9 +597,9 @@ async def post_cp( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="cp", args={ @@ -625,9 +625,9 @@ async def get_download( ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="download", args={ @@ -665,9 +665,9 @@ async def post_upload( ) return await router.task_adapter.put_task( - user, - resource, - task_models.TaskCommand( + user=user, + resource=resource, + task=task_models.TaskCommand( router=router.get_router_name(), command="upload", args={ diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index f0b5b49c..d4cb6f2e 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -98,11 +98,18 @@ async def current_user( class AuthenticatedAdapter(ABC): + def _warn_on_unused_kwargs(self, func_name: str, kwargs: dict) -> None: + if not kwargs: + return + logging.getLogger().warning("Adapter method '%s' received unused kwargs: %s", func_name, + ", ".join(sorted(kwargs.keys()))) + @abstractmethod async def get_current_user( self : "AuthenticatedAdapter", api_key: str, client_ip: str|None, + **kwargs ) -> str: """ Decode the api_key and return the authenticated user's id. @@ -118,6 +125,7 @@ async def get_user( user_id: str, api_key: str, client_ip: str|None, + **kwargs ) -> User: """ Retrieve additional user information (name, email, etc.) for the given user_id. diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index d7358c50..1d774cb0 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -25,6 +25,7 @@ async def get_resources( resource_type: status_models.ResourceType = Query(default=None), current_status: status_models.Status = Query(default=None), capability: Capability | None = None, + **kwargs ) -> list[status_models.Resource]: pass @@ -32,7 +33,8 @@ async def get_resources( @abstractmethod async def get_resource( self : "FacilityAdapter", - id : str + id_ : str, + **kwargs ) -> status_models.Resource: pass @@ -49,8 +51,9 @@ async def get_events( status : status_models.Status | None = None, from_ : datetime.datetime | None = None, to : datetime.datetime | None = None, - time : datetime.datetime | None = None, + time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, + **kwargs ) -> list[status_models.Event]: pass @@ -59,7 +62,8 @@ async def get_events( async def get_event( self : "FacilityAdapter", incident_id : str, - id : str + id_ : str, + **kwargs ) -> status_models.Event: pass @@ -79,6 +83,7 @@ async def get_incidents( modified_since : datetime.datetime | None = None, resource_id : str | None = None, resolution: status_models.Resolution | None = None, + **kwargs ) -> list[status_models.Incident]: pass @@ -86,6 +91,7 @@ async def get_incidents( @abstractmethod async def get_incident( self : "FacilityAdapter", - id : str + id_ : str, + **kwargs ) -> status_models.Incident: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 3650451a..013323f7 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -1,6 +1,6 @@ import datetime import enum -from pydantic import BaseModel, computed_field, Field +from pydantic import BaseModel, computed_field, Field, field_validator from ... import config from ..common import NamedObject @@ -29,33 +29,47 @@ class ResourceType(enum.Enum): class Resource(NamedObject): def _self_path(self) -> str: + """ Return the API path for this resource. """ return f"/status/resources/{self.id}" - capability_ids: list[str] = Field(exclude=True) + capability_ids: list[str] = Field(default_factory=list, exclude=True) group: str | None - current_status: Status | None = Field("The current status comes from the status of the last event for this resource") + current_status: Status | None = Field(default=None, description="The current status comes from the status of the last event for this resource") resource_type: ResourceType @computed_field(description="The list of capabilities in this resource") @property def capability_uris(self) -> list[str]: + """ Return the list of capability URIs for this resource. """ return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] - @staticmethod - def find(resources, name, description, group, modified_since, resource_type): - a = NamedObject.find(resources, name, description, modified_since) + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, group=None, resource_type=None, current_status=None, capability=None) -> list: + items = super().find(items, name=name, description=description, modified_since=modified_since) if group: - a = [aa for aa in a if aa.group == group] + items = [item for item in items if item.group == group] if resource_type: - a = [aa for aa in a if aa.resource_type == resource_type] - return a - + if isinstance(resource_type, str): + resource_type = ResourceType(resource_type) + items = [item for item in items if item.resource_type == resource_type] + if current_status: + items = [item for item in items if item.current_status == current_status] + if capability: + items = [item for item in items + if any(cap_id in item.capability_ids for cap_id in capability)] + return items class Event(NamedObject): def _self_path(self) -> str: + """ Return the API path for this event. """ return f"/status/incidents/{self.incident_id}/events/{self.id}" + @field_validator("occurred_at", mode="before") + @classmethod + def _norm_dt_field(cls, v): + return cls.normalize_dt(v) + occurred_at : datetime.datetime status : Status resource_id : str = Field(exclude=True) @@ -64,38 +78,39 @@ def _self_path(self) -> str: @computed_field(description="The resource belonging to this event") @property def resource_uri(self) -> str: + """ Return the resource URI for this event. """ return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.resource_id}" @computed_field(description="The event's incident") @property def incident_uri(self) -> str|None: + """ Return the incident URI for this event. """ return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}" if self.incident_id else None - @staticmethod - def find( - events : list, - resource_id : str | None = None, - name : str | None = None, - description : str | None = None, - status : Status | None = None, - from_ : datetime.datetime | None = None, - to : datetime.datetime | None = None, - time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - ) -> list: - events = NamedObject.find(events, name, description, modified_since) + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, + resource_id=None, status=None, from_=None, to=None, time_=None) -> list: + items = super().find(items, name=name, description=description, modified_since=modified_since) + if resource_id: - events = [e for e in events if e.resource_id == resource_id] + items = [e for e in items if e.resource_id == resource_id] if status: - events = [e for e in events if e.status == status] + if isinstance(status, str): + status = Status(status) + items = [e for e in items if e.status == status] + + from_ = cls.normalize_dt(from_) if from_ else None + to = cls.normalize_dt(to) if to else None + time_ = cls.normalize_dt(time_) if time_ else None + if from_: - events = [e for e in events if e.occurred_at >= from_] + items = [e for e in items if e.occurred_at >= from_] if to: - events = [e for e in events if e.occurred_at < to] + items = [e for e in items if e.occurred_at < to] if time_: - events = [e for e in events if e.occurred_at == time_] - return events + items = [e for e in items if e.occurred_at == time_] + return items class IncidentType(enum.Enum): @@ -116,11 +131,17 @@ class Resolution(enum.Enum): class Incident(NamedObject): def _self_path(self) -> str: + """ Return the API path for this incident. """ return f"/status/incidents/{self.id}" + @field_validator("start", "end", mode="before") + @classmethod + def _norm_dt_field(cls, v): + return cls.normalize_dt(v) + status : Status - resource_ids : list[str] = Field(exclude=True) - event_ids : list[str] = Field(exclude=True) + resource_ids : list[str] = Field(default_factory=list, exclude=True) + event_ids : list[str] = Field(default_factory=list, exclude=True) start : datetime.datetime end : datetime.datetime | None type : IncidentType @@ -129,37 +150,39 @@ def _self_path(self) -> str: @computed_field(description="The list of past events in this incident") @property def event_uris(self) -> list[str]: + """ Return the list of event URIs for this incident. """ return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}/events/{e}" for e in self.event_ids] @computed_field(description="The list of resources that may be impacted by this incident") @property def resource_uris(self) -> list[str]: + """ Return the list of resource URIs for this incident. """ return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] - @staticmethod - def find( - incidents : list, - name : str | None = None, - description : str | None = None, - status : Status | None = None, - type_ : IncidentType | None = None, - from_ : datetime.datetime | None = None, - to : datetime.datetime | None = None, - time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - resource_id : str | None = None, - ) -> list: - incidents = NamedObject.find(incidents, name, description, modified_since) + @classmethod + def find(cls, items, name=None, description=None, modified_since=None, status=None, + type_=None, from_= None, to = None, time_ = None, resource_id = None, resolution=None) -> list: + items = super().find(items, name=name, description=description, modified_since=modified_since) + if resource_id: - incidents = [e for e in incidents if resource_id in e.resource_ids] + items = [e for e in items if resource_id in e.resource_ids] if status: - incidents = [e for e in incidents if e.status == status] + items = [e for e in items if e.status == status] if type_: - incidents = [e for e in incidents if e.type == type_] + items = [e for e in items if e.type == type_] + if resolution: + items = [e for e in items if e.resolution == resolution] + + from_ = cls.normalize_dt(from_) if from_ else None + to = cls.normalize_dt(to) if to else None + time_ = cls.normalize_dt(time_) if time_ else None + if from_: - incidents = [e for e in incidents if e.start >= from_] + items = [e for e in items if e.start >= from_] if to: - incidents = [e for e in incidents if e.end < to] + items = [e for e in items if e.end and e.end < to] + if time_: - incidents = [e for e in incidents if e.start <= time_ and e.end > time_] - return incidents + items = [e for e in items + if e.start <= time_ and (e.end is None or e.end > time_)] + return items \ No newline at end of file diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 6a0e9489..eb78109a 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -31,7 +31,8 @@ async def get_resources( capability: List[AllocationUnit] = Query(default=None, min_length=1), _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: - return await router.adapter.get_resources(offset, limit, name, description, group, modified_since, resource_type, current_status, capability) + return await router.adapter.get_resources(offset=offset, limit=limit, name=name, description=description, group=group, modified_since=modified_since, + resource_type=resource_type, current_status=current_status, capability=capability) @router.get( @@ -77,7 +78,8 @@ async def get_incidents( _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), ) -> list[models.Incident]: - return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id, resolution) + return await router.adapter.get_incidents(offset=offset, limit=limit, name=name, description=description, status=status, type_=type_, from_=from_, to=to, + time_=time_, modified_since=modified_since, resource_id=resource_id, resolution=resolution) @router.get( @@ -120,7 +122,8 @@ async def get_events( limit : int = Query(default=100, ge=0, le=1000), _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: - return await router.adapter.get_events(incident_id, offset, limit, resource_id, name, description, status, from_, to, time_, modified_since) + return await router.adapter.get_events(incident_id, offset=offset, limit=limit, resource_id=resource_id, name=name, description=description, status=status, + from_=from_, to=to, time_=time_, modified_since=modified_since) @router.get( diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index 6659d154..27654c21 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -1,5 +1,4 @@ from abc import abstractmethod -from typing import Any from . import models as task_models from ..account import models as account_models from ..status import models as status_models @@ -20,6 +19,7 @@ async def get_task( self : "FacilityAdapter", user: account_models.User, task_id: str, + **kwargs ) -> task_models.Task|None: pass @@ -28,6 +28,7 @@ async def get_task( async def get_tasks( self : "FacilityAdapter", user: account_models.User, + **kwargs ) -> list[task_models.Task]: pass @@ -37,7 +38,8 @@ async def put_task( self: "FacilityAdapter", user: account_models.User, resource: status_models.Resource|None, - command: task_models.TaskCommand + task: task_models.TaskCommand, + **kwargs ) -> str: pass @@ -46,78 +48,79 @@ async def put_task( async def on_task( resource: status_models.Resource, user: account_models.User, - cmd: task_models.TaskCommand, + task: task_models.TaskCommand, + **kwargs ) -> tuple[str, task_models.TaskStatus]: # Handle a task from the facility message queue. # Returns: (result, status) try: r = None - if cmd.router == "filesystem": - fs_adapter = IriRouter.create_adapter(cmd.router, filesystem_adapter.FacilityAdapter) - if cmd.command == "chmod": - request_model = filesystem_models.PutFileChmodRequest.model_validate(cmd.args["request_model"]) + if task.router == "filesystem": + fs_adapter = IriRouter.create_adapter(task.router, filesystem_adapter.FacilityAdapter) + if task.command == "chmod": + request_model = filesystem_models.PutFileChmodRequest.model_validate(task.args["request_model"]) o = await fs_adapter.chmod(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "chown": - request_model = filesystem_models.PutFileChownRequest.model_validate(cmd.args["request_model"]) + elif task.command == "chown": + request_model = filesystem_models.PutFileChownRequest.model_validate(task.args["request_model"]) o = await fs_adapter.chown(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "file": - o = await fs_adapter.file(resource, user, **cmd.args) + elif task.command == "file": + o = await fs_adapter.file(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "stat": - o = await fs_adapter.stat(resource, user, **cmd.args) + elif task.command == "stat": + o = await fs_adapter.stat(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "mkdir": - request_model = filesystem_models.PostMakeDirRequest.model_validate(cmd.args["request_model"]) + elif task.command == "mkdir": + request_model = filesystem_models.PostMakeDirRequest.model_validate(task.args["request_model"]) o = await fs_adapter.mkdir(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "symlink": - request_model = filesystem_models.PostFileSymlinkRequest.model_validate(cmd.args["request_model"]) + elif task.command == "symlink": + request_model = filesystem_models.PostFileSymlinkRequest.model_validate(task.args["request_model"]) o = await fs_adapter.symlink(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "ls": - o = await fs_adapter.ls(resource, user, **cmd.args) + elif task.command == "ls": + o = await fs_adapter.ls(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "head": - o = await fs_adapter.head(resource, user, **cmd.args) + elif task.command == "head": + o = await fs_adapter.head(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "view": - o = await fs_adapter.view(resource, user, **cmd.args) + elif task.command == "view": + o = await fs_adapter.view(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "tail": - o = await fs_adapter.tail(resource, user, **cmd.args) + elif task.command == "tail": + o = await fs_adapter.tail(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "checksum": - o = await fs_adapter.checksum(resource, user, **cmd.args) + elif task.command == "checksum": + o = await fs_adapter.checksum(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "rm": - o = await fs_adapter.rm(resource, user, **cmd.args) + elif task.command == "rm": + o = await fs_adapter.rm(resource, user, **task.args) r = o.model_dump_json() - elif cmd.command == "compress": - request_model = filesystem_models.PostCompressRequest.model_validate(cmd.args["request_model"]) + elif task.command == "compress": + request_model = filesystem_models.PostCompressRequest.model_validate(task.args["request_model"]) o = await fs_adapter.compress(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "extract": - request_model = filesystem_models.PostExtractRequest.model_validate(cmd.args["request_model"]) + elif task.command == "extract": + request_model = filesystem_models.PostExtractRequest.model_validate(task.args["request_model"]) o = await fs_adapter.extract(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "mv": - request_model = filesystem_models.PostMoveRequest.model_validate(cmd.args["request_model"]) + elif task.command == "mv": + request_model = filesystem_models.PostMoveRequest.model_validate(task.args["request_model"]) o = await fs_adapter.mv(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "cp": - request_model = filesystem_models.PostCopyRequest.model_validate(cmd.args["request_model"]) + elif task.command == "cp": + request_model = filesystem_models.PostCopyRequest.model_validate(task.args["request_model"]) o = await fs_adapter.cp(resource, user, request_model) r = o.model_dump_json() - elif cmd.command == "download": - r = await fs_adapter.download(resource, user, **cmd.args) - elif cmd.command == "upload": - o = await fs_adapter.upload(resource, user, **cmd.args) + elif task.command == "download": + r = await fs_adapter.download(resource, user, **task.args) + elif task.command == "upload": + o = await fs_adapter.upload(resource, user, **task.args) r = "File uploaded successfully" if r: return (r, task_models.TaskStatus.completed) else: - return (f"Task was cancelled due to unknown router/command: {cmd.router}:{cmd.command}", task_models.TaskStatus.failed) + return (f"Task was cancelled due to unknown router/command: {task.router}:{task.command}", task_models.TaskStatus.failed) except Exception as exc: return (f"Error: {exc}", task_models.TaskStatus.failed) diff --git a/app/routers/task/task.py b/app/routers/task/task.py index 094cdd0f..1a65459a 100644 --- a/app/routers/task/task.py +++ b/app/routers/task/task.py @@ -22,10 +22,10 @@ async def get_task( task_id : str, ) -> models.Task: """Get a task""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - task = await router.adapter.get_task(user, task_id) + task = await router.adapter.get_task(user=user, task_id=task_id) if not task: raise HTTPException(status_code=404, detail=f"Task {task_id} not found") return task @@ -42,7 +42,7 @@ async def get_tasks( request : Request, ) -> list[models.Task]: """Get all tasks""" - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - return await router.adapter.get_tasks(user) + return await router.adapter.get_tasks(user=user) \ No newline at end of file From ccfd2fe1218ceda731484c11050c13b24b0b5b8d Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 3 Feb 2026 20:14:19 -0600 Subject: [PATCH 059/133] Remove kwargs (versioning agreed) --- app/demo_adapter.py | 175 +++++---------------- app/routers/account/facility_adapter.py | 12 +- app/routers/common.py | 11 +- app/routers/compute/facility_adapter.py | 17 +- app/routers/compute/models.py | 52 +----- app/routers/facility/facility_adapter.py | 28 ++-- app/routers/filesystem/facility_adapter.py | 83 ++++------ app/routers/iri_router.py | 12 +- app/routers/status/facility_adapter.py | 18 +-- app/routers/task/facility_adapter.py | 16 +- 10 files changed, 120 insertions(+), 304 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 1a520e8e..1a996799 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -275,10 +275,8 @@ def _init_state(self): async def get_facility( self: "DemoAdapter", - modified_since: str | None = None, - **kwargs - ) -> facility_models.Facility: - self._warn_on_unused_kwargs("get_facility", kwargs) + modified_since: str | None = None + ) -> facility_models.Facility: return self.facility @@ -288,10 +286,8 @@ async def list_sites( name: str | None = None, offset: int | None = None, limit: int | None = None, - short_name: str | None = None, - **kwargs - ) -> list[facility_models.Site]: - self._warn_on_unused_kwargs("list_sites", kwargs) + short_name: str | None = None + ) -> list[facility_models.Site]: sites = self.sites if name: @@ -312,10 +308,8 @@ async def list_sites( async def get_site( self: "DemoAdapter", site_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Site: - self._warn_on_unused_kwargs("get_site", kwargs) + modified_since: str | None = None + ) -> facility_models.Site: site = next((s for s in self.sites if s.id == site_id), None) if not site: raise HTTPException(status_code=404, detail="Site not found") @@ -331,10 +325,8 @@ async def get_site( async def get_site_location( self: "DemoAdapter", site_id: str, - modified_since: str | None = None, - **kwargs + modified_since: str | None = None ) -> facility_models.Location: - self._warn_on_unused_kwargs("get_site_location", kwargs) site = await self.get_site(site_id) if not site.location_uri: @@ -360,10 +352,8 @@ async def list_locations( offset: int | None = None, limit: int | None = None, short_name: str | None = None, - country_name: str | None = None, - **kwargs - ) -> list[facility_models.Location]: - self._warn_on_unused_kwargs("list_locations", kwargs) + country_name: str | None = None + ) -> list[facility_models.Location]: locs = self.locations if name: @@ -387,10 +377,8 @@ async def list_locations( async def get_location( self: "DemoAdapter", location_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Location: - self._warn_on_unused_kwargs("get_location", kwargs) + modified_since: str | None = None + ) -> facility_models.Location: location = next((l for l in self.locations if l.id == location_id), None) if not location: @@ -416,10 +404,8 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, current_status : status_models.Status | None = None, - capability: Capability | None = None, - **kwargs + capability: Capability | None = None ) -> list[status_models.Resource]: - self._warn_on_unused_kwargs("get_resources", kwargs) resources = status_models.Resource.find(self.resources, name=name, description=description, group=group, modified_since=modified_since, resource_type=resource_type, current_status=current_status, capability=capability) return paginate_list(resources, offset, limit) @@ -427,10 +413,8 @@ async def get_resources( async def get_resource( self : "DemoAdapter", - id_ : str, - **kwargs + id_ : str ) -> status_models.Resource: - self._warn_on_unused_kwargs("get_resource", kwargs) return status_models.Resource.find_by_id(self.resources, id_) async def get_events( @@ -445,10 +429,8 @@ async def get_events( from_ : datetime.datetime | None = None, to : datetime.datetime | None = None, time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - **kwargs + modified_since : datetime.datetime | None = None ) -> list[status_models.Event]: - self._warn_on_unused_kwargs("get_events", kwargs) events = status_models.Event.find([e for e in self.events if e.incident_id == incident_id], resource_id=resource_id, name=name, description=description, status=status, from_=from_, to=to, time_=time_, modified_since=modified_since) return paginate_list(events, offset, limit) @@ -457,10 +439,8 @@ async def get_events( async def get_event( self : "DemoAdapter", incident_id : str, - id_ : str, - **kwargs + id_ : str ) -> status_models.Event: - self._warn_on_unused_kwargs("get_event", kwargs) return status_models.Event.find_by_id(self.events, id_) @@ -477,10 +457,8 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, - resolution: status_models.Resolution | None = None, - **kwargs + resolution: status_models.Resolution | None = None ) -> list[status_models.Incident]: - self._warn_on_unused_kwargs("get_incidents", kwargs) incidents = status_models.Incident.find(self.incidents, name=name, description=description, status=status, type_=type_,from_=from_, to=to, time_=time_, modified_since=modified_since, resource_id=resource_id, resolution=resolution) return paginate_list(incidents, offset, limit) @@ -488,10 +466,8 @@ async def get_incidents( async def get_incident( self : "DemoAdapter", - id_ : str, - **kwargs + id_ : str ) -> status_models.Incident: - self._warn_on_unused_kwargs("get_incident", kwargs) return status_models.Incident.find_by_id(self.incidents, id_) @@ -501,9 +477,7 @@ async def get_capabilities( modified_since : str | None = None, offset : int = 0, limit : int = 1000, - **kwargs ) -> list[Capability]: - self._warn_on_unused_kwargs("get_capabilities", kwargs) caps = list(self.capabilities.values()) if name: caps = [c for c in caps if name.lower() in c.name.lower()] @@ -515,13 +489,11 @@ async def get_current_user( self : "DemoAdapter", api_key: str, client_ip: str, - **kwargs ) -> str: """ In a real deployment, this would decode the api_key jwt and return the current user's id. This method is not async. """ - self._warn_on_unused_kwargs("get_current_user", kwargs) return "gtorok" @@ -530,9 +502,7 @@ async def get_user( user_id: str, api_key: str, client_ip: str|None, - **kwargs ) -> account_models.User: - self._warn_on_unused_kwargs("get_user", kwargs) if user_id != self.user.id: raise HTTPException(status_code=401, detail="User not found") if api_key != self.user.api_key: @@ -542,10 +512,8 @@ async def get_user( async def get_projects( self : "DemoAdapter", - user: account_models.User, - **kwargs + user: account_models.User ) -> list[account_models.Project]: - self._warn_on_unused_kwargs("get_projects", kwargs) return self.projects @@ -553,9 +521,7 @@ async def get_project_allocations( self : "DemoAdapter", project: account_models.Project, user: account_models.User, - **kwargs ) -> list[account_models.ProjectAllocation]: - self._warn_on_unused_kwargs("get_project_allocations", kwargs) return [pa for pa in self.project_allocations if pa.project_id == project.id] @@ -563,9 +529,7 @@ async def get_user_allocations( self : "DemoAdapter", user: account_models.User, project_allocation: account_models.ProjectAllocation, - **kwargs ) -> list[account_models.UserAllocation]: - self._warn_on_unused_kwargs("get_user_allocations", kwargs) return [ua for ua in self.user_allocations if ua.project_allocation_id == project_allocation.id] @@ -574,9 +538,7 @@ async def submit_job( resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, - **kwargs, ) -> compute_models.Job: - self._warn_on_unused_kwargs("submit_job", kwargs) return compute_models.Job( id="job_123", status=compute_models.JobStatus( @@ -595,9 +557,7 @@ async def submit_job_script( user: account_models.User, job_script_path: str, args: list[str] = [], - **kwargs ) -> compute_models.Job: - self._warn_on_unused_kwargs("submit_job_script", kwargs) return compute_models.Job( id="job_123", status=compute_models.JobStatus( @@ -616,9 +576,7 @@ async def update_job( user: account_models.User, job_spec: compute_models.JobSpec, job_id: str, - **kwargs, ) -> compute_models.Job: - self._warn_on_unused_kwargs("update_job", kwargs) return compute_models.Job( id=job_id, status=compute_models.JobStatus( @@ -638,9 +596,7 @@ async def get_job( job_id: str, historical: bool = False, include_spec: bool = False, - **kwargs, ) -> compute_models.Job: - self._warn_on_unused_kwargs("get_job", kwargs) return compute_models.Job( id=job_id, status=compute_models.JobStatus( @@ -662,9 +618,7 @@ async def get_jobs( filters: dict[str, object] | None = None, historical: bool = False, include_spec: bool = False, - **kwargs, ) -> list[compute_models.Job]: - self._warn_on_unused_kwargs("get_jobs", kwargs) return [compute_models.Job( id=f"job_{i}", status=compute_models.JobStatus( @@ -682,9 +636,7 @@ async def cancel_job( resource: status_models.Resource, user: account_models.User, job_id: str, - **kwargs, ) -> bool: - self._warn_on_unused_kwargs("cancel_job", kwargs) # call slurm/etc. to cancel job return True @@ -754,10 +706,8 @@ async def chmod( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest, - **kwargs, + request_model: filesystem_models.PutFileChmodRequest ) -> filesystem_models.PutFileChmodResponse: - self._warn_on_unused_kwargs("chmod", kwargs) rp = self.validate_path(request_model.path) os.chmod(rp, int(request_model.mode, 8)) return filesystem_models.PutFileChmodResponse( @@ -770,9 +720,7 @@ async def chown( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChownRequest, - **kwargs, ) -> filesystem_models.PutFileChownResponse: - self._warn_on_unused_kwargs("chown", kwargs) rp = self.validate_path(request_model.path) os.chown(rp, request_model.owner, request_model.group) return filesystem_models.PutFileChmodResponse( @@ -789,9 +737,7 @@ async def ls( numeric_uid: bool, recursive: bool, dereference: bool, - **kwargs, ) -> filesystem_models.GetDirectoryLsResponse: - self._warn_on_unused_kwargs("ls", kwargs) rp = self.validate_path(path) files = glob.glob(rp, recursive=recursive) return filesystem_models.GetDirectoryLsResponse( @@ -832,9 +778,7 @@ async def head( file_bytes: int | None, lines: int | None, skip_trailing: bool, - **kwargs, ) -> Tuple[Any, int]: - self._warn_on_unused_kwargs("head", kwargs) return self._headtail("head", path, file_bytes, lines) @@ -845,10 +789,8 @@ async def tail( path: str, file_bytes: int | None, lines: int | None, - skip_trailing: bool, - **kwargs + skip_trailing: bool ) -> Tuple[Any, int]: - self._warn_on_unused_kwargs("tail", kwargs) return self._headtail("tail", path, file_bytes, lines) @@ -858,10 +800,8 @@ async def view( user: account_models.User, path: str, size: int, - offset: int, - **kwargs - ) -> filesystem_models.GetViewFileResponse: - self._warn_on_unused_kwargs("view", kwargs) + offset: int + ) -> filesystem_models.GetViewFileResponse: rp = self.validate_path(path) result = subprocess.run( f"tail -c +{offset+1} {rp} | head -c {size}", @@ -879,10 +819,8 @@ async def checksum( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ) -> filesystem_models.GetFileChecksumResponse: - self._warn_on_unused_kwargs("checksum", kwargs) + path: str + ) -> filesystem_models.GetFileChecksumResponse: rp = self.validate_path(path) result = subprocess.run( ["sha256sum", rp], @@ -901,10 +839,8 @@ async def file( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs + path: str ) -> filesystem_models.GetFileTypeResponse: - self._warn_on_unused_kwargs("file", kwargs) rp = self.validate_path(path) result = subprocess.run( ["file", "-b", rp], @@ -921,10 +857,8 @@ async def stat( resource: status_models.Resource, user: account_models.User, path: str, - dereference: bool, - **kwargs + dereference: bool ) -> filesystem_models.GetFileStatResponse: - self._warn_on_unused_kwargs("stat", kwargs) rp = self.validate_path(path) if dereference: stat_info = os.stat(rp) @@ -951,9 +885,7 @@ async def rm( resource: status_models.Resource, user: account_models.User, path: str, - **kwargs ): - self._warn_on_unused_kwargs("rm", kwargs) rp = self.validate_path(path) if rp == PathSandbox.get_base_temp_dir(): raise HTTPException(status_code=400, detail="Cannot delete sandbox") @@ -965,10 +897,8 @@ async def mkdir( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostMakeDirRequest, - **kwargs - ) -> filesystem_models.PostMkdirResponse: - self._warn_on_unused_kwargs("mkdir", kwargs) + request_model: filesystem_models.PostMakeDirRequest + ) -> filesystem_models.PostMkdirResponse: rp = self.validate_path(request_model.path) args = ["mkdir"] if request_model.parent: @@ -984,10 +914,8 @@ async def symlink( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostFileSymlinkRequest, - **kwargs + request_model: filesystem_models.PostFileSymlinkRequest ) -> filesystem_models.PostFileSymlinkResponse: - self._warn_on_unused_kwargs("symlink", kwargs) rp_src = self.validate_path(request_model.path) rp_dst = self.validate_path(request_model.link_path) subprocess.run(["ln", "-s", rp_src, rp_dst], check=True) @@ -1000,10 +928,8 @@ async def download( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs + path: str ) -> Any: - self._warn_on_unused_kwargs("download", kwargs) rp = self.validate_path(path) raw_content = pathlib.Path(rp).read_bytes() @@ -1018,10 +944,8 @@ async def upload( resource: status_models.Resource, user: account_models.User, path: str, - content: str, - **kwargs + content: str ) -> None: - self._warn_on_unused_kwargs("upload", kwargs) rp = self.validate_path(path) if isinstance(content, bytes): pathlib.Path(rp).write_bytes(content) @@ -1035,10 +959,8 @@ async def compress( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostCompressRequest, - **kwargs - ) -> filesystem_models.PostCompressResponse: - self._warn_on_unused_kwargs("compress", kwargs) + request_model: filesystem_models.PostCompressRequest + ) -> filesystem_models.PostCompressResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -1070,10 +992,8 @@ async def extract( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostExtractRequest, - **kwargs - ) -> filesystem_models.PostExtractResponse: - self._warn_on_unused_kwargs("extract", kwargs) + request_model: filesystem_models.PostExtractRequest + ) -> filesystem_models.PostExtractResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -1100,10 +1020,8 @@ async def mv( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostMoveRequest, - **kwargs - ) -> filesystem_models.PostMoveResponse: - self._warn_on_unused_kwargs("mv", kwargs) + request_model: filesystem_models.PostMoveRequest + ) -> filesystem_models.PostMoveResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) subprocess.run(["mv", src_rp, dst_rp], check=True) @@ -1116,10 +1034,8 @@ async def cp( self : "DemoAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostCopyRequest, - **kwargs - ) -> filesystem_models.PostCopyResponse: - self._warn_on_unused_kwargs("cp", kwargs) + request_model: filesystem_models.PostCopyRequest + ) -> filesystem_models.PostCopyResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) args = ["cp"] @@ -1136,20 +1052,16 @@ async def cp( async def get_task( self : "DemoAdapter", user: account_models.User, - task_id: str, - **kwargs + task_id: str ) -> task_models.Task|None: - self._warn_on_unused_kwargs("get_task", kwargs) await DemoTaskQueue._process_tasks(self) return next((t for t in DemoTaskQueue.tasks if t.user.name == user.name and t.id == task_id), None) async def get_tasks( self : "DemoAdapter", - user: account_models.User, - **kwargs + user: account_models.User ) -> list[task_models.Task]: - self._warn_on_unused_kwargs("get_tasks", kwargs) await DemoTaskQueue._process_tasks(self) return [t for t in DemoTaskQueue.tasks if t.user.name == user.name] @@ -1158,10 +1070,7 @@ async def put_task( self: "DemoAdapter", user: account_models.User, resource: status_models.Resource, - task: str, - **kwargs, - ) -> str: - self._warn_on_unused_kwargs("put_task", kwargs) + task: str) -> str: await DemoTaskQueue._process_tasks(self) return DemoTaskQueue._create_task(user, resource, task) diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 3abca875..334e6827 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -17,8 +17,7 @@ async def get_capabilities( name : str | None = None, modified_since : str | None = None, offset : int = 0, - limit : int = 1000, - **kwargs + limit : int = 1000 ) -> list[Capability]: pass @@ -26,8 +25,7 @@ async def get_capabilities( @abstractmethod async def get_projects( self : "FacilityAdapter", - user: account_models.User, - **kwargs + user: account_models.User ) -> list[account_models.Project]: pass @@ -36,8 +34,7 @@ async def get_projects( async def get_project_allocations( self : "FacilityAdapter", project: account_models.Project, - user: account_models.User, - **kwargs + user: account_models.User ) -> list[account_models.ProjectAllocation]: pass @@ -46,7 +43,6 @@ async def get_project_allocations( async def get_user_allocations( self : "FacilityAdapter", user: account_models.User, - project_allocation: account_models.ProjectAllocation, - **kwargs + project_allocation: account_models.ProjectAllocation ) -> list[account_models.UserAllocation]: pass diff --git a/app/routers/common.py b/app/routers/common.py index 2826dab7..62e2fe75 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -4,7 +4,7 @@ import enum from typing import Optional from urllib.parse import parse_qs - +from collections.abc import Iterable from pydantic_core import core_schema from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer, field_validator from fastapi import Request, HTTPException, status @@ -245,8 +245,14 @@ def find_by_id(cls, items, id_, allow_name: bool = False): @classmethod def find(cls, items, name=None, description=None, modified_since=None): """ Find objects matching the given criteria. """ + single = False if not any((name, description, modified_since)): return items + + if not isinstance(items, Iterable) or isinstance(items, BaseModel): + items = [items] + single = True + if name: items = [item for item in items if item.name == name] if description: @@ -254,6 +260,9 @@ def find(cls, items, name=None, description=None, modified_since=None): if modified_since: modified_since = cls.normalize_dt(modified_since) items = [item for item in items if item.last_modified >= modified_since] + + if single: + return items[0] if items else None return items diff --git a/app/routers/compute/facility_adapter.py b/app/routers/compute/facility_adapter.py index d70ec517..910cacd0 100644 --- a/app/routers/compute/facility_adapter.py +++ b/app/routers/compute/facility_adapter.py @@ -18,9 +18,8 @@ async def submit_job( self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - job_spec: compute_models.JobSpec, - **kwargs - ) -> compute_models.Job: + job_spec: compute_models.JobSpec + ) -> compute_models.Job: pass @@ -30,8 +29,7 @@ async def submit_job_script( resource: status_models.Resource, user: account_models.User, job_script_path: str, - args: list[str] = [], - **kwargs + args: list[str] = [] ) -> compute_models.Job: pass @@ -42,8 +40,7 @@ async def update_job( resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, - job_id: str, - **kwargs + job_id: str ) -> compute_models.Job: pass @@ -69,8 +66,7 @@ async def get_jobs( limit : int, filters: dict[str, object] | None = None, historical: bool = False, - include_spec: bool = False, - **kwargs + include_spec: bool = False ) -> list[compute_models.Job]: pass @@ -80,7 +76,6 @@ async def cancel_job( self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - job_id: str, - **kwargs + job_id: str ) -> bool: pass diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 3aa4121b..3ec1053e 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,11 +1,10 @@ from typing import Annotated from enum import IntEnum -from pydantic import field_serializer, StrictBool, Field +from pydantic import field_serializer, ConfigDict, StrictBool, Field from ..common import IRIBaseModel class ResourceSpec(IRIBaseModel): -<<<<<<< HEAD """ Specification of computational resources required for a job. """ @@ -28,37 +27,13 @@ class JobAttributes(IRIBaseModel): reservation_id: Annotated[str | None, Field(min_length=1, description="ID of a reservation to use for the job")] = None custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs")] = {} -======= - node_count: int | None = Field(default=None, description="Number of compute nodes to allocate") - process_count: int | None = Field(default=None, description="Total number of processes to launch") - processes_per_node: int | None = Field(default=None, description="Number of processes to launch per node") - cpu_cores_per_process: int | None = Field(default=None, description="Number of CPU cores to allocate per process") - gpu_cores_per_process: int | None = Field(default=None, description="Number of GPU cores to allocate per process") - exclusive_node_use: StrictBool = Field(default=True, description="Whether to request exclusive use of allocated nodes") - memory: int | None = Field(default=None, description="Amount of memory to allocate in bytes") - - -class JobAttributes(IRIBaseModel): - duration: int = Field(default=None, ge=1, description="Duration in seconds", examples=[30, 60, 120]) - queue_name: str | None = Field(default=None, min_length=1, description="Name of the queue or partition to submit the job to") - account: str | None = Field(default=None, min_length=1, description="Account or project to charge for resource usage") - reservation_id: str | None = Field(default=None, min_length=1, description="ID of a reservation to use for the job") - custom_attributes: dict[str, str] = Field(default={}, description="Custom scheduler-specific attributes as key-value pairs") ->>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) - class VolumeMount(IRIBaseModel): """ Represents a volume mount for a container. """ -<<<<<<< HEAD source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount")] target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted")] read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only")] = True -======= - source: str = Field(description="The source path on the host system to mount") - target: str = Field(description="The target path inside the container where the volume will be mounted") - read_only: StrictBool = Field(default=True, description="Whether the mount should be read-only") ->>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class Container(IRIBaseModel): """ @@ -69,11 +44,9 @@ class Container(IRIBaseModel): to determine if the container should be run with MPI support. The container should by default. be run with host networking. """ -<<<<<<< HEAD image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] - class JobSpec(IRIBaseModel): """ Specification for job. @@ -94,28 +67,6 @@ class JobSpec(IRIBaseModel): pre_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run before launching the job")] = None post_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run after the job completes")] = None launcher: Annotated[str | None, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None -======= - image: str = Field(description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')") - volume_mounts: list[VolumeMount] = Field(default=[], description="List of volume mounts for the container") - - -class JobSpec(IRIBaseModel): - executable: str | None = Field(default=None, description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.") - container: Container | None = Field(default=None, description="Container specification for containerized execution") - arguments: list[str] = Field(default=[], description="Command-line arguments to pass to the executable or container") - directory: str | None = Field(default=None, description="Working directory for the job") - name: str | None = Field(default=None, description="Name of the job") - inherit_environment: StrictBool = Field(default=True, description="Whether to inherit the environment variables from the submission environment") - environment: dict[str, str] = Field(default={}, description="Environment variables to set for the job. If container is specified, these will be set inside the container.") - stdin_path: str | None = Field(default=None, description="Path to file to use as standard input") - stdout_path: str | None = Field(default=None, description="Path to file to write standard output") - stderr_path: str | None = Field(default=None, description="Path to file to write standard error") - resources: ResourceSpec | None = Field(default=None, description="Resource requirements for the job") - attributes: JobAttributes | None = Field(default=None, description="Additional job attributes such as duration, queue, and account") - pre_launch: str | None = Field(default=None, description="Script or commands to run before launching the job") - post_launch: str | None = Field(default=None, description="Script or commands to run after the job completes") - launcher: str | None = Field(default=None, description="Job launcher to use (e.g., 'mpirun', 'srun')") ->>>>>>> b7b104f (Make adapter forward compatible. add filtering, pagination, datetime normalization) class CommandResult(IRIBaseModel): @@ -157,7 +108,6 @@ class JobState(IntEnum): CANCELED = 5 """Represents a job that was canceled by a call to :func:`~psij.Job.cancel()`.""" - class JobStatus(IRIBaseModel): state : JobState time : float | None = None diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index fc6777fb..0bb9ca33 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -13,9 +13,8 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def get_facility( self: "FacilityAdapter", - modified_since: str | None = None, - **kwargs - ) -> facility_models.Facility | None: + modified_since: str | None = None + ) -> facility_models.Facility | None: pass @abstractmethod @@ -25,27 +24,24 @@ async def list_sites( name: str | None = None, offset: int | None = None, limit: int | None = None, - short_name: str | None = None, - **kwargs - ) -> list[facility_models.Site]: + short_name: str | None = None + ) -> list[facility_models.Site]: pass @abstractmethod async def get_site( self: "FacilityAdapter", site_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Site | None: + modified_since: str | None = None + ) -> facility_models.Site | None: pass @abstractmethod async def get_site_location( self: "FacilityAdapter", site_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Location | None: + modified_since: str | None = None + ) -> facility_models.Location | None: pass @abstractmethod @@ -56,8 +52,7 @@ async def list_locations( offset: int | None = None, limit: int | None = None, short_name: str | None = None, - country_name: str | None = None, - **kwargs + country_name: str | None = None ) -> list[facility_models.Location]: pass @@ -65,7 +60,6 @@ async def list_locations( async def get_location( self: "FacilityAdapter", location_id: str, - modified_since: str | None = None, - **kwargs - ) -> facility_models.Location | None: + modified_since: str | None = None + ) -> facility_models.Location | None: pass diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index 8506f1c7..636b0a9b 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -29,9 +29,8 @@ async def chmod( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest, - **kwargs - ) -> filesystem_models.PutFileChmodResponse: + request_model: filesystem_models.PutFileChmodRequest + ) -> filesystem_models.PutFileChmodResponse: pass @@ -40,9 +39,8 @@ async def chown( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PutFileChownRequest, - **kwargs - ) -> filesystem_models.PutFileChownResponse: + request_model: filesystem_models.PutFileChownRequest + ) -> filesystem_models.PutFileChownResponse: pass @@ -55,8 +53,7 @@ async def ls( show_hidden: bool, numeric_uid: bool, recursive: bool, - dereference: bool, - **kwargs + dereference: bool ) -> filesystem_models.GetDirectoryLsResponse: pass @@ -69,8 +66,7 @@ async def head( path: str, file_bytes: int, lines: int, - skip_trailing: bool, - **kwargs + skip_trailing: bool ) -> Tuple[Any, int]: pass @@ -83,9 +79,8 @@ async def tail( path: str, file_bytes: int | None, lines: int | None, - skip_trailing: bool, - **kwargs - ) -> Tuple[Any, int]: + skip_trailing: bool + ) -> Tuple[Any, int]: pass @@ -96,9 +91,8 @@ async def view( user: account_models.User, path: str, size: int, - offset: int, - **kwargs - ) -> filesystem_models.GetViewFileResponse: + offset: int + ) -> filesystem_models.GetViewFileResponse: pass @@ -107,9 +101,8 @@ async def checksum( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ) -> filesystem_models.GetFileChecksumResponse: + path: str + ) -> filesystem_models.GetFileChecksumResponse: pass @@ -118,9 +111,8 @@ async def file( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ) -> filesystem_models.GetFileTypeResponse: + path: str + ) -> filesystem_models.GetFileTypeResponse: pass @@ -130,9 +122,8 @@ async def stat( resource: status_models.Resource, user: account_models.User, path: str, - dereference: bool, - **kwargs - ) -> filesystem_models.GetFileStatResponse: + dereference: bool + ) -> filesystem_models.GetFileStatResponse: pass @@ -141,9 +132,7 @@ async def rm( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ): + path: str): pass @@ -152,9 +141,8 @@ async def mkdir( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostMakeDirRequest, - **kwargs - ) -> filesystem_models.PostMkdirResponse: + request_model: filesystem_models.PostMakeDirRequest + ) -> filesystem_models.PostMkdirResponse: pass @@ -164,8 +152,7 @@ async def symlink( resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostFileSymlinkRequest, - **kwargs - ) -> filesystem_models.PostFileSymlinkResponse: + ) -> filesystem_models.PostFileSymlinkResponse: pass @@ -174,9 +161,8 @@ async def download( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - path: str, - **kwargs - ) -> Any: + path: str + ) -> Any: pass @@ -186,9 +172,8 @@ async def upload( resource: status_models.Resource, user: account_models.User, path: str, - content: str, - **kwargs - ) -> None: + content: str + ) -> None: pass @@ -197,9 +182,8 @@ async def compress( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostCompressRequest, - **kwargs - ) -> filesystem_models.PostCompressResponse: + request_model: filesystem_models.PostCompressRequest + ) -> filesystem_models.PostCompressResponse: pass @@ -208,9 +192,8 @@ async def extract( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostExtractRequest, - **kwargs - ) -> filesystem_models.PostExtractResponse: + request_model: filesystem_models.PostExtractRequest + ) -> filesystem_models.PostExtractResponse: pass @@ -219,9 +202,8 @@ async def mv( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostMoveRequest, - **kwargs - ) -> filesystem_models.PostMoveResponse: + request_model: filesystem_models.PostMoveRequest + ) -> filesystem_models.PostMoveResponse: pass @@ -230,7 +212,6 @@ async def cp( self : "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - request_model: filesystem_models.PostCopyRequest, - **kwargs - ) -> filesystem_models.PostCopyResponse: + request_model: filesystem_models.PostCopyRequest + ) -> filesystem_models.PostCopyResponse: pass diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index d4cb6f2e..be317624 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -98,18 +98,11 @@ async def current_user( class AuthenticatedAdapter(ABC): - def _warn_on_unused_kwargs(self, func_name: str, kwargs: dict) -> None: - if not kwargs: - return - logging.getLogger().warning("Adapter method '%s' received unused kwargs: %s", func_name, - ", ".join(sorted(kwargs.keys()))) - @abstractmethod async def get_current_user( self : "AuthenticatedAdapter", api_key: str, - client_ip: str|None, - **kwargs + client_ip: str|None ) -> str: """ Decode the api_key and return the authenticated user's id. @@ -124,8 +117,7 @@ async def get_user( self : "AuthenticatedAdapter", user_id: str, api_key: str, - client_ip: str|None, - **kwargs + client_ip: str|None ) -> User: """ Retrieve additional user information (name, email, etc.) for the given user_id. diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 1d774cb0..274e9f5c 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -24,8 +24,7 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type: status_models.ResourceType = Query(default=None), current_status: status_models.Status = Query(default=None), - capability: Capability | None = None, - **kwargs + capability: Capability | None = None ) -> list[status_models.Resource]: pass @@ -33,8 +32,7 @@ async def get_resources( @abstractmethod async def get_resource( self : "FacilityAdapter", - id_ : str, - **kwargs + id_ : str ) -> status_models.Resource: pass @@ -52,8 +50,7 @@ async def get_events( from_ : datetime.datetime | None = None, to : datetime.datetime | None = None, time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - **kwargs + modified_since : datetime.datetime | None = None ) -> list[status_models.Event]: pass @@ -62,8 +59,7 @@ async def get_events( async def get_event( self : "FacilityAdapter", incident_id : str, - id_ : str, - **kwargs + id_ : str ) -> status_models.Event: pass @@ -82,8 +78,7 @@ async def get_incidents( time_ : datetime.datetime | None = None, modified_since : datetime.datetime | None = None, resource_id : str | None = None, - resolution: status_models.Resolution | None = None, - **kwargs + resolution: status_models.Resolution | None = None ) -> list[status_models.Incident]: pass @@ -91,7 +86,6 @@ async def get_incidents( @abstractmethod async def get_incident( self : "FacilityAdapter", - id_ : str, - **kwargs + id_ : str ) -> status_models.Incident: pass diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index 27654c21..47f686eb 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -18,8 +18,7 @@ class FacilityAdapter(AuthenticatedAdapter): async def get_task( self : "FacilityAdapter", user: account_models.User, - task_id: str, - **kwargs + task_id: str ) -> task_models.Task|None: pass @@ -27,8 +26,7 @@ async def get_task( @abstractmethod async def get_tasks( self : "FacilityAdapter", - user: account_models.User, - **kwargs + user: account_models.User ) -> list[task_models.Task]: pass @@ -38,9 +36,8 @@ async def put_task( self: "FacilityAdapter", user: account_models.User, resource: status_models.Resource|None, - task: task_models.TaskCommand, - **kwargs - ) -> str: + task: task_models.TaskCommand + ) -> str: pass @@ -48,9 +45,8 @@ async def put_task( async def on_task( resource: status_models.Resource, user: account_models.User, - task: task_models.TaskCommand, - **kwargs - ) -> tuple[str, task_models.TaskStatus]: + task: task_models.TaskCommand + ) -> tuple[str, task_models.TaskStatus]: # Handle a task from the facility message queue. # Returns: (result, status) try: From 1206be92768c653a05065702a4803628a4086130 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 4 Feb 2026 05:39:49 -0600 Subject: [PATCH 060/133] get_resource single id --- app/routers/compute/compute.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 804f9a5f..5f44d522 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -41,7 +41,7 @@ async def submit_job( raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs @@ -78,7 +78,7 @@ async def submit_job( # raise HTTPException(status_code=404, detail="User not found") # # # look up the resource (todo: maybe ensure it's available) -# resource = await status_router.adapter.get_resource(resource_id=resource_id) +# resource = await status_router.adapter.get_resource(resource_id) # # # the handler can use whatever means it wants to submit the job and then fill in its id # # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs @@ -113,7 +113,7 @@ async def update_job( raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs @@ -143,7 +143,7 @@ async def get_job_status( # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) job = await router.adapter.get_job(resource=resource, user=user, job_id=job_id, historical=historical, include_spec=include_spec) @@ -175,7 +175,7 @@ async def get_job_statuses( # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) jobs = await router.adapter.get_jobs(resource=resource, user=user, offset=offset, limit=limit, filters=filters, historical=historical, include_spec=include_spec) @@ -203,7 +203,7 @@ async def cancel_job( raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) await router.adapter.cancel_job(resource=resource, user=user, job_id=job_id) From b141dbdd807aaf50e9c571e791e5f686c90bfcc8 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 4 Feb 2026 19:28:14 -0600 Subject: [PATCH 061/133] Add paginate in demo_adapter. Fix number of params for override method --- app/demo_adapter.py | 12 ++++++++++++ app/routers/common.py | 1 - 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index a36c27e3..81b84603 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -22,6 +22,14 @@ DEMO_QUEUE_UPDATE_SECS = 5 +def paginate_list(items, offset: int | None, limit: int | None): + """Return a sliced items using offset and limit.""" + if offset is not None and offset > 0: + items = items[offset:] + if limit is not None and limit >= 0: + items = items[:limit] + return items + class PathSandbox: _base_temp_dir = None @@ -473,6 +481,10 @@ async def get_incident( async def get_capabilities( self : "DemoAdapter", + name : str | None = None, + modified_since : str | None = None, + offset : int = 0, + limit : int = 1000 ) -> list[Capability]: return self.capabilities.values() diff --git a/app/routers/common.py b/app/routers/common.py index ce91f4a5..75879b95 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -11,7 +11,6 @@ from .. import config - # These are Pydantic custom types for strict validation # that are not implmented in Pydantic by default. # ----------------------------------------------------------------------- From ef5c6e24c1ad2ebe926799c6bc38eaecbda0b6de Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 5 Feb 2026 06:51:48 -0600 Subject: [PATCH 062/133] Harden validation and response allingment with spec This includes: - application/problem+json for error responses - Generate URL-safe instance values - Normalize non-string validation error into strings (fastapi validation can give list/dict) - Exclude None/empty fields from locations/capabilities reponses - Capability (according to spec) should use NamedObject, but overrides last_modified as optional. --- app/routers/account/account.py | 3 +-- app/routers/common.py | 15 +++++++----- app/routers/error_handlers.py | 41 +++++++++++++++++++++++++++----- app/routers/facility/facility.py | 2 +- 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index e13cbde2..6aa6e233 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -18,8 +18,7 @@ description="Get a list of capabilities at this facility.", responses=DEFAULT_RESPONSES, operation_id="getCapabilities", - -) + response_model_exclude_none=True) async def get_capabilities( request : Request, name : str = Query(default=None, min_length=1), diff --git a/app/routers/common.py b/app/routers/common.py index 75879b95..a6a91ec2 100644 --- a/app/routers/common.py +++ b/app/routers/common.py @@ -229,7 +229,7 @@ def find_by_id(cls, items, id_, allow_name: bool = False): if not matches: return None if len(matches) > 1: - raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id}'") + raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id_}'") return matches[0] @@ -250,8 +250,8 @@ def find(cls, items, name=None, description=None, modified_since=None): items = [item for item in items if item.description and description in item.description] if modified_since: modified_since = cls.normalize_dt(modified_since) - items = [item for item in items if item.last_modified >= modified_since] - + items = [item for item in items + if item.last_modified and item.last_modified >= modified_since] if single: return items[0] if items else None return items @@ -263,7 +263,7 @@ class AllocationUnit(enum.Enum): inodes = "inodes" -class Capability(IRIBaseModel): +class Capability(NamedObject): """ An aspect of a resource that can have an allocation. For example, Perlmutter nodes with GPUs @@ -271,6 +271,9 @@ class Capability(IRIBaseModel): It is a way to further subdivide a resource into allocatable sub-resources. The word "capability" is also known to users as something they need for a job to run. (eg. gpu) """ - id: str - name: str + def _self_path(self) -> str: + return f"/account/capabilities/{self.id}" + + last_modified: StrictDateTime | None = Field(default=None, description="ISO 8601 timestamp when this object was last modified.") + units: list[AllocationUnit] diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 337b5fc2..bf781be8 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -3,7 +3,7 @@ Default problem schema and example responses for various HTTP status codes. """ import logging -from urllib.parse import unquote +from urllib.parse import urlsplit, urlunsplit, quote from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse from fastapi.exceptions import RequestValidationError @@ -16,12 +16,36 @@ def get_url_base(request: Request) -> str: proto = request.headers.get("x-forwarded-proto") or request.url.scheme return f"{proto}://{host}/problems" +def safe_instance_url(request: Request) -> str: + """Return a URL-safe version of the request URL for the 'instance' field.""" + parts = urlsplit(str(request.url)) + + # Encode unsafe characters in each component + safe_path = quote(parts.path, safe="/:@&+$,;=-._~") + safe_query = quote(parts.query, safe="=&?/:@+$,;=-._~") + safe_fragment = quote(parts.fragment, safe="=&?/:@+$,;=-._~") + + return urlunsplit((parts.scheme, parts.netloc, safe_path, safe_query, safe_fragment)) + def problem_response(*, request: Request, status: int, - title: str, detail: str, problem_type: str, + title, detail, problem_type: str, invalid_params=None, extra_headers=None): """Return a JSON problem response with the given status, title, and detail.""" - instance = unquote(str(request.url)) + instance = safe_instance_url(request) url_base = get_url_base(request) + + # Normalize title and detail to strings (Official spec says they must be strings) + # but fastapi validation errors may provide lists/dicts + if not isinstance(title, str): + title = "Error" + + if not isinstance(detail, str): + if isinstance(detail, list): + detail = ", ".join(err.get("msg", str(err)) if isinstance(err, dict) else str(err) + for err in detail) + else: + detail = str(detail) + body = { "type": f"{url_base}/{problem_type}", "title": title, @@ -34,7 +58,13 @@ def problem_response(*, request: Request, status: int, body["invalid_params"] = invalid_params headers = extra_headers or {} - return JSONResponse(status_code=status, content=body, headers=headers) + return JSONResponse( + status_code=status, + content=body, + headers=headers, + media_type="application/problem+json" + ) + def install_error_handlers(app: FastAPI): @@ -45,8 +75,7 @@ async def validation_error_handler(request: Request, exc: RequestValidationError invalid_params = [] for err in exc.errors(): - loc = err.get("loc", []) - name = loc[-1] if loc else "unknown" + name = str((err.get("loc") or ["unknown"])[-1]) reason = err.get("msg", "Invalid parameter") invalid_params.append({"name": name, "reason": reason}) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 7d264af3..d804b9c0 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -51,7 +51,7 @@ async def get_site_location( """Get site location by site ID""" return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) -@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") +@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations", response_model_exclude_none=True) async def list_locations( request : Request, modified_since: StrictDateTime = Query(default=None), From 2040b483ffb839416996fb0c4cb4476ab67b6ae8 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 5 Feb 2026 12:54:35 -0600 Subject: [PATCH 063/133] Update Facility endpoint (based on new Format discussed during the meeting) --- .github/workflows/api-validation.yml | 4 +- app/demo_adapter.py | 124 +++-------------------- app/routers/facility/facility.py | 36 +------ app/routers/facility/facility_adapter.py | 30 +----- app/routers/facility/models.py | 21 +--- app/routers/status/models.py | 11 +- 6 files changed, 26 insertions(+), 200 deletions(-) diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml index e4d688b4..51ed3312 100644 --- a/.github/workflows/api-validation.yml +++ b/.github/workflows/api-validation.yml @@ -80,8 +80,6 @@ jobs: --report-name schemathesis-local echo "exitcode=$?" >> $GITHUB_OUTPUT - # TODO: Change back to https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json - # Once https://github.com/doe-iri/iri-facility-api-docs/pull/12 merged. - name: Run Schemathesis validation (official spec) id: schemathesis_official env: @@ -91,7 +89,7 @@ jobs: source .venv/bin/activate python schema-validator/verification/api-validator.py \ --baseurl http://localhost:8000 \ - --schema-url https://raw.githubusercontent.com/juztas/iri-facility-api-docs/refs/heads/newspec/specification/openapi/openapi_iri_facility_api_v1.json \ + --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ --report-name schemathesis-official echo "exitcode=$?" >> $GITHUB_OUTPUT diff --git a/app/demo_adapter.py b/app/demo_adapter.py index e68635e6..50a92674 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -76,30 +76,6 @@ def __init__(self): def _init_state(self): now = utc_now() - loc1 = facility_models.Location( - id=demo_uuid("location", "demo_location_1"), - name="Demo Location 1", - description="The first demo location", - last_modified=now, - short_name="DL1", - country_name="USA", - locality_name="Demo City", - state_or_province_name="DC", - latitude=36.173357, - longitude=-234.51452) - - loc2 = facility_models.Location( - id=demo_uuid("location", "demo_location_2"), - name="Demo Location 2", - description="The second demo location", - last_modified=now, - short_name="DL2", - country_name="USA", - locality_name="Example Town", - state_or_province_name="ET", - latitude=38.410558, - longitude=-286.36999) - site1 = facility_models.Site( id=demo_uuid("site", "demo_site_1"), name="Demo Site 1", @@ -107,8 +83,13 @@ def _init_state(self): last_modified=now, short_name="DS1", operating_organization="Demo Org", - location_uri=loc1.self_uri, + country_name="USA", + locality_name="Demo City", + state_or_province_name="DC", + latitude=36.173357, + longitude=-234.51452, resource_uris=[]) + site2 = facility_models.Site( id=demo_uuid("site", "demo_site_2"), name="Demo Site 2", @@ -116,7 +97,11 @@ def _init_state(self): last_modified=now, short_name="DS2", operating_organization="Demo Org", - location_uri=loc2.self_uri, + country_name="USA", + locality_name="Example Town", + state_or_province_name="ET", + latitude=38.410558, + longitude=-286.36999, resource_uris=[]) self.facility = facility_models.Facility( @@ -127,20 +112,9 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - site_uris=[site1.self_uri, site2.self_uri], - location_uris=[loc1.self_uri, loc2.self_uri], - resource_uris=[], - event_uris=[], - incident_uris=[], - capability_uris=[], - project_uris=[], - project_allocation_uris=[], - user_allocation_uris=[], + site_uris=[site1.self_uri, site2.self_uri] ) - loc1.site_uris.append(site1.self_uri) - loc2.site_uris.append(site2.self_uri) - self.locations = [loc1, loc2] self.sites = [site1, site2] @@ -324,80 +298,6 @@ async def get_site( return site - async def get_site_location( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - site = await self.get_site(site_id) - - if not site.location_uri: - raise HTTPException(status_code=404, detail="Site has no location") - - location = next((l for l in self.locations if l.self_uri == str(site.location_uri)), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - - return location - - - async def list_locations( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - - locs = self.locations - - if name: - locs = [l for l in locs if name.lower() in l.name.lower()] - - if short_name: - locs = [l for l in locs if l.short_name == short_name] - - if country_name: - locs = [l for l in locs if l.country_name == country_name] - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - locs = [l for l in locs if l.last_modified > ms] - - o = offset or 0 - l = limit or len(locs) - return locs[o:o+l] - - - async def get_location( - self: "DemoAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location: - - location = next((l for l in self.locations if l.id == location_id), None) - - if not location: - raise HTTPException(status_code=404, detail="Location not found") - - if modified_since: - ms = datetime.datetime.fromisoformat(str(modified_since)) - if location.last_modified <= ms: - raise HTTPException(status_code=304, headers={"Last-Modified": location.last_modified.isoformat()}) - return location - - - - # ---------------------------- # Status API # ---------------------------- diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 7c685e15..acd4d394 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -41,38 +41,4 @@ async def get_site( _forbid = Depends(forbidExtraQueryParams("modified_since")), )-> models.Site: """Get site by ID""" - return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) - -@router.get("/sites/{site_id}/location", responses=DEFAULT_RESPONSES, operation_id="getLocationBySite") -async def get_site_location( - request : Request, - site_id: str, - modified_since: StrictDateTime = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get site location by site ID""" - return await router.adapter.get_site_location(site_id=site_id, modified_since=modified_since) - -@router.get("/locations", responses=DEFAULT_RESPONSES, operation_id="getLocations") -async def list_locations( - request : Request, - modified_since: StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), - limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), - country_name: str = Query(default=None, min_length=1), - _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name", "country_name")), - )-> list[models.Location]: - """List locations""" - return await router.adapter.list_locations(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name, country_name=country_name) - -@router.get("/locations/{location_id}", responses=DEFAULT_RESPONSES, operation_id="getLocation") -async def get_location( - request : Request, - location_id: str, - modified_since: StrictDateTime = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("modified_since")), - )-> models.Location: - """Get location by ID""" - return await router.adapter.get_location(location_id=location_id, modified_since=modified_since) \ No newline at end of file + return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) \ No newline at end of file diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index d316674d..7758f24f 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -34,32 +34,4 @@ async def get_site( site_id: str, modified_since: str | None = None, ) -> facility_models.Site | None: - pass - - @abstractmethod - async def get_site_location( - self: "FacilityAdapter", - site_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass - - @abstractmethod - async def list_locations( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None, - country_name: str | None = None, - ) -> list[facility_models.Location]: - pass - - @abstractmethod - async def get_location( - self: "FacilityAdapter", - location_id: str, - modified_since: str | None = None, - ) -> facility_models.Location | None: - pass + pass \ No newline at end of file diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index fd164fad..a3a781c6 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -10,13 +10,6 @@ def _self_path(self) -> str: return f"/facility/sites/{self.id}" short_name: Optional[str] = Field(None, description="Common or short name of the Site.") operating_organization: str = Field(..., description="Organization operating the Site.") - location_uri: Optional[HttpUrl] = Field(None, description="URI of Location containing this Site.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") - -class Location(NamedObject): - def _self_path(self) -> str: - return f"/facility/locations/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Location.") country_name: Optional[str] = Field(None, description="Country name of the Location.") locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") @@ -25,21 +18,13 @@ def _self_path(self) -> str: altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Sites contained in this Location.") + resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + class Facility(NamedObject): def _self_path(self) -> str: - return f"/facility/facilities/{self.id}" + return "/facility" short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") organization_name: Optional[str] = Field(None, description="Operating organization’s name.") support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") - location_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Locations.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of contained Resources.") - event_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Events in this Facility.") - incident_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Incidents in this Facility.") - capability_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Capabilities offered by the Facility.") - project_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Projects associated with this Facility.") - project_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Project Allocations.") - user_allocation_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of User Allocations.") - diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 3650451a..15a9e36e 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -1,9 +1,11 @@ import datetime import enum -from pydantic import BaseModel, computed_field, Field +from typing import Optional +from pydantic import BaseModel, computed_field, Field, HttpUrl from ... import config from ..common import NamedObject + class Link(BaseModel): rel : str href : str @@ -32,9 +34,12 @@ def _self_path(self) -> str: return f"/status/resources/{self.id}" capability_ids: list[str] = Field(exclude=True) - group: str | None - current_status: Status | None = Field("The current status comes from the status of the last event for this resource") resource_type: ResourceType + group: str | None = Field(None, description="Group this resource belongs to") + current_status: Status | None = Field(None, description="The current status comes from the status of the last event for this resource") + located_at_uri: Optional[HttpUrl] = Field(None, description="Resource located at specific Site") + + @computed_field(description="The list of capabilities in this resource") @property From e85c2660fd3e72a41837d4cfcca842ca34de368e Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 5 Feb 2026 15:57:55 -0600 Subject: [PATCH 064/133] Fix operation id: getJob, getJobs --- app/routers/compute/compute.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index cca45be5..e915048e 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -126,7 +126,7 @@ async def update_job( response_model=models.Job, response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, - operation_id="getJobs", + operation_id="getJob", ) async def get_job_status( resource_id : str, @@ -156,7 +156,7 @@ async def get_job_status( response_model=list[models.Job], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, - operation_id="getAllJobs", + operation_id="getJobs", ) async def get_job_statuses( resource_id : str, From 431da63708a06d68b9218d6c66945969716f6ae6 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 5 Feb 2026 17:43:49 -0600 Subject: [PATCH 065/133] Remove resource_uri and event_uris --- app/routers/status/status.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 6a0e9489..34cc80e5 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -72,10 +72,8 @@ async def get_incidents( offset : int = Query(default=0, ge=0, le=1000), limit : int = Query(default=100, ge=0, le=1000), resolution : models.Resolution = Query(default=None), - resource_uris: Optional[List[str]] = Query(default=None, min_length=1), - event_uris: Optional[List[str]] = Query(default=None, min_length=1), _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", - "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), + "offset", "limit", "resolution")), ) -> list[models.Incident]: return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id, resolution) From 889272489bd764f3eaf6daed692fbf0a42586d57 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 5 Feb 2026 17:43:58 -0600 Subject: [PATCH 066/133] Remove resource_uri and event_uris --- app/routers/status/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 34cc80e5..7f4fae62 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -73,7 +73,7 @@ async def get_incidents( limit : int = Query(default=100, ge=0, le=1000), resolution : models.Resolution = Query(default=None), _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", - "offset", "limit", "resolution")), + "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), ) -> list[models.Incident]: return await router.adapter.get_incidents(offset, limit, name, description, status, type_, from_, to, time_, modified_since, resource_id, resolution) From 3273b8af3e97d41ff72441f5c044ec802a104b5a Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 6 Feb 2026 22:41:12 -0600 Subject: [PATCH 067/133] Add site_id under resource. Fix schemathesis run with excluding none values --- app/demo_adapter.py | 44 ++++++++++++++++---------- app/routers/status/facility_adapter.py | 3 +- app/routers/status/models.py | 8 ++++- app/routers/status/status.py | 1 + 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index d9c72fda..9779e9e8 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -134,21 +134,30 @@ def _init_state(self): "gpfs": Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), } - pm = status_models.Resource(id=str(uuid.uuid4()), group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", capability_ids=[ - self.capabilities["cpu"].id, - self.capabilities["gpu"].id, - ], current_status=status_models.Status.degraded, last_modified=day_ago, resource_type=status_models.ResourceType.compute) - hpss = status_models.Resource(id=str(uuid.uuid4()), group="hpss", name="hpss", description="hpss tape storage", capability_ids=[self.capabilities["hpss"].id], current_status=status_models.Status.up, last_modified=day_ago, resource_type=status_models.ResourceType.storage) - cfs = status_models.Resource(id=str(uuid.uuid4()), group="cfs", name="cfs", description="cfs storage", capability_ids=[self.capabilities["gpfs"].id], current_status=status_models.Status.up, last_modified=day_ago, resource_type=status_models.ResourceType.storage) - - self.resources = [ - pm, - hpss, - cfs, - status_models.Resource(id=str(uuid.uuid4()), group="perlmutter", name="login nodes", description="the perlmutter computer login nodes", capability_ids=[], current_status=status_models.Status.degraded, last_modified=day_ago, resource_type=status_models.ResourceType.system), - status_models.Resource(id=str(uuid.uuid4()), group="services", name="Iris", description="Iris webapp", capability_ids=[], current_status=status_models.Status.down, last_modified=day_ago, resource_type=status_models.ResourceType.website), - status_models.Resource(id=str(uuid.uuid4()), group="services", name="sfapi", description="the Superfacility API", capability_ids=[], current_status=status_models.Status.up, last_modified=day_ago, resource_type=status_models.ResourceType.service), - ] + pm = status_models.Resource(id=str(uuid.uuid4()), site_id=site1.id, group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", + capability_ids=[self.capabilities["cpu"].id, self.capabilities["gpu"].id,], current_status=status_models.Status.degraded, + last_modified=day_ago, resource_type=status_models.ResourceType.compute, located_at_uri=site1.self_uri) + + hpss = status_models.Resource(id=str(uuid.uuid4()), site_id=site1.id, group="hpss", name="hpss", description="hpss tape storage", + capability_ids=[self.capabilities["hpss"].id], current_status=status_models.Status.up, + last_modified=day_ago, resource_type=status_models.ResourceType.storage, located_at_uri=site1.self_uri) + + cfs = status_models.Resource(id=str(uuid.uuid4()), site_id=site1.id, group="cfs", name="cfs", description="cfs storage", + capability_ids=[self.capabilities["gpfs"].id], current_status=status_models.Status.up, + last_modified=day_ago, resource_type=status_models.ResourceType.storage, located_at_uri=site1.self_uri) + + login = status_models.Resource(id=str(uuid.uuid4()), site_id=site2.id, group="perlmutter", name="login nodes", description="the perlmutter computer login nodes", + capability_ids=[], current_status=status_models.Status.degraded, + last_modified=day_ago, resource_type=status_models.ResourceType.system, located_at_uri=site2.self_uri) + + iris = status_models.Resource(id=str(uuid.uuid4()), site_id=site2.id, group="services", name="Iris", description="Iris webapp", + capability_ids=[], current_status=status_models.Status.down, + last_modified=day_ago, resource_type=status_models.ResourceType.website, located_at_uri=site2.self_uri) + sfapi = status_models.Resource(id=str(uuid.uuid4()), site_id=site2.id, group="services", name="sfapi", description="the Superfacility API", + capability_ids=[], current_status=status_models.Status.up, + last_modified=day_ago, resource_type=status_models.ResourceType.service, located_at_uri=site2.self_uri) + + self.resources = [pm, hpss, cfs, login, iris, sfapi] self.projects = [ account_models.Project( @@ -318,10 +327,11 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type : status_models.ResourceType | None = None, current_status : status_models.Status | None = None, - capability: Capability | None = None + capability: Capability | None = None, + site_id: str | None = None ) -> list[status_models.Resource]: resources = status_models.Resource.find(self.resources, name=name, description=description, group=group, modified_since=modified_since, - resource_type=resource_type, current_status=current_status, capability=capability) + resource_type=resource_type, current_status=current_status, capability=capability, site_id=site_id) return paginate_list(resources, offset, limit) diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 274e9f5c..a17f6108 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -24,7 +24,8 @@ async def get_resources( modified_since : datetime.datetime | None = None, resource_type: status_models.ResourceType = Query(default=None), current_status: status_models.Status = Query(default=None), - capability: Capability | None = None + capability: Capability | None = None, + site_id: str | None = None ) -> list[status_models.Resource]: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 88fb3b46..d7213067 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -34,6 +34,9 @@ def _self_path(self) -> str: """ Return the API path for this resource. """ return f"/status/resources/{self.id}" + # NOTE (TBR): If site_id is required, then located_at_uri should be also required. This can be easily identified by Site.self_uri + # Is there a specific Resource, that has no Site? + site_id: str = Field(..., description="The site identifier this resource is located at") capability_ids: list[str] = Field(default_factory=list, exclude=True) group: str | None current_status: Status | None = Field(default=None, description="The current status comes from the status of the last event for this resource") @@ -49,7 +52,8 @@ def capability_uris(self) -> list[str]: return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] @classmethod - def find(cls, items, name=None, description=None, modified_since=None, group=None, resource_type=None, current_status=None, capability=None) -> list: + def find(cls, items, name=None, description=None, modified_since=None, group=None, + resource_type=None, current_status=None, capability=None, site_id=None) -> list: items = super().find(items, name=name, description=description, modified_since=modified_since) if group: items = [item for item in items if item.group == group] @@ -62,6 +66,8 @@ def find(cls, items, name=None, description=None, modified_since=None, group=Non if capability: items = [item for item in items if any(cap_id in item.capability_ids for cap_id in capability)] + if site_id: + items = [item for item in items if item.site_id == site_id] return items class Event(NamedObject): diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 5b50a4a1..424564a0 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -17,6 +17,7 @@ description="Get a list of all resources at this facility. You can optionally filter the returned list by specifying attribtes.", responses=DEFAULT_RESPONSES, operation_id="getResources", + response_model_exclude_none=True ) async def get_resources( request : Request, From 68a72bc1fc14465280820c0e92e42bf1bdbcbf4c Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Sat, 7 Feb 2026 08:15:38 -0600 Subject: [PATCH 068/133] Split common.py into it's own types --- app/demo_adapter.py | 39 ++-- app/routers/account/account.py | 10 +- app/routers/account/facility_adapter.py | 5 +- app/routers/account/models.py | 6 +- app/routers/common.py | 279 ------------------------ app/routers/compute/compute.py | 18 +- app/routers/compute/models.py | 8 +- app/routers/facility/facility.py | 9 +- app/routers/facility/models.py | 7 +- app/routers/status/facility_adapter.py | 6 +- app/routers/status/models.py | 6 +- app/routers/status/status.py | 11 +- app/types/__init__.py | 0 app/types/base.py | 101 +++++++++ app/types/http.py | 90 ++++++++ app/types/models.py | 21 ++ app/types/scalars.py | 97 ++++++++ 17 files changed, 383 insertions(+), 330 deletions(-) delete mode 100644 app/routers/common.py create mode 100644 app/types/__init__.py create mode 100644 app/types/base.py create mode 100644 app/types/http.py create mode 100644 app/types/models.py create mode 100644 app/types/scalars.py diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 9779e9e8..b0285540 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -1,24 +1,33 @@ +import base64 import datetime -import random -import uuid +import glob +import grp import os -import stat +import pathlib import pwd -import grp -import glob +import random +import stat import subprocess -import pathlib -import base64 +import uuid from typing import Any, Tuple -from pydantic import BaseModel + from fastapi import HTTPException -from .routers.common import AllocationUnit, Capability -from .routers.facility import models as facility_models, facility_adapter as facility_adapter -from .routers.status import models as status_models, facility_adapter as status_adapter -from .routers.account import models as account_models, facility_adapter as account_adapter -from .routers.compute import models as compute_models, facility_adapter as compute_adapter -from .routers.filesystem import models as filesystem_models, facility_adapter as filesystem_adapter -from .routers.task import models as task_models, facility_adapter as task_adapter +from pydantic import BaseModel + +from .routers.account import facility_adapter as account_adapter +from .routers.account import models as account_models +from .routers.compute import facility_adapter as compute_adapter +from .routers.compute import models as compute_models +from .routers.facility import facility_adapter +from .routers.facility import models as facility_models +from .routers.filesystem import facility_adapter as filesystem_adapter +from .routers.filesystem import models as filesystem_models +from .routers.status import facility_adapter as status_adapter +from .routers.status import models as status_models +from .routers.task import facility_adapter as task_adapter +from .routers.task import models as task_models +from .types.models import Capability +from .types.scalars import AllocationUnit DEMO_QUEUE_UPDATE_SECS = 5 diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 6aa6e233..abd55ad5 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -1,9 +1,11 @@ -from fastapi import HTTPException, Request, Depends, Query -from . import models, facility_adapter +from fastapi import Depends, HTTPException, Query, Request + +from ...types.http import forbidExtraQueryParams +from ...types.models import Capability +from ...types.scalars import StrictDateTime from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import forbidExtraQueryParams, StrictDateTime, Capability - +from . import facility_adapter, models router = iri_router.IriRouter( facility_adapter.FacilityAdapter, diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 334e6827..fb2a66f3 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -1,7 +1,8 @@ from abc import abstractmethod -from . import models as account_models -from ..common import Capability + +from ...types.models import Capability from ..iri_router import AuthenticatedAdapter +from . import models as account_models class FacilityAdapter(AuthenticatedAdapter): diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 1a9333d8..6bde3be4 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,6 +1,8 @@ -from pydantic import computed_field, Field +from pydantic import Field, computed_field + from ... import config -from ..common import IRIBaseModel, AllocationUnit +from ...types.base import IRIBaseModel +from ...types.scalars import AllocationUnit class User(IRIBaseModel): diff --git a/app/routers/common.py b/app/routers/common.py deleted file mode 100644 index a6a91ec2..00000000 --- a/app/routers/common.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Default models used by multiple routers.""" -import datetime -from email.utils import parsedate_to_datetime -import enum -from typing import Optional -from urllib.parse import parse_qs -from collections.abc import Iterable -from pydantic_core import core_schema -from pydantic import BaseModel, ConfigDict, Field, computed_field, model_serializer, field_validator -from fastapi import Request, HTTPException, status - -from .. import config - -# These are Pydantic custom types for strict validation -# that are not implmented in Pydantic by default. -# ----------------------------------------------------------------------- -# StrictBool: a strict boolean type -class StrictBool: - """Strict boolean: - - Accepts: real booleans, 'true', 'false' - - Rejects everything else. - """ - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - return core_schema.no_info_plain_validator_function(cls.validate) - - @staticmethod - def validate(value): - """Validate the input value as a strict boolean.""" - if isinstance(value, bool): - return value - if isinstance(value, str): - v = value.strip().lower() - if v == "true": - return True - if v == "false": - return False - raise ValueError("Invalid boolean value. Expected 'true' or 'false'.") - raise ValueError("Invalid boolean value. Expected true/false or 'true'/'false'.") - - @classmethod - def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "boolean", - "description": "Strict boolean. Only true/false allowed (bool or string)." - } - -# ----------------------------------------------------------------------- -# StrictDateTime: a strict ISO8601 datetime type - -class StrictDateTime: - """ - Strict ISO8601 datetime: - - Accepts datetime objects - - Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 - - Converts 'Z' → UTC - - Converts naive datetimes → UTC - - Rejects integers ("0"), null, garbage strings, etc. - """ - - @classmethod - def __get_pydantic_core_schema__(cls, source, handler): - return core_schema.no_info_plain_validator_function(cls.validate) - - @staticmethod - def validate(value): - if isinstance(value, datetime.datetime): - return StrictDateTime._normalize(value) - if not isinstance(value, str): - raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") - v = value.strip() - if v.endswith("Z"): - v = v[:-1] + "+00:00" - try: - dt = datetime.datetime.fromisoformat(v) - except Exception as ex: - raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex - - return StrictDateTime._normalize(dt) - - - @staticmethod - def modifiedSinceDatetime( - modified_since: str | None, - header_modified_since: str | None - ) -> datetime.datetime | None: - """ - Combine modified_since (ISO8601) and If-Modified-Since (RFC1123). - If both are provided, the most recent timestamp is used. - """ - - parsed_times: list[datetime.datetime] = [] - - # Query param (ISO 8601) - if modified_since is not None: - try: - dt = StrictDateTime.validate(modified_since) - parsed_times.append(dt) - except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid modified_since query param: {exc}", - ) from exc - - # Header (RFC 1123) - if header_modified_since is not None: - try: - dt = parsedate_to_datetime(header_modified_since) - if dt is None: - raise ValueError("Invalid RFC1123 date") - if dt.tzinfo is None: - dt = dt.replace(tzinfo=datetime.timezone.utc) - parsed_times.append(dt.astimezone(datetime.timezone.utc)) - except Exception as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Invalid If-Modified-Since header format (must be RFC1123)", - ) from exc - - if not parsed_times: - return None - - # Stricter constraint wins - return max(parsed_times) - - @staticmethod - def _normalize(dt: datetime.datetime) -> datetime.datetime: - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - - @classmethod - def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "string", - "format": "date-time", - "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." - } - - -def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): - multiParams = multiParams or set() - - async def checker(req: Request): - if "*" in allowedParams: - return - - raw_qs = req.scope.get("query_string", b"") - parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) - - allowed = set(allowedParams) - - for key, values in parsed.items(): - if key not in allowed: - raise HTTPException(status_code=422, - detail=[{"type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}"}]) - - - if len(values) > 1 and key not in multiParams: - raise HTTPException(status_code=422, - detail=[{"type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}"}]) - - return checker - - -class IRIBaseModel(BaseModel): - """Base model for IRI models.""" - model_config = ConfigDict(extra="allow") - - @model_serializer(mode="wrap") - def _hide_extra(self, handler, info): - data = handler(self) - - model_fields = set(self.model_fields or {}) - computed_fields = set(self.model_computed_fields or {}) - extra = getattr(self, "__pydantic_extra__", {}) or {} - for k in extra: - if k not in model_fields and k not in computed_fields: - data.pop(k, None) - return data - - def get_extra(self, key, default=None): - return getattr(self, "__pydantic_extra__", {}).get(key, default) - - -class NamedObject(IRIBaseModel): - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - def _self_path(self) -> str: - raise NotImplementedError - - @classmethod - def normalize_dt(cls, dt: datetime | None) -> datetime | None: - """Normalize datetime to UTC-aware.""" - # Convert naive datetimes into UTC-aware versions - if dt is None: - return None - if isinstance(dt, str): - dt = StrictDateTime.validate(dt) - if dt.tzinfo is None: - return dt.replace(tzinfo=datetime.timezone.utc) - return dt - - @field_validator("last_modified", mode="before") - @classmethod - def _norm_dt_field(cls, v): - return cls.normalize_dt(v) - - @computed_field(description="The canonical URL of this object") - @property - def self_uri(self) -> str: - """Computed self URI property.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") - - @classmethod - def find_by_id(cls, items, id_, allow_name: bool = False): - """ Find an object by its id or name == id. """ - # Find a resource by its id. - # If allow_name is True, the id parameter can also match the resource's name. - matches = [r for r in items if r.id == id_ or (allow_name and r.name == id_)] - if not matches: - return None - if len(matches) > 1: - raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id_}'") - - return matches[0] - - @classmethod - def find(cls, items, name=None, description=None, modified_since=None): - """ Find objects matching the given criteria. """ - single = False - if not any((name, description, modified_since)): - return items - - if not isinstance(items, Iterable) or isinstance(items, BaseModel): - items = [items] - single = True - - if name: - items = [item for item in items if item.name == name] - if description: - items = [item for item in items if item.description and description in item.description] - if modified_since: - modified_since = cls.normalize_dt(modified_since) - items = [item for item in items - if item.last_modified and item.last_modified >= modified_since] - if single: - return items[0] if items else None - return items - - -class AllocationUnit(enum.Enum): - node_hours = "node_hours" - bytes = "bytes" - inodes = "inodes" - - -class Capability(NamedObject): - """ - An aspect of a resource that can have an allocation. - For example, Perlmutter nodes with GPUs - For some resources at a facility, this will be 1 to 1 with the resource. - It is a way to further subdivide a resource into allocatable sub-resources. - The word "capability" is also known to users as something they need for a job to run. (eg. gpu) - """ - def _self_path(self) -> str: - return f"/account/capabilities/{self.id}" - - last_modified: StrictDateTime | None = Field(default=None, description="ISO 8601 timestamp when this object was last modified.") - - units: list[AllocationUnit] diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 5a11c8a7..d1eadf4b 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,12 +1,12 @@ """Compute resource API router""" -from fastapi import HTTPException, Request, Depends, status, Query -from . import models, facility_adapter -from .. import iri_router +from fastapi import Depends, HTTPException, Query, Request, status +from ...types.http import forbidExtraQueryParams +from ...types.scalars import StrictHTTPBool +from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router -from ..common import forbidExtraQueryParams, StrictBool - +from . import facility_adapter, models router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -132,8 +132,8 @@ async def get_job_status( resource_id : str, job_id : str, request : Request, - historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), - include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + historical : StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictHTTPBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" @@ -164,8 +164,8 @@ async def get_job_statuses( offset : int = Query(default=0, ge=0, le=1000), limit : int = Query(default=100, ge=0, le=1000), filters : dict[str, object] | None = None, - historical : StrictBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), - include_spec: StrictBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + historical : StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictHTTPBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index a56d4fe4..87142a45 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,7 +1,9 @@ -from typing import Annotated from enum import IntEnum -from pydantic import field_serializer, ConfigDict, StrictBool, Field -from ..common import IRIBaseModel +from typing import Annotated + +from pydantic import ConfigDict, Field, StrictBool, field_serializer + +from ...types.base import IRIBaseModel class ResourceSpec(IRIBaseModel): diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index d8422240..7a1a1ea1 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -1,9 +1,10 @@ -from fastapi import Request, Depends, Query +from fastapi import Depends, Query, Request + +from ...types.http import forbidExtraQueryParams +from ...types.scalars import StrictDateTime from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from .import models, facility_adapter -from ..common import StrictDateTime, forbidExtraQueryParams - +from . import facility_adapter, models router = iri_router.IriRouter(facility_adapter.FacilityAdapter, prefix="/facility", diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 32bc866b..021d9a24 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,8 +1,9 @@ """Facility-related models.""" -from typing import Optional, List +from typing import List, Optional + from pydantic import Field, HttpUrl -from ..common import NamedObject +from ...types.base import NamedObject class Site(NamedObject): @@ -20,7 +21,6 @@ def _self_path(self) -> str: longitude: Optional[float] = Field(None, description="Longitude of the Location.") resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") - @classmethod def find(cls, items, name=None, description=None, modified_since=None, short_name=None, country_name=None): """ Find Locations matching the given criteria. """ @@ -32,7 +32,6 @@ def find(cls, items, name=None, description=None, modified_since=None, short_nam return items - class Facility(NamedObject): def _self_path(self) -> str: return "/facility" diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index a17f6108..6be6c883 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -1,8 +1,10 @@ -from abc import ABC, abstractmethod import datetime +from abc import ABC, abstractmethod + from fastapi import Query + +from ...types.models import Capability from . import models as status_models -from ..common import Capability class FacilityAdapter(ABC): diff --git a/app/routers/status/models.py b/app/routers/status/models.py index d7213067..7f0d51d0 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -1,9 +1,11 @@ import datetime import enum from typing import Optional -from pydantic import BaseModel, computed_field, Field, field_validator, HttpUrl + +from pydantic import BaseModel, Field, HttpUrl, computed_field, field_validator + from ... import config -from ..common import NamedObject +from ...types.base import NamedObject class Link(BaseModel): diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 424564a0..6ffa0bad 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -1,9 +1,12 @@ -from typing import Optional, List, Annotated -from fastapi import HTTPException, Request, Query, Depends -from . import models, facility_adapter +from typing import Annotated, List, Optional + +from fastapi import Depends, HTTPException, Query, Request + +from ...types.http import forbidExtraQueryParams +from ...types.scalars import AllocationUnit, StrictDateTime from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from ..common import StrictDateTime, forbidExtraQueryParams, AllocationUnit +from . import facility_adapter, models router = iri_router.IriRouter( facility_adapter.FacilityAdapter, diff --git a/app/types/__init__.py b/app/types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/types/base.py b/app/types/base.py new file mode 100644 index 00000000..4a3b245d --- /dev/null +++ b/app/types/base.py @@ -0,0 +1,101 @@ +"""Default models used by multiple routers.""" +import datetime +from collections.abc import Iterable +from typing import Optional + +from pydantic import (BaseModel, ConfigDict, Field, computed_field, + field_validator, model_serializer) + +from .. import config +from .scalars import StrictDateTime + + +class IRIBaseModel(BaseModel): + """Base model for IRI models.""" + model_config = ConfigDict(extra="allow") + + @model_serializer(mode="wrap") + def _hide_extra(self, handler, info): + data = handler(self) + + model_fields = set(self.model_fields or {}) + computed_fields = set(self.model_computed_fields or {}) + extra = getattr(self, "__pydantic_extra__", {}) or {} + for k in extra: + if k not in model_fields and k not in computed_fields: + data.pop(k, None) + return data + + def get_extra(self, key, default=None): + """Get an extra field value that is not defined in the model. Returns default if not found.""" + return getattr(self, "__pydantic_extra__", {}).get(key, default) + + +class NamedObject(IRIBaseModel): + """Base model for named objects.""" + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: + raise NotImplementedError + + @classmethod + def normalize_dt(cls, dt: datetime | None) -> datetime | None: + """Normalize datetime to UTC-aware.""" + # Convert naive datetimes into UTC-aware versions + if dt is None: + return None + if isinstance(dt, str): + dt = StrictDateTime.validate(dt) + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @field_validator("last_modified", mode="before") + @classmethod + def _norm_dt_field(cls, v): + return cls.normalize_dt(v) + + @computed_field(description="The canonical URL of this object") + @property + def self_uri(self) -> str: + """Computed self URI property.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + + name: Optional[str] = Field(None, description="The long name of the object.") + description: Optional[str] = Field(None, description="Human-readable description of the object.") + last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + + @classmethod + def find_by_id(cls, items, id_, allow_name: bool = False): + """ Find an object by its id or name == id. """ + # Find a resource by its id. + # If allow_name is True, the id parameter can also match the resource's name. + matches = [r for r in items if r.id == id_ or (allow_name and r.name == id_)] + if not matches: + return None + if len(matches) > 1: + raise ValueError(f"Multiple {cls.__name__} objects matched identifier '{id_}'") + + return matches[0] + + @classmethod + def find(cls, items, name=None, description=None, modified_since=None): + """ Find objects matching the given criteria. """ + single = False + if not any((name, description, modified_since)): + return items + + if not isinstance(items, Iterable) or isinstance(items, BaseModel): + items = [items] + single = True + + if name: + items = [item for item in items if item.name == name] + if description: + items = [item for item in items if item.description and description in item.description] + if modified_since: + modified_since = cls.normalize_dt(modified_since) + items = [item for item in items + if item.last_modified and item.last_modified >= modified_since] + if single: + return items[0] if items else None + return items diff --git a/app/types/http.py b/app/types/http.py new file mode 100644 index 00000000..c59c8ef1 --- /dev/null +++ b/app/types/http.py @@ -0,0 +1,90 @@ +"""HTTP-related types and utilities for the IRI Facility API""" +import datetime +from email.utils import parsedate_to_datetime +from urllib.parse import parse_qs + +from fastapi import HTTPException, Request, status + +from .scalars import StrictDateTime + +# ----------------------------------------------------------------------- +# modifiedSinceDatetime: combine modified_since (ISO8601) and If-Modified-Since (RFC1123) +# If both are provided, the most recent timestamp is used. Strict validation is applied to both formats. +# modified_since must be a valid ISO8601 datetime string. +# If-Modified-Since must be a valid RFC1123 datetime string. +# TODO: If-Modified-Since is not yet supported/used by the API. + +def modifiedSinceDatetime( + modified_since: str | None, + header_modified_since: str | None +) -> datetime.datetime | None: + """ + Combine modified_since (ISO8601) and If-Modified-Since (RFC1123). + If both are provided, the most recent timestamp is used. + """ + + parsed_times: list[datetime.datetime] = [] + + # Query param (ISO 8601) + if modified_since is not None: + try: + dt = StrictDateTime.validate(modified_since) + parsed_times.append(dt) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid modified_since query param: {exc}", + ) from exc + + # Header (RFC 1123) + if header_modified_since is not None: + try: + dt = parsedate_to_datetime(header_modified_since) + if dt is None: + raise ValueError("Invalid RFC1123 date") + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + parsed_times.append(dt.astimezone(datetime.timezone.utc)) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid If-Modified-Since header format (must be RFC1123)", + ) from exc + + if not parsed_times: + return None + + # Stricter constraint wins + return max(parsed_times) + +# ----------------------------------------------------------------------- +# forbidExtraQueryParams: a dependency to forbid extra query parameters + +def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): + """Dependency to forbid extra query parameters. If allowedParams contains "*", all params are allowed.""" + multiParams = multiParams or set() + + async def checker(req: Request): + if "*" in allowedParams: + return + + raw_qs = req.scope.get("query_string", b"") + parsed = parse_qs(raw_qs.decode("utf-8", errors="strict"), keep_blank_values=True) + + allowed = set(allowedParams) + + for key, values in parsed.items(): + if key not in allowed: + raise HTTPException(status_code=422, + detail=[{"type": "extra_forbidden", + "loc": ["query", key], + "msg": f"Unexpected query parameter: {key}"}]) + + + if len(values) > 1 and key not in multiParams: + raise HTTPException(status_code=422, + detail=[{"type": "duplicate_forbidden", + "loc": ["query", key], + "msg": f"Duplicate query parameter: {key}"}]) + + return checker diff --git a/app/types/models.py b/app/types/models.py new file mode 100644 index 00000000..9378f62a --- /dev/null +++ b/app/types/models.py @@ -0,0 +1,21 @@ +"""Models for the IRI Facility API.""" +from pydantic import Field + +from .base import NamedObject +from .scalars import AllocationUnit, StrictDateTime + + +class Capability(NamedObject): + """ + An aspect of a resource that can have an allocation. + For example, Perlmutter nodes with GPUs + For some resources at a facility, this will be 1 to 1 with the resource. + It is a way to further subdivide a resource into allocatable sub-resources. + The word "capability" is also known to users as something they need for a job to run. (eg. gpu) + """ + def _self_path(self) -> str: + return f"/account/capabilities/{self.id}" + + last_modified: StrictDateTime | None = Field(default=None, description="ISO 8601 timestamp when this object was last modified.") + + units: list[AllocationUnit] diff --git a/app/types/scalars.py b/app/types/scalars.py new file mode 100644 index 00000000..582efcee --- /dev/null +++ b/app/types/scalars.py @@ -0,0 +1,97 @@ +"""Scalar types for the IRI Facility API""" +# pylint: disable=unused-argument +import datetime +import enum + +from pydantic_core import core_schema + +# ----------------------------------------------------------------------- +# StrictHTTPBool: a strict boolean type + +class StrictHTTPBool: + """Strict boolean: + - Accepts: real booleans, 'true', 'false' + - Rejects everything else. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + """Validate the input value as a strict boolean.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + v = value.strip().lower() + if v == "true": + return True + if v == "false": + return False + raise ValueError("Invalid boolean value. Expected 'true' or 'false'.") + raise ValueError("Invalid boolean value. Expected true/false or 'true'/'false'.") + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "boolean", + "description": "Strict boolean. Only true/false allowed (bool or string)." + } + +# ----------------------------------------------------------------------- +# StrictDateTime: a strict ISO8601 datetime type + +class StrictDateTime: + """ + Strict ISO8601 datetime: + - Accepts datetime objects + - Accepts ISO8601 strings: 2025-12-06T10:00:00Z, 2025-12-06T10:00:00+00:00 + - Converts 'Z' → UTC + - Converts naive datetimes → UTC + - Rejects integers ("0"), null, garbage strings, etc. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, source, handler): + return core_schema.no_info_plain_validator_function(cls.validate) + + @staticmethod + def validate(value): + """Validate the input value as a strict ISO8601 datetime.""" + if isinstance(value, datetime.datetime): + return StrictDateTime._normalize(value) + if not isinstance(value, str): + raise ValueError("Invalid datetime value. Expected ISO8601 datetime string.") + v = value.strip() + if v.endswith("Z"): + v = v[:-1] + "+00:00" + try: + dt = datetime.datetime.fromisoformat(v) + except Exception as ex: + raise ValueError("Invalid datetime format. Expected ISO8601 string.") from ex + + return StrictDateTime._normalize(dt) + + @staticmethod + def _normalize(dt: datetime.datetime) -> datetime.datetime: + if dt.tzinfo is None: + return dt.replace(tzinfo=datetime.timezone.utc) + return dt + + @classmethod + def __get_pydantic_json_schema__(cls, schema, handler): + return { + "type": "string", + "format": "date-time", + "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." + } + +# ----------------------------------------------------------------------- +# AllocationUnit: an enum for allocation units + +class AllocationUnit(enum.Enum): + """Units for allocation""" + node_hours = "node_hours" + bytes = "bytes" + inodes = "inodes" From 9e65878bf7e297c9aee0abf79e8a840ec7258898 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Sat, 7 Feb 2026 17:21:20 -0600 Subject: [PATCH 069/133] ruff, lint, bandit, pip-audit all codebase. Update Makefile to allow --- Makefile | 78 ++- app/__init__.py | 5 +- app/config.py | 7 +- app/demo_adapter.py | 636 +++++++++------------ app/main.py | 6 +- app/routers/account/account.py | 69 +-- app/routers/account/facility_adapter.py | 30 +- app/routers/account/models.py | 22 +- app/routers/compute/compute.py | 62 +- app/routers/compute/facility_adapter.py | 42 +- app/routers/compute/models.py | 30 +- app/routers/error_handlers.py | 74 +-- app/routers/facility/facility.py | 19 +- app/routers/facility/facility_adapter.py | 14 +- app/routers/facility/models.py | 5 +- app/routers/filesystem/facility_adapter.py | 151 +---- app/routers/filesystem/filesystem.py | 195 +++---- app/routers/filesystem/models.py | 49 +- app/routers/iri_router.py | 40 +- app/routers/status/facility_adapter.py | 96 ++-- app/routers/status/models.py | 69 +-- app/routers/status/status.py | 140 +++-- app/routers/task/facility_adapter.py | 28 +- app/routers/task/models.py | 7 +- app/routers/task/task.py | 14 +- app/types/base.py | 14 +- app/types/http.py | 20 +- app/types/models.py | 12 +- app/types/scalars.py | 22 +- gunicorn.config.py | 2 +- pylintrc | 581 +++++++++++++++++++ pyproject.toml | 5 +- 32 files changed, 1385 insertions(+), 1159 deletions(-) create mode 100644 pylintrc diff --git a/Makefile b/Makefile index abd87e4a..e9bb6ff6 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,69 @@ -dev : .venv - @source ./.venv/bin/activate && \ - IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ - IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ - IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ - IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ - IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ - IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ - OPENTELEMETRY_ENABLED=true \ - API_URL_ROOT='http://127.0.0.1:8000' fastapi dev +PYTHON := python3.14 +VENV := .venv +BIN := $(VENV)/bin +UV := uv +PIP := $(BIN)/pip +STAMP_VENV := $(VENV)/.created +STAMP_DEPS := $(VENV)/.deps -.venv: - @uv venv - @uv pip install -e . +.DEFAULT_GOAL := dev +$(STAMP_VENV): + $(UV) venv $(VENV) + touch $(STAMP_VENV) + +.venv: $(STAMP_VENV) + +$(STAMP_DEPS): $(STAMP_VENV) pyproject.toml + $(UV) pip install --python $(BIN)/python -e . + $(UV) pip install --python $(BIN)/python \ + ruff \ + pylint \ + bandit + touch $(STAMP_DEPS) + +deps: $(STAMP_DEPS) + +dev: deps + @source $(BIN)/activate && \ + IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ + IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ + IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ + IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ + IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ + IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + OPENTELEMETRY_ENABLED=true \ + API_URL_ROOT='http://127.0.0.1:8000' fastapi dev .PHONY: clean clean: - @rm -rf iri_sandbox - @rm -rf .venv + rm -rf iri_sandbox + rm -rf .venv + +# Format and lint +format: deps + $(BIN)/ruff format --line-length 200 . + +ruff: deps + $(BIN)/ruff check . --fix || true + +pylint: deps + find . -path ./$(VENV) -prune -o -type f -name "*.py" -print0 | while IFS= read -r -d '' f; do \ + echo "Pylint $$f"; \ + $(BIN)/pylint $$f --rcfile pylintrc || true; \ + done + +# Security +audit: deps + uv pip compile pyproject.toml -o requirements.txt + uv pip sync requirements.txt + uv pip install pip-audit + $(BIN)/pip-audit || true + rm -f requirements.txt + +bandit: deps + $(BIN)/bandit -r app || true + +# Full validation bundle +lint: clean format ruff pylint audit bandit diff --git a/app/__init__.py b/app/__init__.py index 39fe7626..b36383a6 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,2 +1,3 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/app/config.py b/app/config.py index 71a7c20d..c8c53025 100644 --- a/app/config.py +++ b/app/config.py @@ -23,11 +23,8 @@ "description": description, "version": API_VERSION, "docs_url": "/", - "contact": { - "name": "Facility API contact", - "url": "https://www.somefacility.gov/about/contact-us/" - }, - "terms_of_service": "https://www.somefacility.gov/terms-of-service" + "contact": {"name": "Facility API contact", "url": "https://www.somefacility.gov/about/contact-us/"}, + "terms_of_service": "https://www.somefacility.gov/terms-of-service", } try: # optionally overload the init params diff --git a/app/demo_adapter.py b/app/demo_adapter.py index b0285540..390f2661 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -31,6 +31,7 @@ DEMO_QUEUE_UPDATE_SECS = 5 + def paginate_list(items, offset: int | None, limit: int | None): """Return a sliced items using offset and limit.""" if offset is not None and offset > 0: @@ -39,6 +40,7 @@ def paginate_list(items, offset: int | None, limit: int | None): items = items[:limit] return items + class PathSandbox: _base_temp_dir = None @@ -46,10 +48,7 @@ class PathSandbox: def get_base_temp_dir(cls): if cls._base_temp_dir is None: # Create in system temp with a fixed name - cls._base_temp_dir = os.path.join( - os.getcwd(), - "iri_sandbox" - ) + cls._base_temp_dir = os.path.join(os.getcwd(), "iri_sandbox") os.makedirs(cls._base_temp_dir, exist_ok=True) # create a test file @@ -72,9 +71,9 @@ def utc_timestamp() -> int: return int(utc_now().timestamp()) -class DemoAdapter(status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, - compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, - task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter): +class DemoAdapter( + status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter +): def __init__(self): self.resources = [] self.incidents = [] @@ -89,7 +88,6 @@ def __init__(self): self.sites = [] self._init_state() - def _init_state(self): now = utc_now() @@ -105,7 +103,8 @@ def _init_state(self): state_or_province_name="DC", latitude=36.173357, longitude=-234.51452, - resource_uris=[]) + resource_uris=[], + ) site2 = facility_models.Site( id=demo_uuid("site", "demo_site_2"), @@ -119,7 +118,8 @@ def _init_state(self): state_or_province_name="ET", latitude=38.410558, longitude=-286.36999, - resource_uris=[]) + resource_uris=[], + ) self.facility = facility_models.Facility( id=demo_uuid("facility", "demo_facility"), @@ -129,12 +129,11 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - site_uris=[site1.self_uri, site2.self_uri] + site_uris=[site1.self_uri, site2.self_uri], ) self.sites = [site1, site2] - day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { "cpu": Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[AllocationUnit.node_hours]), @@ -143,28 +142,85 @@ def _init_state(self): "gpfs": Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), } - pm = status_models.Resource(id=str(uuid.uuid4()), site_id=site1.id, group="perlmutter", name="compute nodes", description="the perlmutter computer compute nodes", - capability_ids=[self.capabilities["cpu"].id, self.capabilities["gpu"].id,], current_status=status_models.Status.degraded, - last_modified=day_ago, resource_type=status_models.ResourceType.compute, located_at_uri=site1.self_uri) + pm = status_models.Resource( + id=str(uuid.uuid4()), + site_id=site1.id, + group="perlmutter", + name="compute nodes", + description="the perlmutter computer compute nodes", + capability_ids=[ + self.capabilities["cpu"].id, + self.capabilities["gpu"].id, + ], + current_status=status_models.Status.degraded, + last_modified=day_ago, + resource_type=status_models.ResourceType.compute, + located_at_uri=site1.self_uri, + ) - hpss = status_models.Resource(id=str(uuid.uuid4()), site_id=site1.id, group="hpss", name="hpss", description="hpss tape storage", - capability_ids=[self.capabilities["hpss"].id], current_status=status_models.Status.up, - last_modified=day_ago, resource_type=status_models.ResourceType.storage, located_at_uri=site1.self_uri) + hpss = status_models.Resource( + id=str(uuid.uuid4()), + site_id=site1.id, + group="hpss", + name="hpss", + description="hpss tape storage", + capability_ids=[self.capabilities["hpss"].id], + current_status=status_models.Status.up, + last_modified=day_ago, + resource_type=status_models.ResourceType.storage, + located_at_uri=site1.self_uri, + ) - cfs = status_models.Resource(id=str(uuid.uuid4()), site_id=site1.id, group="cfs", name="cfs", description="cfs storage", - capability_ids=[self.capabilities["gpfs"].id], current_status=status_models.Status.up, - last_modified=day_ago, resource_type=status_models.ResourceType.storage, located_at_uri=site1.self_uri) + cfs = status_models.Resource( + id=str(uuid.uuid4()), + site_id=site1.id, + group="cfs", + name="cfs", + description="cfs storage", + capability_ids=[self.capabilities["gpfs"].id], + current_status=status_models.Status.up, + last_modified=day_ago, + resource_type=status_models.ResourceType.storage, + located_at_uri=site1.self_uri, + ) - login = status_models.Resource(id=str(uuid.uuid4()), site_id=site2.id, group="perlmutter", name="login nodes", description="the perlmutter computer login nodes", - capability_ids=[], current_status=status_models.Status.degraded, - last_modified=day_ago, resource_type=status_models.ResourceType.system, located_at_uri=site2.self_uri) + login = status_models.Resource( + id=str(uuid.uuid4()), + site_id=site2.id, + group="perlmutter", + name="login nodes", + description="the perlmutter computer login nodes", + capability_ids=[], + current_status=status_models.Status.degraded, + last_modified=day_ago, + resource_type=status_models.ResourceType.system, + located_at_uri=site2.self_uri, + ) - iris = status_models.Resource(id=str(uuid.uuid4()), site_id=site2.id, group="services", name="Iris", description="Iris webapp", - capability_ids=[], current_status=status_models.Status.down, - last_modified=day_ago, resource_type=status_models.ResourceType.website, located_at_uri=site2.self_uri) - sfapi = status_models.Resource(id=str(uuid.uuid4()), site_id=site2.id, group="services", name="sfapi", description="the Superfacility API", - capability_ids=[], current_status=status_models.Status.up, - last_modified=day_ago, resource_type=status_models.ResourceType.service, located_at_uri=site2.self_uri) + iris = status_models.Resource( + id=str(uuid.uuid4()), + site_id=site2.id, + group="services", + name="Iris", + description="Iris webapp", + capability_ids=[], + current_status=status_models.Status.down, + last_modified=day_ago, + resource_type=status_models.ResourceType.website, + located_at_uri=site2.self_uri, + ) + sfapi = status_models.Resource( + id=str(uuid.uuid4()), + site_id=site2.id, + group="services", + name="sfapi", + description="the Superfacility API", + capability_ids=[], + current_status=status_models.Status.up, + last_modified=day_ago, + resource_type=status_models.ResourceType.service, + located_at_uri=site2.self_uri, + ) self.resources = [pm, hpss, cfs, login, iris, sfapi] @@ -173,13 +229,13 @@ def _init_state(self): id=str(uuid.uuid4()), name="Staff research project", description="Compute and storage allocation for staff research use", - user_ids=[ "gtorok" ], + user_ids=["gtorok"], ), account_models.Project( id=str(uuid.uuid4()), name="Test project", description="Compute and storage allocation for testing use", - user_ids=[ "gtorok" ], + user_ids=["gtorok"], ), ] @@ -196,7 +252,7 @@ def _init_state(self): unit=cu, ) for cu in c.units - ] + ], ) self.project_allocations.append(pa) self.user_allocations.append( @@ -205,18 +261,11 @@ def _init_state(self): project_id=pa.project_id, project_allocation_id=pa.id, user_id="gtorok", - entries=[ - account_models.AllocationEntry( - allocation=a.allocation/10, - usage=a.usage/10, - unit=a.unit - ) - for a in pa.entries - ] + entries=[account_models.AllocationEntry(allocation=a.allocation / 10, usage=a.usage / 10, unit=a.unit) for a in pa.entries], ) ) - statuses = { r.name: status_models.Status.up for r in self.resources } + statuses = {r.name: status_models.Status.up for r in self.resources} last_incidents = {} d = datetime.datetime(2025, 3, 1, 10, 0, 0, tzinfo=datetime.timezone.utc) @@ -261,33 +310,23 @@ def _init_state(self): end=d, type=random.choice(list(status_models.IncidentType)), resolution=random.choice(list(status_models.Resolution)), - last_modified=d + last_modified=d, ) self.incidents.append(incident) last_incidents[r.name] = incident - d += datetime.timedelta(minutes=int(random.random() * 15 + 1)) # ---------------------------- # Facility API # ---------------------------- - async def get_facility( - self: "DemoAdapter", - modified_since: str | None = None - ) -> facility_models.Facility: + async def get_facility(self: "DemoAdapter", modified_since: str | None = None) -> facility_models.Facility: return self.facility - async def list_sites( - self: "DemoAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None - ) -> list[facility_models.Site]: + self: "DemoAdapter", modified_since: str | None = None, name: str | None = None, offset: int | None = None, limit: int | None = None, short_name: str | None = None + ) -> list[facility_models.Site]: sites = self.sites if name: @@ -302,14 +341,9 @@ async def list_sites( o = offset or 0 l = limit or len(sites) - return sites[o:o+l] + return sites[o : o + l] - - async def get_site( - self: "DemoAdapter", - site_id: str, - modified_since: str | None = None - ) -> facility_models.Site: + async def get_site(self: "DemoAdapter", site_id: str, modified_since: str | None = None) -> facility_models.Site: site = next((s for s in self.sites if s.id == site_id), None) if not site: raise HTTPException(status_code=404, detail="Site not found") @@ -321,147 +355,145 @@ async def get_site( return site - # ---------------------------- # Status API # ---------------------------- async def get_resources( - self : "DemoAdapter", - offset : int, - limit : int, - name : str | None = None, - description : str | None = None, - group : str | None = None, - modified_since : datetime.datetime | None = None, - resource_type : status_models.ResourceType | None = None, - current_status : status_models.Status | None = None, + self: "DemoAdapter", + offset: int, + limit: int, + name: str | None = None, + description: str | None = None, + group: str | None = None, + modified_since: datetime.datetime | None = None, + resource_type: status_models.ResourceType | None = None, + current_status: status_models.Status | None = None, capability: Capability | None = None, - site_id: str | None = None - ) -> list[status_models.Resource]: - resources = status_models.Resource.find(self.resources, name=name, description=description, group=group, modified_since=modified_since, - resource_type=resource_type, current_status=current_status, capability=capability, site_id=site_id) + site_id: str | None = None, + ) -> list[status_models.Resource]: + resources = status_models.Resource.find( + self.resources, + name=name, + description=description, + group=group, + modified_since=modified_since, + resource_type=resource_type, + current_status=current_status, + capability=capability, + site_id=site_id, + ) return paginate_list(resources, offset, limit) - - async def get_resource( - self : "DemoAdapter", - id_ : str - ) -> status_models.Resource: + async def get_resource(self: "DemoAdapter", id_: str) -> status_models.Resource: return status_models.Resource.find_by_id(self.resources, id_) async def get_events( - self : "DemoAdapter", - incident_id : str, - offset : int, - limit : int, - resource_id : str | None = None, - name : str | None = None, - description : str | None = None, - status : status_models.Status | None = None, - from_ : datetime.datetime | None = None, - to : datetime.datetime | None = None, - time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None - ) -> list[status_models.Event]: - events = status_models.Event.find([e for e in self.events if e.incident_id == incident_id], resource_id=resource_id, name=name, description=description, - status=status, from_=from_, to=to, time_=time_, modified_since=modified_since) + self: "DemoAdapter", + incident_id: str, + offset: int, + limit: int, + resource_id: str | None = None, + name: str | None = None, + description: str | None = None, + status: status_models.Status | None = None, + from_: datetime.datetime | None = None, + to: datetime.datetime | None = None, + time_: datetime.datetime | None = None, + modified_since: datetime.datetime | None = None, + ) -> list[status_models.Event]: + events = status_models.Event.find( + [e for e in self.events if e.incident_id == incident_id], + resource_id=resource_id, + name=name, + description=description, + status=status, + from_=from_, + to=to, + time_=time_, + modified_since=modified_since, + ) return paginate_list(events, offset, limit) - - async def get_event( - self : "DemoAdapter", - incident_id : str, - id_ : str - ) -> status_models.Event: + async def get_event(self: "DemoAdapter", incident_id: str, id_: str) -> status_models.Event: return status_models.Event.find_by_id(self.events, id_) - async def get_incidents( - self : "DemoAdapter", - offset : int, - limit : int, - name : str | None = None, - description : str | None = None, - status : status_models.Status | None = None, - type_ : status_models.IncidentType | None = None, - from_ : datetime.datetime | None = None, - to : datetime.datetime | None = None, - time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - resource_id : str | None = None, - resolution: status_models.Resolution | None = None - ) -> list[status_models.Incident]: - incidents = status_models.Incident.find(self.incidents, name=name, description=description, status=status, type_=type_,from_=from_, to=to, - time_=time_, modified_since=modified_since, resource_id=resource_id, resolution=resolution) + self: "DemoAdapter", + offset: int, + limit: int, + name: str | None = None, + description: str | None = None, + status: status_models.Status | None = None, + type_: status_models.IncidentType | None = None, + from_: datetime.datetime | None = None, + to: datetime.datetime | None = None, + time_: datetime.datetime | None = None, + modified_since: datetime.datetime | None = None, + resource_id: str | None = None, + resolution: status_models.Resolution | None = None, + ) -> list[status_models.Incident]: + incidents = status_models.Incident.find( + self.incidents, + name=name, + description=description, + status=status, + type_=type_, + from_=from_, + to=to, + time_=time_, + modified_since=modified_since, + resource_id=resource_id, + resolution=resolution, + ) return paginate_list(incidents, offset, limit) - - async def get_incident( - self : "DemoAdapter", - id_ : str - ) -> status_models.Incident: + async def get_incident(self: "DemoAdapter", id_: str) -> status_models.Incident: return status_models.Incident.find_by_id(self.incidents, id_) - - async def get_capabilities( - self : "DemoAdapter", - name : str | None = None, - modified_since : str | None = None, - offset : int = 0, - limit : int = 1000 - ) -> list[Capability]: + async def get_capabilities(self: "DemoAdapter", name: str | None = None, modified_since: str | None = None, offset: int = 0, limit: int = 1000) -> list[Capability]: return self.capabilities.values() - async def get_current_user( - self : "DemoAdapter", - api_key: str, - client_ip: str, - ) -> str: + self: "DemoAdapter", + api_key: str, + client_ip: str, + ) -> str: """ - In a real deployment, this would decode the api_key jwt and return the current user's id. - This method is not async. + In a real deployment, this would decode the api_key jwt and return the current user's id. + This method is not async. """ return "gtorok" - async def get_user( - self : "DemoAdapter", - user_id: str, - api_key: str, - client_ip: str|None, - ) -> account_models.User: + self: "DemoAdapter", + user_id: str, + api_key: str, + client_ip: str | None, + ) -> account_models.User: if user_id != self.user.id: raise HTTPException(status_code=401, detail="User not found") if api_key != self.user.api_key: raise HTTPException(status_code=403, detail="Invalid API key") return self.user - - async def get_projects( - self : "DemoAdapter", - user: account_models.User - ) -> list[account_models.Project]: + async def get_projects(self: "DemoAdapter", user: account_models.User) -> list[account_models.Project]: return self.projects - async def get_project_allocations( - self : "DemoAdapter", + self: "DemoAdapter", project: account_models.Project, user: account_models.User, - ) -> list[account_models.ProjectAllocation]: + ) -> list[account_models.ProjectAllocation]: return [pa for pa in self.project_allocations if pa.project_id == project.id] - async def get_user_allocations( - self : "DemoAdapter", + self: "DemoAdapter", user: account_models.User, project_allocation: account_models.ProjectAllocation, - ) -> list[account_models.UserAllocation]: + ) -> list[account_models.UserAllocation]: return [ua for ua in self.user_allocations if ua.project_allocation_id == project_allocation.id] - async def submit_job( self: "DemoAdapter", resource: status_models.Resource, @@ -475,11 +507,10 @@ async def submit_job( time=utc_timestamp(), message="job submitted", exit_code=None, - meta_data={ "account": "account1" }, - ) + meta_data={"account": "account1"}, + ), ) - async def submit_job_script( self: "DemoAdapter", resource: status_models.Resource, @@ -494,11 +525,10 @@ async def submit_job_script( time=utc_timestamp(), message="job submitted", exit_code=None, - meta_data={ "account": "account1" }, - ) + meta_data={"account": "account1"}, + ), ) - async def update_job( self: "DemoAdapter", resource: status_models.Resource, @@ -513,11 +543,10 @@ async def update_job( time=utc_timestamp(), message="job updated", exit_code=None, - meta_data={ "account": "account1" }, - ) + meta_data={"account": "account1"}, + ), ) - async def get_job( self: "DemoAdapter", resource: status_models.Resource, @@ -533,32 +562,33 @@ async def get_job( time=utc_timestamp(), message="job completed successfully", exit_code=0, - meta_data={ "account": "account1" }, - ) + meta_data={"account": "account1"}, + ), ) - async def get_jobs( self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, - offset : int, - limit : int, + offset: int, + limit: int, filters: dict[str, object] | None = None, historical: bool = False, include_spec: bool = False, ) -> list[compute_models.Job]: - return [compute_models.Job( - id=f"job_{i}", - status=compute_models.JobStatus( - state=random.choice([s for s in compute_models.JobState]), - time=utc_timestamp() - int(random.random() * 100), - message="", - exit_code=random.choice([0, 0, 0, 0, 0, 1, 1, 128, 127]), - meta_data={ "account": "account1" }, + return [ + compute_models.Job( + id=f"job_{i}", + status=compute_models.JobStatus( + state=random.choice([s for s in compute_models.JobState]), + time=utc_timestamp() - int(random.random() * 100), + message="", + exit_code=random.choice([0, 0, 0, 0, 0, 1, 1, 128, 127]), + meta_data={"account": "account1"}, + ), ) - ) for i in range(random.randint(3, 10))] - + for i in range(random.randint(3, 10)) + ] async def cancel_job( self: "DemoAdapter", @@ -569,7 +599,6 @@ async def cancel_job( # call slurm/etc. to cancel job return True - def validate_path(self, path: str, allow_symlinks: bool = True) -> str: basedir = PathSandbox.get_base_temp_dir() real_path = os.path.realpath(os.path.join(basedir, path)) @@ -586,7 +615,6 @@ def validate_path(self, path: str, allow_symlinks: bool = True) -> str: return real_path - def _file(self, path: str) -> filesystem_models.File: # Get file stats (follows symlinks by default) rp = self.validate_path(path) @@ -615,50 +643,30 @@ def _file(self, path: str) -> filesystem_models.File: permissions = stat.filemode(file_stat.st_mode) # Get last modified time - last_modified = datetime.datetime.fromtimestamp(file_stat.st_mtime).strftime('%Y-%m-%d %H:%M:%S') + last_modified = datetime.datetime.fromtimestamp(file_stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S") # Get size size = str(file_stat.st_size) - return filesystem_models.File( - name=os.path.basename(rp), - type=file_type, - link_target=link_target, - user=user, - group=group, - permissions=permissions, - last_modified=last_modified, - size=size - ) + return filesystem_models.File(name=os.path.basename(rp), type=file_type, link_target=link_target, user=user, group=group, permissions=permissions, last_modified=last_modified, size=size) - async def chmod( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest - ) -> filesystem_models.PutFileChmodResponse: + async def chmod(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChmodRequest) -> filesystem_models.PutFileChmodResponse: rp = self.validate_path(request_model.path) os.chmod(rp, int(request_model.mode, 8)) - return filesystem_models.PutFileChmodResponse( - output=self._file(rp) - ) - + return filesystem_models.PutFileChmodResponse(output=self._file(rp)) async def chown( - self : "DemoAdapter", + self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChownRequest, ) -> filesystem_models.PutFileChownResponse: rp = self.validate_path(request_model.path) os.chown(rp, request_model.owner, request_model.group) - return filesystem_models.PutFileChmodResponse( - output=self._file(rp) - ) - + return filesystem_models.PutFileChmodResponse(output=self._file(rp)) async def ls( - self : "DemoAdapter", + self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, @@ -669,13 +677,10 @@ async def ls( ) -> filesystem_models.GetDirectoryLsResponse: rp = self.validate_path(path) files = glob.glob(rp, recursive=recursive) - return filesystem_models.GetDirectoryLsResponse( - output=[self._file(f) for f in files] - ) - + return filesystem_models.GetDirectoryLsResponse(output=[self._file(f) for f in files]) def _headtail( - self : "DemoAdapter", + self: "DemoAdapter", cmd: str, path: str, file_bytes: int | None, @@ -690,17 +695,12 @@ def _headtail( args.append(str(lines)) rp = self.validate_path(path) args.append(rp) - result = subprocess.run( - args, - capture_output=True, - text=True - ) + result = subprocess.run(args, capture_output=True, text=True) content = result.stdout return content, -len(content) - async def head( - self : "DemoAdapter", + self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, @@ -710,52 +710,20 @@ async def head( ) -> Tuple[Any, int]: return self._headtail("head", path, file_bytes, lines) - - async def tail( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - file_bytes: int | None, - lines: int | None, - skip_trailing: bool - ) -> Tuple[Any, int]: + async def tail(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int | None, lines: int | None, skip_trailing: bool) -> Tuple[Any, int]: return self._headtail("tail", path, file_bytes, lines) - - async def view( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - size: int, - offset: int - ) -> filesystem_models.GetViewFileResponse: + async def view(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, size: int, offset: int) -> filesystem_models.GetViewFileResponse: rp = self.validate_path(path) - result = subprocess.run( - f"tail -c +{offset+1} {rp} | head -c {size}", - shell=True, - capture_output=True, - text=True - ) + result = subprocess.run(f"tail -c +{offset + 1} {rp} | head -c {size}", shell=True, capture_output=True, text=True) content = result.stdout return filesystem_models.GetViewFileResponse( output=content, ) - - async def checksum( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str - ) -> filesystem_models.GetFileChecksumResponse: + async def checksum(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileChecksumResponse: rp = self.validate_path(path) - result = subprocess.run( - ["sha256sum", rp], - capture_output=True, - text=True - ) + result = subprocess.run(["sha256sum", rp], capture_output=True, text=True) checksum = result.stdout.split()[0] return filesystem_models.GetFileChecksumResponse( output=filesystem_models.FileChecksum( @@ -763,38 +731,21 @@ async def checksum( ) ) - - async def file( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str - ) -> filesystem_models.GetFileTypeResponse: + async def file(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileTypeResponse: rp = self.validate_path(path) - result = subprocess.run( - ["file", "-b", rp], - capture_output=True, - text=True - ) + result = subprocess.run(["file", "-b", rp], capture_output=True, text=True) return filesystem_models.GetFileTypeResponse( output=result.stdout.strip(), ) - - async def stat( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - dereference: bool - ) -> filesystem_models.GetFileStatResponse: + async def stat(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, dereference: bool) -> filesystem_models.GetFileStatResponse: rp = self.validate_path(path) if dereference: stat_info = os.stat(rp) else: stat_info = os.lstat(rp) return filesystem_models.GetFileStatResponse( - output=filesystem_models.FileStat( + output=filesystem_models.FileStat( mode=stat_info.st_mode, ino=stat_info.st_ino, dev=stat_info.st_dev, @@ -804,13 +755,12 @@ async def stat( size=stat_info.st_size, atime=int(stat_info.st_atime), ctime=int(stat_info.st_ctime), - mtime=int(stat_info.st_mtime) + mtime=int(stat_info.st_mtime), ) ) - async def rm( - self : "DemoAdapter", + self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, @@ -821,60 +771,33 @@ async def rm( subprocess.run(["rm", "-rf", rp], check=True) return None - - async def mkdir( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostMakeDirRequest - ) -> filesystem_models.PostMkdirResponse: + async def mkdir(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest) -> filesystem_models.PostMkdirResponse: rp = self.validate_path(request_model.path) args = ["mkdir"] if request_model.parent: args.append("-p") args.append(rp) subprocess.run(args, check=True) - return filesystem_models.PostMkdirResponse( - output=self._file(rp) - ) - + return filesystem_models.PostMkdirResponse(output=self._file(rp)) async def symlink( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostFileSymlinkRequest + self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostFileSymlinkRequest ) -> filesystem_models.PostFileSymlinkResponse: rp_src = self.validate_path(request_model.path) rp_dst = self.validate_path(request_model.link_path) subprocess.run(["ln", "-s", rp_src, rp_dst], check=True) - return filesystem_models.PostFileSymlinkResponse( - output=self._file(rp_dst) - ) + return filesystem_models.PostFileSymlinkResponse(output=self._file(rp_dst)) - - async def download( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str - ) -> Any: + async def download(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> Any: rp = self.validate_path(path) raw_content = pathlib.Path(rp).read_bytes() if len(raw_content) > filesystem_adapter.OPS_SIZE_LIMIT: raise Exception("File to download is too large.") - return base64.b64encode(raw_content).decode('utf-8') - + return base64.b64encode(raw_content).decode("utf-8") - async def upload( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - content: str - ) -> None: + async def upload(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, content: str) -> None: rp = self.validate_path(path) if isinstance(content, bytes): pathlib.Path(rp).write_bytes(content) @@ -883,17 +806,13 @@ async def upload( else: raise Exception(f"Don't know how to handle variable of type: {type(content)}") - async def compress( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostCompressRequest - ) -> filesystem_models.PostCompressResponse: + self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCompressRequest + ) -> filesystem_models.PostCompressResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) - args = [ "tar" ] + args = ["tar"] if request_model.compression == filesystem_models.CompressionType.gzip: args.append("-czf") elif request_model.compression == filesystem_models.CompressionType.bzip2: @@ -912,21 +831,13 @@ async def compress( args.append(p.relative_to(PathSandbox.get_base_temp_dir())) subprocess.run(args, check=True) - return filesystem_models.PostCompressResponse( - output=self._file(dst_rp) - ) - + return filesystem_models.PostCompressResponse(output=self._file(dst_rp)) - async def extract( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostExtractRequest - ) -> filesystem_models.PostExtractResponse: + async def extract(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostExtractRequest) -> filesystem_models.PostExtractResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) - args = [ "tar" ] + args = ["tar"] if request_model.compression == filesystem_models.CompressionType.gzip: args.append("-xzf") elif request_model.compression == filesystem_models.CompressionType.bzip2: @@ -940,31 +851,15 @@ async def extract( args.append(dst_rp) subprocess.run(args, check=True) - return filesystem_models.PostExtractResponse( - output=self._file(dst_rp) - ) - + return filesystem_models.PostExtractResponse(output=self._file(dst_rp)) - async def mv( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostMoveRequest - ) -> filesystem_models.PostMoveResponse: + async def mv(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMoveRequest) -> filesystem_models.PostMoveResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) subprocess.run(["mv", src_rp, dst_rp], check=True) - return filesystem_models.PostMoveResponse( - output=self._file(dst_rp) - ) - + return filesystem_models.PostMoveResponse(output=self._file(dst_rp)) - async def cp( - self : "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostCopyRequest - ) -> filesystem_models.PostCopyResponse: + async def cp(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCopyRequest) -> filesystem_models.PostCopyResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) args = ["cp"] @@ -973,33 +868,17 @@ async def cp( args.append(src_rp) args.append(dst_rp) subprocess.run(args, check=True) - return filesystem_models.PostCopyResponse( - output=self._file(dst_rp) - ) - + return filesystem_models.PostCopyResponse(output=self._file(dst_rp)) - async def get_task( - self : "DemoAdapter", - user: account_models.User, - task_id: str - ) -> task_models.Task|None: + async def get_task(self: "DemoAdapter", user: account_models.User, task_id: str) -> task_models.Task | None: await DemoTaskQueue._process_tasks(self) return next((t for t in DemoTaskQueue.tasks if t.user.name == user.name and t.id == task_id), None) - - async def get_tasks( - self : "DemoAdapter", - user: account_models.User - ) -> list[task_models.Task]: + async def get_tasks(self: "DemoAdapter", user: account_models.User) -> list[task_models.Task]: await DemoTaskQueue._process_tasks(self) return [t for t in DemoTaskQueue.tasks if t.user.name == user.name] - - async def put_task( - self: "DemoAdapter", - user: account_models.User, - resource: status_models.Resource, - task: str) -> str: + async def put_task(self: "DemoAdapter", user: account_models.User, resource: status_models.Resource, task: str) -> str: await DemoTaskQueue._process_tasks(self) return DemoTaskQueue._create_task(user, resource, task) @@ -1010,8 +889,8 @@ class DemoTask(BaseModel): resource: status_models.Resource user: account_models.User start: float - status: task_models.TaskStatus=task_models.TaskStatus.pending - result: str|None=None + status: task_models.TaskStatus = task_models.TaskStatus.pending + result: str | None = None class DemoTaskQueue: @@ -1036,7 +915,6 @@ async def _process_tasks(da: DemoAdapter): _tasks.append(t) DemoTaskQueue.tasks = _tasks - @staticmethod def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> str: task_id = f"task_{len(DemoTaskQueue.tasks)}" diff --git a/app/main.py b/app/main.py index fa3f1eda..f6eda7bf 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Main API application""" + import logging from fastapi import FastAPI from opentelemetry import trace @@ -24,10 +25,7 @@ # OpenTelemetry Tracing Configuration # ------------------------------------------------------------------ if config.OPENTELEMETRY_ENABLED: - resource = Resource.create({ - "service.name": "iri-facility-api", - "service.version": config.API_VERSION, - "service.endpoint": config.API_URL_ROOT}) + resource = Resource.create({"service.name": "iri-facility-api", "service.version": config.API_VERSION, "service.endpoint": config.API_URL_ROOT}) samplerate = "1.0" if config.OPENTELEMETRY_DEBUG else config.OTEL_SAMPLE_RATE provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(samplerate))) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index abd55ad5..c030d635 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -20,15 +20,16 @@ description="Get a list of capabilities at this facility.", responses=DEFAULT_RESPONSES, operation_id="getCapabilities", - response_model_exclude_none=True) + response_model_exclude_none=True, +) async def get_capabilities( - request : Request, - name : str = Query(default=None, min_length=1), + request: Request, + name: str = Query(default=None, min_length=1), modified_since: StrictDateTime = Query(default=None), - offset : int = Query(default=0, ge=0, le=1000), - limit : int = Query(default=100, ge=0, le=1000), - _forbid = Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), - ) -> list[Capability]: + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + _forbid=Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), +) -> list[Capability]: return await router.adapter.get_capabilities(name=name, modified_since=modified_since, offset=offset, limit=limit) @@ -40,11 +41,11 @@ async def get_capabilities( operation_id="getCapability", ) async def get_capability( - capability_id : str, - request : Request, + capability_id: str, + request: Request, modified_since: StrictDateTime = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("modified_since")), - ) -> Capability: + _forbid=Depends(forbidExtraQueryParams("modified_since")), +) -> Capability: caps = await router.adapter.get_capabilities(name=None, modified_since=modified_since, offset=0, limit=100) cc = next((c for c in caps if c.id == capability_id), None) if not cc: @@ -61,9 +62,9 @@ async def get_capability( operation_id="getProjects", ) async def get_projects( - request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> list[models.Project]: + request: Request, + _forbid=Depends(forbidExtraQueryParams()), +) -> list[models.Project]: user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") @@ -79,10 +80,10 @@ async def get_projects( operation_id="getProject", ) async def get_project( - project_id : str, - request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> models.Project: + project_id: str, + request: Request, + _forbid=Depends(forbidExtraQueryParams()), +) -> models.Project: user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") @@ -103,9 +104,9 @@ async def get_project( ) async def get_project_allocations( project_id: str, - request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> list[models.ProjectAllocation]: + request: Request, + _forbid=Depends(forbidExtraQueryParams()), +) -> list[models.ProjectAllocation]: user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") @@ -126,10 +127,10 @@ async def get_project_allocations( ) async def get_project_allocation( project_id: str, - project_allocation_id : str, - request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> models.ProjectAllocation: + project_allocation_id: str, + request: Request, + _forbid=Depends(forbidExtraQueryParams()), +) -> models.ProjectAllocation: user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") @@ -152,10 +153,10 @@ async def get_project_allocation( ) async def get_user_allocations( project_id: str, - project_allocation_id : str, - request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> list[models.UserAllocation]: + project_allocation_id: str, + request: Request, + _forbid=Depends(forbidExtraQueryParams()), +) -> list[models.UserAllocation]: user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") @@ -180,11 +181,11 @@ async def get_user_allocations( ) async def get_user_allocation( project_id: str, - project_allocation_id : str, - user_allocation_id : str, - request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ) -> models.UserAllocation: + project_allocation_id: str, + user_allocation_id: str, + request: Request, + _forbid=Depends(forbidExtraQueryParams()), +) -> models.UserAllocation: user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index fb2a66f3..98f815b0 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -8,42 +8,22 @@ class FacilityAdapter(AuthenticatedAdapter): """ Facility-specific code is handled by the implementation of this interface. - Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) + Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) to install your facility adapter before the API starts. """ @abstractmethod - async def get_capabilities( - self : "FacilityAdapter", - name : str | None = None, - modified_since : str | None = None, - offset : int = 0, - limit : int = 1000 - ) -> list[Capability]: + async def get_capabilities(self: "FacilityAdapter", name: str | None = None, modified_since: str | None = None, offset: int = 0, limit: int = 1000) -> list[Capability]: pass - @abstractmethod - async def get_projects( - self : "FacilityAdapter", - user: account_models.User - ) -> list[account_models.Project]: + async def get_projects(self: "FacilityAdapter", user: account_models.User) -> list[account_models.Project]: pass - @abstractmethod - async def get_project_allocations( - self : "FacilityAdapter", - project: account_models.Project, - user: account_models.User - ) -> list[account_models.ProjectAllocation]: + async def get_project_allocations(self: "FacilityAdapter", project: account_models.Project, user: account_models.User) -> list[account_models.ProjectAllocation]: pass - @abstractmethod - async def get_user_allocations( - self : "FacilityAdapter", - user: account_models.User, - project_allocation: account_models.ProjectAllocation - ) -> list[account_models.UserAllocation]: + async def get_user_allocations(self: "FacilityAdapter", user: account_models.User, project_allocation: account_models.ProjectAllocation) -> list[account_models.UserAllocation]: pass diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 6bde3be4..55406dc1 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -7,15 +7,17 @@ class User(IRIBaseModel): """A user of the facility""" + id: str name: str api_key: str - client_ip: str|None + client_ip: str | None # we could expose more fields here (eg. email) but it might be against policy class Project(IRIBaseModel): """A project and its users at a facility""" + id: str name: str description: str @@ -24,30 +26,30 @@ class Project(IRIBaseModel): class AllocationEntry(IRIBaseModel): """Base class for allocations.""" + allocation: float # how much this allocation can spend - usage: float # how much this allocation has spent + usage: float # how much this allocation has spent unit: AllocationUnit class ProjectAllocation(IRIBaseModel): """ - A project's allocation for a capability. (aka. repo) - This allocation is a piece of the total allocation for the capability. (eg. 5% of the total node hours of Perlmutter GPU nodes) - A project would at least have a storage and job repos, maybe more than 1 of each. + A project's allocation for a capability. (aka. repo) + This allocation is a piece of the total allocation for the capability. (eg. 5% of the total node hours of Perlmutter GPU nodes) + A project would at least have a storage and job repos, maybe more than 1 of each. """ + # how much this allocation can spend id: str project_id: str = Field(exclude=True) capability_id: str = Field(exclude=True) entries: list[AllocationEntry] - @computed_field(description="The list of past events in this incident") @property def project_uri(self) -> str: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/projects/{self.project_id}" - @computed_field(description="The list of past events in this incident") @property def capability_uri(self) -> str: @@ -56,16 +58,16 @@ def capability_uri(self) -> str: class UserAllocation(IRIBaseModel): """ - A user's allcation in a project. - This allocation is a piece of the project's allocation. + A user's allcation in a project. + This allocation is a piece of the project's allocation. """ + id: str project_id: str = Field(exclude=True) project_allocation_id: str = Field(exclude=True) user_id: str entries: list[AllocationEntry] - @computed_field(description="The list of past events in this incident") @property def project_allocation_uri(self) -> str: diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index d1eadf4b..0115e352 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,4 +1,5 @@ """Compute resource API router""" + from fastapi import Depends, HTTPException, Query, Request, status from ...types.http import forbidExtraQueryParams @@ -14,6 +15,7 @@ tags=["compute"], ) + @router.post( "/job/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -24,10 +26,10 @@ ) async def submit_job( resource_id: str, - job_spec : models.JobSpec, - request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ): + job_spec: models.JobSpec, + request: Request, + _forbid=Depends(forbidExtraQueryParams()), +): """ Submit a job on a compute resource @@ -49,15 +51,15 @@ async def submit_job( # TODO: this conflicts with PUT commented out while we finalize the API design -#@router.post( +# @router.post( # "/job/script/{resource_id:str}", # dependencies=[Depends(router.current_user)], # response_model=models.Job, # response_model_exclude_unset=True, # responses=DEFAULT_RESPONSES, # operation_id="launchJobScript", -#) -#async def submit_job_path( +# ) +# async def submit_job_path( # resource_id: str, # job_script_path : str, # request : Request, @@ -96,10 +98,10 @@ async def submit_job( async def update_job( resource_id: str, job_id: str, - job_spec : models.JobSpec, - request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ): + job_spec: models.JobSpec, + request: Request, + _forbid=Depends(forbidExtraQueryParams()), +): """ Update a previously submitted job for a resource. Note that only some attributes of a scheduled job can be updated. Check the facility documentation for details. @@ -129,13 +131,13 @@ async def update_job( operation_id="getJob", ) async def get_job_status( - resource_id : str, - job_id : str, - request : Request, - historical : StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + resource_id: str, + job_id: str, + request: Request, + historical: StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), include_spec: StrictHTTPBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), - _forbid = Depends(forbidExtraQueryParams("historical", "include_spec")), - ): + _forbid=Depends(forbidExtraQueryParams("historical", "include_spec")), +): """Get a job's status""" user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: @@ -159,15 +161,15 @@ async def get_job_status( operation_id="getJobs", ) async def get_job_statuses( - resource_id : str, - request : Request, - offset : int = Query(default=0, ge=0, le=1000), - limit : int = Query(default=100, ge=0, le=1000), - filters : dict[str, object] | None = None, - historical : StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + resource_id: str, + request: Request, + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + filters: dict[str, object] | None = None, + historical: StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), include_spec: StrictHTTPBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), - _forbid = Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), - ): + _forbid=Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), +): """Get multiple jobs' statuses""" user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: @@ -192,11 +194,11 @@ async def get_job_statuses( operation_id="cancelJob", ) async def cancel_job( - resource_id : str, - job_id : str, - request : Request, - _forbid = Depends(forbidExtraQueryParams()), - ): + resource_id: str, + job_id: str, + request: Request, + _forbid=Depends(forbidExtraQueryParams()), +): """Cancel a job""" user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: diff --git a/app/routers/compute/facility_adapter.py b/app/routers/compute/facility_adapter.py index 910cacd0..1370aedb 100644 --- a/app/routers/compute/facility_adapter.py +++ b/app/routers/compute/facility_adapter.py @@ -12,39 +12,18 @@ class FacilityAdapter(AuthenticatedAdapter): to install your facility adapter before the API starts. """ - @abstractmethod - async def submit_job( - self: "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - job_spec: compute_models.JobSpec - ) -> compute_models.Job: + async def submit_job(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec) -> compute_models.Job: pass - @abstractmethod - async def submit_job_script( - self: "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - job_script_path: str, - args: list[str] = [] - ) -> compute_models.Job: + async def submit_job_script(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_script_path: str, args: list[str] = []) -> compute_models.Job: pass - @abstractmethod - async def update_job( - self: "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - job_spec: compute_models.JobSpec, - job_id: str - ) -> compute_models.Job: + async def update_job(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, job_id: str) -> compute_models.Job: pass - @abstractmethod async def get_job( self: "FacilityAdapter", @@ -56,26 +35,19 @@ async def get_job( ) -> compute_models.Job: pass - @abstractmethod async def get_jobs( self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, - offset : int, - limit : int, + offset: int, + limit: int, filters: dict[str, object] | None = None, historical: bool = False, - include_spec: bool = False + include_spec: bool = False, ) -> list[compute_models.Job]: pass - @abstractmethod - async def cancel_job( - self: "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - job_id: str - ) -> bool: + async def cancel_job(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_id: str) -> bool: pass diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 87142a45..38eca02d 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -10,19 +10,21 @@ class ResourceSpec(IRIBaseModel): """ Specification of computational resources required for a job. """ + node_count: Annotated[int | None, Field(ge=1, description="Number of compute nodes to allocate")] = None process_count: Annotated[int | None, Field(ge=1, description="Total number of processes to launch")] = None processes_per_node: Annotated[int | None, Field(ge=1, description="Number of processes to launch per node")] = None cpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of CPU cores to allocate per process")] = None gpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of GPU cores to allocate per process")] = None exclusive_node_use: Annotated[StrictBool, Field(description="Whether to request exclusive use of allocated nodes")] = True - memory: Annotated[int | None, Field(ge=1,description="Amount of memory to allocate in bytes")] = None + memory: Annotated[int | None, Field(ge=1, description="Amount of memory to allocate in bytes")] = None class JobAttributes(IRIBaseModel): """ Additional attributes and scheduling parameters for a job. """ + duration: Annotated[int | None, Field(description="Duration in seconds", ge=1, examples=[30, 60, 120])] = None queue_name: Annotated[str | None, Field(min_length=1, description="Name of the queue or partition to submit the job to")] = None account: Annotated[str | None, Field(min_length=1, description="Account or project to charge for resource usage")] = None @@ -34,10 +36,12 @@ class VolumeMount(IRIBaseModel): """ Represents a volume mount for a container. """ + source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount")] target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted")] read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only")] = True + class Container(IRIBaseModel): """ Represents a container specification for job execution. @@ -47,6 +51,7 @@ class Container(IRIBaseModel): to determine if the container should be run with MPI support. The container should by default. be run with host networking. """ + image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] @@ -55,6 +60,7 @@ class JobSpec(IRIBaseModel): """ Specification for job. """ + model_config = ConfigDict(extra="forbid") executable: Annotated[str | None, Field(min_length=1, description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.")] = None container: Annotated[Container | None, Field(description="Container specification for containerized execution")] = None @@ -74,8 +80,8 @@ class JobSpec(IRIBaseModel): class CommandResult(IRIBaseModel): - status : str - result : str | None = None + status: str + result: str | None = None class JobState(IntEnum): @@ -114,18 +120,18 @@ class JobState(IntEnum): class JobStatus(IRIBaseModel): - state : JobState - time : float | None = None - message : str | None = None - exit_code : int | None = None - meta_data : dict[str, object] | None = None + state: JobState + time: float | None = None + message: str | None = None + exit_code: int | None = None + meta_data: dict[str, object] | None = None - @field_serializer('state') + @field_serializer("state") def serialize_state(self, state: JobState): return state.name class Job(IRIBaseModel): - id : str - status : JobStatus | None = None - job_spec : JobSpec | None = None + id: str + status: JobStatus | None = None + job_spec: JobSpec | None = None diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index bf781be8..1dcc5f6d 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -2,6 +2,7 @@ """ Default problem schema and example responses for various HTTP status codes. """ + import logging from urllib.parse import urlsplit, urlunsplit, quote from fastapi import FastAPI, HTTPException, Request @@ -9,6 +10,7 @@ from fastapi.exceptions import RequestValidationError from starlette.exceptions import HTTPException as StarletteHTTPException + def get_url_base(request: Request) -> str: """Return the base URL for the API.""" # If behind a proxy (and x-forwarded-* headers present), use the forwarded host and protocol @@ -16,6 +18,7 @@ def get_url_base(request: Request) -> str: proto = request.headers.get("x-forwarded-proto") or request.url.scheme return f"{proto}://{host}/problems" + def safe_instance_url(request: Request) -> str: """Return a URL-safe version of the request URL for the 'instance' field.""" parts = urlsplit(str(request.url)) @@ -27,9 +30,8 @@ def safe_instance_url(request: Request) -> str: return urlunsplit((parts.scheme, parts.netloc, safe_path, safe_query, safe_fragment)) -def problem_response(*, request: Request, status: int, - title, detail, problem_type: str, - invalid_params=None, extra_headers=None): + +def problem_response(*, request: Request, status: int, title, detail, problem_type: str, invalid_params=None, extra_headers=None): """Return a JSON problem response with the given status, title, and detail.""" instance = safe_instance_url(request) url_base = get_url_base(request) @@ -41,8 +43,7 @@ def problem_response(*, request: Request, status: int, if not isinstance(detail, str): if isinstance(detail, list): - detail = ", ".join(err.get("msg", str(err)) if isinstance(err, dict) else str(err) - for err in detail) + detail = ", ".join(err.get("msg", str(err)) if isinstance(err, dict) else str(err) for err in detail) else: detail = str(detail) @@ -58,17 +59,12 @@ def problem_response(*, request: Request, status: int, body["invalid_params"] = invalid_params headers = extra_headers or {} - return JSONResponse( - status_code=status, - content=body, - headers=headers, - media_type="application/problem+json" - ) - + return JSONResponse(status_code=status, content=body, headers=headers, media_type="application/problem+json") def install_error_handlers(app: FastAPI): """Install custom error handlers for the FastAPI app.""" + # 400 — VALIDATION ERRORS @app.exception_handler(RequestValidationError) async def validation_error_handler(request: Request, exc: RequestValidationError): @@ -95,10 +91,7 @@ async def validation_error_handler(request: Request, exc: RequestValidationError async def http_exception_handler(request: Request, exc: HTTPException): if exc.status_code == 304: - return JSONResponse( - status_code=304, - content=None, - headers=exc.headers or {}) + return JSONResponse(status_code=304, content=None, headers=exc.headers or {}) if exc.status_code == 401: return problem_response( @@ -229,25 +222,17 @@ async def global_handler(request: Request, exc: Exception): "status": 400, "detail": "modified_since must be in ISO 8601 format.", "instance": "/api/v1/status/resources?modified_since=BADVALUE", - "invalid_params": [ - {"name": "modified_since", "reason": "Invalid datetime format"} - ] + "invalid_params": [{"name": "modified_since", "reason": "Invalid datetime format"}], } -EXAMPLE_401 = { - "type": "https://iri.example.com/problems/unauthorized", - "title": "Unauthorized", - "status": 401, - "detail": "Bearer token is missing or invalid.", - "instance": "/api/v1/status/resources" -} +EXAMPLE_401 = {"type": "https://iri.example.com/problems/unauthorized", "title": "Unauthorized", "status": 401, "detail": "Bearer token is missing or invalid.", "instance": "/api/v1/status/resources"} EXAMPLE_403 = { "type": "https://iri.example.com/problems/forbidden", "title": "Forbidden", "status": 403, "detail": "Caller is authenticated but lacks required role.", - "instance": "/api/v1/status/resources" + "instance": "/api/v1/status/resources", } EXAMPLE_404 = { @@ -255,7 +240,7 @@ async def global_handler(request: Request, exc: Exception): "title": "Not Found", "status": 404, "detail": "The resource ID 'abc123' does not exist.", - "instance": "/api/v1/status/resources/abc123" + "instance": "/api/v1/status/resources/abc123", } EXAMPLE_405 = { @@ -263,7 +248,7 @@ async def global_handler(request: Request, exc: Exception): "title": "Method Not Allowed", "status": 405, "detail": "HTTP method TRACE is not allowed for this endpoint.", - "instance": "/api/v1/status/resources" + "instance": "/api/v1/status/resources", } EXAMPLE_409 = { @@ -271,7 +256,7 @@ async def global_handler(request: Request, exc: Exception): "title": "Conflict", "status": 409, "detail": "A job with this ID already exists.", - "instance": "/api/v1/compute/job/perlmutter/123" + "instance": "/api/v1/compute/job/perlmutter/123", } EXAMPLE_422 = { @@ -280,9 +265,7 @@ async def global_handler(request: Request, exc: Exception): "status": 422, "detail": "The PSIJ JobSpec is syntactically correct but invalid.", "instance": "/api/v1/compute/job/perlmutter", - "invalid_params": [ - {"name": "job_spec.executable", "reason": "Executable must be provided"} - ] + "invalid_params": [{"name": "job_spec.executable", "reason": "Executable must be provided"}], } EXAMPLE_500 = { @@ -290,7 +273,7 @@ async def global_handler(request: Request, exc: Exception): "title": "Internal Server Error", "status": 500, "detail": "An unexpected error occurred.", - "instance": "/api/v1/status/resources" + "instance": "/api/v1/status/resources", } EXAMPLE_501 = { @@ -298,7 +281,7 @@ async def global_handler(request: Request, exc: Exception): "title": "Not Implemented", "status": 501, "detail": "This functionality is not implemented.", - "instance": "/api/v1/status/resources" + "instance": "/api/v1/status/resources", } EXAMPLE_503 = { @@ -306,7 +289,7 @@ async def global_handler(request: Request, exc: Exception): "title": "Service Unavailable", "status": 503, "detail": "The service is temporarily unavailable.", - "instance": "/api/v1/status/resources" + "instance": "/api/v1/status/resources", } EXAMPLE_504 = { @@ -314,7 +297,7 @@ async def global_handler(request: Request, exc: Exception): "title": "Gateway Timeout", "status": 504, "detail": "The server did not receive a timely response.", - "instance": "/api/v1/status/resources" + "instance": "/api/v1/status/resources", } DEFAULT_RESPONSES = { @@ -327,7 +310,6 @@ async def global_handler(request: Request, exc: Exception): } }, }, - 401: { "description": "Unauthorized", "headers": { @@ -343,7 +325,6 @@ async def global_handler(request: Request, exc: Exception): } }, }, - 403: { "description": "Forbidden", "content": { @@ -353,7 +334,6 @@ async def global_handler(request: Request, exc: Exception): } }, }, - 404: { "description": "Not Found", "content": { @@ -363,7 +343,6 @@ async def global_handler(request: Request, exc: Exception): } }, }, - 405: { "description": "Method Not Allowed", "headers": { @@ -379,7 +358,6 @@ async def global_handler(request: Request, exc: Exception): } }, }, - 409: { "description": "Conflict", "content": { @@ -389,7 +367,6 @@ async def global_handler(request: Request, exc: Exception): } }, }, - 422: { "description": "Unprocessable Entity", "content": { @@ -399,7 +376,6 @@ async def global_handler(request: Request, exc: Exception): } }, }, - 500: { "description": "Internal Server Error", "content": { @@ -409,7 +385,6 @@ async def global_handler(request: Request, exc: Exception): } }, }, - 501: { "description": "Not Implemented", "content": { @@ -417,9 +392,8 @@ async def global_handler(request: Request, exc: Exception): "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_501, } - } + }, }, - 503: { "description": "Service Unavailable", "content": { @@ -427,9 +401,8 @@ async def global_handler(request: Request, exc: Exception): "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_503, } - } + }, }, - 504: { "description": "Gateway Timeout", "content": { @@ -437,8 +410,7 @@ async def global_handler(request: Request, exc: Exception): "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_504, } - } + }, }, - 304: {"description": "Not Modified"}, } diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 7a1a1ea1..03331cf8 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -6,19 +6,19 @@ from ..error_handlers import DEFAULT_RESPONSES from . import facility_adapter, models -router = iri_router.IriRouter(facility_adapter.FacilityAdapter, - prefix="/facility", - tags=["facility"]) +router = iri_router.IriRouter(facility_adapter.FacilityAdapter, prefix="/facility", tags=["facility"]) + @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") async def get_facility( request: Request, modified_since: StrictDateTime = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("modified_since")), - ) -> models.Facility: + _forbid=Depends(forbidExtraQueryParams("modified_since")), +) -> models.Facility: """Get facility information""" return await router.adapter.get_facility(modified_since=modified_since) + @router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") async def list_sites( request: Request, @@ -27,17 +27,18 @@ async def list_sites( offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), short_name: str = Query(default=None, min_length=1), - _forbid = Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), - )-> list[models.Site]: + _forbid=Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), +) -> list[models.Site]: """List sites""" return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + @router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") async def get_site( request: Request, site_id: str, modified_since: StrictDateTime = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("modified_since")), - )-> models.Site: + _forbid=Depends(forbidExtraQueryParams("modified_since")), +) -> models.Site: """Get site by ID""" return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index b7656404..e04e9cf8 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -11,21 +11,13 @@ class FacilityAdapter(AuthenticatedAdapter): """ @abstractmethod - async def get_facility( - self: "FacilityAdapter", - modified_since: str | None = None - ) -> facility_models.Facility | None: + async def get_facility(self: "FacilityAdapter", modified_since: str | None = None) -> facility_models.Facility | None: pass @abstractmethod async def list_sites( - self: "FacilityAdapter", - modified_since: str | None = None, - name: str | None = None, - offset: int | None = None, - limit: int | None = None, - short_name: str | None = None - ) -> list[facility_models.Site]: + self: "FacilityAdapter", modified_since: str | None = None, name: str | None = None, offset: int | None = None, limit: int | None = None, short_name: str | None = None + ) -> list[facility_models.Site]: pass @abstractmethod diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 021d9a24..c260b203 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,4 +1,5 @@ """Facility-related models.""" + from typing import List, Optional from pydantic import Field, HttpUrl @@ -9,6 +10,7 @@ class Site(NamedObject): def _self_path(self) -> str: return f"/facility/sites/{self.id}" + short_name: Optional[str] = Field(None, description="Common or short name of the Site.") operating_organization: str = Field(..., description="Organization operating the Site.") country_name: Optional[str] = Field(None, description="Country name of the Location.") @@ -23,7 +25,7 @@ def _self_path(self) -> str: @classmethod def find(cls, items, name=None, description=None, modified_since=None, short_name=None, country_name=None): - """ Find Locations matching the given criteria. """ + """Find Locations matching the given criteria.""" items = super().find(items, name=name, description=description, modified_since=modified_since) if short_name: items = [item for item in items if item.short_name == short_name] @@ -35,6 +37,7 @@ def find(cls, items, name=None, description=None, modified_since=None, short_nam class Facility(NamedObject): def _self_path(self) -> str: return "/facility" + short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") organization_name: Optional[str] = Field(None, description="Operating organization’s name.") support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index 636b0a9b..afa02907 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -26,192 +26,87 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def chmod( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PutFileChmodRequest - ) -> filesystem_models.PutFileChmodResponse: + self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChmodRequest + ) -> filesystem_models.PutFileChmodResponse: pass - @abstractmethod async def chown( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PutFileChownRequest - ) -> filesystem_models.PutFileChownResponse: + self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChownRequest + ) -> filesystem_models.PutFileChownResponse: pass - @abstractmethod async def ls( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - show_hidden: bool, - numeric_uid: bool, - recursive: bool, - dereference: bool + self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, show_hidden: bool, numeric_uid: bool, recursive: bool, dereference: bool ) -> filesystem_models.GetDirectoryLsResponse: pass - @abstractmethod - async def head( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - file_bytes: int, - lines: int, - skip_trailing: bool - ) -> Tuple[Any, int]: + async def head(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int, lines: int, skip_trailing: bool) -> Tuple[Any, int]: pass - @abstractmethod - async def tail( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - file_bytes: int | None, - lines: int | None, - skip_trailing: bool - ) -> Tuple[Any, int]: + async def tail(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int | None, lines: int | None, skip_trailing: bool) -> Tuple[Any, int]: pass - @abstractmethod - async def view( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - size: int, - offset: int - ) -> filesystem_models.GetViewFileResponse: + async def view(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, size: int, offset: int) -> filesystem_models.GetViewFileResponse: pass - @abstractmethod - async def checksum( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str - ) -> filesystem_models.GetFileChecksumResponse: + async def checksum(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileChecksumResponse: pass - @abstractmethod - async def file( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str - ) -> filesystem_models.GetFileTypeResponse: + async def file(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileTypeResponse: pass - @abstractmethod - async def stat( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - dereference: bool - ) -> filesystem_models.GetFileStatResponse: + async def stat(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, dereference: bool) -> filesystem_models.GetFileStatResponse: pass - @abstractmethod - async def rm( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str): + async def rm(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str): pass - @abstractmethod - async def mkdir( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostMakeDirRequest - ) -> filesystem_models.PostMkdirResponse: + async def mkdir(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest) -> filesystem_models.PostMkdirResponse: pass - @abstractmethod async def symlink( - self : "FacilityAdapter", + self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostFileSymlinkRequest, - ) -> filesystem_models.PostFileSymlinkResponse: + ) -> filesystem_models.PostFileSymlinkResponse: pass - @abstractmethod - async def download( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str - ) -> Any: + async def download(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> Any: pass - @abstractmethod - async def upload( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - path: str, - content: str - ) -> None: + async def upload(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, content: str) -> None: pass - @abstractmethod async def compress( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostCompressRequest - ) -> filesystem_models.PostCompressResponse: + self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCompressRequest + ) -> filesystem_models.PostCompressResponse: pass - @abstractmethod async def extract( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostExtractRequest - ) -> filesystem_models.PostExtractResponse: + self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostExtractRequest + ) -> filesystem_models.PostExtractResponse: pass - @abstractmethod - async def mv( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostMoveRequest - ) -> filesystem_models.PostMoveResponse: + async def mv(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMoveRequest) -> filesystem_models.PostMoveResponse: pass - @abstractmethod - async def cp( - self : "FacilityAdapter", - resource: status_models.Resource, - user: account_models.User, - request_model: filesystem_models.PostCopyRequest - ) -> filesystem_models.PostCopyResponse: + async def cp(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCopyRequest) -> filesystem_models.PostCopyResponse: pass diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index fb27427e..ba04d8db 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -6,21 +6,13 @@ # SPDX-License-Identifier: BSD-3-Clause import base64 from typing import Annotated -from fastapi import ( - Depends, - HTTPException, - status, - Query, - Request, - File, - UploadFile -) +from fastapi import Depends, HTTPException, status, Query, Request, File, UploadFile from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from ..status.status import router as status_router, models as status_models from ..account.account import models as account_models from ..task import facility_adapter as task_facility_adapter, models as task_models -from .import models, facility_adapter +from . import models, facility_adapter router = iri_router.IriRouter( @@ -30,10 +22,11 @@ tags=["filesystem"], ) + async def _user_resource( - resource_id: str, - request: Request, - ) -> tuple[account_models.User, status_models.Resource]: + resource_id: str, + request: Request, +) -> tuple[account_models.User, status_models.Resource]: user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") @@ -58,7 +51,7 @@ async def _user_resource( async def put_chmod( resource_id: str, request_model: models.PutFileChmodRequest, - request : Request, + request: Request, ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( @@ -69,8 +62,8 @@ async def put_chmod( command="chmod", args={ "request_model": request_model, - } - ) + }, + ), ) @@ -87,7 +80,7 @@ async def put_chmod( async def put_chown( resource_id: str, request_model: models.PutFileChownRequest, - request : Request, + request: Request, ) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( @@ -98,12 +91,11 @@ async def put_chown( command="chown", args={ "request_model": request_model, - } - ) + }, + ), ) - @router.get( "/file/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -116,7 +108,7 @@ async def put_chown( ) async def get_file( resource_id: str, - request : Request, + request: Request, path: Annotated[str, Query(description="A file or folder path")], ) -> str: user, resource = await _user_resource(resource_id, request) @@ -128,8 +120,8 @@ async def get_file( command="file", args={ "path": path, - } - ) + }, + ), ) @@ -145,7 +137,7 @@ async def get_file( ) async def get_stat( resource_id: str, - request : Request, + request: Request, path: Annotated[str, Query(description="A file or folder path")], dereference: Annotated[bool, Query(description="Follow symbolic links")] = False, ) -> str: @@ -159,8 +151,8 @@ async def get_stat( args={ "path": path, "dereference": dereference, - } - ) + }, + ), ) @@ -176,7 +168,7 @@ async def get_stat( ) async def post_mkdir( resource_id: str, - request : Request, + request: Request, request_model: models.PostMakeDirRequest, ) -> str: user, resource = await _user_resource(resource_id, request) @@ -188,12 +180,11 @@ async def post_mkdir( command="mkdir", args={ "request_model": request_model, - } - ) + }, + ), ) - @router.post( "/symlink/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -206,7 +197,7 @@ async def post_mkdir( ) async def post_symlink( resource_id: str, - request : Request, + request: Request, request_model: models.PostFileSymlinkRequest, ) -> str: user, resource = await _user_resource(resource_id, request) @@ -218,8 +209,8 @@ async def post_symlink( command="symlink", args={ "request_model": request_model, - } - ) + }, + ), ) @@ -236,17 +227,11 @@ async def post_symlink( ) async def get_ls_async( resource_id: str, - request : Request, + request: Request, path: Annotated[str, Query(description="The path to list")], - show_hidden: Annotated[ - bool, Query(alias="showHidden", description="Show hidden files") - ] = False, - numeric_uid: Annotated[ - bool, Query(alias="numericUid", description="List numeric user and group IDs") - ] = False, - recursive: Annotated[ - bool, Query(alias="recursive", description="Recursively list files and folders") - ] = False, + show_hidden: Annotated[bool, Query(alias="showHidden", description="Show hidden files")] = False, + numeric_uid: Annotated[bool, Query(alias="numericUid", description="List numeric user and group IDs")] = False, + recursive: Annotated[bool, Query(alias="recursive", description="Recursively list files and folders")] = False, dereference: Annotated[ bool, Query( @@ -260,16 +245,8 @@ async def get_ls_async( user=user, resource=resource, task=task_models.TaskCommand( - router=router.get_router_name(), - command="ls", - args={ - "path": path, - "show_hidden": show_hidden, - "numeric_uid": numeric_uid, - "recursive": recursive, - "dereference": dereference - } - ) + router=router.get_router_name(), command="ls", args={"path": path, "show_hidden": show_hidden, "numeric_uid": numeric_uid, "recursive": recursive, "dereference": dereference} + ), ) @@ -285,7 +262,7 @@ async def get_ls_async( ) async def get_head( resource_id: str, - request : Request, + request: Request, path: Annotated[str, Query(description="File path")], # TODO Should we allow bytes and lines to be strings? The head allows the following: # NUM may have a multiplier suffix: b 512, kB 1000, K 1024, MB @@ -309,21 +286,14 @@ async def get_head( bool, Query( alias="skipTrailing", - description=( - "The output will be the whole file, without the last NUM " - "bytes/lines of each file. NUM should be specified in the " - "respective argument through `bytes` or `lines`." - ), + description=("The output will be the whole file, without the last NUM bytes/lines of each file. NUM should be specified in the respective argument through `bytes` or `lines`."), ), ] = False, ) -> str: user, resource = await _user_resource(resource_id, request) # Enforce that exactly one of `bytes` or `lines` is specified if (file_bytes is None and lines is None) or (file_bytes is not None and lines is not None): - raise HTTPException( - status_code=400, - detail="Exactly one of `bytes` or `lines` must be specified." - ) + raise HTTPException(status_code=400, detail="Exactly one of `bytes` or `lines` must be specified.") return await router.task_adapter.put_task( user=user, resource=resource, @@ -335,8 +305,8 @@ async def get_head( "file_bytes": file_bytes, "lines": lines, "skip_trailing": skip_trailing, - } - ) + }, + ), ) @@ -352,14 +322,11 @@ async def get_head( ) async def get_view( resource_id: str, - request : Request, + request: Request, path: Annotated[str, Query(description="File path")], - - size: Annotated[int, Query(description="Value, in bytes, of the size of data to be retrieved from the file.", - ge=1, le=facility_adapter.OPS_SIZE_LIMIT)] = facility_adapter.OPS_SIZE_LIMIT, - - offset: Annotated[int, Query( description="Value in bytes of the offset.", ge=0)] = 0 - ) -> str: + size: Annotated[int, Query(description="Value, in bytes, of the size of data to be retrieved from the file.", ge=1, le=facility_adapter.OPS_SIZE_LIMIT)] = facility_adapter.OPS_SIZE_LIMIT, + offset: Annotated[int, Query(description="Value in bytes of the offset.", ge=0)] = 0, +) -> str: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( @@ -372,8 +339,8 @@ async def get_view( "path": path, "size": size, "offset": offset, - } - ) + }, + ), ) @@ -389,11 +356,11 @@ async def get_view( ) async def get_tail( resource_id: str, - request : Request, + request: Request, path: Annotated[str, Query(description="File path", min_length=1)], - file_bytes: Annotated[int, Query(alias="bytes", - description="The output will be the last NUM bytes of each file.", - ge=1), + file_bytes: Annotated[ + int, + Query(alias="bytes", description="The output will be the last NUM bytes of each file.", ge=1), ] = None, lines: Annotated[ int, @@ -406,21 +373,14 @@ async def get_tail( bool, Query( alias="skipHeading", - description=( - "The output will be the whole file, without the first NUM " - "bytes/lines of each file. NUM should be specified in the " - "respective argument through `bytes` or `lines`." - ), + description=("The output will be the whole file, without the first NUM bytes/lines of each file. NUM should be specified in the respective argument through `bytes` or `lines`."), ), ] = False, ) -> str: user, resource = await _user_resource(resource_id, request) # Enforce that exactly one of `bytes` or `lines` is specified if (file_bytes is None and lines is None) or (file_bytes is not None and lines is not None): - raise HTTPException( - status_code=400, - detail="Exactly one of `bytes` or `lines` must be specified." - ) + raise HTTPException(status_code=400, detail="Exactly one of `bytes` or `lines` must be specified.") return await router.task_adapter.put_task( user=user, resource=resource, @@ -432,9 +392,8 @@ async def get_tail( "file_bytes": file_bytes, "lines": lines, "skip_heading": skip_heading, - - } - ) + }, + ), ) @@ -450,7 +409,7 @@ async def get_tail( ) async def get_checksum( resource_id: str, - request : Request, + request: Request, path: Annotated[str, Query(description="Target system")], ) -> str: user, resource = await _user_resource(resource_id, request) @@ -462,10 +421,11 @@ async def get_checksum( command="checksum", args={ "path": path, - } - ) + }, + ), ) + @router.delete( "/rm/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -476,7 +436,7 @@ async def get_checksum( ) async def delete_rm( resource_id: str, - request : Request, + request: Request, path: Annotated[str, Query(description="The path to delete")], ) -> str: user, resource = await _user_resource(resource_id, request) @@ -488,8 +448,8 @@ async def delete_rm( command="rm", args={ "path": path, - } - ) + }, + ), ) @@ -505,7 +465,7 @@ async def delete_rm( ) async def post_compress( resource_id: str, - request : Request, + request: Request, request_model: models.PostCompressRequest, ) -> str: user, resource = await _user_resource(resource_id, request) @@ -517,8 +477,8 @@ async def post_compress( command="compress", args={ "request_model": request_model, - } - ) + }, + ), ) @@ -534,7 +494,7 @@ async def post_compress( ) async def post_extract( resource_id: str, - request : Request, + request: Request, request_model: models.PostExtractRequest, ) -> str: user, resource = await _user_resource(resource_id, request) @@ -546,8 +506,8 @@ async def post_extract( command="extract", args={ "request_model": request_model, - } - ) + }, + ), ) @@ -563,7 +523,7 @@ async def post_extract( ) async def move_mv( resource_id: str, - request : Request, + request: Request, request_model: models.PostMoveRequest, ) -> str: user, resource = await _user_resource(resource_id, request) @@ -575,8 +535,8 @@ async def move_mv( command="mv", args={ "request_model": request_model, - } - ) + }, + ), ) @@ -592,7 +552,7 @@ async def move_mv( ) async def post_cp( resource_id: str, - request : Request, + request: Request, request_model: models.PostCopyRequest, ) -> str: user, resource = await _user_resource(resource_id, request) @@ -604,10 +564,11 @@ async def post_cp( command="cp", args={ "request_model": request_model, - } - ) + }, + ), ) + @router.get( "/download/{resource_id:str}", dependencies=[Depends(router.current_user)], @@ -620,7 +581,7 @@ async def post_cp( ) async def get_download( resource_id: str, - request : Request, + request: Request, path: Annotated[str, Query(description="A file to download")], ) -> str: user, resource = await _user_resource(resource_id, request) @@ -632,8 +593,8 @@ async def get_download( command="download", args={ "path": path, - } - ) + }, + ), ) @@ -649,10 +610,8 @@ async def get_download( ) async def post_upload( resource_id: str, - request : Request, - path: Annotated[ - str, Query(description="Specify path where file should be uploaded.") - ], + request: Request, + path: Annotated[str, Query(description="Specify path where file should be uploaded.")], file: UploadFile = File(description="File to be uploaded as `multipart/form-data`"), ) -> str: user, resource = await _user_resource(resource_id, request) @@ -672,7 +631,7 @@ async def post_upload( command="upload", args={ "path": path, - "content": base64.b64encode(raw_content).decode('utf-8'), - } - ) + "content": base64.b64encode(raw_content).decode("utf-8"), + }, + ), ) diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index d32ad943..5c2cc0a4 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -1,5 +1,5 @@ # Copied from: https://github.com/eth-cscs/firecrest-v2/blob/master/src/firecrest/filesystem/ops/models.py -# +# # Copyright (c) 2025, ETH Zurich. All rights reserved. # # Please, refer to the LICENSE file in the root directory. @@ -17,6 +17,7 @@ class CompressionType(str, Enum): gzip = "gzip" xz = "xz" + class ContentUnit(str, Enum): lines = "lines" bytes = "bytes" @@ -30,6 +31,7 @@ class CamelModel(BaseModel): validate_assignment=True, ) + class File(CamelModel): name: str type: str @@ -110,19 +112,12 @@ class PatchFileMetadataResponse(CamelModel): class FilesystemRequestBase(CamelModel): - path: Optional[str] = Field( - validation_alias=AliasChoices("sourcePath", "source_path"), - example="/home/user/dir" - ) + path: Optional[str] = Field(validation_alias=AliasChoices("sourcePath", "source_path"), example="/home/user/dir") class PutFileChmodRequest(FilesystemRequestBase): mode: str = Field(..., description="Mode in octal permission format") - model_config = { - "json_schema_extra": { - "examples": [{"path": "/home/user/dir/file.out", "mode": "777"}] - } - } + model_config = {"json_schema_extra": {"examples": [{"path": "/home/user/dir/file.out", "mode": "777"}]}} class PutFileChmodResponse(CamelModel): @@ -130,12 +125,8 @@ class PutFileChmodResponse(CamelModel): class PutFileChownRequest(FilesystemRequestBase): - owner: Optional[str] = Field( - default="", description="User name of the new user owner of the file" - ) - group: Optional[str] = Field( - default="", description="Group name of the new group owner of the file" - ) + owner: Optional[str] = Field(default="", description="User name of the new user owner of the file") + group: Optional[str] = Field(default="", description="Group name of the new group owner of the file") model_config = { "json_schema_extra": { "examples": [ @@ -158,20 +149,12 @@ class PostMakeDirRequest(FilesystemRequestBase): default=False, description="If set to `true` creates all its parent directories if they do not already exist", ) - model_config = { - "json_schema_extra": { - "examples": [{"path": "/home/user/dir/newdir", "parent": "true"}] - } - } + model_config = {"json_schema_extra": {"examples": [{"path": "/home/user/dir/newdir", "parent": "true"}]}} class PostFileSymlinkRequest(FilesystemRequestBase): link_path: str = Field(..., description="Path to the new symlink") - model_config = { - "json_schema_extra": { - "examples": [{"path": "/home/user/dir", "link_path": "/home/user/newlink"}] - } - } + model_config = {"json_schema_extra": {"examples": [{"path": "/home/user/dir", "link_path": "/home/user/newlink"}]}} class PostFileSymlinkResponse(CamelModel): @@ -192,9 +175,7 @@ class PostCompressResponse(CamelModel): class PostCompressRequest(FilesystemRequestBase): target_path: str = Field(..., description="Path to the compressed file") - match_pattern: Optional[str] = Field( - default=None, description="Regex pattern to filter files to compress" - ) + match_pattern: Optional[str] = Field(default=None, description="Regex pattern to filter files to compress") dereference: Optional[bool] = Field( default=False, description="If set to `true`, it follows symbolic links and archive the files they point to instead of the links themselves.", @@ -223,9 +204,7 @@ class PostExtractResponse(CamelModel): class PostExtractRequest(FilesystemRequestBase): - target_path: str = Field( - ..., description="Path to the directory where to extract the compressed file" - ) + target_path: str = Field(..., description="Path to the directory where to extract the compressed file") compression: Optional[CompressionType] = Field( default="gzip", description="Defines the type of compression to be used. By default gzip is used.", @@ -247,10 +226,7 @@ class PostCopyRequest(FilesystemRequestBase): target_path: str = Field(..., description="Target path of the copy operation") dereference: Optional[bool] = Field( default=False, - description=( - "If set to `true`, it follows symbolic links and copies the " - "files they point to instead of the links themselves." - ), + description=("If set to `true`, it follows symbolic links and copies the files they point to instead of the links themselves."), ) model_config = { "json_schema_extra": { @@ -285,4 +261,3 @@ class PostMoveRequest(FilesystemRequestBase): class PostMoveResponse(CamelModel): output: Optional[File] - diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index be317624..950f68b6 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -11,14 +11,14 @@ bearer_token = APIKeyHeader(name="Authorization") -def get_client_ip(request : Request) -> str|None: +def get_client_ip(request: Request) -> str | None: forwarded_for = request.headers.get("X-Forwarded-For") if forwarded_for: return forwarded_for.split(",")[0].strip() else: - ip_addr = request.headers.get('HTTP_X_REAL_IP') + ip_addr = request.headers.get("HTTP_X_REAL_IP") if not ip_addr: - ip_addr = request.headers.get('x-real-ip') + ip_addr = request.headers.get("x-real-ip") if not ip_addr: ip_addr = request.client.host return ip_addr @@ -38,16 +38,14 @@ def __init__(self, router_adapter=None, task_router_adapter=None, **kwargs): if task_router_adapter: self.task_adapter = IriRouter.create_adapter("task", task_router_adapter) if not self.task_adapter: - logging.getLogger().info(f"Hiding {router_name} because \"task\" adapter was not found") + logging.getLogger().info(f'Hiding {router_name} because "task" adapter was not found') self.include_in_schema = False - def get_router_name(self): return self.prefix.replace("/", "").strip() - @staticmethod - def _get_adapter_name(router_name: str) -> str|None: + def _get_adapter_name(router_name: str) -> str | None: """Return the adapter name, or None if it's not configured and IRI_SHOW_MISSING_ROUTES is true""" # if there is no adapter specified for this router, # and IRI_SHOW_MISSING_ROUTES is not true, @@ -59,7 +57,6 @@ def _get_adapter_name(router_name: str) -> str|None: # find and load the actual implementation return os.environ.get(env_var, "app.demo_adapter.DemoAdapter") - @staticmethod def create_adapter(router_name, router_adapter): # Load the facility-specific adapter @@ -67,7 +64,6 @@ def create_adapter(router_name, router_adapter): if not adapter_name: return None - parts = adapter_name.rsplit(".", 1) module = importlib.import_module(parts[0]) AdapterClass = getattr(module, parts[1]) @@ -77,10 +73,9 @@ def create_adapter(router_name, router_adapter): # assign it return AdapterClass() - async def current_user( self, - request : Request, + request: Request, api_key: str = Depends(bearer_token), ): user_id = None @@ -97,29 +92,18 @@ async def current_user( class AuthenticatedAdapter(ABC): - @abstractmethod - async def get_current_user( - self : "AuthenticatedAdapter", - api_key: str, - client_ip: str|None - ) -> str: + async def get_current_user(self: "AuthenticatedAdapter", api_key: str, client_ip: str | None) -> str: """ - Decode the api_key and return the authenticated user's id. - This method is not called directly, rather authorized endpoints "depend" on it. - (https://fastapi.tiangolo.com/tutorial/dependencies/) + Decode the api_key and return the authenticated user's id. + This method is not called directly, rather authorized endpoints "depend" on it. + (https://fastapi.tiangolo.com/tutorial/dependencies/) """ pass - @abstractmethod - async def get_user( - self : "AuthenticatedAdapter", - user_id: str, - api_key: str, - client_ip: str|None - ) -> User: + async def get_user(self: "AuthenticatedAdapter", user_id: str, api_key: str, client_ip: str | None) -> User: """ - Retrieve additional user information (name, email, etc.) for the given user_id. + Retrieve additional user information (name, email, etc.) for the given user_id. """ pass diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 6be6c883..f0b9e9f3 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -10,85 +10,69 @@ class FacilityAdapter(ABC): """ Facility-specific code is handled by the implementation of this interface. - Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) + Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) to install your facility adapter before the API starts. """ - @abstractmethod async def get_resources( - self : "FacilityAdapter", - offset : int, - limit : int, - name : str | None = None, - description : str | None = None, - group : str | None = None, - modified_since : datetime.datetime | None = None, + self: "FacilityAdapter", + offset: int, + limit: int, + name: str | None = None, + description: str | None = None, + group: str | None = None, + modified_since: datetime.datetime | None = None, resource_type: status_models.ResourceType = Query(default=None), current_status: status_models.Status = Query(default=None), capability: Capability | None = None, - site_id: str | None = None - ) -> list[status_models.Resource]: + site_id: str | None = None, + ) -> list[status_models.Resource]: pass - @abstractmethod - async def get_resource( - self : "FacilityAdapter", - id_ : str - ) -> status_models.Resource: + async def get_resource(self: "FacilityAdapter", id_: str) -> status_models.Resource: pass - @abstractmethod async def get_events( - self : "FacilityAdapter", - incident_id : str, - offset : int, - limit : int, - resource_id : str | None = None, - name : str | None = None, - description : str | None = None, - status : status_models.Status | None = None, - from_ : datetime.datetime | None = None, - to : datetime.datetime | None = None, - time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None - ) -> list[status_models.Event]: + self: "FacilityAdapter", + incident_id: str, + offset: int, + limit: int, + resource_id: str | None = None, + name: str | None = None, + description: str | None = None, + status: status_models.Status | None = None, + from_: datetime.datetime | None = None, + to: datetime.datetime | None = None, + time_: datetime.datetime | None = None, + modified_since: datetime.datetime | None = None, + ) -> list[status_models.Event]: pass - @abstractmethod - async def get_event( - self : "FacilityAdapter", - incident_id : str, - id_ : str - ) -> status_models.Event: + async def get_event(self: "FacilityAdapter", incident_id: str, id_: str) -> status_models.Event: pass - @abstractmethod async def get_incidents( - self : "FacilityAdapter", - offset : int, - limit : int, - name : str | None = None, - description : str | None = None, - status : status_models.Status | None = None, - type_ : status_models.IncidentType | None = None, - from_ : datetime.datetime | None = None, - to : datetime.datetime | None = None, - time_ : datetime.datetime | None = None, - modified_since : datetime.datetime | None = None, - resource_id : str | None = None, - resolution: status_models.Resolution | None = None - ) -> list[status_models.Incident]: + self: "FacilityAdapter", + offset: int, + limit: int, + name: str | None = None, + description: str | None = None, + status: status_models.Status | None = None, + type_: status_models.IncidentType | None = None, + from_: datetime.datetime | None = None, + to: datetime.datetime | None = None, + time_: datetime.datetime | None = None, + modified_since: datetime.datetime | None = None, + resource_id: str | None = None, + resolution: status_models.Resolution | None = None, + ) -> list[status_models.Incident]: pass - @abstractmethod - async def get_incident( - self : "FacilityAdapter", - id_ : str - ) -> status_models.Incident: + async def get_incident(self: "FacilityAdapter", id_: str) -> status_models.Incident: pass diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 7f0d51d0..566a8106 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -9,8 +9,8 @@ class Link(BaseModel): - rel : str - href : str + rel: str + href: str class Status(enum.Enum): @@ -31,9 +31,8 @@ class ResourceType(enum.Enum): class Resource(NamedObject): - def _self_path(self) -> str: - """ Return the API path for this resource. """ + """Return the API path for this resource.""" return f"/status/resources/{self.id}" # NOTE (TBR): If site_id is required, then located_at_uri should be also required. This can be easily identified by Site.self_uri @@ -45,17 +44,14 @@ def _self_path(self) -> str: resource_type: ResourceType located_at_uri: Optional[HttpUrl] = Field(None, description="Resource located at specific Site") - - @computed_field(description="The list of capabilities in this resource") @property def capability_uris(self) -> list[str]: - """ Return the list of capability URIs for this resource. """ + """Return the list of capability URIs for this resource.""" return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] @classmethod - def find(cls, items, name=None, description=None, modified_since=None, group=None, - resource_type=None, current_status=None, capability=None, site_id=None) -> list: + def find(cls, items, name=None, description=None, modified_since=None, group=None, resource_type=None, current_status=None, capability=None, site_id=None) -> list: items = super().find(items, name=name, description=description, modified_since=modified_since) if group: items = [item for item in items if item.group == group] @@ -66,16 +62,15 @@ def find(cls, items, name=None, description=None, modified_since=None, group=Non if current_status: items = [item for item in items if item.current_status == current_status] if capability: - items = [item for item in items - if any(cap_id in item.capability_ids for cap_id in capability)] + items = [item for item in items if any(cap_id in item.capability_ids for cap_id in capability)] if site_id: items = [item for item in items if item.site_id == site_id] return items -class Event(NamedObject): +class Event(NamedObject): def _self_path(self) -> str: - """ Return the API path for this event. """ + """Return the API path for this event.""" return f"/status/incidents/{self.incident_id}/events/{self.id}" @field_validator("occurred_at", mode="before") @@ -83,27 +78,25 @@ def _self_path(self) -> str: def _norm_dt_field(cls, v): return cls.normalize_dt(v) - occurred_at : datetime.datetime - status : Status - resource_id : str = Field(exclude=True) - incident_id : str | None = Field(exclude=True, default=None) + occurred_at: datetime.datetime + status: Status + resource_id: str = Field(exclude=True) + incident_id: str | None = Field(exclude=True, default=None) @computed_field(description="The resource belonging to this event") @property def resource_uri(self) -> str: - """ Return the resource URI for this event. """ + """Return the resource URI for this event.""" return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.resource_id}" @computed_field(description="The event's incident") @property - def incident_uri(self) -> str|None: - """ Return the incident URI for this event. """ + def incident_uri(self) -> str | None: + """Return the incident URI for this event.""" return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}" if self.incident_id else None - @classmethod - def find(cls, items, name=None, description=None, modified_since=None, - resource_id=None, status=None, from_=None, to=None, time_=None) -> list: + def find(cls, items, name=None, description=None, modified_since=None, resource_id=None, status=None, from_=None, to=None, time_=None) -> list: items = super().find(items, name=name, description=description, modified_since=modified_since) if resource_id: @@ -140,11 +133,9 @@ class Resolution(enum.Enum): pending = "pending" - class Incident(NamedObject): - def _self_path(self) -> str: - """ Return the API path for this incident. """ + """Return the API path for this incident.""" return f"/status/incidents/{self.id}" @field_validator("start", "end", mode="before") @@ -152,29 +143,28 @@ def _self_path(self) -> str: def _norm_dt_field(cls, v): return cls.normalize_dt(v) - status : Status - resource_ids : list[str] = Field(default_factory=list, exclude=True) - event_ids : list[str] = Field(default_factory=list, exclude=True) - start : datetime.datetime - end : datetime.datetime | None - type : IncidentType - resolution : Resolution + status: Status + resource_ids: list[str] = Field(default_factory=list, exclude=True) + event_ids: list[str] = Field(default_factory=list, exclude=True) + start: datetime.datetime + end: datetime.datetime | None + type: IncidentType + resolution: Resolution @computed_field(description="The list of past events in this incident") @property def event_uris(self) -> list[str]: - """ Return the list of event URIs for this incident. """ + """Return the list of event URIs for this incident.""" return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}/events/{e}" for e in self.event_ids] @computed_field(description="The list of resources that may be impacted by this incident") @property def resource_uris(self) -> list[str]: - """ Return the list of resource URIs for this incident. """ + """Return the list of resource URIs for this incident.""" return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] @classmethod - def find(cls, items, name=None, description=None, modified_since=None, status=None, - type_=None, from_= None, to = None, time_ = None, resource_id = None, resolution=None) -> list: + def find(cls, items, name=None, description=None, modified_since=None, status=None, type_=None, from_=None, to=None, time_=None, resource_id=None, resolution=None) -> list: items = super().find(items, name=name, description=description, modified_since=modified_since) if resource_id: @@ -196,6 +186,5 @@ def find(cls, items, name=None, description=None, modified_since=None, status=No items = [e for e in items if e.end and e.end < to] if time_: - items = [e for e in items - if e.start <= time_ and (e.end is None or e.end > time_)] - return items \ No newline at end of file + items = [e for e in items if e.start <= time_ and (e.end is None or e.end > time_)] + return items diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 6ffa0bad..e313d3ee 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -1,4 +1,4 @@ -from typing import Annotated, List, Optional +from typing import List from fastapi import Depends, HTTPException, Query, Request @@ -14,29 +14,31 @@ tags=["status"], ) + @router.get( "/resources", summary="Get all resources", description="Get a list of all resources at this facility. You can optionally filter the returned list by specifying attribtes.", responses=DEFAULT_RESPONSES, operation_id="getResources", - response_model_exclude_none=True + response_model_exclude_none=True, ) async def get_resources( - request : Request, - name : str = Query(default=None, min_length=1), - description : str = Query(default=None, min_length=1), - group : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0, le=1000), - limit : int = Query(default=100, ge=0, le=1000), + request: Request, + name: str = Query(default=None, min_length=1), + description: str = Query(default=None, min_length=1), + group: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), current_status: models.Status = Query(default=None), capability: List[AllocationUnit] = Query(default=None, min_length=1), - _forbid = Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), - ) -> list[models.Resource]: - return await router.adapter.get_resources(offset=offset, limit=limit, name=name, description=description, group=group, modified_since=modified_since, - resource_type=resource_type, current_status=current_status, capability=capability) + _forbid=Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), +) -> list[models.Resource]: + return await router.adapter.get_resources( + offset=offset, limit=limit, name=name, description=description, group=group, modified_since=modified_since, resource_type=resource_type, current_status=current_status, capability=capability + ) @router.get( @@ -47,16 +49,16 @@ async def get_resources( operation_id="getResource", ) async def get_resource( - request : Request, - resource_id : str, - ) -> models.Resource: + request: Request, + resource_id: str, +) -> models.Resource: item = await router.adapter.get_resource(resource_id) if not item: raise HTTPException(status_code=404, detail="Item not found") return item -@router.get( +@router.get( "/incidents", summary="Get all incidents without their events", description="Get a list of all incidents. Each incident will be returned without its events. You can optionally filter the returned list by specifying attributes.", @@ -64,24 +66,53 @@ async def get_resource( operation_id="getIncidents", ) async def get_incidents( - request : Request, - name : str = Query(default=None, min_length=1), - description : str = Query(default=None, min_length=1), - status : models.Status = Query(default=None), - type_: models.IncidentType = Query(alias="type", default=None), + request: Request, + name: str = Query(default=None, min_length=1), + description: str = Query(default=None, min_length=1), + status: models.Status = Query(default=None), + type_: models.IncidentType = Query(alias="type", default=None), from_: StrictDateTime = Query(alias="from", default=None), - time_ : StrictDateTime = Query(alias="time", default=None), - to : StrictDateTime = Query(default=None), - modified_since : StrictDateTime = Query(default=None), - resource_id : str = Query(default=None, min_length=1), - offset : int = Query(default=0, ge=0, le=1000), - limit : int = Query(default=100, ge=0, le=1000), - resolution : models.Resolution = Query(default=None), - _forbid = Depends(forbidExtraQueryParams("name", "description", "status", "type", "from", "to", "time", "modified_since", "resource_id", - "offset", "limit", "resolution", "resource_uris", "event_uris", multiParams={"resource_uris", "event_uris"})), - ) -> list[models.Incident]: - return await router.adapter.get_incidents(offset=offset, limit=limit, name=name, description=description, status=status, type_=type_, from_=from_, to=to, - time_=time_, modified_since=modified_since, resource_id=resource_id, resolution=resolution) + time_: StrictDateTime = Query(alias="time", default=None), + to: StrictDateTime = Query(default=None), + modified_since: StrictDateTime = Query(default=None), + resource_id: str = Query(default=None, min_length=1), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + resolution: models.Resolution = Query(default=None), + _forbid=Depends( + forbidExtraQueryParams( + "name", + "description", + "status", + "type", + "from", + "to", + "time", + "modified_since", + "resource_id", + "offset", + "limit", + "resolution", + "resource_uris", + "event_uris", + multiParams={"resource_uris", "event_uris"}, + ) + ), +) -> list[models.Incident]: + return await router.adapter.get_incidents( + offset=offset, + limit=limit, + name=name, + description=description, + status=status, + type_=type_, + from_=from_, + to=to, + time_=time_, + modified_since=modified_since, + resource_id=resource_id, + resolution=resolution, + ) @router.get( @@ -90,12 +121,8 @@ async def get_incidents( description="Get a specific incident for a given id. The incident's events will also be included. You can optionally filter the returned list by specifying attributes.", responses=DEFAULT_RESPONSES, operation_id="getIncident", - ) -async def get_incident( - request : Request, - incident_id : str - ) -> models.Incident: +async def get_incident(request: Request, incident_id: str) -> models.Incident: item = await router.adapter.get_incident(incident_id) if not item: raise HTTPException(status_code=404, detail="Item not found") @@ -110,22 +137,23 @@ async def get_incident( operation_id="getEventsByIncident", ) async def get_events( - request : Request, - incident_id : str, - resource_id : str = Query(default=None, min_length=1), - name : str = Query(default=None, min_length=1), - description : str = Query(default=None, min_length=1), - status : models.Status = Query(default=None), + request: Request, + incident_id: str, + resource_id: str = Query(default=None, min_length=1), + name: str = Query(default=None, min_length=1), + description: str = Query(default=None, min_length=1), + status: models.Status = Query(default=None), from_: StrictDateTime = Query(alias="from", default=None), - time_ : StrictDateTime = Query(alias="time", default=None), - to : StrictDateTime = Query(default=None), - modified_since : StrictDateTime = Query(default=None), - offset : int = Query(default=0, ge=0, le=1000), - limit : int = Query(default=100, ge=0, le=1000), - _forbid = Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), - ) -> list[models.Event]: - return await router.adapter.get_events(incident_id, offset=offset, limit=limit, resource_id=resource_id, name=name, description=description, status=status, - from_=from_, to=to, time_=time_, modified_since=modified_since) + time_: StrictDateTime = Query(alias="time", default=None), + to: StrictDateTime = Query(default=None), + modified_since: StrictDateTime = Query(default=None), + offset: int = Query(default=0, ge=0, le=1000), + limit: int = Query(default=100, ge=0, le=1000), + _forbid=Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), +) -> list[models.Event]: + return await router.adapter.get_events( + incident_id, offset=offset, limit=limit, resource_id=resource_id, name=name, description=description, status=status, from_=from_, to=to, time_=time_, modified_since=modified_since + ) @router.get( @@ -135,11 +163,7 @@ async def get_events( responses=DEFAULT_RESPONSES, operation_id="getEventByIncident", ) -async def get_event( - request : Request, - incident_id : str, - event_id : str - ) -> models.Event: +async def get_event(request: Request, incident_id: str, event_id: str) -> models.Event: item = await router.adapter.get_event(incident_id, event_id) if not item: raise HTTPException(status_code=404, detail="Item not found") diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index 47f686eb..5eb92c2f 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -13,40 +13,20 @@ class FacilityAdapter(AuthenticatedAdapter): to install your facility adapter before the API starts. """ - @abstractmethod - async def get_task( - self : "FacilityAdapter", - user: account_models.User, - task_id: str - ) -> task_models.Task|None: + async def get_task(self: "FacilityAdapter", user: account_models.User, task_id: str) -> task_models.Task | None: pass - @abstractmethod - async def get_tasks( - self : "FacilityAdapter", - user: account_models.User - ) -> list[task_models.Task]: + async def get_tasks(self: "FacilityAdapter", user: account_models.User) -> list[task_models.Task]: pass - @abstractmethod - async def put_task( - self: "FacilityAdapter", - user: account_models.User, - resource: status_models.Resource|None, - task: task_models.TaskCommand - ) -> str: + async def put_task(self: "FacilityAdapter", user: account_models.User, resource: status_models.Resource | None, task: task_models.TaskCommand) -> str: pass - @staticmethod - async def on_task( - resource: status_models.Resource, - user: account_models.User, - task: task_models.TaskCommand - ) -> tuple[str, task_models.TaskStatus]: + async def on_task(resource: status_models.Resource, user: account_models.User, task: task_models.TaskCommand) -> tuple[str, task_models.TaskStatus]: # Handle a task from the facility message queue. # Returns: (result, status) try: diff --git a/app/routers/task/models.py b/app/routers/task/models.py index cea97873..409a0721 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -9,6 +9,7 @@ class TaskStatus(str, enum.Enum): failed = "failed" canceled = "canceled" + class TaskCommand(BaseModel): router: str command: str @@ -17,6 +18,6 @@ class TaskCommand(BaseModel): class Task(BaseModel): id: str - status: TaskStatus=TaskStatus.pending - result: str|None=None - command: TaskCommand|None=None + status: TaskStatus = TaskStatus.pending + result: str | None = None + command: TaskCommand | None = None diff --git a/app/routers/task/task.py b/app/routers/task/task.py index 1a65459a..8fa8a495 100644 --- a/app/routers/task/task.py +++ b/app/routers/task/task.py @@ -1,7 +1,7 @@ from fastapi import Request, HTTPException, Depends from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES -from .import models, facility_adapter +from . import models, facility_adapter router = iri_router.IriRouter( facility_adapter.FacilityAdapter, @@ -18,9 +18,9 @@ operation_id="getTask", ) async def get_task( - request : Request, - task_id : str, - ) -> models.Task: + request: Request, + task_id: str, +) -> models.Task: """Get a task""" user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: @@ -39,10 +39,10 @@ async def get_task( operation_id="getTasks", ) async def get_tasks( - request : Request, - ) -> list[models.Task]: + request: Request, +) -> list[models.Task]: """Get all tasks""" user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) if not user: raise HTTPException(status_code=404, detail="User not found") - return await router.adapter.get_tasks(user=user) \ No newline at end of file + return await router.adapter.get_tasks(user=user) diff --git a/app/types/base.py b/app/types/base.py index 4a3b245d..5052d99d 100644 --- a/app/types/base.py +++ b/app/types/base.py @@ -1,10 +1,10 @@ """Default models used by multiple routers.""" + import datetime from collections.abc import Iterable from typing import Optional -from pydantic import (BaseModel, ConfigDict, Field, computed_field, - field_validator, model_serializer) +from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator, model_serializer from .. import config from .scalars import StrictDateTime @@ -12,6 +12,7 @@ class IRIBaseModel(BaseModel): """Base model for IRI models.""" + model_config = ConfigDict(extra="allow") @model_serializer(mode="wrap") @@ -33,7 +34,9 @@ def get_extra(self, key, default=None): class NamedObject(IRIBaseModel): """Base model for named objects.""" + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + def _self_path(self) -> str: raise NotImplementedError @@ -66,7 +69,7 @@ def self_uri(self) -> str: @classmethod def find_by_id(cls, items, id_, allow_name: bool = False): - """ Find an object by its id or name == id. """ + """Find an object by its id or name == id.""" # Find a resource by its id. # If allow_name is True, the id parameter can also match the resource's name. matches = [r for r in items if r.id == id_ or (allow_name and r.name == id_)] @@ -79,7 +82,7 @@ def find_by_id(cls, items, id_, allow_name: bool = False): @classmethod def find(cls, items, name=None, description=None, modified_since=None): - """ Find objects matching the given criteria. """ + """Find objects matching the given criteria.""" single = False if not any((name, description, modified_since)): return items @@ -94,8 +97,7 @@ def find(cls, items, name=None, description=None, modified_since=None): items = [item for item in items if item.description and description in item.description] if modified_since: modified_since = cls.normalize_dt(modified_since) - items = [item for item in items - if item.last_modified and item.last_modified >= modified_since] + items = [item for item in items if item.last_modified and item.last_modified >= modified_since] if single: return items[0] if items else None return items diff --git a/app/types/http.py b/app/types/http.py index c59c8ef1..056dc51a 100644 --- a/app/types/http.py +++ b/app/types/http.py @@ -1,4 +1,5 @@ """HTTP-related types and utilities for the IRI Facility API""" + import datetime from email.utils import parsedate_to_datetime from urllib.parse import parse_qs @@ -14,10 +15,8 @@ # If-Modified-Since must be a valid RFC1123 datetime string. # TODO: If-Modified-Since is not yet supported/used by the API. -def modifiedSinceDatetime( - modified_since: str | None, - header_modified_since: str | None -) -> datetime.datetime | None: + +def modifiedSinceDatetime(modified_since: str | None, header_modified_since: str | None) -> datetime.datetime | None: """ Combine modified_since (ISO8601) and If-Modified-Since (RFC1123). If both are provided, the most recent timestamp is used. @@ -57,9 +56,11 @@ def modifiedSinceDatetime( # Stricter constraint wins return max(parsed_times) + # ----------------------------------------------------------------------- # forbidExtraQueryParams: a dependency to forbid extra query parameters + def forbidExtraQueryParams(*allowedParams: str, multiParams: set[str] | None = None): """Dependency to forbid extra query parameters. If allowedParams contains "*", all params are allowed.""" multiParams = multiParams or set() @@ -75,16 +76,9 @@ async def checker(req: Request): for key, values in parsed.items(): if key not in allowed: - raise HTTPException(status_code=422, - detail=[{"type": "extra_forbidden", - "loc": ["query", key], - "msg": f"Unexpected query parameter: {key}"}]) - + raise HTTPException(status_code=422, detail=[{"type": "extra_forbidden", "loc": ["query", key], "msg": f"Unexpected query parameter: {key}"}]) if len(values) > 1 and key not in multiParams: - raise HTTPException(status_code=422, - detail=[{"type": "duplicate_forbidden", - "loc": ["query", key], - "msg": f"Duplicate query parameter: {key}"}]) + raise HTTPException(status_code=422, detail=[{"type": "duplicate_forbidden", "loc": ["query", key], "msg": f"Duplicate query parameter: {key}"}]) return checker diff --git a/app/types/models.py b/app/types/models.py index 9378f62a..070b2dbd 100644 --- a/app/types/models.py +++ b/app/types/models.py @@ -1,4 +1,5 @@ """Models for the IRI Facility API.""" + from pydantic import Field from .base import NamedObject @@ -7,12 +8,13 @@ class Capability(NamedObject): """ - An aspect of a resource that can have an allocation. - For example, Perlmutter nodes with GPUs - For some resources at a facility, this will be 1 to 1 with the resource. - It is a way to further subdivide a resource into allocatable sub-resources. - The word "capability" is also known to users as something they need for a job to run. (eg. gpu) + An aspect of a resource that can have an allocation. + For example, Perlmutter nodes with GPUs + For some resources at a facility, this will be 1 to 1 with the resource. + It is a way to further subdivide a resource into allocatable sub-resources. + The word "capability" is also known to users as something they need for a job to run. (eg. gpu) """ + def _self_path(self) -> str: return f"/account/capabilities/{self.id}" diff --git a/app/types/scalars.py b/app/types/scalars.py index 582efcee..2b9b6fe5 100644 --- a/app/types/scalars.py +++ b/app/types/scalars.py @@ -1,4 +1,5 @@ """Scalar types for the IRI Facility API""" + # pylint: disable=unused-argument import datetime import enum @@ -8,10 +9,11 @@ # ----------------------------------------------------------------------- # StrictHTTPBool: a strict boolean type + class StrictHTTPBool: """Strict boolean: - - Accepts: real booleans, 'true', 'false' - - Rejects everything else. + - Accepts: real booleans, 'true', 'false' + - Rejects everything else. """ @classmethod @@ -34,14 +36,13 @@ def validate(value): @classmethod def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "boolean", - "description": "Strict boolean. Only true/false allowed (bool or string)." - } + return {"type": "boolean", "description": "Strict boolean. Only true/false allowed (bool or string)."} + # ----------------------------------------------------------------------- # StrictDateTime: a strict ISO8601 datetime type + class StrictDateTime: """ Strict ISO8601 datetime: @@ -81,17 +82,16 @@ def _normalize(dt: datetime.datetime) -> datetime.datetime: @classmethod def __get_pydantic_json_schema__(cls, schema, handler): - return { - "type": "string", - "format": "date-time", - "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted." - } + return {"type": "string", "format": "date-time", "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted."} + # ----------------------------------------------------------------------- # AllocationUnit: an enum for allocation units + class AllocationUnit(enum.Enum): """Units for allocation""" + node_hours = "node_hours" bytes = "bytes" inodes = "inodes" diff --git a/gunicorn.config.py b/gunicorn.config.py index 81077cee..e05cf888 100644 --- a/gunicorn.config.py +++ b/gunicorn.config.py @@ -4,6 +4,6 @@ logging.getLogger().setLevel(logging.INFO) workers = 8 -worker_class = "uvicorn.workers.UvicornWorker" +worker_class = "uvicorn.workers.UvicornWorker" bind = "0.0.0.0:8000" timeout = 60 diff --git a/pylintrc b/pylintrc new file mode 100644 index 00000000..4a9fbf34 --- /dev/null +++ b/pylintrc @@ -0,0 +1,581 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths= + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. The default value ignores emacs file +# locks +ignore-patterns=^\.# + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.14 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=missing-function-docstring, + too-many-positional-arguments, + too-many-arguments, + too-many-positional-arguments, + missing-class-docstring, + invalid-name, + missing-module-docstring + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=200 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=no + +# Signatures are removed from the similarity computation +ignore-signatures=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear and the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# class is considered mixin if its name matches the mixin-class-rgx option. +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins ignore-mixin- +# members is set to 'yes' +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=camelCase + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=camelCase + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=camelCase + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=camelCase +method-rgx=^(_{1,2}[a-z][a-zA-Z0-9]*|[a-z][a-zA-Z0-9]*)$ + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=camelCase + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=camelCase + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=builtins.BaseException, + builtins.Exception diff --git a/pyproject.toml b/pyproject.toml index 63f6c020..ec0ccb32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,4 +10,7 @@ dependencies = [ "opentelemetry-sdk>=1.39.1,<1.40.0", "opentelemetry-instrumentation-fastapi>=0.60b1,<0.61b0", "opentelemetry-exporter-otlp>=1.39.1,<1.40.0" -] \ No newline at end of file +] +[tool.ruff] +line-length = 200 +exclude = [".venv", "__pycache__", "build", "dist"] From f721ab1afba0e6862299b0847f1231090ebca67f Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 9 Feb 2026 06:26:09 -0600 Subject: [PATCH 070/133] Add response exclude none. Esnet endpoint fails validation with none data --- app/routers/facility/facility.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 03331cf8..aa9bd2c9 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -9,7 +9,7 @@ router = iri_router.IriRouter(facility_adapter.FacilityAdapter, prefix="/facility", tags=["facility"]) -@router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility") +@router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility", response_model_exclude_none=True,) async def get_facility( request: Request, modified_since: StrictDateTime = Query(default=None), @@ -19,7 +19,7 @@ async def get_facility( return await router.adapter.get_facility(modified_since=modified_since) -@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites") +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites", response_model_exclude_none=True,) async def list_sites( request: Request, modified_since: StrictDateTime = Query(default=None), @@ -33,7 +33,7 @@ async def list_sites( return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite") +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite", response_model_exclude_none=True,) async def get_site( request: Request, site_id: str, From 117b96e94edb8598ec6e6a851bd112f3634047ab Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Mon, 9 Feb 2026 11:12:48 -0800 Subject: [PATCH 071/133] Changed site_url to site_id --- app/demo_adapter.py | 16 +++++++--------- app/routers/facility/models.py | 23 ++++++++++++++++++----- app/routers/status/models.py | 14 ++++++++------ 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 390f2661..9bbe825e 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -103,7 +103,7 @@ def _init_state(self): state_or_province_name="DC", latitude=36.173357, longitude=-234.51452, - resource_uris=[], + resource_ids=[], ) site2 = facility_models.Site( @@ -118,7 +118,7 @@ def _init_state(self): state_or_province_name="ET", latitude=38.410558, longitude=-286.36999, - resource_uris=[], + resource_ids=[], ) self.facility = facility_models.Facility( @@ -129,7 +129,7 @@ def _init_state(self): short_name="DEMO", organization_name="Demo Organization", support_uri="https://support.demo.example", - site_uris=[site1.self_uri, site2.self_uri], + site_ids=[site1.id, site2.id], ) self.sites = [site1, site2] @@ -155,7 +155,6 @@ def _init_state(self): current_status=status_models.Status.degraded, last_modified=day_ago, resource_type=status_models.ResourceType.compute, - located_at_uri=site1.self_uri, ) hpss = status_models.Resource( @@ -168,7 +167,6 @@ def _init_state(self): current_status=status_models.Status.up, last_modified=day_ago, resource_type=status_models.ResourceType.storage, - located_at_uri=site1.self_uri, ) cfs = status_models.Resource( @@ -181,7 +179,6 @@ def _init_state(self): current_status=status_models.Status.up, last_modified=day_ago, resource_type=status_models.ResourceType.storage, - located_at_uri=site1.self_uri, ) login = status_models.Resource( @@ -194,7 +191,6 @@ def _init_state(self): current_status=status_models.Status.degraded, last_modified=day_ago, resource_type=status_models.ResourceType.system, - located_at_uri=site2.self_uri, ) iris = status_models.Resource( @@ -207,7 +203,6 @@ def _init_state(self): current_status=status_models.Status.down, last_modified=day_ago, resource_type=status_models.ResourceType.website, - located_at_uri=site2.self_uri, ) sfapi = status_models.Resource( id=str(uuid.uuid4()), @@ -219,11 +214,14 @@ def _init_state(self): current_status=status_models.Status.up, last_modified=day_ago, resource_type=status_models.ResourceType.service, - located_at_uri=site2.self_uri, ) self.resources = [pm, hpss, cfs, login, iris, sfapi] + # Populate site resource_ids based on which resources are at each site + site1.resource_ids = [r.id for r in self.resources if r.site_id == site1.id] + site2.resource_ids = [r.id for r in self.resources if r.site_id == site2.id] + self.projects = [ account_models.Project( id=str(uuid.uuid4()), diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index c260b203..bf917db2 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,9 +1,10 @@ """Facility-related models.""" -from typing import List, Optional +from typing import Optional -from pydantic import Field, HttpUrl +from pydantic import Field, HttpUrl, computed_field +from ... import config from ...types.base import NamedObject @@ -21,7 +22,13 @@ def _self_path(self) -> str: altitude: Optional[float] = Field(None, description="Altitude of the Location.") latitude: Optional[float] = Field(None, description="Latitude of the Location.") longitude: Optional[float] = Field(None, description="Longitude of the Location.") - resource_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of Resources hosted at this Site.") + resource_ids: list[str] = Field(default_factory=list, exclude=True) + + @computed_field(description="URIs of Resources hosted at this Site.") + @property + def resource_uris(self) -> list[str]: + """Return the list of resource URIs for this site.""" + return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{resource_id}" for resource_id in self.resource_ids] @classmethod def find(cls, items, name=None, description=None, modified_since=None, short_name=None, country_name=None): @@ -39,6 +46,12 @@ def _self_path(self) -> str: return "/facility" short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization’s name.") + organization_name: Optional[str] = Field(None, description="Operating organization's name.") support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") - site_uris: List[HttpUrl] = Field(default_factory=list, description="URIs of associated Sites.") + site_ids: list[str] = Field(default_factory=list, exclude=True) + + @computed_field(description="URIs of associated Sites.") + @property + def site_uris(self) -> list[str]: + """Return the list of site URIs for this facility.""" + return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/facility/sites/{site_id}" for site_id in self.site_ids] diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 566a8106..930892c6 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -1,8 +1,7 @@ import datetime import enum -from typing import Optional -from pydantic import BaseModel, Field, HttpUrl, computed_field, field_validator +from pydantic import BaseModel, Field, computed_field, field_validator from ... import config from ...types.base import NamedObject @@ -35,14 +34,17 @@ def _self_path(self) -> str: """Return the API path for this resource.""" return f"/status/resources/{self.id}" - # NOTE (TBR): If site_id is required, then located_at_uri should be also required. This can be easily identified by Site.self_uri - # Is there a specific Resource, that has no Site? - site_id: str = Field(..., description="The site identifier this resource is located at") + site_id: str = Field(..., description="The site identifier this resource is located at", exclude=True) capability_ids: list[str] = Field(default_factory=list, exclude=True) group: str | None current_status: Status | None = Field(default=None, description="The current status comes from the status of the last event for this resource") resource_type: ResourceType - located_at_uri: Optional[HttpUrl] = Field(None, description="Resource located at specific Site") + + @computed_field(description="URI of the site where this resource is located") + @property + def site_uri(self) -> str: + """Return the site URI for this resource.""" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/facility/sites/{self.site_id}" @computed_field(description="The list of capabilities in this resource") @property From 346cf0d0aabc682d9ec8e079850b4c1c9a747c65 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Mon, 9 Feb 2026 13:23:58 -0800 Subject: [PATCH 072/133] the facility api doesn't require authentication --- app/routers/facility/facility_adapter.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/routers/facility/facility_adapter.py b/app/routers/facility/facility_adapter.py index e04e9cf8..e43de0ea 100644 --- a/app/routers/facility/facility_adapter.py +++ b/app/routers/facility/facility_adapter.py @@ -1,9 +1,8 @@ -from abc import abstractmethod +from abc import ABC, abstractmethod from . import models as facility_models -from ..iri_router import AuthenticatedAdapter -class FacilityAdapter(AuthenticatedAdapter): +class FacilityAdapter(ABC): """ Facility-specific code is handled by the implementation of this interface. Use the `IRI_API_ADAPTER` environment variable (defaults to `app.demo_adapter.FacilityAdapter`) From 2cd36d0ea4658df46124518cbb46c1451aba1e5c Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Mon, 9 Feb 2026 14:50:03 -0800 Subject: [PATCH 073/133] updated api endpoint url for NERSC --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 02b796c9..72aa0515 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ See it live: - NERSC instance: - API docs: https://api.iri.nersc.gov - - API requests: https://api.iri.nersc.gov/nersc/api/v1/ + - API requests: https://api.iri.nersc.gov/api/v1/ - ALCF instance: - API docs: https://api.alcf.anl.gov - API requests: https://api.alcf.anl.gov/api/v1/ From 3b3651fe537d4821270f1835e34ec7759118f41a Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 10 Feb 2026 13:54:53 -0800 Subject: [PATCH 074/133] minor readme update --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 72aa0515..a0d4e645 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,8 @@ ENV IRI_API_PARAMS='{ \ ## Next steps - Learn more about [fastapi](https://fastapi.tiangolo.com/), including how to run it [in production](https://fastapi.tiangolo.com/advanced/behind-a-proxy/) -- Instead of the simulated state, keep real data in a [database](/Users/gtorok/dev/iri-api-python/README.md) -- Add monitoring by [integrating with OpenTelemetry](https://opentelemetry.io/docs/zero-code/python/) +- Instead of the simulated state, keep real data in a database +- Specify the monitoring endpoint by setting the [OpenTelemetry](https://opentelemetry.io/docs/zero-code/python/) env vars - Add additional routers for other API-s - Add authenticated API-s via an [OAuth2 integration](https://fastapi.tiangolo.com/tutorial/security/oauth2-jwt/) From 821a681e5e6b2a66481391fe27d38166b10122ab Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 10 Feb 2026 17:05:25 -0800 Subject: [PATCH 075/133] Facility api fix in case the adapter returns null --- app/routers/facility/facility.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index aa9bd2c9..cd294296 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -1,4 +1,4 @@ -from fastapi import Depends, Query, Request +from fastapi import Depends, Query, Request, HTTPException from ...types.http import forbidExtraQueryParams from ...types.scalars import StrictDateTime @@ -10,13 +10,17 @@ @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility", response_model_exclude_none=True,) +@router.get("/", responses=DEFAULT_RESPONSES, operation_id="getFacilityWithSlash", response_model_exclude_none=True,) async def get_facility( request: Request, modified_since: StrictDateTime = Query(default=None), _forbid=Depends(forbidExtraQueryParams("modified_since")), ) -> models.Facility: """Get facility information""" - return await router.adapter.get_facility(modified_since=modified_since) + facility = await router.adapter.get_facility(modified_since=modified_since) + if not facility: + raise HTTPException(status_code=404, detail="Facility not found") + return facility @router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites", response_model_exclude_none=True,) @@ -41,4 +45,7 @@ async def get_site( _forbid=Depends(forbidExtraQueryParams("modified_since")), ) -> models.Site: """Get site by ID""" - return await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + site = await router.adapter.get_site(site_id=site_id, modified_since=modified_since) + if not site: + raise HTTPException(status_code=404, detail="Site not found") + return site From 195387999aaf7968fca64bf454d687f77ef4e1bf Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 10 Feb 2026 17:09:10 -0800 Subject: [PATCH 076/133] hide facility/ from swagger --- app/routers/facility/facility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index cd294296..1ab5b5c2 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -10,7 +10,7 @@ @router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility", response_model_exclude_none=True,) -@router.get("/", responses=DEFAULT_RESPONSES, operation_id="getFacilityWithSlash", response_model_exclude_none=True,) +@router.get("/", responses=DEFAULT_RESPONSES, operation_id="getFacilityWithSlash", response_model_exclude_none=True, include_in_schema=False,) async def get_facility( request: Request, modified_since: StrictDateTime = Query(default=None), From d55147d2de4f53333bf24130d1526d55cb68838f Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 16 Feb 2026 12:00:29 -0600 Subject: [PATCH 077/133] Task Improvements (response format, task_uri, fix get_resource call) --- app/demo_adapter.py | 14 ++++-- app/routers/filesystem/filesystem.py | 74 ++++++++++++++-------------- app/routers/task/facility_adapter.py | 6 ++- app/routers/task/models.py | 14 +++++- app/routers/task/task.py | 27 +++++++--- 5 files changed, 87 insertions(+), 48 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 9bbe825e..9d217bae 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -876,10 +876,18 @@ async def get_tasks(self: "DemoAdapter", user: account_models.User) -> list[task await DemoTaskQueue._process_tasks(self) return [t for t in DemoTaskQueue.tasks if t.user.name == user.name] - async def put_task(self: "DemoAdapter", user: account_models.User, resource: status_models.Resource, task: str) -> str: + async def put_task(self: "DemoAdapter", user: account_models.User, resource: status_models.Resource, task: str) -> task_models.TaskSubmitResponse: await DemoTaskQueue._process_tasks(self) return DemoTaskQueue._create_task(user, resource, task) + async def delete_task(self: "DemoAdapter", user: account_models.User, task_id: str) -> None: + await DemoTaskQueue._process_tasks(self) + for t in DemoTaskQueue.tasks: + if t.user.name == user.name and t.id == task_id: + t.status = task_models.TaskStatus.canceled + t.result = None + break + class DemoTask(BaseModel): id: str @@ -914,7 +922,7 @@ async def _process_tasks(da: DemoAdapter): DemoTaskQueue.tasks = _tasks @staticmethod - def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> str: + def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> task_models.TaskSubmitResponse: task_id = f"task_{len(DemoTaskQueue.tasks)}" DemoTaskQueue.tasks.append(DemoTask(id=task_id, task=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) - return task_id + return task_models.TaskSubmitResponse(task_id=task_id) diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index ba04d8db..50f226ef 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -32,7 +32,7 @@ async def _user_resource( raise HTTPException(status_code=404, detail="User not found") # look up the resource (todo: maybe ensure it's available) - resource = await status_router.adapter.get_resource(resource_id=resource_id) + resource = await status_router.adapter.get_resource(resource_id) if not resource: raise HTTPException(status_code=404, detail="Resource not found") return (user, resource) @@ -43,7 +43,7 @@ async def _user_resource( dependencies=[Depends(router.current_user)], description="Change the permission mode of a file(`chmod`)", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="File permissions changed successfully", responses=DEFAULT_RESPONSES, operation_id="chmod", @@ -52,7 +52,7 @@ async def put_chmod( resource_id: str, request_model: models.PutFileChmodRequest, request: Request, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -72,7 +72,7 @@ async def put_chmod( dependencies=[Depends(router.current_user)], description="Change the ownership of a given file (`chown`)", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="File ownership changed successfully", responses=DEFAULT_RESPONSES, operation_id="chown", @@ -81,7 +81,7 @@ async def put_chown( resource_id: str, request_model: models.PutFileChownRequest, request: Request, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -101,7 +101,7 @@ async def put_chown( dependencies=[Depends(router.current_user)], description="Output the type of a file or directory", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="Type returned successfully", responses=DEFAULT_RESPONSES, operation_id="file", @@ -110,7 +110,7 @@ async def get_file( resource_id: str, request: Request, path: Annotated[str, Query(description="A file or folder path")], -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -130,7 +130,7 @@ async def get_file( dependencies=[Depends(router.current_user)], description="Output the `stat` of a file", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="Stat returned successfully", responses=DEFAULT_RESPONSES, operation_id="stat", @@ -140,7 +140,7 @@ async def get_stat( request: Request, path: Annotated[str, Query(description="A file or folder path")], dereference: Annotated[bool, Query(description="Follow symbolic links")] = False, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -161,7 +161,7 @@ async def get_stat( dependencies=[Depends(router.current_user)], description="Create directory operation (`mkdir`)", status_code=status.HTTP_201_CREATED, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="Directory created successfully", responses=DEFAULT_RESPONSES, operation_id="mkdir", @@ -170,7 +170,7 @@ async def post_mkdir( resource_id: str, request: Request, request_model: models.PostMakeDirRequest, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -190,7 +190,7 @@ async def post_mkdir( dependencies=[Depends(router.current_user)], description="Create symlink operation (`ln`)", status_code=status.HTTP_201_CREATED, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="Symlink created successfully", responses=DEFAULT_RESPONSES, operation_id="symlink", @@ -199,7 +199,7 @@ async def post_symlink( resource_id: str, request: Request, request_model: models.PostFileSymlinkRequest, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -219,7 +219,7 @@ async def post_symlink( dependencies=[Depends(router.current_user)], description="List the contents of the given directory (`ls`) asynchronously", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="Directory listed successfully", include_in_schema=router.task_adapter is not None, responses=DEFAULT_RESPONSES, @@ -239,13 +239,15 @@ async def get_ls_async( description="Show information for the file the link references.", ), ] = False, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, resource=resource, task=task_models.TaskCommand( - router=router.get_router_name(), command="ls", args={"path": path, "show_hidden": show_hidden, "numeric_uid": numeric_uid, "recursive": recursive, "dereference": dereference} + router=router.get_router_name(), + command="ls", + args={"path": path, "show_hidden": show_hidden, "numeric_uid": numeric_uid, "recursive": recursive, "dereference": dereference} ), ) @@ -255,7 +257,7 @@ async def get_ls_async( dependencies=[Depends(router.current_user)], description="Output the first part of file/s (`head`)", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="Head operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="head", @@ -289,7 +291,7 @@ async def get_head( description=("The output will be the whole file, without the last NUM bytes/lines of each file. NUM should be specified in the respective argument through `bytes` or `lines`."), ), ] = False, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) # Enforce that exactly one of `bytes` or `lines` is specified if (file_bytes is None and lines is None) or (file_bytes is not None and lines is not None): @@ -315,7 +317,7 @@ async def get_head( dependencies=[Depends(router.current_user)], description=f"View file content (up to max {facility_adapter.OPS_SIZE_LIMIT} bytes)", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="View operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="view", @@ -326,7 +328,7 @@ async def get_view( path: Annotated[str, Query(description="File path")], size: Annotated[int, Query(description="Value, in bytes, of the size of data to be retrieved from the file.", ge=1, le=facility_adapter.OPS_SIZE_LIMIT)] = facility_adapter.OPS_SIZE_LIMIT, offset: Annotated[int, Query(description="Value in bytes of the offset.", ge=0)] = 0, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( @@ -349,7 +351,7 @@ async def get_view( dependencies=[Depends(router.current_user)], description="Output the last part of a file (`tail`)", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="`tail` operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="tail", @@ -376,7 +378,7 @@ async def get_tail( description=("The output will be the whole file, without the first NUM bytes/lines of each file. NUM should be specified in the respective argument through `bytes` or `lines`."), ), ] = False, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) # Enforce that exactly one of `bytes` or `lines` is specified if (file_bytes is None and lines is None) or (file_bytes is not None and lines is not None): @@ -402,7 +404,7 @@ async def get_tail( dependencies=[Depends(router.current_user)], description="Output the checksum of a file (using SHA-256 algotithm)", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="Checksum returned successfully", responses=DEFAULT_RESPONSES, operation_id="checksum", @@ -411,7 +413,7 @@ async def get_checksum( resource_id: str, request: Request, path: Annotated[str, Query(description="Target system")], -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -458,7 +460,7 @@ async def delete_rm( dependencies=[Depends(router.current_user)], description="Compress files and directories using `tar` command", status_code=status.HTTP_201_CREATED, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="File and/or directories compressed successfully", responses=DEFAULT_RESPONSES, operation_id="compress", @@ -467,7 +469,7 @@ async def post_compress( resource_id: str, request: Request, request_model: models.PostCompressRequest, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -487,7 +489,7 @@ async def post_compress( dependencies=[Depends(router.current_user)], description="Extract `tar` `gzip` archives", status_code=status.HTTP_201_CREATED, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="File extracted successfully", responses=DEFAULT_RESPONSES, operation_id="extract", @@ -496,7 +498,7 @@ async def post_extract( resource_id: str, request: Request, request_model: models.PostExtractRequest, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -516,7 +518,7 @@ async def post_extract( dependencies=[Depends(router.current_user)], description="Create move file or directory operation (`mv`)", status_code=status.HTTP_201_CREATED, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="Move file or directory operation created successfully", responses=DEFAULT_RESPONSES, operation_id="mv", @@ -525,7 +527,7 @@ async def move_mv( resource_id: str, request: Request, request_model: models.PostMoveRequest, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -545,7 +547,7 @@ async def move_mv( dependencies=[Depends(router.current_user)], description="Create copy file or directory operation (`cp`)", status_code=status.HTTP_201_CREATED, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="Copy file or directory operation created successfully", responses=DEFAULT_RESPONSES, operation_id="cp", @@ -554,7 +556,7 @@ async def post_cp( resource_id: str, request: Request, request_model: models.PostCopyRequest, -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -574,7 +576,7 @@ async def post_cp( dependencies=[Depends(router.current_user)], description=f"Download a small file (max {facility_adapter.OPS_SIZE_LIMIT} Bytes)", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="File downloaded successfully", responses=DEFAULT_RESPONSES, operation_id="download", @@ -583,7 +585,7 @@ async def get_download( resource_id: str, request: Request, path: Annotated[str, Query(description="A file to download")], -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, @@ -603,7 +605,7 @@ async def get_download( dependencies=[Depends(router.current_user)], description=f"Upload a small file (max {facility_adapter.OPS_SIZE_LIMIT} Bytes)", status_code=status.HTTP_200_OK, - response_model=str, + response_model=task_models.TaskSubmitResponse, response_description="File uploaded successfully", responses=DEFAULT_RESPONSES, operation_id="upload", @@ -613,7 +615,7 @@ async def post_upload( request: Request, path: Annotated[str, Query(description="Specify path where file should be uploaded.")], file: UploadFile = File(description="File to be uploaded as `multipart/form-data`"), -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) raw_content = file.file.read() diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index 5eb92c2f..bfca880e 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -22,7 +22,11 @@ async def get_tasks(self: "FacilityAdapter", user: account_models.User) -> list[ pass @abstractmethod - async def put_task(self: "FacilityAdapter", user: account_models.User, resource: status_models.Resource | None, task: task_models.TaskCommand) -> str: + async def put_task(self: "FacilityAdapter", user: account_models.User, resource: status_models.Resource | None, task: task_models.TaskCommand) -> task_models.TaskSubmitResponse: + pass + + @abstractmethod + async def delete_task(self: "FacilityAdapter", user: account_models.User, task_id: str) -> None: pass @staticmethod diff --git a/app/routers/task/models.py b/app/routers/task/models.py index 409a0721..30fe9ca2 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,5 +1,17 @@ import enum -from pydantic import BaseModel +from pydantic import BaseModel, computed_field + +from ... import config + + +class TaskSubmitResponse(BaseModel): + """Response model for submitting a task""" + task_id: str + + @computed_field(description="The list of past events in this incident") + @property + def task_uri(self) -> str: + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/tasks/{self.task_id}" class TaskStatus(str, enum.Enum): diff --git a/app/routers/task/task.py b/app/routers/task/task.py index 8fa8a495..74cfa6d4 100644 --- a/app/routers/task/task.py +++ b/app/routers/task/task.py @@ -31,13 +31,9 @@ async def get_task( return task -@router.get( - "", - dependencies=[Depends(router.current_user)], - response_model_exclude_unset=True, - responses=DEFAULT_RESPONSES, - operation_id="getTasks", -) +@router.get("", dependencies=[Depends(router.current_user)], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getTasks") +@router.get("/", responses=DEFAULT_RESPONSES, operation_id="getTasksWithSlash", include_in_schema=False) + async def get_tasks( request: Request, ) -> list[models.Task]: @@ -46,3 +42,20 @@ async def get_tasks( if not user: raise HTTPException(status_code=404, detail="User not found") return await router.adapter.get_tasks(user=user) + +@router.delete( + "/{task_id:str}", + dependencies=[Depends(router.current_user)], + responses=DEFAULT_RESPONSES, + operation_id="deleteTask", +) +async def delete_task( + request: Request, + task_id: str, +) -> str: + """Delete a task""" + user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) + if not user: + raise HTTPException(status_code=404, detail="User not found") + await router.adapter.delete_task(user=user, task_id=task_id) + return f"Task {task_id} deleted successfully" \ No newline at end of file From 8d4f0afb6d52bdebda75fb17a4f45109fbfb92af Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 16 Feb 2026 20:15:43 -0600 Subject: [PATCH 078/133] Do not swallow error message returned by backend driver. Pass it back to client --- app/routers/error_handlers.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 1dcc5f6d..338eae6a 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -89,6 +89,9 @@ async def validation_error_handler(request: Request, exc: RequestValidationError # FASTAPI HTTP EXCEPTIONS @app.exception_handler(HTTPException) async def http_exception_handler(request: Request, exc: HTTPException): + err_msg = "" + if hasattr(exc, "detail") and exc.detail: + err_msg = exc.detail if exc.status_code == 304: return JSONResponse(status_code=304, content=None, headers=exc.headers or {}) @@ -98,7 +101,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): request=request, status=401, title="Unauthorized", - detail="Bearer token is missing or invalid.", + detail=err_msg or "Bearer token is missing or invalid.", problem_type="unauthorized", extra_headers={"WWW-Authenticate": "Bearer"}, ) @@ -108,7 +111,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): request=request, status=403, title="Forbidden", - detail="Caller is authenticated but lacks required role.", + detail=err_msg or "Caller is authenticated but lacks required role.", problem_type="forbidden", ) @@ -117,7 +120,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): request=request, status=404, title="Not Found", - detail=exc.detail or "Invalid resource identifier.", + detail=err_msg or "Invalid resource identifier.", problem_type="not-found", ) @@ -126,7 +129,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): request=request, status=405, title="Method Not Allowed", - detail="HTTP method is not allowed for this resource.", + detail=err_msg or "HTTP method is not allowed for this resource.", problem_type="method-not-allowed", extra_headers={"Allow": "GET, HEAD"}, ) @@ -136,7 +139,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): request=request, status=409, title="Conflict", - detail=exc.detail or "Conflict occurred.", + detail=err_msg or "Conflict occurred.", problem_type="conflict", ) @@ -144,21 +147,23 @@ async def http_exception_handler(request: Request, exc: HTTPException): return problem_response( request=request, status=exc.status_code, - title=exc.detail or "Error", - detail=exc.detail or "An error occurred.", + title=err_msg or "Error", + detail=err_msg or "An error occurred.", problem_type="generic-error", ) # STARLETTE HTTP EXCEPTIONS @app.exception_handler(StarletteHTTPException) async def starlette_handler(request: Request, exc: StarletteHTTPException): - + err_msg = "" + if hasattr(exc, "detail") and exc.detail: + err_msg = exc.detail if exc.status_code == 404: return problem_response( request=request, status=404, title="Not Found", - detail="Invalid resource identifier.", + detail=err_msg or "Invalid resource identifier.", problem_type="not-found", ) @@ -167,7 +172,7 @@ async def starlette_handler(request: Request, exc: StarletteHTTPException): request=request, status=405, title="Method Not Allowed", - detail="HTTP method is not allowed for this resource.", + detail=err_msg or "HTTP method is not allowed for this resource.", problem_type="method-not-allowed", extra_headers={"Allow": "GET, HEAD"}, ) @@ -175,8 +180,8 @@ async def starlette_handler(request: Request, exc: StarletteHTTPException): return problem_response( request=request, status=exc.status_code, - title=exc.detail or "Error", - detail=exc.detail or "An error occurred.", + title=err_msg or "Error", + detail=err_msg or "An error occurred.", problem_type="generic-error", ) From f26c6de288ae3bfadfc6688d830078023d7ccd7b Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 17 Feb 2026 09:44:03 -0600 Subject: [PATCH 079/133] Use TaskSubmitResponse for rm --- app/routers/filesystem/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index 50f226ef..b53521e8 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -440,7 +440,7 @@ async def delete_rm( resource_id: str, request: Request, path: Annotated[str, Query(description="The path to delete")], -) -> str: +) -> task_models.TaskSubmitResponse: user, resource = await _user_resource(resource_id, request) return await router.task_adapter.put_task( user=user, From 8a7a743c107bf2a5789388dbab558925294be860 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 17 Feb 2026 14:36:20 -0800 Subject: [PATCH 080/133] minor bugfix --- app/routers/task/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/task/models.py b/app/routers/task/models.py index 30fe9ca2..9d64e3bf 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -11,7 +11,7 @@ class TaskSubmitResponse(BaseModel): @computed_field(description="The list of past events in this incident") @property def task_uri(self) -> str: - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/tasks/{self.task_id}" + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/task/{self.task_id}" class TaskStatus(str, enum.Enum): From da54e0c5686e29fd7df0cd7bc33d847f31e2f54b Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 17 Feb 2026 17:06:09 -0600 Subject: [PATCH 081/133] RFC 7807 OpenAPI native error modeling Refactor error handling to use Pydantic model instead of manually defined JSON schemas. (fixes an issue with automatic code generator tools) --- app/routers/error_handlers.py | 65 ++++++++++++++++------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 338eae6a..2f567902 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -5,11 +5,29 @@ import logging from urllib.parse import urlsplit, urlunsplit, quote + +from typing import Optional, List +from pydantic import BaseModel, constr + from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse from fastapi.exceptions import RequestValidationError from starlette.exceptions import HTTPException as StarletteHTTPException +class InvalidParam(BaseModel): + name: str + reason: str + +class Problem(BaseModel): + type: constr(min_length=1) + title: Optional[str] = None + status: int + detail: Optional[str] = None + instance: Optional[str] = None + invalid_params: Optional[List[InvalidParam]] = None + + class Config: + extra = "allow" def get_url_base(request: Request) -> str: """Return the base URL for the API.""" @@ -158,6 +176,7 @@ async def starlette_handler(request: Request, exc: StarletteHTTPException): err_msg = "" if hasattr(exc, "detail") and exc.detail: err_msg = exc.detail + if exc.status_code == 404: return problem_response( request=request, @@ -197,30 +216,6 @@ async def global_handler(request: Request, exc: Exception): problem_type="internal-error", ) - -DEFAULT_PROBLEM_SCHEMA = { - "type": "object", - "properties": { - "type": {"type": "string"}, - "title": {"type": "string"}, - "status": {"type": "integer"}, - "detail": {"type": "string"}, - "instance": {"type": "string"}, - "invalid_params": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "reason": {"type": "string"}, - }, - "required": ["name", "reason"], - }, - }, - }, - "required": ["type", "title", "status", "detail", "instance"], -} - EXAMPLE_400 = { "type": "https://iri.example.com/problems/invalid-parameter", "title": "Invalid parameter", @@ -308,15 +303,16 @@ async def global_handler(request: Request, exc: Exception): DEFAULT_RESPONSES = { 400: { "description": "Invalid request parameters", + "model": Problem, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_400, } }, }, 401: { "description": "Unauthorized", + "model": Problem, "headers": { "WWW-Authenticate": { "description": "Bearer authentication challenge", @@ -325,31 +321,31 @@ async def global_handler(request: Request, exc: Exception): }, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_401, } }, }, 403: { "description": "Forbidden", + "model": Problem, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_403, } }, }, 404: { "description": "Not Found", + "model": Problem, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_404, } }, }, 405: { "description": "Method Not Allowed", + "model": Problem, "headers": { "Allow": { "description": "Allowed HTTP methods", @@ -358,61 +354,60 @@ async def global_handler(request: Request, exc: Exception): }, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_405, } }, }, 409: { "description": "Conflict", + "model": Problem, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_409, } }, }, 422: { "description": "Unprocessable Entity", + "model": Problem, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_422, } }, }, 500: { "description": "Internal Server Error", + "model": Problem, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_500, } }, }, 501: { "description": "Not Implemented", + "model": Problem, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_501, } }, }, 503: { "description": "Service Unavailable", + "model": Problem, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_503, } }, }, 504: { "description": "Gateway Timeout", + "model": Problem, "content": { "application/problem+json": { - "schema": DEFAULT_PROBLEM_SCHEMA, "example": EXAMPLE_504, } }, From 594d758328a330b6701b5bdaddf476564eb1dbfa Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 18 Feb 2026 05:58:51 -0600 Subject: [PATCH 082/133] Add servers url inside the spec --- app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index f6eda7bf..a024798d 100644 --- a/app/main.py +++ b/app/main.py @@ -41,7 +41,7 @@ tracer = trace.get_tracer(__name__) # ------------------------------------------------------------------ -APP = FastAPI(**config.API_CONFIG) +APP = FastAPI(servers=[{"url": config.API_URL_ROOT}], **config.API_CONFIG) if config.OPENTELEMETRY_ENABLED: FastAPIInstrumentor.instrument_app(APP) From 341c557406889db869264063794c8315d4213c82 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 18 Feb 2026 09:30:09 -0600 Subject: [PATCH 083/133] Fixes while implementing filesystem for ESnet endpoint - Deterministic uuid in demo adapter; - Allow DEMO_QUEUE_UPDATE_SECS change via env variable; - Fix Auth schemantics. Check api_key if Bearer used. extract it (only demo_adapter.py) - Fix filesystem api: use skip_heading for tail; define all return models for all filesystem calls; Nested results. use jsonable_encoder to parse it. - Break extract in demo_adapter if dir is present. - Catch project None; - JobStates use enum. Cleaner schema. no custom serializer. (fails schema validation run output != documented contract) --- Makefile | 3 +- app/demo_adapter.py | 126 +++++++++---- app/routers/account/account.py | 2 + app/routers/compute/models.py | 20 +- app/routers/filesystem/facility_adapter.py | 10 +- app/routers/filesystem/models.py | 10 + app/routers/task/models.py | 3 +- test/test_filesystem.py | 201 +++++++++++++++++++++ 8 files changed, 319 insertions(+), 56 deletions(-) create mode 100644 test/test_filesystem.py diff --git a/Makefile b/Makefile index e9bb6ff6..60f47819 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,9 @@ dev: deps IRI_API_ADAPTER_compute=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_filesystem=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_task=app.demo_adapter.DemoAdapter \ + DEMO_QUEUE_UPDATE_SECS=2 \ OPENTELEMETRY_ENABLED=true \ - API_URL_ROOT='http://127.0.0.1:8000' fastapi dev + API_URL_ROOT='http://localhost:8000' fastapi dev .PHONY: clean clean: diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 9d217bae..8b0e54e3 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -12,6 +12,7 @@ from typing import Any, Tuple from fastapi import HTTPException +from fastapi.encoders import jsonable_encoder from pydantic import BaseModel from .routers.account import facility_adapter as account_adapter @@ -29,7 +30,7 @@ from .types.models import Capability from .types.scalars import AllocationUnit -DEMO_QUEUE_UPDATE_SECS = 5 +DEMO_QUEUE_UPDATE_SECS = int(os.environ.get("DEMO_QUEUE_UPDATE_SECS", 5)) def paginate_list(items, offset: int | None, limit: int | None): @@ -136,14 +137,14 @@ def _init_state(self): day_ago = utc_now() - datetime.timedelta(days=1) self.capabilities = { - "cpu": Capability(id=str(uuid.uuid4()), name="CPU Nodes", units=[AllocationUnit.node_hours]), - "gpu": Capability(id=str(uuid.uuid4()), name="GPU Nodes", units=[AllocationUnit.node_hours]), - "hpss": Capability(id=str(uuid.uuid4()), name="Tape Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), - "gpfs": Capability(id=str(uuid.uuid4()), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), + "cpu": Capability(id=demo_uuid("capability", "cpu"), name="CPU Nodes", units=[AllocationUnit.node_hours]), + "gpu": Capability(id=demo_uuid("capability", "gpu"), name="GPU Nodes", units=[AllocationUnit.node_hours]), + "hpss": Capability(id=demo_uuid("capability", "hpss"), name="Tape Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), + "gpfs": Capability(id=demo_uuid("capability", "gpfs"), name="GPFS Storage", units=[AllocationUnit.bytes, AllocationUnit.inodes]), } pm = status_models.Resource( - id=str(uuid.uuid4()), + id=demo_uuid("resource", "perlmutter_compute_nodes"), site_id=site1.id, group="perlmutter", name="compute nodes", @@ -158,7 +159,7 @@ def _init_state(self): ) hpss = status_models.Resource( - id=str(uuid.uuid4()), + id=demo_uuid("resource", "hpss"), site_id=site1.id, group="hpss", name="hpss", @@ -170,7 +171,7 @@ def _init_state(self): ) cfs = status_models.Resource( - id=str(uuid.uuid4()), + id=demo_uuid("resource", "cfs"), site_id=site1.id, group="cfs", name="cfs", @@ -182,7 +183,7 @@ def _init_state(self): ) login = status_models.Resource( - id=str(uuid.uuid4()), + id=demo_uuid("resource", "login_nodes"), site_id=site2.id, group="perlmutter", name="login nodes", @@ -194,7 +195,7 @@ def _init_state(self): ) iris = status_models.Resource( - id=str(uuid.uuid4()), + id=demo_uuid("resource", "iris"), site_id=site2.id, group="services", name="Iris", @@ -205,7 +206,7 @@ def _init_state(self): resource_type=status_models.ResourceType.website, ) sfapi = status_models.Resource( - id=str(uuid.uuid4()), + id=demo_uuid("resource", "sfapi"), site_id=site2.id, group="services", name="sfapi", @@ -224,13 +225,13 @@ def _init_state(self): self.projects = [ account_models.Project( - id=str(uuid.uuid4()), + id=demo_uuid("project", "staff_research"), name="Staff research project", description="Compute and storage allocation for staff research use", user_ids=["gtorok"], ), account_models.Project( - id=str(uuid.uuid4()), + id=demo_uuid("project", "test_project"), name="Test project", description="Compute and storage allocation for testing use", user_ids=["gtorok"], @@ -240,7 +241,7 @@ def _init_state(self): for p in self.projects: for c in self.capabilities.values(): pa = account_models.ProjectAllocation( - id=str(uuid.uuid4()), + id=demo_uuid("project_allocation", f"{p.id}_{c.id}"), project_id=p.id, capability_id=c.id, entries=[ @@ -255,7 +256,7 @@ def _init_state(self): self.project_allocations.append(pa) self.user_allocations.append( account_models.UserAllocation( - id=str(uuid.uuid4()), + id=demo_uuid("user_allocation", f"{pa.id}_gtorok"), project_id=pa.project_id, project_allocation_id=pa.id, user_id="gtorok", @@ -274,7 +275,7 @@ def _init_state(self): r = random.choice(self.resources) status = statuses[r.name] event = status_models.Event( - id=str(uuid.uuid4()), + id=demo_uuid("event", f"{r.name}_{d.isoformat()}"), name=f"{r.name} is {status.value}", description=f"{r.name} is {status.value}", occurred_at=d, @@ -298,7 +299,7 @@ def _init_state(self): statuses[r.name] = status_models.Status.down dstr = d.strftime("%Y-%m-%d %H:%M:%S.%f%z") incident = status_models.Incident( - id=str(uuid.uuid4()), + id=demo_uuid("incident", f"{r.name}_{dstr}"), name=f"{r.name} incident at {dstr}", description=f"{r.name} incident at {dstr}", status=status_models.Status.down, @@ -470,9 +471,11 @@ async def get_user( client_ip: str | None, ) -> account_models.User: if user_id != self.user.id: - raise HTTPException(status_code=401, detail="User not found") + raise HTTPException(status_code=403, detail="User not found") + if api_key.startswith("Bearer "): + api_key = api_key[len("Bearer ") :] if api_key != self.user.api_key: - raise HTTPException(status_code=403, detail="Invalid API key") + raise HTTPException(status_code=401, detail="Invalid API key") return self.user async def get_projects(self: "DemoAdapter", user: account_models.User) -> list[account_models.Project]: @@ -683,19 +686,27 @@ def _headtail( path: str, file_bytes: int | None, lines: int | None, + skip_heading: bool = False, ) -> Tuple[Any, int]: args = [cmd] - if file_bytes: - args.append("-c") - args.append(str(file_bytes)) - elif lines: - args.append("-n") - args.append(str(lines)) + + if cmd == "tail" and skip_heading: + if file_bytes is not None: + args.extend(["-c", f"+{file_bytes + 1}"]) + elif lines is not None: + args.extend(["-n", f"+{lines + 1}"]) + else: + if file_bytes is not None: + args.extend(["-c", str(file_bytes)]) + elif lines is not None: + args.extend(["-n", str(lines)]) + rp = self.validate_path(path) args.append(rp) + result = subprocess.run(args, capture_output=True, text=True) content = result.stdout - return content, -len(content) + return content, len(content) async def head( self: "DemoAdapter", @@ -705,11 +716,42 @@ async def head( file_bytes: int | None, lines: int | None, skip_trailing: bool, - ) -> Tuple[Any, int]: - return self._headtail("head", path, file_bytes, lines) + ) -> filesystem_models.GetFileHeadResponse: + content, offset = self._headtail("head", path, file_bytes, lines) + + fc = filesystem_models.FileContent( + content=content, + content_type=(filesystem_models.ContentUnit.bytes + if file_bytes is not None + else filesystem_models.ContentUnit.lines), + start_position=0, + end_position=len(content)) + + return filesystem_models.GetFileHeadResponse(output=fc, offset=offset) + + async def tail( + self: "DemoAdapter", + resource: status_models.Resource, + user: account_models.User, + path: str, + file_bytes: int | None, + lines: int | None, + skip_heading: bool, + ) -> filesystem_models.GetFileTailResponse: + + content, offset = self._headtail("tail", path, file_bytes, lines, skip_heading=skip_heading) + + fc = filesystem_models.FileContent( + content=content, + content_type=(filesystem_models.ContentUnit.bytes + if file_bytes is not None + else filesystem_models.ContentUnit.lines), + start_position=0, + end_position=len(content)) + + return filesystem_models.GetFileTailResponse(output=fc, offset=offset) + - async def tail(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int | None, lines: int | None, skip_trailing: bool) -> Tuple[Any, int]: - return self._headtail("tail", path, file_bytes, lines) async def view(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, size: int, offset: int) -> filesystem_models.GetViewFileResponse: rp = self.validate_path(path) @@ -762,12 +804,12 @@ async def rm( resource: status_models.Resource, user: account_models.User, path: str, - ): + ) -> filesystem_models.RemoveResponse: rp = self.validate_path(path) if rp == PathSandbox.get_base_temp_dir(): raise HTTPException(status_code=400, detail="Cannot delete sandbox") subprocess.run(["rm", "-rf", rp], check=True) - return None + return filesystem_models.RemoveResponse(output=f"Removed {rp}") async def mkdir(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest) -> filesystem_models.PostMkdirResponse: rp = self.validate_path(request_model.path) @@ -786,16 +828,18 @@ async def symlink( subprocess.run(["ln", "-s", rp_src, rp_dst], check=True) return filesystem_models.PostFileSymlinkResponse(output=self._file(rp_dst)) - async def download(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> Any: + async def download(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileDownloadResponse: rp = self.validate_path(path) raw_content = pathlib.Path(rp).read_bytes() if len(raw_content) > filesystem_adapter.OPS_SIZE_LIMIT: raise Exception("File to download is too large.") - return base64.b64encode(raw_content).decode("utf-8") + return filesystem_models.GetFileDownloadResponse( + output=base64.b64encode(raw_content).decode("utf-8"), + ) - async def upload(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, content: str) -> None: + async def upload(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, content: str) -> filesystem_models.PutFileUploadResponse: rp = self.validate_path(path) if isinstance(content, bytes): pathlib.Path(rp).write_bytes(content) @@ -803,6 +847,7 @@ async def upload(self: "DemoAdapter", resource: status_models.Resource, user: ac pathlib.Path(rp).write_bytes(base64.b64decode(content)) else: raise Exception(f"Don't know how to handle variable of type: {type(content)}") + return filesystem_models.PutFileUploadResponse(output=f"Uploaded to {rp}") async def compress( self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCompressRequest @@ -835,6 +880,13 @@ async def extract(self: "DemoAdapter", resource: status_models.Resource, user: a src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) + if os.path.exists(dst_rp): + if os.path.isdir(dst_rp): + raise Exception(f"Target path already exists: {request_model.target_path}") + else: + raise Exception(f"Target path already exists and is not a directory: {request_model.target_path}") + os.makedirs(dst_rp) + args = ["tar"] if request_model.compression == filesystem_models.CompressionType.gzip: args.append("-xzf") @@ -896,7 +948,7 @@ class DemoTask(BaseModel): user: account_models.User start: float status: task_models.TaskStatus = task_models.TaskStatus.pending - result: str | None = None + result: Any | None = None class DemoTaskQueue: @@ -916,7 +968,7 @@ async def _process_tasks(da: DemoAdapter): elif t.status == task_models.TaskStatus.active and now - t.start > DEMO_QUEUE_UPDATE_SECS: cmd = task_models.TaskCommand.model_validate_json(t.task) (result, status) = await DemoAdapter.on_task(t.resource, t.user, cmd) - t.result = result + t.result = jsonable_encoder(result) t.status = status _tasks.append(t) DemoTaskQueue.tasks = _tasks diff --git a/app/routers/account/account.py b/app/routers/account/account.py index c030d635..f3b36963 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -136,6 +136,8 @@ async def get_project_allocation( raise HTTPException(status_code=404, detail="User not found") projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) + if not project: + raise HTTPException(status_code=404, detail="Project not found") pas = await router.adapter.get_project_allocations(project=project, user=user) pa = next((pa for pa in pas if pa.id == project_allocation_id), None) if not pa: diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 38eca02d..968e0af6 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,4 +1,4 @@ -from enum import IntEnum +from enum import Enum from typing import Annotated from pydantic import ConfigDict, Field, StrictBool, field_serializer @@ -84,7 +84,7 @@ class CommandResult(IRIBaseModel): result: str | None = None -class JobState(IntEnum): +class JobState(str, Enum): """ from: https://exaworks.org/psij-python/docs/v/0.9.11/_modules/psij/job_state.html#JobState @@ -93,29 +93,29 @@ class JobState(IntEnum): The possible states are: `NEW`, `QUEUED`, `ACTIVE`, `COMPLETED`, `FAILED`, and `CANCELED`. """ - NEW = 0 + NEW = "new" """ This is the state of a job immediately after the :class:`~psij.Job` object is created and before being submitted to a :class:`~psij.JobExecutor`. """ - QUEUED = 1 + QUEUED = "queued" """ This is the state of the job after being accepted by a backend for execution, but before the execution of the job begins. """ - ACTIVE = 2 + ACTIVE = "active" """This state represents an actively running job.""" - COMPLETED = 3 + COMPLETED = "completed" """ This state represents a job that has completed *successfully* (i.e., with a zero exit code). In other words, a job with the executable set to `/bin/false` cannot enter this state. """ - FAILED = 4 + FAILED = "failed" """ Represents a job that has either completed unsuccessfully (with a non-zero exit code) or a job whose handling and/or execution by the backend has failed in some way. """ - CANCELED = 5 + CANCELED = "canceled" """Represents a job that was canceled by a call to :func:`~psij.Job.cancel()`.""" @@ -126,10 +126,6 @@ class JobStatus(IRIBaseModel): exit_code: int | None = None meta_data: dict[str, object] | None = None - @field_serializer("state") - def serialize_state(self, state: JobState): - return state.name - class Job(IRIBaseModel): id: str diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index afa02907..bc1d74d8 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -43,11 +43,11 @@ async def ls( pass @abstractmethod - async def head(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int, lines: int, skip_trailing: bool) -> Tuple[Any, int]: + async def head(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int, lines: int, skip_trailing: bool) -> filesystem_models.GetFileHeadResponse: pass @abstractmethod - async def tail(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int | None, lines: int | None, skip_trailing: bool) -> Tuple[Any, int]: + async def tail(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int | None, lines: int | None, skip_trailing: bool) -> filesystem_models.GetFileTailResponse: pass @abstractmethod @@ -67,7 +67,7 @@ async def stat(self: "FacilityAdapter", resource: status_models.Resource, user: pass @abstractmethod - async def rm(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str): + async def rm(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.RemoveResponse: pass @abstractmethod @@ -84,11 +84,11 @@ async def symlink( pass @abstractmethod - async def download(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> Any: + async def download(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileDownloadResponse: pass @abstractmethod - async def upload(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, content: str) -> None: + async def upload(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, content: str) -> filesystem_models.PutFileUploadResponse: pass @abstractmethod diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index 5c2cc0a4..b653b90e 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -89,10 +89,12 @@ class GetDirectoryLsResponse(CamelModel): class GetFileHeadResponse(CamelModel): output: Optional[FileContent] + offset: Optional[int] = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content") class GetFileTailResponse(CamelModel): output: Optional[FileContent] + offset: Optional[int] = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content") class GetFileChecksumResponse(CamelModel): @@ -107,6 +109,9 @@ class GetFileStatResponse(CamelModel): output: Optional[FileStat] +class GetFileDownloadResponse(CamelModel): + output: Optional[str] + class PatchFileMetadataResponse(CamelModel): output: Optional[PatchFile] @@ -143,6 +148,8 @@ class PutFileChownRequest(FilesystemRequestBase): class PutFileChownResponse(CamelModel): output: Optional[File] +class PutFileUploadResponse(CamelModel): + output: Optional[str] class PostMakeDirRequest(FilesystemRequestBase): parent: Optional[bool] = Field( @@ -261,3 +268,6 @@ class PostMoveRequest(FilesystemRequestBase): class PostMoveResponse(CamelModel): output: Optional[File] + +class RemoveResponse(CamelModel): + output: Optional[str] \ No newline at end of file diff --git a/app/routers/task/models.py b/app/routers/task/models.py index 9d64e3bf..322557d3 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,5 +1,6 @@ import enum from pydantic import BaseModel, computed_field +from typing import Any from ... import config @@ -31,5 +32,5 @@ class TaskCommand(BaseModel): class Task(BaseModel): id: str status: TaskStatus = TaskStatus.pending - result: str | None = None + result: Any | None = None command: TaskCommand | None = None diff --git a/test/test_filesystem.py b/test/test_filesystem.py new file mode 100644 index 00000000..53064c07 --- /dev/null +++ b/test/test_filesystem.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +""" +IRI Filesystem API smoke test via async tasks. +""" + +import sys +import time +import datetime as dt +import requests + + +# ========================= +# CONFIG — EDIT THESE AS NEEDED +# ========================= + +BASE_URL = "http://localhost:8000/api/v1" +TOKEN = "12345" +RESOURCE_ID = "bcc15dde-0ab6-5d31-9f20-7336adb60968" +# ========================= + +HEADERS = {"Authorization": f"Bearer {TOKEN}", "Accept": "application/json"} + +POLL_INTERVAL = 2 +TIMEOUT = 180 + + +def die(msg): + """Print error message and exit.""" + print(f"\nERROR: {msg}") + sys.exit(1) + + +def submit(method, path, **kwargs): + """Submit a task and return its ID.""" + url = f"{BASE_URL}{path}" + r = requests.request(method, url, headers=HEADERS, timeout=TIMEOUT, **kwargs) + + if not r.ok: + die(f"{method} {url} failed: {r.status_code} {r.text}") + + data = r.json() + if not data.get("task_id"): + die(f"No task_id in response: {data}") + if not data.get("task_uri"): + die(f"No task_uri in response: {data}") + + return data + + +def wait_task(task): + """Wait for a task to complete and return its result.""" + deadline = time.time() + TIMEOUT + + while time.time() < deadline: + r = requests.get(task["task_uri"], headers=HEADERS, timeout=TIMEOUT) + + if not r.ok: + die(f"Task query failed: {r.status_code} {r.text}") + + t = r.json() + status = t["status"] + + print(f" Task {task['task_id']}: {status}") + + if status == "completed": + return t.get("result") + + if status in ("failed", "canceled"): + die(f"Task {task['task_id']} ended with status {status}: {t}") + + time.sleep(POLL_INTERVAL) + + die(f"Task {task['task_id']} timed out") + + +# ============================================================ +# Sandbox setup +# ============================================================ + +timestamp = dt.datetime.utcnow().strftime("%Y%m%d-%H%M%S") + +# NOTE/TODO: /Users/jbalcas/work/amsc/iri/iri-facility-api-python/iri_sandbox/ +# While we can use absolute paths, there is a need to return relative paths from the API +# As this directory can be mounted at different locations at different facilities +base_dir = f"iri-fs-test-{timestamp}" +file_path = f"{base_dir}/hello.txt" +copy_path = f"{base_dir}/hello_copy.txt" +moved_path = f"{base_dir}/hello_moved.txt" +link_path = f"{base_dir}/hello_link.txt" +archive_path = f"{base_dir}.tar.gz" +extract_dir = f"{base_dir}_extracted" + +content = f"hello world {timestamp}\n" + + +print("\n" + "="*40) +print("=== CREATE DIRECTORY ===") + +task = submit("POST", f"/filesystem/mkdir/{RESOURCE_ID}", json={"path": base_dir, "parent": True}) +wait_task(task) + +print("\n" + "="*40) +print("=== UPLOAD FILE ===") + +task = submit("POST", f"/filesystem/upload/{RESOURCE_ID}?path={file_path}", files={"file": ("hello.txt", content.encode())}) +wait_task(task) + +print("\n" + "="*40) +print("=== FILE TYPE ===") + +task = submit("GET", f"/filesystem/file/{RESOURCE_ID}", params={"path": file_path}) +wait_task(task) + +print("\n" + "="*40) +print("=== STAT ===") + +task = submit("GET", f"/filesystem/stat/{RESOURCE_ID}", params={"path": file_path}) +wait_task(task) + +print("\n" + "="*40) +print("=== LS ===") + +task = submit("GET", f"/filesystem/ls/{RESOURCE_ID}", params={"path": base_dir}) +wait_task(task) + +print("\n" + "="*40) +print("=== CHMOD ===") + +task = submit("PUT", f"/filesystem/chmod/{RESOURCE_ID}", json={"path": file_path, "mode": "644"}) +wait_task(task) + +print("\n" + "="*40) +print("=== HEAD ===") + +task = submit("GET", f"/filesystem/head/{RESOURCE_ID}", params={"path": file_path, "lines": 1}) +wait_task(task) + +print("\n" + "="*40) +print("=== TAIL ===") + +task = submit("GET", f"/filesystem/tail/{RESOURCE_ID}", params={"path": file_path, "lines": 1}) +wait_task(task) + +print("\n" + "="*40) +print("=== VIEW ===") + +task = submit("GET", f"/filesystem/view/{RESOURCE_ID}", params={"path": file_path, "size": 4096, "offset": 0}) +print(wait_task(task)) + +print("\n" + "="*40) +print("=== CHECKSUM ===") + +task = submit("GET", f"/filesystem/checksum/{RESOURCE_ID}", params={"path": file_path}) +print(wait_task(task)) + +print("\n" + "="*40) +print("=== COPY FILE ===") + +task = submit("POST", f"/filesystem/cp/{RESOURCE_ID}", json={"sourcePath": file_path, "targetPath": copy_path}) +wait_task(task) + +print("\n" + "="*40) +print("=== MOVE FILE ===") + +task = submit("POST", f"/filesystem/mv/{RESOURCE_ID}", json={"sourcePath": copy_path, "targetPath": moved_path}) +wait_task(task) + +print("\n" + "="*40) +print("=== CREATE SYMLINK ===") + +task = submit("POST", f"/filesystem/symlink/{RESOURCE_ID}", json={"path": moved_path, "linkPath": link_path}) +wait_task(task) + +print("\n" + "="*40) +print("=== COMPRESS DIRECTORY ===") + +task = submit("POST", f"/filesystem/compress/{RESOURCE_ID}", json={"sourcePath": base_dir, "targetPath": archive_path, "compression": "gzip"}) +wait_task(task) + +print("\n" + "="*40) +print("=== EXTRACT ARCHIVE ===") + +task = submit("POST", f"/filesystem/extract/{RESOURCE_ID}", json={"sourcePath": archive_path, "targetPath": extract_dir, "compression": "gzip"}) +wait_task(task) + +print("\n" + "="*40) +print("=== DOWNLOAD FILE ===") + +task = submit("GET", f"/filesystem/download/{RESOURCE_ID}", params={"path": moved_path}) +print(wait_task(task)) + +print("\n" + "="*40) +print("=== CLEANUP ===") + +for p in [base_dir, archive_path, extract_dir]: + task = submit("DELETE", f"/filesystem/rm/{RESOURCE_ID}", params={"path": p}) + wait_task(task) + +print("\n" + "="*40) +print("ALL FILESYSTEM TESTS COMPLETED SUCCESSFULLY") +print("="*40) From 730b740bcd6193f8af3364a60ae45e7b8d281396 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 18 Feb 2026 10:29:35 -0600 Subject: [PATCH 084/133] Include last_modified and self_uri in Projects. As required by official spec --- app/demo_adapter.py | 2 ++ app/routers/account/models.py | 15 ++++++++++++++- app/types/base.py | 18 +++++++++--------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 8b0e54e3..fa2d9cfe 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -229,12 +229,14 @@ def _init_state(self): name="Staff research project", description="Compute and storage allocation for staff research use", user_ids=["gtorok"], + last_modified=day_ago, ), account_models.Project( id=demo_uuid("project", "test_project"), name="Test project", description="Compute and storage allocation for testing use", user_ids=["gtorok"], + last_modified=day_ago, ), ] diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 55406dc1..0ba13b2a 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,4 +1,6 @@ -from pydantic import Field, computed_field +import datetime + +from pydantic import Field, computed_field, field_validator from ... import config from ...types.base import IRIBaseModel @@ -23,6 +25,17 @@ class Project(IRIBaseModel): description: str user_ids: list[str] + @field_validator("last_modified", mode="before") + @classmethod + def _norm_dt_field(cls, v): + return cls.normalize_dt(v) + + last_modified: datetime.datetime + + @computed_field(description="URI to this project resource") + @property + def self_uri(self) -> str: + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/projects/{self.id}" class AllocationEntry(IRIBaseModel): """Base class for allocations.""" diff --git a/app/types/base.py b/app/types/base.py index 5052d99d..c0a55ddd 100644 --- a/app/types/base.py +++ b/app/types/base.py @@ -31,15 +31,6 @@ def get_extra(self, key, default=None): """Get an extra field value that is not defined in the model. Returns default if not found.""" return getattr(self, "__pydantic_extra__", {}).get(key, default) - -class NamedObject(IRIBaseModel): - """Base model for named objects.""" - - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") - - def _self_path(self) -> str: - raise NotImplementedError - @classmethod def normalize_dt(cls, dt: datetime | None) -> datetime | None: """Normalize datetime to UTC-aware.""" @@ -52,6 +43,15 @@ def normalize_dt(cls, dt: datetime | None) -> datetime | None: return dt.replace(tzinfo=datetime.timezone.utc) return dt + +class NamedObject(IRIBaseModel): + """Base model for named objects.""" + + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + + def _self_path(self) -> str: + raise NotImplementedError + @field_validator("last_modified", mode="before") @classmethod def _norm_dt_field(cls, v): From fffeb6858baa92ece5e8813439f758af597c4c67 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 19 Feb 2026 06:54:57 -0600 Subject: [PATCH 085/133] Filesystem validation and fixes (pyhumps, aliasChoices, testscript - upgrade humps to pyhumps 3.8.0 (support not only camelize, but also decamelize) - Add aliasChoices for target/link{Path}. Match source path - Update test script to choose storage randomly. Test camelCase, snakeCase --- app/routers/filesystem/models.py | 18 ++++++------ app/routers/task/models.py | 21 +++++++++++++- pyproject.toml | 2 +- test/test_filesystem.py | 48 +++++++++++++++++++++++++++----- 4 files changed, 72 insertions(+), 17 deletions(-) diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index b653b90e..9668ce42 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -7,7 +7,7 @@ from enum import Enum from typing import Optional -from humps.camel import case +from humps import camelize from pydantic import Field, AliasChoices, ConfigDict, BaseModel @@ -22,10 +22,12 @@ class ContentUnit(str, Enum): lines = "lines" bytes = "bytes" - +# TODO: consider using a common base model with camelCase aliasing (or snake_case) for all the models in the API +# Right now that is an issue for the worker to accept both (as client can pass either camelCase or snake_case).abs +# There is a function in Task to decamelize the args, but that is not ideal (back and forth...) class CamelModel(BaseModel): model_config = ConfigDict( - alias_generator=case, + alias_generator=camelize, arbitrary_types_allowed=True, populate_by_name=True, validate_assignment=True, @@ -160,7 +162,7 @@ class PostMakeDirRequest(FilesystemRequestBase): class PostFileSymlinkRequest(FilesystemRequestBase): - link_path: str = Field(..., description="Path to the new symlink") + link_path: str = Field(validation_alias=AliasChoices("linkPath", "link_path"), description="Path to the new symlink") model_config = {"json_schema_extra": {"examples": [{"path": "/home/user/dir", "link_path": "/home/user/newlink"}]}} @@ -181,7 +183,7 @@ class PostCompressResponse(CamelModel): class PostCompressRequest(FilesystemRequestBase): - target_path: str = Field(..., description="Path to the compressed file") + target_path: str = Field(validation_alias=AliasChoices("targetPath", "target_path"), description="Path to the compressed file") match_pattern: Optional[str] = Field(default=None, description="Regex pattern to filter files to compress") dereference: Optional[bool] = Field( default=False, @@ -211,7 +213,7 @@ class PostExtractResponse(CamelModel): class PostExtractRequest(FilesystemRequestBase): - target_path: str = Field(..., description="Path to the directory where to extract the compressed file") + target_path: str = Field(validation_alias=AliasChoices("targetPath", "target_path"), description="Path to the directory where to extract the compressed file") compression: Optional[CompressionType] = Field( default="gzip", description="Defines the type of compression to be used. By default gzip is used.", @@ -230,7 +232,7 @@ class PostExtractRequest(FilesystemRequestBase): class PostCopyRequest(FilesystemRequestBase): - target_path: str = Field(..., description="Target path of the copy operation") + target_path: str = Field(validation_alias=AliasChoices("targetPath", "target_path"), description="Target path of the copy operation") dereference: Optional[bool] = Field( default=False, description=("If set to `true`, it follows symbolic links and copies the files they point to instead of the links themselves."), @@ -253,7 +255,7 @@ class PostCopyResponse(CamelModel): class PostMoveRequest(FilesystemRequestBase): - target_path: str = Field(..., description="Target path of the move operation") + target_path: str = Field(validation_alias=AliasChoices("targetPath", "target_path"), description="Target path of the move operation") model_config = { "json_schema_extra": { "examples": [ diff --git a/app/routers/task/models.py b/app/routers/task/models.py index 322557d3..d7b4955b 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,7 +1,9 @@ import enum -from pydantic import BaseModel, computed_field +from pydantic import BaseModel, computed_field, field_validator from typing import Any +from humps import decamelize + from ... import config @@ -28,6 +30,23 @@ class TaskCommand(BaseModel): command: str args: dict + @field_validator("args", mode="before") + @classmethod + def normalize_args(cls, v): + if v is None or not isinstance(v, dict): + return v + + v = v.copy() + rm = v.get("request_model") + + if hasattr(rm, "model_dump"): + v["request_model"] = rm.model_dump(by_alias=False) + elif isinstance(rm, dict): + v["request_model"] = decamelize(rm) + + + return v + class Task(BaseModel): id: str diff --git a/pyproject.toml b/pyproject.toml index ec0ccb32..356763e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires-python = ">=3.14,<3.15" dependencies = [ "fastapi[standard]>=0.128.0,<0.129.0", "uvicorn[standard]>=0.40.0,<0.41.0", - "humps>=0.2.2,<0.3.0", + "pyhumps>=3.8.0,<3.9.0", "opentelemetry-api>=1.39.1,<1.40.0", "opentelemetry-sdk>=1.39.1,<1.40.0", "opentelemetry-instrumentation-fastapi>=0.60b1,<0.61b0", diff --git a/test/test_filesystem.py b/test/test_filesystem.py index 53064c07..790e14bf 100644 --- a/test/test_filesystem.py +++ b/test/test_filesystem.py @@ -2,9 +2,10 @@ """ IRI Filesystem API smoke test via async tasks. """ - +import os import sys import time +import random import datetime as dt import requests @@ -14,8 +15,7 @@ # ========================= BASE_URL = "http://localhost:8000/api/v1" -TOKEN = "12345" -RESOURCE_ID = "bcc15dde-0ab6-5d31-9f20-7336adb60968" +TOKEN = os.environ.get("IRI_API_TOKEN", "12345") # ========================= HEADERS = {"Authorization": f"Bearer {TOKEN}", "Accept": "application/json"} @@ -24,6 +24,37 @@ TIMEOUT = 180 +def getAnyStorageResource(): + """Get the ID of any storage resource available in the facility by looking at the project allocations and resource capabilities.""" + projects = requests.get(f"{BASE_URL}/account/projects", headers=HEADERS, timeout=TIMEOUT).json() + caps = requests.get(f"{BASE_URL}/account/capabilities", headers=HEADERS, timeout=TIMEOUT).json() + storageCaps = {c["self_uri"] for c in caps if c["name"] == "storage"} + if not storageCaps: + raise RuntimeError("No storage capabilities defined") + + projectStorageCaps = set() + for p in projects: + allocs = requests.get(f"{BASE_URL}/account/projects/{p['id']}/project_allocations", headers=HEADERS, timeout=TIMEOUT).json() + for a in allocs: + if a["capability_uri"] in storageCaps: + projectStorageCaps.add(a["capability_uri"]) + + if not projectStorageCaps: + raise RuntimeError("No storage allocations found in any project") + + resources = requests.get(f"{BASE_URL}/status/resources?offset=0&limit=100", headers=HEADERS, timeout=TIMEOUT).json() + matchingResources = [r["id"] for r in resources if any(cap in r["capability_uris"] for cap in projectStorageCaps)] + if not matchingResources: + raise RuntimeError("No storage resources found") + + return random.choice(matchingResources) + + +RESOURCE_ID = getAnyStorageResource() +print("Chosen storage resource ID:", RESOURCE_ID) + + + def die(msg): """Print error message and exit.""" print(f"\nERROR: {msg}") @@ -32,6 +63,7 @@ def die(msg): def submit(method, path, **kwargs): """Submit a task and return its ID.""" + print(f"Submitting {method} {path} with {kwargs}") url = f"{BASE_URL}{path}" r = requests.request(method, url, headers=HEADERS, timeout=TIMEOUT, **kwargs) @@ -63,6 +95,7 @@ def wait_task(task): print(f" Task {task['task_id']}: {status}") if status == "completed": + print(f" Task result: {t.get('result')}") return t.get("result") if status in ("failed", "canceled"): @@ -145,18 +178,19 @@ def wait_task(task): print("=== VIEW ===") task = submit("GET", f"/filesystem/view/{RESOURCE_ID}", params={"path": file_path, "size": 4096, "offset": 0}) -print(wait_task(task)) +wait_task(task) print("\n" + "="*40) print("=== CHECKSUM ===") task = submit("GET", f"/filesystem/checksum/{RESOURCE_ID}", params={"path": file_path}) -print(wait_task(task)) +wait_task(task) print("\n" + "="*40) print("=== COPY FILE ===") -task = submit("POST", f"/filesystem/cp/{RESOURCE_ID}", json={"sourcePath": file_path, "targetPath": copy_path}) +# Keep this as source_path. Server accepts both, so making sure it works. +task = submit("POST", f"/filesystem/cp/{RESOURCE_ID}", json={"source_path": file_path, "targetPath": copy_path}) wait_task(task) print("\n" + "="*40) @@ -187,7 +221,7 @@ def wait_task(task): print("=== DOWNLOAD FILE ===") task = submit("GET", f"/filesystem/download/{RESOURCE_ID}", params={"path": moved_path}) -print(wait_task(task)) +wait_task(task) print("\n" + "="*40) print("=== CLEANUP ===") From 5f5c6c38271b0fe1e347e4a579258b07ef3db2fd Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 19 Feb 2026 07:00:43 -0600 Subject: [PATCH 086/133] RFC 9457 compliant --- app/routers/error_handlers.py | 87 ++++++++--------------------------- 1 file changed, 20 insertions(+), 67 deletions(-) diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index 2f567902..a30b2756 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -10,7 +10,7 @@ from pydantic import BaseModel, constr from fastapi import FastAPI, HTTPException, Request -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, Response from fastapi.exceptions import RequestValidationError from starlette.exceptions import HTTPException as StarletteHTTPException @@ -19,9 +19,9 @@ class InvalidParam(BaseModel): reason: str class Problem(BaseModel): - type: constr(min_length=1) + type: str = "about:blank" title: Optional[str] = None - status: int + status: Optional[int] = None detail: Optional[str] = None instance: Optional[str] = None invalid_params: Optional[List[InvalidParam]] = None @@ -32,8 +32,8 @@ class Config: def get_url_base(request: Request) -> str: """Return the base URL for the API.""" # If behind a proxy (and x-forwarded-* headers present), use the forwarded host and protocol - host = request.headers.get("x-forwarded-host") or request.headers.get("host") - proto = request.headers.get("x-forwarded-proto") or request.url.scheme + host = (request.headers.get("x-forwarded-host") or request.headers.get("host", "")).split(",")[0].strip() + proto = (request.headers.get("x-forwarded-proto") or request.url.scheme).split(",")[0].strip() return f"{proto}://{host}/problems" @@ -57,7 +57,13 @@ def problem_response(*, request: Request, status: int, title, detail, problem_ty # Normalize title and detail to strings (Official spec says they must be strings) # but fastapi validation errors may provide lists/dicts if not isinstance(title, str): - title = "Error" + if status >= 500: + title = "Internal Server Error" + elif status >= 400: + title = "Bad Request" + else: + title = "Error" + if not isinstance(detail, str): if isinstance(detail, list): @@ -112,7 +118,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): err_msg = exc.detail if exc.status_code == 304: - return JSONResponse(status_code=304, content=None, headers=exc.headers or {}) + return Response(status_code=304, headers=exc.headers or {}) if exc.status_code == 401: return problem_response( @@ -165,7 +171,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): return problem_response( request=request, status=exc.status_code, - title=err_msg or "Error", + title="Error", detail=err_msg or "An error occurred.", problem_type="generic-error", ) @@ -199,7 +205,7 @@ async def starlette_handler(request: Request, exc: StarletteHTTPException): return problem_response( request=request, status=exc.status_code, - title=err_msg or "Error", + title="Error", detail=err_msg or "An error occurred.", problem_type="generic-error", ) @@ -216,6 +222,7 @@ async def global_handler(request: Request, exc: Exception): problem_type="internal-error", ) + EXAMPLE_400 = { "type": "https://iri.example.com/problems/invalid-parameter", "title": "Invalid parameter", @@ -304,113 +311,59 @@ async def global_handler(request: Request, exc: Exception): 400: { "description": "Invalid request parameters", "model": Problem, - "content": { - "application/problem+json": { - "example": EXAMPLE_400, - } - }, }, 401: { "description": "Unauthorized", - "model": Problem, "headers": { "WWW-Authenticate": { "description": "Bearer authentication challenge", "schema": {"type": "string"}, } }, - "content": { - "application/problem+json": { - "example": EXAMPLE_401, - } - }, + "model": Problem, + }, 403: { "description": "Forbidden", "model": Problem, - "content": { - "application/problem+json": { - "example": EXAMPLE_403, - } - }, }, 404: { "description": "Not Found", "model": Problem, - "content": { - "application/problem+json": { - "example": EXAMPLE_404, - } - }, }, 405: { "description": "Method Not Allowed", - "model": Problem, "headers": { "Allow": { "description": "Allowed HTTP methods", "schema": {"type": "string"}, } }, - "content": { - "application/problem+json": { - "example": EXAMPLE_405, - } - }, + "model": Problem, }, 409: { "description": "Conflict", "model": Problem, - "content": { - "application/problem+json": { - "example": EXAMPLE_409, - } - }, }, 422: { "description": "Unprocessable Entity", "model": Problem, - "content": { - "application/problem+json": { - "example": EXAMPLE_422, - } - }, }, 500: { "description": "Internal Server Error", "model": Problem, - "content": { - "application/problem+json": { - "example": EXAMPLE_500, - } - }, }, 501: { "description": "Not Implemented", "model": Problem, - "content": { - "application/problem+json": { - "example": EXAMPLE_501, - } - }, }, 503: { "description": "Service Unavailable", "model": Problem, - "content": { - "application/problem+json": { - "example": EXAMPLE_503, - } - }, }, 504: { "description": "Gateway Timeout", "model": Problem, - "content": { - "application/problem+json": { - "example": EXAMPLE_504, - } - }, }, 304: {"description": "Not Modified"}, -} +} \ No newline at end of file From f55cb80660da34754fc7c5ad2841ab5d3b1cf73e Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 20 Feb 2026 08:15:40 -0600 Subject: [PATCH 087/133] RFC9457 and consistent with Java expectations --- app/routers/error_handlers.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index a30b2756..cbc75167 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -6,28 +6,22 @@ import logging from urllib.parse import urlsplit, urlunsplit, quote -from typing import Optional, List -from pydantic import BaseModel, constr +from pydantic import BaseModel, Field, ConfigDict from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse, Response from fastapi.exceptions import RequestValidationError from starlette.exceptions import HTTPException as StarletteHTTPException -class InvalidParam(BaseModel): - name: str - reason: str class Problem(BaseModel): - type: str = "about:blank" - title: Optional[str] = None - status: Optional[int] = None - detail: Optional[str] = None - instance: Optional[str] = None - invalid_params: Optional[List[InvalidParam]] = None + model_config = ConfigDict(extra="allow", json_schema_extra={"description": 'Error structure for REST interface based on RFC 9457, "Problem Details for HTTP APIs."'}) + type: str = Field(..., description="A URI reference that identifies the problem type.", example="https://example.com/notFound", json_schema_extra={"format": "uri", "default": "about:blank"}) + status: int = Field(..., ge=100, le=599, description="The HTTP status code for this occurrence.", example=404) + title: str = Field(default=None, description="Short human-readable summary.", example="Not Found") + detail: str = Field(default=None, description="Human-readable explanation.", example="Descriptive text.") + instance: str = Field(..., description="A URI reference identifying this occurrence.", example="http://localhost/api/v1/resource/123") - class Config: - extra = "allow" def get_url_base(request: Request) -> str: """Return the base URL for the API.""" @@ -83,7 +77,7 @@ def problem_response(*, request: Request, status: int, title, detail, problem_ty body["invalid_params"] = invalid_params headers = extra_headers or {} - return JSONResponse(status_code=status, content=body, headers=headers, media_type="application/problem+json") + return JSONResponse(status_code=status, content=Problem(**body).model_dump(), headers=headers, media_type="application/problem+json") def install_error_handlers(app: FastAPI): From b2a5c29bc349bab471929d5d59081c122b8356d5 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 20 Feb 2026 08:56:19 -0600 Subject: [PATCH 088/133] Everything is snake_case for fs operations --- app/routers/filesystem/models.py | 97 ++++++++++++++------------------ app/routers/task/models.py | 20 +------ test/test_filesystem.py | 12 ++-- 3 files changed, 50 insertions(+), 79 deletions(-) diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index 9668ce42..57b0634d 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -7,8 +7,7 @@ from enum import Enum from typing import Optional -from humps import camelize -from pydantic import Field, AliasChoices, ConfigDict, BaseModel +from pydantic import Field, AliasChoices, BaseModel class CompressionType(str, Enum): @@ -22,19 +21,8 @@ class ContentUnit(str, Enum): lines = "lines" bytes = "bytes" -# TODO: consider using a common base model with camelCase aliasing (or snake_case) for all the models in the API -# Right now that is an issue for the worker to accept both (as client can pass either camelCase or snake_case).abs -# There is a function in Task to decamelize the args, but that is not ideal (back and forth...) -class CamelModel(BaseModel): - model_config = ConfigDict( - alias_generator=camelize, - arbitrary_types_allowed=True, - populate_by_name=True, - validate_assignment=True, - ) - -class File(CamelModel): +class File(BaseModel): name: str type: str link_target: Optional[str] @@ -45,19 +33,19 @@ class File(CamelModel): size: str -class FileContent(CamelModel): +class FileContent(BaseModel): content: str content_type: ContentUnit start_position: int end_position: int -class FileChecksum(CamelModel): +class FileChecksum(BaseModel): algorithm: str = "SHA-256" checksum: str -class FileStat(CamelModel): +class FileStat(BaseModel): # message: str mode: int ino: int @@ -72,54 +60,55 @@ class FileStat(CamelModel): # birthtime: int -class PatchFile(CamelModel): +class PatchFile(BaseModel): message: str new_filepath: str new_permissions: str new_owner: str -class PatchFileMetadataRequest(CamelModel): +class PatchFileMetadataRequest(BaseModel): new_filename: Optional[str] = None new_permissions: Optional[str] = None new_owner: Optional[str] = None -class GetDirectoryLsResponse(CamelModel): +class GetDirectoryLsResponse(BaseModel): output: Optional[list[File]] -class GetFileHeadResponse(CamelModel): +class GetFileHeadResponse(BaseModel): output: Optional[FileContent] offset: Optional[int] = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content") -class GetFileTailResponse(CamelModel): +class GetFileTailResponse(BaseModel): output: Optional[FileContent] offset: Optional[int] = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content") -class GetFileChecksumResponse(CamelModel): +class GetFileChecksumResponse(BaseModel): output: Optional[FileChecksum] -class GetFileTypeResponse(CamelModel): +class GetFileTypeResponse(BaseModel): output: Optional[str] = Field(example="directory") -class GetFileStatResponse(CamelModel): +class GetFileStatResponse(BaseModel): output: Optional[FileStat] -class GetFileDownloadResponse(CamelModel): +class GetFileDownloadResponse(BaseModel): output: Optional[str] -class PatchFileMetadataResponse(CamelModel): +class PatchFileMetadataResponse(BaseModel): output: Optional[PatchFile] -class FilesystemRequestBase(CamelModel): - path: Optional[str] = Field(validation_alias=AliasChoices("sourcePath", "source_path"), example="/home/user/dir") +class FilesystemRequestBase(BaseModel): + # Should we allow both: path and source_path? Or just one of them? + path: Optional[str] = Field(validation_alias=AliasChoices("path", "source_path"), example="/home/user/dir") class PutFileChmodRequest(FilesystemRequestBase): @@ -127,7 +116,7 @@ class PutFileChmodRequest(FilesystemRequestBase): model_config = {"json_schema_extra": {"examples": [{"path": "/home/user/dir/file.out", "mode": "777"}]}} -class PutFileChmodResponse(CamelModel): +class PutFileChmodResponse(BaseModel): output: Optional[File] @@ -147,10 +136,10 @@ class PutFileChownRequest(FilesystemRequestBase): } -class PutFileChownResponse(CamelModel): +class PutFileChownResponse(BaseModel): output: Optional[File] -class PutFileUploadResponse(CamelModel): +class PutFileUploadResponse(BaseModel): output: Optional[str] class PostMakeDirRequest(FilesystemRequestBase): @@ -162,28 +151,28 @@ class PostMakeDirRequest(FilesystemRequestBase): class PostFileSymlinkRequest(FilesystemRequestBase): - link_path: str = Field(validation_alias=AliasChoices("linkPath", "link_path"), description="Path to the new symlink") + link_path: str = Field(description="Path to the new symlink") model_config = {"json_schema_extra": {"examples": [{"path": "/home/user/dir", "link_path": "/home/user/newlink"}]}} -class PostFileSymlinkResponse(CamelModel): +class PostFileSymlinkResponse(BaseModel): output: Optional[File] -class GetViewFileResponse(CamelModel): +class GetViewFileResponse(BaseModel): output: Optional[str] -class PostMkdirResponse(CamelModel): +class PostMkdirResponse(BaseModel): output: Optional[File] -class PostCompressResponse(CamelModel): +class PostCompressResponse(BaseModel): output: Optional[File] class PostCompressRequest(FilesystemRequestBase): - target_path: str = Field(validation_alias=AliasChoices("targetPath", "target_path"), description="Path to the compressed file") + target_path: str = Field(description="Path to the compressed file") match_pattern: Optional[str] = Field(default=None, description="Regex pattern to filter files to compress") dereference: Optional[bool] = Field( default=False, @@ -197,9 +186,9 @@ class PostCompressRequest(FilesystemRequestBase): "json_schema_extra": { "examples": [ { - "sourcePath": "/home/user/dir", - "targetPath": "/home/user/file.tar.gz", - "matchPattern": "*./[ab].*\\.txt", + "source_path": "/home/user/dir", + "target_path": "/home/user/file.tar.gz", + "match_pattern": "*./[ab].*\\.txt", "dereference": "true", "compression": "none", } @@ -208,12 +197,12 @@ class PostCompressRequest(FilesystemRequestBase): } -class PostExtractResponse(CamelModel): +class PostExtractResponse(BaseModel): output: Optional[File] class PostExtractRequest(FilesystemRequestBase): - target_path: str = Field(validation_alias=AliasChoices("targetPath", "target_path"), description="Path to the directory where to extract the compressed file") + target_path: str = Field(description="Path to the directory where to extract the compressed file") compression: Optional[CompressionType] = Field( default="gzip", description="Defines the type of compression to be used. By default gzip is used.", @@ -222,8 +211,8 @@ class PostExtractRequest(FilesystemRequestBase): "json_schema_extra": { "examples": [ { - "sourcePath": "/home/user/dir/file.tar.gz", - "targetPath": "/home/user/dir", + "source_path": "/home/user/dir/file.tar.gz", + "target_path": "/home/user/dir", "compression": "none", } ] @@ -232,7 +221,7 @@ class PostExtractRequest(FilesystemRequestBase): class PostCopyRequest(FilesystemRequestBase): - target_path: str = Field(validation_alias=AliasChoices("targetPath", "target_path"), description="Target path of the copy operation") + target_path: str = Field(description="Target path of the copy operation") dereference: Optional[bool] = Field( default=False, description=("If set to `true`, it follows symbolic links and copies the files they point to instead of the links themselves."), @@ -241,8 +230,8 @@ class PostCopyRequest(FilesystemRequestBase): "json_schema_extra": { "examples": [ { - "sourcePath": "/home/user/dir/file.orig", - "targetPath": "/home/user/dir/file.new", + "source_path": "/home/user/dir/file.orig", + "target_path": "/home/user/dir/file.new", "dereference": "true", } ] @@ -250,26 +239,26 @@ class PostCopyRequest(FilesystemRequestBase): } -class PostCopyResponse(CamelModel): +class PostCopyResponse(BaseModel): output: Optional[File] class PostMoveRequest(FilesystemRequestBase): - target_path: str = Field(validation_alias=AliasChoices("targetPath", "target_path"), description="Target path of the move operation") + target_path: str = Field(description="Target path of the move operation") model_config = { "json_schema_extra": { "examples": [ { - "sourcePath": "/home/user/dir/file.orig", - "targetPath": "/home/user/dir/file.new", + "source_path": "/home/user/dir/file.orig", + "target_path": "/home/user/dir/file.new", } ] } } -class PostMoveResponse(CamelModel): +class PostMoveResponse(BaseModel): output: Optional[File] -class RemoveResponse(CamelModel): +class RemoveResponse(BaseModel): output: Optional[str] \ No newline at end of file diff --git a/app/routers/task/models.py b/app/routers/task/models.py index d7b4955b..ba124bc6 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,8 +1,7 @@ import enum -from pydantic import BaseModel, computed_field, field_validator from typing import Any +from pydantic import BaseModel, computed_field -from humps import decamelize from ... import config @@ -30,23 +29,6 @@ class TaskCommand(BaseModel): command: str args: dict - @field_validator("args", mode="before") - @classmethod - def normalize_args(cls, v): - if v is None or not isinstance(v, dict): - return v - - v = v.copy() - rm = v.get("request_model") - - if hasattr(rm, "model_dump"): - v["request_model"] = rm.model_dump(by_alias=False) - elif isinstance(rm, dict): - v["request_model"] = decamelize(rm) - - - return v - class Task(BaseModel): id: str diff --git a/test/test_filesystem.py b/test/test_filesystem.py index 790e14bf..62f04ad0 100644 --- a/test/test_filesystem.py +++ b/test/test_filesystem.py @@ -28,7 +28,7 @@ def getAnyStorageResource(): """Get the ID of any storage resource available in the facility by looking at the project allocations and resource capabilities.""" projects = requests.get(f"{BASE_URL}/account/projects", headers=HEADERS, timeout=TIMEOUT).json() caps = requests.get(f"{BASE_URL}/account/capabilities", headers=HEADERS, timeout=TIMEOUT).json() - storageCaps = {c["self_uri"] for c in caps if c["name"] == "storage"} + storageCaps = {c["self_uri"] for c in caps if c["name"] == "GPFS Storage"} if not storageCaps: raise RuntimeError("No storage capabilities defined") @@ -190,31 +190,31 @@ def wait_task(task): print("=== COPY FILE ===") # Keep this as source_path. Server accepts both, so making sure it works. -task = submit("POST", f"/filesystem/cp/{RESOURCE_ID}", json={"source_path": file_path, "targetPath": copy_path}) +task = submit("POST", f"/filesystem/cp/{RESOURCE_ID}", json={"source_path": file_path, "target_path": copy_path}) wait_task(task) print("\n" + "="*40) print("=== MOVE FILE ===") -task = submit("POST", f"/filesystem/mv/{RESOURCE_ID}", json={"sourcePath": copy_path, "targetPath": moved_path}) +task = submit("POST", f"/filesystem/mv/{RESOURCE_ID}", json={"source_path": copy_path, "target_path": moved_path}) wait_task(task) print("\n" + "="*40) print("=== CREATE SYMLINK ===") -task = submit("POST", f"/filesystem/symlink/{RESOURCE_ID}", json={"path": moved_path, "linkPath": link_path}) +task = submit("POST", f"/filesystem/symlink/{RESOURCE_ID}", json={"path": moved_path, "link_path": link_path}) wait_task(task) print("\n" + "="*40) print("=== COMPRESS DIRECTORY ===") -task = submit("POST", f"/filesystem/compress/{RESOURCE_ID}", json={"sourcePath": base_dir, "targetPath": archive_path, "compression": "gzip"}) +task = submit("POST", f"/filesystem/compress/{RESOURCE_ID}", json={"source_path": base_dir, "target_path": archive_path, "compression": "gzip"}) wait_task(task) print("\n" + "="*40) print("=== EXTRACT ARCHIVE ===") -task = submit("POST", f"/filesystem/extract/{RESOURCE_ID}", json={"sourcePath": archive_path, "targetPath": extract_dir, "compression": "gzip"}) +task = submit("POST", f"/filesystem/extract/{RESOURCE_ID}", json={"source_path": archive_path, "target_path": extract_dir, "compression": "gzip"}) wait_task(task) print("\n" + "="*40) From 42cea9f33fad259f21a641422de964a622943650 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Sun, 22 Feb 2026 19:19:04 -0600 Subject: [PATCH 089/133] OpenAPI 3.1 update (doc, anyOf remove). Logger, DemoAdapter upd - Avoid anyOf where possible; - Add logger and print all log files; Allow control logger level; - Demo adapter: add CommandError for subprocess run. run subprocess with check=True, raise error if fails; - Add descriptions to all models, get rid of str | None - this produces anyOf. Make spec compliant with Johns code. - Facility worker (skip None values). If field str, cant assign None. - Remove pyhums (no need capitalize, decapitalize) --- app/apilogger.py | 28 ++++ app/config.py | 25 +++- app/demo_adapter.py | 132 +++++++++++----- app/routers/account/models.py | 54 ++++--- app/routers/compute/facility_adapter.py | 4 - app/routers/compute/models.py | 93 ++++++------ app/routers/facility/models.py | 31 ++-- app/routers/filesystem/models.py | 191 ++++++++++++++---------- app/routers/status/models.py | 41 ++--- app/routers/task/facility_adapter.py | 35 ++++- app/routers/task/models.py | 24 +-- app/types/base.py | 10 +- app/types/http.py | 14 +- app/types/models.py | 4 +- app/types/scalars.py | 11 +- pyproject.toml | 1 - 16 files changed, 417 insertions(+), 281 deletions(-) create mode 100644 app/apilogger.py diff --git a/app/apilogger.py b/app/apilogger.py new file mode 100644 index 00000000..80403a9f --- /dev/null +++ b/app/apilogger.py @@ -0,0 +1,28 @@ +"""Logging utilities for the IRI Facility API.""" +import logging + +LEVELS = {"FATAL": logging.FATAL, + "ERROR": logging.ERROR, + "WARNING": logging.WARNING, + "INFO": logging.INFO, + "DEBUG": logging.DEBUG} + + +def get_stream_logger(name: str = __name__, level: str = "DEBUG") -> logging.Logger: + """ + Return a configured Stream logger. + """ + logger = logging.getLogger(name) + + if not logger.handlers: + handler = logging.StreamHandler() + + formatter = logging.Formatter("%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s", datefmt="%a, %d %b %Y %H:%M:%S") + + handler.setFormatter(formatter) + logger.addHandler(handler) + + logger.setLevel(LEVELS.get(level, logging.DEBUG)) + logger.propagate = False + + return logger \ No newline at end of file diff --git a/app/config.py b/app/config.py index c8c53025..35c17c93 100644 --- a/app/config.py +++ b/app/config.py @@ -1,9 +1,11 @@ +"""Configuration for the IRI Facility API reference implementation.""" import os -import logging import json +from .apilogger import get_stream_logger -logging.basicConfig() -logging.getLogger().setLevel(logging.INFO) +LOG_LEVEL = os.environ.get("LOG_LEVEL", "DEBUG") + +logger = get_stream_logger(__name__, LOG_LEVEL) API_VERSION = "1.0.0" @@ -31,7 +33,7 @@ d2 = json.loads(os.environ.get("IRI_API_PARAMS", "{}")) API_CONFIG.update(d2) except Exception as exc: - logging.getLogger().error(f"Error parsing IRI_API_PARAMS: {exc}") + logger.error(f"Error parsing IRI_API_PARAMS: {exc}") API_URL_ROOT = os.environ.get("API_URL_ROOT", "https://api.iri.nersc.gov") @@ -42,3 +44,18 @@ OPENTELEMETRY_DEBUG = os.environ.get("OPENTELEMETRY_DEBUG", "false").lower() == "true" OTLP_ENDPOINT = os.environ.get("OTLP_ENDPOINT", "") OTEL_SAMPLE_RATE = float(os.environ.get("OTEL_SAMPLE_RATE", "0.2")) + +# Print all startup config for debugging +logger.info("IRI Facility API starting with config:") +logger.info("="*40) +logger.info(f"API_VERSION={API_VERSION}") +logger.info(f"API_CONFIG={API_CONFIG}") +logger.info(f"API_URL_ROOT={API_URL_ROOT}") +logger.info(f"API_PREFIX={API_PREFIX}") +logger.info(f"API_URL={API_URL}") +logger.info(f"LOG_LEVEL={LOG_LEVEL}") +logger.info(f"OPENTELEMETRY_ENABLED={OPENTELEMETRY_ENABLED}") +logger.info(f"OPENTELEMETRY_DEBUG={OPENTELEMETRY_DEBUG}") +logger.info(f"OTLP_ENDPOINT={OTLP_ENDPOINT}") +logger.info(f"OTEL_SAMPLE_RATE={OTEL_SAMPLE_RATE}") +logger.info("="*40) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index fa2d9cfe..ec86fbdd 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -1,3 +1,7 @@ +""" +A demo adapter for the IRI Facility API that returns hardcoded data. +This is useful for testing and development of the API without needing to connect to real resources +""" import base64 import datetime import glob @@ -29,6 +33,10 @@ from .routers.task import models as task_models from .types.models import Capability from .types.scalars import AllocationUnit +from .apilogger import get_stream_logger +from .config import LOG_LEVEL + +logger = get_stream_logger(__name__, LOG_LEVEL) DEMO_QUEUE_UPDATE_SECS = int(os.environ.get("DEMO_QUEUE_UPDATE_SECS", 5)) @@ -41,12 +49,25 @@ def paginate_list(items, offset: int | None, limit: int | None): items = items[:limit] return items +class CommandError(RuntimeError): + """Raised when an external subprocess command fails.""" + + def __init__(self, cmd, returncode=None, stdout=None, stderr=None): + self.cmd = cmd + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + super().__init__(f"Command failed: {cmd} (rc={returncode})") + class PathSandbox: + """A simple sandbox for file operations.""" _base_temp_dir = None @classmethod def get_base_temp_dir(cls): + """Get the base temporary directory for the sandbox.""" if cls._base_temp_dir is None: # Create in system temp with a fixed name cls._base_temp_dir = os.path.join(os.getcwd(), "iri_sandbox") @@ -55,10 +76,12 @@ def get_base_temp_dir(cls): # create a test file with open(f"{cls._base_temp_dir}/test.txt", encoding="utf-8", mode="w") as f: f.write("hello world") + logger.info(f"Created test file in sandbox: {cls._base_temp_dir}/test.txt") return cls._base_temp_dir def demo_uuid(kind: str, name: str) -> str: + """Generate a deterministic UUID based on the kind and name.""" return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"demo:{kind}:{name}")) @@ -75,6 +98,7 @@ def utc_timestamp() -> int: class DemoAdapter( status_adapter.FacilityAdapter, account_adapter.FacilityAdapter, compute_adapter.FacilityAdapter, filesystem_adapter.FacilityAdapter, task_adapter.FacilityAdapter, facility_adapter.FacilityAdapter ): + """A demo implementation of the FacilityAdapter that returns hardcoded data.""" def __init__(self): self.resources = [] self.incidents = [] @@ -331,7 +355,7 @@ async def list_sites( sites = self.sites if name: - sites = [s for s in sites if name.lower() in s.name.lower()] + sites = [s for s in sites if name.lower() in s.name.lower()] # pylint: disable=no-member if short_name: sites = [s for s in sites if s.short_name == short_name] @@ -509,25 +533,7 @@ async def submit_job( state=compute_models.JobState.NEW, time=utc_timestamp(), message="job submitted", - exit_code=None, - meta_data={"account": "account1"}, - ), - ) - - async def submit_job_script( - self: "DemoAdapter", - resource: status_models.Resource, - user: account_models.User, - job_script_path: str, - args: list[str] = [], - ) -> compute_models.Job: - return compute_models.Job( - id="job_123", - status=compute_models.JobStatus( - state=compute_models.JobState.NEW, - time=utc_timestamp(), - message="job submitted", - exit_code=None, + exit_code=0, meta_data={"account": "account1"}, ), ) @@ -545,7 +551,7 @@ async def update_job( state=compute_models.JobState.ACTIVE, time=utc_timestamp(), message="job updated", - exit_code=None, + exit_code=0, meta_data={"account": "account1"}, ), ) @@ -603,6 +609,7 @@ async def cancel_job( return True def validate_path(self, path: str, allow_symlinks: bool = True) -> str: + """Validate that the given path is within the sandbox base directory and optionally check for symlinks.""" basedir = PathSandbox.get_base_temp_dir() real_path = os.path.realpath(os.path.join(basedir, path)) @@ -618,6 +625,27 @@ def validate_path(self, path: str, allow_symlinks: bool = True) -> str: return real_path +# ---------------------------------------------- +# Filesystem API +# ---------------------------------------------- + def _run(self, args, *, shell: bool = False, timeout: int | None = 3600, text: bool = True) -> subprocess.CompletedProcess: + """ + Run a subprocess command and catch exceptions. + Raises CommandError on failure with captured diagnostics. + """ + try: + return subprocess.run(args, shell=shell, capture_output=True, text=text, check=True, timeout=timeout) + except subprocess.TimeoutExpired as exc: + logger.warning(f"Command timed out: {args} (after {timeout} seconds)") + raise CommandError(cmd=args, returncode=None, stdout=exc.stdout, stderr=exc.stderr) from exc + except subprocess.CalledProcessError as exc: + logger.warning(f"Command failed: {args} (rc={exc.returncode})\nstdout: {exc.stdout}\nstderr: {exc.stderr}") + raise CommandError(cmd=args, returncode=exc.returncode, stdout=exc.stdout, stderr=exc.stderr) from exc + except OSError as exc: + logger.warning(f"OS error running command: {args}\nError: {exc}") + raise CommandError(cmd=args, returncode=None, stdout=None, stderr=str(exc)) from exc + + def _file(self, path: str) -> filesystem_models.File: # Get file stats (follows symlinks by default) rp = self.validate_path(path) @@ -650,8 +678,21 @@ def _file(self, path: str) -> filesystem_models.File: # Get size size = str(file_stat.st_size) + data = dict( + name=os.path.basename(rp), + type=file_type, + user=user, + group=group, + permissions=permissions, + last_modified=last_modified, + size=size, + ) + + if link_target is not None: + data["link_target"] = link_target + + return filesystem_models.File(**data) - return filesystem_models.File(name=os.path.basename(rp), type=file_type, link_target=link_target, user=user, group=group, permissions=permissions, last_modified=last_modified, size=size) async def chmod(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChmodRequest) -> filesystem_models.PutFileChmodResponse: rp = self.validate_path(request_model.path) @@ -666,7 +707,7 @@ async def chown( ) -> filesystem_models.PutFileChownResponse: rp = self.validate_path(request_model.path) os.chown(rp, request_model.owner, request_model.group) - return filesystem_models.PutFileChmodResponse(output=self._file(rp)) + return filesystem_models.PutFileChownResponse(output=self._file(rp)) async def ls( self: "DemoAdapter", @@ -689,6 +730,7 @@ def _headtail( file_bytes: int | None, lines: int | None, skip_heading: bool = False, + skip_trailing: bool = False, ) -> Tuple[Any, int]: args = [cmd] @@ -697,6 +739,11 @@ def _headtail( args.extend(["-c", f"+{file_bytes + 1}"]) elif lines is not None: args.extend(["-n", f"+{lines + 1}"]) + if cmd == "head" and skip_trailing: + if file_bytes is not None: + args.extend(["-c", f"-{file_bytes}"]) + elif lines is not None: + args.extend(["-n", f"-{lines}"]) else: if file_bytes is not None: args.extend(["-c", str(file_bytes)]) @@ -706,7 +753,7 @@ def _headtail( rp = self.validate_path(path) args.append(rp) - result = subprocess.run(args, capture_output=True, text=True) + result = self._run(args) content = result.stdout return content, len(content) @@ -717,9 +764,9 @@ async def head( path: str, file_bytes: int | None, lines: int | None, - skip_trailing: bool, + skip_trailing: bool = False, ) -> filesystem_models.GetFileHeadResponse: - content, offset = self._headtail("head", path, file_bytes, lines) + content, offset = self._headtail("head", path, file_bytes, lines, skip_trailing=skip_trailing) fc = filesystem_models.FileContent( content=content, @@ -738,7 +785,7 @@ async def tail( path: str, file_bytes: int | None, lines: int | None, - skip_heading: bool, + skip_heading: bool = False, ) -> filesystem_models.GetFileTailResponse: content, offset = self._headtail("tail", path, file_bytes, lines, skip_heading=skip_heading) @@ -757,7 +804,7 @@ async def tail( async def view(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, size: int, offset: int) -> filesystem_models.GetViewFileResponse: rp = self.validate_path(path) - result = subprocess.run(f"tail -c +{offset + 1} {rp} | head -c {size}", shell=True, capture_output=True, text=True) + result = self._run(f"tail -c +{offset + 1} {rp} | head -c {size}", shell=True) content = result.stdout return filesystem_models.GetViewFileResponse( output=content, @@ -765,7 +812,7 @@ async def view(self: "DemoAdapter", resource: status_models.Resource, user: acco async def checksum(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileChecksumResponse: rp = self.validate_path(path) - result = subprocess.run(["sha256sum", rp], capture_output=True, text=True) + result = self._run(["sha256sum", rp]) checksum = result.stdout.split()[0] return filesystem_models.GetFileChecksumResponse( output=filesystem_models.FileChecksum( @@ -775,7 +822,7 @@ async def checksum(self: "DemoAdapter", resource: status_models.Resource, user: async def file(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileTypeResponse: rp = self.validate_path(path) - result = subprocess.run(["file", "-b", rp], capture_output=True, text=True) + result = self._run(["file", "-b", rp]) return filesystem_models.GetFileTypeResponse( output=result.stdout.strip(), ) @@ -810,7 +857,7 @@ async def rm( rp = self.validate_path(path) if rp == PathSandbox.get_base_temp_dir(): raise HTTPException(status_code=400, detail="Cannot delete sandbox") - subprocess.run(["rm", "-rf", rp], check=True) + self._run(["rm", "-rf", rp]) return filesystem_models.RemoveResponse(output=f"Removed {rp}") async def mkdir(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest) -> filesystem_models.PostMkdirResponse: @@ -819,7 +866,7 @@ async def mkdir(self: "DemoAdapter", resource: status_models.Resource, user: acc if request_model.parent: args.append("-p") args.append(rp) - subprocess.run(args, check=True) + self._run(args) return filesystem_models.PostMkdirResponse(output=self._file(rp)) async def symlink( @@ -827,7 +874,7 @@ async def symlink( ) -> filesystem_models.PostFileSymlinkResponse: rp_src = self.validate_path(request_model.path) rp_dst = self.validate_path(request_model.link_path) - subprocess.run(["ln", "-s", rp_src, rp_dst], check=True) + self._run(["ln", "-s", rp_src, rp_dst]) return filesystem_models.PostFileSymlinkResponse(output=self._file(rp_dst)) async def download(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileDownloadResponse: @@ -923,19 +970,19 @@ async def cp(self: "DemoAdapter", resource: status_models.Resource, user: accoun return filesystem_models.PostCopyResponse(output=self._file(dst_rp)) async def get_task(self: "DemoAdapter", user: account_models.User, task_id: str) -> task_models.Task | None: - await DemoTaskQueue._process_tasks(self) + await DemoTaskQueue.process_tasks(self) return next((t for t in DemoTaskQueue.tasks if t.user.name == user.name and t.id == task_id), None) async def get_tasks(self: "DemoAdapter", user: account_models.User) -> list[task_models.Task]: - await DemoTaskQueue._process_tasks(self) + await DemoTaskQueue.process_tasks(self) return [t for t in DemoTaskQueue.tasks if t.user.name == user.name] async def put_task(self: "DemoAdapter", user: account_models.User, resource: status_models.Resource, task: str) -> task_models.TaskSubmitResponse: - await DemoTaskQueue._process_tasks(self) - return DemoTaskQueue._create_task(user, resource, task) + await DemoTaskQueue.process_tasks(self) + return DemoTaskQueue.create_task(user, resource, task) async def delete_task(self: "DemoAdapter", user: account_models.User, task_id: str) -> None: - await DemoTaskQueue._process_tasks(self) + await DemoTaskQueue.process_tasks(self) for t in DemoTaskQueue.tasks: if t.user.name == user.name and t.id == task_id: t.status = task_models.TaskStatus.canceled @@ -944,6 +991,7 @@ async def delete_task(self: "DemoAdapter", user: account_models.User, task_id: s class DemoTask(BaseModel): + """A simple in-memory task queue for demonstration purposes.""" id: str task: str resource: status_models.Resource @@ -954,10 +1002,12 @@ class DemoTask(BaseModel): class DemoTaskQueue: + """A simple in-memory task queue for demonstration purposes.""" tasks = [] @staticmethod - async def _process_tasks(da: DemoAdapter): + async def process_tasks(da: DemoAdapter): + """Process tasks in the queue, simulating task execution and completion.""" now = utc_timestamp() _tasks = [] for t in DemoTaskQueue.tasks: @@ -976,7 +1026,9 @@ async def _process_tasks(da: DemoAdapter): DemoTaskQueue.tasks = _tasks @staticmethod - def _create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> task_models.TaskSubmitResponse: + def create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> task_models.TaskSubmitResponse: + """Create a new task in the queue.""" task_id = f"task_{len(DemoTaskQueue.tasks)}" DemoTaskQueue.tasks.append(DemoTask(id=task_id, task=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) + logger.info(f"Created task: {task_id}") return task_models.TaskSubmitResponse(task_id=task_id) diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 0ba13b2a..cd7cf812 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,3 +1,4 @@ +"""Models for account-related API endpoints, including users, projects, and allocations.""" import datetime from pydantic import Field, computed_field, field_validator @@ -10,39 +11,41 @@ class User(IRIBaseModel): """A user of the facility""" - id: str - name: str - api_key: str - client_ip: str | None + id: str = Field(..., description="Unique identifier of the user.", example="user-123") + name: str = Field(..., description="Name of the user.", example="Jane Doe") + api_key: str = Field(..., description="API key associated with this user.", example="AKIAIOSFODNN7EXAMPLE") + client_ip: str = Field(default=None, description="IP address from which the user connects.", example="192.0.2.10") # we could expose more fields here (eg. email) but it might be against policy class Project(IRIBaseModel): """A project and its users at a facility""" - id: str - name: str - description: str - user_ids: list[str] + id: str = Field(..., description="Unique identifier of the project.", example="proj-abc123") + name: str = Field(..., description="Human-readable name of the project.", example="Climate Simulation") + description: str = Field(..., description="Detailed description of the project.", example="Research project studying atmospheric dynamics.") + user_ids: list[str] = Field(..., description="List of user identifiers participating in the project.", example=["user-123", "user-456"]) @field_validator("last_modified", mode="before") @classmethod def _norm_dt_field(cls, v): return cls.normalize_dt(v) - last_modified: datetime.datetime + last_modified: datetime.datetime = Field(..., description="Timestamp of the last modification of the project.", example="2026-02-21T14:30:00Z") @computed_field(description="URI to this project resource") @property def self_uri(self) -> str: + """Return the URI for this project resource.""" return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/projects/{self.id}" + class AllocationEntry(IRIBaseModel): """Base class for allocations.""" - allocation: float # how much this allocation can spend - usage: float # how much this allocation has spent - unit: AllocationUnit + allocation: float = Field(..., description="Total allocation amount granted.", example=100000.0) # how much this allocation can spend + usage: float = Field(..., description="Amount of allocation consumed.", example=52342.5) # how much this allocation has spent + unit: AllocationUnit = Field(..., description="Unit of the allocation (e.g., node_hours, bytes).", example="node_hours") class ProjectAllocation(IRIBaseModel): @@ -53,19 +56,21 @@ class ProjectAllocation(IRIBaseModel): """ # how much this allocation can spend - id: str - project_id: str = Field(exclude=True) - capability_id: str = Field(exclude=True) - entries: list[AllocationEntry] + id: str = Field(..., description="Unique identifier of the project allocation.", example="alloc-001") + project_id: str = Field(exclude=True, description="Internal identifier of the associated project.") + capability_id: str = Field(exclude=True, description="Internal identifier of the associated capability.") + entries: list[AllocationEntry] = Field(..., description="Allocation entries describing usage and limits.") - @computed_field(description="The list of past events in this incident") + @computed_field(description="URI of the associated project resource") @property def project_uri(self) -> str: + """Return the URI for the associated project resource.""" return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/projects/{self.project_id}" - @computed_field(description="The list of past events in this incident") + @computed_field(description="URI of the associated capability resource") @property def capability_uri(self) -> str: + """Return the URI for the associated capability.""" return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{self.capability_id}" @@ -75,13 +80,14 @@ class UserAllocation(IRIBaseModel): This allocation is a piece of the project's allocation. """ - id: str - project_id: str = Field(exclude=True) - project_allocation_id: str = Field(exclude=True) - user_id: str - entries: list[AllocationEntry] + id: str = Field(..., description="Unique identifier of the user allocation.", example="user-alloc-42") + project_id: str = Field(exclude=True, description="Internal identifier of the associated project.") + project_allocation_id: str = Field(exclude=True, description="Internal identifier of the associated project allocation.") + user_id: str = Field(..., description="Identifier of the user receiving this allocation.", example="user-123") + entries: list[AllocationEntry] = Field(..., description="Allocation entries describing usage and limits.") - @computed_field(description="The list of past events in this incident") + @computed_field(description="URI of the associated project allocation") @property def project_allocation_uri(self) -> str: + """Return the URI for the associated project allocation.""" return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/projects/{self.project_id}/project_allocations/{self.project_allocation_id}" diff --git a/app/routers/compute/facility_adapter.py b/app/routers/compute/facility_adapter.py index 1370aedb..608217e1 100644 --- a/app/routers/compute/facility_adapter.py +++ b/app/routers/compute/facility_adapter.py @@ -16,10 +16,6 @@ class FacilityAdapter(AuthenticatedAdapter): async def submit_job(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec) -> compute_models.Job: pass - @abstractmethod - async def submit_job_script(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_script_path: str, args: list[str] = []) -> compute_models.Job: - pass - @abstractmethod async def update_job(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, job_id: str) -> compute_models.Job: pass diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 968e0af6..c687c4a7 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -1,7 +1,8 @@ +"""Models for compute router, including job specifications, job status, and related data structures.""" from enum import Enum from typing import Annotated -from pydantic import ConfigDict, Field, StrictBool, field_serializer +from pydantic import ConfigDict, Field, StrictBool from ...types.base import IRIBaseModel @@ -11,13 +12,13 @@ class ResourceSpec(IRIBaseModel): Specification of computational resources required for a job. """ - node_count: Annotated[int | None, Field(ge=1, description="Number of compute nodes to allocate")] = None - process_count: Annotated[int | None, Field(ge=1, description="Total number of processes to launch")] = None - processes_per_node: Annotated[int | None, Field(ge=1, description="Number of processes to launch per node")] = None - cpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of CPU cores to allocate per process")] = None - gpu_cores_per_process: Annotated[int | None, Field(ge=1, description="Number of GPU cores to allocate per process")] = None - exclusive_node_use: Annotated[StrictBool, Field(description="Whether to request exclusive use of allocated nodes")] = True - memory: Annotated[int | None, Field(ge=1, description="Amount of memory to allocate in bytes")] = None + node_count: Annotated[int, Field(ge=1, description="Number of compute nodes to allocate", example=2)] = None + process_count: Annotated[int, Field(ge=1, description="Total number of processes to launch", example=64)] = None + processes_per_node: Annotated[int, Field(ge=1, description="Number of processes to launch per node", example=32)] = None + cpu_cores_per_process: Annotated[int, Field(ge=1, description="Number of CPU cores to allocate per process", example=2)] = None + gpu_cores_per_process: Annotated[int, Field(ge=1, description="Number of GPU cores to allocate per process", example=1)] = None + exclusive_node_use: Annotated[StrictBool, Field(description="Whether to request exclusive use of allocated nodes", example=True)] = True + memory: Annotated[int, Field(ge=1, description="Amount of memory to allocate in bytes", example=17179869184)] = None class JobAttributes(IRIBaseModel): @@ -25,11 +26,11 @@ class JobAttributes(IRIBaseModel): Additional attributes and scheduling parameters for a job. """ - duration: Annotated[int | None, Field(description="Duration in seconds", ge=1, examples=[30, 60, 120])] = None - queue_name: Annotated[str | None, Field(min_length=1, description="Name of the queue or partition to submit the job to")] = None - account: Annotated[str | None, Field(min_length=1, description="Account or project to charge for resource usage")] = None - reservation_id: Annotated[str | None, Field(min_length=1, description="ID of a reservation to use for the job")] = None - custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs")] = {} + duration: Annotated[int, Field(description="Duration in seconds", ge=1, examples=[30, 60, 120])] = None + queue_name: Annotated[str, Field(min_length=1, description="Name of the queue or partition to submit the job to", example="debug")] = None + account: Annotated[str, Field(min_length=1, description="Account or project to charge for resource usage", example="proj123")] = None + reservation_id: Annotated[str, Field(min_length=1, description="ID of a reservation to use for the job", example="resv-42")] = None + custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs", example={"constraint": "gpu"})] = {} class VolumeMount(IRIBaseModel): @@ -37,9 +38,9 @@ class VolumeMount(IRIBaseModel): Represents a volume mount for a container. """ - source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount")] - target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted")] - read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only")] = True + source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount", example="/data/project")] + target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted", example="/mnt/data")] + read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only", example=True)] = True class Container(IRIBaseModel): @@ -52,36 +53,34 @@ class Container(IRIBaseModel): host networking. """ - image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')")] + image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')", example="docker.io/library/ubuntu:latest")] volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] class JobSpec(IRIBaseModel): """ - Specification for job. + Specification for a job. """ model_config = ConfigDict(extra="forbid") - executable: Annotated[str | None, Field(min_length=1, description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.")] = None - container: Annotated[Container | None, Field(description="Container specification for containerized execution")] = None - arguments: Annotated[list[str], Field(description="Command-line arguments to pass to the executable or container")] = [] - directory: Annotated[str | None, Field(min_length=1, description="Working directory for the job")] = None - name: Annotated[str | None, Field(min_length=1, description="Name of the job")] = None - inherit_environment: Annotated[StrictBool, Field(description="Whether to inherit the environment variables from the submission environment")] = True - environment: Annotated[dict[str, str], Field(description="Environment variables to set for the job. If container is specified, these will be set inside the container.")] = {} - stdin_path: Annotated[str | None, Field(min_length=1, description="Path to file to use as standard input")] = None - stdout_path: Annotated[str | None, Field(min_length=1, description="Path to file to write standard output")] = None - stderr_path: Annotated[str | None, Field(min_length=1, description="Path to file to write standard error")] = None - resources: Annotated[ResourceSpec | None, Field(description="Resource requirements for the job")] = None - attributes: Annotated[JobAttributes | None, Field(description="Additional job attributes such as duration, queue, and account")] = None - pre_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run before launching the job")] = None - post_launch: Annotated[str | None, Field(min_length=1, description="Script or commands to run after the job completes")] = None - launcher: Annotated[str | None, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')")] = None - - -class CommandResult(IRIBaseModel): - status: str - result: str | None = None + executable: Annotated[str, Field(min_length=1, + description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.", + example="/usr/bin/python")] = None + container: Annotated[Container, Field(description="Container specification for containerized execution")] = None + arguments: Annotated[list[str], Field(description="Command-line arguments to pass to the executable or container", example=["-n", "100"])] = [] + directory: Annotated[str, Field(min_length=1, description="Working directory for the job", example="/home/user/work")] = None + name: Annotated[str, Field(min_length=1, description="Name of the job", example="my-job")] = None + inherit_environment: Annotated[StrictBool, Field(description="Whether to inherit the environment variables from the submission environment", example=True)] = True + environment: Annotated[dict[str, str], Field(description="Environment variables to set for the job. If container is specified, these will be set inside the container.", + example={"OMP_NUM_THREADS": "4"})] = {} + stdin_path: Annotated[str, Field(min_length=1, description="Path to file to use as standard input", example="/home/user/input.txt")] = None + stdout_path: Annotated[str, Field(min_length=1, description="Path to file to write standard output", example="/home/user/output.txt")] = None + stderr_path: Annotated[str, Field(min_length=1, description="Path to file to write standard error", example="/home/user/error.txt")] = None + resources: Annotated[ResourceSpec, Field(description="Resource requirements for the job")] = None + attributes: Annotated[JobAttributes, Field(description="Additional job attributes such as duration, queue, and account")] = None + pre_launch: Annotated[str, Field(min_length=1, description="Script or commands to run before launching the job", example="module load cuda")] = None + post_launch: Annotated[str, Field(min_length=1, description="Script or commands to run after the job completes", example="echo done")] = None + launcher: Annotated[str, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')", example="srun")] = None class JobState(str, Enum): @@ -120,14 +119,16 @@ class JobState(str, Enum): class JobStatus(IRIBaseModel): - state: JobState - time: float | None = None - message: str | None = None - exit_code: int | None = None - meta_data: dict[str, object] | None = None + """Represents the status of a job.""" + state: JobState = Field(..., description="Current state of the job", example="queued") + time: float = Field(default=None, description="Timestamp associated with the status (seconds since epoch)", example=1708531200.0) + message: str = Field(default=None, description="Human-readable status message", example="Job is waiting in queue") + exit_code: int = Field(default=None, description="Process exit code if the job has finished", example=0) + meta_data: dict[str, object] = Field(default=None, description="Backend-specific metadata associated with the job status") class Job(IRIBaseModel): - id: str - status: JobStatus | None = None - job_spec: JobSpec | None = None + """Represents a job in the system.""" + id: str = Field(..., description="Unique identifier of the job", example="job-12345") + status: JobStatus = Field(default=None, description="Current status of the job") + job_spec: JobSpec = Field(default=None, description="Specification used to create the job") diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index bf917db2..8efe0197 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,4 @@ """Facility-related models.""" - -from typing import Optional - from pydantic import Field, HttpUrl, computed_field from ... import config @@ -9,19 +6,20 @@ class Site(NamedObject): + """A physical site that hosts resources and is part of a facility.""" def _self_path(self) -> str: return f"/facility/sites/{self.id}" - short_name: Optional[str] = Field(None, description="Common or short name of the Site.") - operating_organization: str = Field(..., description="Organization operating the Site.") - country_name: Optional[str] = Field(None, description="Country name of the Location.") - locality_name: Optional[str] = Field(None, description="City or locality name of the Location.") - state_or_province_name: Optional[str] = Field(None, description="State or province name of the Location.") - street_address: Optional[str] = Field(None, description="Street address of the Location.") - unlocode: Optional[str] = Field(None, description="United Nations trade and transport location code.") - altitude: Optional[float] = Field(None, description="Altitude of the Location.") - latitude: Optional[float] = Field(None, description="Latitude of the Location.") - longitude: Optional[float] = Field(None, description="Longitude of the Location.") + short_name: str = Field(default=None, description="Common or short name of the Site.", example="NERSC") + operating_organization: str = Field(..., description="Organization operating the Site.", example="Lawrence Berkeley National Laboratory") + country_name: str = Field(default=None, description="Country name of the Location.", example="United States") + locality_name: str = Field(default=None, description="City or locality name of the Location.", example="Berkeley") + state_or_province_name: str = Field(default=None, description="State or province name of the Location.", example="California") + street_address: str = Field(default=None, description="Street address of the Location.", example="1 Cyclotron Rd") + unlocode: str = Field(default=None, description="United Nations trade and transport location code.", example="USOAK") + altitude: float = Field(default=None, description="Altitude of the Location.", example=52.0) + latitude: float = Field(default=None, description="Latitude of the Location.", example=37.8762) + longitude: float = Field(default=None, description="Longitude of the Location.", example=-122.2506) resource_ids: list[str] = Field(default_factory=list, exclude=True) @computed_field(description="URIs of Resources hosted at this Site.") @@ -42,12 +40,13 @@ def find(cls, items, name=None, description=None, modified_since=None, short_nam class Facility(NamedObject): + """ Facility representation, including associated Sites.""" def _self_path(self) -> str: return "/facility" - short_name: Optional[str] = Field(None, description="Common or short name of the Facility.") - organization_name: Optional[str] = Field(None, description="Operating organization's name.") - support_uri: Optional[HttpUrl] = Field(None, description="Link to facility support portal.") + short_name: str = Field(default=None, description="Common or short name of the Facility.", example="ESnet") + organization_name: str = Field(default=None, description="Operating organization's name.", example="Energy Sciences Network") + support_uri: HttpUrl = Field(default=None, description="Link to facility support portal.", example="https://support.es.net") site_ids: list[str] = Field(default_factory=list, exclude=True) @computed_field(description="URIs of associated Sites.") diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index 57b0634d..12569937 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -1,3 +1,4 @@ +"""Filesystem-related models.""" # Copied from: https://github.com/eth-cscs/firecrest-v2/blob/master/src/firecrest/filesystem/ops/models.py # # Copyright (c) 2025, ETH Zurich. All rights reserved. @@ -6,11 +7,11 @@ # SPDX-License-Identifier: BSD-3-Clause from enum import Enum -from typing import Optional from pydantic import Field, AliasChoices, BaseModel class CompressionType(str, Enum): + """Defines the type of compression to be used for compressing or extracting files.""" none = "none" bzip2 = "bzip2" gzip = "gzip" @@ -18,111 +19,131 @@ class CompressionType(str, Enum): class ContentUnit(str, Enum): + """Defines the unit of content for file operations.""" lines = "lines" bytes = "bytes" class File(BaseModel): - name: str - type: str - link_target: Optional[str] - user: str - group: str - permissions: str - last_modified: str - size: str + """Represents a file or directory in the filesystem.""" + name: str = Field(..., description="File name", example="file.txt") + type: str = Field(..., description="File type", example="file") + link_target: str = Field(default=None, description="Target path if the file is a symbolic link", example="/data/file.txt") + user: str = Field(..., description="Owner username", example="user") + group: str = Field(..., description="Owner group", example="users") + permissions: str = Field(..., description="POSIX permission string", example="rwxr-xr-x") + last_modified: str = Field(..., description="Last modification timestamp", example="2026-02-21T12:00:00Z") + size: str = Field(..., description="File size in bytes as string", example="1024") class FileContent(BaseModel): - content: str - content_type: ContentUnit - start_position: int - end_position: int + """Represents the content of a file, along with metadata about the content.""" + content: str = Field(..., description="File content segment", example="Hello world") + content_type: ContentUnit = Field(..., description="Unit used for content slicing", example="lines") + start_position: int = Field(..., description="Start position of the returned content", example=0) + end_position: int = Field(..., description="End position of the returned content", example=10) class FileChecksum(BaseModel): - algorithm: str = "SHA-256" - checksum: str + """Represents the checksum information of a file.""" + algorithm: str = Field(default="SHA-256", description="Checksum algorithm", example="SHA-256") + checksum: str = Field(..., description="Checksum value", example="3a7bd3e2360a3d...") class FileStat(BaseModel): + """Represents the metadata information of a file.""" # message: str - mode: int - ino: int - dev: int - nlink: int - uid: int - gid: int - size: int - atime: int - ctime: int - mtime: int + mode: int = Field(..., description="File mode", example=33188) + ino: int = Field(..., description="Inode number", example=123456) + dev: int = Field(..., description="Device ID", example=2049) + nlink: int = Field(..., description="Number of hard links", example=1) + uid: int = Field(..., description="User ID of owner", example=1000) + gid: int = Field(..., description="Group ID of owner", example=1000) + size: int = Field(..., description="File size in bytes", example=1024) + atime: int = Field(..., description="Last access time (epoch seconds)", example=1708531200) + ctime: int = Field(..., description="Last metadata change time (epoch seconds)", example=1708531200) + mtime: int = Field(..., description="Last modification time (epoch seconds)", example=1708531200) # birthtime: int class PatchFile(BaseModel): - message: str - new_filepath: str - new_permissions: str - new_owner: str + """Represents the result of a file patch operation.""" + message: str = Field(..., description="Result message", example="File updated") + new_filepath: str = Field(..., description="New file path", example="/home/user/file.new") + new_permissions: str = Field(..., description="Updated permissions", example="755") + new_owner: str = Field(..., description="Updated owner", example="user") class PatchFileMetadataRequest(BaseModel): - new_filename: Optional[str] = None - new_permissions: Optional[str] = None - new_owner: Optional[str] = None + """Represents a request to update file metadata.""" + new_filename: str = Field(default=None, description="New file name", example="file.new") + new_permissions: str = Field(default=None, description="New permissions", example="755") + new_owner: str = Field(default=None, description="New owner", example="user") class GetDirectoryLsResponse(BaseModel): - output: Optional[list[File]] + """Represents the response for a directory listing.""" + output: list[File] = Field(default=None, description="Directory listing") class GetFileHeadResponse(BaseModel): - output: Optional[FileContent] - offset: Optional[int] = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content") + """Represents the response for reading the beginning of a file.""" + output: FileContent = Field(default=None, description="File content from the beginning") + offset: int = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content", example=0) class GetFileTailResponse(BaseModel): - output: Optional[FileContent] - offset: Optional[int] = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content") + """Represents the response for reading the end of a file.""" + output: FileContent = Field(default=None, description="File content from the end") + offset: int = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content", example=0) class GetFileChecksumResponse(BaseModel): - output: Optional[FileChecksum] + """Represents the response for getting file checksum information.""" + output: FileChecksum = Field(default=None, description="File checksum information") class GetFileTypeResponse(BaseModel): - output: Optional[str] = Field(example="directory") + """Represents the response for getting the type of a file.""" + output: str = Field(default=None, description="Type of the file", example="directory") class GetFileStatResponse(BaseModel): - output: Optional[FileStat] + """Represents the response for getting file metadata information.""" + output: FileStat = Field(default=None, description="File stat information") class GetFileDownloadResponse(BaseModel): - output: Optional[str] + """Represents the response for downloading a file.""" + output: str = Field(default=None, description="Download URL or identifier", example="https://example.com/download/file") + class PatchFileMetadataResponse(BaseModel): - output: Optional[PatchFile] + """Represents the response for updating file metadata.""" + output: PatchFile = Field(default=None, description="Updated file metadata") class FilesystemRequestBase(BaseModel): + """Base class for filesystem operation requests.""" # Should we allow both: path and source_path? Or just one of them? - path: Optional[str] = Field(validation_alias=AliasChoices("path", "source_path"), example="/home/user/dir") + path: str = Field(default=None, validation_alias=AliasChoices("path", "source_path"), description="Source file or directory path", example="/home/user/dir") class PutFileChmodRequest(FilesystemRequestBase): - mode: str = Field(..., description="Mode in octal permission format") + """Represents a request to change file permissions.""" + mode: str = Field(..., description="Mode in octal permission format", example="777") model_config = {"json_schema_extra": {"examples": [{"path": "/home/user/dir/file.out", "mode": "777"}]}} class PutFileChmodResponse(BaseModel): - output: Optional[File] + """Represents the response for changing file permissions.""" + output: File = Field(default=None, description="Updated file metadata") class PutFileChownRequest(FilesystemRequestBase): - owner: Optional[str] = Field(default="", description="User name of the new user owner of the file") - group: Optional[str] = Field(default="", description="Group name of the new group owner of the file") + """Represents a request to change file ownership.""" + owner: str = Field(default="", description="User name of the new user owner of the file", example="user") + group: str = Field(default="", description="Group name of the new group owner of the file", example="my-group") model_config = { "json_schema_extra": { "examples": [ @@ -137,51 +158,53 @@ class PutFileChownRequest(FilesystemRequestBase): class PutFileChownResponse(BaseModel): - output: Optional[File] + """Represents the response for changing file ownership.""" + output: File = Field(default=None, description="Updated file metadata") + class PutFileUploadResponse(BaseModel): - output: Optional[str] + """Represents the response for uploading a file.""" + output: str = Field(default=None, description="Upload result or identifier") + class PostMakeDirRequest(FilesystemRequestBase): - parent: Optional[bool] = Field( - default=False, - description="If set to `true` creates all its parent directories if they do not already exist", - ) + """Represents a request to create a directory.""" + parent: bool = Field(default=False, description="If set to `true` creates all its parent directories if they do not already exist", example=True) model_config = {"json_schema_extra": {"examples": [{"path": "/home/user/dir/newdir", "parent": "true"}]}} class PostFileSymlinkRequest(FilesystemRequestBase): - link_path: str = Field(description="Path to the new symlink") + """Represents a request to create a symbolic link.""" + link_path: str = Field(..., description="Path to the new symlink", example="/home/user/newlink") model_config = {"json_schema_extra": {"examples": [{"path": "/home/user/dir", "link_path": "/home/user/newlink"}]}} class PostFileSymlinkResponse(BaseModel): - output: Optional[File] + """Represents the response for creating a symbolic link.""" + output: File = Field(default=None, description="Created symlink metadata") class GetViewFileResponse(BaseModel): - output: Optional[str] + """Represents the response for viewing a file.""" + output: str = Field(default=None, description="File content") class PostMkdirResponse(BaseModel): - output: Optional[File] + """Represents the response for creating a directory.""" + output: File = Field(default=None, description="Created directory metadata") class PostCompressResponse(BaseModel): - output: Optional[File] + """Represents the response for compressing a file.""" + output: File = Field(default=None, description="Compressed file metadata") class PostCompressRequest(FilesystemRequestBase): - target_path: str = Field(description="Path to the compressed file") - match_pattern: Optional[str] = Field(default=None, description="Regex pattern to filter files to compress") - dereference: Optional[bool] = Field( - default=False, - description="If set to `true`, it follows symbolic links and archive the files they point to instead of the links themselves.", - ) - compression: Optional[CompressionType] = Field( - default="gzip", - description="Defines the type of compression to be used. By default gzip is used.", - ) + """Represents a request to compress a file.""" + target_path: str = Field(..., description="Path to the compressed file", example="/home/user/file.tar.gz") + match_pattern: str = Field(default=None, description="Regex pattern to filter files to compress", example=".*\\.txt$") + dereference: bool = Field(default=False, description="If set to `true`, it follows symbolic links and archive the files they point to instead of the links themselves.", example=True) + compression: CompressionType = Field(default="gzip", description="Defines the type of compression to be used. By default gzip is used.", example="gzip") model_config = { "json_schema_extra": { "examples": [ @@ -198,15 +221,14 @@ class PostCompressRequest(FilesystemRequestBase): class PostExtractResponse(BaseModel): - output: Optional[File] + """Represents the response for extracting a compressed file.""" + output: File = Field(default=None, description="Extracted file metadata") class PostExtractRequest(FilesystemRequestBase): - target_path: str = Field(description="Path to the directory where to extract the compressed file") - compression: Optional[CompressionType] = Field( - default="gzip", - description="Defines the type of compression to be used. By default gzip is used.", - ) + """Represents a request to extract a compressed file.""" + target_path: str = Field(..., description="Path to the directory where to extract the compressed file", example="/home/user/dir") + compression: CompressionType = Field(default="gzip", description="Defines the type of compression to be used. By default gzip is used.", example="gzip") model_config = { "json_schema_extra": { "examples": [ @@ -221,11 +243,9 @@ class PostExtractRequest(FilesystemRequestBase): class PostCopyRequest(FilesystemRequestBase): - target_path: str = Field(description="Target path of the copy operation") - dereference: Optional[bool] = Field( - default=False, - description=("If set to `true`, it follows symbolic links and copies the files they point to instead of the links themselves."), - ) + """Represents a request to copy a file.""" + target_path: str = Field(..., description="Target path of the copy operation", example="/home/user/dir/file.new") + dereference: bool = Field(default=False, description=("If set to `true`, it follows symbolic links and copies the files they point to instead of the links themselves."), example=True) model_config = { "json_schema_extra": { "examples": [ @@ -240,11 +260,13 @@ class PostCopyRequest(FilesystemRequestBase): class PostCopyResponse(BaseModel): - output: Optional[File] + """Represents the response for copying a file.""" + output: File = Field(default=None, description="Copied file metadata") class PostMoveRequest(FilesystemRequestBase): - target_path: str = Field(description="Target path of the move operation") + """Represents a request to move a file.""" + target_path: str = Field(..., description="Target path of the move operation", example="/home/user/dir/file.new") model_config = { "json_schema_extra": { "examples": [ @@ -258,7 +280,10 @@ class PostMoveRequest(FilesystemRequestBase): class PostMoveResponse(BaseModel): - output: Optional[File] + """Represents the response for moving a file.""" + output: File = Field(default=None, description="Moved file metadata") + class RemoveResponse(BaseModel): - output: Optional[str] \ No newline at end of file + """Represents the response for removing a file or directory.""" + output: str = Field(default=None, description="Removal result message") diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 930892c6..0e051d65 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -1,18 +1,15 @@ +"""Models for the status API.""" import datetime import enum -from pydantic import BaseModel, Field, computed_field, field_validator +from pydantic import Field, computed_field, field_validator from ... import config from ...types.base import NamedObject -class Link(BaseModel): - rel: str - href: str - - class Status(enum.Enum): + """Represents the status of a resource.""" up = "up" down = "down" degraded = "degraded" @@ -20,6 +17,7 @@ class Status(enum.Enum): class ResourceType(enum.Enum): + """Represents the type of a resource.""" website = "website" service = "service" compute = "compute" @@ -30,15 +28,16 @@ class ResourceType(enum.Enum): class Resource(NamedObject): + """Represents a resource in the system.""" def _self_path(self) -> str: """Return the API path for this resource.""" return f"/status/resources/{self.id}" - site_id: str = Field(..., description="The site identifier this resource is located at", exclude=True) + site_id: str = Field(..., description="The site identifier this resource is located at", exclude=True, example="site-1") capability_ids: list[str] = Field(default_factory=list, exclude=True) - group: str | None - current_status: Status | None = Field(default=None, description="The current status comes from the status of the last event for this resource") - resource_type: ResourceType + group: str = Field(default=None, description="Logical grouping of the resource", example="frontend") + current_status: Status = Field(default=None, description="The current status comes from the status of the last event for this resource", example="up") + resource_type: ResourceType = Field(..., description="Type of the resource", example="service") @computed_field(description="URI of the site where this resource is located") @property @@ -71,6 +70,7 @@ def find(cls, items, name=None, description=None, modified_since=None, group=Non class Event(NamedObject): + """Represents an event that occurred to a resource, which may be part of an incident.""" def _self_path(self) -> str: """Return the API path for this event.""" return f"/status/incidents/{self.incident_id}/events/{self.id}" @@ -80,10 +80,10 @@ def _self_path(self) -> str: def _norm_dt_field(cls, v): return cls.normalize_dt(v) - occurred_at: datetime.datetime - status: Status - resource_id: str = Field(exclude=True) - incident_id: str | None = Field(exclude=True, default=None) + occurred_at: datetime.datetime = Field(..., description="Timestamp when the event occurred", example="2026-02-21T12:00:00Z") + status: Status = Field(..., description="Status of the resource at the time of the event", example="down") + resource_id: str = Field(..., exclude=True, description="Identifier of the affected resource", example="res-1") + incident_id: str = Field(default=None, exclude=True, description="Identifier of the related incident", example="inc-1") @computed_field(description="The resource belonging to this event") @property @@ -122,12 +122,14 @@ def find(cls, items, name=None, description=None, modified_since=None, resource_ class IncidentType(enum.Enum): + """Represents the type of an incident.""" planned = "planned" unplanned = "unplanned" reservation = "reservation" class Resolution(enum.Enum): + """Represents the resolution status of an incident.""" unresolved = "unresolved" cancelled = "cancelled" completed = "completed" @@ -136,6 +138,7 @@ class Resolution(enum.Enum): class Incident(NamedObject): + """Represents an incident that may impact one or more resources.""" def _self_path(self) -> str: """Return the API path for this incident.""" return f"/status/incidents/{self.id}" @@ -145,13 +148,13 @@ def _self_path(self) -> str: def _norm_dt_field(cls, v): return cls.normalize_dt(v) - status: Status + status: Status = Field(..., description="Current status of the incident", example="degraded") resource_ids: list[str] = Field(default_factory=list, exclude=True) event_ids: list[str] = Field(default_factory=list, exclude=True) - start: datetime.datetime - end: datetime.datetime | None - type: IncidentType - resolution: Resolution + start: datetime.datetime = Field(..., description="Incident start time", example="2026-02-21T12:00:00Z") + end: datetime.datetime = Field(default=None, description="Incident end time", example="2026-02-21T14:00:00Z") + type: IncidentType = Field(..., description="Type of incident", example="planned") + resolution: Resolution = Field(..., description="Resolution status of the incident", example="pending") @computed_field(description="The list of past events in this incident") @property diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index bfca880e..afec203a 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -1,3 +1,4 @@ +import traceback from abc import abstractmethod from . import models as task_models from ..account import models as account_models @@ -5,6 +6,9 @@ from ..filesystem import models as filesystem_models, facility_adapter as filesystem_adapter from ..iri_router import AuthenticatedAdapter, IriRouter +from ...apilogger import get_stream_logger + +logger = get_stream_logger(__name__) class FacilityAdapter(AuthenticatedAdapter): """ @@ -33,16 +37,22 @@ async def delete_task(self: "FacilityAdapter", user: account_models.User, task_i async def on_task(resource: status_models.Resource, user: account_models.User, task: task_models.TaskCommand) -> tuple[str, task_models.TaskStatus]: # Handle a task from the facility message queue. # Returns: (result, status) + def _extractNull(ind): + data = {k: v for k, v in ind.items() if v is not None} + return data try: r = None + logger.info(f"Received task: {task.router}:{task.command} with args: {task.args}") if task.router == "filesystem": fs_adapter = IriRouter.create_adapter(task.router, filesystem_adapter.FacilityAdapter) if task.command == "chmod": - request_model = filesystem_models.PutFileChmodRequest.model_validate(task.args["request_model"]) + data = _extractNull(task.args["request_model"]) + request_model = filesystem_models.PutFileChmodRequest.model_validate(data) o = await fs_adapter.chmod(resource, user, request_model) r = o.model_dump_json() elif task.command == "chown": - request_model = filesystem_models.PutFileChownRequest.model_validate(task.args["request_model"]) + data = _extractNull(task.args["request_model"]) + request_model = filesystem_models.PutFileChownRequest.model_validate(data) o = await fs_adapter.chown(resource, user, request_model) r = o.model_dump_json() elif task.command == "file": @@ -52,11 +62,13 @@ async def on_task(resource: status_models.Resource, user: account_models.User, t o = await fs_adapter.stat(resource, user, **task.args) r = o.model_dump_json() elif task.command == "mkdir": - request_model = filesystem_models.PostMakeDirRequest.model_validate(task.args["request_model"]) + data = _extractNull(task.args["request_model"]) + request_model = filesystem_models.PostMakeDirRequest.model_validate(data) o = await fs_adapter.mkdir(resource, user, request_model) r = o.model_dump_json() elif task.command == "symlink": - request_model = filesystem_models.PostFileSymlinkRequest.model_validate(task.args["request_model"]) + data = _extractNull(task.args["request_model"]) + request_model = filesystem_models.PostFileSymlinkRequest.model_validate(data) o = await fs_adapter.symlink(resource, user, request_model) r = o.model_dump_json() elif task.command == "ls": @@ -78,19 +90,23 @@ async def on_task(resource: status_models.Resource, user: account_models.User, t o = await fs_adapter.rm(resource, user, **task.args) r = o.model_dump_json() elif task.command == "compress": - request_model = filesystem_models.PostCompressRequest.model_validate(task.args["request_model"]) + data = _extractNull(task.args["request_model"]) + request_model = filesystem_models.PostCompressRequest.model_validate(data) o = await fs_adapter.compress(resource, user, request_model) r = o.model_dump_json() elif task.command == "extract": - request_model = filesystem_models.PostExtractRequest.model_validate(task.args["request_model"]) + data = _extractNull(task.args["request_model"]) + request_model = filesystem_models.PostExtractRequest.model_validate(data) o = await fs_adapter.extract(resource, user, request_model) r = o.model_dump_json() elif task.command == "mv": - request_model = filesystem_models.PostMoveRequest.model_validate(task.args["request_model"]) + data = _extractNull(task.args["request_model"]) + request_model = filesystem_models.PostMoveRequest.model_validate(data) o = await fs_adapter.mv(resource, user, request_model) r = o.model_dump_json() elif task.command == "cp": - request_model = filesystem_models.PostCopyRequest.model_validate(task.args["request_model"]) + data = _extractNull(task.args["request_model"]) + request_model = filesystem_models.PostCopyRequest.model_validate(data) o = await fs_adapter.cp(resource, user, request_model) r = o.model_dump_json() elif task.command == "download": @@ -103,4 +119,7 @@ async def on_task(resource: status_models.Resource, user: account_models.User, t else: return (f"Task was cancelled due to unknown router/command: {task.router}:{task.command}", task_models.TaskStatus.failed) except Exception as exc: + traceback_str = traceback.format_exc() + logger.warning(f"Error handling task {task.router}:{task.command} with args: {task.args}\nError: {exc}") + logger.debug(f"Traceback:\n{traceback_str}") return (f"Error: {exc}", task_models.TaskStatus.failed) diff --git a/app/routers/task/models.py b/app/routers/task/models.py index ba124bc6..a5a5072c 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,22 +1,24 @@ +""" Task models for the IRI Facility API """ import enum from typing import Any -from pydantic import BaseModel, computed_field - +from pydantic import BaseModel, Field, computed_field from ... import config class TaskSubmitResponse(BaseModel): """Response model for submitting a task""" - task_id: str + task_id: str = Field(..., description="Identifier of the submitted task", example="task-123") @computed_field(description="The list of past events in this incident") @property def task_uri(self) -> str: + """Return the URI for this task.""" return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/task/{self.task_id}" class TaskStatus(str, enum.Enum): + """Represents the status of a task.""" pending = "pending" active = "active" completed = "completed" @@ -25,13 +27,15 @@ class TaskStatus(str, enum.Enum): class TaskCommand(BaseModel): - router: str - command: str - args: dict + """Represents a command to be executed as part of a task.""" + router: str = Field(..., description="Router name the task comes from", example="filesystem") + command: str = Field(..., description="Command to execute", example="chmod") + args: dict = Field(..., description="Command arguments as key-value pairs", example={"path": "/home/user/file", "mode": "755"}) class Task(BaseModel): - id: str - status: TaskStatus = TaskStatus.pending - result: Any | None = None - command: TaskCommand | None = None + """Represents a task in the system.""" + id: str = Field(..., description="Unique identifier of the task", example="task-123") + status: TaskStatus = Field(default=TaskStatus.pending, description="Current status of the task", example="pending") + result: Any = Field(default=None, description="Result of the task execution, if available") + command: TaskCommand = Field(default=None, description="Command associated with this task") diff --git a/app/types/base.py b/app/types/base.py index c0a55ddd..f78e7849 100644 --- a/app/types/base.py +++ b/app/types/base.py @@ -1,8 +1,6 @@ """Default models used by multiple routers.""" - import datetime from collections.abc import Iterable -from typing import Optional from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator, model_serializer @@ -47,7 +45,7 @@ def normalize_dt(cls, dt: datetime | None) -> datetime | None: class NamedObject(IRIBaseModel): """Base model for named objects.""" - id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.") + id: str = Field(..., description="The unique identifier for the object. Typically a UUID or URN.", example="urn:iri:object:1234") def _self_path(self) -> str: raise NotImplementedError @@ -63,9 +61,9 @@ def self_uri(self) -> str: """Computed self URI property.""" return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - name: Optional[str] = Field(None, description="The long name of the object.") - description: Optional[str] = Field(None, description="Human-readable description of the object.") - last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.") + name: str = Field(default=None, description="The long name of the object.", example="Perlmutter GPU") + description: str = Field(default=None, description="Human-readable description of the object.", example="High-performance GPU compute resource") + last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.", example="2026-02-21T12:00:00Z") @classmethod def find_by_id(cls, items, id_, allow_name: bool = False): diff --git a/app/types/http.py b/app/types/http.py index 056dc51a..4ee2a989 100644 --- a/app/types/http.py +++ b/app/types/http.py @@ -30,10 +30,7 @@ def modifiedSinceDatetime(modified_since: str | None, header_modified_since: str dt = StrictDateTime.validate(modified_since) parsed_times.append(dt) except ValueError as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid modified_since query param: {exc}", - ) from exc + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Invalid modified_since query param: {exc}") from exc # Header (RFC 1123) if header_modified_since is not None: @@ -45,10 +42,7 @@ def modifiedSinceDatetime(modified_since: str | None, header_modified_since: str dt = dt.replace(tzinfo=datetime.timezone.utc) parsed_times.append(dt.astimezone(datetime.timezone.utc)) except Exception as exc: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Invalid If-Modified-Since header format (must be RFC1123)", - ) from exc + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid If-Modified-Since header format (must be RFC1123)") from exc if not parsed_times: return None @@ -76,9 +70,9 @@ async def checker(req: Request): for key, values in parsed.items(): if key not in allowed: - raise HTTPException(status_code=422, detail=[{"type": "extra_forbidden", "loc": ["query", key], "msg": f"Unexpected query parameter: {key}"}]) + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=[{"type": "extra_forbidden", "loc": ["query", key], "msg": f"Unexpected query parameter: {key}"}]) if len(values) > 1 and key not in multiParams: - raise HTTPException(status_code=422, detail=[{"type": "duplicate_forbidden", "loc": ["query", key], "msg": f"Duplicate query parameter: {key}"}]) + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=[{"type": "duplicate_forbidden", "loc": ["query", key], "msg": f"Duplicate query parameter: {key}"}]) return checker diff --git a/app/types/models.py b/app/types/models.py index 070b2dbd..6240b785 100644 --- a/app/types/models.py +++ b/app/types/models.py @@ -18,6 +18,6 @@ class Capability(NamedObject): def _self_path(self) -> str: return f"/account/capabilities/{self.id}" - last_modified: StrictDateTime | None = Field(default=None, description="ISO 8601 timestamp when this object was last modified.") + last_modified: StrictDateTime = Field(default=None, description="ISO 8601 timestamp when this object was last modified.", example="2026-02-21T12:00:00Z") - units: list[AllocationUnit] + units: list[AllocationUnit] = Field(..., description="Allocation units supported by this capability", example=["node_hours"]) diff --git a/app/types/scalars.py b/app/types/scalars.py index 2b9b6fe5..365be066 100644 --- a/app/types/scalars.py +++ b/app/types/scalars.py @@ -6,10 +6,9 @@ from pydantic_core import core_schema + # ----------------------------------------------------------------------- # StrictHTTPBool: a strict boolean type - - class StrictHTTPBool: """Strict boolean: - Accepts: real booleans, 'true', 'false' @@ -36,13 +35,11 @@ def validate(value): @classmethod def __get_pydantic_json_schema__(cls, schema, handler): - return {"type": "boolean", "description": "Strict boolean. Only true/false allowed (bool or string)."} + return {"type": "boolean", "description": "Strict boolean. Only true/false allowed (bool or string).", "example": True} # ----------------------------------------------------------------------- # StrictDateTime: a strict ISO8601 datetime type - - class StrictDateTime: """ Strict ISO8601 datetime: @@ -82,13 +79,11 @@ def _normalize(dt: datetime.datetime) -> datetime.datetime: @classmethod def __get_pydantic_json_schema__(cls, schema, handler): - return {"type": "string", "format": "date-time", "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted."} + return {"type": "string", "format": "date-time", "description": "Strict ISO8601 datetime. Only valid ISO8601 datetime strings are accepted.", "example": "2026-02-21T12:00:00Z"} # ----------------------------------------------------------------------- # AllocationUnit: an enum for allocation units - - class AllocationUnit(enum.Enum): """Units for allocation""" diff --git a/pyproject.toml b/pyproject.toml index 356763e5..af2f4b11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,6 @@ requires-python = ">=3.14,<3.15" dependencies = [ "fastapi[standard]>=0.128.0,<0.129.0", "uvicorn[standard]>=0.40.0,<0.41.0", - "pyhumps>=3.8.0,<3.9.0", "opentelemetry-api>=1.39.1,<1.40.0", "opentelemetry-sdk>=1.39.1,<1.40.0", "opentelemetry-instrumentation-fastapi>=0.60b1,<0.61b0", From 3308337248efeac127e47bdc224242cad10536ae Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 23 Feb 2026 14:35:52 -0600 Subject: [PATCH 090/133] Downgrade to py313 --- .github/workflows/api-validation.yml | 2 +- Dockerfile | 2 +- Makefile | 2 +- pylintrc | 2 +- pyproject.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml index 51ed3312..673a1128 100644 --- a/.github/workflows/api-validation.yml +++ b/.github/workflows/api-validation.yml @@ -26,7 +26,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.14" + python-version: "3.13" - name: Install uv run: pip install uv diff --git a/Dockerfile b/Dockerfile index 93c80d90..9d980c40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.14 +FROM python:3.13 RUN mkdir /app COPY . /app diff --git a/Makefile b/Makefile index 60f47819..febce8ef 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON := python3.14 +PYTHON := python3.13 VENV := .venv BIN := $(VENV)/bin UV := uv diff --git a/pylintrc b/pylintrc index 4a9fbf34..99b31968 100644 --- a/pylintrc +++ b/pylintrc @@ -53,7 +53,7 @@ persistent=yes # Minimum Python version to use for version dependent checks. Will default to # the version used to run pylint. -py-version=3.14 +py-version=3.13 # Discover python modules and packages in the file system subtree. recursive=no diff --git a/pyproject.toml b/pyproject.toml index 356763e5..b8cf29c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "iri-api-python" version = "0.1.0" -requires-python = ">=3.14,<3.15" +requires-python = ">=3.13,<3.14" dependencies = [ "fastapi[standard]>=0.128.0,<0.129.0", "uvicorn[standard]>=0.40.0,<0.41.0", From 822af942b5e1151d07b2fce9201e6d9bc45db0fd Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 23 Feb 2026 15:01:20 -0600 Subject: [PATCH 091/133] Remove unsupported operand. Py3.13 vs 3.14 --- app/types/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/types/base.py b/app/types/base.py index c0a55ddd..d28d1651 100644 --- a/app/types/base.py +++ b/app/types/base.py @@ -32,7 +32,7 @@ def get_extra(self, key, default=None): return getattr(self, "__pydantic_extra__", {}).get(key, default) @classmethod - def normalize_dt(cls, dt: datetime | None) -> datetime | None: + def normalize_dt(cls, dt: datetime) -> datetime: """Normalize datetime to UTC-aware.""" # Convert naive datetimes into UTC-aware versions if dt is None: From 98f3dc6fcd6c786716333537870b7ccb318df902 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 23 Feb 2026 15:25:16 -0600 Subject: [PATCH 092/133] Dummy force retest --- app/routers/account/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/routers/account/models.py b/app/routers/account/models.py index cd7cf812..73fb147c 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -1,6 +1,5 @@ """Models for account-related API endpoints, including users, projects, and allocations.""" import datetime - from pydantic import Field, computed_field, field_validator from ... import config From 3c3676586b2944ab3de54199674d818894dd288c Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 25 Feb 2026 08:45:05 -0600 Subject: [PATCH 093/133] Update fs test script. Print all resources and provide option to choose one --- test/test_filesystem.py | 120 +++++++++++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 27 deletions(-) diff --git a/test/test_filesystem.py b/test/test_filesystem.py index 62f04ad0..46ce5a08 100644 --- a/test/test_filesystem.py +++ b/test/test_filesystem.py @@ -5,7 +5,6 @@ import os import sys import time -import random import datetime as dt import requests @@ -14,45 +13,113 @@ # CONFIG — EDIT THESE AS NEEDED # ========================= -BASE_URL = "http://localhost:8000/api/v1" +#BASE_URL = "http://localhost:8000/api/v1" +BASE_URL = "https://api.iri.nersc.gov/api/v1" +#BASE_URL = "https://iri-dev.ppg.es.net/api/v1" TOKEN = os.environ.get("IRI_API_TOKEN", "12345") # ========================= - -HEADERS = {"Authorization": f"Bearer {TOKEN}", "Accept": "application/json"} - +HEADERS = {"Authorization": f"{TOKEN}", "Accept": "application/json"} POLL_INTERVAL = 2 TIMEOUT = 180 -def getAnyStorageResource(): - """Get the ID of any storage resource available in the facility by looking at the project allocations and resource capabilities.""" +def print_table(headers, rows): + """Pretty-print a table with dynamic column width.""" + widths = [len(h) for h in headers] + + for row in rows: + for i, cell in enumerate(row): + widths[i] = max(widths[i], len(str(cell))) + + fmt = " | ".join(f"{{:<{w}}}" for w in widths) + sep = "-+-".join("-" * w for w in widths) + + print(fmt.format(*headers)) + print(sep) + for row in rows: + print(fmt.format(*[str(c) for c in row])) + print() + +def cap_name(allcaps, uri): + """Get capability name from URI, or return URI if not found.""" + c = allcaps.get(uri) + return c.get("name") if c else uri + +def getAllResources(): + """Get and print all storage resources available to the user, along with their capabilities.""" + print("\n" + "="*40) + print("=== DISCOVERING RESOURCES AND CAPABILITIES ===") + # ----------------------------- + # Projects + # ----------------------------- projects = requests.get(f"{BASE_URL}/account/projects", headers=HEADERS, timeout=TIMEOUT).json() + project_rows = [[p.get("id"), p.get("name", ""), p.get("description", "")] for p in projects] + + print("\n=== PROJECTS ===") + print_table(["Project ID", "Name", "Description"], project_rows) + + # ----------------------------- + # Capabilities + # ----------------------------- caps = requests.get(f"{BASE_URL}/account/capabilities", headers=HEADERS, timeout=TIMEOUT).json() - storageCaps = {c["self_uri"] for c in caps if c["name"] == "GPFS Storage"} - if not storageCaps: - raise RuntimeError("No storage capabilities defined") + cap_rows = [[c.get("self_uri"), c.get("name"), c.get("description", "")] for c in caps] + cap_by_uri = {c["self_uri"]: c for c in caps} + + print("\n=== CAPABILITIES ===") + print_table(["Capability URI", "Name", "Description"], cap_rows) + + + # ----------------------------- + # Allocations per project + # ----------------------------- + alloc_rows = [] projectStorageCaps = set() - for p in projects: - allocs = requests.get(f"{BASE_URL}/account/projects/{p['id']}/project_allocations", headers=HEADERS, timeout=TIMEOUT).json() + + for pr in projects: + allocs = requests.get(f"{BASE_URL}/account/projects/{pr['id']}/project_allocations", headers=HEADERS, timeout=TIMEOUT).json() + for a in allocs: - if a["capability_uri"] in storageCaps: + alloc_rows.append([pr["id"], a.get("id"), cap_name(cap_by_uri, a.get("capability_uri"))]) + + if a.get("capability_uri") in cap_by_uri: projectStorageCaps.add(a["capability_uri"]) + print("\n=== PROJECT ALLOCATIONS ===") + print_table(["Project ID", "Allocation ID", "Capability URI"], alloc_rows) + if not projectStorageCaps: - raise RuntimeError("No storage allocations found in any project") + die("No storage allocations found") + # ----------------------------- + # Resources + # ----------------------------- resources = requests.get(f"{BASE_URL}/status/resources?offset=0&limit=100", headers=HEADERS, timeout=TIMEOUT).json() - matchingResources = [r["id"] for r in resources if any(cap in r["capability_uris"] for cap in projectStorageCaps)] - if not matchingResources: - raise RuntimeError("No storage resources found") - return random.choice(matchingResources) + resource_rows = [] + matching = [] + for r in resources: + caps = r.get("capability_uris", []) + cap_names = [cap_name(cap_by_uri, c) for c in caps] -RESOURCE_ID = getAnyStorageResource() -print("Chosen storage resource ID:", RESOURCE_ID) + match = any(cap in caps for cap in projectStorageCaps) + + if match: + resource_rows.append([r.get("id"), r.get("name", ""), r.get("resource_type", ""), + r.get("description", ""), ", ".join(cap_names), r.get("current_status", "")]) + matching.append(r["id"]) + print("\n=== RESOURCES ===") + print_table(["Resource ID", "Name", "Type", "Description", "Capability URIs", "Current Status"], resource_rows) + + if not matching: + die("No storage resources found") + +getAllResources() +print("\n" + "="*40) +RESOURCE_ID = input("Enter the ID of the storage resource to test against: \n").strip() +print("Chosen storage resource ID:", RESOURCE_ID) def die(msg): @@ -79,12 +146,12 @@ def submit(method, path, **kwargs): return data -def wait_task(task): +def wait_task(taskin): """Wait for a task to complete and return its result.""" deadline = time.time() + TIMEOUT while time.time() < deadline: - r = requests.get(task["task_uri"], headers=HEADERS, timeout=TIMEOUT) + r = requests.get(taskin["task_uri"], headers=HEADERS, timeout=TIMEOUT) if not r.ok: die(f"Task query failed: {r.status_code} {r.text}") @@ -92,25 +159,24 @@ def wait_task(task): t = r.json() status = t["status"] - print(f" Task {task['task_id']}: {status}") + print(f" Task {taskin['task_id']}: {status}") if status == "completed": print(f" Task result: {t.get('result')}") return t.get("result") if status in ("failed", "canceled"): - die(f"Task {task['task_id']} ended with status {status}: {t}") + die(f"Task {taskin['task_id']} ended with status {status}: {t}") time.sleep(POLL_INTERVAL) - die(f"Task {task['task_id']} timed out") + die(f"Task {taskin['task_id']} timed out") # ============================================================ # Sandbox setup # ============================================================ - -timestamp = dt.datetime.utcnow().strftime("%Y%m%d-%H%M%S") +timestamp = dt.datetime.now(dt.UTC).strftime("%Y%m%d-%H%M%S") # NOTE/TODO: /Users/jbalcas/work/amsc/iri/iri-facility-api-python/iri_sandbox/ # While we can use absolute paths, there is a need to return relative paths from the API From da6f0be08d074e5deae3db41b0ea5f1d30277e1b Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Wed, 25 Feb 2026 15:27:17 -0800 Subject: [PATCH 094/133] Remove offset from head/tail calls. This info is already present in the FileContent --- app/demo_adapter.py | 13 ++++++------- app/routers/filesystem/models.py | 2 -- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index ec86fbdd..fd112968 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -731,7 +731,7 @@ def _headtail( lines: int | None, skip_heading: bool = False, skip_trailing: bool = False, - ) -> Tuple[Any, int]: + ) -> Any: args = [cmd] if cmd == "tail" and skip_heading: @@ -754,8 +754,7 @@ def _headtail( args.append(rp) result = self._run(args) - content = result.stdout - return content, len(content) + return result.stdout async def head( self: "DemoAdapter", @@ -766,7 +765,7 @@ async def head( lines: int | None, skip_trailing: bool = False, ) -> filesystem_models.GetFileHeadResponse: - content, offset = self._headtail("head", path, file_bytes, lines, skip_trailing=skip_trailing) + content = self._headtail("head", path, file_bytes, lines, skip_trailing=skip_trailing) fc = filesystem_models.FileContent( content=content, @@ -776,7 +775,7 @@ async def head( start_position=0, end_position=len(content)) - return filesystem_models.GetFileHeadResponse(output=fc, offset=offset) + return filesystem_models.GetFileHeadResponse(output=fc) async def tail( self: "DemoAdapter", @@ -788,7 +787,7 @@ async def tail( skip_heading: bool = False, ) -> filesystem_models.GetFileTailResponse: - content, offset = self._headtail("tail", path, file_bytes, lines, skip_heading=skip_heading) + content = self._headtail("tail", path, file_bytes, lines, skip_heading=skip_heading) fc = filesystem_models.FileContent( content=content, @@ -798,7 +797,7 @@ async def tail( start_position=0, end_position=len(content)) - return filesystem_models.GetFileTailResponse(output=fc, offset=offset) + return filesystem_models.GetFileTailResponse(output=fc) diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index 12569937..c978a2ad 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -89,13 +89,11 @@ class GetDirectoryLsResponse(BaseModel): class GetFileHeadResponse(BaseModel): """Represents the response for reading the beginning of a file.""" output: FileContent = Field(default=None, description="File content from the beginning") - offset: int = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content", example=0) class GetFileTailResponse(BaseModel): """Represents the response for reading the end of a file.""" output: FileContent = Field(default=None, description="File content from the end") - offset: int = Field(default=0, description="Offset in bytes from the beginning of the file where to start reading the content", example=0) class GetFileChecksumResponse(BaseModel): From a8fa1434bedb25c0837a9841b1e435f8ba75f122 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Fri, 27 Feb 2026 10:22:17 -0800 Subject: [PATCH 095/133] use bearer tokens --- app/routers/iri_router.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 950f68b6..3ea6c289 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -4,11 +4,11 @@ import logging import importlib from fastapi import Request, Depends, HTTPException, APIRouter -from fastapi.security import APIKeyHeader +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from .account.models import User -bearer_token = APIKeyHeader(name="Authorization") +bearer_scheme = HTTPBearer() def get_client_ip(request: Request) -> str | None: @@ -76,11 +76,13 @@ def create_adapter(router_name, router_adapter): async def current_user( self, request: Request, - api_key: str = Depends(bearer_token), + credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme), ): + token = credentials.credentials + user_id = None try: - user_id = await self.adapter.get_current_user(api_key, get_client_ip(request)) + user_id = await self.adapter.get_current_user(token, get_client_ip(request)) except Exception as exc: logging.getLogger().error(f"Error parsing IRI_API_PARAMS: {exc}") traceback.print_exc() @@ -88,7 +90,7 @@ async def current_user( if not user_id: raise HTTPException(status_code=403, detail="Unauthorized access") request.state.current_user_id = user_id - request.state.api_key = api_key + request.state.api_key = token class AuthenticatedAdapter(ABC): From 60aaf0ef70cf1a0c5ed3c367ac27cde24dc926a9 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Wed, 25 Feb 2026 13:38:30 -0800 Subject: [PATCH 096/133] restore null-s to models/endpoints that can be null. This is needed to correctly represent objects by facilities --- app/routers/account/account.py | 6 ++-- app/routers/account/models.py | 2 +- app/routers/compute/models.py | 12 +++---- app/routers/error_handlers.py | 4 +-- app/routers/facility/facility.py | 8 ++--- app/routers/facility/models.py | 26 +++++++------- app/routers/filesystem/models.py | 50 +++++++++++++------------- app/routers/status/facility_adapter.py | 4 +-- app/routers/status/models.py | 8 ++--- app/routers/status/status.py | 50 +++++++++++++------------- app/routers/task/models.py | 4 +-- app/types/base.py | 4 +-- app/types/models.py | 2 +- 13 files changed, 90 insertions(+), 90 deletions(-) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index f3b36963..075b46a0 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -24,8 +24,8 @@ ) async def get_capabilities( request: Request, - name: str = Query(default=None, min_length=1), - modified_since: StrictDateTime = Query(default=None), + name: str|None = Query(default=None, min_length=1), + modified_since: StrictDateTime|None = Query(default=None), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), _forbid=Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), @@ -43,7 +43,7 @@ async def get_capabilities( async def get_capability( capability_id: str, request: Request, - modified_since: StrictDateTime = Query(default=None), + modified_since: StrictDateTime|None = Query(default=None), _forbid=Depends(forbidExtraQueryParams("modified_since")), ) -> Capability: caps = await router.adapter.get_capabilities(name=None, modified_since=modified_since, offset=0, limit=100) diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 73fb147c..774fd2b2 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -13,7 +13,7 @@ class User(IRIBaseModel): id: str = Field(..., description="Unique identifier of the user.", example="user-123") name: str = Field(..., description="Name of the user.", example="Jane Doe") api_key: str = Field(..., description="API key associated with this user.", example="AKIAIOSFODNN7EXAMPLE") - client_ip: str = Field(default=None, description="IP address from which the user connects.", example="192.0.2.10") + client_ip: str|None = Field(default=None, description="IP address from which the user connects.", example="192.0.2.10") # we could expose more fields here (eg. email) but it might be against policy diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index c687c4a7..94b511ce 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -121,14 +121,14 @@ class JobState(str, Enum): class JobStatus(IRIBaseModel): """Represents the status of a job.""" state: JobState = Field(..., description="Current state of the job", example="queued") - time: float = Field(default=None, description="Timestamp associated with the status (seconds since epoch)", example=1708531200.0) - message: str = Field(default=None, description="Human-readable status message", example="Job is waiting in queue") - exit_code: int = Field(default=None, description="Process exit code if the job has finished", example=0) - meta_data: dict[str, object] = Field(default=None, description="Backend-specific metadata associated with the job status") + time: float|None = Field(default=None, description="Timestamp associated with the status (seconds since epoch)", example=1708531200.0) + message: str|None = Field(default=None, description="Human-readable status message", example="Job is waiting in queue") + exit_code: int|None = Field(default=None, description="Process exit code if the job has finished", example=0) + meta_data: dict[str, object]|None = Field(default=None, description="Backend-specific metadata associated with the job status") class Job(IRIBaseModel): """Represents a job in the system.""" id: str = Field(..., description="Unique identifier of the job", example="job-12345") - status: JobStatus = Field(default=None, description="Current status of the job") - job_spec: JobSpec = Field(default=None, description="Specification used to create the job") + status: JobStatus|None = Field(default=None, description="Current status of the job") + job_spec: JobSpec|None = Field(default=None, description="Specification used to create the job") diff --git a/app/routers/error_handlers.py b/app/routers/error_handlers.py index cbc75167..a94334da 100644 --- a/app/routers/error_handlers.py +++ b/app/routers/error_handlers.py @@ -18,8 +18,8 @@ class Problem(BaseModel): model_config = ConfigDict(extra="allow", json_schema_extra={"description": 'Error structure for REST interface based on RFC 9457, "Problem Details for HTTP APIs."'}) type: str = Field(..., description="A URI reference that identifies the problem type.", example="https://example.com/notFound", json_schema_extra={"format": "uri", "default": "about:blank"}) status: int = Field(..., ge=100, le=599, description="The HTTP status code for this occurrence.", example=404) - title: str = Field(default=None, description="Short human-readable summary.", example="Not Found") - detail: str = Field(default=None, description="Human-readable explanation.", example="Descriptive text.") + title: str|None = Field(default=None, description="Short human-readable summary.", example="Not Found") + detail: str|None = Field(default=None, description="Human-readable explanation.", example="Descriptive text.") instance: str = Field(..., description="A URI reference identifying this occurrence.", example="http://localhost/api/v1/resource/123") diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 1ab5b5c2..61e124c2 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -26,11 +26,11 @@ async def get_facility( @router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites", response_model_exclude_none=True,) async def list_sites( request: Request, - modified_since: StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), + modified_since: StrictDateTime|None = Query(default=None), + name: str|None = Query(default=None, min_length=1), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), + short_name: str|None = Query(default=None, min_length=1), _forbid=Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), ) -> list[models.Site]: """List sites""" @@ -41,7 +41,7 @@ async def list_sites( async def get_site( request: Request, site_id: str, - modified_since: StrictDateTime = Query(default=None), + modified_since: StrictDateTime|None = Query(default=None), _forbid=Depends(forbidExtraQueryParams("modified_since")), ) -> models.Site: """Get site by ID""" diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 8efe0197..5d306751 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -10,16 +10,16 @@ class Site(NamedObject): def _self_path(self) -> str: return f"/facility/sites/{self.id}" - short_name: str = Field(default=None, description="Common or short name of the Site.", example="NERSC") - operating_organization: str = Field(..., description="Organization operating the Site.", example="Lawrence Berkeley National Laboratory") - country_name: str = Field(default=None, description="Country name of the Location.", example="United States") - locality_name: str = Field(default=None, description="City or locality name of the Location.", example="Berkeley") - state_or_province_name: str = Field(default=None, description="State or province name of the Location.", example="California") - street_address: str = Field(default=None, description="Street address of the Location.", example="1 Cyclotron Rd") - unlocode: str = Field(default=None, description="United Nations trade and transport location code.", example="USOAK") - altitude: float = Field(default=None, description="Altitude of the Location.", example=52.0) - latitude: float = Field(default=None, description="Latitude of the Location.", example=37.8762) - longitude: float = Field(default=None, description="Longitude of the Location.", example=-122.2506) + short_name: str|None = Field(default=None, description="Common or short name of the Site.", example="NERSC") + operating_organization: str|None = Field(..., description="Organization operating the Site.", example="Lawrence Berkeley National Laboratory") + country_name: str|None = Field(default=None, description="Country name of the Location.", example="United States") + locality_name: str|None = Field(default=None, description="City or locality name of the Location.", example="Berkeley") + state_or_province_name: str|None = Field(default=None, description="State or province name of the Location.", example="California") + street_address: str|None = Field(default=None, description="Street address of the Location.", example="1 Cyclotron Rd") + unlocode: str|None = Field(default=None, description="United Nations trade and transport location code.", example="USOAK") + altitude: float|None = Field(default=None, description="Altitude of the Location.", example=52.0) + latitude: float|None = Field(default=None, description="Latitude of the Location.", example=37.8762) + longitude: float|None = Field(default=None, description="Longitude of the Location.", example=-122.2506) resource_ids: list[str] = Field(default_factory=list, exclude=True) @computed_field(description="URIs of Resources hosted at this Site.") @@ -44,9 +44,9 @@ class Facility(NamedObject): def _self_path(self) -> str: return "/facility" - short_name: str = Field(default=None, description="Common or short name of the Facility.", example="ESnet") - organization_name: str = Field(default=None, description="Operating organization's name.", example="Energy Sciences Network") - support_uri: HttpUrl = Field(default=None, description="Link to facility support portal.", example="https://support.es.net") + short_name: str|None = Field(default=None, description="Common or short name of the Facility.", example="ESnet") + organization_name: str|None = Field(default=None, description="Operating organization's name.", example="Energy Sciences Network") + support_uri: HttpUrl|None = Field(default=None, description="Link to facility support portal.", example="https://support.es.net") site_ids: list[str] = Field(default_factory=list, exclude=True) @computed_field(description="URIs of associated Sites.") diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index c978a2ad..59411351 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -28,7 +28,7 @@ class File(BaseModel): """Represents a file or directory in the filesystem.""" name: str = Field(..., description="File name", example="file.txt") type: str = Field(..., description="File type", example="file") - link_target: str = Field(default=None, description="Target path if the file is a symbolic link", example="/data/file.txt") + link_target: str|None = Field(default=None, description="Target path if the file is a symbolic link", example="/data/file.txt") user: str = Field(..., description="Owner username", example="user") group: str = Field(..., description="Owner group", example="users") permissions: str = Field(..., description="POSIX permission string", example="rwxr-xr-x") @@ -76,55 +76,55 @@ class PatchFile(BaseModel): class PatchFileMetadataRequest(BaseModel): """Represents a request to update file metadata.""" - new_filename: str = Field(default=None, description="New file name", example="file.new") - new_permissions: str = Field(default=None, description="New permissions", example="755") - new_owner: str = Field(default=None, description="New owner", example="user") + new_filename: str|None = Field(default=None, description="New file name", example="file.new") + new_permissions: str|None = Field(default=None, description="New permissions", example="755") + new_owner: str|None = Field(default=None, description="New owner", example="user") class GetDirectoryLsResponse(BaseModel): """Represents the response for a directory listing.""" - output: list[File] = Field(default=None, description="Directory listing") + output: list[File]|None = Field(default=None, description="Directory listing") class GetFileHeadResponse(BaseModel): """Represents the response for reading the beginning of a file.""" - output: FileContent = Field(default=None, description="File content from the beginning") + output: FileContent|None = Field(default=None, description="File content from the beginning") class GetFileTailResponse(BaseModel): """Represents the response for reading the end of a file.""" - output: FileContent = Field(default=None, description="File content from the end") + output: FileContent|None = Field(default=None, description="File content from the end") class GetFileChecksumResponse(BaseModel): """Represents the response for getting file checksum information.""" - output: FileChecksum = Field(default=None, description="File checksum information") + output: FileChecksum|None = Field(default=None, description="File checksum information") class GetFileTypeResponse(BaseModel): """Represents the response for getting the type of a file.""" - output: str = Field(default=None, description="Type of the file", example="directory") + output: str|None = Field(default=None, description="Type of the file", example="directory") class GetFileStatResponse(BaseModel): """Represents the response for getting file metadata information.""" - output: FileStat = Field(default=None, description="File stat information") + output: FileStat|None = Field(default=None, description="File stat information") class GetFileDownloadResponse(BaseModel): """Represents the response for downloading a file.""" - output: str = Field(default=None, description="Download URL or identifier", example="https://example.com/download/file") + output: str|None = Field(default=None, description="Download URL or identifier", example="https://example.com/download/file") class PatchFileMetadataResponse(BaseModel): """Represents the response for updating file metadata.""" - output: PatchFile = Field(default=None, description="Updated file metadata") + output: PatchFile|None = Field(default=None, description="Updated file metadata") class FilesystemRequestBase(BaseModel): """Base class for filesystem operation requests.""" # Should we allow both: path and source_path? Or just one of them? - path: str = Field(default=None, validation_alias=AliasChoices("path", "source_path"), description="Source file or directory path", example="/home/user/dir") + path: str|None = Field(default=None, validation_alias=AliasChoices("path", "source_path"), description="Source file or directory path", example="/home/user/dir") class PutFileChmodRequest(FilesystemRequestBase): @@ -135,7 +135,7 @@ class PutFileChmodRequest(FilesystemRequestBase): class PutFileChmodResponse(BaseModel): """Represents the response for changing file permissions.""" - output: File = Field(default=None, description="Updated file metadata") + output: File|None = Field(default=None, description="Updated file metadata") class PutFileChownRequest(FilesystemRequestBase): @@ -157,12 +157,12 @@ class PutFileChownRequest(FilesystemRequestBase): class PutFileChownResponse(BaseModel): """Represents the response for changing file ownership.""" - output: File = Field(default=None, description="Updated file metadata") + output: File|None = Field(default=None, description="Updated file metadata") class PutFileUploadResponse(BaseModel): """Represents the response for uploading a file.""" - output: str = Field(default=None, description="Upload result or identifier") + output: str|None = Field(default=None, description="Upload result or identifier") class PostMakeDirRequest(FilesystemRequestBase): @@ -179,28 +179,28 @@ class PostFileSymlinkRequest(FilesystemRequestBase): class PostFileSymlinkResponse(BaseModel): """Represents the response for creating a symbolic link.""" - output: File = Field(default=None, description="Created symlink metadata") + output: File|None = Field(default=None, description="Created symlink metadata") class GetViewFileResponse(BaseModel): """Represents the response for viewing a file.""" - output: str = Field(default=None, description="File content") + output: str|None = Field(default=None, description="File content") class PostMkdirResponse(BaseModel): """Represents the response for creating a directory.""" - output: File = Field(default=None, description="Created directory metadata") + output: File|None = Field(default=None, description="Created directory metadata") class PostCompressResponse(BaseModel): """Represents the response for compressing a file.""" - output: File = Field(default=None, description="Compressed file metadata") + output: File|None = Field(default=None, description="Compressed file metadata") class PostCompressRequest(FilesystemRequestBase): """Represents a request to compress a file.""" target_path: str = Field(..., description="Path to the compressed file", example="/home/user/file.tar.gz") - match_pattern: str = Field(default=None, description="Regex pattern to filter files to compress", example=".*\\.txt$") + match_pattern: str|None = Field(default=None, description="Regex pattern to filter files to compress", example=".*\\.txt$") dereference: bool = Field(default=False, description="If set to `true`, it follows symbolic links and archive the files they point to instead of the links themselves.", example=True) compression: CompressionType = Field(default="gzip", description="Defines the type of compression to be used. By default gzip is used.", example="gzip") model_config = { @@ -220,7 +220,7 @@ class PostCompressRequest(FilesystemRequestBase): class PostExtractResponse(BaseModel): """Represents the response for extracting a compressed file.""" - output: File = Field(default=None, description="Extracted file metadata") + output: File|None = Field(default=None, description="Extracted file metadata") class PostExtractRequest(FilesystemRequestBase): @@ -259,7 +259,7 @@ class PostCopyRequest(FilesystemRequestBase): class PostCopyResponse(BaseModel): """Represents the response for copying a file.""" - output: File = Field(default=None, description="Copied file metadata") + output: File|None = Field(default=None, description="Copied file metadata") class PostMoveRequest(FilesystemRequestBase): @@ -279,9 +279,9 @@ class PostMoveRequest(FilesystemRequestBase): class PostMoveResponse(BaseModel): """Represents the response for moving a file.""" - output: File = Field(default=None, description="Moved file metadata") + output: File|None = Field(default=None, description="Moved file metadata") class RemoveResponse(BaseModel): """Represents the response for removing a file or directory.""" - output: str = Field(default=None, description="Removal result message") + output: str|None = Field(default=None, description="Removal result message") diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index f0b9e9f3..4bc88a11 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -23,8 +23,8 @@ async def get_resources( description: str | None = None, group: str | None = None, modified_since: datetime.datetime | None = None, - resource_type: status_models.ResourceType = Query(default=None), - current_status: status_models.Status = Query(default=None), + resource_type: status_models.ResourceType|None = Query(default=None), + current_status: status_models.Status|None = Query(default=None), capability: Capability | None = None, site_id: str | None = None, ) -> list[status_models.Resource]: diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 0e051d65..e020f4df 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -35,8 +35,8 @@ def _self_path(self) -> str: site_id: str = Field(..., description="The site identifier this resource is located at", exclude=True, example="site-1") capability_ids: list[str] = Field(default_factory=list, exclude=True) - group: str = Field(default=None, description="Logical grouping of the resource", example="frontend") - current_status: Status = Field(default=None, description="The current status comes from the status of the last event for this resource", example="up") + group: str|None = Field(default=None, description="Logical grouping of the resource", example="frontend") + current_status: Status|None = Field(default=None, description="The current status comes from the status of the last event for this resource", example="up") resource_type: ResourceType = Field(..., description="Type of the resource", example="service") @computed_field(description="URI of the site where this resource is located") @@ -83,7 +83,7 @@ def _norm_dt_field(cls, v): occurred_at: datetime.datetime = Field(..., description="Timestamp when the event occurred", example="2026-02-21T12:00:00Z") status: Status = Field(..., description="Status of the resource at the time of the event", example="down") resource_id: str = Field(..., exclude=True, description="Identifier of the affected resource", example="res-1") - incident_id: str = Field(default=None, exclude=True, description="Identifier of the related incident", example="inc-1") + incident_id: str|None = Field(default=None, exclude=True, description="Identifier of the related incident", example="inc-1") @computed_field(description="The resource belonging to this event") @property @@ -152,7 +152,7 @@ def _norm_dt_field(cls, v): resource_ids: list[str] = Field(default_factory=list, exclude=True) event_ids: list[str] = Field(default_factory=list, exclude=True) start: datetime.datetime = Field(..., description="Incident start time", example="2026-02-21T12:00:00Z") - end: datetime.datetime = Field(default=None, description="Incident end time", example="2026-02-21T14:00:00Z") + end: datetime.datetime|None = Field(default=None, description="Incident end time", example="2026-02-21T14:00:00Z") type: IncidentType = Field(..., description="Type of incident", example="planned") resolution: Resolution = Field(..., description="Resolution status of the incident", example="pending") diff --git a/app/routers/status/status.py b/app/routers/status/status.py index e313d3ee..62a9ea19 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -25,15 +25,15 @@ ) async def get_resources( request: Request, - name: str = Query(default=None, min_length=1), - description: str = Query(default=None, min_length=1), - group: str = Query(default=None, min_length=1), + name: str|None = Query(default=None, min_length=1), + description: str|None = Query(default=None, min_length=1), + group: str|None = Query(default=None, min_length=1), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), - modified_since: StrictDateTime = Query(default=None), - resource_type: models.ResourceType = Query(default=None), - current_status: models.Status = Query(default=None), - capability: List[AllocationUnit] = Query(default=None, min_length=1), + modified_since: StrictDateTime|None = Query(default=None), + resource_type: models.ResourceType|None = Query(default=None), + current_status: models.Status|None = Query(default=None), + capability: List[AllocationUnit]|None = Query(default=None, min_length=1), _forbid=Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: return await router.adapter.get_resources( @@ -67,18 +67,18 @@ async def get_resource( ) async def get_incidents( request: Request, - name: str = Query(default=None, min_length=1), - description: str = Query(default=None, min_length=1), - status: models.Status = Query(default=None), - type_: models.IncidentType = Query(alias="type", default=None), - from_: StrictDateTime = Query(alias="from", default=None), - time_: StrictDateTime = Query(alias="time", default=None), - to: StrictDateTime = Query(default=None), - modified_since: StrictDateTime = Query(default=None), - resource_id: str = Query(default=None, min_length=1), + name: str|None = Query(default=None, min_length=1), + description: str|None = Query(default=None, min_length=1), + status: models.Status|None = Query(default=None), + type_: models.IncidentType|None = Query(alias="type", default=None), + from_: StrictDateTime|None = Query(alias="from", default=None), + time_: StrictDateTime|None = Query(alias="time", default=None), + to: StrictDateTime|None = Query(default=None), + modified_since: StrictDateTime|None = Query(default=None), + resource_id: str|None = Query(default=None, min_length=1), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), - resolution: models.Resolution = Query(default=None), + resolution: models.Resolution|None = Query(default=None), _forbid=Depends( forbidExtraQueryParams( "name", @@ -139,14 +139,14 @@ async def get_incident(request: Request, incident_id: str) -> models.Incident: async def get_events( request: Request, incident_id: str, - resource_id: str = Query(default=None, min_length=1), - name: str = Query(default=None, min_length=1), - description: str = Query(default=None, min_length=1), - status: models.Status = Query(default=None), - from_: StrictDateTime = Query(alias="from", default=None), - time_: StrictDateTime = Query(alias="time", default=None), - to: StrictDateTime = Query(default=None), - modified_since: StrictDateTime = Query(default=None), + resource_id: str|None = Query(default=None, min_length=1), + name: str|None = Query(default=None, min_length=1), + description: str|None = Query(default=None, min_length=1), + status: models.Status|None = Query(default=None), + from_: StrictDateTime|None = Query(alias="from", default=None), + time_: StrictDateTime|None = Query(alias="time", default=None), + to: StrictDateTime|None = Query(default=None), + modified_since: StrictDateTime|None = Query(default=None), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), _forbid=Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), diff --git a/app/routers/task/models.py b/app/routers/task/models.py index a5a5072c..1e042307 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -37,5 +37,5 @@ class Task(BaseModel): """Represents a task in the system.""" id: str = Field(..., description="Unique identifier of the task", example="task-123") status: TaskStatus = Field(default=TaskStatus.pending, description="Current status of the task", example="pending") - result: Any = Field(default=None, description="Result of the task execution, if available") - command: TaskCommand = Field(default=None, description="Command associated with this task") + result: Any|None = Field(default=None, description="Result of the task execution, if available") + command: TaskCommand|None = Field(default=None, description="Command associated with this task") diff --git a/app/types/base.py b/app/types/base.py index 254edda7..e25734a3 100644 --- a/app/types/base.py +++ b/app/types/base.py @@ -61,8 +61,8 @@ def self_uri(self) -> str: """Computed self URI property.""" return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" - name: str = Field(default=None, description="The long name of the object.", example="Perlmutter GPU") - description: str = Field(default=None, description="Human-readable description of the object.", example="High-performance GPU compute resource") + name: str|None = Field(default=None, description="The long name of the object.", example="Perlmutter GPU") + description: str|None = Field(default=None, description="Human-readable description of the object.", example="High-performance GPU compute resource") last_modified: StrictDateTime = Field(..., description="ISO 8601 timestamp when this object was last modified.", example="2026-02-21T12:00:00Z") @classmethod diff --git a/app/types/models.py b/app/types/models.py index 6240b785..113e5dc5 100644 --- a/app/types/models.py +++ b/app/types/models.py @@ -18,6 +18,6 @@ class Capability(NamedObject): def _self_path(self) -> str: return f"/account/capabilities/{self.id}" - last_modified: StrictDateTime = Field(default=None, description="ISO 8601 timestamp when this object was last modified.", example="2026-02-21T12:00:00Z") + last_modified: StrictDateTime|None = Field(default=None, description="ISO 8601 timestamp when this object was last modified.", example="2026-02-21T12:00:00Z") units: list[AllocationUnit] = Field(..., description="Allocation units supported by this capability", example=["node_hours"]) From c43431be0d71a44bd8c152ee4a520024573c1b9f Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Wed, 25 Feb 2026 15:45:24 -0800 Subject: [PATCH 097/133] removed or-none from Query args --- app/routers/account/account.py | 6 ++-- app/routers/facility/facility.py | 8 ++--- app/routers/status/facility_adapter.py | 6 ++-- app/routers/status/status.py | 50 +++++++++++++------------- 4 files changed, 34 insertions(+), 36 deletions(-) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index 075b46a0..f3b36963 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -24,8 +24,8 @@ ) async def get_capabilities( request: Request, - name: str|None = Query(default=None, min_length=1), - modified_since: StrictDateTime|None = Query(default=None), + name: str = Query(default=None, min_length=1), + modified_since: StrictDateTime = Query(default=None), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), _forbid=Depends(forbidExtraQueryParams("name", "modified_since", "offset", "limit")), @@ -43,7 +43,7 @@ async def get_capabilities( async def get_capability( capability_id: str, request: Request, - modified_since: StrictDateTime|None = Query(default=None), + modified_since: StrictDateTime = Query(default=None), _forbid=Depends(forbidExtraQueryParams("modified_since")), ) -> Capability: caps = await router.adapter.get_capabilities(name=None, modified_since=modified_since, offset=0, limit=100) diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 61e124c2..1ab5b5c2 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -26,11 +26,11 @@ async def get_facility( @router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites", response_model_exclude_none=True,) async def list_sites( request: Request, - modified_since: StrictDateTime|None = Query(default=None), - name: str|None = Query(default=None, min_length=1), + modified_since: StrictDateTime = Query(default=None), + name: str = Query(default=None, min_length=1), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), - short_name: str|None = Query(default=None, min_length=1), + short_name: str = Query(default=None, min_length=1), _forbid=Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), ) -> list[models.Site]: """List sites""" @@ -41,7 +41,7 @@ async def list_sites( async def get_site( request: Request, site_id: str, - modified_since: StrictDateTime|None = Query(default=None), + modified_since: StrictDateTime = Query(default=None), _forbid=Depends(forbidExtraQueryParams("modified_since")), ) -> models.Site: """Get site by ID""" diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 4bc88a11..96c32003 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -1,8 +1,6 @@ import datetime from abc import ABC, abstractmethod -from fastapi import Query - from ...types.models import Capability from . import models as status_models @@ -23,8 +21,8 @@ async def get_resources( description: str | None = None, group: str | None = None, modified_since: datetime.datetime | None = None, - resource_type: status_models.ResourceType|None = Query(default=None), - current_status: status_models.Status|None = Query(default=None), + resource_type: status_models.ResourceType|None = None, + current_status: status_models.Status|None = None, capability: Capability | None = None, site_id: str | None = None, ) -> list[status_models.Resource]: diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 62a9ea19..e313d3ee 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -25,15 +25,15 @@ ) async def get_resources( request: Request, - name: str|None = Query(default=None, min_length=1), - description: str|None = Query(default=None, min_length=1), - group: str|None = Query(default=None, min_length=1), + name: str = Query(default=None, min_length=1), + description: str = Query(default=None, min_length=1), + group: str = Query(default=None, min_length=1), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), - modified_since: StrictDateTime|None = Query(default=None), - resource_type: models.ResourceType|None = Query(default=None), - current_status: models.Status|None = Query(default=None), - capability: List[AllocationUnit]|None = Query(default=None, min_length=1), + modified_since: StrictDateTime = Query(default=None), + resource_type: models.ResourceType = Query(default=None), + current_status: models.Status = Query(default=None), + capability: List[AllocationUnit] = Query(default=None, min_length=1), _forbid=Depends(forbidExtraQueryParams("name", "description", "group", "offset", "limit", "modified_since", "resource_type", "current_status", "capability", multiParams={"capability"})), ) -> list[models.Resource]: return await router.adapter.get_resources( @@ -67,18 +67,18 @@ async def get_resource( ) async def get_incidents( request: Request, - name: str|None = Query(default=None, min_length=1), - description: str|None = Query(default=None, min_length=1), - status: models.Status|None = Query(default=None), - type_: models.IncidentType|None = Query(alias="type", default=None), - from_: StrictDateTime|None = Query(alias="from", default=None), - time_: StrictDateTime|None = Query(alias="time", default=None), - to: StrictDateTime|None = Query(default=None), - modified_since: StrictDateTime|None = Query(default=None), - resource_id: str|None = Query(default=None, min_length=1), + name: str = Query(default=None, min_length=1), + description: str = Query(default=None, min_length=1), + status: models.Status = Query(default=None), + type_: models.IncidentType = Query(alias="type", default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_: StrictDateTime = Query(alias="time", default=None), + to: StrictDateTime = Query(default=None), + modified_since: StrictDateTime = Query(default=None), + resource_id: str = Query(default=None, min_length=1), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), - resolution: models.Resolution|None = Query(default=None), + resolution: models.Resolution = Query(default=None), _forbid=Depends( forbidExtraQueryParams( "name", @@ -139,14 +139,14 @@ async def get_incident(request: Request, incident_id: str) -> models.Incident: async def get_events( request: Request, incident_id: str, - resource_id: str|None = Query(default=None, min_length=1), - name: str|None = Query(default=None, min_length=1), - description: str|None = Query(default=None, min_length=1), - status: models.Status|None = Query(default=None), - from_: StrictDateTime|None = Query(alias="from", default=None), - time_: StrictDateTime|None = Query(alias="time", default=None), - to: StrictDateTime|None = Query(default=None), - modified_since: StrictDateTime|None = Query(default=None), + resource_id: str = Query(default=None, min_length=1), + name: str = Query(default=None, min_length=1), + description: str = Query(default=None, min_length=1), + status: models.Status = Query(default=None), + from_: StrictDateTime = Query(alias="from", default=None), + time_: StrictDateTime = Query(alias="time", default=None), + to: StrictDateTime = Query(default=None), + modified_since: StrictDateTime = Query(default=None), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), _forbid=Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), From 0a5ddd2c425e696e5115ef3f95b6b5791ad0997b Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 2 Mar 2026 09:24:48 -0600 Subject: [PATCH 098/133] Tighten query types and handle empty retsults with 404 For optional filters, use `str | None`. Empty results return 404 and message --- app/routers/account/account.py | 2 +- app/routers/facility/facility.py | 9 ++++++--- app/routers/status/status.py | 23 ++++++++++++++--------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index f3b36963..b2fc4101 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -24,7 +24,7 @@ ) async def get_capabilities( request: Request, - name: str = Query(default=None, min_length=1), + name: str | None = Query(default=None, min_length=1), modified_since: StrictDateTime = Query(default=None), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 1ab5b5c2..7abdb47a 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -27,14 +27,17 @@ async def get_facility( async def list_sites( request: Request, modified_since: StrictDateTime = Query(default=None), - name: str = Query(default=None, min_length=1), + name: str | None = Query(default=None, min_length=1), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), - short_name: str = Query(default=None, min_length=1), + short_name: str | None = Query(default=None, min_length=1), _forbid=Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), ) -> list[models.Site]: """List sites""" - return await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + sites = await router.adapter.list_sites(modified_since=modified_since, name=name, offset=offset, limit=limit, short_name=short_name) + if not sites: + raise HTTPException(status_code=404, detail="No sites found") + return sites @router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite", response_model_exclude_none=True,) diff --git a/app/routers/status/status.py b/app/routers/status/status.py index e313d3ee..46ff4b4a 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -67,15 +67,15 @@ async def get_resource( ) async def get_incidents( request: Request, - name: str = Query(default=None, min_length=1), - description: str = Query(default=None, min_length=1), + name: str | None = Query(default=None, min_length=1), + description: str | None = Query(default=None, min_length=1), status: models.Status = Query(default=None), type_: models.IncidentType = Query(alias="type", default=None), from_: StrictDateTime = Query(alias="from", default=None), time_: StrictDateTime = Query(alias="time", default=None), to: StrictDateTime = Query(default=None), modified_since: StrictDateTime = Query(default=None), - resource_id: str = Query(default=None, min_length=1), + resource_id: str | None = Query(default=None, min_length=1), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), resolution: models.Resolution = Query(default=None), @@ -99,7 +99,7 @@ async def get_incidents( ) ), ) -> list[models.Incident]: - return await router.adapter.get_incidents( + incidents = await router.adapter.get_incidents( offset=offset, limit=limit, name=name, @@ -113,7 +113,9 @@ async def get_incidents( resource_id=resource_id, resolution=resolution, ) - + if not incidents: + raise HTTPException(status_code=404, detail="No incidents found") + return incidents @router.get( "/incidents/{incident_id}", @@ -139,9 +141,9 @@ async def get_incident(request: Request, incident_id: str) -> models.Incident: async def get_events( request: Request, incident_id: str, - resource_id: str = Query(default=None, min_length=1), - name: str = Query(default=None, min_length=1), - description: str = Query(default=None, min_length=1), + resource_id: str | None = Query(default=None, min_length=1), + name: str | None = Query(default=None, min_length=1), + description: str | None = Query(default=None, min_length=1), status: models.Status = Query(default=None), from_: StrictDateTime = Query(alias="from", default=None), time_: StrictDateTime = Query(alias="time", default=None), @@ -151,9 +153,12 @@ async def get_events( limit: int = Query(default=100, ge=0, le=1000), _forbid=Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: - return await router.adapter.get_events( + events = await router.adapter.get_events( incident_id, offset=offset, limit=limit, resource_id=resource_id, name=name, description=description, status=status, from_=from_, to=to, time_=time_, modified_since=modified_since ) + if not events: + raise HTTPException(status_code=404, detail="No events found") + return events @router.get( From 7a2c87183e37cb50ca479ab617fc26a32607db79 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 2 Mar 2026 17:46:38 -0600 Subject: [PATCH 099/133] Fix schemathesis errors --- app/routers/compute/compute.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 0115e352..6023ddd6 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -134,8 +134,8 @@ async def get_job_status( resource_id: str, job_id: str, request: Request, - historical: StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), - include_spec: StrictHTTPBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + historical: StrictHTTPBool | None = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictHTTPBool | None = Query(default=False, description="Whether to include the job specification. Defaults to false"), _forbid=Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" @@ -166,8 +166,8 @@ async def get_job_statuses( offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), filters: dict[str, object] | None = None, - historical: StrictHTTPBool = Query(default=False, description="Whether to include historical jobs. Defaults to false"), - include_spec: StrictHTTPBool = Query(default=False, description="Whether to include the job specification. Defaults to false"), + historical: StrictHTTPBool | None = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + include_spec: StrictHTTPBool | None = Query(default=False, description="Whether to include the job specification. Defaults to false"), _forbid=Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" From 5d924bac95cf98a7df676bff1ce60c620dfc7fee Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 2 Mar 2026 17:48:00 -0600 Subject: [PATCH 100/133] Do not use Any in return Use of Any produces the following OpenAPI schema: ``` "result": { "anyOf": [ {}, { "type": "null" } ], ... } ``` and that empty {} breaks tool generators. Any task output is always dict, so it is either dict or None. Facilities are required to json.load before return --- app/demo_adapter.py | 5 ++--- app/routers/filesystem/facility_adapter.py | 1 - app/routers/task/models.py | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index fd112968..231d18d9 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -13,7 +13,6 @@ import stat import subprocess import uuid -from typing import Any, Tuple from fastapi import HTTPException from fastapi.encoders import jsonable_encoder @@ -731,7 +730,7 @@ def _headtail( lines: int | None, skip_heading: bool = False, skip_trailing: bool = False, - ) -> Any: + ) -> str: args = [cmd] if cmd == "tail" and skip_heading: @@ -997,7 +996,7 @@ class DemoTask(BaseModel): user: account_models.User start: float status: task_models.TaskStatus = task_models.TaskStatus.pending - result: Any | None = None + result: dict | None = None class DemoTaskQueue: diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index bc1d74d8..497f16e4 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -1,6 +1,5 @@ import os from abc import abstractmethod -from typing import Any, Tuple from ..status import models as status_models from ..account import models as account_models from . import models as filesystem_models diff --git a/app/routers/task/models.py b/app/routers/task/models.py index 1e042307..7a13ac0e 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -1,6 +1,5 @@ """ Task models for the IRI Facility API """ import enum -from typing import Any from pydantic import BaseModel, Field, computed_field from ... import config @@ -37,5 +36,5 @@ class Task(BaseModel): """Represents a task in the system.""" id: str = Field(..., description="Unique identifier of the task", example="task-123") status: TaskStatus = Field(default=TaskStatus.pending, description="Current status of the task", example="pending") - result: Any|None = Field(default=None, description="Result of the task execution, if available") + result: dict | None = Field(default=None, description="Result of the task execution, if available") command: TaskCommand|None = Field(default=None, description="Command associated with this task") From 6b56ffa90f2102077a49ea7fd28ec79443f5cb35 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 2 Mar 2026 20:40:29 -0600 Subject: [PATCH 101/133] Fix FS Demo adapter to return in json --- app/demo_adapter.py | 7 +++- app/routers/task/facility_adapter.py | 59 ++++++++++------------------ test/test_filesystem.py | 6 +-- 3 files changed, 30 insertions(+), 42 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 231d18d9..6b32869a 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -1018,7 +1018,12 @@ async def process_tasks(da: DemoAdapter): elif t.status == task_models.TaskStatus.active and now - t.start > DEMO_QUEUE_UPDATE_SECS: cmd = task_models.TaskCommand.model_validate_json(t.task) (result, status) = await DemoAdapter.on_task(t.resource, t.user, cmd) - t.result = jsonable_encoder(result) + if isinstance(result, BaseModel): + t.result = result.model_dump() + elif isinstance(result, dict): + t.result = result + else: + t.result = {"output": result} t.status = status _tasks.append(t) DemoTaskQueue.tasks = _tasks diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index afec203a..43f9a5e6 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -34,7 +34,7 @@ async def delete_task(self: "FacilityAdapter", user: account_models.User, task_i pass @staticmethod - async def on_task(resource: status_models.Resource, user: account_models.User, task: task_models.TaskCommand) -> tuple[str, task_models.TaskStatus]: + async def on_task(resource: status_models.Resource, user: account_models.User, task: task_models.TaskCommand) -> tuple[object, task_models.TaskStatus]: # Handle a task from the facility message queue. # Returns: (result, status) def _extractNull(ind): @@ -48,78 +48,61 @@ def _extractNull(ind): if task.command == "chmod": data = _extractNull(task.args["request_model"]) request_model = filesystem_models.PutFileChmodRequest.model_validate(data) - o = await fs_adapter.chmod(resource, user, request_model) - r = o.model_dump_json() + r = await fs_adapter.chmod(resource, user, request_model) elif task.command == "chown": data = _extractNull(task.args["request_model"]) request_model = filesystem_models.PutFileChownRequest.model_validate(data) - o = await fs_adapter.chown(resource, user, request_model) - r = o.model_dump_json() + r = await fs_adapter.chown(resource, user, request_model) elif task.command == "file": - o = await fs_adapter.file(resource, user, **task.args) - r = o.model_dump_json() + r = await fs_adapter.file(resource, user, **task.args) elif task.command == "stat": - o = await fs_adapter.stat(resource, user, **task.args) - r = o.model_dump_json() + r = await fs_adapter.stat(resource, user, **task.args) elif task.command == "mkdir": data = _extractNull(task.args["request_model"]) request_model = filesystem_models.PostMakeDirRequest.model_validate(data) - o = await fs_adapter.mkdir(resource, user, request_model) - r = o.model_dump_json() + r = await fs_adapter.mkdir(resource, user, request_model) elif task.command == "symlink": data = _extractNull(task.args["request_model"]) request_model = filesystem_models.PostFileSymlinkRequest.model_validate(data) - o = await fs_adapter.symlink(resource, user, request_model) - r = o.model_dump_json() + r = await fs_adapter.symlink(resource, user, request_model) elif task.command == "ls": - o = await fs_adapter.ls(resource, user, **task.args) - r = o.model_dump_json() + r = await fs_adapter.ls(resource, user, **task.args) elif task.command == "head": - o = await fs_adapter.head(resource, user, **task.args) - r = o.model_dump_json() + r = await fs_adapter.head(resource, user, **task.args) elif task.command == "view": - o = await fs_adapter.view(resource, user, **task.args) - r = o.model_dump_json() + r = await fs_adapter.view(resource, user, **task.args) elif task.command == "tail": - o = await fs_adapter.tail(resource, user, **task.args) - r = o.model_dump_json() + r = await fs_adapter.tail(resource, user, **task.args) elif task.command == "checksum": - o = await fs_adapter.checksum(resource, user, **task.args) - r = o.model_dump_json() + r = await fs_adapter.checksum(resource, user, **task.args) elif task.command == "rm": - o = await fs_adapter.rm(resource, user, **task.args) - r = o.model_dump_json() + r = await fs_adapter.rm(resource, user, **task.args) elif task.command == "compress": data = _extractNull(task.args["request_model"]) request_model = filesystem_models.PostCompressRequest.model_validate(data) - o = await fs_adapter.compress(resource, user, request_model) - r = o.model_dump_json() + r = await fs_adapter.compress(resource, user, request_model) elif task.command == "extract": data = _extractNull(task.args["request_model"]) request_model = filesystem_models.PostExtractRequest.model_validate(data) - o = await fs_adapter.extract(resource, user, request_model) - r = o.model_dump_json() + r = await fs_adapter.extract(resource, user, request_model) elif task.command == "mv": data = _extractNull(task.args["request_model"]) request_model = filesystem_models.PostMoveRequest.model_validate(data) - o = await fs_adapter.mv(resource, user, request_model) - r = o.model_dump_json() + r = await fs_adapter.mv(resource, user, request_model) elif task.command == "cp": data = _extractNull(task.args["request_model"]) request_model = filesystem_models.PostCopyRequest.model_validate(data) - o = await fs_adapter.cp(resource, user, request_model) - r = o.model_dump_json() + r = await fs_adapter.cp(resource, user, request_model) elif task.command == "download": r = await fs_adapter.download(resource, user, **task.args) elif task.command == "upload": - o = await fs_adapter.upload(resource, user, **task.args) - r = "File uploaded successfully" - if r: + r = await fs_adapter.upload(resource, user, **task.args) + if r is not None: return (r, task_models.TaskStatus.completed) else: - return (f"Task was cancelled due to unknown router/command: {task.router}:{task.command}", task_models.TaskStatus.failed) + return ({"output": f"Task was cancelled due to unknown router/command: {task.router}:{task.command}"}, task_models.TaskStatus.failed) except Exception as exc: traceback_str = traceback.format_exc() logger.warning(f"Error handling task {task.router}:{task.command} with args: {task.args}\nError: {exc}") logger.debug(f"Traceback:\n{traceback_str}") - return (f"Error: {exc}", task_models.TaskStatus.failed) + return ({"output": f"Error: {exc}"}, task_models.TaskStatus.failed) diff --git a/test/test_filesystem.py b/test/test_filesystem.py index 46ce5a08..e83f6137 100644 --- a/test/test_filesystem.py +++ b/test/test_filesystem.py @@ -13,12 +13,12 @@ # CONFIG — EDIT THESE AS NEEDED # ========================= -#BASE_URL = "http://localhost:8000/api/v1" -BASE_URL = "https://api.iri.nersc.gov/api/v1" +BASE_URL = "http://localhost:8000/api/v1" +#BASE_URL = "https://api.iri.nersc.gov/api/v1" #BASE_URL = "https://iri-dev.ppg.es.net/api/v1" TOKEN = os.environ.get("IRI_API_TOKEN", "12345") # ========================= -HEADERS = {"Authorization": f"{TOKEN}", "Accept": "application/json"} +HEADERS = {"Authorization": f"Bearer {TOKEN}", "Accept": "application/json"} POLL_INTERVAL = 2 TIMEOUT = 180 From f31ff76f69b4df3ed371867778620889f668b252 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 3 Mar 2026 09:51:48 -0800 Subject: [PATCH 102/133] view to use FileContent to return output --- app/demo_adapter.py | 7 ++++++- app/routers/filesystem/models.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index fd112968..345999d3 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -806,7 +806,12 @@ async def view(self: "DemoAdapter", resource: status_models.Resource, user: acco result = self._run(f"tail -c +{offset + 1} {rp} | head -c {size}", shell=True) content = result.stdout return filesystem_models.GetViewFileResponse( - output=content, + output=filesystem_models.FileContent( + content=content, + content_type=filesystem_models.ContentUnit.bytes, + start_position=offset, + end_position=offset + len(content) + ), ) async def checksum(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileChecksumResponse: diff --git a/app/routers/filesystem/models.py b/app/routers/filesystem/models.py index 59411351..fc36ebca 100644 --- a/app/routers/filesystem/models.py +++ b/app/routers/filesystem/models.py @@ -184,7 +184,7 @@ class PostFileSymlinkResponse(BaseModel): class GetViewFileResponse(BaseModel): """Represents the response for viewing a file.""" - output: str|None = Field(default=None, description="File content") + output: FileContent|None = Field(default=None, description="File content") class PostMkdirResponse(BaseModel): From b694b52b65cf5373eddc79b2b4da256c7265b9d5 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 3 Mar 2026 16:16:38 -0800 Subject: [PATCH 103/133] renamed incorrectly named param --- app/routers/filesystem/facility_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index bc1d74d8..d525aa10 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -47,7 +47,7 @@ async def head(self: "FacilityAdapter", resource: status_models.Resource, user: pass @abstractmethod - async def tail(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int | None, lines: int | None, skip_trailing: bool) -> filesystem_models.GetFileTailResponse: + async def tail(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int | None, lines: int | None, skip_heading: bool) -> filesystem_models.GetFileTailResponse: pass @abstractmethod From 86c0f9001395baabb7294bd66c93a342c2659795 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 4 Mar 2026 08:36:56 -0600 Subject: [PATCH 104/133] Change object to dict --- app/routers/task/facility_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index 43f9a5e6..2bd0cf59 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -34,7 +34,7 @@ async def delete_task(self: "FacilityAdapter", user: account_models.User, task_i pass @staticmethod - async def on_task(resource: status_models.Resource, user: account_models.User, task: task_models.TaskCommand) -> tuple[object, task_models.TaskStatus]: + async def on_task(resource: status_models.Resource, user: account_models.User, task: task_models.TaskCommand) -> tuple[dict, task_models.TaskStatus]: # Handle a task from the facility message queue. # Returns: (result, status) def _extractNull(ind): From b0f50b8998a281d67253068894189526136208a6 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Wed, 4 Mar 2026 13:57:24 -0800 Subject: [PATCH 105/133] on_task bugfix. Refs https://github.com/doe-iri/iri-facility-api-python/issues/52 --- app/routers/task/facility_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index afec203a..c0dd7757 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -38,7 +38,7 @@ async def on_task(resource: status_models.Resource, user: account_models.User, t # Handle a task from the facility message queue. # Returns: (result, status) def _extractNull(ind): - data = {k: v for k, v in ind.items() if v is not None} + data = {k: v for k, v in ind.model_dump().items() if v is not None} return data try: r = None From 623e207c13f7b7c1f27a949c7415c5daca05f069 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Wed, 4 Mar 2026 14:14:32 -0800 Subject: [PATCH 106/133] handle both dict and BaseModel --- app/routers/task/facility_adapter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index c0dd7757..ed119eb6 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -38,8 +38,11 @@ async def on_task(resource: status_models.Resource, user: account_models.User, t # Handle a task from the facility message queue. # Returns: (result, status) def _extractNull(ind): - data = {k: v for k, v in ind.model_dump().items() if v is not None} - return data + if hasattr(ind, "model_dump"): + data = ind.model_dump() + else: + data = ind + return {k: v for k, v in data.items() if v is not None} try: r = None logger.info(f"Received task: {task.router}:{task.command} with args: {task.args}") From c762036094b421148b291dfd90e1c5b3e3d240b1 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Wed, 4 Mar 2026 14:41:31 -0800 Subject: [PATCH 107/133] JobSpec is used as both input and output model. Refs https://github.com/doe-iri/iri-facility-api-python/issues/57 --- app/routers/compute/models.py | 72 ++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 94b511ce..0dba2f98 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -12,13 +12,13 @@ class ResourceSpec(IRIBaseModel): Specification of computational resources required for a job. """ - node_count: Annotated[int, Field(ge=1, description="Number of compute nodes to allocate", example=2)] = None - process_count: Annotated[int, Field(ge=1, description="Total number of processes to launch", example=64)] = None - processes_per_node: Annotated[int, Field(ge=1, description="Number of processes to launch per node", example=32)] = None - cpu_cores_per_process: Annotated[int, Field(ge=1, description="Number of CPU cores to allocate per process", example=2)] = None - gpu_cores_per_process: Annotated[int, Field(ge=1, description="Number of GPU cores to allocate per process", example=1)] = None - exclusive_node_use: Annotated[StrictBool, Field(description="Whether to request exclusive use of allocated nodes", example=True)] = True - memory: Annotated[int, Field(ge=1, description="Amount of memory to allocate in bytes", example=17179869184)] = None + node_count: int|None = Field(default=None, ge=1, description="Number of compute nodes to allocate", example=2) + process_count: int|None = Field(default=None, ge=1, description="Total number of processes to launch", example=64) + processes_per_node: int|None = Field(default=None, ge=1, description="Number of processes to launch per node", example=32) + cpu_cores_per_process: int|None = Field(default=None, ge=1, description="Number of CPU cores to allocate per process", example=2) + gpu_cores_per_process: int|None = Field(default=None, ge=1, description="Number of GPU cores to allocate per process", example=1) + exclusive_node_use: StrictBool = Field(default=True, description="Whether to request exclusive use of allocated nodes", example=True) + memory: int|None = Field(default=None, ge=1, description="Amount of memory to allocate in bytes", example=17179869184) class JobAttributes(IRIBaseModel): @@ -26,11 +26,11 @@ class JobAttributes(IRIBaseModel): Additional attributes and scheduling parameters for a job. """ - duration: Annotated[int, Field(description="Duration in seconds", ge=1, examples=[30, 60, 120])] = None - queue_name: Annotated[str, Field(min_length=1, description="Name of the queue or partition to submit the job to", example="debug")] = None - account: Annotated[str, Field(min_length=1, description="Account or project to charge for resource usage", example="proj123")] = None - reservation_id: Annotated[str, Field(min_length=1, description="ID of a reservation to use for the job", example="resv-42")] = None - custom_attributes: Annotated[dict[str, str], Field(description="Custom scheduler-specific attributes as key-value pairs", example={"constraint": "gpu"})] = {} + duration: int|None = Field(default=None, description="Duration in seconds", ge=1, examples=[30, 60, 120]) + queue_name: str|None = Field(default=None, min_length=1, description="Name of the queue or partition to submit the job to", example="debug") + account: str|None = Field(default=None, min_length=1, description="Account or project to charge for resource usage", example="proj123") + reservation_id: str|None = Field(default=None, min_length=1, description="ID of a reservation to use for the job", example="resv-42") + custom_attributes: dict[str, str] = Field(default={}, description="Custom scheduler-specific attributes as key-value pairs", example={"constraint": "gpu"}) class VolumeMount(IRIBaseModel): @@ -38,9 +38,9 @@ class VolumeMount(IRIBaseModel): Represents a volume mount for a container. """ - source: Annotated[str, Field(min_length=1, description="The source path on the host system to mount", example="/data/project")] - target: Annotated[str, Field(min_length=1, description="The target path inside the container where the volume will be mounted", example="/mnt/data")] - read_only: Annotated[StrictBool, Field(description="Whether the mount should be read-only", example=True)] = True + source: str = Field(min_length=1, description="The source path on the host system to mount", example="/data/project") + target: str = Field(min_length=1, description="The target path inside the container where the volume will be mounted", example="/mnt/data") + read_only: StrictBool = Field(default=True, description="Whether the mount should be read-only", example=True) class Container(IRIBaseModel): @@ -53,8 +53,8 @@ class Container(IRIBaseModel): host networking. """ - image: Annotated[str, Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')", example="docker.io/library/ubuntu:latest")] - volume_mounts: Annotated[list[VolumeMount], Field(description="List of volume mounts for the container")] = [] + image: str = Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')", example="docker.io/library/ubuntu:latest") + volume_mounts: list[VolumeMount] = Field(default=[], description="List of volume mounts for the container") class JobSpec(IRIBaseModel): @@ -63,24 +63,26 @@ class JobSpec(IRIBaseModel): """ model_config = ConfigDict(extra="forbid") - executable: Annotated[str, Field(min_length=1, - description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.", - example="/usr/bin/python")] = None - container: Annotated[Container, Field(description="Container specification for containerized execution")] = None - arguments: Annotated[list[str], Field(description="Command-line arguments to pass to the executable or container", example=["-n", "100"])] = [] - directory: Annotated[str, Field(min_length=1, description="Working directory for the job", example="/home/user/work")] = None - name: Annotated[str, Field(min_length=1, description="Name of the job", example="my-job")] = None - inherit_environment: Annotated[StrictBool, Field(description="Whether to inherit the environment variables from the submission environment", example=True)] = True - environment: Annotated[dict[str, str], Field(description="Environment variables to set for the job. If container is specified, these will be set inside the container.", - example={"OMP_NUM_THREADS": "4"})] = {} - stdin_path: Annotated[str, Field(min_length=1, description="Path to file to use as standard input", example="/home/user/input.txt")] = None - stdout_path: Annotated[str, Field(min_length=1, description="Path to file to write standard output", example="/home/user/output.txt")] = None - stderr_path: Annotated[str, Field(min_length=1, description="Path to file to write standard error", example="/home/user/error.txt")] = None - resources: Annotated[ResourceSpec, Field(description="Resource requirements for the job")] = None - attributes: Annotated[JobAttributes, Field(description="Additional job attributes such as duration, queue, and account")] = None - pre_launch: Annotated[str, Field(min_length=1, description="Script or commands to run before launching the job", example="module load cuda")] = None - post_launch: Annotated[str, Field(min_length=1, description="Script or commands to run after the job completes", example="echo done")] = None - launcher: Annotated[str, Field(min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')", example="srun")] = None + executable: str|None = Field(default=None, + min_length=1, + description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.", + example="/usr/bin/python") + container: Container|None = Field(default=None, description="Container specification for containerized execution") + arguments: list[str] = Field(default=[], description="Command-line arguments to pass to the executable or container", example=["-n", "100"]) + directory: str|None = Field(default=None, min_length=1, description="Working directory for the job", example="/home/user/work") + name: str|None = Field(default=None, min_length=1, description="Name of the job", example="my-job") + inherit_environment: StrictBool = Field(default=True, description="Whether to inherit the environment variables from the submission environment", example=True) + environment: dict[str, str] = Field(default={}, + description="Environment variables to set for the job. If container is specified, these will be set inside the container.", + example={"OMP_NUM_THREADS": "4"}) + stdin_path: str|None = Field(default=None, min_length=1, description="Path to file to use as standard input", example="/home/user/input.txt") + stdout_path: str|None = Field(default=None, min_length=1, description="Path to file to write standard output", example="/home/user/output.txt") + stderr_path: str|None = Field(default=None, min_length=1, description="Path to file to write standard error", example="/home/user/error.txt") + resources: ResourceSpec|None = Field(default=None, description="Resource requirements for the job") + attributes: JobAttributes|None = Field(default=None, description="Additional job attributes such as duration, queue, and account") + pre_launch: str|None = Field(default=None, min_length=1, description="Script or commands to run before launching the job", example="module load cuda") + post_launch: str|None = Field(default=None, min_length=1, description="Script or commands to run after the job completes", example="echo done") + launcher: str|None = Field(default=None, min_length=1, description="Job launcher to use (e.g., 'mpirun', 'srun')", example="srun") class JobState(str, Enum): From 23af870506fa07c022d83188daf21582c29fb958 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Thu, 5 Mar 2026 10:15:53 -0800 Subject: [PATCH 108/133] use default_factory for list/dict params --- app/routers/compute/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 0dba2f98..87631e2f 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -30,7 +30,7 @@ class JobAttributes(IRIBaseModel): queue_name: str|None = Field(default=None, min_length=1, description="Name of the queue or partition to submit the job to", example="debug") account: str|None = Field(default=None, min_length=1, description="Account or project to charge for resource usage", example="proj123") reservation_id: str|None = Field(default=None, min_length=1, description="ID of a reservation to use for the job", example="resv-42") - custom_attributes: dict[str, str] = Field(default={}, description="Custom scheduler-specific attributes as key-value pairs", example={"constraint": "gpu"}) + custom_attributes: dict[str, str] = Field(default_factory=dict, description="Custom scheduler-specific attributes as key-value pairs", example={"constraint": "gpu"}) class VolumeMount(IRIBaseModel): @@ -54,7 +54,7 @@ class Container(IRIBaseModel): """ image: str = Field(min_length=1, description="The container image to use (e.g., 'docker.io/library/ubuntu:latest')", example="docker.io/library/ubuntu:latest") - volume_mounts: list[VolumeMount] = Field(default=[], description="List of volume mounts for the container") + volume_mounts: list[VolumeMount] = Field(default_factory=list, description="List of volume mounts for the container") class JobSpec(IRIBaseModel): @@ -68,11 +68,11 @@ class JobSpec(IRIBaseModel): description="Path to the executable to run. If container is specified, this will be used as the entrypoint to the container.", example="/usr/bin/python") container: Container|None = Field(default=None, description="Container specification for containerized execution") - arguments: list[str] = Field(default=[], description="Command-line arguments to pass to the executable or container", example=["-n", "100"]) + arguments: list[str] = Field(default_factory=list, description="Command-line arguments to pass to the executable or container", example=["-n", "100"]) directory: str|None = Field(default=None, min_length=1, description="Working directory for the job", example="/home/user/work") name: str|None = Field(default=None, min_length=1, description="Name of the job", example="my-job") inherit_environment: StrictBool = Field(default=True, description="Whether to inherit the environment variables from the submission environment", example=True) - environment: dict[str, str] = Field(default={}, + environment: dict[str, str] = Field(default_factory=dict, description="Environment variables to set for the job. If container is specified, these will be set inside the container.", example={"OMP_NUM_THREADS": "4"}) stdin_path: str|None = Field(default=None, min_length=1, description="Path to file to use as standard input", example="/home/user/input.txt") From 0d8d8e9cf144ca0ea18f5bb30d72bd8cb08d4c89 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Mon, 9 Mar 2026 11:44:17 -0700 Subject: [PATCH 109/133] make events a top level object --- app/demo_adapter.py | 7 ++++--- app/routers/status/facility_adapter.py | 4 ++-- app/routers/status/models.py | 4 +++- app/routers/status/status.py | 18 +++++++++--------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index b9c06a80..6e8419df 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -414,9 +414,9 @@ async def get_resource(self: "DemoAdapter", id_: str) -> status_models.Resource: async def get_events( self: "DemoAdapter", - incident_id: str, offset: int, limit: int, + incident_id: str | None = None, resource_id: str | None = None, name: str | None = None, description: str | None = None, @@ -427,7 +427,8 @@ async def get_events( modified_since: datetime.datetime | None = None, ) -> list[status_models.Event]: events = status_models.Event.find( - [e for e in self.events if e.incident_id == incident_id], + self.events, + incident_id=incident_id, resource_id=resource_id, name=name, description=description, @@ -439,7 +440,7 @@ async def get_events( ) return paginate_list(events, offset, limit) - async def get_event(self: "DemoAdapter", incident_id: str, id_: str) -> status_models.Event: + async def get_event(self: "DemoAdapter", id_: str) -> status_models.Event: return status_models.Event.find_by_id(self.events, id_) async def get_incidents( diff --git a/app/routers/status/facility_adapter.py b/app/routers/status/facility_adapter.py index 96c32003..65b87c4c 100644 --- a/app/routers/status/facility_adapter.py +++ b/app/routers/status/facility_adapter.py @@ -35,9 +35,9 @@ async def get_resource(self: "FacilityAdapter", id_: str) -> status_models.Resou @abstractmethod async def get_events( self: "FacilityAdapter", - incident_id: str, offset: int, limit: int, + incident_id: str | None = None, resource_id: str | None = None, name: str | None = None, description: str | None = None, @@ -50,7 +50,7 @@ async def get_events( pass @abstractmethod - async def get_event(self: "FacilityAdapter", incident_id: str, id_: str) -> status_models.Event: + async def get_event(self: "FacilityAdapter", id_: str) -> status_models.Event: pass @abstractmethod diff --git a/app/routers/status/models.py b/app/routers/status/models.py index e020f4df..cfa17e3a 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -98,9 +98,11 @@ def incident_uri(self) -> str | None: return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}" if self.incident_id else None @classmethod - def find(cls, items, name=None, description=None, modified_since=None, resource_id=None, status=None, from_=None, to=None, time_=None) -> list: + def find(cls, items, incident_id=None, name=None, description=None, modified_since=None, resource_id=None, status=None, from_=None, to=None, time_=None) -> list: items = super().find(items, name=name, description=description, modified_since=modified_since) + if incident_id: + items = [e for e in items if e.incident_id == incident_id] if resource_id: items = [e for e in items if e.resource_id == resource_id] if status: diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 46ff4b4a..018492f9 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -132,15 +132,15 @@ async def get_incident(request: Request, incident_id: str) -> models.Incident: @router.get( - "/incidents/{incident_id}/events", - summary="Get all events for an incident", - description="Get a list of all events in this incident. You can optionally filter the returned list by specifying attribtes.", + "/events", + summary="Get all events", + description="Get a list of all events. You can optionally filter the returned list by specifying attribtes.", responses=DEFAULT_RESPONSES, operation_id="getEventsByIncident", ) async def get_events( request: Request, - incident_id: str, + incident_id: str | None = Query(default=None, min_length=1), resource_id: str | None = Query(default=None, min_length=1), name: str | None = Query(default=None, min_length=1), description: str | None = Query(default=None, min_length=1), @@ -151,10 +151,10 @@ async def get_events( modified_since: StrictDateTime = Query(default=None), offset: int = Query(default=0, ge=0, le=1000), limit: int = Query(default=100, ge=0, le=1000), - _forbid=Depends(forbidExtraQueryParams("resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), + _forbid=Depends(forbidExtraQueryParams("incident_id", "resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: events = await router.adapter.get_events( - incident_id, offset=offset, limit=limit, resource_id=resource_id, name=name, description=description, status=status, from_=from_, to=to, time_=time_, modified_since=modified_since + incident_id=incident_id, offset=offset, limit=limit, resource_id=resource_id, name=name, description=description, status=status, from_=from_, to=to, time_=time_, modified_since=modified_since ) if not events: raise HTTPException(status_code=404, detail="No events found") @@ -162,14 +162,14 @@ async def get_events( @router.get( - "/incidents/{incident_id}/events/{event_id}", + "/events/{event_id}", summary="Get a specific event", description="Get a specific event for a given id", responses=DEFAULT_RESPONSES, operation_id="getEventByIncident", ) -async def get_event(request: Request, incident_id: str, event_id: str) -> models.Event: - item = await router.adapter.get_event(incident_id, event_id) +async def get_event(request: Request, event_id: str) -> models.Event: + item = await router.adapter.get_event(event_id) if not item: raise HTTPException(status_code=404, detail="Item not found") return item From 3d88afb2cb620224a159a9dcef8831cf4d79a8c8 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Mon, 9 Mar 2026 17:09:37 -0700 Subject: [PATCH 110/133] removed upper limit on offset param --- app/routers/compute/compute.py | 2 +- app/routers/facility/facility.py | 2 +- app/routers/status/status.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 6023ddd6..6a63c11f 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -163,7 +163,7 @@ async def get_job_status( async def get_job_statuses( resource_id: str, request: Request, - offset: int = Query(default=0, ge=0, le=1000), + offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=0, le=1000), filters: dict[str, object] | None = None, historical: StrictHTTPBool | None = Query(default=False, description="Whether to include historical jobs. Defaults to false"), diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 7abdb47a..19f6b282 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -28,7 +28,7 @@ async def list_sites( request: Request, modified_since: StrictDateTime = Query(default=None), name: str | None = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), + offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=0, le=1000), short_name: str | None = Query(default=None, min_length=1), _forbid=Depends(forbidExtraQueryParams("modified_since", "name", "offset", "limit", "short_name")), diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 018492f9..4b8759b1 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -28,7 +28,7 @@ async def get_resources( name: str = Query(default=None, min_length=1), description: str = Query(default=None, min_length=1), group: str = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), + offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=0, le=1000), modified_since: StrictDateTime = Query(default=None), resource_type: models.ResourceType = Query(default=None), @@ -76,7 +76,7 @@ async def get_incidents( to: StrictDateTime = Query(default=None), modified_since: StrictDateTime = Query(default=None), resource_id: str | None = Query(default=None, min_length=1), - offset: int = Query(default=0, ge=0, le=1000), + offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=0, le=1000), resolution: models.Resolution = Query(default=None), _forbid=Depends( @@ -149,7 +149,7 @@ async def get_events( time_: StrictDateTime = Query(alias="time", default=None), to: StrictDateTime = Query(default=None), modified_since: StrictDateTime = Query(default=None), - offset: int = Query(default=0, ge=0, le=1000), + offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=0, le=1000), _forbid=Depends(forbidExtraQueryParams("incident_id", "resource_id", "name", "description", "status", "from", "to", "time", "modified_since", "offset", "limit")), ) -> list[models.Event]: From 500195775fa2400b2084fe5919a9ead3dac4671f Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 10 Mar 2026 13:21:11 -0700 Subject: [PATCH 111/133] held job state --- app/routers/compute/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/routers/compute/models.py b/app/routers/compute/models.py index 87631e2f..cea26492 100644 --- a/app/routers/compute/models.py +++ b/app/routers/compute/models.py @@ -104,6 +104,10 @@ class JobState(str, Enum): This is the state of the job after being accepted by a backend for execution, but before the execution of the job begins. """ + HELD = "held" + """ + This is the state of a job that is queued but ineligible to run. + """ ACTIVE = "active" """This state represents an actively running job.""" COMPLETED = "completed" From 600f9850579a60ab18b626072abd0a0e36b78b22 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Fri, 13 Mar 2026 16:01:55 -0700 Subject: [PATCH 112/133] globus integration --- .gitignore | 3 +- Makefile | 11 ++++++ README.md | 12 +++++++ app/demo_adapter.py | 18 ++++++++-- app/routers/iri_router.py | 61 ++++++++++++++++++++++++++++++--- pyproject.toml | 4 ++- tools/globus.py | 58 ++++++++++++++++++++++++++++++++ tools/manage_globus.py | 71 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 230 insertions(+), 8 deletions(-) create mode 100644 tools/globus.py create mode 100644 tools/manage_globus.py diff --git a/.gitignore b/.gitignore index c8d4802f..8f87ec10 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ __pycache__ iri_api_python.egg-info docker-build.sh iri_sandbox -.env \ No newline at end of file +.env +local.env \ No newline at end of file diff --git a/Makefile b/Makefile index febce8ef..14195d59 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ deps: $(STAMP_DEPS) dev: deps @source $(BIN)/activate && \ + [ -f local.env ] && source local.env || true && \ IRI_API_ADAPTER_facility=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_status=app.demo_adapter.DemoAdapter \ IRI_API_ADAPTER_account=app.demo_adapter.DemoAdapter \ @@ -68,3 +69,13 @@ bandit: deps # Full validation bundle lint: clean format ruff pylint audit bandit + +globus: deps + @source local.env && $(BIN)/python ./tools/globus.py + +ARGS ?= + +# call it via: make manage-globus ARGS=scopes-show +manage-globus: deps + @source local.env && $(BIN)/python ./tools/manage_globus.py $(ARGS) + diff --git a/README.md b/README.md index a0d4e645..935730ef 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,18 @@ ENV IRI_API_PARAMS='{ \ }' ``` +## Globus auth integration + +You can optionally use globus for authorization. Steps to use globus: +- ask someone to add your globus account to the IRI Resource Server +- log into globus and make a secret for yourself for the IRI Resource Server +- if you want to create tokens during developent, also create a separate globus app +- `cp local-template.env local.env` and fill in the missing values +- to mint a token, run `make globus`, click the link and copy the code from the browser url bar back into the terminal +- you can also run `make manage-globus` but be sure to not accidentally delete the `iri-api` scope. (Maybe it's better if you don't run this app) +- now you can run `make` for the dev server and enjoy using your globus iri access tokens (in the demo adapter they will all resolve to the user `gtorok`) +- for your facility, implement the `get_current_user_globus` method (see iri_adapter.py). Here you can look at the linked globus identities and session info to determine what the local username is + ## Next steps - Learn more about [fastapi](https://fastapi.tiangolo.com/), including how to run it [in production](https://fastapi.tiangolo.com/advanced/behind-a-proxy/) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 6e8419df..2ec1f715 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -488,6 +488,22 @@ async def get_current_user( In a real deployment, this would decode the api_key jwt and return the current user's id. This method is not async. """ + if api_key != self.user.api_key: + raise HTTPException(status_code=401, detail="Invalid API key") + return "gtorok" + + async def get_current_user_globus( + self: "DemoAdapter", + api_key: str, + client_ip: str, + globus_linked_identities: list | None, + globus_session_info: dict | None, + ) -> str: + """ + Decode the api_key and return the authenticated user's id from information returned by introspecting a globus token. + This method is not called directly, rather authorized endpoints "depend" on it. + (https://fastapi.tiangolo.com/tutorial/dependencies/) + """ return "gtorok" async def get_user( @@ -500,8 +516,6 @@ async def get_user( raise HTTPException(status_code=403, detail="User not found") if api_key.startswith("Bearer "): api_key = api_key[len("Bearer ") :] - if api_key != self.user.api_key: - raise HTTPException(status_code=401, detail="Invalid API key") return self.user async def get_projects(self: "DemoAdapter", user: account_models.User) -> list[account_models.Project]: diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 3ea6c289..818822e6 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod -import traceback import os import logging import importlib +import globus_sdk from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from .account.models import User @@ -11,6 +11,11 @@ bearer_scheme = HTTPBearer() +GLOBUS_RS_ID = os.environ.get("GLOBUS_RS_ID") +GLOBUS_RS_SECRET = os.environ.get("GLOBUS_RS_SECRET") +GLOBUS_RS_SCOPE_SUFFIX = os.environ.get("GLOBUS_RS_SCOPE_SUFFIX") + + def get_client_ip(request: Request) -> str | None: forwarded_for = request.headers.get("X-Forwarded-For") if forwarded_for: @@ -73,6 +78,38 @@ def create_adapter(router_name, router_adapter): # assign it return AdapterClass() + + async def get_globus_info(self, api_key: str) -> tuple[list, dict]: + """Returns the linked identities and the session info objects""" + # Introspect the IRI API token using resource server credentials + globus_client = globus_sdk.ConfidentialAppAuthClient(GLOBUS_RS_ID, GLOBUS_RS_SECRET) + # grab identity_set_detail for linked identities and session_info to see how the user logged in + introspect = globus_client.oauth2_token_introspect(api_key, include="identity_set_detail,session_info") + logging.getLogger().info("IRI TOKEN INTROSPECTION:") + logging.getLogger().info(introspect) + if not introspect.get("active"): + raise Exception("Inactive token") + + # Check if token has the required IRI scope + token_scope = introspect.get("scope", "") + GLOBUS_SCOPE = f"https://auth.globus.org/scopes/{GLOBUS_RS_ID}/{GLOBUS_RS_SCOPE_SUFFIX}" + if GLOBUS_SCOPE not in token_scope: + raise Exception(f"Token missing required scope: {GLOBUS_SCOPE}") + + # Get linked identities from the introspection response + identity_set = introspect.get('identity_set_detail', []) + + # If no identity_set_detail, fall back to primary identity + if not identity_set: + email = introspect.get('email') + username = introspect.get('username') + if email or username: + identity_set = [{'email': email, 'username': username}] + + logging.getLogger().info(f"Found {len(identity_set)} linked identities") + return identity_set, introspect.get("session_info", {}) + + async def current_user( self, request: Request, @@ -82,10 +119,17 @@ async def current_user( user_id = None try: - user_id = await self.adapter.get_current_user(token, get_client_ip(request)) + ip_address = get_client_ip(request) + if GLOBUS_RS_ID and GLOBUS_RS_SECRET and GLOBUS_RS_SCOPE_SUFFIX: + try: + globus_linked_identities, globus_session_info = await self.get_globus_info(token) + user_id = await self.adapter.get_current_user_globus(token, ip_address, globus_linked_identities, globus_session_info) + except Exception as globus_exc: + logging.getLogger().exception("Globus error:", exc_info=globus_exc) + if not user_id: + user_id = await self.adapter.get_current_user(token, ip_address) except Exception as exc: - logging.getLogger().error(f"Error parsing IRI_API_PARAMS: {exc}") - traceback.print_exc() + logging.getLogger().exception(f"Error parsing IRI_API_PARAMS: ", exc_info=exc) raise HTTPException(status_code=401, detail="Invalid or malformed Authorization parameters") from exc if not user_id: raise HTTPException(status_code=403, detail="Unauthorized access") @@ -103,6 +147,15 @@ async def get_current_user(self: "AuthenticatedAdapter", api_key: str, client_ip """ pass + @abstractmethod + async def get_current_user_globus(self: "AuthenticatedAdapter", api_key: str, client_ip: str | None, globus_linked_identities: list | None, globus_session_info: dict | None) -> str: + """ + Decode the api_key and return the authenticated user's id from information returned by introspecting a globus token. + This method is not called directly, rather authorized endpoints "depend" on it. + (https://fastapi.tiangolo.com/tutorial/dependencies/) + """ + pass + @abstractmethod async def get_user(self: "AuthenticatedAdapter", user_id: str, api_key: str, client_ip: str | None) -> User: """ diff --git a/pyproject.toml b/pyproject.toml index a5a3ac07..4e34d7e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,9 @@ dependencies = [ "opentelemetry-api>=1.39.1,<1.40.0", "opentelemetry-sdk>=1.39.1,<1.40.0", "opentelemetry-instrumentation-fastapi>=0.60b1,<0.61b0", - "opentelemetry-exporter-otlp>=1.39.1,<1.40.0" + "opentelemetry-exporter-otlp>=1.39.1,<1.40.0", + "globus-sdk>=4.3.1", + "typer>=0.24.1", ] [tool.ruff] line-length = 200 diff --git a/tools/globus.py b/tools/globus.py new file mode 100644 index 00000000..328b9720 --- /dev/null +++ b/tools/globus.py @@ -0,0 +1,58 @@ +import globus_sdk +import datetime +import time +import os + +GLOBUS_APP_ID = os.environ.get("GLOBUS_APP_ID") +GLOBUS_APP_SECRET = os.environ.get("GLOBUS_APP_SECRET") +GLOBUS_RS_ID = os.environ.get("GLOBUS_RS_ID") +GLOBUS_RS_SCOPE_SUFFIX = os.environ.get("GLOBUS_RS_SCOPE_SUFFIX") +GLOBUS_SCOPE = f"https://auth.globus.org/scopes/{GLOBUS_RS_ID}/{GLOBUS_RS_SCOPE_SUFFIX}" + +# Create a confidential client +client = globus_sdk.ConfidentialAppAuthClient(GLOBUS_APP_ID, GLOBUS_APP_SECRET) + +# Start the OAuth flow +client.oauth2_start_flow( + redirect_uri="http://localhost:5000/callback", # or your registered redirect URI + requested_scopes=["openid", "profile", "email", GLOBUS_SCOPE] +) + +# Get the authorization URL +authorize_url = client.oauth2_get_authorize_url() +print(f"Visit this URL in your browser:\n{authorize_url}\n") + +# After visiting the URL and authorizing, you'll be redirected to a URL with a code parameter +auth_code = input("Paste the 'code' parameter from the redirect URL: ") + +# Exchange the code for tokens +token_response = client.oauth2_exchange_code_for_tokens(auth_code) + +# Print all resource servers and their tokens +print("\n=== ALL TOKENS ===") +for resource_server, token_data in token_response.by_resource_server.items(): + print(f"\nResource Server: {resource_server}") + print(f" Access Token: {token_data['access_token'][:50]}...") + print(f" Scope: {token_data.get('scope', 'N/A')}") + print(f" Expires at: {datetime.datetime.fromtimestamp(token_data['expires_at_seconds'])}") + +# Extract the IRI API token to send to the API +if GLOBUS_RS_ID in token_response.by_resource_server: + iri_token_data = token_response.by_resource_server[GLOBUS_RS_ID] + iri_token = iri_token_data['access_token'] + expires_at = iri_token_data['expires_at_seconds'] + + # Convert to human-readable time + expiration_time = datetime.datetime.fromtimestamp(expires_at) + print(f"\nIRI API token expires at: {expiration_time}") + + # Calculate how long until expiration + seconds_until_expiration = expires_at - time.time() + hours_until_expiration = seconds_until_expiration / 3600 + print(f"Token expires in {hours_until_expiration:.2f} hours") + + print(f"\n=== USE THIS IRI API TOKEN ===") + print(f"IMPORTANT: You must log in with your NERSC-linked Globus identity") + print(f"\n{iri_token}") +else: + print(f"\nERROR: No IRI API token found. Make sure you requested the correct scope.") diff --git a/tools/manage_globus.py b/tools/manage_globus.py new file mode 100644 index 00000000..818ef572 --- /dev/null +++ b/tools/manage_globus.py @@ -0,0 +1,71 @@ +""" +This script manages the resource server for the data movement API. It allows you to view, create, and delete scopes +for the resource server. + +It's only useful to admins of the data movement API. +""" + +import typer +import globus_sdk +import pprint +import os + +GLOBUS_RS_ID = os.environ.get("GLOBUS_RS_ID") +GLOBUS_RS_SECRET = os.environ.get("GLOBUS_RS_SECRET") +GLOBUS_RS_SCOPE_SUFFIX = os.environ.get("GLOBUS_RS_SCOPE_SUFFIX") +GLOBUS_SCOPE = f"https://auth.globus.org/scopes/{GLOBUS_RS_ID}/{GLOBUS_RS_SCOPE_SUFFIX}" + +global client +app = typer.Typer(no_args_is_help=True) + + +@app.command() +def client_show(): + print(client.get_client(client_id=GLOBUS_RS_ID)) + + +@app.command() +def scopes_show(): + scope_ids = client.get_client(client_id=GLOBUS_RS_ID)["client"]["scopes"] + scopes = [client.get_scope(scope_id).data for scope_id in scope_ids] + pprint.pprint(scopes) + + +@app.command() +def scope_show(scope: str = None, scope_string: str = None): + if not scope and not scope_string: + scope_string = GLOBUS_SCOPE + if scope_string: + print(client.get_scopes(scope_strings=[scope_string])) + else: + print(client.get_scope(scope_id=scope)) + + +@app.command() +def scope_create_iri(): + if typer.confirm("Create a new IRI API scope?"): + print( + client.create_scope( + GLOBUS_RS_ID, + "IRI API", + "Access to IRI API services", + "iri_api", + advertised=True, + allows_refresh_token=True, + ) + ) + + +@app.command() +def scope_delete(scope: str): + print(client.delete_scope(scope_id=scope)) + + +if __name__ == "__main__": + if not GLOBUS_RS_SECRET: + print("Error: No CLIENT_SECRET detected on env. Please set 'export CLIENT_SECRET=your_secret'") + else: + globus_app = globus_sdk.ClientApp("manage-ap", client_id=GLOBUS_RS_ID, client_secret=GLOBUS_RS_SECRET) + client = globus_sdk.AuthClient(app=globus_app) + client.add_app_scope(globus_sdk.AuthClient.scopes.manage_projects) + app() From 96a7eeea6d3fa46703e78d03a1508e1acd52555f Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Fri, 13 Mar 2026 16:12:11 -0700 Subject: [PATCH 113/133] globus integration --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 935730ef..b47226df 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,9 @@ You can optionally use globus for authorization. Steps to use globus: - to mint a token, run `make globus`, click the link and copy the code from the browser url bar back into the terminal - you can also run `make manage-globus` but be sure to not accidentally delete the `iri-api` scope. (Maybe it's better if you don't run this app) - now you can run `make` for the dev server and enjoy using your globus iri access tokens (in the demo adapter they will all resolve to the user `gtorok`) -- for your facility, implement the `get_current_user_globus` method (see iri_adapter.py). Here you can look at the linked globus identities and session info to determine what the local username is +- for your facility: + - implement the `get_current_user_globus` method (see iri_adapter.py). Here you can look at the linked globus identities and session info to determine what the local username is + - make sure the values in `local.env` are available in the deployed app ## Next steps From 15b00b830e76e3866567ea2e9a67a01711093d42 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Mon, 16 Mar 2026 14:16:53 -0700 Subject: [PATCH 114/133] exp and nbf checks --- app/routers/iri_router.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 818822e6..51602dc0 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -2,6 +2,7 @@ import os import logging import importlib +import time import globus_sdk from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials @@ -90,6 +91,16 @@ async def get_globus_info(self, api_key: str) -> tuple[list, dict]: if not introspect.get("active"): raise Exception("Inactive token") + # Check exp (expiration time) claim + exp = introspect.get("exp") + if exp and time.time() >= exp: + raise Exception("Token has expired") + + # Check nbf (not before) claim + nbf = introspect.get("nbf") + if nbf and time.time() < nbf: + raise Exception("Token not yet valid") + # Check if token has the required IRI scope token_scope = introspect.get("scope", "") GLOBUS_SCOPE = f"https://auth.globus.org/scopes/{GLOBUS_RS_ID}/{GLOBUS_RS_SCOPE_SUFFIX}" From 626f05e38d40fd5feb31786c4daa7b644a13ee46 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 17 Mar 2026 09:16:08 -0700 Subject: [PATCH 115/133] added missing file --- local-template.env | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 local-template.env diff --git a/local-template.env b/local-template.env new file mode 100644 index 00000000..68ad9b0c --- /dev/null +++ b/local-template.env @@ -0,0 +1,9 @@ +# globus app credentials +export GLOBUS_APP_ID= +export GLOBUS_APP_SECRET= + +# the resource server's credentials +export GLOBUS_RS_ID=ed3e577d-f7f3-4639-b96e-ff5a8445d699 +export GLOBUS_RS_SECRET= + +export GLOBUS_RS_SCOPE_SUFFIX=iri_api From 0c8dd797eed9cd0ff223a4a3324ef35e30432711 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 17 Mar 2026 10:18:08 -0700 Subject: [PATCH 116/133] logging and token splitting --- app/main.py | 5 +++++ app/routers/iri_router.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index a024798d..62147590 100644 --- a/app/main.py +++ b/app/main.py @@ -21,6 +21,11 @@ from . import config +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s" +) + # ------------------------------------------------------------------ # OpenTelemetry Tracing Configuration # ------------------------------------------------------------------ diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 51602dc0..24bdfb6b 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -102,7 +102,7 @@ async def get_globus_info(self, api_key: str) -> tuple[list, dict]: raise Exception("Token not yet valid") # Check if token has the required IRI scope - token_scope = introspect.get("scope", "") + token_scope = introspect.get("scope", "").split() GLOBUS_SCOPE = f"https://auth.globus.org/scopes/{GLOBUS_RS_ID}/{GLOBUS_RS_SCOPE_SUFFIX}" if GLOBUS_SCOPE not in token_scope: raise Exception(f"Token missing required scope: {GLOBUS_SCOPE}") From a52d79b9394c40bf86aa7d49ea16746d05d3b625 Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 17 Mar 2026 13:19:33 -0700 Subject: [PATCH 117/133] remove primary identity hack --- app/routers/iri_router.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 24bdfb6b..2fd1fbb1 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -109,15 +109,8 @@ async def get_globus_info(self, api_key: str) -> tuple[list, dict]: # Get linked identities from the introspection response identity_set = introspect.get('identity_set_detail', []) - - # If no identity_set_detail, fall back to primary identity - if not identity_set: - email = introspect.get('email') - username = introspect.get('username') - if email or username: - identity_set = [{'email': email, 'username': username}] - logging.getLogger().info(f"Found {len(identity_set)} linked identities") + return identity_set, introspect.get("session_info", {}) From 2511b2ee70c803bb916cd873746d6791a6dfa15f Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Thu, 19 Mar 2026 10:26:06 -0700 Subject: [PATCH 118/133] pass entire introspection result to facility + exception if session_info is empty --- app/demo_adapter.py | 3 +-- app/routers/iri_router.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 2ec1f715..264fc467 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -496,8 +496,7 @@ async def get_current_user_globus( self: "DemoAdapter", api_key: str, client_ip: str, - globus_linked_identities: list | None, - globus_session_info: dict | None, + globus_introspect: dict | None, ) -> str: """ Decode the api_key and return the authenticated user's id from information returned by introspecting a globus token. diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 2fd1fbb1..48ca4aa6 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -80,7 +80,7 @@ def create_adapter(router_name, router_adapter): return AdapterClass() - async def get_globus_info(self, api_key: str) -> tuple[list, dict]: + async def get_globus_info(self, api_key: str) -> dict: """Returns the linked identities and the session info objects""" # Introspect the IRI API token using resource server credentials globus_client = globus_sdk.ConfidentialAppAuthClient(GLOBUS_RS_ID, GLOBUS_RS_SECRET) @@ -107,11 +107,11 @@ async def get_globus_info(self, api_key: str) -> tuple[list, dict]: if GLOBUS_SCOPE not in token_scope: raise Exception(f"Token missing required scope: {GLOBUS_SCOPE}") - # Get linked identities from the introspection response - identity_set = introspect.get('identity_set_detail', []) - logging.getLogger().info(f"Found {len(identity_set)} linked identities") + session_info = introspect.get("session_info", {}) + if not session_info or not session_info.get("authentications", {}): + raise Exception(f"Empty session_info.authentications block") - return identity_set, introspect.get("session_info", {}) + return introspect async def current_user( @@ -126,8 +126,8 @@ async def current_user( ip_address = get_client_ip(request) if GLOBUS_RS_ID and GLOBUS_RS_SECRET and GLOBUS_RS_SCOPE_SUFFIX: try: - globus_linked_identities, globus_session_info = await self.get_globus_info(token) - user_id = await self.adapter.get_current_user_globus(token, ip_address, globus_linked_identities, globus_session_info) + globus_introspect = await self.get_globus_info(token) + user_id = await self.adapter.get_current_user_globus(token, ip_address, globus_introspect) except Exception as globus_exc: logging.getLogger().exception("Globus error:", exc_info=globus_exc) if not user_id: @@ -152,7 +152,7 @@ async def get_current_user(self: "AuthenticatedAdapter", api_key: str, client_ip pass @abstractmethod - async def get_current_user_globus(self: "AuthenticatedAdapter", api_key: str, client_ip: str | None, globus_linked_identities: list | None, globus_session_info: dict | None) -> str: + async def get_current_user_globus(self: "AuthenticatedAdapter", api_key: str, client_ip: str | None, globus_introspect: dict | None) -> str: """ Decode the api_key and return the authenticated user's id from information returned by introspecting a globus token. This method is not called directly, rather authorized endpoints "depend" on it. From 562b33d40fe0834d05c2ad795d71deba87741a56 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Fri, 13 Mar 2026 07:13:21 -0500 Subject: [PATCH 119/133] Add x-iri meta extra --- app/routers/account/account.py | 9 +++++++ app/routers/compute/compute.py | 6 +++++ app/routers/facility/facility.py | 17 ++++++++++--- app/routers/filesystem/filesystem.py | 19 ++++++++++++++ app/routers/iri_meta.py | 38 ++++++++++++++++++++++++++++ app/routers/status/status.py | 7 +++++ app/routers/task/task.py | 9 ++++++- 7 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 app/routers/iri_meta.py diff --git a/app/routers/account/account.py b/app/routers/account/account.py index b2fc4101..c584839f 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -5,6 +5,7 @@ from ...types.scalars import StrictDateTime from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..iri_meta import iri_meta_dict from . import facility_adapter, models router = iri_router.IriRouter( @@ -21,6 +22,7 @@ responses=DEFAULT_RESPONSES, operation_id="getCapabilities", response_model_exclude_none=True, + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_capabilities( request: Request, @@ -39,6 +41,7 @@ async def get_capabilities( description="Get a single capability at this facility.", responses=DEFAULT_RESPONSES, operation_id="getCapability", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_capability( capability_id: str, @@ -60,6 +63,7 @@ async def get_capability( description="Get a list of projects for the currently authenticated user at this facility.", responses=DEFAULT_RESPONSES, operation_id="getProjects", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_projects( request: Request, @@ -78,6 +82,7 @@ async def get_projects( description="Get a single project at this facility.", responses=DEFAULT_RESPONSES, operation_id="getProject", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_project( project_id: str, @@ -101,6 +106,7 @@ async def get_project( description="Get a list of allocations for the currently authenticated user's projects at this facility.", responses=DEFAULT_RESPONSES, operation_id="getProjectAllocationsByProject", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_project_allocations( project_id: str, @@ -124,6 +130,7 @@ async def get_project_allocations( description="Get a single project allocation at this facility for this user.", responses=DEFAULT_RESPONSES, operation_id="getProjectAllocationByProject", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_project_allocation( project_id: str, @@ -152,6 +159,7 @@ async def get_project_allocation( description="Get a list of user allocations for the currently authenticated user's projects at this facility.", responses=DEFAULT_RESPONSES, operation_id="getUserAllocationsByProjectAllocation", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_user_allocations( project_id: str, @@ -180,6 +188,7 @@ async def get_user_allocations( description="Get a user allocation for the currently authenticated user's projects at this facility.", responses=DEFAULT_RESPONSES, operation_id="getUserAllocationByProjectAllocation", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_user_allocation( project_id: str, diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 6a63c11f..ac67853d 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -6,6 +6,7 @@ from ...types.scalars import StrictHTTPBool from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..iri_meta import iri_meta_dict from ..status.status import router as status_router from . import facility_adapter, models @@ -23,6 +24,7 @@ response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="launchJob", + openapi_extra=iri_meta_dict("incubator", "required") ) async def submit_job( resource_id: str, @@ -94,6 +96,7 @@ async def submit_job( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="updateJob", + openapi_extra=iri_meta_dict("incubator", "required") ) async def update_job( resource_id: str, @@ -129,6 +132,7 @@ async def update_job( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getJob", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_job_status( resource_id: str, @@ -159,6 +163,7 @@ async def get_job_status( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getJobs", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_job_statuses( resource_id: str, @@ -192,6 +197,7 @@ async def get_job_statuses( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="cancelJob", + openapi_extra=iri_meta_dict("incubator", "required") ) async def cancel_job( resource_id: str, diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 19f6b282..626d1bef 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -4,13 +4,22 @@ from ...types.scalars import StrictDateTime from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..iri_meta import iri_meta_dict from . import facility_adapter, models router = iri_router.IriRouter(facility_adapter.FacilityAdapter, prefix="/facility", tags=["facility"]) -@router.get("", responses=DEFAULT_RESPONSES, operation_id="getFacility", response_model_exclude_none=True,) -@router.get("/", responses=DEFAULT_RESPONSES, operation_id="getFacilityWithSlash", response_model_exclude_none=True, include_in_schema=False,) +@router.get("", + responses=DEFAULT_RESPONSES, + operation_id="getFacility", + response_model_exclude_none=True, + openapi_extra=iri_meta_dict("graduated", "required")) +@router.get("/", + responses=DEFAULT_RESPONSES, + operation_id="getFacilityWithSlash", + response_model_exclude_none=True, + include_in_schema=False) async def get_facility( request: Request, modified_since: StrictDateTime = Query(default=None), @@ -23,7 +32,7 @@ async def get_facility( return facility -@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites", response_model_exclude_none=True,) +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites", response_model_exclude_none=True, openapi_extra=iri_meta_dict("graduated", "required")) async def list_sites( request: Request, modified_since: StrictDateTime = Query(default=None), @@ -40,7 +49,7 @@ async def list_sites( return sites -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite", response_model_exclude_none=True,) +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite", response_model_exclude_none=True, openapi_extra=iri_meta_dict("graduated", "required")) async def get_site( request: Request, site_id: str, diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index b53521e8..635f438d 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -9,6 +9,7 @@ from fastapi import Depends, HTTPException, status, Query, Request, File, UploadFile from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..iri_meta import iri_meta_dict from ..status.status import router as status_router, models as status_models from ..account.account import models as account_models from ..task import facility_adapter as task_facility_adapter, models as task_models @@ -47,6 +48,7 @@ async def _user_resource( response_description="File permissions changed successfully", responses=DEFAULT_RESPONSES, operation_id="chmod", + openapi_extra=iri_meta_dict("incubator", "required") ) async def put_chmod( resource_id: str, @@ -76,6 +78,7 @@ async def put_chmod( response_description="File ownership changed successfully", responses=DEFAULT_RESPONSES, operation_id="chown", + openapi_extra=iri_meta_dict("incubator", "required") ) async def put_chown( resource_id: str, @@ -105,6 +108,7 @@ async def put_chown( response_description="Type returned successfully", responses=DEFAULT_RESPONSES, operation_id="file", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_file( resource_id: str, @@ -134,6 +138,7 @@ async def get_file( response_description="Stat returned successfully", responses=DEFAULT_RESPONSES, operation_id="stat", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_stat( resource_id: str, @@ -165,6 +170,7 @@ async def get_stat( response_description="Directory created successfully", responses=DEFAULT_RESPONSES, operation_id="mkdir", + openapi_extra=iri_meta_dict("incubator", "required") ) async def post_mkdir( resource_id: str, @@ -194,6 +200,7 @@ async def post_mkdir( response_description="Symlink created successfully", responses=DEFAULT_RESPONSES, operation_id="symlink", + openapi_extra=iri_meta_dict("incubator", "required") ) async def post_symlink( resource_id: str, @@ -224,6 +231,7 @@ async def post_symlink( include_in_schema=router.task_adapter is not None, responses=DEFAULT_RESPONSES, operation_id="ls", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_ls_async( resource_id: str, @@ -261,6 +269,7 @@ async def get_ls_async( response_description="Head operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="head", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_head( resource_id: str, @@ -321,6 +330,7 @@ async def get_head( response_description="View operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="view", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_view( resource_id: str, @@ -355,6 +365,7 @@ async def get_view( response_description="`tail` operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="tail", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_tail( resource_id: str, @@ -408,6 +419,7 @@ async def get_tail( response_description="Checksum returned successfully", responses=DEFAULT_RESPONSES, operation_id="checksum", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_checksum( resource_id: str, @@ -435,6 +447,7 @@ async def get_checksum( response_description="File or directory deleted successfully", responses=DEFAULT_RESPONSES, operation_id="rm", + openapi_extra=iri_meta_dict("incubator", "required") ) async def delete_rm( resource_id: str, @@ -464,6 +477,7 @@ async def delete_rm( response_description="File and/or directories compressed successfully", responses=DEFAULT_RESPONSES, operation_id="compress", + openapi_extra=iri_meta_dict("incubator", "required") ) async def post_compress( resource_id: str, @@ -493,6 +507,7 @@ async def post_compress( response_description="File extracted successfully", responses=DEFAULT_RESPONSES, operation_id="extract", + openapi_extra=iri_meta_dict("incubator", "required") ) async def post_extract( resource_id: str, @@ -522,6 +537,7 @@ async def post_extract( response_description="Move file or directory operation created successfully", responses=DEFAULT_RESPONSES, operation_id="mv", + openapi_extra=iri_meta_dict("incubator", "required") ) async def move_mv( resource_id: str, @@ -551,6 +567,7 @@ async def move_mv( response_description="Copy file or directory operation created successfully", responses=DEFAULT_RESPONSES, operation_id="cp", + openapi_extra=iri_meta_dict("incubator", "required") ) async def post_cp( resource_id: str, @@ -580,6 +597,7 @@ async def post_cp( response_description="File downloaded successfully", responses=DEFAULT_RESPONSES, operation_id="download", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_download( resource_id: str, @@ -609,6 +627,7 @@ async def get_download( response_description="File uploaded successfully", responses=DEFAULT_RESPONSES, operation_id="upload", + openapi_extra=iri_meta_dict("incubator", "required") ) async def post_upload( resource_id: str, diff --git a/app/routers/iri_meta.py b/app/routers/iri_meta.py new file mode 100644 index 00000000..db3bbdfb --- /dev/null +++ b/app/routers/iri_meta.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +""" +Utility for generating the IRI OpenAPI extension metadata. +It generates: +{ + "x-iri": { + "maturity": "graduated", + "implementation": { + "level": "required", + "required_if_capability": "dpu" + } + } +} +""" + + +def iri_meta_dict( + maturity: str | None = None, + implementation_level: str | None = None, + required_if: str | None = None, +) -> dict: + """Generate the IRI OpenAPI extension metadata.""" + + out_obj = {} + + if maturity is not None: + out_obj["maturity"] = maturity + + if implementation_level is not None: + out_obj.setdefault("implementation", {})["level"] = implementation_level + + if required_if is not None: + out_obj.setdefault("implementation", {})["required_if_capability"] = required_if + + if not out_obj: + return {} + + return {"x-iri": out_obj} \ No newline at end of file diff --git a/app/routers/status/status.py b/app/routers/status/status.py index 4b8759b1..c8b1872c 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -6,6 +6,7 @@ from ...types.scalars import AllocationUnit, StrictDateTime from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..iri_meta import iri_meta_dict from . import facility_adapter, models router = iri_router.IriRouter( @@ -22,6 +23,7 @@ responses=DEFAULT_RESPONSES, operation_id="getResources", response_model_exclude_none=True, + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_resources( request: Request, @@ -47,6 +49,7 @@ async def get_resources( description="Get a specific resource for a given id", responses=DEFAULT_RESPONSES, operation_id="getResource", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_resource( request: Request, @@ -64,6 +67,7 @@ async def get_resource( description="Get a list of all incidents. Each incident will be returned without its events. You can optionally filter the returned list by specifying attributes.", responses=DEFAULT_RESPONSES, operation_id="getIncidents", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_incidents( request: Request, @@ -123,6 +127,7 @@ async def get_incidents( description="Get a specific incident for a given id. The incident's events will also be included. You can optionally filter the returned list by specifying attributes.", responses=DEFAULT_RESPONSES, operation_id="getIncident", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_incident(request: Request, incident_id: str) -> models.Incident: item = await router.adapter.get_incident(incident_id) @@ -137,6 +142,7 @@ async def get_incident(request: Request, incident_id: str) -> models.Incident: description="Get a list of all events. You can optionally filter the returned list by specifying attribtes.", responses=DEFAULT_RESPONSES, operation_id="getEventsByIncident", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_events( request: Request, @@ -167,6 +173,7 @@ async def get_events( description="Get a specific event for a given id", responses=DEFAULT_RESPONSES, operation_id="getEventByIncident", + openapi_extra=iri_meta_dict("graduated", "required") ) async def get_event(request: Request, event_id: str) -> models.Event: item = await router.adapter.get_event(event_id) diff --git a/app/routers/task/task.py b/app/routers/task/task.py index 74cfa6d4..e662c447 100644 --- a/app/routers/task/task.py +++ b/app/routers/task/task.py @@ -1,6 +1,7 @@ from fastapi import Request, HTTPException, Depends from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES +from ..iri_meta import iri_meta_dict from . import models, facility_adapter router = iri_router.IriRouter( @@ -16,6 +17,7 @@ response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getTask", + openapi_extra=iri_meta_dict("incubator", "required") ) async def get_task( request: Request, @@ -31,7 +33,11 @@ async def get_task( return task -@router.get("", dependencies=[Depends(router.current_user)], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getTasks") +@router.get("", + dependencies=[Depends(router.current_user)], + response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, + operation_id="getTasks", + openapi_extra=iri_meta_dict("incubator", "required")) @router.get("/", responses=DEFAULT_RESPONSES, operation_id="getTasksWithSlash", include_in_schema=False) async def get_tasks( @@ -48,6 +54,7 @@ async def get_tasks( dependencies=[Depends(router.current_user)], responses=DEFAULT_RESPONSES, operation_id="deleteTask", + openapi_extra=iri_meta_dict("incubator", "required") ) async def delete_task( request: Request, From eb55c606fe5ef828c52578b19464c3c36f18b694 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 18 Mar 2026 10:00:09 -0500 Subject: [PATCH 120/133] Use new labels same as DoE reporting --- app/routers/account/account.py | 16 ++++++------- app/routers/compute/compute.py | 10 ++++---- app/routers/facility/facility.py | 6 ++--- app/routers/filesystem/filesystem.py | 36 ++++++++++++++-------------- app/routers/status/status.py | 12 +++++----- app/routers/task/task.py | 6 ++--- 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index c584839f..e7b0af16 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -22,7 +22,7 @@ responses=DEFAULT_RESPONSES, operation_id="getCapabilities", response_model_exclude_none=True, - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_capabilities( request: Request, @@ -41,7 +41,7 @@ async def get_capabilities( description="Get a single capability at this facility.", responses=DEFAULT_RESPONSES, operation_id="getCapability", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_capability( capability_id: str, @@ -63,7 +63,7 @@ async def get_capability( description="Get a list of projects for the currently authenticated user at this facility.", responses=DEFAULT_RESPONSES, operation_id="getProjects", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_projects( request: Request, @@ -82,7 +82,7 @@ async def get_projects( description="Get a single project at this facility.", responses=DEFAULT_RESPONSES, operation_id="getProject", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_project( project_id: str, @@ -106,7 +106,7 @@ async def get_project( description="Get a list of allocations for the currently authenticated user's projects at this facility.", responses=DEFAULT_RESPONSES, operation_id="getProjectAllocationsByProject", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_project_allocations( project_id: str, @@ -130,7 +130,7 @@ async def get_project_allocations( description="Get a single project allocation at this facility for this user.", responses=DEFAULT_RESPONSES, operation_id="getProjectAllocationByProject", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_project_allocation( project_id: str, @@ -159,7 +159,7 @@ async def get_project_allocation( description="Get a list of user allocations for the currently authenticated user's projects at this facility.", responses=DEFAULT_RESPONSES, operation_id="getUserAllocationsByProjectAllocation", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_user_allocations( project_id: str, @@ -188,7 +188,7 @@ async def get_user_allocations( description="Get a user allocation for the currently authenticated user's projects at this facility.", responses=DEFAULT_RESPONSES, operation_id="getUserAllocationByProjectAllocation", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_user_allocation( project_id: str, diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index ac67853d..a186c92d 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -24,7 +24,7 @@ response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="launchJob", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def submit_job( resource_id: str, @@ -96,7 +96,7 @@ async def submit_job( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="updateJob", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def update_job( resource_id: str, @@ -132,7 +132,7 @@ async def update_job( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getJob", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_job_status( resource_id: str, @@ -163,7 +163,7 @@ async def get_job_status( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getJobs", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_job_statuses( resource_id: str, @@ -197,7 +197,7 @@ async def get_job_statuses( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="cancelJob", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def cancel_job( resource_id: str, diff --git a/app/routers/facility/facility.py b/app/routers/facility/facility.py index 626d1bef..f86cd9df 100644 --- a/app/routers/facility/facility.py +++ b/app/routers/facility/facility.py @@ -14,7 +14,7 @@ responses=DEFAULT_RESPONSES, operation_id="getFacility", response_model_exclude_none=True, - openapi_extra=iri_meta_dict("graduated", "required")) + openapi_extra=iri_meta_dict("production", "required")) @router.get("/", responses=DEFAULT_RESPONSES, operation_id="getFacilityWithSlash", @@ -32,7 +32,7 @@ async def get_facility( return facility -@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites", response_model_exclude_none=True, openapi_extra=iri_meta_dict("graduated", "required")) +@router.get("/sites", responses=DEFAULT_RESPONSES, operation_id="getSites", response_model_exclude_none=True, openapi_extra=iri_meta_dict("production", "required")) async def list_sites( request: Request, modified_since: StrictDateTime = Query(default=None), @@ -49,7 +49,7 @@ async def list_sites( return sites -@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite", response_model_exclude_none=True, openapi_extra=iri_meta_dict("graduated", "required")) +@router.get("/sites/{site_id}", responses=DEFAULT_RESPONSES, operation_id="getSite", response_model_exclude_none=True, openapi_extra=iri_meta_dict("production", "required")) async def get_site( request: Request, site_id: str, diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index 635f438d..0ba16958 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -48,7 +48,7 @@ async def _user_resource( response_description="File permissions changed successfully", responses=DEFAULT_RESPONSES, operation_id="chmod", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def put_chmod( resource_id: str, @@ -78,7 +78,7 @@ async def put_chmod( response_description="File ownership changed successfully", responses=DEFAULT_RESPONSES, operation_id="chown", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def put_chown( resource_id: str, @@ -108,7 +108,7 @@ async def put_chown( response_description="Type returned successfully", responses=DEFAULT_RESPONSES, operation_id="file", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_file( resource_id: str, @@ -138,7 +138,7 @@ async def get_file( response_description="Stat returned successfully", responses=DEFAULT_RESPONSES, operation_id="stat", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_stat( resource_id: str, @@ -170,7 +170,7 @@ async def get_stat( response_description="Directory created successfully", responses=DEFAULT_RESPONSES, operation_id="mkdir", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def post_mkdir( resource_id: str, @@ -200,7 +200,7 @@ async def post_mkdir( response_description="Symlink created successfully", responses=DEFAULT_RESPONSES, operation_id="symlink", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def post_symlink( resource_id: str, @@ -231,7 +231,7 @@ async def post_symlink( include_in_schema=router.task_adapter is not None, responses=DEFAULT_RESPONSES, operation_id="ls", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_ls_async( resource_id: str, @@ -269,7 +269,7 @@ async def get_ls_async( response_description="Head operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="head", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_head( resource_id: str, @@ -330,7 +330,7 @@ async def get_head( response_description="View operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="view", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_view( resource_id: str, @@ -365,7 +365,7 @@ async def get_view( response_description="`tail` operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="tail", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_tail( resource_id: str, @@ -419,7 +419,7 @@ async def get_tail( response_description="Checksum returned successfully", responses=DEFAULT_RESPONSES, operation_id="checksum", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_checksum( resource_id: str, @@ -447,7 +447,7 @@ async def get_checksum( response_description="File or directory deleted successfully", responses=DEFAULT_RESPONSES, operation_id="rm", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def delete_rm( resource_id: str, @@ -477,7 +477,7 @@ async def delete_rm( response_description="File and/or directories compressed successfully", responses=DEFAULT_RESPONSES, operation_id="compress", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def post_compress( resource_id: str, @@ -507,7 +507,7 @@ async def post_compress( response_description="File extracted successfully", responses=DEFAULT_RESPONSES, operation_id="extract", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def post_extract( resource_id: str, @@ -537,7 +537,7 @@ async def post_extract( response_description="Move file or directory operation created successfully", responses=DEFAULT_RESPONSES, operation_id="mv", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def move_mv( resource_id: str, @@ -567,7 +567,7 @@ async def move_mv( response_description="Copy file or directory operation created successfully", responses=DEFAULT_RESPONSES, operation_id="cp", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def post_cp( resource_id: str, @@ -597,7 +597,7 @@ async def post_cp( response_description="File downloaded successfully", responses=DEFAULT_RESPONSES, operation_id="download", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_download( resource_id: str, @@ -627,7 +627,7 @@ async def get_download( response_description="File uploaded successfully", responses=DEFAULT_RESPONSES, operation_id="upload", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def post_upload( resource_id: str, diff --git a/app/routers/status/status.py b/app/routers/status/status.py index c8b1872c..ce5958e4 100644 --- a/app/routers/status/status.py +++ b/app/routers/status/status.py @@ -23,7 +23,7 @@ responses=DEFAULT_RESPONSES, operation_id="getResources", response_model_exclude_none=True, - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_resources( request: Request, @@ -49,7 +49,7 @@ async def get_resources( description="Get a specific resource for a given id", responses=DEFAULT_RESPONSES, operation_id="getResource", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_resource( request: Request, @@ -67,7 +67,7 @@ async def get_resource( description="Get a list of all incidents. Each incident will be returned without its events. You can optionally filter the returned list by specifying attributes.", responses=DEFAULT_RESPONSES, operation_id="getIncidents", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_incidents( request: Request, @@ -127,7 +127,7 @@ async def get_incidents( description="Get a specific incident for a given id. The incident's events will also be included. You can optionally filter the returned list by specifying attributes.", responses=DEFAULT_RESPONSES, operation_id="getIncident", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_incident(request: Request, incident_id: str) -> models.Incident: item = await router.adapter.get_incident(incident_id) @@ -142,7 +142,7 @@ async def get_incident(request: Request, incident_id: str) -> models.Incident: description="Get a list of all events. You can optionally filter the returned list by specifying attribtes.", responses=DEFAULT_RESPONSES, operation_id="getEventsByIncident", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_events( request: Request, @@ -173,7 +173,7 @@ async def get_events( description="Get a specific event for a given id", responses=DEFAULT_RESPONSES, operation_id="getEventByIncident", - openapi_extra=iri_meta_dict("graduated", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_event(request: Request, event_id: str) -> models.Event: item = await router.adapter.get_event(event_id) diff --git a/app/routers/task/task.py b/app/routers/task/task.py index e662c447..14bee3fa 100644 --- a/app/routers/task/task.py +++ b/app/routers/task/task.py @@ -17,7 +17,7 @@ response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getTask", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def get_task( request: Request, @@ -37,7 +37,7 @@ async def get_task( dependencies=[Depends(router.current_user)], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getTasks", - openapi_extra=iri_meta_dict("incubator", "required")) + openapi_extra=iri_meta_dict("beta", "required")) @router.get("/", responses=DEFAULT_RESPONSES, operation_id="getTasksWithSlash", include_in_schema=False) async def get_tasks( @@ -54,7 +54,7 @@ async def get_tasks( dependencies=[Depends(router.current_user)], responses=DEFAULT_RESPONSES, operation_id="deleteTask", - openapi_extra=iri_meta_dict("incubator", "required") + openapi_extra=iri_meta_dict("beta", "required") ) async def delete_task( request: Request, From c3b4fe20aed0e871f7f946e3bd15dda9ecf46951 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Wed, 18 Mar 2026 10:02:12 -0500 Subject: [PATCH 121/133] change example label --- app/routers/iri_meta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/routers/iri_meta.py b/app/routers/iri_meta.py index db3bbdfb..bc97b9f3 100644 --- a/app/routers/iri_meta.py +++ b/app/routers/iri_meta.py @@ -4,7 +4,7 @@ It generates: { "x-iri": { - "maturity": "graduated", + "maturity": "production", "implementation": { "level": "required", "required_if_capability": "dpu" @@ -35,4 +35,4 @@ def iri_meta_dict( if not out_obj: return {} - return {"x-iri": out_obj} \ No newline at end of file + return {"x-iri": out_obj} From 9455d9a415230154ddd284836af6acdf4ff5d8cc Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 19 Mar 2026 12:47:29 -0500 Subject: [PATCH 122/133] Update schema URL in API validation workflow --- .github/workflows/api-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/api-validation.yml b/.github/workflows/api-validation.yml index 673a1128..adfcd92a 100644 --- a/.github/workflows/api-validation.yml +++ b/.github/workflows/api-validation.yml @@ -89,7 +89,7 @@ jobs: source .venv/bin/activate python schema-validator/verification/api-validator.py \ --baseurl http://localhost:8000 \ - --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/openapi_iri_facility_api_v1.json \ + --schema-url https://raw.githubusercontent.com/doe-iri/iri-facility-api-docs/refs/heads/main/specification/openapi/all_spec.yaml \ --report-name schemathesis-official echo "exitcode=$?" >> $GITHUB_OUTPUT From 0db75b7daba6e6ed2fec27168a48fd55acc9a2ef Mon Sep 17 00:00:00 2001 From: BenoitCote Date: Thu, 19 Mar 2026 13:15:50 -0700 Subject: [PATCH 123/133] removed /incidents/incident_id from event uris --- app/routers/status/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/routers/status/models.py b/app/routers/status/models.py index cfa17e3a..357db3a6 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -73,7 +73,7 @@ class Event(NamedObject): """Represents an event that occurred to a resource, which may be part of an incident.""" def _self_path(self) -> str: """Return the API path for this event.""" - return f"/status/incidents/{self.incident_id}/events/{self.id}" + return f"/status/events/{self.id}" @field_validator("occurred_at", mode="before") @classmethod @@ -162,7 +162,7 @@ def _norm_dt_field(cls, v): @property def event_uris(self) -> list[str]: """Return the list of event URIs for this incident.""" - return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.id}/events/{e}" for e in self.event_ids] + return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/events/{e}" for e in self.event_ids] @computed_field(description="The list of resources that may be impacted by this incident") @property From aca8a20b9363db41f067f52271067f7a3dcad29c Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 23 Mar 2026 10:51:19 -0500 Subject: [PATCH 124/133] Auth Once at the FastAPI Level --- app/demo_adapter.py | 70 +++++++++--------- app/routers/account/account.py | 31 ++------ app/routers/account/facility_adapter.py | 7 +- app/routers/account/models.py | 10 --- app/routers/compute/compute.py | 68 ++--------------- app/routers/compute/facility_adapter.py | 12 +-- app/routers/filesystem/facility_adapter.py | 38 +++++----- app/routers/filesystem/filesystem.py | 86 +++++++++++----------- app/routers/iri_router.py | 21 ++++-- app/routers/task/facility_adapter.py | 12 +-- app/routers/task/task.py | 14 +--- 11 files changed, 145 insertions(+), 224 deletions(-) diff --git a/app/demo_adapter.py b/app/demo_adapter.py index 264fc467..fa9a23ae 100644 --- a/app/demo_adapter.py +++ b/app/demo_adapter.py @@ -31,6 +31,7 @@ from .routers.task import facility_adapter as task_adapter from .routers.task import models as task_models from .types.models import Capability +from .types.user import User from .types.scalars import AllocationUnit from .apilogger import get_stream_logger from .config import LOG_LEVEL @@ -103,7 +104,7 @@ def __init__(self): self.incidents = [] self.events = [] self.capabilities = {} - self.user = account_models.User(id="gtorok", name="Gabor Torok", api_key="12345", client_ip="1.2.3.4") + self.user = User(id="gtorok", name="Gabor Torok", api_key="12345", client_ip="1.2.3.4") self.projects = [] self.project_allocations = [] self.user_allocations = [] @@ -510,26 +511,27 @@ async def get_user( user_id: str, api_key: str, client_ip: str | None, - ) -> account_models.User: + globus_introspect: dict | None, + ) -> User: if user_id != self.user.id: raise HTTPException(status_code=403, detail="User not found") if api_key.startswith("Bearer "): api_key = api_key[len("Bearer ") :] return self.user - async def get_projects(self: "DemoAdapter", user: account_models.User) -> list[account_models.Project]: + async def get_projects(self: "DemoAdapter", user: User) -> list[account_models.Project]: return self.projects async def get_project_allocations( self: "DemoAdapter", project: account_models.Project, - user: account_models.User, + user: User, ) -> list[account_models.ProjectAllocation]: return [pa for pa in self.project_allocations if pa.project_id == project.id] async def get_user_allocations( self: "DemoAdapter", - user: account_models.User, + user: User, project_allocation: account_models.ProjectAllocation, ) -> list[account_models.UserAllocation]: return [ua for ua in self.user_allocations if ua.project_allocation_id == project_allocation.id] @@ -537,7 +539,7 @@ async def get_user_allocations( async def submit_job( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, job_spec: compute_models.JobSpec, ) -> compute_models.Job: return compute_models.Job( @@ -554,7 +556,7 @@ async def submit_job( async def update_job( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, job_spec: compute_models.JobSpec, job_id: str, ) -> compute_models.Job: @@ -572,7 +574,7 @@ async def update_job( async def get_job( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, job_id: str, historical: bool = False, include_spec: bool = False, @@ -591,7 +593,7 @@ async def get_job( async def get_jobs( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, offset: int, limit: int, filters: dict[str, object] | None = None, @@ -615,7 +617,7 @@ async def get_jobs( async def cancel_job( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, job_id: str, ) -> bool: # call slurm/etc. to cancel job @@ -707,7 +709,7 @@ def _file(self, path: str) -> filesystem_models.File: return filesystem_models.File(**data) - async def chmod(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChmodRequest) -> filesystem_models.PutFileChmodResponse: + async def chmod(self: "DemoAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PutFileChmodRequest) -> filesystem_models.PutFileChmodResponse: rp = self.validate_path(request_model.path) os.chmod(rp, int(request_model.mode, 8)) return filesystem_models.PutFileChmodResponse(output=self._file(rp)) @@ -715,7 +717,7 @@ async def chmod(self: "DemoAdapter", resource: status_models.Resource, user: acc async def chown( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, request_model: filesystem_models.PutFileChownRequest, ) -> filesystem_models.PutFileChownResponse: rp = self.validate_path(request_model.path) @@ -725,7 +727,7 @@ async def chown( async def ls( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, path: str, show_hidden: bool, numeric_uid: bool, @@ -772,7 +774,7 @@ def _headtail( async def head( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, path: str, file_bytes: int | None, lines: int | None, @@ -793,7 +795,7 @@ async def head( async def tail( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, path: str, file_bytes: int | None, lines: int | None, @@ -814,7 +816,7 @@ async def tail( - async def view(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, size: int, offset: int) -> filesystem_models.GetViewFileResponse: + async def view(self: "DemoAdapter", resource: status_models.Resource, user: User, path: str, size: int, offset: int) -> filesystem_models.GetViewFileResponse: rp = self.validate_path(path) result = self._run(f"tail -c +{offset + 1} {rp} | head -c {size}", shell=True) content = result.stdout @@ -827,7 +829,7 @@ async def view(self: "DemoAdapter", resource: status_models.Resource, user: acco ), ) - async def checksum(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileChecksumResponse: + async def checksum(self: "DemoAdapter", resource: status_models.Resource, user: User, path: str) -> filesystem_models.GetFileChecksumResponse: rp = self.validate_path(path) result = self._run(["sha256sum", rp]) checksum = result.stdout.split()[0] @@ -837,14 +839,14 @@ async def checksum(self: "DemoAdapter", resource: status_models.Resource, user: ) ) - async def file(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileTypeResponse: + async def file(self: "DemoAdapter", resource: status_models.Resource, user: User, path: str) -> filesystem_models.GetFileTypeResponse: rp = self.validate_path(path) result = self._run(["file", "-b", rp]) return filesystem_models.GetFileTypeResponse( output=result.stdout.strip(), ) - async def stat(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, dereference: bool) -> filesystem_models.GetFileStatResponse: + async def stat(self: "DemoAdapter", resource: status_models.Resource, user: User, path: str, dereference: bool) -> filesystem_models.GetFileStatResponse: rp = self.validate_path(path) if dereference: stat_info = os.stat(rp) @@ -868,7 +870,7 @@ async def stat(self: "DemoAdapter", resource: status_models.Resource, user: acco async def rm( self: "DemoAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, path: str, ) -> filesystem_models.RemoveResponse: rp = self.validate_path(path) @@ -877,7 +879,7 @@ async def rm( self._run(["rm", "-rf", rp]) return filesystem_models.RemoveResponse(output=f"Removed {rp}") - async def mkdir(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest) -> filesystem_models.PostMkdirResponse: + async def mkdir(self: "DemoAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostMakeDirRequest) -> filesystem_models.PostMkdirResponse: rp = self.validate_path(request_model.path) args = ["mkdir"] if request_model.parent: @@ -887,14 +889,14 @@ async def mkdir(self: "DemoAdapter", resource: status_models.Resource, user: acc return filesystem_models.PostMkdirResponse(output=self._file(rp)) async def symlink( - self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostFileSymlinkRequest + self: "DemoAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostFileSymlinkRequest ) -> filesystem_models.PostFileSymlinkResponse: rp_src = self.validate_path(request_model.path) rp_dst = self.validate_path(request_model.link_path) self._run(["ln", "-s", rp_src, rp_dst]) return filesystem_models.PostFileSymlinkResponse(output=self._file(rp_dst)) - async def download(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileDownloadResponse: + async def download(self: "DemoAdapter", resource: status_models.Resource, user: User, path: str) -> filesystem_models.GetFileDownloadResponse: rp = self.validate_path(path) raw_content = pathlib.Path(rp).read_bytes() @@ -905,7 +907,7 @@ async def download(self: "DemoAdapter", resource: status_models.Resource, user: output=base64.b64encode(raw_content).decode("utf-8"), ) - async def upload(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, path: str, content: str) -> filesystem_models.PutFileUploadResponse: + async def upload(self: "DemoAdapter", resource: status_models.Resource, user: User, path: str, content: str) -> filesystem_models.PutFileUploadResponse: rp = self.validate_path(path) if isinstance(content, bytes): pathlib.Path(rp).write_bytes(content) @@ -916,7 +918,7 @@ async def upload(self: "DemoAdapter", resource: status_models.Resource, user: ac return filesystem_models.PutFileUploadResponse(output=f"Uploaded to {rp}") async def compress( - self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCompressRequest + self: "DemoAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostCompressRequest ) -> filesystem_models.PostCompressResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -942,7 +944,7 @@ async def compress( return filesystem_models.PostCompressResponse(output=self._file(dst_rp)) - async def extract(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostExtractRequest) -> filesystem_models.PostExtractResponse: + async def extract(self: "DemoAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostExtractRequest) -> filesystem_models.PostExtractResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) @@ -969,13 +971,13 @@ async def extract(self: "DemoAdapter", resource: status_models.Resource, user: a return filesystem_models.PostExtractResponse(output=self._file(dst_rp)) - async def mv(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMoveRequest) -> filesystem_models.PostMoveResponse: + async def mv(self: "DemoAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostMoveRequest) -> filesystem_models.PostMoveResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) subprocess.run(["mv", src_rp, dst_rp], check=True) return filesystem_models.PostMoveResponse(output=self._file(dst_rp)) - async def cp(self: "DemoAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCopyRequest) -> filesystem_models.PostCopyResponse: + async def cp(self: "DemoAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostCopyRequest) -> filesystem_models.PostCopyResponse: src_rp = self.validate_path(request_model.path) dst_rp = self.validate_path(request_model.target_path) args = ["cp"] @@ -986,19 +988,19 @@ async def cp(self: "DemoAdapter", resource: status_models.Resource, user: accoun subprocess.run(args, check=True) return filesystem_models.PostCopyResponse(output=self._file(dst_rp)) - async def get_task(self: "DemoAdapter", user: account_models.User, task_id: str) -> task_models.Task | None: + async def get_task(self: "DemoAdapter", user: User, task_id: str) -> task_models.Task | None: await DemoTaskQueue.process_tasks(self) return next((t for t in DemoTaskQueue.tasks if t.user.name == user.name and t.id == task_id), None) - async def get_tasks(self: "DemoAdapter", user: account_models.User) -> list[task_models.Task]: + async def get_tasks(self: "DemoAdapter", user: User) -> list[task_models.Task]: await DemoTaskQueue.process_tasks(self) return [t for t in DemoTaskQueue.tasks if t.user.name == user.name] - async def put_task(self: "DemoAdapter", user: account_models.User, resource: status_models.Resource, task: str) -> task_models.TaskSubmitResponse: + async def put_task(self: "DemoAdapter", user: User, resource: status_models.Resource, task: str) -> task_models.TaskSubmitResponse: await DemoTaskQueue.process_tasks(self) return DemoTaskQueue.create_task(user, resource, task) - async def delete_task(self: "DemoAdapter", user: account_models.User, task_id: str) -> None: + async def delete_task(self: "DemoAdapter", user: User, task_id: str) -> None: await DemoTaskQueue.process_tasks(self) for t in DemoTaskQueue.tasks: if t.user.name == user.name and t.id == task_id: @@ -1012,7 +1014,7 @@ class DemoTask(BaseModel): id: str task: str resource: status_models.Resource - user: account_models.User + user: User start: float status: task_models.TaskStatus = task_models.TaskStatus.pending result: dict | None = None @@ -1048,7 +1050,7 @@ async def process_tasks(da: DemoAdapter): DemoTaskQueue.tasks = _tasks @staticmethod - def create_task(user: account_models.User, resource: status_models.Resource, command: task_models.TaskCommand) -> task_models.TaskSubmitResponse: + def create_task(user: User, resource: status_models.Resource, command: task_models.TaskCommand) -> task_models.TaskSubmitResponse: """Create a new task in the queue.""" task_id = f"task_{len(DemoTaskQueue.tasks)}" DemoTaskQueue.tasks.append(DemoTask(id=task_id, task=command.model_dump_json(), user=user, resource=resource, start=utc_timestamp())) diff --git a/app/routers/account/account.py b/app/routers/account/account.py index e7b0af16..35ba9cb8 100644 --- a/app/routers/account/account.py +++ b/app/routers/account/account.py @@ -3,6 +3,7 @@ from ...types.http import forbidExtraQueryParams from ...types.models import Capability from ...types.scalars import StrictDateTime +from ...types.user import User from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from ..iri_meta import iri_meta_dict @@ -58,7 +59,6 @@ async def get_capability( @router.get( "/projects", - dependencies=[Depends(router.current_user)], summary="Get the projects of the current user", description="Get a list of projects for the currently authenticated user at this facility.", responses=DEFAULT_RESPONSES, @@ -67,17 +67,14 @@ async def get_capability( ) async def get_projects( request: Request, + user: User = Depends(router.current_user), _forbid=Depends(forbidExtraQueryParams()), ) -> list[models.Project]: - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") return await router.adapter.get_projects(user) @router.get( "/projects/{project_id}", - dependencies=[Depends(router.current_user)], summary="Get a single project", description="Get a single project at this facility.", responses=DEFAULT_RESPONSES, @@ -87,11 +84,9 @@ async def get_projects( async def get_project( project_id: str, request: Request, + user: User = Depends(router.current_user), _forbid=Depends(forbidExtraQueryParams()), ) -> models.Project: - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") projects = await router.adapter.get_projects(user=user) pp = next((p for p in projects if p.id == project_id), None) if not pp: @@ -101,7 +96,6 @@ async def get_project( @router.get( "/projects/{project_id}/project_allocations", - dependencies=[Depends(router.current_user)], summary="Get the allocations of the current user's projects", description="Get a list of allocations for the currently authenticated user's projects at this facility.", responses=DEFAULT_RESPONSES, @@ -111,11 +105,9 @@ async def get_project( async def get_project_allocations( project_id: str, request: Request, + user: User = Depends(router.current_user), _forbid=Depends(forbidExtraQueryParams()), ) -> list[models.ProjectAllocation]: - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: @@ -125,7 +117,6 @@ async def get_project_allocations( @router.get( "/projects/{project_id}/project_allocations/{project_allocation_id}", - dependencies=[Depends(router.current_user)], summary="Get a single project allocation", description="Get a single project allocation at this facility for this user.", responses=DEFAULT_RESPONSES, @@ -136,11 +127,9 @@ async def get_project_allocation( project_id: str, project_allocation_id: str, request: Request, + user: User = Depends(router.current_user), _forbid=Depends(forbidExtraQueryParams()), ) -> models.ProjectAllocation: - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: @@ -154,7 +143,6 @@ async def get_project_allocation( @router.get( "/projects/{project_id}/project_allocations/{project_allocation_id}/user_allocations", - dependencies=[Depends(router.current_user)], summary="Get the user allocations of the current user's projects", description="Get a list of user allocations for the currently authenticated user's projects at this facility.", responses=DEFAULT_RESPONSES, @@ -166,10 +154,8 @@ async def get_user_allocations( project_allocation_id: str, request: Request, _forbid=Depends(forbidExtraQueryParams()), + user: User = Depends(router.current_user), ) -> list[models.UserAllocation]: - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: @@ -183,7 +169,6 @@ async def get_user_allocations( @router.get( "/projects/{project_id}/project_allocations/{project_allocation_id}/user_allocations/{user_allocation_id}", - dependencies=[Depends(router.current_user)], summary="Get a user allocation of the current user's projects", description="Get a user allocation for the currently authenticated user's projects at this facility.", responses=DEFAULT_RESPONSES, @@ -196,10 +181,8 @@ async def get_user_allocation( user_allocation_id: str, request: Request, _forbid=Depends(forbidExtraQueryParams()), + user: User = Depends(router.current_user), ) -> models.UserAllocation: - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") projects = await router.adapter.get_projects(user=user) project = next((p for p in projects if p.id == project_id), None) if not project: diff --git a/app/routers/account/facility_adapter.py b/app/routers/account/facility_adapter.py index 98f815b0..109c60bd 100644 --- a/app/routers/account/facility_adapter.py +++ b/app/routers/account/facility_adapter.py @@ -1,6 +1,7 @@ from abc import abstractmethod from ...types.models import Capability +from ...types.user import User from ..iri_router import AuthenticatedAdapter from . import models as account_models @@ -17,13 +18,13 @@ async def get_capabilities(self: "FacilityAdapter", name: str | None = None, mod pass @abstractmethod - async def get_projects(self: "FacilityAdapter", user: account_models.User) -> list[account_models.Project]: + async def get_projects(self: "FacilityAdapter", user: User) -> list[account_models.Project]: pass @abstractmethod - async def get_project_allocations(self: "FacilityAdapter", project: account_models.Project, user: account_models.User) -> list[account_models.ProjectAllocation]: + async def get_project_allocations(self: "FacilityAdapter", project: account_models.Project, user: User) -> list[account_models.ProjectAllocation]: pass @abstractmethod - async def get_user_allocations(self: "FacilityAdapter", user: account_models.User, project_allocation: account_models.ProjectAllocation) -> list[account_models.UserAllocation]: + async def get_user_allocations(self: "FacilityAdapter", user: User, project_allocation: account_models.ProjectAllocation) -> list[account_models.UserAllocation]: pass diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 774fd2b2..0c6beba2 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -7,16 +7,6 @@ from ...types.scalars import AllocationUnit -class User(IRIBaseModel): - """A user of the facility""" - - id: str = Field(..., description="Unique identifier of the user.", example="user-123") - name: str = Field(..., description="Name of the user.", example="Jane Doe") - api_key: str = Field(..., description="API key associated with this user.", example="AKIAIOSFODNN7EXAMPLE") - client_ip: str|None = Field(default=None, description="IP address from which the user connects.", example="192.0.2.10") - # we could expose more fields here (eg. email) but it might be against policy - - class Project(IRIBaseModel): """A project and its users at a facility""" diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index a186c92d..d95f6a96 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -4,6 +4,7 @@ from ...types.http import forbidExtraQueryParams from ...types.scalars import StrictHTTPBool +from ...types.user import User from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from ..iri_meta import iri_meta_dict @@ -19,7 +20,6 @@ @router.post( "/job/{resource_id:str}", - dependencies=[Depends(router.current_user)], response_model=models.Job, response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, @@ -30,6 +30,7 @@ async def submit_job( resource_id: str, job_spec: models.JobSpec, request: Request, + user: User = Depends(router.current_user), _forbid=Depends(forbidExtraQueryParams()), ): """ @@ -40,10 +41,6 @@ async def submit_job( This command will attempt to submit a job and return its id. """ - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - # look up the resource (todo: maybe ensure it's available) resource = await status_router.adapter.get_resource(resource_id) @@ -52,46 +49,8 @@ async def submit_job( return await router.adapter.submit_job(resource=resource, user=user, job_spec=job_spec) -# TODO: this conflicts with PUT commented out while we finalize the API design -# @router.post( -# "/job/script/{resource_id:str}", -# dependencies=[Depends(router.current_user)], -# response_model=models.Job, -# response_model_exclude_unset=True, -# responses=DEFAULT_RESPONSES, -# operation_id="launchJobScript", -# ) -# async def submit_job_path( -# resource_id: str, -# job_script_path : str, -# request : Request, -# args : Annotated[List[str], Form()] = [], -# _forbid = Depends(iri_router.forbidExtraQueryParams("job_script_path")), -# ): -# """ -# Submit a job on a compute resource -# -# - **resource**: the name of the compute resource to use -# - **job_script_path**: path to the job script on the compute resource -# - **args**: optional arguments to the job script -# -# This command will attempt to submit a job and return its id. -# """ -# user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) -# if not user: -# raise HTTPException(status_code=404, detail="User not found") -# -# # look up the resource (todo: maybe ensure it's available) -# resource = await status_router.adapter.get_resource(resource_id) -# -# # the handler can use whatever means it wants to submit the job and then fill in its id -# # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs -# return await router.adapter.submit_job_script(resource=resource, user=user, job_script_path=job_script_path, args=args) - - @router.put( "/job/{resource_id:str}/{job_id:str}", - dependencies=[Depends(router.current_user)], response_model=models.Job, response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, @@ -103,6 +62,7 @@ async def update_job( job_id: str, job_spec: models.JobSpec, request: Request, + user: User = Depends(router.current_user), _forbid=Depends(forbidExtraQueryParams()), ): """ @@ -113,10 +73,6 @@ async def update_job( - **job_request**: a PSIJ job spec as defined here """ - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - # look up the resource (todo: maybe ensure it's available) resource = await status_router.adapter.get_resource(resource_id) @@ -127,7 +83,6 @@ async def update_job( @router.get( "/status/{resource_id:str}/{job_id:str}", - dependencies=[Depends(router.current_user)], response_model=models.Job, response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, @@ -138,15 +93,12 @@ async def get_job_status( resource_id: str, job_id: str, request: Request, + user: User = Depends(router.current_user), historical: StrictHTTPBool | None = Query(default=False, description="Whether to include historical jobs. Defaults to false"), include_spec: StrictHTTPBool | None = Query(default=False, description="Whether to include the job specification. Defaults to false"), _forbid=Depends(forbidExtraQueryParams("historical", "include_spec")), ): """Get a job's status""" - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) resource = await status_router.adapter.get_resource(resource_id) @@ -158,7 +110,6 @@ async def get_job_status( @router.post( "/status/{resource_id:str}", - dependencies=[Depends(router.current_user)], response_model=list[models.Job], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, @@ -168,6 +119,7 @@ async def get_job_status( async def get_job_statuses( resource_id: str, request: Request, + user: User = Depends(router.current_user), offset: int = Query(default=0, ge=0), limit: int = Query(default=100, ge=0, le=1000), filters: dict[str, object] | None = None, @@ -176,10 +128,6 @@ async def get_job_statuses( _forbid=Depends(forbidExtraQueryParams("offset", "limit", "filters", "historical", "include_spec")), ): """Get multiple jobs' statuses""" - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) resource = await status_router.adapter.get_resource(resource_id) @@ -191,7 +139,6 @@ async def get_job_statuses( @router.delete( "/cancel/{resource_id:str}/{job_id:str}", - dependencies=[Depends(router.current_user)], status_code=status.HTTP_204_NO_CONTENT, response_model=None, response_model_exclude_unset=True, @@ -203,13 +150,10 @@ async def cancel_job( resource_id: str, job_id: str, request: Request, + user: User = Depends(router.current_user), _forbid=Depends(forbidExtraQueryParams()), ): """Cancel a job""" - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - # look up the resource (todo: maybe ensure it's available) resource = await status_router.adapter.get_resource(resource_id) diff --git a/app/routers/compute/facility_adapter.py b/app/routers/compute/facility_adapter.py index 608217e1..a8cbf9aa 100644 --- a/app/routers/compute/facility_adapter.py +++ b/app/routers/compute/facility_adapter.py @@ -1,6 +1,6 @@ from abc import abstractmethod +from ...types.user import User from ..status import models as status_models -from ..account import models as account_models from . import models as compute_models from ..iri_router import AuthenticatedAdapter @@ -13,18 +13,18 @@ class FacilityAdapter(AuthenticatedAdapter): """ @abstractmethod - async def submit_job(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec) -> compute_models.Job: + async def submit_job(self: "FacilityAdapter", resource: status_models.Resource, user: User, job_spec: compute_models.JobSpec) -> compute_models.Job: pass @abstractmethod - async def update_job(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_spec: compute_models.JobSpec, job_id: str) -> compute_models.Job: + async def update_job(self: "FacilityAdapter", resource: status_models.Resource, user: User, job_spec: compute_models.JobSpec, job_id: str) -> compute_models.Job: pass @abstractmethod async def get_job( self: "FacilityAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, job_id: str, historical: bool = False, include_spec: bool = False, @@ -35,7 +35,7 @@ async def get_job( async def get_jobs( self: "FacilityAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, offset: int, limit: int, filters: dict[str, object] | None = None, @@ -45,5 +45,5 @@ async def get_jobs( pass @abstractmethod - async def cancel_job(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, job_id: str) -> bool: + async def cancel_job(self: "FacilityAdapter", resource: status_models.Resource, user: User, job_id: str) -> bool: pass diff --git a/app/routers/filesystem/facility_adapter.py b/app/routers/filesystem/facility_adapter.py index 0e272563..df545a19 100644 --- a/app/routers/filesystem/facility_adapter.py +++ b/app/routers/filesystem/facility_adapter.py @@ -1,7 +1,7 @@ import os from abc import abstractmethod +from ...types.user import User from ..status import models as status_models -from ..account import models as account_models from . import models as filesystem_models from ..iri_router import AuthenticatedAdapter @@ -25,87 +25,87 @@ class FacilityAdapter(AuthenticatedAdapter): @abstractmethod async def chmod( - self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChmodRequest + self: "FacilityAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PutFileChmodRequest ) -> filesystem_models.PutFileChmodResponse: pass @abstractmethod async def chown( - self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PutFileChownRequest + self: "FacilityAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PutFileChownRequest ) -> filesystem_models.PutFileChownResponse: pass @abstractmethod async def ls( - self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, show_hidden: bool, numeric_uid: bool, recursive: bool, dereference: bool + self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str, show_hidden: bool, numeric_uid: bool, recursive: bool, dereference: bool ) -> filesystem_models.GetDirectoryLsResponse: pass @abstractmethod - async def head(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int, lines: int, skip_trailing: bool) -> filesystem_models.GetFileHeadResponse: + async def head(self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str, file_bytes: int, lines: int, skip_trailing: bool) -> filesystem_models.GetFileHeadResponse: pass @abstractmethod - async def tail(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, file_bytes: int | None, lines: int | None, skip_heading: bool) -> filesystem_models.GetFileTailResponse: + async def tail(self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str, file_bytes: int | None, lines: int | None, skip_heading: bool) -> filesystem_models.GetFileTailResponse: pass @abstractmethod - async def view(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, size: int, offset: int) -> filesystem_models.GetViewFileResponse: + async def view(self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str, size: int, offset: int) -> filesystem_models.GetViewFileResponse: pass @abstractmethod - async def checksum(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileChecksumResponse: + async def checksum(self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str) -> filesystem_models.GetFileChecksumResponse: pass @abstractmethod - async def file(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileTypeResponse: + async def file(self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str) -> filesystem_models.GetFileTypeResponse: pass @abstractmethod - async def stat(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, dereference: bool) -> filesystem_models.GetFileStatResponse: + async def stat(self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str, dereference: bool) -> filesystem_models.GetFileStatResponse: pass @abstractmethod - async def rm(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.RemoveResponse: + async def rm(self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str) -> filesystem_models.RemoveResponse: pass @abstractmethod - async def mkdir(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMakeDirRequest) -> filesystem_models.PostMkdirResponse: + async def mkdir(self: "FacilityAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostMakeDirRequest) -> filesystem_models.PostMkdirResponse: pass @abstractmethod async def symlink( self: "FacilityAdapter", resource: status_models.Resource, - user: account_models.User, + user: User, request_model: filesystem_models.PostFileSymlinkRequest, ) -> filesystem_models.PostFileSymlinkResponse: pass @abstractmethod - async def download(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str) -> filesystem_models.GetFileDownloadResponse: + async def download(self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str) -> filesystem_models.GetFileDownloadResponse: pass @abstractmethod - async def upload(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, path: str, content: str) -> filesystem_models.PutFileUploadResponse: + async def upload(self: "FacilityAdapter", resource: status_models.Resource, user: User, path: str, content: str) -> filesystem_models.PutFileUploadResponse: pass @abstractmethod async def compress( - self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCompressRequest + self: "FacilityAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostCompressRequest ) -> filesystem_models.PostCompressResponse: pass @abstractmethod async def extract( - self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostExtractRequest + self: "FacilityAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostExtractRequest ) -> filesystem_models.PostExtractResponse: pass @abstractmethod - async def mv(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostMoveRequest) -> filesystem_models.PostMoveResponse: + async def mv(self: "FacilityAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostMoveRequest) -> filesystem_models.PostMoveResponse: pass @abstractmethod - async def cp(self: "FacilityAdapter", resource: status_models.Resource, user: account_models.User, request_model: filesystem_models.PostCopyRequest) -> filesystem_models.PostCopyResponse: + async def cp(self: "FacilityAdapter", resource: status_models.Resource, user: User, request_model: filesystem_models.PostCopyRequest) -> filesystem_models.PostCopyResponse: pass diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index 0ba16958..e3c35037 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -7,11 +7,11 @@ import base64 from typing import Annotated from fastapi import Depends, HTTPException, status, Query, Request, File, UploadFile +from ...types.user import User from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from ..iri_meta import iri_meta_dict from ..status.status import router as status_router, models as status_models -from ..account.account import models as account_models from ..task import facility_adapter as task_facility_adapter, models as task_models from . import models, facility_adapter @@ -26,22 +26,17 @@ async def _user_resource( resource_id: str, - request: Request, -) -> tuple[account_models.User, status_models.Resource]: - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - + user: User, +) -> status_models.Resource: # look up the resource (todo: maybe ensure it's available) resource = await status_router.adapter.get_resource(resource_id) if not resource: raise HTTPException(status_code=404, detail="Resource not found") - return (user, resource) + return resource @router.put( "/chmod/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Change the permission mode of a file(`chmod`)", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -54,8 +49,9 @@ async def put_chmod( resource_id: str, request_model: models.PutFileChmodRequest, request: Request, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -71,7 +67,6 @@ async def put_chmod( @router.put( "/chown/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Change the ownership of a given file (`chown`)", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -84,8 +79,9 @@ async def put_chown( resource_id: str, request_model: models.PutFileChownRequest, request: Request, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -101,7 +97,6 @@ async def put_chown( @router.get( "/file/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Output the type of a file or directory", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -114,8 +109,9 @@ async def get_file( resource_id: str, request: Request, path: Annotated[str, Query(description="A file or folder path")], + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -131,7 +127,6 @@ async def get_file( @router.get( "/stat/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Output the `stat` of a file", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -145,8 +140,9 @@ async def get_stat( request: Request, path: Annotated[str, Query(description="A file or folder path")], dereference: Annotated[bool, Query(description="Follow symbolic links")] = False, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -163,7 +159,6 @@ async def get_stat( @router.post( "/mkdir/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Create directory operation (`mkdir`)", status_code=status.HTTP_201_CREATED, response_model=task_models.TaskSubmitResponse, @@ -176,8 +171,9 @@ async def post_mkdir( resource_id: str, request: Request, request_model: models.PostMakeDirRequest, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -193,7 +189,6 @@ async def post_mkdir( @router.post( "/symlink/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Create symlink operation (`ln`)", status_code=status.HTTP_201_CREATED, response_model=task_models.TaskSubmitResponse, @@ -206,8 +201,9 @@ async def post_symlink( resource_id: str, request: Request, request_model: models.PostFileSymlinkRequest, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -223,7 +219,6 @@ async def post_symlink( @router.get( "/ls/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="List the contents of the given directory (`ls`) asynchronously", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -247,8 +242,10 @@ async def get_ls_async( description="Show information for the file the link references.", ), ] = False, + user: User = Depends(router.current_user), + ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -262,7 +259,6 @@ async def get_ls_async( @router.get( "/head/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Output the first part of file/s (`head`)", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -300,8 +296,9 @@ async def get_head( description=("The output will be the whole file, without the last NUM bytes/lines of each file. NUM should be specified in the respective argument through `bytes` or `lines`."), ), ] = False, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) # Enforce that exactly one of `bytes` or `lines` is specified if (file_bytes is None and lines is None) or (file_bytes is not None and lines is not None): raise HTTPException(status_code=400, detail="Exactly one of `bytes` or `lines` must be specified.") @@ -323,7 +320,6 @@ async def get_head( @router.get( "/view/{resource_id:str}", - dependencies=[Depends(router.current_user)], description=f"View file content (up to max {facility_adapter.OPS_SIZE_LIMIT} bytes)", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -338,8 +334,9 @@ async def get_view( path: Annotated[str, Query(description="File path")], size: Annotated[int, Query(description="Value, in bytes, of the size of data to be retrieved from the file.", ge=1, le=facility_adapter.OPS_SIZE_LIMIT)] = facility_adapter.OPS_SIZE_LIMIT, offset: Annotated[int, Query(description="Value in bytes of the offset.", ge=0)] = 0, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, @@ -358,7 +355,6 @@ async def get_view( @router.get( "/tail/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Output the last part of a file (`tail`)", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -389,8 +385,10 @@ async def get_tail( description=("The output will be the whole file, without the first NUM bytes/lines of each file. NUM should be specified in the respective argument through `bytes` or `lines`."), ), ] = False, + user: User = Depends(router.current_user), + ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) # Enforce that exactly one of `bytes` or `lines` is specified if (file_bytes is None and lines is None) or (file_bytes is not None and lines is not None): raise HTTPException(status_code=400, detail="Exactly one of `bytes` or `lines` must be specified.") @@ -412,7 +410,6 @@ async def get_tail( @router.get( "/checksum/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Output the checksum of a file (using SHA-256 algotithm)", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -425,8 +422,9 @@ async def get_checksum( resource_id: str, request: Request, path: Annotated[str, Query(description="Target system")], + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -442,7 +440,6 @@ async def get_checksum( @router.delete( "/rm/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Delete file or directory operation (`rm`)", response_description="File or directory deleted successfully", responses=DEFAULT_RESPONSES, @@ -453,8 +450,9 @@ async def delete_rm( resource_id: str, request: Request, path: Annotated[str, Query(description="The path to delete")], + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -470,7 +468,6 @@ async def delete_rm( @router.post( "/compress/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Compress files and directories using `tar` command", status_code=status.HTTP_201_CREATED, response_model=task_models.TaskSubmitResponse, @@ -483,8 +480,9 @@ async def post_compress( resource_id: str, request: Request, request_model: models.PostCompressRequest, + user: str = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -500,7 +498,6 @@ async def post_compress( @router.post( "/extract/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Extract `tar` `gzip` archives", status_code=status.HTTP_201_CREATED, response_model=task_models.TaskSubmitResponse, @@ -513,8 +510,9 @@ async def post_extract( resource_id: str, request: Request, request_model: models.PostExtractRequest, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -530,7 +528,6 @@ async def post_extract( @router.post( "/mv/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Create move file or directory operation (`mv`)", status_code=status.HTTP_201_CREATED, response_model=task_models.TaskSubmitResponse, @@ -543,8 +540,9 @@ async def move_mv( resource_id: str, request: Request, request_model: models.PostMoveRequest, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -560,7 +558,6 @@ async def move_mv( @router.post( "/cp/{resource_id:str}", - dependencies=[Depends(router.current_user)], description="Create copy file or directory operation (`cp`)", status_code=status.HTTP_201_CREATED, response_model=task_models.TaskSubmitResponse, @@ -573,8 +570,9 @@ async def post_cp( resource_id: str, request: Request, request_model: models.PostCopyRequest, + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -590,7 +588,6 @@ async def post_cp( @router.get( "/download/{resource_id:str}", - dependencies=[Depends(router.current_user)], description=f"Download a small file (max {facility_adapter.OPS_SIZE_LIMIT} Bytes)", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -603,8 +600,9 @@ async def get_download( resource_id: str, request: Request, path: Annotated[str, Query(description="A file to download")], + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) return await router.task_adapter.put_task( user=user, resource=resource, @@ -620,7 +618,6 @@ async def get_download( @router.post( "/upload/{resource_id:str}", - dependencies=[Depends(router.current_user)], description=f"Upload a small file (max {facility_adapter.OPS_SIZE_LIMIT} Bytes)", status_code=status.HTTP_200_OK, response_model=task_models.TaskSubmitResponse, @@ -634,8 +631,9 @@ async def post_upload( request: Request, path: Annotated[str, Query(description="Specify path where file should be uploaded.")], file: UploadFile = File(description="File to be uploaded as `multipart/form-data`"), + user: User = Depends(router.current_user), ) -> task_models.TaskSubmitResponse: - user, resource = await _user_resource(resource_id, request) + resource = await _user_resource(resource_id, user) raw_content = file.file.read() if len(raw_content) > facility_adapter.OPS_SIZE_LIMIT: diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 48ca4aa6..4fa19c2f 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -6,8 +6,8 @@ import globus_sdk from fastapi import Request, Depends, HTTPException, APIRouter from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -from .account.models import User +from ..types.user import User bearer_scheme = HTTPBearer() @@ -120,10 +120,10 @@ async def current_user( credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme), ): token = credentials.credentials - + ip_address = get_client_ip(request) user_id = None + globus_introspect = None try: - ip_address = get_client_ip(request) if GLOBUS_RS_ID and GLOBUS_RS_SECRET and GLOBUS_RS_SCOPE_SUFFIX: try: globus_introspect = await self.get_globus_info(token) @@ -137,8 +137,17 @@ async def current_user( raise HTTPException(status_code=401, detail="Invalid or malformed Authorization parameters") from exc if not user_id: raise HTTPException(status_code=403, detail="Unauthorized access") - request.state.current_user_id = user_id - request.state.api_key = token + + user = await self.adapter.get_user( + user_id=user_id, + api_key=token, + client_ip=ip_address, + globus_introspect=globus_introspect, + ) + + if not user: + raise HTTPException(status_code=404, detail="User not found") + return user class AuthenticatedAdapter(ABC): @@ -161,7 +170,7 @@ async def get_current_user_globus(self: "AuthenticatedAdapter", api_key: str, cl pass @abstractmethod - async def get_user(self: "AuthenticatedAdapter", user_id: str, api_key: str, client_ip: str | None) -> User: + async def get_user(self: "AuthenticatedAdapter", user_id: str, api_key: str, client_ip: str | None, globus_introspect: dict | None) -> User: """ Retrieve additional user information (name, email, etc.) for the given user_id. """ diff --git a/app/routers/task/facility_adapter.py b/app/routers/task/facility_adapter.py index fba4206d..bec55bca 100644 --- a/app/routers/task/facility_adapter.py +++ b/app/routers/task/facility_adapter.py @@ -1,7 +1,7 @@ import traceback from abc import abstractmethod +from ...types.user import User from . import models as task_models -from ..account import models as account_models from ..status import models as status_models from ..filesystem import models as filesystem_models, facility_adapter as filesystem_adapter from ..iri_router import AuthenticatedAdapter, IriRouter @@ -18,23 +18,23 @@ class FacilityAdapter(AuthenticatedAdapter): """ @abstractmethod - async def get_task(self: "FacilityAdapter", user: account_models.User, task_id: str) -> task_models.Task | None: + async def get_task(self: "FacilityAdapter", user: User, task_id: str) -> task_models.Task | None: pass @abstractmethod - async def get_tasks(self: "FacilityAdapter", user: account_models.User) -> list[task_models.Task]: + async def get_tasks(self: "FacilityAdapter", user: User) -> list[task_models.Task]: pass @abstractmethod - async def put_task(self: "FacilityAdapter", user: account_models.User, resource: status_models.Resource | None, task: task_models.TaskCommand) -> task_models.TaskSubmitResponse: + async def put_task(self: "FacilityAdapter", user: User, resource: status_models.Resource | None, task: task_models.TaskCommand) -> task_models.TaskSubmitResponse: pass @abstractmethod - async def delete_task(self: "FacilityAdapter", user: account_models.User, task_id: str) -> None: + async def delete_task(self: "FacilityAdapter", user: User, task_id: str) -> None: pass @staticmethod - async def on_task(resource: status_models.Resource, user: account_models.User, task: task_models.TaskCommand) -> tuple[dict, task_models.TaskStatus]: + async def on_task(resource: status_models.Resource, user: User, task: task_models.TaskCommand) -> tuple[dict, task_models.TaskStatus]: # Handle a task from the facility message queue. # Returns: (result, status) def _extractNull(ind): diff --git a/app/routers/task/task.py b/app/routers/task/task.py index 14bee3fa..ff2a5627 100644 --- a/app/routers/task/task.py +++ b/app/routers/task/task.py @@ -1,4 +1,5 @@ from fastapi import Request, HTTPException, Depends +from ...types.user import User from .. import iri_router from ..error_handlers import DEFAULT_RESPONSES from ..iri_meta import iri_meta_dict @@ -13,7 +14,6 @@ @router.get( "/{task_id:str}", - dependencies=[Depends(router.current_user)], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getTask", @@ -22,11 +22,9 @@ async def get_task( request: Request, task_id: str, + user: User = Depends(router.current_user) ) -> models.Task: """Get a task""" - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") task = await router.adapter.get_task(user=user, task_id=task_id) if not task: raise HTTPException(status_code=404, detail=f"Task {task_id} not found") @@ -42,11 +40,9 @@ async def get_task( async def get_tasks( request: Request, + user: User = Depends(router.current_user) ) -> list[models.Task]: """Get all tasks""" - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") return await router.adapter.get_tasks(user=user) @router.delete( @@ -59,10 +55,8 @@ async def get_tasks( async def delete_task( request: Request, task_id: str, + user: User = Depends(router.current_user) ) -> str: """Delete a task""" - user = await router.adapter.get_user(user_id=request.state.current_user_id, api_key=request.state.api_key, client_ip=iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") await router.adapter.delete_task(user=user, task_id=task_id) return f"Task {task_id} deleted successfully" \ No newline at end of file From 49ff4fdabb2dac5bf28e6222f996ae05f086f9a6 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Mon, 23 Mar 2026 11:06:44 -0500 Subject: [PATCH 125/133] Add user type --- app/types/user.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 app/types/user.py diff --git a/app/types/user.py b/app/types/user.py new file mode 100644 index 00000000..fa9db2b4 --- /dev/null +++ b/app/types/user.py @@ -0,0 +1,12 @@ +from pydantic import Field +from ..types.base import IRIBaseModel + + +class User(IRIBaseModel): + """A user of the facility""" + + id: str = Field(..., description="Unique identifier of the user.", example="user-123") + name: str = Field(..., description="Name of the user.", example="Jane Doe") + api_key: str = Field(..., description="API key associated with this user.", example="AKIAIOSFODNN7EXAMPLE") + client_ip: str|None = Field(default=None, description="IP address from which the user connects.", example="192.0.2.10") + # we could expose more fields here (eg. email) but it might be against policy \ No newline at end of file From 93878dd37eede067608db3dd482b736623978ecb Mon Sep 17 00:00:00 2001 From: Gabor Torok Date: Tue, 24 Mar 2026 10:00:09 -0700 Subject: [PATCH 126/133] force login so session_info.authentications is not empty --- tools/globus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/globus.py b/tools/globus.py index 328b9720..772a1b75 100644 --- a/tools/globus.py +++ b/tools/globus.py @@ -19,7 +19,7 @@ ) # Get the authorization URL -authorize_url = client.oauth2_get_authorize_url() +authorize_url = client.oauth2_get_authorize_url(query_params={"prompt": "login"}) print(f"Visit this URL in your browser:\n{authorize_url}\n") # After visiting the URL and authorizing, you'll be redirected to a URL with a code parameter From 8a1e0658ab796afb6f6d84b11ba33826891b5dd5 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 31 Mar 2026 05:57:29 -0500 Subject: [PATCH 127/133] Set Compute, Filesystem, Task to Production status --- app/routers/compute/compute.py | 10 ++++---- app/routers/filesystem/filesystem.py | 36 ++++++++++++++-------------- app/routers/task/task.py | 6 ++--- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index d95f6a96..f8a4cb53 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -24,7 +24,7 @@ response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="launchJob", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def submit_job( resource_id: str, @@ -55,7 +55,7 @@ async def submit_job( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="updateJob", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def update_job( resource_id: str, @@ -87,7 +87,7 @@ async def update_job( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getJob", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_job_status( resource_id: str, @@ -114,7 +114,7 @@ async def get_job_status( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getJobs", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_job_statuses( resource_id: str, @@ -144,7 +144,7 @@ async def get_job_statuses( response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="cancelJob", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def cancel_job( resource_id: str, diff --git a/app/routers/filesystem/filesystem.py b/app/routers/filesystem/filesystem.py index e3c35037..cd6448a3 100644 --- a/app/routers/filesystem/filesystem.py +++ b/app/routers/filesystem/filesystem.py @@ -43,7 +43,7 @@ async def _user_resource( response_description="File permissions changed successfully", responses=DEFAULT_RESPONSES, operation_id="chmod", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def put_chmod( resource_id: str, @@ -73,7 +73,7 @@ async def put_chmod( response_description="File ownership changed successfully", responses=DEFAULT_RESPONSES, operation_id="chown", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def put_chown( resource_id: str, @@ -103,7 +103,7 @@ async def put_chown( response_description="Type returned successfully", responses=DEFAULT_RESPONSES, operation_id="file", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_file( resource_id: str, @@ -133,7 +133,7 @@ async def get_file( response_description="Stat returned successfully", responses=DEFAULT_RESPONSES, operation_id="stat", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_stat( resource_id: str, @@ -165,7 +165,7 @@ async def get_stat( response_description="Directory created successfully", responses=DEFAULT_RESPONSES, operation_id="mkdir", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def post_mkdir( resource_id: str, @@ -195,7 +195,7 @@ async def post_mkdir( response_description="Symlink created successfully", responses=DEFAULT_RESPONSES, operation_id="symlink", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def post_symlink( resource_id: str, @@ -226,7 +226,7 @@ async def post_symlink( include_in_schema=router.task_adapter is not None, responses=DEFAULT_RESPONSES, operation_id="ls", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_ls_async( resource_id: str, @@ -265,7 +265,7 @@ async def get_ls_async( response_description="Head operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="head", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_head( resource_id: str, @@ -326,7 +326,7 @@ async def get_head( response_description="View operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="view", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_view( resource_id: str, @@ -361,7 +361,7 @@ async def get_view( response_description="`tail` operation finished successfully", responses=DEFAULT_RESPONSES, operation_id="tail", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_tail( resource_id: str, @@ -416,7 +416,7 @@ async def get_tail( response_description="Checksum returned successfully", responses=DEFAULT_RESPONSES, operation_id="checksum", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_checksum( resource_id: str, @@ -444,7 +444,7 @@ async def get_checksum( response_description="File or directory deleted successfully", responses=DEFAULT_RESPONSES, operation_id="rm", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def delete_rm( resource_id: str, @@ -474,7 +474,7 @@ async def delete_rm( response_description="File and/or directories compressed successfully", responses=DEFAULT_RESPONSES, operation_id="compress", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def post_compress( resource_id: str, @@ -504,7 +504,7 @@ async def post_compress( response_description="File extracted successfully", responses=DEFAULT_RESPONSES, operation_id="extract", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def post_extract( resource_id: str, @@ -534,7 +534,7 @@ async def post_extract( response_description="Move file or directory operation created successfully", responses=DEFAULT_RESPONSES, operation_id="mv", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def move_mv( resource_id: str, @@ -564,7 +564,7 @@ async def move_mv( response_description="Copy file or directory operation created successfully", responses=DEFAULT_RESPONSES, operation_id="cp", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def post_cp( resource_id: str, @@ -594,7 +594,7 @@ async def post_cp( response_description="File downloaded successfully", responses=DEFAULT_RESPONSES, operation_id="download", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_download( resource_id: str, @@ -624,7 +624,7 @@ async def get_download( response_description="File uploaded successfully", responses=DEFAULT_RESPONSES, operation_id="upload", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def post_upload( resource_id: str, diff --git a/app/routers/task/task.py b/app/routers/task/task.py index ff2a5627..b3a81166 100644 --- a/app/routers/task/task.py +++ b/app/routers/task/task.py @@ -17,7 +17,7 @@ response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getTask", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def get_task( request: Request, @@ -35,7 +35,7 @@ async def get_task( dependencies=[Depends(router.current_user)], response_model_exclude_unset=True, responses=DEFAULT_RESPONSES, operation_id="getTasks", - openapi_extra=iri_meta_dict("beta", "required")) + openapi_extra=iri_meta_dict("production", "required")) @router.get("/", responses=DEFAULT_RESPONSES, operation_id="getTasksWithSlash", include_in_schema=False) async def get_tasks( @@ -50,7 +50,7 @@ async def get_tasks( dependencies=[Depends(router.current_user)], responses=DEFAULT_RESPONSES, operation_id="deleteTask", - openapi_extra=iri_meta_dict("beta", "required") + openapi_extra=iri_meta_dict("production", "required") ) async def delete_task( request: Request, From 4e5745f20429ab04254f3f04a74c7c398272ce46 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 31 Mar 2026 06:33:34 -0500 Subject: [PATCH 128/133] Pass Error Message back of failed auth --- app/routers/iri_router.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index 4fa19c2f..f839416a 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -123,6 +123,7 @@ async def current_user( ip_address = get_client_ip(request) user_id = None globus_introspect = None + exc_msg = "" try: if GLOBUS_RS_ID and GLOBUS_RS_SECRET and GLOBUS_RS_SCOPE_SUFFIX: try: @@ -130,13 +131,15 @@ async def current_user( user_id = await self.adapter.get_current_user_globus(token, ip_address, globus_introspect) except Exception as globus_exc: logging.getLogger().exception("Globus error:", exc_info=globus_exc) + exc_msg = f"Globus authentication failed: {str(globus_exc)}. || " if not user_id: user_id = await self.adapter.get_current_user(token, ip_address) except Exception as exc: - logging.getLogger().exception(f"Error parsing IRI_API_PARAMS: ", exc_info=exc) - raise HTTPException(status_code=401, detail="Invalid or malformed Authorization parameters") from exc + logging.getLogger().exception("Facility Specific auth failed: ", exc_info=exc) + exc_msg += f"Facility Specific authentication failed: {str(exc)}" + raise HTTPException(status_code=401, detail=exc_msg) from exc if not user_id: - raise HTTPException(status_code=403, detail="Unauthorized access") + raise HTTPException(status_code=403, detail="Authentication succeeded but no user ID was identified. Contact Facility Admin.") user = await self.adapter.get_user( user_id=user_id, From 8fb98a4833e0f6d981392a6d91cd900c1ec6cdc9 Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Tue, 31 Mar 2026 12:31:01 -0500 Subject: [PATCH 129/133] More clear error message for missing session info --- app/routers/iri_router.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/app/routers/iri_router.py b/app/routers/iri_router.py index f839416a..8abcc4b5 100644 --- a/app/routers/iri_router.py +++ b/app/routers/iri_router.py @@ -107,9 +107,16 @@ async def get_globus_info(self, api_key: str) -> dict: if GLOBUS_SCOPE not in token_scope: raise Exception(f"Token missing required scope: {GLOBUS_SCOPE}") - session_info = introspect.get("session_info", {}) - if not session_info or not session_info.get("authentications", {}): - raise Exception(f"Empty session_info.authentications block") + session_info = introspect.get("session_info") + + if not session_info: + raise Exception("No recent login was found in the token (missing session_info). " + "Please re-authenticate to obtain a valid session.") + + authentications = session_info.get("authentications") + if not authentications: + raise Exception("No recent login was found in the token (empty session_info.authentications). " + "Please re-authenticate to obtain a valid session.") return introspect From 7667d954326a3d99a6330d5bcb43421e4bc6aeff Mon Sep 17 00:00:00 2001 From: Justas Balcas Date: Thu, 2 Apr 2026 07:06:08 -0500 Subject: [PATCH 130/133] [Feat] Dynamic URI generated from forwarded headers (Kong, Nginx,...) Introduce request-scoped URL derived from forwarded headers (host, proto, prefix). This makes API responses proxy aware. --- app/main.py | 17 ++++++++++++++++- app/request_context.py | 30 ++++++++++++++++++++++++++++++ app/routers/account/models.py | 10 +++++----- app/routers/facility/models.py | 6 +++--- app/routers/status/models.py | 14 +++++++------- app/routers/task/models.py | 4 ++-- app/types/base.py | 3 ++- 7 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 app/request_context.py diff --git a/app/main.py b/app/main.py index 62147590..2019bece 100644 --- a/app/main.py +++ b/app/main.py @@ -2,8 +2,9 @@ """Main API application""" import logging -from fastapi import FastAPI +from fastapi import FastAPI, Request from opentelemetry import trace +from starlette.middleware.base import BaseHTTPMiddleware from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ConsoleSpanExporter, BatchSpanProcessor, SimpleSpanProcessor @@ -20,6 +21,7 @@ from app.routers.task import task from . import config +from .request_context import set_api_url_base, _api_url_base logging.basicConfig( level=logging.INFO, @@ -48,6 +50,19 @@ APP = FastAPI(servers=[{"url": config.API_URL_ROOT}], **config.API_CONFIG) + +class _ExternalRequestContextMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + token = _api_url_base.set(None) + try: + set_api_url_base(request) + return await call_next(request) + finally: + _api_url_base.reset(token) + + +APP.add_middleware(_ExternalRequestContextMiddleware) + if config.OPENTELEMETRY_ENABLED: FastAPIInstrumentor.instrument_app(APP) diff --git a/app/request_context.py b/app/request_context.py new file mode 100644 index 00000000..cb35184b --- /dev/null +++ b/app/request_context.py @@ -0,0 +1,30 @@ +"""Per-request URL context derived from forwarding headers. (e.g. for Kong or other API gateways)""" +from contextvars import ContextVar + +from fastapi import Request + +from . import config + +_api_url_base: ContextVar[str | None] = ContextVar("_api_url_base", default=None) + + +def set_api_url_base(request: Request) -> None: + """Set the per-request API URL base from forwarding headers.""" + host = (request.headers.get("x-forwarded-host") or + request.headers.get("host", "")).split(",")[0].strip() + proto = (request.headers.get("x-forwarded-proto") or + request.url.scheme).split(",")[0].strip() + prefix = (request.headers.get("x-forwarded-prefix") + or request.headers.get("x-script-name") + or "").rstrip("/") + api_url = config.API_URL.strip("/") + if host: + _api_url_base.set(f"{proto}://{host}{prefix}/{api_url}") + + +def get_url_prefix() -> str: + """Return the per-request API URL base, or fall back to static config.""" + value = _api_url_base.get() + if value: + return value + return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}" diff --git a/app/routers/account/models.py b/app/routers/account/models.py index 0c6beba2..e4c6d4f7 100644 --- a/app/routers/account/models.py +++ b/app/routers/account/models.py @@ -2,7 +2,7 @@ import datetime from pydantic import Field, computed_field, field_validator -from ... import config +from ...request_context import get_url_prefix from ...types.base import IRIBaseModel from ...types.scalars import AllocationUnit @@ -26,7 +26,7 @@ def _norm_dt_field(cls, v): @property def self_uri(self) -> str: """Return the URI for this project resource.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/projects/{self.id}" + return f"{get_url_prefix()}/account/projects/{self.id}" class AllocationEntry(IRIBaseModel): @@ -54,13 +54,13 @@ class ProjectAllocation(IRIBaseModel): @property def project_uri(self) -> str: """Return the URI for the associated project resource.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/projects/{self.project_id}" + return f"{get_url_prefix()}/account/projects/{self.project_id}" @computed_field(description="URI of the associated capability resource") @property def capability_uri(self) -> str: """Return the URI for the associated capability.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{self.capability_id}" + return f"{get_url_prefix()}/account/capabilities/{self.capability_id}" class UserAllocation(IRIBaseModel): @@ -79,4 +79,4 @@ class UserAllocation(IRIBaseModel): @property def project_allocation_uri(self) -> str: """Return the URI for the associated project allocation.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/projects/{self.project_id}/project_allocations/{self.project_allocation_id}" + return f"{get_url_prefix()}/account/projects/{self.project_id}/project_allocations/{self.project_allocation_id}" diff --git a/app/routers/facility/models.py b/app/routers/facility/models.py index 5d306751..a2527abd 100644 --- a/app/routers/facility/models.py +++ b/app/routers/facility/models.py @@ -1,7 +1,7 @@ """Facility-related models.""" from pydantic import Field, HttpUrl, computed_field -from ... import config +from ...request_context import get_url_prefix from ...types.base import NamedObject @@ -26,7 +26,7 @@ def _self_path(self) -> str: @property def resource_uris(self) -> list[str]: """Return the list of resource URIs for this site.""" - return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{resource_id}" for resource_id in self.resource_ids] + return [f"{get_url_prefix()}/status/resources/{resource_id}" for resource_id in self.resource_ids] @classmethod def find(cls, items, name=None, description=None, modified_since=None, short_name=None, country_name=None): @@ -53,4 +53,4 @@ def _self_path(self) -> str: @property def site_uris(self) -> list[str]: """Return the list of site URIs for this facility.""" - return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/facility/sites/{site_id}" for site_id in self.site_ids] + return [f"{get_url_prefix()}/facility/sites/{site_id}" for site_id in self.site_ids] diff --git a/app/routers/status/models.py b/app/routers/status/models.py index 357db3a6..4fda8015 100644 --- a/app/routers/status/models.py +++ b/app/routers/status/models.py @@ -4,7 +4,7 @@ from pydantic import Field, computed_field, field_validator -from ... import config +from ...request_context import get_url_prefix from ...types.base import NamedObject @@ -43,13 +43,13 @@ def _self_path(self) -> str: @property def site_uri(self) -> str: """Return the site URI for this resource.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/facility/sites/{self.site_id}" + return f"{get_url_prefix()}/facility/sites/{self.site_id}" @computed_field(description="The list of capabilities in this resource") @property def capability_uris(self) -> list[str]: """Return the list of capability URIs for this resource.""" - return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/account/capabilities/{e}" for e in self.capability_ids] + return [f"{get_url_prefix()}/account/capabilities/{e}" for e in self.capability_ids] @classmethod def find(cls, items, name=None, description=None, modified_since=None, group=None, resource_type=None, current_status=None, capability=None, site_id=None) -> list: @@ -89,13 +89,13 @@ def _norm_dt_field(cls, v): @property def resource_uri(self) -> str: """Return the resource URI for this event.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{self.resource_id}" + return f"{get_url_prefix()}/status/resources/{self.resource_id}" @computed_field(description="The event's incident") @property def incident_uri(self) -> str | None: """Return the incident URI for this event.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/incidents/{self.incident_id}" if self.incident_id else None + return f"{get_url_prefix()}/status/incidents/{self.incident_id}" if self.incident_id else None @classmethod def find(cls, items, incident_id=None, name=None, description=None, modified_since=None, resource_id=None, status=None, from_=None, to=None, time_=None) -> list: @@ -162,13 +162,13 @@ def _norm_dt_field(cls, v): @property def event_uris(self) -> list[str]: """Return the list of event URIs for this incident.""" - return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/events/{e}" for e in self.event_ids] + return [f"{get_url_prefix()}/status/events/{e}" for e in self.event_ids] @computed_field(description="The list of resources that may be impacted by this incident") @property def resource_uris(self) -> list[str]: """Return the list of resource URIs for this incident.""" - return [f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/status/resources/{r}" for r in self.resource_ids] + return [f"{get_url_prefix()}/status/resources/{r}" for r in self.resource_ids] @classmethod def find(cls, items, name=None, description=None, modified_since=None, status=None, type_=None, from_=None, to=None, time_=None, resource_id=None, resolution=None) -> list: diff --git a/app/routers/task/models.py b/app/routers/task/models.py index 7a13ac0e..ee85b979 100644 --- a/app/routers/task/models.py +++ b/app/routers/task/models.py @@ -2,7 +2,7 @@ import enum from pydantic import BaseModel, Field, computed_field -from ... import config +from ...request_context import get_url_prefix class TaskSubmitResponse(BaseModel): @@ -13,7 +13,7 @@ class TaskSubmitResponse(BaseModel): @property def task_uri(self) -> str: """Return the URI for this task.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}/task/{self.task_id}" + return f"{get_url_prefix()}/task/{self.task_id}" class TaskStatus(str, enum.Enum): diff --git a/app/types/base.py b/app/types/base.py index e25734a3..c5619599 100644 --- a/app/types/base.py +++ b/app/types/base.py @@ -5,6 +5,7 @@ from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator, model_serializer from .. import config +from ..request_context import get_url_prefix from .scalars import StrictDateTime @@ -59,7 +60,7 @@ def _norm_dt_field(cls, v): @property def self_uri(self) -> str: """Computed self URI property.""" - return f"{config.API_URL_ROOT}{config.API_PREFIX}{config.API_URL}{self._self_path()}" + return f"{get_url_prefix()}{self._self_path()}" name: str|None = Field(default=None, description="The long name of the object.", example="Perlmutter GPU") description: str|None = Field(default=None, description="Human-readable description of the object.", example="High-performance GPU compute resource") From d5a6c56ce2caadb79c6fe5ee5be0d6d559c9eda9 Mon Sep 17 00:00:00 2001 From: BenoitCote Date: Thu, 9 Apr 2026 08:26:40 -0700 Subject: [PATCH 131/133] default historical now True by default for get_job --- app/routers/compute/compute.py | 2 +- app/routers/compute/facility_adapter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index f8a4cb53..71c80b45 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -94,7 +94,7 @@ async def get_job_status( job_id: str, request: Request, user: User = Depends(router.current_user), - historical: StrictHTTPBool | None = Query(default=False, description="Whether to include historical jobs. Defaults to false"), + historical: StrictHTTPBool | None = Query(default=True, description="Whether to include historical jobs. Defaults to true"), include_spec: StrictHTTPBool | None = Query(default=False, description="Whether to include the job specification. Defaults to false"), _forbid=Depends(forbidExtraQueryParams("historical", "include_spec")), ): diff --git a/app/routers/compute/facility_adapter.py b/app/routers/compute/facility_adapter.py index a8cbf9aa..32adbdbf 100644 --- a/app/routers/compute/facility_adapter.py +++ b/app/routers/compute/facility_adapter.py @@ -26,7 +26,7 @@ async def get_job( resource: status_models.Resource, user: User, job_id: str, - historical: bool = False, + historical: bool = True, include_spec: bool = False, ) -> compute_models.Job: pass From 2cee5446f011b9c011cfc85267037f3d6c74dc22 Mon Sep 17 00:00:00 2001 From: Amith Murthy Date: Mon, 4 May 2026 10:21:40 -0700 Subject: [PATCH 132/133] worked through integration issues as a result of upstream merge --- VALIDATION.MD | 23 ------------------ app/s3df/account_adapter.py | 33 ++++++++++++++------------ app/s3df/auth/authenticated_adapter.py | 7 +++++- app/s3df/compute_adapter.py | 3 +-- 4 files changed, 25 insertions(+), 41 deletions(-) delete mode 100644 VALIDATION.MD diff --git a/VALIDATION.MD b/VALIDATION.MD deleted file mode 100644 index 26f77ef6..00000000 --- a/VALIDATION.MD +++ /dev/null @@ -1,23 +0,0 @@ -# API Validation with Schemathesis - -On every pull request or push to `main` branch, Github Actions run the following steps below that validates an IRI Facility API implementation against OpenAPI spec using Schemathesis. - -1. Builds the Facility API Docker image from Dockerfile. -2. Runs the API container with demo adapter. -3. Waits for `/openapi.json` to become available on localhost:8000. -4. Runs Schemathesis validation twice: - - Against Facilities API’s OpenAPI spec. (http://localhost:8000/openapi.json) - - Against the official IRI Facility API OpenAPI spec. (https://github.com/doe-iri/iri-facility-api-docs/blob/main/specification/openapi/openapi_iri_facility_api_v1.json) -5. Fails the workflow if either validation fails. -6. Saves Schemathesis HTML/XML reports as artifacts (or saves it locally when run with `act`). -7. Dumps API container logs and do clean up to stop container. - -## Running locally - -```bash -act -W .github/workflows/api-validator.yml -s GITHUB_TOKEN= -``` - -## Known issues - -Python implementation not fully aligns with the official Specification. Running against Official Spec will continue to fail, until Spec or Py implementation is fixed. diff --git a/app/s3df/account_adapter.py b/app/s3df/account_adapter.py index 106e2d8a..34a96b9d 100644 --- a/app/s3df/account_adapter.py +++ b/app/s3df/account_adapter.py @@ -18,6 +18,9 @@ from fastapi import HTTPException +from app.types.user import User +from app.types.models import Capability +from app.types.scalars import AllocationUnit from ..routers.account import models as account_models from ..routers.account import facility_adapter as account_adapter from app.s3df.auth.authenticated_adapter import S3DFAuthenticatedAdapter @@ -38,7 +41,7 @@ def __init__(self, coact_client: CoactClient | None = None): # AuthenticatedAdapter methods # ------------------------------------------------------------------------- - async def get_user(self, user_id: str, api_key: str, client_ip: str | None) -> account_models.User: + async def get_user(self, user_id: str, api_key: str, client_ip: str | None, globus_introspect: dict | None = None) -> User: """ coact.User → IRI.User mapping: - username → id @@ -47,7 +50,7 @@ async def get_user(self, user_id: str, api_key: str, client_ip: str | None) -> a coact_user = await self.coact_client.get_user(user_id) if not coact_user: raise HTTPException(status_code=403, detail="User not authorized") - return account_models.User( + return User( id=coact_user["username"], name=coact_user.get("fullname", user_id), api_key=api_key, @@ -58,7 +61,7 @@ async def get_user(self, user_id: str, api_key: str, client_ip: str | None) -> a # AccountFacilityAdapter methods # ------------------------------------------------------------------------- - async def get_capabilities(self) -> list[account_models.Capability]: + async def get_capabilities(self) -> list[Capability]: """ coact.Cluster → IRI.Capability (compute) Static storage types → IRI.Capability (storage) @@ -70,35 +73,35 @@ async def get_capabilities(self) -> list[account_models.Capability]: for cluster in clusters: gpu_info = f", {cluster['nodegpucount']} GPUs" if cluster.get('nodegpucount', 0) > 0 else "" - capabilities.append(account_models.Capability( + capabilities.append(Capability( id=cluster["name"], name=f"{cluster['name'].upper()} ({cluster['nodecpucount']} CPUs{gpu_info}, {cluster['nodememgb']}GB/node)", - units=[account_models.AllocationUnit.node_hours] + units=[AllocationUnit.node_hours] )) # Add storage capabilities # capabilities.extend([ - # account_models.Capability( + # Capability( # id="sdf-data", # name="S3DF Data Storage - /sdf/data", - # units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes] + # units=[AllocationUnit.bytes, AllocationUnit.inodes] # ), - # account_models.Capability( + # Capability( # id="sdf-group", # name="S3DF Group Storage - /sdf/group", - # units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes] + # units=[AllocationUnit.bytes, AllocationUnit.inodes] # ), - # account_models.Capability( + # Capability( # id="sdf-scratch", # name="S3DF Scratch Storage - /sdf/scratch", - # units=[account_models.AllocationUnit.bytes, account_models.AllocationUnit.inodes] + # units=[AllocationUnit.bytes, AllocationUnit.inodes] # ), # ]) return capabilities - async def get_projects(self, user: account_models.User) -> list[account_models.Project]: + async def get_projects(self, user: User) -> list[account_models.Project]: """ coact.Repo → IRI.Project mapping: - Id → id @@ -127,7 +130,7 @@ async def get_projects(self, user: account_models.User) -> list[account_models.P async def get_project_allocations( self, project: account_models.Project, - user: account_models.User + user: User ) -> list[account_models.ProjectAllocation]: """ coact.RepoComputeAllocation → IRI.ProjectAllocation (node_hours) @@ -153,7 +156,7 @@ async def get_project_allocations( entries=[account_models.AllocationEntry( allocation=comp_alloc.get("allocated", 0), usage= overall_usage.get("resourceHours", 0) if overall_usage else 0, - unit=account_models.AllocationUnit.node_hours + unit=AllocationUnit.node_hours )] )) @@ -162,7 +165,7 @@ async def get_project_allocations( async def get_user_allocations( self, - user: account_models.User, + user: User, project_allocation: account_models.ProjectAllocation ) -> list[account_models.UserAllocation]: """ diff --git a/app/s3df/auth/authenticated_adapter.py b/app/s3df/auth/authenticated_adapter.py index 5e10fd86..b7bdfb61 100644 --- a/app/s3df/auth/authenticated_adapter.py +++ b/app/s3df/auth/authenticated_adapter.py @@ -23,7 +23,7 @@ class S3DFAuthenticatedAdapter: """Mixin: implements AuthenticatedAdapter.get_current_user via Dex JWT + CoAct.""" - async def get_current_user(self, api_key: str, client_ip: str | None) -> str: + async def get_current_user(self, api_key: str, client_ip: str | None, globus_introspect: dict | None = None) -> str: if not api_key: raise HTTPException(status_code=401, detail="Missing Authorization header") @@ -36,3 +36,8 @@ async def get_current_user(self, api_key: str, client_ip: str | None) -> str: # raise HTTPException(status_code=403, detail="User not authorized") return username + + async def get_current_user_globus(self, api_key: str, client_ip: str | None, globus_introspect: dict | None) -> str: + # Not implemented currently, return 501: Not Implemented + + raise HTTPException(status_code=501, detail="Globus authentication not implemented yet") diff --git a/app/s3df/compute_adapter.py b/app/s3df/compute_adapter.py index dee75105..037152fa 100644 --- a/app/s3df/compute_adapter.py +++ b/app/s3df/compute_adapter.py @@ -18,7 +18,6 @@ import logging import base64 from typing import Optional -from ..routers.compute import models as compute_models from ..routers.compute import facility_adapter as compute_adapter from ..routers.compute.models import JobState from app.s3df.auth.authenticated_adapter import S3DFAuthenticatedAdapter @@ -199,7 +198,7 @@ class SLACComputeAdapter(S3DFAuthenticatedAdapter, compute_adapter.FacilityAdapt # -- AuthenticatedAdapter methods --------------------------------------- - async def get_user(self, user_id: str, api_key: str, client_ip: str | None): + async def get_user(self, user_id: str, api_key: str, client_ip: str | None, globus_introspect: dict | None = None): """ Return a minimal user object. The unix_username is the critical field — it becomes the `sun` claim in the Slurm JWT. From fd81babfd93b7c658d66a73dac5e670c47640a8d Mon Sep 17 00:00:00 2001 From: Amith Murthy Date: Mon, 4 May 2026 10:33:33 -0700 Subject: [PATCH 133/133] removed changes to compute.py --- app/routers/compute/compute.py | 59 ++++------------------------------ 1 file changed, 7 insertions(+), 52 deletions(-) diff --git a/app/routers/compute/compute.py b/app/routers/compute/compute.py index 9d68155a..fb29dd88 100644 --- a/app/routers/compute/compute.py +++ b/app/routers/compute/compute.py @@ -1,8 +1,6 @@ """Compute resource API router""" -from typing import Annotated, List - -from fastapi import Depends, Form, HTTPException, Query, Request, status +from fastapi import Depends, HTTPException, Query, Request, status from ...types.http import forbidExtraQueryParams from ...types.scalars import StrictHTTPBool @@ -13,14 +11,6 @@ from ..status.status import router as status_router from . import facility_adapter, models - -async def _lookup_resource(resource_id: str): - if status_router.adapter is None: - return None - return await status_router.adapter.get_resource(resource_id) - - - router = iri_router.IriRouter( facility_adapter.FacilityAdapter, prefix="/compute", @@ -52,48 +42,13 @@ async def submit_job( This command will attempt to submit a job and return its id. """ # look up the resource (todo: maybe ensure it's available) - resource = await _lookup_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs return await router.adapter.submit_job(resource=resource, user=user, job_spec=job_spec) -@router.post( - "/job/script/{resource_id:str}", - dependencies=[Depends(router.current_user)], - response_model=models.Job, - response_model_exclude_unset=True, - responses=DEFAULT_RESPONSES, - operation_id="launchJobScript", -) -async def submit_job_path( - resource_id: str, - job_script_path : str, - request : Request, - args : Annotated[List[str], Form()] = [], - ): - """ - Submit a job on a compute resource - - - **resource**: the name of the compute resource to use - - **job_script_path**: path to the job script on the compute resource - - **args**: optional arguments to the job script - - This command will attempt to submit a job and return its id. - """ - user = await router.adapter.get_user(request.state.current_user_id, request.state.api_key, iri_router.get_client_ip(request)) - if not user: - raise HTTPException(status_code=404, detail="User not found") - - # look up the resource (todo: maybe ensure it's available) - resource = await _lookup_resource(resource_id) - - # the handler can use whatever means it wants to submit the job and then fill in its id - # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs - return await router.adapter.submit_job_script(resource, user, job_script_path, args) - - @router.put( "/job/{resource_id:str}/{job_id:str}", response_model=models.Job, @@ -119,7 +74,7 @@ async def update_job( """ # look up the resource (todo: maybe ensure it's available) - resource = await _lookup_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id) # the handler can use whatever means it wants to submit the job and then fill in its id # see: https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#submitting-jobs @@ -146,7 +101,7 @@ async def get_job_status( """Get a job's status""" # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await _lookup_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id) job = await router.adapter.get_job(resource=resource, user=user, job_id=job_id, historical=historical, include_spec=include_spec) @@ -175,7 +130,7 @@ async def get_job_statuses( """Get multiple jobs' statuses""" # look up the resource (todo: maybe ensure it's available) # This could be done via slurm (in the adapter) or via psij's "attach" (https://exaworks.org/psij-python/docs/v/0.9.11/user_guide.html#detaching-and-attaching-jobs) - resource = await _lookup_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id) jobs = await router.adapter.get_jobs(resource=resource, user=user, offset=offset, limit=limit, filters=filters, historical=historical, include_spec=include_spec) @@ -200,8 +155,8 @@ async def cancel_job( ): """Cancel a job""" # look up the resource (todo: maybe ensure it's available) - resource = await _lookup_resource(resource_id) + resource = await status_router.adapter.get_resource(resource_id) await router.adapter.cancel_job(resource=resource, user=user, job_id=job_id) - return None + return None \ No newline at end of file