diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..57cdb58 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,29 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + commit-message: + prefix: "chore(deps):" + + - package-ecosystem: "pip" + directory: "/services/api" + schedule: + interval: "monthly" + commit-message: + prefix: "chore(deps):" + + - package-ecosystem: "pip" + directory: "/services/worker" + schedule: + interval: "monthly" + commit-message: + prefix: "chore(deps):" + + - package-ecosystem: "pip" + directory: "/services/jupyter" + schedule: + interval: "monthly" + commit-message: + prefix: "chore(deps):" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baca9c7..31f09b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: run: pip install ruff - name: Run Lint - run: ruff check services/api/ services/worker/ --exclude __pycache__,*.apagar,*.pyc || true + run: ruff check services/api/ services/worker/ --exclude __pycache__,*.apagar,*.pyc typecheck: @@ -52,7 +52,7 @@ jobs: - name: Run Mypy run: | # Focamos apenas na lógica da API e Worker da plataforma - mypy --ignore-missing-imports services/api/main.py services/worker/worker.py || true + PYTHONPATH=$PWD/services mypy --ignore-missing-imports services/api/main.py services/worker/worker.py security: @@ -70,7 +70,7 @@ jobs: run: pip install bandit - name: Run Bandit - run: bandit -r services/api/ services/worker/ -ll -ii -f txt -o bandit-report.txt || true + run: bandit -r services/api/ services/worker/ -ll -ii -f txt -o bandit-report.txt - name: Upload Bandit Report uses: actions/upload-artifact@v4 @@ -193,11 +193,11 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - - name: Build Frontend + - name: Build Jupyter uses: docker/build-push-action@v5 with: - context: ./services/frontend - file: ./services/frontend/Dockerfile + context: ./services/jupyter + file: ./services/jupyter/Dockerfile push: false cache-from: type=gha cache-to: type=gha,mode=max diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..78f0e68 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,45 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.0] - 2026-06-13 + +First public release of the DisSModel Platform MVP. + +### Added + +- **REST API** (`services/api`) — FastAPI gateway with `X-API-Key` authentication applied to all routes; endpoints for job submission (async and inline), status polling, reproduction, publishing, model listing, file upload/download, and admin sync. +- **Worker** (`services/worker`) — Redis-queue consumer that delegates execution to `dissmodel.executor.runner.execute_lifecycle`; publishes profiling metrics and `ExperimentRecord` JSON to MinIO. +- **JupyterLab** (`services/jupyter`) — Containerised notebook environment (port 8888) with `dissmodel`, `ipyleaflet`, `ipywidgets`, and `folium` pre-installed. +- **Streamlit CA Explorer** (`services/streamlit-ca`) — Interactive explorer for Cellular Automata models. +- **Streamlit SysDyn Explorer** (`services/streamlit-sysdyn`) — Interactive explorer for System Dynamics models. +- **Nginx reverse proxy** (`services/nginx`) — Routes `/dissmodel/jupyter`, `/dissmodel/api`, `/dissmodel/minio` in production. +- **Docker Compose** — Development (`docker-compose.yml`) and production (`docker-compose.prod.yml`) stacks with Redis, MinIO, config-sync sidecar, and all services. +- **CI pipeline** (`.github/workflows/ci.yml`) — Lint (ruff), type-check (mypy), security scan (bandit), API tests, worker executor validation, and Docker build jobs; all gates are enforced (no `|| true` bypasses). +- **Dependabot** — Automated dependency updates for `services/api`, `services/worker`, and `services/jupyter`. +- **Presigned URL generation** — Local HMAC signing for MinIO download links without extra network round-trips. +- **Config-sync sidecar** — Git-backed model registry auto-pulled into all services at runtime. +- **Executor contract validation** (`scripts/validate_executors.py`) — CI step that checks registered executors comply with the dissmodel interface before tests run. + +### Fixed + +- Moved mid-file imports (`hmac`, `hashlib`, `urllib.parse`, `datetime.timezone`) to module top in `services/api/main.py`. +- Removed unused imports (`json`, `timedelta`, `S3Error`, `start_sync_scheduler`, `reproduce_experiment`, `run_experiment`) from `services/api/main.py` and `services/worker/storage.py`. +- Added `# type: ignore[misc]` for redis-py sync/async stub ambiguity in `services/worker/worker.py`. +- Added `# nosec B104` and `# nosec B310` for intentional false positives in bandit scan. + +### Changed + +- Renamed `services/frontend/` → `services/jupyter/` to reflect that the service is JupyterLab, not a generic web frontend. +- Updated all repository URLs from `LambdaGeo/dissmodel-platform` → `DisSModel/dissmodel-platform` in `README.md` and `docs/deployment.md`. +- Updated core library link from `LambdaGeo/dissmodel` → `DisSModel/dissmodel`. +- Updated organisation name from `LambdaGeo / INPE` → `DisSModel / INPE` in `README.md` contact section. +- `typecheck` CI job now sets `PYTHONPATH=$PWD/services` so worker imports resolve correctly without stubs. + +[Unreleased]: https://github.com/DisSModel/dissmodel-platform/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/DisSModel/dissmodel-platform/releases/tag/v0.1.0 diff --git a/README.md b/README.md index 6645b05..52a2d05 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ An integrated environment for developing and running geospatial models, featurin ```bash # 1. Clone the repository -git clone https://github.com/LambdaGeo/dissmodel-platform.git +git clone https://github.com/DisSModel/dissmodel-platform.git cd dissmodel-platform # 2. Configure environment variables @@ -175,13 +175,13 @@ MIT License — see [LICENSE](LICENSE) ## 🙏 Acknowledgements -- [DisSModel](https://github.com/LambdaGeo/dissmodel) — Core modelling library +- [DisSModel](https://github.com/DisSModel/dissmodel) — Core modelling library - [Jupyter Project](https://jupyter.org/) — Development environment - [MinIO](https://min.io/) — S3-compatible object storage - [Pangeo](https://pangeo.io/) — Inspiration for cloud-native architecture ## 📞 Contact -- **Organisation:** LambdaGeo / INPE -- **Issues:** https://github.com/LambdaGeo/dissmodel-platform/issues -- **Discussions:** https://github.com/LambdaGeo/dissmodel-platform/discussions +- **Organisation:** DisSModel / INPE +- **Issues:** https://github.com/DisSModel/dissmodel-platform/issues +- **Discussions:** https://github.com/DisSModel/dissmodel-platform/discussions diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 5071523..ac6d5a1 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -19,7 +19,7 @@ services: #sudo chmod -R 775 ./workspace jupyter: build: - context: ./services/frontend + context: ./services/jupyter dockerfile: Dockerfile container_name: dissmodel-jupyter restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index be6c280..5995613 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: jupyter: build: - context: ./services/frontend + context: ./services/jupyter dockerfile: Dockerfile container_name: dissmodel-jupyter restart: unless-stopped diff --git a/docs/deployment.md b/docs/deployment.md index b3d14d6..2d6caad 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -11,7 +11,7 @@ ```bash # Clonar -git clone https://github.com/LambdaGeo/dissmodel-platform.git +git clone https://github.com/DisSModel/dissmodel-platform.git cd dissmodel-platform # Configurar diff --git a/services/api/main.py b/services/api/main.py index 1ae1f4a..4d3a4e8 100644 --- a/services/api/main.py +++ b/services/api/main.py @@ -1,13 +1,15 @@ # services/api/main.py from __future__ import annotations +import hashlib +import hmac import io -import json import logging import os from contextlib import asynccontextmanager -from datetime import datetime, timedelta +from datetime import datetime, timezone from typing import Optional +from urllib.parse import quote, urlencode import redis from fastapi import Depends, FastAPI, File, Form, HTTPException, UploadFile @@ -15,10 +17,9 @@ from fastapi.security import APIKeyHeader from minio import Minio -from minio.error import S3Error -from worker.api_registry import list_models, load_model_spec, start_sync_scheduler, sync_configs -from worker.runner import build_record, build_record_inline, reproduce_experiment, run_experiment +from worker.api_registry import list_models, load_model_spec, sync_configs +from worker.runner import build_record, build_record_inline from dissmodel.executor.schemas import ExperimentRecord, InlineJobRequest, JobRequest, JobResponse # ── Logging ─────────────────────────────────────────────────────────────────── @@ -308,11 +309,6 @@ async def upload_dataset( } -import hmac -import hashlib -from urllib.parse import urlencode, quote -from datetime import timezone - def _presign_url(bucket: str, key: str, expires_seconds: int = 3600) -> str: """Gera presigned URL sem conexão de rede — cálculo local puro.""" server_url = os.getenv("MINIO_URL", "http://localhost:19000").rstrip("/") @@ -399,4 +395,4 @@ async def general_exception_handler(request, exc): if __name__ == "__main__": import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=8000) # nosec B104 \ No newline at end of file diff --git a/services/frontend/Dockerfile b/services/jupyter/Dockerfile similarity index 100% rename from services/frontend/Dockerfile rename to services/jupyter/Dockerfile diff --git a/services/frontend/jupyter_config.py b/services/jupyter/jupyter_config.py similarity index 100% rename from services/frontend/jupyter_config.py rename to services/jupyter/jupyter_config.py diff --git a/services/frontend/requirements.txt b/services/jupyter/requirements.txt similarity index 100% rename from services/frontend/requirements.txt rename to services/jupyter/requirements.txt diff --git a/services/worker/storage.py b/services/worker/storage.py index efc4fec..9d7b3ef 100644 --- a/services/worker/storage.py +++ b/services/worker/storage.py @@ -6,7 +6,6 @@ import os from minio import Minio -from minio.error import S3Error # ── Client ──────────────────────────────────────────────────────────────────── @@ -35,7 +34,7 @@ def download_to_file(uri: str, dest: str) -> str: if uri.startswith("http://") or uri.startswith("https://"): import urllib.request - urllib.request.urlretrieve(uri, dest) + urllib.request.urlretrieve(uri, dest) # nosec B310 return dest return uri # local path — return as-is diff --git a/services/worker/worker.py b/services/worker/worker.py index 1e5d320..3dd08da 100644 --- a/services/worker/worker.py +++ b/services/worker/worker.py @@ -86,10 +86,10 @@ def main() -> None: while True: try: # brpop blocks up to 5s and respects queue priority order - result = redis_client.brpop(QUEUES, timeout=5) + result = redis_client.brpop(QUEUES, timeout=5) # type: ignore[misc] if result: - _, experiment_id = result + _, experiment_id = result # type: ignore[misc] process_job(experiment_id) except KeyboardInterrupt: