From 748b0ee61f32e8907ed5f215b5ea2f34d7285650 Mon Sep 17 00:00:00 2001 From: Sergio Souza Costa Date: Sat, 13 Jun 2026 10:30:47 -0300 Subject: [PATCH 1/2] chore: housekeeping pre-release v0.1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix all LambdaGeo → DisSModel org URLs in README and docs/deployment.md - Rename services/frontend/ → services/jupyter/ (reflects actual JupyterLab service) - Update docker-compose.yml, docker-compose.prod.yml and CI to reference new path - Remove || true from lint/typecheck/security CI jobs; fix underlying errors: - ruff: remove unused imports, move mid-file imports to module top (main.py) - mypy: add type: ignore[misc] for redis-py sync/async stub ambiguity (worker.py) - bandit: add nosec B104/B310 for intentional container bind and guarded urlretrieve - Add PYTHONPATH=$PWD/services to mypy CI step so worker imports resolve - Add CHANGELOG.md for v0.1.0 Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/ci.yml | 12 ++--- CHANGELOG.md | 45 +++++++++++++++++++ README.md | 10 ++--- docker-compose.prod.yml | 2 +- docker-compose.yml | 2 +- docs/deployment.md | 2 +- services/api/main.py | 18 +++----- services/{frontend => jupyter}/Dockerfile | 0 .../{frontend => jupyter}/jupyter_config.py | 0 .../{frontend => jupyter}/requirements.txt | 0 services/worker/storage.py | 3 +- services/worker/worker.py | 4 +- 12 files changed, 69 insertions(+), 29 deletions(-) create mode 100644 CHANGELOG.md rename services/{frontend => jupyter}/Dockerfile (100%) rename services/{frontend => jupyter}/jupyter_config.py (100%) rename services/{frontend => jupyter}/requirements.txt (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baca9c7..31f09b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: run: pip install ruff - name: Run Lint - run: ruff check services/api/ services/worker/ --exclude __pycache__,*.apagar,*.pyc || true + run: ruff check services/api/ services/worker/ --exclude __pycache__,*.apagar,*.pyc typecheck: @@ -52,7 +52,7 @@ jobs: - name: Run Mypy run: | # Focamos apenas na lógica da API e Worker da plataforma - mypy --ignore-missing-imports services/api/main.py services/worker/worker.py || true + PYTHONPATH=$PWD/services mypy --ignore-missing-imports services/api/main.py services/worker/worker.py security: @@ -70,7 +70,7 @@ jobs: run: pip install bandit - name: Run Bandit - run: bandit -r services/api/ services/worker/ -ll -ii -f txt -o bandit-report.txt || true + run: bandit -r services/api/ services/worker/ -ll -ii -f txt -o bandit-report.txt - name: Upload Bandit Report uses: actions/upload-artifact@v4 @@ -193,11 +193,11 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - - name: Build Frontend + - name: Build Jupyter uses: docker/build-push-action@v5 with: - context: ./services/frontend - file: ./services/frontend/Dockerfile + context: ./services/jupyter + file: ./services/jupyter/Dockerfile push: false cache-from: type=gha cache-to: type=gha,mode=max diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..78f0e68 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,45 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.0] - 2026-06-13 + +First public release of the DisSModel Platform MVP. + +### Added + +- **REST API** (`services/api`) — FastAPI gateway with `X-API-Key` authentication applied to all routes; endpoints for job submission (async and inline), status polling, reproduction, publishing, model listing, file upload/download, and admin sync. +- **Worker** (`services/worker`) — Redis-queue consumer that delegates execution to `dissmodel.executor.runner.execute_lifecycle`; publishes profiling metrics and `ExperimentRecord` JSON to MinIO. +- **JupyterLab** (`services/jupyter`) — Containerised notebook environment (port 8888) with `dissmodel`, `ipyleaflet`, `ipywidgets`, and `folium` pre-installed. +- **Streamlit CA Explorer** (`services/streamlit-ca`) — Interactive explorer for Cellular Automata models. +- **Streamlit SysDyn Explorer** (`services/streamlit-sysdyn`) — Interactive explorer for System Dynamics models. +- **Nginx reverse proxy** (`services/nginx`) — Routes `/dissmodel/jupyter`, `/dissmodel/api`, `/dissmodel/minio` in production. +- **Docker Compose** — Development (`docker-compose.yml`) and production (`docker-compose.prod.yml`) stacks with Redis, MinIO, config-sync sidecar, and all services. +- **CI pipeline** (`.github/workflows/ci.yml`) — Lint (ruff), type-check (mypy), security scan (bandit), API tests, worker executor validation, and Docker build jobs; all gates are enforced (no `|| true` bypasses). +- **Dependabot** — Automated dependency updates for `services/api`, `services/worker`, and `services/jupyter`. +- **Presigned URL generation** — Local HMAC signing for MinIO download links without extra network round-trips. +- **Config-sync sidecar** — Git-backed model registry auto-pulled into all services at runtime. +- **Executor contract validation** (`scripts/validate_executors.py`) — CI step that checks registered executors comply with the dissmodel interface before tests run. + +### Fixed + +- Moved mid-file imports (`hmac`, `hashlib`, `urllib.parse`, `datetime.timezone`) to module top in `services/api/main.py`. +- Removed unused imports (`json`, `timedelta`, `S3Error`, `start_sync_scheduler`, `reproduce_experiment`, `run_experiment`) from `services/api/main.py` and `services/worker/storage.py`. +- Added `# type: ignore[misc]` for redis-py sync/async stub ambiguity in `services/worker/worker.py`. +- Added `# nosec B104` and `# nosec B310` for intentional false positives in bandit scan. + +### Changed + +- Renamed `services/frontend/` → `services/jupyter/` to reflect that the service is JupyterLab, not a generic web frontend. +- Updated all repository URLs from `LambdaGeo/dissmodel-platform` → `DisSModel/dissmodel-platform` in `README.md` and `docs/deployment.md`. +- Updated core library link from `LambdaGeo/dissmodel` → `DisSModel/dissmodel`. +- Updated organisation name from `LambdaGeo / INPE` → `DisSModel / INPE` in `README.md` contact section. +- `typecheck` CI job now sets `PYTHONPATH=$PWD/services` so worker imports resolve correctly without stubs. + +[Unreleased]: https://github.com/DisSModel/dissmodel-platform/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/DisSModel/dissmodel-platform/releases/tag/v0.1.0 diff --git a/README.md b/README.md index 6645b05..52a2d05 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ An integrated environment for developing and running geospatial models, featurin ```bash # 1. Clone the repository -git clone https://github.com/LambdaGeo/dissmodel-platform.git +git clone https://github.com/DisSModel/dissmodel-platform.git cd dissmodel-platform # 2. Configure environment variables @@ -175,13 +175,13 @@ MIT License — see [LICENSE](LICENSE) ## 🙏 Acknowledgements -- [DisSModel](https://github.com/LambdaGeo/dissmodel) — Core modelling library +- [DisSModel](https://github.com/DisSModel/dissmodel) — Core modelling library - [Jupyter Project](https://jupyter.org/) — Development environment - [MinIO](https://min.io/) — S3-compatible object storage - [Pangeo](https://pangeo.io/) — Inspiration for cloud-native architecture ## 📞 Contact -- **Organisation:** LambdaGeo / INPE -- **Issues:** https://github.com/LambdaGeo/dissmodel-platform/issues -- **Discussions:** https://github.com/LambdaGeo/dissmodel-platform/discussions +- **Organisation:** DisSModel / INPE +- **Issues:** https://github.com/DisSModel/dissmodel-platform/issues +- **Discussions:** https://github.com/DisSModel/dissmodel-platform/discussions diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 5071523..ac6d5a1 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -19,7 +19,7 @@ services: #sudo chmod -R 775 ./workspace jupyter: build: - context: ./services/frontend + context: ./services/jupyter dockerfile: Dockerfile container_name: dissmodel-jupyter restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index be6c280..5995613 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: jupyter: build: - context: ./services/frontend + context: ./services/jupyter dockerfile: Dockerfile container_name: dissmodel-jupyter restart: unless-stopped diff --git a/docs/deployment.md b/docs/deployment.md index b3d14d6..2d6caad 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -11,7 +11,7 @@ ```bash # Clonar -git clone https://github.com/LambdaGeo/dissmodel-platform.git +git clone https://github.com/DisSModel/dissmodel-platform.git cd dissmodel-platform # Configurar diff --git a/services/api/main.py b/services/api/main.py index 1ae1f4a..4d3a4e8 100644 --- a/services/api/main.py +++ b/services/api/main.py @@ -1,13 +1,15 @@ # services/api/main.py from __future__ import annotations +import hashlib +import hmac import io -import json import logging import os from contextlib import asynccontextmanager -from datetime import datetime, timedelta +from datetime import datetime, timezone from typing import Optional +from urllib.parse import quote, urlencode import redis from fastapi import Depends, FastAPI, File, Form, HTTPException, UploadFile @@ -15,10 +17,9 @@ from fastapi.security import APIKeyHeader from minio import Minio -from minio.error import S3Error -from worker.api_registry import list_models, load_model_spec, start_sync_scheduler, sync_configs -from worker.runner import build_record, build_record_inline, reproduce_experiment, run_experiment +from worker.api_registry import list_models, load_model_spec, sync_configs +from worker.runner import build_record, build_record_inline from dissmodel.executor.schemas import ExperimentRecord, InlineJobRequest, JobRequest, JobResponse # ── Logging ─────────────────────────────────────────────────────────────────── @@ -308,11 +309,6 @@ async def upload_dataset( } -import hmac -import hashlib -from urllib.parse import urlencode, quote -from datetime import timezone - def _presign_url(bucket: str, key: str, expires_seconds: int = 3600) -> str: """Gera presigned URL sem conexão de rede — cálculo local puro.""" server_url = os.getenv("MINIO_URL", "http://localhost:19000").rstrip("/") @@ -399,4 +395,4 @@ async def general_exception_handler(request, exc): if __name__ == "__main__": import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8000) \ No newline at end of file + uvicorn.run(app, host="0.0.0.0", port=8000) # nosec B104 \ No newline at end of file diff --git a/services/frontend/Dockerfile b/services/jupyter/Dockerfile similarity index 100% rename from services/frontend/Dockerfile rename to services/jupyter/Dockerfile diff --git a/services/frontend/jupyter_config.py b/services/jupyter/jupyter_config.py similarity index 100% rename from services/frontend/jupyter_config.py rename to services/jupyter/jupyter_config.py diff --git a/services/frontend/requirements.txt b/services/jupyter/requirements.txt similarity index 100% rename from services/frontend/requirements.txt rename to services/jupyter/requirements.txt diff --git a/services/worker/storage.py b/services/worker/storage.py index efc4fec..9d7b3ef 100644 --- a/services/worker/storage.py +++ b/services/worker/storage.py @@ -6,7 +6,6 @@ import os from minio import Minio -from minio.error import S3Error # ── Client ──────────────────────────────────────────────────────────────────── @@ -35,7 +34,7 @@ def download_to_file(uri: str, dest: str) -> str: if uri.startswith("http://") or uri.startswith("https://"): import urllib.request - urllib.request.urlretrieve(uri, dest) + urllib.request.urlretrieve(uri, dest) # nosec B310 return dest return uri # local path — return as-is diff --git a/services/worker/worker.py b/services/worker/worker.py index 1e5d320..3dd08da 100644 --- a/services/worker/worker.py +++ b/services/worker/worker.py @@ -86,10 +86,10 @@ def main() -> None: while True: try: # brpop blocks up to 5s and respects queue priority order - result = redis_client.brpop(QUEUES, timeout=5) + result = redis_client.brpop(QUEUES, timeout=5) # type: ignore[misc] if result: - _, experiment_id = result + _, experiment_id = result # type: ignore[misc] process_job(experiment_id) except KeyboardInterrupt: From b9613b95fb0aac9042ae8c9641ec5bba67864f27 Mon Sep 17 00:00:00 2001 From: Sergio Souza Costa Date: Sat, 13 Jun 2026 10:57:37 -0300 Subject: [PATCH 2/2] chore: reactivate dependabot with monthly schedule and add services/jupyter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Renamed dependabot.yml.disabled → dependabot.yml - Changed all pip and github-actions intervals: weekly → monthly - Added /services/jupyter to pip coverage Co-Authored-By: Claude Sonnet 4.6 --- .github/dependabot.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..57cdb58 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,29 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + commit-message: + prefix: "chore(deps):" + + - package-ecosystem: "pip" + directory: "/services/api" + schedule: + interval: "monthly" + commit-message: + prefix: "chore(deps):" + + - package-ecosystem: "pip" + directory: "/services/worker" + schedule: + interval: "monthly" + commit-message: + prefix: "chore(deps):" + + - package-ecosystem: "pip" + directory: "/services/jupyter" + schedule: + interval: "monthly" + commit-message: + prefix: "chore(deps):"