From 8a770826f0b7b48549ec8693aed0078e56f7f062 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 28 Jun 2026 12:04:14 +0530 Subject: [PATCH 1/6] Fast api Port Change --- AGENT.md | 2 +- Dockerfile | 4 +- README.md | 10 +- diagram.html | 610 ++++++++++++++++++ docker-compose.prod.yml | 14 +- docker-compose.pull.yml | 12 +- docker-compose.yml | 12 +- docker-entrypoint.sh | 4 +- docs/MCP.md | 14 +- docs/OPS.md | 2 +- docs/README.md | 4 +- scripts/local-prod.sh | 10 +- scripts/local-run-common.sh | 10 +- scripts/local-run.ps1 | 10 +- scripts/local-run.sh | 10 +- services/AiService/README.md | 6 +- .../src/AiService.Api/appsettings.json | 2 +- .../AiService.Tools/Options/FastApiOptions.cs | 2 +- services/Bff/src/Bff.Api/appsettings.json | 2 +- .../Options/UpstreamOptions.cs | 2 +- services/FileService/README.md | 6 +- .../src/FileService.Api/appsettings.json | 2 +- .../Options/ReportApiOptions.cs | 2 +- .../ServiceRegistrationValidationTests.cs | 2 +- services/IntegrationsService/README.md | 2 +- .../Google/FastApiPythonBridge.cs | 2 +- .../src/ReportService.Api/appsettings.json | 2 +- .../Options/FastApiOptions.cs | 2 +- .../ServiceRegistrationTestEnvironment.cs | 2 +- .../commands/report_build.py | 2 +- tests/test_ai_migration_coverage.py | 2 +- web/src/lib/mcpClientConfig.test.ts | 2 +- web/src/lib/mcpClientConfig.ts | 4 +- 33 files changed, 692 insertions(+), 82 deletions(-) create mode 100644 diagram.html diff --git a/AGENT.md b/AGENT.md index 8cc715a3..73225adf 100644 --- a/AGENT.md +++ b/AGENT.md @@ -21,7 +21,7 @@ Developer reference for agents and contributors. User-facing overview: [README.m - `web/src/` -- React SPA (`AppRoutes.tsx`, `views/`, `components/`); pipeline UI: `PipelineRunnerFab`, `PipelineContext` - `alembic/` -- schema migrations -**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, FileService `:8080`, Data `:8091`, AiService `:8092`, ReportService `:8094`, FastAPI `:8001`, BFF `:8090`, Vite `:3000`; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web and .NET checks — mirrors CI; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`. +**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, FileService `:8080`, Data `:8091`, AiService `:8092`, ReportService `:8094`, FastAPI `:8096`, BFF `:8090`, Vite `:3000`; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web and .NET checks — mirrors CI; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`. **JavaScript crawl (optional):** Config keys `crawl_render_mode` (`static` | `javascript` | `auto`) and `crawl_js_*` in pipeline config / `pipelineConfigSchema.ts`. JS/auto crawls can capture browser console errors and uncaught exceptions (`crawl_js_capture_console`, stored under `page_analysis.browser`). **Auto mode** uses static-first fetch, pre-parse SPA heuristics (`needs_js_render`), then post-parse low-outlink fallback (`needs_js_render_after_parse`) in `crawler.py`. **Preflight:** `GET /api/crawl/browser-status` (localhost) spawns Python `browser_status()`; Run audit settings/run validation calls it when render mode is `javascript` or `auto`. Browser deps: Playwright from `requirements.txt` (installed by `./local-run setup` and `./local-test`). Runtime needs Chromium on `PATH` or `CHROME_PATH` (Docker sets `CHROME_PATH=/usr/bin/chromium`). Integration tests: `@pytest.mark.browser` — excluded by default in `pytest.ini`; Docker CI runs `tests/test_crawl_fetchers.py` and `tests/test_crawler_browser_e2e.py -m browser`; locally `./local-test browser`. diff --git a/Dockerfile b/Dockerfile index d2dc985d..5fcd75b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -# WebsiteProfiling: FastAPI (port 8001) + Python worker + pipeline. +# WebsiteProfiling: FastAPI (port 8096) + Python worker + pipeline. # Web UI is a separate image: web/Dockerfile (Vite SPA + nginx). # Build from repository root: docker build -t website-profiling . # BuildKit cache mounts (default in Docker Desktop) reuse pip/npm downloads across rebuilds. @@ -72,6 +72,6 @@ ENV NODE_ENV=production # Persisted data directory (secrets + shadow config) RUN mkdir -p /data && chmod +x /app/docker-entrypoint.sh -EXPOSE 8001 +EXPOSE 8096 CMD ["/app/docker-entrypoint.sh"] diff --git a/README.md b/README.md index e5c61239..37f11836 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ flowchart TB subgraph BffRoutes["BFF upstream routing"] direction LR - BFF --> FastAPI["fastapi :8001
Crawl · jobs · pipeline settings"] + BFF --> FastAPI["fastapi :8096
Crawl · jobs · pipeline settings"] BFF --> Report["report :8094
Report build · orchestration"] BFF --> Integrations["integrations :8093
Google/Bing · GSC/GA4 · keywords"] BFF --> Ai["ai :8092
Chat · LLM · secrets · MCP"] @@ -208,7 +208,7 @@ flowchart TB ``` WebsiteProfiling/ ├── src/website_profiling/ # Python audit engine (CLI: python -m src) -│ ├── api/ # FastAPI app (uvicorn :8001) +│ ├── api/ # FastAPI app (uvicorn :8096) │ ├── worker/ # Background pipeline job runner │ ├── crawl/ # Crawler, fetchers, JS rendering │ ├── reporting/ # Report builder (Python bridge; native build in ReportService) @@ -291,7 +291,7 @@ Build and run the full dev stack from source: docker compose up --build ``` -Services: **postgres**, **fastapi** (`:8001`, internal), **worker**, **report** (`:8094`, internal), **integrations** (`:8093`, internal), **ai** (`:8092`, internal), **data** (`:8091`, internal), **bff** (`:8090`), **web** (`:3000`), **files** (`:8080`, internal). +Services: **postgres**, **fastapi** (`:8096`, internal), **worker**, **report** (`:8094`, internal), **integrations** (`:8093`, internal), **ai** (`:8092`, internal), **data** (`:8091`, internal), **bff** (`:8090`), **web** (`:3000`), **files** (`:8080`, internal). Open [http://localhost:3000/home](http://localhost:3000/home). The browser talks only to the **BFF** (`:8090`); the BFF proxies to FastAPI (crawl/pipeline), IntegrationsService (Google/Bing), AiService (AI/secrets), Data (report reads), and FileService (PDF/workbook export). @@ -308,7 +308,7 @@ Production deployment: `docker-compose.prod.yml` — set `POSTGRES_USER`, `POSTG ./local-prod # Same DB, Vite production build + preview (no hot reload) ``` -`./local-run` starts (in order): **FileService** `:8080`, **Data** `:8091`, **AiService** `:8092` (MCP HTTP enabled), **ReportService** `:8094`, **IntegrationsService** `:8093`, **ConfigService** `:8095`, **FastAPI** `:8001` (Python bridge), **BFF** `:8090`, and **Vite** `:3000`. Use `localhost` (not `127.0.0.1`) for pipeline APIs so CORS and cookies match the BFF origin. +`./local-run` starts (in order): **FileService** `:8080`, **Data** `:8091`, **AiService** `:8092` (MCP HTTP enabled), **ReportService** `:8094`, **IntegrationsService** `:8093`, **ConfigService** `:8095`, **FastAPI** `:8096` (Python bridge), **BFF** `:8090`, and **Vite** `:3000`. Use `localhost` (not `127.0.0.1`) for pipeline APIs so CORS and cookies match the BFF origin. Default local `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling` (Docker Compose dev stack uses `profiling:profiling`). @@ -325,7 +325,7 @@ With `./local-run`, each service runs in **Development** and exposes interactive | **ReportService** | 8094 | [http://localhost:8094/docs](http://localhost:8094/docs) | `/swagger/v1/swagger.json` | | **ConfigService** | 8095 | [http://localhost:8095/docs](http://localhost:8095/docs) | `/swagger/v1/swagger.json` | | **FileService** | 8080 | [http://localhost:8080/docs](http://localhost:8080/docs) | `/swagger/v1/swagger.json` | -| **Python bridge** (FastAPI) | 8001 | [http://localhost:8001/docs](http://localhost:8001/docs) | `/openapi.json` | +| **Python bridge** (FastAPI) | 8096 | [http://localhost:8096/docs](http://localhost:8096/docs) | `/openapi.json` | Swagger UI is disabled when `ASPNETCORE_ENVIRONMENT=Production`. The legacy `web/openapi.json` reflects the old monolithic FastAPI app and is not a complete map of the current C# services. diff --git a/diagram.html b/diagram.html new file mode 100644 index 00000000..96b32eef --- /dev/null +++ b/diagram.html @@ -0,0 +1,610 @@ + + + + + + Interactive Architecture Canvas + + + + + + + +
+ + +
+
+ +
+ + + + + + + + + + + +
+
+
+ + +
+ + +
+ +
+ + + +
+ + + + \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b8c76630..9ba277cc 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -40,7 +40,7 @@ services: volumes: - profiling-data:/data healthcheck: - test: ['CMD', 'node', '-e', "require('http').get('http://127.0.0.1:8001/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] + test: ['CMD', 'node', '-e', "require('http').get('http://127.0.0.1:8096/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s timeout: 5s retries: 3 @@ -71,7 +71,7 @@ services: dockerfile: AiService/Dockerfile environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 ASPNETCORE_URLS: http://+:8092 WP_MCP_HTTP: "1" depends_on: @@ -94,7 +94,7 @@ services: environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} ASPNETCORE_URLS: http://+:8093 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 USE_FASTAPI_PYTHON_BRIDGE: "1" ASPNETCORE_ENVIRONMENT: Production AUTH_SECRET: ${AUTH_SECRET:?set AUTH_SECRET} @@ -120,7 +120,7 @@ services: environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} ASPNETCORE_URLS: http://+:8094 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 INTEGRATIONS_SERVICE_URL: http://integrations:8093 AISERVICE_URL: http://ai:8092 REPORT_SERVICE_USE_PYTHON_BRIDGE: "0" @@ -147,7 +147,7 @@ services: dockerfile: FileService/Dockerfile environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} - REPORT_API_URL: http://fastapi:8001 + REPORT_API_URL: http://fastapi:8096 depends_on: postgres: condition: service_healthy @@ -174,7 +174,7 @@ services: ports: - '${BFF_PORT:-8090}:8090' environment: - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 FILE_SERVICE_URL: http://files:8080 DATA_SERVICE_URL: http://data:8091 AI_SERVICE_URL: http://ai:8092 @@ -220,7 +220,7 @@ services: condition: service_healthy environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 ASPNETCORE_URLS: http://+:8092 WP_MCP_HTTP: "1" WP_MCP_DOMAIN: ${WP_MCP_DOMAIN:-core} diff --git a/docker-compose.pull.yml b/docker-compose.pull.yml index c258d7fe..43c1715b 100644 --- a/docker-compose.pull.yml +++ b/docker-compose.pull.yml @@ -42,7 +42,7 @@ services: volumes: - profiling-data:/data healthcheck: - test: ["CMD", "node", "-e", "require('http').get('http://127.0.0.1:8001/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] + test: ["CMD", "node", "-e", "require('http').get('http://127.0.0.1:8096/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s timeout: 5s retries: 3 @@ -73,7 +73,7 @@ services: dockerfile: AiService/Dockerfile environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 ASPNETCORE_URLS: http://+:8092 depends_on: postgres: @@ -95,7 +95,7 @@ services: environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling ASPNETCORE_URLS: http://+:8093 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 USE_FASTAPI_PYTHON_BRIDGE: "1" AUTH_SECRET: ${AUTH_SECRET:-} GOOGLE_REDIRECT_URI: ${GOOGLE_REDIRECT_URI:-http://localhost:8090/api/integrations/google/callback} @@ -120,7 +120,7 @@ services: environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling ASPNETCORE_URLS: http://+:8094 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 INTEGRATIONS_SERVICE_URL: http://integrations:8093 REPORT_SERVICE_USE_PYTHON_BRIDGE: "0" USE_FASTAPI_PYTHON_BRIDGE: "1" @@ -146,7 +146,7 @@ services: dockerfile: FileService/Dockerfile environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling - REPORT_API_URL: http://fastapi:8001 + REPORT_API_URL: http://fastapi:8096 depends_on: postgres: condition: service_healthy @@ -160,7 +160,7 @@ services: ports: - "8090:8090" environment: - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 FILE_SERVICE_URL: http://files:8080 DATA_SERVICE_URL: http://data:8091 AI_SERVICE_URL: http://ai:8092 diff --git a/docker-compose.yml b/docker-compose.yml index 4cba43e6..65f61f7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,7 @@ services: volumes: - profiling-data:/data healthcheck: - test: ["CMD", "node", "-e", "require('http').get('http://127.0.0.1:8001/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] + test: ["CMD", "node", "-e", "require('http').get('http://127.0.0.1:8096/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s timeout: 5s retries: 3 @@ -74,7 +74,7 @@ services: dockerfile: AiService/Dockerfile environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 ASPNETCORE_URLS: http://+:8092 WP_MCP_HTTP: "1" depends_on: @@ -97,7 +97,7 @@ services: environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling ASPNETCORE_URLS: http://+:8093 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 USE_FASTAPI_PYTHON_BRIDGE: "1" AUTH_SECRET: ${AUTH_SECRET:-} GOOGLE_REDIRECT_URI: ${GOOGLE_REDIRECT_URI:-http://localhost:8090/api/integrations/google/callback} @@ -122,7 +122,7 @@ services: environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling ASPNETCORE_URLS: http://+:8094 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 INTEGRATIONS_SERVICE_URL: http://integrations:8093 AISERVICE_URL: http://ai:8092 REPORT_SERVICE_USE_PYTHON_BRIDGE: "0" @@ -169,7 +169,7 @@ services: ports: - "8090:8090" environment: - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 FILE_SERVICE_URL: http://files:8080 BFF_ALLOWED_ORIGINS: "http://localhost:3000" DATA_SERVICE_URL: http://data:8091 @@ -226,7 +226,7 @@ services: dockerfile: FileService/Dockerfile environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling - REPORT_API_URL: http://fastapi:8001 + REPORT_API_URL: http://fastapi:8096 depends_on: postgres: condition: service_healthy diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 6b93de80..c26b722a 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -76,7 +76,7 @@ migrate() { start_uvicorn_foreground() { exec /opt/venv/bin/uvicorn website_profiling.api.main:app \ - --host 0.0.0.0 --port 8001 --workers 1 + --host 0.0.0.0 --port 8096 --workers 1 } case "$ROLE" in @@ -109,7 +109,7 @@ case "$ROLE" in WORKER_PID=$! /opt/venv/bin/uvicorn website_profiling.api.main:app \ - --host 0.0.0.0 --port 8001 --workers 1 & + --host 0.0.0.0 --port 8096 --workers 1 & UVICORN_PID=$! while kill -0 "$UVICORN_PID" 2>/dev/null; do diff --git a/docs/MCP.md b/docs/MCP.md index 5e79c553..f00f7c35 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -28,12 +28,12 @@ The same tool catalog powers in-app **AI Chat** at `/chat` (also served by AiSer ## Prerequisites -[.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, and `./local-run` (or AiService + FastAPI manually). AiService needs FastAPI on `:8001` for the audit-tool bridge until all tools are native C#. +[.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, and `./local-run` (or AiService + FastAPI manually). AiService needs FastAPI on `:8096` for the audit-tool bridge until all tools are native C#. ```bash export DATABASE_URL=postgres://profiling:profiling@localhost:5432/website_profiling # Docker default # ./local-run default: postgres://postgres:dev@127.0.0.1:5432/website_profiling -export FASTAPI_URL=http://127.0.0.1:8001 +export FASTAPI_URL=http://127.0.0.1:8096 ``` ### Local stdio (IDE subprocess) @@ -43,7 +43,7 @@ From the repo root, with Postgres running: ```bash cd services/AiService export DATABASE_URL=postgres://postgres:dev@127.0.0.1:5432/website_profiling -export FASTAPI_URL=http://127.0.0.1:8001 +export FASTAPI_URL=http://127.0.0.1:8096 dotnet run --project src/AiService.Api ``` @@ -85,7 +85,7 @@ Add to `.cursor/mcp.json` or your MCP client settings (stdio — spawn AiService "args": ["run", "--project", "services/AiService/src/AiService.Api", "--no-launch-profile"], "env": { "DATABASE_URL": "postgres://postgres:dev@127.0.0.1:5432/website_profiling", - "FASTAPI_URL": "http://127.0.0.1:8001", + "FASTAPI_URL": "http://127.0.0.1:8096", "WP_MCP_DOMAIN": "core", "WP_PROPERTY_ID": "1" } @@ -95,7 +95,7 @@ Add to `.cursor/mcp.json` or your MCP client settings (stdio — spawn AiService "args": ["run", "--project", "services/AiService/src/AiService.Api", "--no-launch-profile"], "env": { "DATABASE_URL": "postgres://postgres:dev@127.0.0.1:5432/website_profiling", - "FASTAPI_URL": "http://127.0.0.1:8001", + "FASTAPI_URL": "http://127.0.0.1:8096", "WP_MCP_DOMAIN": "google", "WP_PROPERTY_ID": "1" } @@ -114,7 +114,7 @@ Add to `.cursor/mcp.json` or your MCP client settings (stdio — spawn AiService "args": ["run", "--project", "services/AiService/src/AiService.Api", "--no-launch-profile"], "env": { "DATABASE_URL": "postgres://postgres:dev@127.0.0.1:5432/website_profiling", - "FASTAPI_URL": "http://127.0.0.1:8001", + "FASTAPI_URL": "http://127.0.0.1:8096", "WP_MCP_DOMAIN": "full", "WP_PROPERTY_ID": "1" } @@ -135,7 +135,7 @@ Configure access on **MCP settings** (`/mcp`) in the web UI (recommended), or se ```bash export DATABASE_URL=postgres://profiling:profiling@localhost:5432/website_profiling -export FASTAPI_URL=http://127.0.0.1:8001 +export FASTAPI_URL=http://127.0.0.1:8096 export ASPNETCORE_URLS=http://0.0.0.0:8092 export WP_MCP_HTTP=1 export WP_MCP_DOMAIN=core diff --git a/docs/OPS.md b/docs/OPS.md index 2411e258..f2e76944 100644 --- a/docs/OPS.md +++ b/docs/OPS.md @@ -266,7 +266,7 @@ The `files` service (port **8080**) renders audit PDFs and Excel workbooks. It r | Variable | Service | Purpose | |----------|---------|---------| | `FILE_SERVICE_URL` | `web`, MCP | Where clients call FileService (default `http://files:8080` in Compose) | -| `REPORT_API_URL` | `files` | Report API base URL (Compose: `http://web:8001`) | +| `REPORT_API_URL` | `files` | Report API base URL (Compose: `http://web:8096`) | PDF or workbook downloads fail if `files` is not running. See [services/FileService/README.md](../services/FileService/README.md). diff --git a/docs/README.md b/docs/README.md index 5cc4b272..089a0318 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,7 +29,7 @@ All `/api/*` calls from the SPA go to the **BFF** (`:8090`). The BFF forwards su | Upstream | Examples | |----------|----------| -| **FastAPI** (`:8001`) | `/api/run`, `/api/jobs/*`, `/api/pipeline-config`, `/api/pipeline-settings`, `/api/ui-preferences`, crawl, properties | +| **FastAPI** (`:8096`) | `/api/run`, `/api/jobs/*`, `/api/pipeline-config`, `/api/pipeline-settings`, `/api/ui-preferences`, crawl, properties | | **ReportService** (`:8094`, internal) | Report build and full-audit orchestration (worker; not browser-facing) | | **IntegrationsService** (`:8093`) | `/api/integrations/google/*`, `/api/integrations/bing/*`, property Google config | | **AiService** (`:8092`) | `/api/chat`, `/api/secrets`, `/api/llm-settings`, MCP-related APIs | @@ -53,7 +53,7 @@ During local development (`./local-run`, `ASPNETCORE_ENVIRONMENT=Development`), | ReportService | 8094 | [http://localhost:8094/docs](http://localhost:8094/docs) | | ConfigService | 8095 | [http://localhost:8095/docs](http://localhost:8095/docs) | | FileService | 8080 | [http://localhost:8080/docs](http://localhost:8080/docs) | -| Python bridge | 8001 | [http://localhost:8001/docs](http://localhost:8001/docs) | +| Python bridge | 8096 | [http://localhost:8096/docs](http://localhost:8096/docs) | OpenAPI JSON for .NET services: `/swagger/v1/swagger.json`. See the main [README.md](../README.md#api-documentation-swagger) for details. diff --git a/scripts/local-prod.sh b/scripts/local-prod.sh index a408442e..41f39932 100755 --- a/scripts/local-prod.sh +++ b/scripts/local-prod.sh @@ -165,7 +165,7 @@ cmd_start() { stop_service "Vite preview" "$NPM_PID" NPM_PID="" stop_host_dotnet_stack stop_service - stop_service "FastAPI" "$UVICORN_PID" 8001 + stop_service "FastAPI" "$UVICORN_PID" 8096 UVICORN_PID="" stop_service "pipeline worker" "$WORKER_PID" WORKER_PID="" @@ -191,14 +191,14 @@ cmd_start() { export WEBSITE_PROFILING_ROOT="$ROOT" log "Pipeline jobs run in ReportService C# worker" - log "Starting Python bridge (audit-tool + keyword enrich CLI) on port 8001" - export FASTAPI_URL="http://127.0.0.1:8001" + log "Starting Python bridge (audit-tool + keyword enrich CLI) on port 8096" + export FASTAPI_URL="http://127.0.0.1:8096" export FASTAPI_ALLOWED_ORIGINS="http://localhost:8090" "$ROOT/.venv/bin/uvicorn" website_profiling.api.main:app \ - --host 0.0.0.0 --port 8001 --workers 1 & + --host 0.0.0.0 --port 8096 --workers 1 & UVICORN_PID=$! disown_bg "$UVICORN_PID" - wait_for_http "http://127.0.0.1:8001/api/health" "FastAPI" 90 || die "FastAPI failed to start" + wait_for_http "http://127.0.0.1:8096/api/health" "FastAPI" 90 || die "FastAPI failed to start" start_host_integrations_bff "$ROOT" Production disown_bg "$INTEGRATIONS_PID" diff --git a/scripts/local-run-common.sh b/scripts/local-run-common.sh index c2d8abbf..fe500158 100644 --- a/scripts/local-run-common.sh +++ b/scripts/local-run-common.sh @@ -32,7 +32,7 @@ start_host_dotnet_base() { local root="$1" local mode="${2:-Development}" - export REPORT_API_URL="${REPORT_API_URL:-http://127.0.0.1:8001}" + export REPORT_API_URL="${REPORT_API_URL:-http://127.0.0.1:8096}" export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}" export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}" export REPORT_SERVICE_URL="${REPORT_SERVICE_URL:-http://127.0.0.1:8094}" @@ -69,7 +69,7 @@ start_host_dotnet_base() { printf '\033[1;36m→\033[0m Starting AiService on port 8092\n' (cd "$root/services/AiService" && \ DATABASE_URL="$DATABASE_URL" \ - FASTAPI_URL="http://127.0.0.1:8001" \ + FASTAPI_URL="http://127.0.0.1:8096" \ ASPNETCORE_URLS="http://127.0.0.1:8092" \ ASPNETCORE_ENVIRONMENT="$mode" \ WP_MCP_HTTP=1 \ @@ -90,7 +90,7 @@ start_host_report_service() { printf '\033[1;36m→\033[0m Starting ReportService on port 8094\n' (cd "$root/services/ReportService" && \ DATABASE_URL="$DATABASE_URL" \ - FASTAPI_URL="http://127.0.0.1:8001" \ + FASTAPI_URL="http://127.0.0.1:8096" \ INTEGRATIONS_SERVICE_URL="$INTEGRATIONS_SERVICE_URL" \ AISERVICE_URL="${AISERVICE_URL:-${AI_SERVICE_URL:-http://127.0.0.1:8092}}" \ WEBSITE_PROFILING_ROOT="$root" \ @@ -114,7 +114,7 @@ start_host_integrations_bff() { printf '\033[1;36m→\033[0m Starting IntegrationsService on port 8093\n' (cd "$root/services/IntegrationsService" && \ DATABASE_URL="$DATABASE_URL" \ - FASTAPI_URL="http://127.0.0.1:8001" \ + FASTAPI_URL="http://127.0.0.1:8096" \ USE_FASTAPI_PYTHON_BRIDGE="${USE_FASTAPI_PYTHON_BRIDGE:-1}" \ ASPNETCORE_URLS="http://127.0.0.1:8093" \ ASPNETCORE_ENVIRONMENT="$mode" \ @@ -129,7 +129,7 @@ start_host_integrations_bff() { free_port 8090 printf '\033[1;36m→\033[0m Starting BFF on port 8090\n' (cd "$root/services/Bff" && \ - FASTAPI_URL="http://127.0.0.1:8001" \ + FASTAPI_URL="http://127.0.0.1:8096" \ FILE_SERVICE_URL="$FILE_SERVICE_URL" \ DATA_SERVICE_URL="http://127.0.0.1:8091" \ AI_SERVICE_URL="$AI_SERVICE_URL" \ diff --git a/scripts/local-run.ps1 b/scripts/local-run.ps1 index cc6538bc..62b5f0f3 100644 --- a/scripts/local-run.ps1 +++ b/scripts/local-run.ps1 @@ -260,7 +260,7 @@ function Invoke-Start { if (Get-Command dotnet -ErrorAction SilentlyContinue) { Write-Log "Starting FileService on port 8080" - $env:REPORT_API_URL = "http://127.0.0.1:8001" + $env:REPORT_API_URL = "http://127.0.0.1:8096" Start-Process -FilePath "dotnet" ` -ArgumentList @("run", "--project", "src/FileService.Api", "--no-launch-profile") ` -WorkingDirectory (Join-Path $ROOT "services/FileService") ` @@ -270,18 +270,18 @@ function Invoke-Start { } Write-Log "Pipeline jobs run in ReportService C# worker" - Write-Log "Starting Python bridge on port 8001" - $env:FASTAPI_URL = "http://127.0.0.1:8001" + Write-Log "Starting Python bridge on port 8096" + $env:FASTAPI_URL = "http://127.0.0.1:8096" $env:FASTAPI_ALLOWED_ORIGINS = "http://localhost:8090" Start-Process -FilePath $VENV_PYTHON ` - -ArgumentList @("-m", "uvicorn", "website_profiling.api.main:app", "--host", "0.0.0.0", "--port", "8001", "--workers", "1") ` + -ArgumentList @("-m", "uvicorn", "website_profiling.api.main:app", "--host", "0.0.0.0", "--port", "8096", "--workers", "1") ` -WorkingDirectory $ROOT ` -WindowStyle Minimized | Out-Null if (Get-Command dotnet -ErrorAction SilentlyContinue) { Write-Log "Starting BFF on port 8090" $bffDir = Join-Path $ROOT "services/Bff" - $env:FASTAPI_URL = "http://127.0.0.1:8001" + $env:FASTAPI_URL = "http://127.0.0.1:8096" $env:FILE_SERVICE_URL = $fileServiceUrl $env:BFF_ALLOWED_ORIGINS = "http://localhost:3000" $env:ASPNETCORE_URLS = "http://127.0.0.1:8090" diff --git a/scripts/local-run.sh b/scripts/local-run.sh index 5591e75e..fccbac84 100755 --- a/scripts/local-run.sh +++ b/scripts/local-run.sh @@ -234,7 +234,7 @@ cmd_start() { if command -v dotnet >/dev/null 2>&1; then stop_host_dotnet_stack stop_service fi - stop_service "FastAPI" "$UVICORN_PID" 8001 + stop_service "FastAPI" "$UVICORN_PID" 8096 UVICORN_PID="" stop_service "pipeline worker" "$WORKER_PID" WORKER_PID="" @@ -268,16 +268,16 @@ cmd_start() { export WEBSITE_PROFILING_ROOT="$ROOT" log "Pipeline jobs run in ReportService C# worker (Python subprocess for crawl/lighthouse only)" - log "Starting Python bridge (audit-tool + keyword enrich CLI only) on port 8001" - export FASTAPI_URL="http://127.0.0.1:8001" + log "Starting Python bridge (audit-tool + keyword enrich CLI only) on port 8096" + export FASTAPI_URL="http://127.0.0.1:8096" export FASTAPI_ALLOWED_ORIGINS="http://localhost:8090" export DEPRECATE_PYTHON_INTEGRATIONS="${DEPRECATE_PYTHON_INTEGRATIONS:-1}" export DEPRECATE_PYTHON_REPORT_ROUTES="${DEPRECATE_PYTHON_REPORT_ROUTES:-1}" "$VENV/bin/uvicorn" website_profiling.api.main:app \ - --host 0.0.0.0 --port 8001 --workers 1 & + --host 0.0.0.0 --port 8096 --workers 1 & UVICORN_PID=$! disown_bg "$UVICORN_PID" - wait_for_http_logged "http://127.0.0.1:8001/api/health" "FastAPI" + wait_for_http_logged "http://127.0.0.1:8096/api/health" "FastAPI" if command -v dotnet >/dev/null 2>&1; then start_host_integrations_bff "$ROOT" Development diff --git a/services/AiService/README.md b/services/AiService/README.md index 03b4bfa9..67e26f7f 100644 --- a/services/AiService/README.md +++ b/services/AiService/README.md @@ -6,12 +6,12 @@ Python FastAPI retains crawl, pipeline jobs, and integrations OAuth reads. AiSer ## Run locally -Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, FastAPI on port 8001 (audit-tool bridge for tools not yet native in C#). +Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, FastAPI on port 8096 (audit-tool bridge for tools not yet native in C#). ```bash cd services/AiService export DATABASE_URL=postgres://postgres:dev@127.0.0.1:5432/website_profiling -export FASTAPI_URL=http://127.0.0.1:8001 +export FASTAPI_URL=http://127.0.0.1:8096 dotnet run --project src/AiService.Api ``` @@ -60,7 +60,7 @@ Pipeline domain settings (`crawl_settings`, `report_settings`, …) are written | Variable | Default | Description | |----------|---------|-------------| | `DATABASE_URL` | — | Postgres (`llm_settings`, `llm_cache`, `chat_sessions`, typed pipeline tables, `google_app_settings`, `report_payload`) | -| `FASTAPI_URL` | `http://127.0.0.1:8001` | Python audit-tool bridge for unported tools | +| `FASTAPI_URL` | `http://127.0.0.1:8096` | Python audit-tool bridge for unported tools | | `AI_SERVICE_URL` | — | Used by Python worker (`ai_service_client.py`) | | `ASPNETCORE_URLS` | `http://+:8092` | Bind address | | `WP_MCP_HTTP` | — | Set `1` to expose MCP at `/mcp` | diff --git a/services/AiService/src/AiService.Api/appsettings.json b/services/AiService/src/AiService.Api/appsettings.json index ed5a72cd..8d852d66 100644 --- a/services/AiService/src/AiService.Api/appsettings.json +++ b/services/AiService/src/AiService.Api/appsettings.json @@ -7,7 +7,7 @@ "CommandTimeoutSeconds": 30 }, "FastApi": { - "BaseUrl": "http://127.0.0.1:8001" + "BaseUrl": "http://127.0.0.1:8096" }, "Logging": { "LogLevel": { diff --git a/services/AiService/src/AiService.Tools/Options/FastApiOptions.cs b/services/AiService/src/AiService.Tools/Options/FastApiOptions.cs index 265e509d..9872ba84 100644 --- a/services/AiService/src/AiService.Tools/Options/FastApiOptions.cs +++ b/services/AiService/src/AiService.Tools/Options/FastApiOptions.cs @@ -9,5 +9,5 @@ public sealed class FastApiOptions public const string SectionName = "FastApi"; /// FastAPI base URL. Default matches local compose / BFF upstream. - public string BaseUrl { get; set; } = "http://127.0.0.1:8001"; + public string BaseUrl { get; set; } = "http://127.0.0.1:8096"; } diff --git a/services/Bff/src/Bff.Api/appsettings.json b/services/Bff/src/Bff.Api/appsettings.json index 5f86f66b..e66f1283 100644 --- a/services/Bff/src/Bff.Api/appsettings.json +++ b/services/Bff/src/Bff.Api/appsettings.json @@ -1,7 +1,7 @@ { "Urls": "http://+:8090", "Upstream": { - "FastApiBaseUrl": "http://127.0.0.1:8001", + "FastApiBaseUrl": "http://127.0.0.1:8096", "FileServiceBaseUrl": "http://127.0.0.1:8080", "TimeoutSeconds": 120 }, diff --git a/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs b/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs index d1bc527c..831f6808 100644 --- a/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs +++ b/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs @@ -9,7 +9,7 @@ public sealed class UpstreamOptions public const string SectionName = "Upstream"; /// FastAPI base URL (env override: FASTAPI_URL). Default matches the new internal compose service. - public string FastApiBaseUrl { get; set; } = "http://127.0.0.1:8001"; + public string FastApiBaseUrl { get; set; } = "http://127.0.0.1:8096"; /// FileService base URL (env override: FILE_SERVICE_URL). public string FileServiceBaseUrl { get; set; } = "http://127.0.0.1:8080"; diff --git a/services/FileService/README.md b/services/FileService/README.md index d2d84dc2..f575d882 100644 --- a/services/FileService/README.md +++ b/services/FileService/README.md @@ -6,11 +6,11 @@ Python owns crawl data and CSV/JSON exports. FileService renders PDF and Excel w ## Run locally -Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Site Audit report API on port 8001 (started by `./local-run` or Docker `web` service). +Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Site Audit report API on port 8096 (started by `./local-run` or Docker `web` service). ```bash cd services/FileService -export REPORT_API_URL=http://127.0.0.1:8001 +export REPORT_API_URL=http://127.0.0.1:8096 dotnet run --project src/FileService.Api ``` @@ -22,7 +22,7 @@ In **Development**, Swagger UI is at **http://localhost:8080/docs** and the Open | Variable | Default | Description | |----------|---------|-------------| -| `REPORT_API_URL` | `http://127.0.0.1:8001` | Base URL for report payload, meta, and ui-preferences HTTP API | +| `REPORT_API_URL` | `http://127.0.0.1:8096` | Base URL for report payload, meta, and ui-preferences HTTP API | | `ASPNETCORE_URLS` | `http://127.0.0.1:8080` | Bind address (Docker sets `http://+:8080`) | ## Upstream HTTP contract diff --git a/services/FileService/src/FileService.Api/appsettings.json b/services/FileService/src/FileService.Api/appsettings.json index 85bfa513..8523417e 100644 --- a/services/FileService/src/FileService.Api/appsettings.json +++ b/services/FileService/src/FileService.Api/appsettings.json @@ -1,7 +1,7 @@ { "Urls": "http://+:8080", "ReportApi": { - "BaseUrl": "http://127.0.0.1:8001", + "BaseUrl": "http://127.0.0.1:8096", "TimeoutSeconds": 120 }, "Logging": { diff --git a/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs b/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs index 9102692b..32a095f5 100644 --- a/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs +++ b/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs @@ -9,6 +9,6 @@ public sealed class ReportApiOptions { public const string SectionName = "ReportApi"; - public string BaseUrl { get; set; } = "http://127.0.0.1:8001"; + public string BaseUrl { get; set; } = "http://127.0.0.1:8096"; public int TimeoutSeconds { get; set; } = 120; } diff --git a/services/FileService/tests/FileService.Tests/ServiceRegistrationValidationTests.cs b/services/FileService/tests/FileService.Tests/ServiceRegistrationValidationTests.cs index 8cd77d1f..63ec1f10 100644 --- a/services/FileService/tests/FileService.Tests/ServiceRegistrationValidationTests.cs +++ b/services/FileService/tests/FileService.Tests/ServiceRegistrationValidationTests.cs @@ -14,7 +14,7 @@ public void Web_host_resolves_core_services() { using var env = ServiceRegistrationTestEnvironment.Push(); env.SetDefaultsForPostgresServices(); - env.Set("REPORT_API_URL", "http://127.0.0.1:8001"); + env.Set("REPORT_API_URL", "http://127.0.0.1:8096"); using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => { diff --git a/services/IntegrationsService/README.md b/services/IntegrationsService/README.md index 7893093f..59fa6485 100644 --- a/services/IntegrationsService/README.md +++ b/services/IntegrationsService/README.md @@ -11,7 +11,7 @@ Internal .NET microservice for Google Search Console, GA4, Bing sync, and keywor ```bash cd services/IntegrationsService DATABASE_URL=postgres://profiling:profiling@127.0.0.1:5432/website_profiling \ - FASTAPI_URL=http://127.0.0.1:8001 \ + FASTAPI_URL=http://127.0.0.1:8096 \ USE_FASTAPI_PYTHON_BRIDGE=1 \ ASPNETCORE_URLS=http://127.0.0.1:8093 \ dotnet run --project src/IntegrationsService.Api --no-launch-profile diff --git a/services/IntegrationsService/src/IntegrationsService.Application/Google/FastApiPythonBridge.cs b/services/IntegrationsService/src/IntegrationsService.Application/Google/FastApiPythonBridge.cs index a3cca697..6dfb82db 100644 --- a/services/IntegrationsService/src/IntegrationsService.Application/Google/FastApiPythonBridge.cs +++ b/services/IntegrationsService/src/IntegrationsService.Application/Google/FastApiPythonBridge.cs @@ -133,7 +133,7 @@ public async Task RunKeywordEnrichAsync( private HttpClient CreateClient() { var client = httpClientFactory.CreateClient(nameof(FastApiPythonBridge)); - var baseUrl = (Environment.GetEnvironmentVariable("FASTAPI_URL") ?? "http://127.0.0.1:8001").Trim().TrimEnd('/'); + var baseUrl = (Environment.GetEnvironmentVariable("FASTAPI_URL") ?? "http://127.0.0.1:8096").Trim().TrimEnd('/'); client.BaseAddress = new Uri(baseUrl + "/"); client.Timeout = TimeSpan.FromSeconds(120); return client; diff --git a/services/ReportService/src/ReportService.Api/appsettings.json b/services/ReportService/src/ReportService.Api/appsettings.json index ab59f4e2..98069c97 100644 --- a/services/ReportService/src/ReportService.Api/appsettings.json +++ b/services/ReportService/src/ReportService.Api/appsettings.json @@ -6,7 +6,7 @@ } }, "FastApi": { - "BaseUrl": "http://127.0.0.1:8001", + "BaseUrl": "http://127.0.0.1:8096", "TimeoutSeconds": 1800 }, "ReportService": { diff --git a/services/ReportService/src/ReportService.Application/Options/FastApiOptions.cs b/services/ReportService/src/ReportService.Application/Options/FastApiOptions.cs index ad69f508..71cac715 100644 --- a/services/ReportService/src/ReportService.Application/Options/FastApiOptions.cs +++ b/services/ReportService/src/ReportService.Application/Options/FastApiOptions.cs @@ -4,7 +4,7 @@ public sealed class FastApiOptions { public const string SectionName = "FastApi"; - public string BaseUrl { get; set; } = "http://127.0.0.1:8001"; + public string BaseUrl { get; set; } = "http://127.0.0.1:8096"; public int TimeoutSeconds { get; set; } = 1800; } diff --git a/services/Shared/WebsiteProfiling.Testing/ServiceRegistrationTestEnvironment.cs b/services/Shared/WebsiteProfiling.Testing/ServiceRegistrationTestEnvironment.cs index b50affed..090d3ebe 100644 --- a/services/Shared/WebsiteProfiling.Testing/ServiceRegistrationTestEnvironment.cs +++ b/services/Shared/WebsiteProfiling.Testing/ServiceRegistrationTestEnvironment.cs @@ -26,7 +26,7 @@ public void Set(string key, string? value) public void SetDefaultsForPostgresServices() { Set("DATABASE_URL", "Host=127.0.0.1;Port=5432;Database=wp_di_test;Username=test;Password=test"); - Set("FASTAPI_URL", "http://127.0.0.1:8001"); + Set("FASTAPI_URL", "http://127.0.0.1:8096"); } public void Dispose() diff --git a/src/website_profiling/commands/report_build.py b/src/website_profiling/commands/report_build.py index 9fb72569..1e330c4e 100644 --- a/src/website_profiling/commands/report_build.py +++ b/src/website_profiling/commands/report_build.py @@ -149,7 +149,7 @@ def call_fastapi_report_bridge(cfg: dict, property_id: int, crawl_run_id: int | import urllib.error import urllib.request - base = (os.environ.get("FASTAPI_URL") or "http://127.0.0.1:8001").strip().rstrip("/") + base = (os.environ.get("FASTAPI_URL") or "http://127.0.0.1:8096").strip().rstrip("/") body = { "propertyId": int(property_id), "crawlRunId": crawl_run_id, diff --git a/tests/test_ai_migration_coverage.py b/tests/test_ai_migration_coverage.py index 8dc15634..c9cb60c0 100644 --- a/tests/test_ai_migration_coverage.py +++ b/tests/test_ai_migration_coverage.py @@ -418,7 +418,7 @@ def __exit__(self, *_a): with pytest.raises(ConnectionError): report_build.call_report_service({}, 1) - monkeypatch.setenv("FASTAPI_URL", "http://fastapi:8001") + monkeypatch.setenv("FASTAPI_URL", "http://fastapi:8096") with patch("urllib.request.urlopen", return_value=_Resp()): out2 = report_build.call_fastapi_report_bridge({"a": "1"}, 3) assert out2["ok"] is True diff --git a/web/src/lib/mcpClientConfig.test.ts b/web/src/lib/mcpClientConfig.test.ts index 4d66f15f..5d274dd1 100644 --- a/web/src/lib/mcpClientConfig.test.ts +++ b/web/src/lib/mcpClientConfig.test.ts @@ -40,7 +40,7 @@ describe('mcpClientConfig', () => { }); expect(json).toContain('"command": "dotnet"'); expect(json).toContain('services/AiService/src/AiService.Api'); - expect(json).toContain('"FASTAPI_URL": "http://127.0.0.1:8001"'); + expect(json).toContain('"FASTAPI_URL": "http://127.0.0.1:8096"'); expect(json).toContain('"WP_PROPERTY_ID": "2"'); }); diff --git a/web/src/lib/mcpClientConfig.ts b/web/src/lib/mcpClientConfig.ts index fb55ed52..ee19c0b0 100644 --- a/web/src/lib/mcpClientConfig.ts +++ b/web/src/lib/mcpClientConfig.ts @@ -78,7 +78,7 @@ export function buildLocalStdioConfig(input: McpClientConfigInput): string { args: ['run', '--project', 'services/AiService/src/AiService.Api', '--no-launch-profile'], env: { DATABASE_URL: databaseUrl, - FASTAPI_URL: 'http://127.0.0.1:8001', + FASTAPI_URL: 'http://127.0.0.1:8096', WP_MCP_DOMAIN: domain, WP_PROPERTY_ID: propertyId, }, @@ -96,7 +96,7 @@ export function buildHttpStartCommand(): string { return [ 'cd services/AiService', 'export DATABASE_URL=postgres://USER:PASS@localhost:5432/website_profiling', - 'export FASTAPI_URL=http://127.0.0.1:8001', + 'export FASTAPI_URL=http://127.0.0.1:8096', 'export ASPNETCORE_URLS=http://0.0.0.0:8092', 'export WP_MCP_HTTP=1', 'dotnet run --project src/AiService.Api', From e79a0ce57fb07b8bc430d7689dc209c441255bcd Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 28 Jun 2026 12:16:49 +0530 Subject: [PATCH 2/6] Google Cred Refresh --- .../Google/GoogleCredentialFactory.cs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/services/IntegrationsService/src/IntegrationsService.Providers/Google/GoogleCredentialFactory.cs b/services/IntegrationsService/src/IntegrationsService.Providers/Google/GoogleCredentialFactory.cs index 55358b87..00ba4a77 100644 --- a/services/IntegrationsService/src/IntegrationsService.Providers/Google/GoogleCredentialFactory.cs +++ b/services/IntegrationsService/src/IntegrationsService.Providers/Google/GoogleCredentialFactory.cs @@ -51,7 +51,24 @@ public async Task BuildCredentialsAsync( propertyId.ToString(), new TokenResponse { RefreshToken = token }); - if (!await credential.RefreshTokenAsync(cancellationToken)) + bool refreshed; + try + { + refreshed = await credential.RefreshTokenAsync(cancellationToken); + } + catch (TokenResponseException ex) when (ex.Error?.Error == "invalid_grant") + { + await properties.DisconnectGoogleAsync(propertyId, cancellationToken); + throw new InvalidOperationException( + "Google connection expired — reconnect Google for this site."); + } + catch (TokenResponseException) + { + throw new InvalidOperationException( + "Google connection expired — reconnect Google for this site."); + } + + if (!refreshed) { throw new InvalidOperationException( "Google connection expired — reconnect Google for this site."); From 76061e0b0d3c1bdc3298cba5ec6a4179b2da6d65 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 28 Jun 2026 12:25:43 +0530 Subject: [PATCH 3/6] GSC config data --- .../Controllers/PropertyGoogleController.cs | 3 +-- .../Repositories/GoogleAppSettingsRepository.cs | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/services/IntegrationsService/src/IntegrationsService.Api/Controllers/PropertyGoogleController.cs b/services/IntegrationsService/src/IntegrationsService.Api/Controllers/PropertyGoogleController.cs index c046a00c..1670c1d1 100644 --- a/services/IntegrationsService/src/IntegrationsService.Api/Controllers/PropertyGoogleController.cs +++ b/services/IntegrationsService/src/IntegrationsService.Api/Controllers/PropertyGoogleController.cs @@ -31,7 +31,6 @@ public async Task Status(long propertyId, CancellationToken cance return NotFound(new { error = "Property not found" }); } - var appCfg = await appSettings.ReadAsync(cancellationToken); var publicStatus = PropertyGoogleStatusMapper.ToPublicStatus(prop); return Ok(new { @@ -42,7 +41,7 @@ public async Task Status(long propertyId, CancellationToken cance dateRangeDays = publicStatus.DateRangeDays, connectedEmail = publicStatus.ConnectedEmail, connectedAt = publicStatus.ConnectedAt, - hasClientId = !string.IsNullOrWhiteSpace(appCfg.ClientId), + hasClientId = await appSettings.HasClientCredentialsAsync(cancellationToken), lastFetchedAt = await googleData.GetLastFetchedAtAsync(propertyId, cancellationToken), propertyId, }); diff --git a/services/IntegrationsService/src/IntegrationsService.Application/Repositories/GoogleAppSettingsRepository.cs b/services/IntegrationsService/src/IntegrationsService.Application/Repositories/GoogleAppSettingsRepository.cs index 113842be..080532e1 100644 --- a/services/IntegrationsService/src/IntegrationsService.Application/Repositories/GoogleAppSettingsRepository.cs +++ b/services/IntegrationsService/src/IntegrationsService.Application/Repositories/GoogleAppSettingsRepository.cs @@ -33,6 +33,13 @@ public async Task HasServiceAccountAsync(CancellationToken cancellationTok && row.ServiceAccountJson != "null"; } + public async Task HasClientCredentialsAsync(CancellationToken cancellationToken = default) + { + var row = await ReadAsync(cancellationToken); + var clientId = (row.ClientId ?? Environment.GetEnvironmentVariable("GOOGLE_CLIENT_ID") ?? "").Trim(); + return !string.IsNullOrWhiteSpace(clientId); + } + public async Task<(string ClientId, string ClientSecret)> AppClientCredentialsAsync( CancellationToken cancellationToken = default) { From 9c1c2a4d804d8bf3bf687b69abba09a7356a5177 Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 28 Jun 2026 12:56:51 +0530 Subject: [PATCH 4/6] config service --- diagram.html | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/diagram.html b/diagram.html index 96b32eef..f6441aed 100644 --- a/diagram.html +++ b/diagram.html @@ -198,11 +198,12 @@

{ id: 'bff', label: 'BFF', sub: ':8090 • Auth • Proxy', x: 550, y: 280, type: 'gateway', desc: 'Backend-for-Frontend serving as the entry point. Handles authentication, CORS, and reverse proxy routing.' }, { id: 'fastapi', label: 'FastAPI', sub: ':8096 • Crawl/Jobs', x: 50, y: 450, type: 'service', desc: 'Core Python service handling crawling, jobs, and pipeline settings. Acts as the strangler proxy target.' }, - { id: 'report', label: 'Report', sub: ':8094 • Build', x: 250, y: 450, type: 'service', desc: 'Service dedicated to report building and orchestrating complex multi-step processes.' }, - { id: 'integrations', label: 'Integrations', sub: ':8093 • APIs', x: 450, y: 450, type: 'service', desc: 'Manages third-party connections including Google/Bing APIs, GSC, GA4, and keyword processing.' }, - { id: 'ai', label: 'AI', sub: ':8092 • Chat/LLM', x: 650, y: 450, type: 'service', desc: 'AI orchestration service managing chat interfaces, LLM connections, secrets, and MCP.' }, - { id: 'data', label: 'Data', sub: ':8091 • Portfolio', x: 850, y: 450, type: 'service', desc: 'Data access layer for report reads, portfolio management, and system issue tracking.' }, - { id: 'files', label: 'Files', sub: ':8080 • Exports', x: 1050, y: 450, type: 'service', desc: 'Handles document generation, including PDF and Excel exports.' }, + { id: 'config', label: 'ConfigService', sub: ':8095 • Typed Config', x: 250, y: 450, type: 'service', desc: 'Typed configuration service exposing pipeline, crawl, Lighthouse, and report settings stored in PostgreSQL typed tables. Serves GET/PUT /api/pipeline-config and /api/pipeline-settings.' }, + { id: 'report', label: 'Report', sub: ':8094 • Build', x: 450, y: 450, type: 'service', desc: 'Service dedicated to report building and orchestrating complex multi-step processes.' }, + { id: 'integrations', label: 'Integrations', sub: ':8093 • APIs', x: 650, y: 450, type: 'service', desc: 'Manages third-party connections including Google/Bing APIs, GSC, GA4, and keyword processing.' }, + { id: 'ai', label: 'AI', sub: ':8092 • Chat/LLM', x: 850, y: 450, type: 'service', desc: 'AI orchestration service managing chat interfaces, LLM connections, secrets, and MCP.' }, + { id: 'data', label: 'Data', sub: ':8091 • Portfolio', x: 1050, y: 450, type: 'service', desc: 'Data access layer for report reads, portfolio management, and system issue tracking.' }, + { id: 'files', label: 'Files', sub: ':8080 • Exports', x: 1250, y: 450, type: 'service', desc: 'Handles document generation, including PDF and Excel exports.' }, { id: 'worker', label: 'Worker', sub: 'Crawl + Lighthouse', x: 350, y: 640, type: 'worker', desc: 'Background asynchronous pipeline worker executing heavy crawl and Lighthouse audit jobs.' }, @@ -241,7 +242,9 @@

{ id: 'e23', source: 'integrations', target: 'google' }, { id: 'e24', source: 'ai', target: 'llm' }, - { id: 'e25', source: 'mcp', target: 'ai', label: 'MCP stdio/HTTP', dashed: true } + { id: 'e25', source: 'mcp', target: 'ai', label: 'MCP stdio/HTTP', dashed: true }, + { id: 'e26', source: 'bff', target: 'config' }, + { id: 'e27', source: 'config', target: 'postgres' } ]; let transform = { x: 0, y: 0, scale: 0.8 }; From 955e3c97077a501adcf78a208615a2ef33bd47fe Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 28 Jun 2026 13:04:13 +0530 Subject: [PATCH 5/6] File Service Port Change --- AGENT.md | 2 +- AGENTS.md | 2 +- README.md | 10 +++++----- diagram.html | 2 +- docker-compose.prod.yml | 2 +- docker-compose.pull.yml | 2 +- docker-compose.yml | 4 ++-- docs/MCP.md | 2 +- docs/OPS.md | 4 ++-- docs/README.md | 4 ++-- scripts/local-prod.sh | 2 +- scripts/local-run-common.sh | 12 ++++++------ scripts/local-run.ps1 | 4 ++-- scripts/local-run.sh | 6 +++--- services/Bff/src/Bff.Api/appsettings.json | 2 +- .../src/Bff.Application/Options/UpstreamOptions.cs | 2 +- .../Bff.Tests/ServiceRegistrationValidationTests.cs | 2 +- services/FileService/Dockerfile | 4 ++-- services/FileService/README.md | 6 +++--- .../FileService.Api/Properties/launchSettings.json | 2 +- .../FileService/src/FileService.Api/appsettings.json | 2 +- src/website_profiling/clients/file_service.py | 2 +- 22 files changed, 40 insertions(+), 40 deletions(-) diff --git a/AGENT.md b/AGENT.md index 73225adf..4a256855 100644 --- a/AGENT.md +++ b/AGENT.md @@ -21,7 +21,7 @@ Developer reference for agents and contributors. User-facing overview: [README.m - `web/src/` -- React SPA (`AppRoutes.tsx`, `views/`, `components/`); pipeline UI: `PipelineRunnerFab`, `PipelineContext` - `alembic/` -- schema migrations -**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, FileService `:8080`, Data `:8091`, AiService `:8092`, ReportService `:8094`, FastAPI `:8096`, BFF `:8090`, Vite `:3000`; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web and .NET checks — mirrors CI; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`. +**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, FileService `:8097`, Data `:8091`, AiService `:8092`, ReportService `:8094`, FastAPI `:8096`, BFF `:8090`, Vite `:3000`; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web and .NET checks — mirrors CI; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`. **JavaScript crawl (optional):** Config keys `crawl_render_mode` (`static` | `javascript` | `auto`) and `crawl_js_*` in pipeline config / `pipelineConfigSchema.ts`. JS/auto crawls can capture browser console errors and uncaught exceptions (`crawl_js_capture_console`, stored under `page_analysis.browser`). **Auto mode** uses static-first fetch, pre-parse SPA heuristics (`needs_js_render`), then post-parse low-outlink fallback (`needs_js_render_after_parse`) in `crawler.py`. **Preflight:** `GET /api/crawl/browser-status` (localhost) spawns Python `browser_status()`; Run audit settings/run validation calls it when render mode is `javascript` or `auto`. Browser deps: Playwright from `requirements.txt` (installed by `./local-run setup` and `./local-test`). Runtime needs Chromium on `PATH` or `CHROME_PATH` (Docker sets `CHROME_PATH=/usr/bin/chromium`). Integration tests: `@pytest.mark.browser` — excluded by default in `pytest.ini`; Docker CI runs `tests/test_crawl_fetchers.py` and `tests/test_crawler_browser_e2e.py -m browser`; locally `./local-test browser`. diff --git a/AGENTS.md b/AGENTS.md index fb1f98c5..2385fa1b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ This file is the canonical entry point for agents. For full detail see [AGENT.md - `services/ReportService/` — .NET report build + pipeline orchestration (port 8094). See [services/ReportService/README.md](services/ReportService/README.md) - `services/AiService/` — .NET AI service (Microsoft.Extensions.AI, chat, enrichment, MCP, **secrets/llm-settings writes**; port 8092). See [services/AiService/README.md](services/AiService/README.md) - `services/IntegrationsService/` — .NET Google/Bing integrations (GSC/GA4 fetch, OAuth, page-live, keyword reads; port 8093). See [services/IntegrationsService/README.md](services/IntegrationsService/README.md) -- `services/FileService/` — .NET PDF + Excel workbook export (port 8080). HTTP-only via `REPORT_API_URL`; no Postgres. Profiles: `executive|standard|full|premium`. Details: [services/FileService/README.md](services/FileService/README.md). Env: `FILE_SERVICE_URL` (MCP), `REPORT_API_URL` (FileService). +- `services/FileService/` — .NET PDF + Excel workbook export (port 8097). HTTP-only via `REPORT_API_URL`; no Postgres. Profiles: `executive|standard|full|premium`. Details: [services/FileService/README.md](services/FileService/README.md). Env: `FILE_SERVICE_URL` (MCP), `REPORT_API_URL` (FileService). - `alembic/` — DB migrations - `services/WebsiteProfiling.slnx` — unified .NET solution (all six services + shared libs) - `docs/` — documentation index diff --git a/README.md b/README.md index 37f11836..7948734f 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ flowchart TB BFF --> Integrations["integrations :8093
Google/Bing · GSC/GA4 · keywords"] BFF --> Ai["ai :8092
Chat · LLM · secrets · MCP"] BFF --> Data["data :8091
Report reads · portfolio · issues"] - BFF --> Files["files :8080
PDF · Excel export"] + BFF --> Files["files :8097
PDF · Excel export"] end subgraph Pipeline["Background pipeline"] @@ -233,7 +233,7 @@ WebsiteProfiling/ ├── services/IntegrationsService/ # .NET Google/Bing integrations — OAuth, GSC/GA4, keywords (port 8093) ├── services/AiService/ # .NET AI — chat, secrets, LLM config, MCP, enrichment (port 8092) ├── services/Data/ # .NET read service — report/portfolio/issue reads (port 8091) -├── services/FileService/ # .NET PDF + Excel workbook export (port 8080) +├── services/FileService/ # .NET PDF + Excel workbook export (port 8097) ├── alembic/versions/ # PostgreSQL schema migrations ├── tests/ # pytest suite + fixtures ├── docs/ # Glossary, MCP, ops, brand assets @@ -291,7 +291,7 @@ Build and run the full dev stack from source: docker compose up --build ``` -Services: **postgres**, **fastapi** (`:8096`, internal), **worker**, **report** (`:8094`, internal), **integrations** (`:8093`, internal), **ai** (`:8092`, internal), **data** (`:8091`, internal), **bff** (`:8090`), **web** (`:3000`), **files** (`:8080`, internal). +Services: **postgres**, **fastapi** (`:8096`, internal), **worker**, **report** (`:8094`, internal), **integrations** (`:8093`, internal), **ai** (`:8092`, internal), **data** (`:8091`, internal), **bff** (`:8090`), **web** (`:3000`), **files** (`:8097`, internal). Open [http://localhost:3000/home](http://localhost:3000/home). The browser talks only to the **BFF** (`:8090`); the BFF proxies to FastAPI (crawl/pipeline), IntegrationsService (Google/Bing), AiService (AI/secrets), Data (report reads), and FileService (PDF/workbook export). @@ -308,7 +308,7 @@ Production deployment: `docker-compose.prod.yml` — set `POSTGRES_USER`, `POSTG ./local-prod # Same DB, Vite production build + preview (no hot reload) ``` -`./local-run` starts (in order): **FileService** `:8080`, **Data** `:8091`, **AiService** `:8092` (MCP HTTP enabled), **ReportService** `:8094`, **IntegrationsService** `:8093`, **ConfigService** `:8095`, **FastAPI** `:8096` (Python bridge), **BFF** `:8090`, and **Vite** `:3000`. Use `localhost` (not `127.0.0.1`) for pipeline APIs so CORS and cookies match the BFF origin. +`./local-run` starts (in order): **FileService** `:8097`, **Data** `:8091`, **AiService** `:8092` (MCP HTTP enabled), **ReportService** `:8094`, **IntegrationsService** `:8093`, **ConfigService** `:8095`, **FastAPI** `:8096` (Python bridge), **BFF** `:8090`, and **Vite** `:3000`. Use `localhost` (not `127.0.0.1`) for pipeline APIs so CORS and cookies match the BFF origin. Default local `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling` (Docker Compose dev stack uses `profiling:profiling`). @@ -324,7 +324,7 @@ With `./local-run`, each service runs in **Development** and exposes interactive | **IntegrationsService** | 8093 | [http://localhost:8093/docs](http://localhost:8093/docs) | `/swagger/v1/swagger.json` | | **ReportService** | 8094 | [http://localhost:8094/docs](http://localhost:8094/docs) | `/swagger/v1/swagger.json` | | **ConfigService** | 8095 | [http://localhost:8095/docs](http://localhost:8095/docs) | `/swagger/v1/swagger.json` | -| **FileService** | 8080 | [http://localhost:8080/docs](http://localhost:8080/docs) | `/swagger/v1/swagger.json` | +| **FileService** | 8097 | [http://localhost:8097/docs](http://localhost:8097/docs) | `/swagger/v1/swagger.json` | | **Python bridge** (FastAPI) | 8096 | [http://localhost:8096/docs](http://localhost:8096/docs) | `/openapi.json` | Swagger UI is disabled when `ASPNETCORE_ENVIRONMENT=Production`. The legacy `web/openapi.json` reflects the old monolithic FastAPI app and is not a complete map of the current C# services. diff --git a/diagram.html b/diagram.html index f6441aed..01802468 100644 --- a/diagram.html +++ b/diagram.html @@ -203,7 +203,7 @@

{ id: 'integrations', label: 'Integrations', sub: ':8093 • APIs', x: 650, y: 450, type: 'service', desc: 'Manages third-party connections including Google/Bing APIs, GSC, GA4, and keyword processing.' }, { id: 'ai', label: 'AI', sub: ':8092 • Chat/LLM', x: 850, y: 450, type: 'service', desc: 'AI orchestration service managing chat interfaces, LLM connections, secrets, and MCP.' }, { id: 'data', label: 'Data', sub: ':8091 • Portfolio', x: 1050, y: 450, type: 'service', desc: 'Data access layer for report reads, portfolio management, and system issue tracking.' }, - { id: 'files', label: 'Files', sub: ':8080 • Exports', x: 1250, y: 450, type: 'service', desc: 'Handles document generation, including PDF and Excel exports.' }, + { id: 'files', label: 'Files', sub: ':8097 • Exports', x: 1250, y: 450, type: 'service', desc: 'Handles document generation, including PDF and Excel exports.' }, { id: 'worker', label: 'Worker', sub: 'Crawl + Lighthouse', x: 350, y: 640, type: 'worker', desc: 'Background asynchronous pipeline worker executing heavy crawl and Lighthouse audit jobs.' }, diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9ba277cc..dcb08397 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -175,7 +175,7 @@ services: - '${BFF_PORT:-8090}:8090' environment: FASTAPI_URL: http://fastapi:8096 - FILE_SERVICE_URL: http://files:8080 + FILE_SERVICE_URL: http://files:8097 DATA_SERVICE_URL: http://data:8091 AI_SERVICE_URL: http://ai:8092 INTEGRATIONS_SERVICE_URL: http://integrations:8093 diff --git a/docker-compose.pull.yml b/docker-compose.pull.yml index 43c1715b..b46b6057 100644 --- a/docker-compose.pull.yml +++ b/docker-compose.pull.yml @@ -161,7 +161,7 @@ services: - "8090:8090" environment: FASTAPI_URL: http://fastapi:8096 - FILE_SERVICE_URL: http://files:8080 + FILE_SERVICE_URL: http://files:8097 DATA_SERVICE_URL: http://data:8091 AI_SERVICE_URL: http://ai:8092 INTEGRATIONS_SERVICE_URL: http://integrations:8093 diff --git a/docker-compose.yml b/docker-compose.yml index 65f61f7b..0eb48aaf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -170,7 +170,7 @@ services: - "8090:8090" environment: FASTAPI_URL: http://fastapi:8096 - FILE_SERVICE_URL: http://files:8080 + FILE_SERVICE_URL: http://files:8097 BFF_ALLOWED_ORIGINS: "http://localhost:3000" DATA_SERVICE_URL: http://data:8091 AI_SERVICE_URL: http://ai:8092 @@ -233,7 +233,7 @@ services: fastapi: condition: service_started healthcheck: - test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8080/health || exit 1"] + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8097/health || exit 1"] interval: 30s timeout: 5s retries: 3 diff --git a/docs/MCP.md b/docs/MCP.md index f00f7c35..4f259755 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -227,7 +227,7 @@ Export tools write artifact files with a 24-hour TTL; in-app chat renders downlo | Deliverable | Generator | Env / notes | |-------------|-----------|-------------| -| PDF | **FileService** (`GET /v1/reports/{id}/pdf`) | `FILE_SERVICE_URL` on MCP/web (default `http://127.0.0.1:8080`); FileService must be running | +| PDF | **FileService** (`GET /v1/reports/{id}/pdf`) | `FILE_SERVICE_URL` on MCP/web (default `http://127.0.0.1:8097`); FileService must be running | | Excel workbook | **FileService** (`GET /v1/reports/{id}/workbook`) | Same as PDF | | CSV / JSON audit export | Python (`export_audit_report`, `GET /api/report/export`) | Reads Postgres via report payload — no FileService required | | Compare / list CSV | Python | `export_compare_csv`, `export_list_as_csv` | diff --git a/docs/OPS.md b/docs/OPS.md index f2e76944..a35089e9 100644 --- a/docs/OPS.md +++ b/docs/OPS.md @@ -261,11 +261,11 @@ Do not run the application container in isolation with `docker run` unless you p ### FileService (PDF and workbook export) -The `files` service (port **8080**) renders audit PDFs and Excel workbooks. It reads report data over HTTP from the `web` service — no Postgres connection. +The `files` service (port **8097**) renders audit PDFs and Excel workbooks. It reads report data over HTTP from the `web` service — no Postgres connection. | Variable | Service | Purpose | |----------|---------|---------| -| `FILE_SERVICE_URL` | `web`, MCP | Where clients call FileService (default `http://files:8080` in Compose) | +| `FILE_SERVICE_URL` | `web`, MCP | Where clients call FileService (default `http://files:8097` in Compose) | | `REPORT_API_URL` | `files` | Report API base URL (Compose: `http://web:8096`) | PDF or workbook downloads fail if `files` is not running. See [services/FileService/README.md](../services/FileService/README.md). diff --git a/docs/README.md b/docs/README.md index 089a0318..b5b64d08 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,7 +34,7 @@ All `/api/*` calls from the SPA go to the **BFF** (`:8090`). The BFF forwards su | **IntegrationsService** (`:8093`) | `/api/integrations/google/*`, `/api/integrations/bing/*`, property Google config | | **AiService** (`:8092`) | `/api/chat`, `/api/secrets`, `/api/llm-settings`, MCP-related APIs | | **Data** (`:8091`) | Report payload reads, portfolio, issue status, saved filters | -| **FileService** (`:8080`) | PDF and Excel export | +| **FileService** (`:8097`) | PDF and Excel export | **Internal service-to-service:** ReportService reads Google/keyword snapshots from IntegrationsService (`GET /internal/integrations/report/enrichment`) during native report build — not via the BFF. @@ -52,7 +52,7 @@ During local development (`./local-run`, `ASPNETCORE_ENVIRONMENT=Development`), | IntegrationsService | 8093 | [http://localhost:8093/docs](http://localhost:8093/docs) | | ReportService | 8094 | [http://localhost:8094/docs](http://localhost:8094/docs) | | ConfigService | 8095 | [http://localhost:8095/docs](http://localhost:8095/docs) | -| FileService | 8080 | [http://localhost:8080/docs](http://localhost:8080/docs) | +| FileService | 8097 | [http://localhost:8097/docs](http://localhost:8097/docs) | | Python bridge | 8096 | [http://localhost:8096/docs](http://localhost:8096/docs) | OpenAPI JSON for .NET services: `/swagger/v1/swagger.json`. See the main [README.md](../README.md#api-documentation-swagger) for details. diff --git a/scripts/local-prod.sh b/scripts/local-prod.sh index 41f39932..2abbe8dd 100755 --- a/scripts/local-prod.sh +++ b/scripts/local-prod.sh @@ -184,7 +184,7 @@ cmd_start() { export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}" export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}" - export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8097}" export REPORT_SERVICE_URL="${REPORT_SERVICE_URL:-http://127.0.0.1:8094}" export PIPELINE_ORCHESTRATE_VIA_REPORT_SERVICE="${PIPELINE_ORCHESTRATE_VIA_REPORT_SERVICE:-1}" export PYTHON="${PYTHON:-$ROOT/.venv/bin/python}" diff --git a/scripts/local-run-common.sh b/scripts/local-run-common.sh index fe500158..397e9c39 100644 --- a/scripts/local-run-common.sh +++ b/scripts/local-run-common.sh @@ -36,12 +36,12 @@ start_host_dotnet_base() { export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}" export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}" export REPORT_SERVICE_URL="${REPORT_SERVICE_URL:-http://127.0.0.1:8094}" - export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8097}" - free_port 8080 - printf '\033[1;36m→\033[0m Starting FileService on port 8080\n' + free_port 8097 + printf '\033[1;36m→\033[0m Starting FileService on port 8097\n' (cd "$root/services/FileService" && \ - ASPNETCORE_URLS="http://127.0.0.1:8080" \ + ASPNETCORE_URLS="http://127.0.0.1:8097" \ ASPNETCORE_ENVIRONMENT="$mode" \ dotnet run --project src/FileService.Api --no-launch-profile) & FILE_SERVICE_PID=$! @@ -76,7 +76,7 @@ start_host_dotnet_base() { dotnet run --project src/AiService.Api --no-launch-profile) & AI_PID=$! - wait_for_http "http://127.0.0.1:8080/health" "FileService" + wait_for_http "http://127.0.0.1:8097/health" "FileService" wait_for_http "http://127.0.0.1:8091/health" "Data service" wait_for_http "http://127.0.0.1:8095/health" "ConfigService" wait_for_http "http://127.0.0.1:8092/health" "AiService" @@ -175,6 +175,6 @@ stop_host_dotnet_stack() { CONFIG_PID="" "$stop_service_fn" "Data" "${DATA_PID:-}" 8091 DATA_PID="" - "$stop_service_fn" "FileService" "${FILE_SERVICE_PID:-}" 8080 + "$stop_service_fn" "FileService" "${FILE_SERVICE_PID:-}" 8097 FILE_SERVICE_PID="" } diff --git a/scripts/local-run.ps1 b/scripts/local-run.ps1 index 62b5f0f3..4ba40ff8 100644 --- a/scripts/local-run.ps1 +++ b/scripts/local-run.ps1 @@ -256,10 +256,10 @@ function Invoke-Start { Invoke-WebDeps $bffBase = if ($env:VITE_BFF_BASE_URL) { $env:VITE_BFF_BASE_URL } else { "http://localhost:8090" } - $fileServiceUrl = if ($env:FILE_SERVICE_URL) { $env:FILE_SERVICE_URL } else { "http://127.0.0.1:8080" } + $fileServiceUrl = if ($env:FILE_SERVICE_URL) { $env:FILE_SERVICE_URL } else { "http://127.0.0.1:8097" } if (Get-Command dotnet -ErrorAction SilentlyContinue) { - Write-Log "Starting FileService on port 8080" + Write-Log "Starting FileService on port 8097" $env:REPORT_API_URL = "http://127.0.0.1:8096" Start-Process -FilePath "dotnet" ` -ArgumentList @("run", "--project", "src/FileService.Api", "--no-launch-profile") ` diff --git a/scripts/local-run.sh b/scripts/local-run.sh index fccbac84..587e9414 100755 --- a/scripts/local-run.sh +++ b/scripts/local-run.sh @@ -261,7 +261,7 @@ cmd_start() { export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}" export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}" - export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8097}" export REPORT_SERVICE_URL="${REPORT_SERVICE_URL:-http://127.0.0.1:8094}" export PIPELINE_ORCHESTRATE_VIA_REPORT_SERVICE="${PIPELINE_ORCHESTRATE_VIA_REPORT_SERVICE:-1}" export PYTHON="${PYTHON:-$VENV/bin/python}" @@ -292,10 +292,10 @@ cmd_start() { log "DATA_DIR=$DATA_DIR" log "PYTHON=$PYTHON" log "VITE_BFF_BASE_URL=${VITE_BFF_BASE_URL:-http://localhost:8090}" - log "FILE_SERVICE_URL=${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + log "FILE_SERVICE_URL=${FILE_SERVICE_URL:-http://127.0.0.1:8097}" log "REPORT_SERVICE_URL=${REPORT_SERVICE_URL:-http://127.0.0.1:8094}" log "DATA_ROUTES=${DATA_ROUTES:-/api/report/meta,...}" - export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8097}" export VITE_BFF_BASE_URL="${VITE_BFF_BASE_URL:-http://localhost:8090}" cd "$WEB" set +e diff --git a/services/Bff/src/Bff.Api/appsettings.json b/services/Bff/src/Bff.Api/appsettings.json index e66f1283..b8f40616 100644 --- a/services/Bff/src/Bff.Api/appsettings.json +++ b/services/Bff/src/Bff.Api/appsettings.json @@ -2,7 +2,7 @@ "Urls": "http://+:8090", "Upstream": { "FastApiBaseUrl": "http://127.0.0.1:8096", - "FileServiceBaseUrl": "http://127.0.0.1:8080", + "FileServiceBaseUrl": "http://127.0.0.1:8097", "TimeoutSeconds": 120 }, "Auth": { diff --git a/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs b/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs index 831f6808..54c91f31 100644 --- a/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs +++ b/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs @@ -12,7 +12,7 @@ public sealed class UpstreamOptions public string FastApiBaseUrl { get; set; } = "http://127.0.0.1:8096"; /// FileService base URL (env override: FILE_SERVICE_URL). - public string FileServiceBaseUrl { get; set; } = "http://127.0.0.1:8080"; + public string FileServiceBaseUrl { get; set; } = "http://127.0.0.1:8097"; /// Timeout for normal (non-streaming) upstream calls. Parity with the TS proxy (120s). public int TimeoutSeconds { get; set; } = 120; diff --git a/services/Bff/tests/Bff.Tests/ServiceRegistrationValidationTests.cs b/services/Bff/tests/Bff.Tests/ServiceRegistrationValidationTests.cs index 38e27553..12182b2a 100644 --- a/services/Bff/tests/Bff.Tests/ServiceRegistrationValidationTests.cs +++ b/services/Bff/tests/Bff.Tests/ServiceRegistrationValidationTests.cs @@ -13,7 +13,7 @@ public void Web_host_resolves_core_services() { using var env = ServiceRegistrationTestEnvironment.Push(); env.SetDefaultsForPostgresServices(); - env.Set("FILE_SERVICE_URL", "http://127.0.0.1:8080"); + env.Set("FILE_SERVICE_URL", "http://127.0.0.1:8097"); env.Set("DATA_SERVICE_URL", "http://127.0.0.1:8091"); env.Set("AI_SERVICE_URL", "http://127.0.0.1:8092"); env.Set("INTEGRATIONS_SERVICE_URL", "http://127.0.0.1:8093"); diff --git a/services/FileService/Dockerfile b/services/FileService/Dockerfile index c09f2268..b23c0143 100644 --- a/services/FileService/Dockerfile +++ b/services/FileService/Dockerfile @@ -12,7 +12,7 @@ RUN dotnet publish FileService/src/FileService.Api/FileService.Api.csproj -c Rel FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime WORKDIR /app -ENV ASPNETCORE_URLS=http://+:8080 -EXPOSE 8080 +ENV ASPNETCORE_URLS=http://+:8097 +EXPOSE 8097 COPY --from=build /app/publish . ENTRYPOINT ["dotnet", "FileService.Api.dll"] diff --git a/services/FileService/README.md b/services/FileService/README.md index f575d882..e25a200c 100644 --- a/services/FileService/README.md +++ b/services/FileService/README.md @@ -14,16 +14,16 @@ export REPORT_API_URL=http://127.0.0.1:8096 dotnet run --project src/FileService.Api ``` -Service listens on **http://localhost:8080** (`ASPNETCORE_URLS` / `appsettings.json`). +Service listens on **http://localhost:8097** (`ASPNETCORE_URLS` / `appsettings.json`). -In **Development**, Swagger UI is at **http://localhost:8080/docs** and the OpenAPI JSON at **http://localhost:8080/swagger/v1/swagger.json**. +In **Development**, Swagger UI is at **http://localhost:8097/docs** and the OpenAPI JSON at **http://localhost:8097/swagger/v1/swagger.json**. ## Environment variables | Variable | Default | Description | |----------|---------|-------------| | `REPORT_API_URL` | `http://127.0.0.1:8096` | Base URL for report payload, meta, and ui-preferences HTTP API | -| `ASPNETCORE_URLS` | `http://127.0.0.1:8080` | Bind address (Docker sets `http://+:8080`) | +| `ASPNETCORE_URLS` | `http://127.0.0.1:8097` | Bind address (Docker sets `http://+:8097`) | ## Upstream HTTP contract diff --git a/services/FileService/src/FileService.Api/Properties/launchSettings.json b/services/FileService/src/FileService.Api/Properties/launchSettings.json index f90e2975..4bf9048a 100644 --- a/services/FileService/src/FileService.Api/Properties/launchSettings.json +++ b/services/FileService/src/FileService.Api/Properties/launchSettings.json @@ -6,7 +6,7 @@ "dotnetRunMessages": true, "launchBrowser": true, "launchUrl": "docs", - "applicationUrl": "http://localhost:8080", + "applicationUrl": "http://localhost:8097", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } diff --git a/services/FileService/src/FileService.Api/appsettings.json b/services/FileService/src/FileService.Api/appsettings.json index 8523417e..33fa3663 100644 --- a/services/FileService/src/FileService.Api/appsettings.json +++ b/services/FileService/src/FileService.Api/appsettings.json @@ -1,5 +1,5 @@ { - "Urls": "http://+:8080", + "Urls": "http://+:8097", "ReportApi": { "BaseUrl": "http://127.0.0.1:8096", "TimeoutSeconds": 120 diff --git a/src/website_profiling/clients/file_service.py b/src/website_profiling/clients/file_service.py index 3285c3b1..0a7b704a 100644 --- a/src/website_profiling/clients/file_service.py +++ b/src/website_profiling/clients/file_service.py @@ -7,7 +7,7 @@ import requests -_DEFAULT_BASE = "http://127.0.0.1:8080" +_DEFAULT_BASE = "http://127.0.0.1:8097" _TIMEOUT_SECONDS = 120 From 575f665b9b51e81da3e47f37823ccb85ee2568ca Mon Sep 17 00:00:00 2001 From: PrashantUnity Date: Sun, 28 Jun 2026 13:10:24 +0530 Subject: [PATCH 6/6] Organising Diagram --- diagram.html | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/diagram.html b/diagram.html index 01802468..6c7bb005 100644 --- a/diagram.html +++ b/diagram.html @@ -193,24 +193,30 @@

}; const nodesData = [ - { id: 'browser', label: 'Browser', sub: 'Client', x: 550, y: 50, type: 'client', desc: 'End-user web browser interacting with the application.' }, - { id: 'web', label: 'Web', sub: ':3000 • React SPA', x: 550, y: 160, type: 'frontend', desc: 'Main frontend single-page application built in React.' }, - { id: 'bff', label: 'BFF', sub: ':8090 • Auth • Proxy', x: 550, y: 280, type: 'gateway', desc: 'Backend-for-Frontend serving as the entry point. Handles authentication, CORS, and reverse proxy routing.' }, - - { id: 'fastapi', label: 'FastAPI', sub: ':8096 • Crawl/Jobs', x: 50, y: 450, type: 'service', desc: 'Core Python service handling crawling, jobs, and pipeline settings. Acts as the strangler proxy target.' }, - { id: 'config', label: 'ConfigService', sub: ':8095 • Typed Config', x: 250, y: 450, type: 'service', desc: 'Typed configuration service exposing pipeline, crawl, Lighthouse, and report settings stored in PostgreSQL typed tables. Serves GET/PUT /api/pipeline-config and /api/pipeline-settings.' }, - { id: 'report', label: 'Report', sub: ':8094 • Build', x: 450, y: 450, type: 'service', desc: 'Service dedicated to report building and orchestrating complex multi-step processes.' }, - { id: 'integrations', label: 'Integrations', sub: ':8093 • APIs', x: 650, y: 450, type: 'service', desc: 'Manages third-party connections including Google/Bing APIs, GSC, GA4, and keyword processing.' }, - { id: 'ai', label: 'AI', sub: ':8092 • Chat/LLM', x: 850, y: 450, type: 'service', desc: 'AI orchestration service managing chat interfaces, LLM connections, secrets, and MCP.' }, - { id: 'data', label: 'Data', sub: ':8091 • Portfolio', x: 1050, y: 450, type: 'service', desc: 'Data access layer for report reads, portfolio management, and system issue tracking.' }, - { id: 'files', label: 'Files', sub: ':8097 • Exports', x: 1250, y: 450, type: 'service', desc: 'Handles document generation, including PDF and Excel exports.' }, - - { id: 'worker', label: 'Worker', sub: 'Crawl + Lighthouse', x: 350, y: 640, type: 'worker', desc: 'Background asynchronous pipeline worker executing heavy crawl and Lighthouse audit jobs.' }, - - { id: 'postgres', label: 'Postgres', sub: 'Audit • Settings', x: 450, y: 820, type: 'database', desc: 'Primary relational data store holding audit data and typed system settings.' }, - { id: 'google', label: 'Google / Bing APIs', sub: 'External', x: 650, y: 640, type: 'external', desc: 'External search and analytics APIs consumed by the Integrations service.' }, - { id: 'llm', label: 'LLM', sub: 'Ollama', x: 850, y: 640, type: 'external', desc: 'Large Language Model provider, heavily utilized for text and chat generation.' }, - { id: 'mcp', label: 'MCP', sub: 'IDE / Agent', x: 850, y: 280, type: 'client', desc: 'Model Context Protocol or Machine Control client interacting with the AI subsystem via stdio or HTTP.' } + // Top — browser stack, centred on x=650 + { id: 'browser', label: 'Browser', sub: 'Client', x: 560, y: 40, type: 'client', desc: 'End-user web browser interacting with the application.' }, + { id: 'web', label: 'Web', sub: ':3000 • React SPA', x: 560, y: 160, type: 'frontend', desc: 'Main frontend single-page application built in React.' }, + { id: 'bff', label: 'BFF', sub: ':8090 • Auth • Proxy', x: 560, y: 290, type: 'gateway', desc: 'Backend-for-Frontend serving as the entry point. Handles authentication, CORS, and reverse proxy routing.' }, + // MCP to the right of BFF + { id: 'mcp', label: 'MCP', sub: 'IDE / Agent', x: 980, y: 290, type: 'client', desc: 'Model Context Protocol or Machine Control client interacting with the AI subsystem via stdio or HTTP.' }, + + // Services row — 7 nodes, 200 px apart, centred under BFF + // span = 6 × 200 = 1200 → start at 650 − 600 = 50 + { id: 'files', label: 'Files', sub: ':8097 • Exports', x: 50, y: 470, type: 'service', desc: 'Handles document generation, including PDF and Excel exports.' }, + { id: 'fastapi', label: 'FastAPI', sub: ':8096 • Crawl/Jobs', x: 250, y: 470, type: 'service', desc: 'Core Python service handling crawling, jobs, and pipeline settings. Acts as the strangler proxy target.' }, + { id: 'config', label: 'ConfigService',sub: ':8095 • Typed Config',x: 450, y: 470, type: 'service', desc: 'Typed configuration service exposing pipeline, crawl, Lighthouse, and report settings stored in PostgreSQL typed tables. Serves GET/PUT /api/pipeline-config and /api/pipeline-settings.' }, + { id: 'report', label: 'Report', sub: ':8094 • Build', x: 650, y: 470, type: 'service', desc: 'Service dedicated to report building and orchestrating complex multi-step processes.' }, + { id: 'integrations', label: 'Integrations', sub: ':8093 • APIs', x: 850, y: 470, type: 'service', desc: 'Manages third-party connections including Google/Bing APIs, GSC, GA4, and keyword processing.' }, + { id: 'ai', label: 'AI', sub: ':8092 • Chat/LLM', x:1050, y: 470, type: 'service', desc: 'AI orchestration service managing chat interfaces, LLM connections, secrets, and MCP.' }, + { id: 'data', label: 'Data', sub: ':8091 • Portfolio', x:1250, y: 470, type: 'service', desc: 'Data access layer for report reads, portfolio management, and system issue tracking.' }, + + // Middle-lower row — Worker under FastAPI/Config, external services under Integrations/AI + { id: 'worker', label: 'Worker', sub: 'Crawl + Lighthouse', x: 350, y: 660, type: 'worker', desc: 'Background asynchronous pipeline worker executing heavy crawl and Lighthouse audit jobs.' }, + { id: 'google', label: 'Google / Bing APIs', sub: 'External', x: 850, y: 660, type: 'external', desc: 'External search and analytics APIs consumed by the Integrations service.' }, + { id: 'llm', label: 'LLM', sub: 'Ollama', x:1050, y: 660, type: 'external', desc: 'Large Language Model provider, heavily utilized for text and chat generation.' }, + + // Bottom — Postgres centred + { id: 'postgres', label: 'Postgres', sub: 'Audit • Settings', x: 560, y: 860, type: 'database', desc: 'Primary relational data store holding audit data and typed system settings.' }, ]; const edgesData = [