diff --git a/.github/workflows/schema-drift.yml b/.github/workflows/schema-drift.yml new file mode 100644 index 0000000..7e099b1 --- /dev/null +++ b/.github/workflows/schema-drift.yml @@ -0,0 +1,31 @@ +name: Schema drift check + +# Check if the MCD source file and the database schema (DDL) are in sync. +# Fail the build if a Liquibase table column is missing from the Merise MCD +# diagram source. The check is pure-stdlib Python, so no project build is needed. +# Runs only when the database schema, the MCD diagram source, or the check script change. + +on: + pull_request: + branches: [dev] + paths: + - "src/main/resources/db/changelog/changes/**" + - "docs/database/merise/learn-dev.mcd" + - "scripts/check_schema_drift.py" + push: + branches: [dev, main] + paths: + - "src/main/resources/db/changelog/changes/**" + - "docs/database/merise/learn-dev.mcd" + - "scripts/check_schema_drift.py" + +jobs: + schema-drift: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Check MCD vs Liquibase column drift + run: python3 scripts/check_schema_drift.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f2f51d3..c91f5a1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -216,7 +216,9 @@ learn-dev/ │ │ │ ├── application-prod.yml # Prod profile (external DB, stricter security) │ │ │ └── db/ │ │ │ └── changelog/ # Liquibase migration files (when introduced) -│ │ │ └── db.changelog-master.yaml +│ │ │ ├── db.changelog-master.yaml +│ │ │ ├── changes/ +│ │ │ └── V20260608161836-create-users-table.sql # Database migration file │ │ │ │ │ │ │ │ └── test/ @@ -336,6 +338,35 @@ The main advantages in my opinion are: The *learn-dev* platform uses a **PostgreSQL** relational database to persist entities. +#### Database Migrations (Liquibase) + +The database schema is managed with **Liquibase**. Migrations live in +`src/main/resources/db/changelog/`: + +- `db.changelog-master.yaml` includes every change file via `includeAll` on the + `changes/` directory (applied in filename order). +- Change files use **formatted SQL** and are named with a timestamp prefix: + + ``` + VYYYYMMDDHHMMSS-short-description.sql + ``` + + e.g. `V20260608161842-create-idx-user-roles-role-id.sql`. The `V` + UTC + timestamp keeps files ordered chronologically and avoids numbering collisions + when branches add migrations in parallel. +- **One changeset per file** (atomic): each file contains a single + `--changeset` so it can be rolled back independently. +- The **changeset id equals the filename's timestamp**, e.g. + `--changeset ebouchut:V20260608161842`. This keeps the id globally unique and + trivially traceable to its file. +- Migrations are **append-only**: never edit a changeset that has already run on + a shared database — add a new one. Each changeset has a `--rollback`. +- After adding or removing a column, update the MCD diagram source + (`docs/database/merise/learn-dev.mcd`) to match, then run + `make check-schema-drift` to verify every table column is represented in the + diagram. CI runs this check too. + + #### Database Naming Conventions Here are the naming conventions for the **name** of our database **tables**: diff --git a/Makefile b/Makefile index f6f96da..231d2d1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Ignore existing files with the same name as phony targets -.PHONY: help diagrams mcd mld mpd clean +.PHONY: help diagrams mcd mld mpd clean check-schema-drift # Default make target used if none specified .DEFAULT_GOAL := help @@ -12,11 +12,17 @@ help: @echo " make mld — generate MLD" @echo " make mpd — generate MPD" @echo " make clean — remove generated diagrams" + @echo " make check-schema-drift — fail if a Liquibase column is missing from the MCD" # Generate all database diagrams (MCD, MLD, MPD) diagrams: mcd mld mpd @echo "All diagrams generated (MCD, MLD, MPD)" +# Fail if a Liquibase table column is missing from the MCD diagram source. +# Heuristic (column-name presence only); CI-friendly (non-zero exit on drift). +check-schema-drift: + python3 scripts/check_schema_drift.py + # Generate MCD from Mocodo source mcd: @echo "Generating MCD..." diff --git a/scripts/check_schema_drift.py b/scripts/check_schema_drift.py new file mode 100755 index 0000000..8771a2f --- /dev/null +++ b/scripts/check_schema_drift.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Detect drift between the Liquibase schema and the Merise MCD diagram source. + +Forward check only: report every column defined in a Liquibase `CREATE TABLE` +changeset whose name does not appear anywhere in the MCD source. This catches +the common mistake of adding a column to a migration but forgetting to update +`learn-dev.mcd` (so the generated SVG/MLD diagrams omit it). + +Heuristic, not a full schema diff: it checks column-NAME presence only, not the +owning entity, type, nullability, or order. Exit code 1 on drift, 0 when clean. + +Usage: python3 scripts/check_schema_drift.py +""" + +import glob +import re +import sys + +CHANGESETS_GLOB = "src/main/resources/db/changelog/changes/V*-create-*-table.sql" +MCD_PATH = "docs/database/merise/learn-dev.mcd" + +# Leading tokens that start a constraint clause rather than a column definition. +NON_COLUMN_KEYWORDS = {"primary", "constraint", "foreign", "unique", "check"} + + +def changeset_columns() -> dict[str, list[str]]: + """Map each table name to the list of column names in its CREATE TABLE.""" + tables: dict[str, list[str]] = {} + for path in sorted(glob.glob(CHANGESETS_GLOB)): + match = re.search(r"CREATE TABLE (\w+)\s*\((.*?)\);", open(path).read(), re.S) + if not match: + continue + table, body = match.group(1), match.group(2) + columns = [] + for line in body.splitlines(): + token = re.match(r"\s*([a-z_]+)\b", line) + if token and token.group(1).lower() not in NON_COLUMN_KEYWORDS: + columns.append(token.group(1)) + tables[table] = columns + return tables + + +def mcd_tokens() -> set[str]: + """All lowercase identifier tokens present in the MCD source.""" + return set(re.findall(r"[a-z_]+", open(MCD_PATH).read())) + + +def main() -> int: + tables = changeset_columns() + if not tables: + print(f"ERROR: no changesets matched {CHANGESETS_GLOB}", file=sys.stderr) + return 2 + known = mcd_tokens() + + drift = [ + (table, column) + for table, columns in sorted(tables.items()) + for column in columns + if column not in known + ] + + if not drift: + n = sum(len(c) for c in tables.values()) + print(f"OK: MCD represents all {n} columns across {len(tables)} tables.") + return 0 + + print("DRIFT: columns in the schema but missing from the MCD " + f"({MCD_PATH}):", file=sys.stderr) + for table, column in drift: + print(f" - {table}.{column}", file=sys.stderr) + print("\nAdd the column(s) to the matching entity in the MCD, then " + "`make mcd && make mld`.", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/main/resources/application-dev.yaml b/src/main/resources/application-dev.yaml index e90078f..f8ee083 100644 --- a/src/main/resources/application-dev.yaml +++ b/src/main/resources/application-dev.yaml @@ -25,6 +25,13 @@ spring: # create-target: schema.sql # create-source: metadata + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Set the execution context for Liquibase changeset + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + liquibase: + contexts: dev + + # ~~~~~~~~~~~~~~~~~~ # Logging Level # ~~~~~~~~~~~~~~~~~~ diff --git a/src/main/resources/application-prod.yaml b/src/main/resources/application-prod.yaml new file mode 100644 index 0000000..e052302 --- /dev/null +++ b/src/main/resources/application-prod.yaml @@ -0,0 +1,47 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# application-prod.yaml +# +# Application configuration file for the dev profile. +# Properties defined in this file override the ones defined +# in application.yaml. +# +# To activate the dev profile: +# - Add SPRING_PROFILES_ACTIVE=dev to your .env file +# - or set the environment variable SPRING_PROFILES_ACTIVE=dev +# - or run: mvn spring-boot:run -Dspring-boot.run.profiles=dev +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +spring: + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Hibernate (ORM) Configuration + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + jpa: + show-sql: false + + # Generate DDL and write to file + # properties: + # javax.persistence.schema-generation.scripts: + # action: create + # create-target: schema.sql + # create-source: metadata + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Set the execution context for Liquibase changeset + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + liquibase: + contexts: prod + + +# ~~~~~~~~~~~~~~~~~~ +# Logging Level +# ~~~~~~~~~~~~~~~~~~ +logging: + level: + # Log Hibernate SQL statements + org.hibernate.SQL: WARN + + # Log incoming Web request details + web: WARN + + # App logging level + com.ericbouchut.learndev: WARN diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 905dfb2..083cf1f 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -20,14 +20,20 @@ spring: # Hibernate (ORM) Configuration # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ jpa: - hibernate.ddl-auto: validate + drop-first: false # Never drop the database show-sql: false + hibernate: + ddl-auto: validate # Ensure the database and its schema are in sync # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - # Database schema changes Configuration + # Database schema change manager + # See: https://www.liquibase.com/ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ liquibase: - enabled: false + enabled: true + change-log: classpath:db/changelog/db.changelog-master.yaml + drop-first: false + # ~~~~~~~~~~~~~~~~~~ # Logging Level diff --git a/src/main/resources/db/changelog/changes/V20260608161836-create-users-table.sql b/src/main/resources/db/changelog/changes/V20260608161836-create-users-table.sql new file mode 100644 index 0000000..95e59b0 --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161836-create-users-table.sql @@ -0,0 +1,19 @@ +--liquibase formatted sql + +-- users: application accounts. UUID PK — non-enumerable and future-API-safe (ADR-0003). +--changeset ebouchut:V20260608161836 +CREATE TABLE users ( + user_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + username VARCHAR(50) NOT NULL UNIQUE, + email VARCHAR(255) NOT NULL UNIQUE, + password VARCHAR(255) NOT NULL, -- bcrypt/argon2 hash, never plaintext + first_name VARCHAR(100), + last_name VARCHAR(100), + is_active BOOLEAN NOT NULL DEFAULT TRUE, + is_verified BOOLEAN NOT NULL DEFAULT FALSE, + is_locked BOOLEAN NOT NULL DEFAULT FALSE, + failed_login_attempts INTEGER NOT NULL DEFAULT 0, + last_login_at TIMESTAMPTZ, + password_changed_at TIMESTAMPTZ +); +--rollback DROP TABLE users; diff --git a/src/main/resources/db/changelog/changes/V20260608161837-create-roles-table.sql b/src/main/resources/db/changelog/changes/V20260608161837-create-roles-table.sql new file mode 100644 index 0000000..6bb8116 --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161837-create-roles-table.sql @@ -0,0 +1,11 @@ +--liquibase formatted sql + +-- roles: application roles (e.g. STUDENT, INSTRUCTOR, ADMIN). BIGINT identity PK. +--changeset ebouchut:V20260608161837 +CREATE TABLE roles ( + role_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + role_name VARCHAR(50) NOT NULL UNIQUE, + description VARCHAR(255), + is_active BOOLEAN NOT NULL DEFAULT TRUE +); +--rollback DROP TABLE roles; diff --git a/src/main/resources/db/changelog/changes/V20260608161838-create-user-roles-table.sql b/src/main/resources/db/changelog/changes/V20260608161838-create-user-roles-table.sql new file mode 100644 index 0000000..85880b7 --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161838-create-user-roles-table.sql @@ -0,0 +1,12 @@ +--liquibase formatted sql + +-- user_roles: junction table for the N..N relationship between users and roles. +--changeset ebouchut:V20260608161838 +CREATE TABLE user_roles ( + user_id UUID NOT NULL REFERENCES users (user_id) ON DELETE CASCADE, + role_id BIGINT NOT NULL REFERENCES roles (role_id) ON DELETE CASCADE, + assigned_at TIMESTAMPTZ NOT NULL DEFAULT now(), + assigned_by UUID REFERENCES users (user_id) ON DELETE SET NULL, + PRIMARY KEY (user_id, role_id) +); +--rollback DROP TABLE user_roles; diff --git a/src/main/resources/db/changelog/changes/V20260608161839-create-email-tokens-table.sql b/src/main/resources/db/changelog/changes/V20260608161839-create-email-tokens-table.sql new file mode 100644 index 0000000..b2d7ff2 --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161839-create-email-tokens-table.sql @@ -0,0 +1,12 @@ +--liquibase formatted sql + +-- email_tokens: email-verification tokens. The secret is the random `token` column. +--changeset ebouchut:V20260608161839 +CREATE TABLE email_tokens ( + token_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + token VARCHAR(255) NOT NULL UNIQUE, + expires_at TIMESTAMPTZ NOT NULL, + used_at TIMESTAMPTZ, + user_id UUID NOT NULL REFERENCES users (user_id) ON DELETE CASCADE +); +--rollback DROP TABLE email_tokens; diff --git a/src/main/resources/db/changelog/changes/V20260608161840-create-reset-tokens-table.sql b/src/main/resources/db/changelog/changes/V20260608161840-create-reset-tokens-table.sql new file mode 100644 index 0000000..2369bc1 --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161840-create-reset-tokens-table.sql @@ -0,0 +1,13 @@ +--liquibase formatted sql + +-- reset_tokens: password-reset tokens. Like email_tokens, plus the requester's IP. +--changeset ebouchut:V20260608161840 +CREATE TABLE reset_tokens ( + token_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + token VARCHAR(255) NOT NULL UNIQUE, + expires_at TIMESTAMPTZ NOT NULL, + used_at TIMESTAMPTZ, + ip_address INET, + user_id UUID NOT NULL REFERENCES users (user_id) ON DELETE CASCADE +); +--rollback DROP TABLE reset_tokens; diff --git a/src/main/resources/db/changelog/changes/V20260608161841-create-audit-logs-table.sql b/src/main/resources/db/changelog/changes/V20260608161841-create-audit-logs-table.sql new file mode 100644 index 0000000..a5d578a --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161841-create-audit-logs-table.sql @@ -0,0 +1,21 @@ +--liquibase formatted sql + +-- audit_logs: security audit trail. user_id is nullable (system events) and uses +-- ON DELETE SET NULL so the trail survives user deletion. entity_id is text +-- because it may reference a UUID (users) or a BIGINT (other tables). +--changeset ebouchut:V20260608161841 +CREATE TABLE audit_logs ( + log_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + action_type VARCHAR(100) NOT NULL, + entity_type VARCHAR(100), + entity_id VARCHAR(64), + description TEXT, + ip_address INET, + user_agent TEXT, + was_successful BOOLEAN NOT NULL DEFAULT TRUE, + error_message TEXT, + metadata JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + user_id UUID REFERENCES users (user_id) ON DELETE SET NULL +); +--rollback DROP TABLE audit_logs; diff --git a/src/main/resources/db/changelog/changes/V20260608161842-create-idx-user-roles-role-id.sql b/src/main/resources/db/changelog/changes/V20260608161842-create-idx-user-roles-role-id.sql new file mode 100644 index 0000000..5f94374 --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161842-create-idx-user-roles-role-id.sql @@ -0,0 +1,6 @@ +--liquibase formatted sql + +-- Index on the user_roles.role_id foreign key (Postgres does not auto-index FKs). +--changeset ebouchut:V20260608161842 +CREATE INDEX idx_user_roles_role_id ON user_roles (role_id); +--rollback DROP INDEX idx_user_roles_role_id; diff --git a/src/main/resources/db/changelog/changes/V20260608161843-create-idx-email-tokens-user-id.sql b/src/main/resources/db/changelog/changes/V20260608161843-create-idx-email-tokens-user-id.sql new file mode 100644 index 0000000..40d0fd9 --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161843-create-idx-email-tokens-user-id.sql @@ -0,0 +1,6 @@ +--liquibase formatted sql + +-- Index on the email_tokens.user_id foreign key. +--changeset ebouchut:V20260608161843 +CREATE INDEX idx_email_tokens_user_id ON email_tokens (user_id); +--rollback DROP INDEX idx_email_tokens_user_id; diff --git a/src/main/resources/db/changelog/changes/V20260608161844-create-idx-reset-tokens-user-id.sql b/src/main/resources/db/changelog/changes/V20260608161844-create-idx-reset-tokens-user-id.sql new file mode 100644 index 0000000..40e567e --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161844-create-idx-reset-tokens-user-id.sql @@ -0,0 +1,6 @@ +--liquibase formatted sql + +-- Index on the reset_tokens.user_id foreign key. +--changeset ebouchut:V20260608161844 +CREATE INDEX idx_reset_tokens_user_id ON reset_tokens (user_id); +--rollback DROP INDEX idx_reset_tokens_user_id; diff --git a/src/main/resources/db/changelog/changes/V20260608161845-create-idx-audit-logs-user-id.sql b/src/main/resources/db/changelog/changes/V20260608161845-create-idx-audit-logs-user-id.sql new file mode 100644 index 0000000..0573946 --- /dev/null +++ b/src/main/resources/db/changelog/changes/V20260608161845-create-idx-audit-logs-user-id.sql @@ -0,0 +1,6 @@ +--liquibase formatted sql + +-- Index on the audit_logs.user_id foreign key. +--changeset ebouchut:V20260608161845 +CREATE INDEX idx_audit_logs_user_id ON audit_logs (user_id); +--rollback DROP INDEX idx_audit_logs_user_id; diff --git a/src/main/resources/db/changelog/db.changelog-master.yaml b/src/main/resources/db/changelog/db.changelog-master.yaml new file mode 100644 index 0000000..61cf11a --- /dev/null +++ b/src/main/resources/db/changelog/db.changelog-master.yaml @@ -0,0 +1,5 @@ +databaseChangeLog: + - includeAll: + relativeToChangelogFile: true + # `path` (where Liquibase reads migration files) is relative to `db.changelog-master.yaml` + path: changes/