From 9dceea0ea9e36b6ab970dfa35137c733762b3e83 Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Mon, 10 Aug 2026 20:06:00 +0200 Subject: [PATCH 1/2] Add a sqlite3 to PostgreSQL migration and document the database Towards a network based SQL database for the library testing, issue #295. sqlite2postgres.py copies the per-machine sqlite3 databases into PostgreSQL. The PostgreSQL layout is a 1:1 mirror of the sqlite3 one - one table per branch with the same column names, only the types adapted - so that the test scripts can push new results to the network database with the statements they already use. The two machines are merged into the same tables, ripper1 first and ripper2 with --skip-existing, so that the row already in the database wins whenever a key collides. The keys the migration adds - (date, libname, model) for a branch table - are what makes a shared database possible at all: they hold across all 160 million migrated rows, and they stop a run that is pushed twice from storing its results twice. The script needs nothing but the standard library and the psql client, streams the rows through COPY in batches, and commits each batch together with its progress row in [migration_progress], so a migration that is interrupted continues where it stopped instead of duplicating rows. --index creates the keys and the index the reports need, --verify compares the row counts of both sides. --catch-up copies the runs a machine wrote after the migration read its database, which a test still using its sqlite3 file keeps doing until the jobs are switched over: a wasm-jit run of 19527 models finished right after the migration and was missing from the shared database until it was caught up. It picks the runs by date rather than by continuing from the rowid it stopped at, because VACUUM renumbers the rowids of these tables and clean-empty-omcversion-dates.py runs one after every test, and because two machines write into the same table for the branches they both test. Reading a whole database costs two to three minutes, and the keys make repeating it harmless. COPY is used in its text format rather than CSV because an empty CSV field reads back as NULL, which would silently turn the empty libversion strings of the old data into NULLs. For the same reason only the key columns are NOT NULL: the sqlite3 tables declare every column NOT NULL, but tables created by an older test.py kept their laxer declaration and the historical rows do not hold up. doc/README.md documents what is actually in the database: the per-branch tables, omcversion and libversion, what every column means, how finalphase maps to the phase names, how a test run writes its results, and how to run the migration. --- Generated by Claude Code. Signed-off-by: Adrian Pop --- doc/README.md | 350 ++++++++++++++++++++++++++++++++++++++ sqlite2postgres.py | 406 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 756 insertions(+) create mode 100644 doc/README.md create mode 100644 sqlite2postgres.py diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..363906d --- /dev/null +++ b/doc/README.md @@ -0,0 +1,350 @@ +# The OpenModelicaLibraryTesting result database + +This document describes how the library testing results are stored: the current +sqlite3 layout, what every table and column means, how the scripts query it, and +the PostgreSQL layout the data is being migrated to (issue +[#295](https://github.com/OpenModelica/OpenModelicaLibraryTesting/issues/295)). + +## Where the databases live + +Each test machine keeps a single sqlite3 file called `sqlite3.db` in the working +directory of the test run. The files are published at: + +| machine | URL | on omod-r630-2 | size | tables | rows | +| --- | --- | --- | --- | --- | --- | +| ripper1 | | `/var/www/libraries.openmodelica.org/sqlite3/ripper1/sqlite3.db` | 11.8 GB | 34 | ~69 million | +| ripper2 | | `/var/www/libraries.openmodelica.org/sqlite3/ripper2/sqlite3.db` | 15.5 GB | 54 | ~91 million | + +(sizes as of 2026-08-10; both are at `PRAGMA user_version` 3) + +Every branch/configuration tested on a machine ends up in that machine's file, +which is why the files are large and why two machines cannot test the same job +without overwriting each other's results. The tables are the branches currently +tested (`master`, `newInst-newBackend`, `cpp`, `master-fmi`, `gbode`, `cvode`, +`ida`, `daemode`, ...) plus one per historical release (`v1.9` ... `v1.27`, +`v1.11-fmi` ...). `master` alone holds ~43 million rows on ripper1. + +Six table names exist on both machines - `master`, `newInst`, `heavy_tests`, +`v1.17`, `libversion` and `omcversion` - so a shared database mixes rows from +both. They are distinguished by their `date`, which is unique per run and +machine. + +The scripts always open the file by the hardcoded relative name `sqlite3.db`: +`test.py` writes it, `report.py`, `all-reports.py`, `all-plots.py`, +`single-model.py`, `clean-dates.py` and `clean-empty-omcversion-dates.py` read +it. + +## Schema version + +`test.py` keeps the layout version in sqlite's `PRAGMA user_version` and +migrates on startup (`test.py`, around the `CREATE TABLE` block): + +| user_version | meaning | +| --- | --- | +| 0 | empty/new database; `omcversion` and `libversion` are created | +| 1 | `libversion.confighash` added | +| 2 | `parsing` added to every per-branch table | +| 3 | current layout | + +A database with a higher `user_version` makes `test.py` exit rather than guess. + +## Tables + +### Per-branch result tables + +There is **one table per tested branch/configuration**, named after the branch +(`master`, `gbode`, `ida`, `cvode`, `master-fmi`, `newInst-newBackend`, +`heavy_tests`, ...), created on demand by `test.py`: + +```sql +CREATE TABLE if not exists [] ( + date integer NOT NULL, -- unix epoch: start of the test run + libname text NOT NULL, -- library incl. version suffix, e.g. Buildings_9.1.0 + model text NOT NULL, -- full Modelica class name of the tested model + exectime real NOT NULL, -- wall clock for the whole test of this model [s] + frontend real NOT NULL, -- time in the front end [s] + backend real NOT NULL, -- time in the back end [s] + simcode real NOT NULL, -- time generating SimCode [s] + templates real NOT NULL, -- time running the code generation templates [s] + compile real NOT NULL, -- time compiling the generated code (or building the FMU) [s] + simulate real NOT NULL, -- time simulating [s] + verify real NOT NULL, -- time spent in diffSimulationResults [s] + verifyfail integer NOT NULL, -- number of variables that differ from the reference + verifytotal integer NOT NULL, -- number of variables compared against the reference + finalphase integer NOT NULL, -- how far the model got, see below + parsing real NOT NULL -- time loading/parsing the library [s] +) +``` + +Notes on the values, which are produced by `testmodel.py` and written by +`test.py`: + +- `date` is `int(time.time())` taken **once per test run** (`testRunStartTimeAsEpoch`), + so all rows of one run share the same date. It is the join key to + `omcversion`/`libversion` and the x-axis of every history plot. +- The phase times are exclusive, computed by subtracting the nested OMC timers + from each other (`frontend = frontend - backend`, `backend = backend - simcode`, ...). + A phase that was never reached is stored as `0.0`. +- `compile` is the `build` measurement: `make -f .makefile` for the C + runtime, the FMU build for FMI configurations, and the JIT compile time for + wasm-jit. +- `exectime` is the total wall clock of the model's test process. `test.py` + reads back the most recent value (`SELECT exectime ... ORDER BY date DESC LIMIT 1`) + to sort the queue longest-job-first. +- `verifyfail`/`verifytotal` are `len(diff.vars)` and `diff.numCompared`; a model + without reference variables stores `0`/`0` and still reaches phase 7. + +`finalphase` is the last phase completed; `shared.finalphaseName` maps it to: + +| value | name | meaning | +| --- | --- | --- | +| 0 | Failed | the front end did not finish | +| 1 | FrontEnd | front end ok, back end failed | +| 2 | BackEnd | back end ok, SimCode failed | +| 3 | SimCode | SimCode ok, templates/translation failed | +| 4 | Templates | translated, but compilation/build failed | +| 5 | Compile | built, but the simulation failed | +| 6 | Simulate | simulated, but the result does not verify (or was not compared) | +| 7 | Verify | the result matches the reference file | + +Reports count models per phase with `WHERE finalphase >= i`, so the columns of +the HTML tables are cumulative. + +### `omcversion` + +Maps a test run to the compiler that produced it: + +```sql +CREATE TABLE if not exists [omcversion] ( + date integer NOT NULL, -- same epoch as the result rows of that run + branch text NOT NULL, -- branch/configuration name = result table name + omcversion text NOT NULL -- output of getVersion(), e.g. "OMCompiler v1.26.0-dev.42+g0123abc" +) +``` + +One row per run. `report.py` and `all-reports.py` use it to label a run, and +`all-reports.py` walks it in date order to pair consecutive runs when generating +the regression reports. + +### `libversion` + +Maps a test run to the library versions and configuration used: + +```sql +CREATE TABLE if not exists [libversion] ( + date integer NOT NULL, -- same epoch as the result rows of that run + branch text NOT NULL, + libname text NOT NULL, -- as in the result table + libversion text NOT NULL, -- conf["libraryLastChange"]: version + git revision/zip hash + confighash integer NOT NULL -- hash of the configuration and the reference files +) +``` + +One row per (run, library). `confighash` is `strToHashInt()` over the +configuration dictionary plus the hashes of all reference files, so any change +to the config or to a reference file yields a different value. + +This drives the "do we need to test this at all" decision in `test.py`: before +testing a library it looks for + +```sql +SELECT date,libversion,libname,branch,omcversion FROM [libversion] NATURAL JOIN [omcversion] +WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1 +``` + +and skips the library when the exact same combination of library version, OMC +version and configuration was already tested. + +### `datelookup_` (obsolete) + +`datelookup_(date, runDate, libname, branch)` was a cache mapping every +omcversion date to the latest run date of a library. The code that fills it in +`all-plots.py` sits inside a triple-quoted block and is no longer executed; +neither ripper1 nor ripper2 still has such a table. The migration skips them. + +### Indexes + +No index is stored permanently. `test.py` drops `idx__date`, +`idx_omcversion_date` and `idx_libversion_date` on startup (they slow the bulk +insert down), and `report.py`/`all-reports.py`/`all-plots.py` recreate +`idx__date` when they need it. + +## How a test run writes the database + +1. Open `sqlite3.db`, apply the `user_version` migration, `CREATE TABLE IF NOT EXISTS []`. +2. Compute `confighash` per library, skip libraries already covered (query above). +3. Run the tests; each model writes `files/.stat.json`. +4. At the end, in one transaction: one `INSERT` per model into `[]`, one + `INSERT` per library into `[libversion]`, one `INSERT` into `[omcversion]`, + then `conn.commit()`. + +Nothing is written while the tests run, so an aborted run leaves no rows behind, +and two machines running the same job produce two full sets of rows in two +separate files - whichever file is copied back last wins. + +## Housekeeping scripts + +- `clean-dates.py --start --stop`: `DELETE FROM [] WHERE date?` + over every table, then `VACUUM`. Removes a range of bad runs. +- `clean-empty-omcversion-dates.py`: drops `omcversion` rows whose date has no + result rows in the corresponding branch table. + +## PostgreSQL layout + +The PostgreSQL database is a **mirror** of the sqlite3 one: the same tables with +the same names and columns, one table per branch plus `omcversion` and +`libversion`. That way the test scripts can push new results to the network +database with the same statements they use today, and the report scripts need no +query rewriting beyond the sqlite `[name]` / PostgreSQL `"name"` quoting. + +Only the types are adapted: + +| sqlite3 | PostgreSQL | +| --- | --- | +| `integer` | `bigint` (`integer` for `verifyfail`, `verifytotal`, `finalphase`) | +| `real` | `double precision` | +| `text` | `text` | +| `NOT NULL` on every column | only on the key columns, see below | +| `PRAGMA user_version` | not used; the `parsing` column always exists | +| `datelookup_*` | not migrated (derived data, no longer generated) | + +### Keys + +Each table gets a unique key, which sqlite3 never had: + +| table | key | +| --- | --- | +| `` | `(date, libname, model)` - a run tests every model of a library once | +| `omcversion` | `(date, branch)` - one row per run | +| `libversion` | `(date, branch, libname, confighash)` - one row per run and library | + +This is what makes a shared database possible: results from a second test +machine can be merged into a table that already holds another machine's rows, +and a run that is pushed twice cannot produce duplicates. + +Only those key columns are `NOT NULL`. The sqlite3 tables declare every column +`NOT NULL`, but `CREATE TABLE if not exists` means tables created by an older +`test.py` keep their old, laxer declaration, so the historical data does not +necessarily hold up. `libversion.libversion` for instance stores empty strings +for some old runs. + +So a branch table becomes: + +```sql +CREATE TABLE "master" ( + date bigint NOT NULL, + libname text NOT NULL, + model text NOT NULL, + exectime double precision NOT NULL, + frontend double precision NOT NULL, + backend double precision NOT NULL, + simcode double precision NOT NULL, + templates double precision NOT NULL, + compile double precision NOT NULL, + simulate double precision NOT NULL, + verify double precision NOT NULL, + verifyfail integer NOT NULL, + verifytotal integer NOT NULL, + finalphase integer NOT NULL, + parsing double precision NOT NULL +); +``` + +Two things to keep in mind when querying it: + +- Identifiers must be **double quoted**, not bracketed: branch names such as + `newInst-newBackend` contain upper case letters and dashes, which PostgreSQL + would otherwise fold to lower case or reject. +- All test machines write into the same tables, so a run is identified by + `date` (plus `branch`) exactly as before. Use `--pgschema ripper1` if a + machine should be mirrored into a schema of its own instead. + +Indexes are created by `sqlite2postgres.py --index` rather than on the fly: +`(date)` on every table, `(branch, date)` on `omcversion`, +`(branch, libname, date)` on `libversion` and `(libname, date)` on the branch +tables. + +## Migrating + +Run this **on omod-r630-2 (openmodelica.org)**: the sqlite3 files and the +PostgreSQL server are on the same machine, so the data never goes over the +network. Pushing it from a developer machine works too, but a home uplink does +0.3-1.8 MB/s, which means hours for ~27 GB. + +```bash +export PGPASSFILE=~/.pgpass # never put the password on the command line +DB=/var/www/libraries.openmodelica.org/sqlite3 +./sqlite2postgres.py --host localhost --sqlite $DB/ripper1/sqlite3.db --source ripper1 +./sqlite2postgres.py --host localhost --index +./sqlite2postgres.py --host localhost --sqlite $DB/ripper2/sqlite3.db --source ripper2 --skip-existing +./sqlite2postgres.py --host localhost --index +./sqlite2postgres.py --host localhost --sqlite $DB/ripper1/sqlite3.db --verify +``` + +The order matters. ripper1 goes in first and without an index, which is the +fast path. `--index` then creates the unique keys, so that ripper2, loaded with +`--skip-existing`, keeps whatever is already there whenever a key collides - +ripper1 wins. The second `--index` covers the tables that only exist on ripper2. + +On the databases as of 2026-08-10 the priority never actually fires: not one +`(branch, date)` is shared between the two machines, not even for the four +branches both of them test, so the merge is a plain union. The rule matters for +re-runs and for two machines pushing results later on. + +### Disk space + +A branch table row measures **212 bytes** in PostgreSQL (measured on `v1.10`, +average `model` length 57, `libname` 17). The whole migration is therefore about + +- 34 GB of table data for the ~160 million rows, plus +- 10-12 GB for the `(date)` and `(libname, date)` indexes. + +so **plan for ~50 GB**. On omod-r630-2 the cluster lives in +`/var/lib/postgresql/16` on the root LV, which has 14 GB free, while the `data` +ZFS pool has 2.9 TB free. Put the database on the pool before loading, e.g. + +```bash +sudo zfs create -o mountpoint=/data/postgres -o compression=lz4 -o recordsize=16k data/postgres +sudo install -d -o postgres -g postgres /data/postgres/omdb +sudo -u postgres psql -c "CREATE TABLESPACE omdb_ts LOCATION '/data/postgres/omdb'" +sudo -u postgres psql -c "ALTER DATABASE omdb SET TABLESPACE omdb_ts" # needs no open connections +``` + +`lz4` on the dataset typically cuts this data to well under half, since the +model names repeat in every run. + +### Catching up with the jobs that are still running + +The migration is a snapshot: a test run that was started before it, or any run +that still writes its own sqlite3 file, adds rows the shared database has never +seen. `--catch-up` copies them over: + +```bash +./sqlite2postgres.py --host localhost --sqlite $DB/ripper1/sqlite3.db --source ripper1 --catch-up +./sqlite2postgres.py --host localhost --sqlite $DB/ripper2/sqlite3.db --source ripper2 --catch-up +``` + +It reads the database from the start and keeps the runs the shared database +does not have, which takes two to three minutes per machine. Not "everything +past the rowid the migration stopped at", tempting as that is: `VACUUM` +renumbers the rowids of these tables and `clean-empty-omcversion-dates.py` runs +one after every test, so that number does not survive a test run. Picking the +runs by date is also what makes it correct for `master`, `newInst`, +`heavy_tests` and `v1.17`, where both machines write into the same table. + +Repeating it costs nothing but the reading: the keys reject anything already +there. Run it once more right after the jobs are switched to the shared +database; from then on nothing writes the sqlite3 files any more and there is +nothing left to catch up with. + +`--source` only names the machine for the bookkeeping table +`migration_progress(source, tbl, last_rowid, rows_read, done)`, which records how +far each sqlite table has been copied. Each batch is committed together with its +progress row, so an interrupted migration continues from the last rowid that +made it in and never copies a batch twice; the command can simply be run again. +Rows are streamed in batches of `--batch` (200000 by default) through +`COPY ... FROM STDIN`, so memory use does not depend on the size of the database. + +`COPY` is used in its text format rather than CSV on purpose: an empty CSV field +reads back as NULL, which would silently turn the empty `libversion` strings in +the old data into NULLs. diff --git a/sqlite2postgres.py b/sqlite2postgres.py new file mode 100644 index 0000000..850d445 --- /dev/null +++ b/sqlite2postgres.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +""" +Migrate OpenModelicaLibraryTesting sqlite3 result databases into PostgreSQL. + +The PostgreSQL layout is a 1:1 mirror of the sqlite3 one: one table per branch +with the same columns, plus the [omcversion] and [libversion] lookup tables. +Only the types are adapted (integer -> bigint, real -> double precision). + +Only the python standard library is used; the psql client binary does the +talking, so nothing has to be installed on the test machines. The password is +never passed on the command line - use PGPASSFILE or PGPASSWORD, as understood +by psql. + +Typical use: + + export PGPASSFILE=~/.pgpass + ./sqlite2postgres.py --sqlite dbs/ripper1.db --source ripper1 + ./sqlite2postgres.py --index + ./sqlite2postgres.py --sqlite dbs/ripper2.db --source ripper2 --skip-existing + ./sqlite2postgres.py --index + +The machines share the tables of the branches they both test, so they are merged +in that order: ripper1 first, then ripper2 with --skip-existing, which keeps the +row already in the database whenever a key collides. --index in between creates +the unique keys that decide what a collision is - (date, libname, model) for a +branch table - so it has to run before the second machine. + +--source only names the machine for the resume bookkeeping; it is not stored in +the data. Use --pgschema ripper1 to keep a machine in a schema of its own +instead of merging. + +The migration is resumable and safe to re-run: how far each table has been +copied is recorded in [migration_progress], and a restart continues from the +last sqlite rowid that made it in. +""" + +import argparse +import io +import os +import sqlite3 +import subprocess +import sys +import time + +# Columns of a per-branch table, in the order used by test.py. Databases with +# PRAGMA user_version < 3 have no "parsing" column. +# +# The sqlite3 tables declare every column NOT NULL, but tables created before +# that declaration still hold NULLs (libversion.libversion has some), so only +# the key columns are NOT NULL here. +BRANCH_COLUMNS = [ + ("date", "bigint"), + ("libname", "text"), + ("model", "text"), + ("exectime", "double precision"), + ("frontend", "double precision"), + ("backend", "double precision"), + ("simcode", "double precision"), + ("templates", "double precision"), + ("compile", "double precision"), + ("simulate", "double precision"), + ("verify", "double precision"), + ("verifyfail", "integer"), + ("verifytotal", "integer"), + ("finalphase", "integer"), + ("parsing", "double precision"), +] + +LOOKUP_COLUMNS = { + "omcversion": [("date", "bigint"), ("branch", "text"), ("omcversion", "text")], + "libversion": [("date", "bigint"), ("branch", "text"), ("libname", "text"), + ("libversion", "text"), ("confighash", "bigint")], +} + +# Derived data, no longer generated by all-plots.py; not worth migrating. +SKIP_PREFIXES = ("datelookup_", "sqlite_") + + +# What identifies a row. A test run writes one row per model, so a model +# appears once per (date, libname); the lookup tables have one row per run and +# per (run, library). --index enforces these, which is what lets a second test +# machine be merged into a table that already holds another machine's results. +KEYS = { + "omcversion": ["date", "branch"], + "libversion": ["date", "branch", "libname", "confighash"], +} +BRANCH_KEY = ["date", "libname", "model"] + +PROGRESS_TABLE = """ +CREATE TABLE IF NOT EXISTS "migration_progress" ( + source text NOT NULL, + tbl text NOT NULL, + last_rowid bigint NOT NULL, + rows_read bigint NOT NULL, + done boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (source, tbl) +) +""" + + +def lit(s): + """Quote a python value as an SQL string literal.""" + return "'" + str(s).replace("'", "''") + "'" + + +def ident(s): + """Quote an SQL identifier. Branch names contain '-' and upper case.""" + return '"' + s.replace('"', '""') + '"' + + +def copy_text(v): + """Encode one value for COPY's text format. + + Not CSV: an empty CSV field reads back as NULL, and libversion.libversion + does contain empty strings that have to stay empty strings. The text format + spells NULL \\N instead, so both survive. + """ + if v is None: + return "\\N" + return (str(v).replace("\\", "\\\\").replace("\n", "\\n") + .replace("\r", "\\r").replace("\t", "\\t")) + + +class Psql: + """Runs SQL through the psql client, which keeps this script dependency-free.""" + + def __init__(self, args): + self.base = ["psql", "--no-psqlrc", "-w", "-v", "ON_ERROR_STOP=1", + "-h", args.host, "-p", str(args.port), "-U", args.user, "-d", args.dbname] + self.schema = args.pgschema + + def _run(self, argv, stdin=None): + p = subprocess.run(self.base + argv, input=stdin, capture_output=True, text=True) + if p.returncode != 0: + raise RuntimeError("psql failed: %s" % (p.stderr.strip() or p.stdout.strip())) + return p.stdout + + def query(self, sql): + """Execute SQL and return the output unaligned and without headers.""" + return self._run(["-tAq", "-c", "SET search_path TO %s; %s" % (ident(self.schema), sql)]).strip() + + def script(self, sql): + return self._run(["-q", "-f", "-"], stdin="SET search_path TO %s;\n%s" % (ident(self.schema), sql)) + + def copy_rows(self, table, columns, rows, also="", skip_existing=None): + """COPY an iterable of tuples into a table. + + The statements in "also" are committed together with the batch, so that a + migration killed halfway through never copies the same rows twice. + + With skip_existing set to the key columns of the table, the batch goes + through a temporary table first and rows that are already there are + dropped, so the machine migrated first keeps its results. + """ + cols = ",".join(ident(c) for c in columns) + target = ident(table) + buf = io.StringIO() + buf.write("SET search_path TO %s;\nBEGIN;\n" % ident(self.schema)) + if skip_existing: + buf.write("CREATE TEMP TABLE batch (LIKE %s) ON COMMIT DROP;\n" % target) + target = "batch" + buf.write("COPY %s (%s) FROM STDIN;\n" % (target, cols)) + for r in rows: + buf.write("\t".join(copy_text(v) for v in r)) + buf.write("\n") + buf.write("\\.\n") + if skip_existing: + key = ",".join(ident(c) for c in skip_existing) + # DISTINCT ON also drops rows the batch itself holds twice, which the + # unique index would otherwise reject. + buf.write("INSERT INTO %s (%s) SELECT DISTINCT ON (%s) %s FROM batch ORDER BY %s" + " ON CONFLICT DO NOTHING;\n" % (ident(table), cols, key, cols, key)) + if also: + buf.write(also.rstrip().rstrip(";") + ";\n") + buf.write("COMMIT;\n") + if not skip_existing: + self._run(["-q", "-f", "-"], stdin=buf.getvalue()) + return None + # Without -q psql reports every statement, so the INSERT says how many rows + # were new - the interesting number when catching up with a running test. + out = self._run(["-f", "-"], stdin=buf.getvalue()) + for line in out.split("\n"): + if line.startswith("INSERT "): + return int(line.split()[-1]) + return 0 + + +def sqlite_tables(conn): + """The tables of a testing database, split into (branch tables, lookup tables).""" + names = [n for (n,) in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")] + branches, lookups = [], [] + for n in names: + if any(n.startswith(p) for p in SKIP_PREFIXES): + continue + (lookups if n in LOOKUP_COLUMNS else branches).append(n) + return branches, lookups + + +def sqlite_columns(conn, tbl): + return [row[1] for row in conn.execute("PRAGMA table_info(%s)" % ident(tbl))] + + +def create_table(pg, tbl, columns): + key = KEYS.get(tbl, BRANCH_KEY) + cols = ",\n ".join("%s %s%s" % (ident(c), t, " NOT NULL" if c in key else "") + for c, t in columns) + pg.script("CREATE TABLE IF NOT EXISTS %s (\n %s\n);" % (ident(tbl), cols)) + + +def read_progress(pg, source, tbl): + out = pg.query("SELECT last_rowid, rows_read, done FROM migration_progress " + "WHERE source=%s AND tbl=%s" % (lit(source), lit(tbl))) + if not out: + return 0, 0, False + last, rows, done = out.split("|") + return int(last), int(rows), done == "t" + + +def progress_sql(source, tbl, last_rowid, rows_read, done): + return ("""INSERT INTO migration_progress (source, tbl, last_rowid, rows_read, done, updated_at) + VALUES (%s, %s, %d, %d, %s, now()) + ON CONFLICT (source, tbl) DO UPDATE SET + last_rowid = EXCLUDED.last_rowid, rows_read = EXCLUDED.rows_read, + done = EXCLUDED.done, updated_at = now()""" + % (lit(source), lit(tbl), last_rowid, rows_read, "true" if done else "false")) + + +def migrate_table(pg, sconn, source, tbl, columns, batch, quiet, skip_existing=False, + catch_up=False): + """Copy one sqlite table into its PostgreSQL twin, in resumable batches.""" + names = [c for c, _ in columns] + have = sqlite_columns(sconn, tbl) + missing = [c for c in names if c not in have] + # Older databases lack "parsing"; test.py defaults it to 0.0 as well. + select = ",".join(ident(c) if c in have else "0" for c in names) + + create_table(pg, tbl, columns) + last_rowid, rows_read, done = read_progress(pg, source, tbl) + if done and not catch_up: + if not quiet: + print(" %-30s done earlier (%d rows)" % (tbl, rows_read)) + return rows_read, 0 + + stored = None + if catch_up: + # Read the table from the start and keep the runs the database has never + # seen. Not "everything past the rowid we stopped at": VACUUM renumbers + # the rowids of these tables, and clean-empty-omcversion-dates.py runs one + # after every test, so that number cannot be trusted between two runs. + stored = set(int(d) for d in pg.query("SELECT DISTINCT date FROM %s" % ident(tbl)).split("\n") if d) + last_rowid, rows_read = 0, 0 + skip_existing = True + key = KEYS.get(tbl, BRANCH_KEY) if skip_existing else None + if not quiet: + extra = "" + if missing: + extra = ", %s defaulted to 0" % ",".join(missing) + if catch_up: + extra += ", %d runs already stored" % len(stored) + elif last_rowid: + extra += ", resuming after rowid %d" % last_rowid + print(" %-30s starting%s" % (tbl, extra)) + + t0 = time.time() + inserted = 0 + date = names.index("date") + while True: + chunk = sconn.execute( + "SELECT rowid,%s FROM %s WHERE rowid > ? ORDER BY rowid LIMIT %d" + % (select, ident(tbl), batch), (last_rowid,)).fetchall() + if not chunk: + break + last_rowid = chunk[-1][0] + rows_read += len(chunk) + rows = [row[1:] for row in chunk] + if stored is not None: + rows = [r for r in rows if r[date] not in stored] + if rows: + n = pg.copy_rows(tbl, names, rows, + also=progress_sql(source, tbl, last_rowid, rows_read, False), + skip_existing=key) + inserted += n if n is not None else len(rows) + if not quiet: + sys.stdout.write("\r %-30s %d rows read, %d new (%.0f rows/s)" + % (tbl, rows_read, inserted, rows_read / max(time.time() - t0, 1e-9))) + sys.stdout.flush() + + pg.query(progress_sql(source, tbl, last_rowid, rows_read, True)) + if not quiet and (inserted or not catch_up): + print("\r %-30s %d rows read, %d new%s" % (tbl, rows_read, inserted, " " * 30)) + return rows_read, inserted + + +def migrate_database(pg, args): + sconn = sqlite3.connect("file:%s?mode=ro" % os.path.abspath(args.sqlite), uri=True) + branches, lookups = sqlite_tables(sconn) + if args.only: + branches = [b for b in branches if b in args.only] + lookups = [l for l in lookups if l in args.only] + print("%s: %d branch tables, lookup tables: %s" + % (args.sqlite, len(branches), ",".join(lookups) or "none")) + + total = new = 0 + for tbl in lookups + branches: + cols = LOOKUP_COLUMNS.get(tbl, BRANCH_COLUMNS) + (r, i) = migrate_table(pg, sconn, args.source, tbl, cols, args.batch, args.quiet, + args.skip_existing, args.catch_up) + total += r + new += i + sconn.close() + print("%s: %d rows read, %d new" % (args.sqlite, total, new)) + + +def create_indexes(pg): + """The unique key of each table, plus the index the reports need. + + The unique index has to exist before a second test machine is migrated with + --skip-existing. It also covers the [date] index the report scripts create + on the fly in sqlite, since date is its first column. + """ + tables = [t for t in pg.query( + "SELECT tablename FROM pg_tables WHERE schemaname=current_schema() ORDER BY tablename").split("\n") if t] + + def index(tbl, cols, unique=False): + name = ("%s_%s_%s" % ("uq" if unique else "idx", tbl, "_".join(cols)))[:63] + pg.script("CREATE %sINDEX IF NOT EXISTS %s ON %s (%s);" + % ("UNIQUE " if unique else "", ident(name), ident(tbl), + ",".join(ident(c) for c in cols))) + + for tbl in tables: + if tbl == "migration_progress": + continue + index(tbl, KEYS.get(tbl, BRANCH_KEY), unique=True) + if tbl == "omcversion": + index(tbl, ["branch", "date"]) + elif tbl == "libversion": + index(tbl, ["branch", "libname", "date"]) + else: + index(tbl, ["libname", "date"]) + print(" indexed %s" % tbl) + + +def verify(pg, args): + """Compare the sqlite and PostgreSQL row counts table by table.""" + sconn = sqlite3.connect("file:%s?mode=ro" % os.path.abspath(args.sqlite), uri=True) + branches, lookups = sqlite_tables(sconn) + bad = 0 + for tbl in lookups + branches: + (n,) = sconn.execute("SELECT COUNT(*) FROM %s" % ident(tbl)).fetchone() + m = int(pg.query("SELECT COUNT(*) FROM %s" % ident(tbl)) or 0) + flag = "ok" if m >= n else "MISSING" + if m < n: + bad += 1 + print(" %-30s sqlite %10d postgres %10d %s" % (tbl, n, m, flag)) + sconn.close() + print("%d of %d tables incomplete" % (bad, len(lookups) + len(branches))) + return bad + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--sqlite", help="sqlite3 database to migrate") + parser.add_argument("--source", help="machine the database comes from, e.g. ripper1; used for resuming") + parser.add_argument("--host", default=os.environ.get("PGHOST", "openmodelica.org")) + parser.add_argument("--port", type=int, default=int(os.environ.get("PGPORT", 5432))) + parser.add_argument("--user", default=os.environ.get("PGUSER", "om")) + parser.add_argument("--dbname", default=os.environ.get("PGDATABASE", "omdb")) + parser.add_argument("--pgschema", default="public", help="PostgreSQL schema to write to (default public)") + parser.add_argument("--batch", type=int, default=200000, help="rows per COPY batch (default 200000)") + parser.add_argument("--only", action="append", help="migrate only this table (repeatable)") + parser.add_argument("--catch-up", action="store_true", + help="copy the runs written since the last migration, for a test that was " + "still using the sqlite3 database. Reads the whole database but only " + "writes the runs it does not have; safe to repeat") + parser.add_argument("--skip-existing", action="store_true", + help="keep the rows already in the database when a key collides; " + "use it for every machine after the first one, and run --index before") + parser.add_argument("--index", action="store_true", help="create the indexes; do this after loading") + parser.add_argument("--verify", action="store_true", help="compare row counts with --sqlite") + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args() + + pg = Psql(args) + pg.script("CREATE SCHEMA IF NOT EXISTS %s;" % ident(args.pgschema)) + pg.script(PROGRESS_TABLE) + + if args.sqlite and not args.verify: + if not args.source: + parser.error("--sqlite needs --source, the machine name, e.g. ripper1") + migrate_database(pg, args) + + if args.index: + print("creating indexes") + create_indexes(pg) + + if args.verify: + if not args.sqlite: + parser.error("--verify needs --sqlite") + sys.exit(1 if verify(pg, args) else 0) + + +if __name__ == "__main__": + main() From 4b7ec32c4f1bbc4fabe640ab859f3abd85b197e5 Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Tue, 11 Aug 2026 01:55:20 +0200 Subject: [PATCH 2/2] Let the testing use a shared PostgreSQL database instead of sqlite3 files Second part of issue #295: the test machines can now write their results to one network database and coordinate through it, instead of each copying a sqlite3 file in, testing, and copying it back - the step that made two machines overwrite each other depending on which one finished last. resultsdb.py holds both backends behind one interface. The scripts write the same statements for either, with "?" as the placeholder, and ask the connection where the dialects genuinely differ: quoting a branch name, testing whether a table exists, concatenating a group, counting a condition, matching a branch without case, skipping a row that is already there. Every script takes --db, which defaults to the LIBTEST_DB environment variable, so Jenkins sets the database once for the whole pipeline rather than on forty invocations. A machine claims a job in [job_claim] before testing a library, keyed by exactly the question the run already asks: which library, in which version, against which compiler and configuration. Only one machine can win the claim, and the others skip that library and move on. The winner refreshes a heartbeat every minute and marks the claim done when the results are written, so a machine that dies parks its jobs for STALE_CLAIM_MINUTES rather than forever. A local sqlite3 file has a single writer, so there claim() always says yes. Two PostgreSQL specifics were needed for the reports: GROUP_CONCAT relies on sqlite keeping the order of the subquery that feeds it, so the ported query orders inside the aggregate, and COUNT(x or null) becomes COUNT(*) FILTER. Running the report queries against ripper1's sqlite file and against the migrated database gives the same rows, down to the phase counts, the per-phase sums and the regression rows: checked on master, whose table holds 43 million of them, and on heavy_tests, conversion and basemodelica_jl_master. The two only render the doubles with a different number of digits, and the report parses them back with float(). The Jenkinsfile gets a "postgres" parameter, on by default, which selects the database and drops the download and the publishing of sqlite3.db. Unticking it restores the old behaviour unchanged. The password comes from a Jenkins secret file credential, omdb-pgpass, bound to PGPASSFILE, so it never reaches a command line or the build log. --- Generated by Claude Code. Signed-off-by: Adrian Pop --- .CI/Jenkinsfile | 55 +++-- .CI/build-dep/Dockerfile | 2 +- all-plots.py | 20 +- all-reports.py | 38 +-- clean-dates.py | 18 +- clean-empty-omcversion-dates.py | 22 +- doc/README.md | 82 +++++++ report.py | 29 +-- requirements.txt | 1 + resultsdb.py | 405 ++++++++++++++++++++++++++++++++ single-model.py | 21 +- test.py | 70 ++---- 12 files changed, 633 insertions(+), 130 deletions(-) create mode 100644 resultsdb.py diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index 5d99a79..1619e07 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -3,6 +3,8 @@ pipeline { parameters { booleanParam(name: 'OLDLIBS', defaultValue: false, description: 'Also test some outdated libraries') + booleanParam(name: 'postgres', defaultValue: true, description: 'Store the results in the shared PostgreSQL database (omdb on openmodelica.org) rather than in the per-machine sqlite3 file. Machines coordinate through it, so two of them no longer overwrite each other. Untick to go back to the sqlite3 files.') + booleanParam(name: 'v1_26', defaultValue: false, description: 'maintenance/v1.26 branch (ryzen-5950x-1)') booleanParam(name: 'v1_27', defaultValue: false, description: 'maintenance/v1.27 branch (ryzen-5950x-1)') booleanParam(name: 'master', defaultValue: false, description: 'master branch (ryzen-5950x-1)') @@ -40,6 +42,12 @@ pipeline { } environment { LC_ALL = 'C.UTF-8' + // Where the results go. The scripts take it from here instead of a --db + // option on every single invocation. + LIBTEST_DB = "${params.postgres ? 'postgresql://om@openmodelica.org/omdb' : 'sqlite3.db'}" + // A secret file holding one pgpass line; libpq reads the password from it, + // so it never reaches a command line or the build log. + PGPASSFILE = credentials('omdb-pgpass') } stages { stage('test') { parallel { @@ -455,7 +463,12 @@ pipeline { cd OpenModelica git fetch ''' - sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper1/sqlite3.db' + // The reports read the shared database directly when it is in use. + script { + if (!params.postgres) { + sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper1/sqlite3.db' + } + } sh './clean-empty-omcversion-dates.py' sh "./all-reports.py --email --omcgitdir=OpenModelica ${env.GITBRANCHES} conversion heavy_tests" @@ -521,7 +534,11 @@ pipeline { cd OpenModelica git fetch ''' - sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper2/sqlite3.db' + script { + if (!params.postgres) { + sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper2/sqlite3.db' + } + } sh './clean-empty-omcversion-dates.py' sh "./all-reports.py --email --omcgitdir=OpenModelica ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_NEWBACKEND_DAE} ${env.GITBRANCHES_CPP} gbode cvode ida" @@ -978,14 +995,17 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om sh "test -d '${libraryPath}/.openmodelica/libraries/Modelica trunk'" - sh """ - if ! test -f ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db; then - wget -O ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp -q https://libraries.openmodelica.org/sqlite3/${dbPrefix}/sqlite3.db - mv ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db - fi - cp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db OpenModelicaLibraryTesting/sqlite3.db - test -s OpenModelicaLibraryTesting/sqlite3.db - """ + // The shared database needs none of this: the results go straight into it. + if (!params.postgres) { + sh """ + if ! test -f ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db; then + wget -O ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp -q https://libraries.openmodelica.org/sqlite3/${dbPrefix}/sqlite3.db + mv ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db + fi + cp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db OpenModelicaLibraryTesting/sqlite3.db + test -s OpenModelicaLibraryTesting/sqlite3.db + """ + } sh 'date' @@ -1008,11 +1028,16 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 """) sh 'date' - sh "rm -f OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp" - sh "ln OpenModelicaLibraryTesting/sqlite3.db OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp" sh "cd OpenModelicaLibraryTesting/ && ./clean-empty-omcversion-dates.py" - sh "cp OpenModelicaLibraryTesting/sqlite3.db ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db" - sh "rm -f ~/TEST_LIBS_BACKUP/${dbPrefix}-`date +sqlite3.%Y%m%d.db`" - sshPublisher(publishers: [sshPublisherDesc(configName: sshConfig, transfers: [sshTransfer(removePrefix: 'OpenModelicaLibraryTesting', sourceFiles: 'OpenModelicaLibraryTesting/sqlite3.db')])], failOnError: true) + // Copying the file back is what made two machines overwrite each other's + // results, so it only happens while a job still writes its own sqlite3 file. + if (!params.postgres) { + sh "rm -f OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp" + sh "ln OpenModelicaLibraryTesting/sqlite3.db OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp" + sh "cp OpenModelicaLibraryTesting/sqlite3.db ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db" + sh "rm -f ~/TEST_LIBS_BACKUP/${dbPrefix}-`date +sqlite3.%Y%m%d.db`" + + sshPublisher(publishers: [sshPublisherDesc(configName: sshConfig, transfers: [sshTransfer(removePrefix: 'OpenModelicaLibraryTesting', sourceFiles: 'OpenModelicaLibraryTesting/sqlite3.db')])], failOnError: true) + } } diff --git a/.CI/build-dep/Dockerfile b/.CI/build-dep/Dockerfile index 78dbab8..c0f8abe 100644 --- a/.CI/build-dep/Dockerfile +++ b/.CI/build-dep/Dockerfile @@ -1,4 +1,4 @@ FROM docker.openmodelica.org/build-deps:v1.16.3 RUN apt-get update && apt-get install libxml2 libxslt1.1 libxml2-dev libxslt1-dev -RUN pip3 install matplotlib FMPy +RUN pip3 install matplotlib FMPy psycopg2-binary diff --git a/all-plots.py b/all-plots.py index 6397707..e7c8026 100755 --- a/all-plots.py +++ b/all-plots.py @@ -3,7 +3,7 @@ import sys, argparse, subprocess, os import simplejson as json -import shared +import shared, resultsdb import re, time, math from omcommon import friendlyStr @@ -18,6 +18,7 @@ parser = argparse.ArgumentParser(description='OpenModelica model testing report generation tool') parser.add_argument('branches', nargs='*') parser.add_argument('--historypath', default="history") +resultsdb.addArgument(parser) args = parser.parse_args() branches = [branch.split("/")[-1] for branch in args.branches] @@ -25,11 +26,11 @@ libs = {} -import cgi, sqlite3, time, datetime +import cgi, time, datetime from omcommon import friendlyStr, multiple_replace -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() def dateStr(dint): return str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S')) @@ -98,8 +99,7 @@ def plotLibrary(branch, libname, xs, total, frontend,backend,simcode,template,co for branch in branches: try: - cursor.execute("SELECT name FROM [sqlite_master] WHERE type='table' AND name=?", (branch,)) - one = cursor.fetchone() + one = (branch,) if db.tableExists(branch) else None if one == None: print("No such table '%s'; specify it using --branch=XXX when running test.py" % branch) # ignore this table and continue @@ -112,13 +112,13 @@ def plotLibrary(branch, libname, xs, total, frontend,backend,simcode,template,co # ignore this table and continue continue - cursor.execute('''CREATE INDEX IF NOT EXISTS [idx_%s_date] ON [%s](date)''' % (branch,branch)) + db.createDateIndex(branch) libs = {} - for (date,libname,total,frontend,backend,simcode,template,compile,simulate,verify) in cursor.execute("""SELECT date,libname,COUNT(finalphase),COUNT(finalphase>=1 or null),COUNT(finalphase>=2 or null),COUNT(finalphase>=3 or null),COUNT(finalphase>=4 or null),COUNT(finalphase>=5 or null),COUNT(finalphase>=6 or null),COUNT(finalphase>=7 or null) - FROM [%s] + for (date,libname,total,frontend,backend,simcode,template,compile,simulate,verify) in cursor.execute("""SELECT date,libname,COUNT(finalphase),%s + FROM %s GROUP BY date,libname ORDER BY libname,date ASC -""" % (branch)): +""" % (",".join(db.countIf("finalphase>=%d" % i) for i in range(1,8)), db.quote(branch))): if libname not in libs: libs[libname] = ([],[],[],[],[],[],[],[],[]) libs[libname][0].append(datetime.datetime.fromtimestamp(date)) diff --git a/all-reports.py b/all-reports.py index 07f0c6e..dbd1a5b 100755 --- a/all-reports.py +++ b/all-reports.py @@ -5,7 +5,7 @@ import codecs import sys, argparse, subprocess, os, time import simplejson as json -import shared +import shared, resultsdb import re from omcommon import friendlyStr @@ -17,6 +17,7 @@ parser.add_argument('--githuburltesting', default="https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit") parser.add_argument('--omcgitdir', default="../OpenModelica/OpenModelica") parser.add_argument('--email', default=False, action='store_true') +resultsdb.addArgument(parser) args = parser.parse_args() os.environ['TZ'] = 'Europe/Stockholm' @@ -44,11 +45,11 @@ libs = {} -import cgi, sqlite3, time, datetime +import cgi, time, datetime from omcommon import friendlyStr, multiple_replace -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() def dateStr(dint): return str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S')) @@ -70,8 +71,7 @@ def modelLink(libname, modelname, extension, text): emails_to_send = {} for branch in branches: try: - cursor.execute("SELECT name FROM [sqlite_master] WHERE type='table' AND name=?", (branch,)) - one = cursor.fetchone() + one = (branch,) if db.tableExists(branch) else None if one == None: print("No such table '%s'; specify it using --branch=XXX when running test.py" % branch) # ignore this table and continue @@ -86,8 +86,8 @@ def modelLink(libname, modelname, extension, text): missing_branches.append(branch) continue - cursor.execute('''CREATE INDEX IF NOT EXISTS [idx_%s_date] ON [%s](date)''' % (branch,branch)) - cursor.execute("SELECT date,omcversion FROM [omcversion] WHERE branch LIKE ? COLLATE NOCASE ORDER BY date ASC", (branch,)) + db.createDateIndex(branch) + cursor.execute("SELECT date,omcversion FROM omcversion WHERE %s ORDER BY date ASC" % db.likeNoCase("branch"), (branch,)) entries = cursor.fetchall() n = len(entries) urlToOpen = "%s/%s/00_history.html" % (historyurl, branch) @@ -152,11 +152,11 @@ def modelLink(libname, modelname, extension, text): gitloglibrarytesting = "could not get the git log for OpenModelicaLibraryTesting" tpl = tpl.replace("#OMCGITLOG#",gitlog).replace("#NUMCOMMITS#",str(gitlog.count(""))).replace("#3rdParty#",thirdPartyChanged).replace("#OMCLIBRARYTESTINGGITLOG#",gitloglibrarytesting) - libnames = [libname for (libname,) in cursor.execute("""SELECT libname FROM [%s] WHERE date=? GROUP BY libname""" % branch, (d2,))] + libnames = [libname for (libname,) in cursor.execute("""SELECT libname FROM %s WHERE date=? GROUP BY libname""" % db.quote(branch), (d2,))] startdates = {} # Get previous date of each library run and group them together for fast queries later for libname in libnames: - ds = cursor.execute("""SELECT date FROM [%s] WHERE date MAX(finalphase)) OR ((MIN(finalphase) >= ?) AND (MAX(frontend) > ?*MIN(frontend) AND MAX(frontend) > ?) OR @@ -179,7 +183,7 @@ def modelLink(libname, modelname, extension, text): (MAX(compile) > ?*MIN(compile) AND MAX(compile) > ?) OR (MAX(simulate) > ?*MIN(simulate) AND MAX(simulate) > ?) ) - """ % (branch,",".join(["'%s'" % libname for libname in startdates[d1lib]])) + """) % (db.quote(branch),",".join(["'%s'" % libname for libname in startdates[d1lib]])) cursor.execute(query, (d1lib,d2,timeMinPhase,timeRel,timeAbs,timeRel,timeAbs,timeRel,timeAbs,timeRel,timeAbs,timeRel,2*timeAbs,timeRel,timeAbs)) regressions += cursor.fetchall() regressions = sorted(regressions, key = lambda x: (x[1],x[0])) @@ -228,10 +232,10 @@ def modelLink(libname, modelname, extension, text): libstrs = [] for libname in sorted(list(libs)): - cursor.execute("SELECT libversion,confighash FROM [libversion] WHERE branch LIKE ? COLLATE NOCASE AND date<=? AND libname=? ORDER BY date DESC LIMIT 1", (branch,d1,libname)) + cursor.execute("SELECT libversion,confighash FROM libversion WHERE %s AND date<=? AND libname=? ORDER BY date DESC LIMIT 1" % db.likeNoCase("branch"), (branch,d1,libname)) (lv1,lh1) = cursor.fetchone() lv1 = lv1.strip() - cursor.execute("SELECT libversion,confighash FROM [libversion] WHERE branch LIKE ? COLLATE NOCASE AND date<=? AND libname=? ORDER BY date DESC LIMIT 1", (branch,d2,libname)) + cursor.execute("SELECT libversion,confighash FROM libversion WHERE %s AND date<=? AND libname=? ORDER BY date DESC LIMIT 1" % db.likeNoCase("branch"), (branch,d2,libname)) (lv2,lh2) = cursor.fetchone() lv2 = lv2.strip() if lv1 != lv2: diff --git a/clean-dates.py b/clean-dates.py index ddafe63..09b831e 100755 --- a/clean-dates.py +++ b/clean-dates.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import argparse, sqlite3, sys +import argparse, sys +import resultsdb from datetime import datetime parser = argparse.ArgumentParser(description='OpenModelica library testing tool') parser.add_argument('startDate') parser.add_argument('stopDate') +resultsdb.addArgument(parser) args = parser.parse_args() @@ -29,12 +31,12 @@ sys.stdout.write("Please respond with 'yes' or 'no'") sys.exit(1) -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() -tables = [tbl for (tbl,) in cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")] +tables = db.tables() for tbl in tables: - cursor.execute("DELETE FROM [%s] WHERE date?" % tbl, (stopTime.timestamp(),startTime.timestamp())) -conn.commit() -conn.execute("VACUUM") -conn.close() + cursor.execute("DELETE FROM %s WHERE date?" % db.quote(tbl), (stopTime.timestamp(),startTime.timestamp())) +db.commit() +db.vacuum() +db.close() diff --git a/clean-empty-omcversion-dates.py b/clean-empty-omcversion-dates.py index 1b0fef4..63aad13 100755 --- a/clean-empty-omcversion-dates.py +++ b/clean-empty-omcversion-dates.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -import argparse, sqlite3, sys +import argparse, sys +import resultsdb from datetime import datetime parser = argparse.ArgumentParser(description='OpenModelica library testing tool') +resultsdb.addArgument(parser) args = parser.parse_args() -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() entries = cursor.execute("SELECT date,branch FROM omcversion").fetchall() dropped=0 @@ -20,7 +22,11 @@ branchDates[branch] = set() branchDates[branch].add(date) for branch in branches: - data=cursor.execute("SELECT DISTINCT date FROM [%s]" % branch).fetchall() + # The shared database holds the branches of every machine, including ones + # this one never created a result table for. + if not db.tableExists(branch): + continue + data=cursor.execute("SELECT DISTINCT date FROM %s" % db.quote(branch)).fetchall() for (date,) in data: try: branchDates[branch].remove(date) @@ -28,10 +34,10 @@ pass for date in branchDates[branch]: print("Dropping empty omcversion entry (%d,%s)" % (date,branch)) - cursor.execute("DELETE FROM [omcversion] WHERE date=? AND branch=?", (date,branch)) + cursor.execute("DELETE FROM omcversion WHERE date=? AND branch=?", (date,branch)) dropped += 1 -conn.commit() +db.commit() if dropped>0: - conn.execute("VACUUM") -conn.close() + db.vacuum() +db.close() diff --git a/doc/README.md b/doc/README.md index 363906d..5d908a6 100644 --- a/doc/README.md +++ b/doc/README.md @@ -264,6 +264,88 @@ Indexes are created by `sqlite2postgres.py --index` rather than on the fly: `(branch, libname, date)` on `libversion` and `(libname, date)` on the branch tables. +## Using the shared database + +Every script takes `--db`, which is a path to a local sqlite3 file (the default, +`sqlite3.db`) or a `postgresql://` URL: + +```bash +./test.py --branch=master --db=postgresql://om@openmodelica.org/omdb configs/conf.json +./report.py --branches=master --db=postgresql://om@openmodelica.org/omdb configs/conf.json +``` + +The password comes from `PGPASSWORD` or `~/.pgpass`, never from the URL or the +command line. `resultsdb.py` holds the two backends behind one interface; the +scripts write the same statements for both, with `?` as the placeholder, and ask +the connection where the dialects genuinely differ (`quote()`, `tableExists()`, +`groupConcat()`, `countIf()`, `likeNoCase()`, `insertIgnore()`). + +### Job claiming + +The point of the shared database is that two machines can test at the same time +without overwriting each other. Before testing a library, `test.py` claims the +job in + +```sql +CREATE TABLE job_claim ( + branch text, libname text, libversion text, omcversion text, confighash bigint, + host text NOT NULL, state text NOT NULL, + claimed_at timestamptz NOT NULL DEFAULT now(), + heartbeat timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (branch, libname, libversion, omcversion, confighash) +); +``` + +The key is exactly the question "which library, in which version, against which +compiler and configuration": the same combination the run already uses to decide +whether results exist. A claim is taken with `INSERT ... ON CONFLICT DO UPDATE +... WHERE` so that only one machine can win it, and a machine that loses prints + +``` +Skipping Buildings_9.1.0 as ripper2 has been testing it since 2026-08-10 22:14:03 +``` + +and moves on to the next library instead of repeating the work. The winner +refreshes `heartbeat` every minute from a background thread and sets +`state='done'` when the results are written. A machine that dies stops sending +its heartbeat, and after 30 minutes (`STALE_CLAIM_MINUTES`) another machine may +take its jobs over, so a crash does not park a library forever. + +Nothing of this applies to a local sqlite3 file: it has a single writer, and +`claim()` always says yes. + +### In Jenkins + +The pipeline has a `postgres` parameter, on by default, and sets two variables +for every stage: + +```groovy +environment { + LIBTEST_DB = "${params.postgres ? 'postgresql://om@openmodelica.org/omdb' : 'sqlite3.db'}" + PGPASSFILE = credentials('omdb-pgpass') +} +``` + +`LIBTEST_DB` is where `--db` defaults to, so no invocation has to spell it out, +and `omdb-pgpass` is a Jenkins *secret file* credential holding a single line: + +``` +openmodelica.org:5432:omdb:om: +``` + +libpq reads the password from that file, so it never appears on a command line +or in the build log. `resultsdb.py` takes a private copy of the file when its +permissions let anyone else read it, because libpq silently ignores such a file +and then fails with `fe_sendauth: no password supplied`. + +With `postgres` ticked, a job no longer downloads the machine's `sqlite3.db` +before the run nor publishes it back afterwards - the step that made two +machines overwrite each other. Untick the parameter and the old behaviour is +back, unchanged. + +The test machines need `psycopg2`: `pip3 install psycopg2-binary`, or a rebuild +of the images, since it is in `requirements.txt` and in `.CI/build-dep`. + ## Migrating Run this **on omod-r630-2 (openmodelica.org)**: the sqlite3 files and the diff --git a/report.py b/report.py index fe8eafe..97cbd80 100755 --- a/report.py +++ b/report.py @@ -3,11 +3,12 @@ import sys, argparse import simplejson as json -import shared +import shared, resultsdb parser = argparse.ArgumentParser(description='OpenModelica library testing report generation tool') parser.add_argument('configs', nargs='*') parser.add_argument('--branches', default='master') +resultsdb.addArgument(parser) args = parser.parse_args() configs = args.configs @@ -23,7 +24,7 @@ libs = {} -import html, sqlite3, time, datetime +import html, time, datetime from omcommon import friendlyStr, multiple_replace configs_lst = [shared.readConfig(c) for c in configs] @@ -32,8 +33,8 @@ configs = configs + c libnames = set(shared.libname(library,conf) for (library,conf) in configs) -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() nmodels = {} nsimulate = {} @@ -42,7 +43,7 @@ missing_branches = [] for branch in branches: try: - cursor.execute("SELECT date FROM [%s] ORDER BY date DESC LIMIT 1" % branch) + cursor.execute("SELECT date FROM %s ORDER BY date DESC LIMIT 1" % db.quote(branch)) one = cursor.fetchone() if one == None: print("No such table '%s'; specify it using --branch=XXX when running test.py" % branch) @@ -58,18 +59,18 @@ continue dates_str[branch] = str(datetime.datetime.fromtimestamp(v).strftime('%Y-%m-%d %H:%M:%S')) - cursor.execute('''CREATE INDEX IF NOT EXISTS [idx_%s_date] ON [%s](date)''' % (branch,branch)) + db.createDateIndex(branch) dates[branch] = {} branch_nmodels = 0 for libname in libnames: - cursor.execute("SELECT date FROM [%s] WHERE libname=? ORDER BY date DESC LIMIT 1" % branch, (libname,)) + cursor.execute("SELECT date FROM %s WHERE libname=? ORDER BY date DESC LIMIT 1" % db.quote(branch), (libname,)) v = cursor.fetchone() if v is None: dates[branch][libname] = 0 continue dates[branch][libname] = v[0] - for x in cursor.execute("SELECT model FROM [%s] WHERE libname=? AND date=?" % branch, (libname,v[0])): + for x in cursor.execute("SELECT model FROM %s WHERE libname=? AND date=?" % db.quote(branch), (libname,v[0])): if libname not in libs: libs[libname] = set() libs[libname].add(x[0]) @@ -89,7 +90,7 @@ def checkEqual(iterator): for lib in sorted(libs.keys()): models = libs[lib] entries += "

%s

\n" % lib - branches_versions = [(cursor.execute("SELECT libversion FROM [libversion] WHERE libname=? AND branch=? ORDER BY date DESC LIMIT 1", (lib,branch)).fetchone() or ["unknown"])[0] for branch in branches] + branches_versions = [(cursor.execute("SELECT libversion FROM libversion WHERE libname=? AND branch=? ORDER BY date DESC LIMIT 1", (lib,branch)).fetchone() or ["unknown"])[0] for branch in branches] all_equal = checkEqual(branches_versions) if not all_equal: entries += "\n" @@ -109,12 +110,12 @@ def checkEqual(iterator): master_models = [] for i in range(0,8): i_models = set() - for v in cursor.execute("SELECT model FROM [%s] WHERE date=? AND finalphase>=? AND libname=?" % (branch), (dates[branch][lib],i,lib)): + for v in cursor.execute("SELECT model FROM %s WHERE date=? AND finalphase>=? AND libname=?" % (db.quote(branch)), (dates[branch][lib],i,lib)): i_models.add(v[0]) master_models.append(i_models) models[branch] = master_models for branch in branches: - vs = [cursor.execute("SELECT COUNT(*) FROM [%s] WHERE date=? AND finalphase>=? AND libname=?" % (branch), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] + vs = [cursor.execute("SELECT COUNT(*) FROM %s WHERE date=? AND finalphase>=? AND libname=?" % (db.quote(branch)), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] warnings = [] entries += '' % (branch,lib,lib,branch) for i in [0]+list(range(0,len(vs))): @@ -138,8 +139,8 @@ def checkEqual(iterator): entries += "
%s
\n" entries += entryhead for branch in branches: - vs = [cursor.execute("SELECT COUNT(*) FROM [%s] WHERE date=? AND finalphase>=? AND libname=?" % (branch), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] - sums = [cursor.execute("SELECT SUM(%s) FROM [%s] WHERE date=? AND libname=?" % (fields[i],branch), (dates[branch][lib],lib)).fetchone()[0] or 0 for i in range(0,9)] + vs = [cursor.execute("SELECT COUNT(*) FROM %s WHERE date=? AND finalphase>=? AND libname=?" % (db.quote(branch)), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] + sums = [cursor.execute("SELECT SUM(%s) FROM %s WHERE date=? AND libname=?" % (fields[i],db.quote(branch)), (dates[branch][lib],lib)).fetchone()[0] or 0 for i in range(0,9)] entries += '' % (branch,lib,lib,branch) entries += ("\n" % (friendlyStr(sums[0]),friendlyStr(sums[1]),friendlyStr(sums[2]),friendlyStr(sums[3]),friendlyStr(sums[4]),friendlyStr(sums[5]),friendlyStr(sums[6]),friendlyStr(sums[7]),friendlyStr(sums[8]))) exectime[branch] += sums[0] @@ -148,7 +149,7 @@ def checkEqual(iterator): nummodels = sum(len(l) for l in libs.values()) branches_lines = [("%d\n" % (html.escape(branch), html.escape( - (cursor.execute("SELECT omcversion FROM [omcversion] WHERE date=? AND branch=?", (max(dates[branch][lib] for lib in libnames),branch)).fetchone() or ["unknown"])[0] + (cursor.execute("SELECT omcversion FROM omcversion WHERE date=? AND branch=?", (max(dates[branch][lib] for lib in libnames),branch)).fetchone() or ["unknown"])[0] ), html.escape(dates_str[branch]), friendlyStr(exectime[branch]), " class=\"warning\"" if nummodels!=nmodels[branch] else "", nsimulate[branch], diff --git a/requirements.txt b/requirements.txt index bf3dd41..c24fbe9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ monotonic natsort ompython==3.6 psutil +psycopg2-binary simplejson diff --git a/resultsdb.py b/resultsdb.py new file mode 100644 index 0000000..81f817e --- /dev/null +++ b/resultsdb.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +""" +The result database of the library testing, either local or shared. + +Historically every test machine wrote its results to its own sqlite3 file, +copied in before a run and copied back afterwards, so two machines running the +same job overwrote each other. The scripts can now talk to a shared PostgreSQL +database instead, where the machines coordinate through [job_claim] and nobody +overwrites anybody. + + db = connect("sqlite3.db") # the old local file + db = connect("postgresql://om@openmodelica.org/omdb") # the shared database + +Both backends take the same statements, with "?" as the placeholder. Where the +two dialects genuinely differ - quoting a branch name, testing whether a table +exists, concatenating a group, counting a condition - ask the connection +instead of writing it out, see quote()/tableExists()/groupConcat()/countIf(). + +PostgreSQL needs psycopg2 (pip install psycopg2-binary) and reads the password +from PGPASSWORD or ~/.pgpass, never from the URL. +""" + +import os +import re +import socket +import sqlite3 +import threading +import time + +# A claim older than this without a heartbeat belongs to a machine that died, +# and another machine may take the job over. +STALE_CLAIM_MINUTES = 30 +HEARTBEAT_SECONDS = 60 + +# The columns of a per-branch result table, with the type spelled per backend. +BRANCH_COLUMNS = [ + ("date", "bigint"), ("libname", "text"), ("model", "text"), ("exectime", "real"), + ("frontend", "real"), ("backend", "real"), ("simcode", "real"), ("templates", "real"), + ("compile", "real"), ("simulate", "real"), ("verify", "real"), + ("verifyfail", "int"), ("verifytotal", "int"), ("finalphase", "int"), ("parsing", "real"), +] +SQLITE_TYPES = {"bigint": "integer", "int": "integer", "real": "real", "text": "text"} +POSTGRES_TYPES = {"bigint": "bigint", "int": "integer", "real": "double precision", "text": "text"} + +# What identifies a row, so that two machines writing the same shared table +# cannot store the same result twice. Mirrors sqlite2postgres.py. +KEYS = { + "omcversion": ["date", "branch"], + "libversion": ["date", "branch", "libname", "confighash"], +} +BRANCH_KEY = ["date", "libname", "model"] + +JOB_CLAIM = """ +CREATE TABLE IF NOT EXISTS job_claim ( + branch text NOT NULL, + libname text NOT NULL, + libversion text NOT NULL, + omcversion text NOT NULL, + confighash bigint NOT NULL, + host text NOT NULL, + state text NOT NULL, + claimed_at timestamptz NOT NULL DEFAULT now(), + heartbeat timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (branch, libname, libversion, omcversion, confighash) +) +""" + + +# Jenkins sets LIBTEST_DB once for the whole pipeline rather than passing --db +# to every script invocation. +DEFAULT_DB = os.environ.get("LIBTEST_DB") or "sqlite3.db" +DB_HELP = ("Result database: a local sqlite3 file, or a postgresql://user@host/database URL for " + "the shared one, taken from the LIBTEST_DB environment variable when not given. " + "Several machines can write to the shared database at the same time; a library " + "another machine is already testing is skipped instead of tested twice. The password " + "is read from PGPASSWORD or ~/.pgpass, never from the URL.") + + +def addArgument(parser): + """Give a script the --db option, spelled the same way everywhere.""" + parser.add_argument("--db", default=DEFAULT_DB, help=DB_HELP) + + +def _fixPgpassPermissions(): + """libpq ignores a password file others can read, and then fails to connect. + + Jenkins hands the credential to the build as a file it created itself, so + rather than have every job remember to chmod it, take a private copy. + """ + path = os.environ.get("PGPASSFILE") + if not path or not os.path.isfile(path): + return + if not (os.stat(path).st_mode & 0o077): + return + import shutil, tempfile + fd, private = tempfile.mkstemp(prefix="pgpass.") + os.close(fd) + os.chmod(private, 0o600) + shutil.copyfile(path, private) + os.environ["PGPASSFILE"] = private + + +def connect(url): + """Open the result database named by url: a path or a postgresql:// URL.""" + if url.startswith("postgres://") or url.startswith("postgresql://"): + return _Postgres(url) + return _Sqlite(url) + + +class _Db: + """What the testing and report scripts use; the backends fill in the rest.""" + + def cursor(self): + return _Cursor(self, self.conn.cursor()) + + def execute(self, sql, params=()): + return self.cursor().execute(sql, params) + + def commit(self): + self.conn.commit() + + def close(self): + self.conn.close() + + def insertIgnore(self): + """The clause that makes an INSERT skip a row that is already there.""" + return "" + + def createDateIndex(self, branch): + """The index the report queries need; test.py drops it before a run.""" + self.execute("CREATE INDEX IF NOT EXISTS %s ON %s (date)" + % (self.quote("idx_%s_date" % branch), self.quote(branch))) + + def claim(self, branch, libname, libversion, omcversion, confighash): + """True when this machine may test that library, False when another one is. + + Only the shared database can say no; a local file has a single writer. + """ + return True + + def claimedBy(self, branch, libname, libversion, omcversion, confighash): + return ("this machine", None) + + def release(self): + """Mark the claims of this run as finished.""" + + def vacuum(self): + self.conn.execute("VACUUM") + + +class _Cursor: + """A cursor that takes "?" placeholders whatever the backend wants.""" + + def __init__(self, db, cursor): + self.db = db + self.cursor = cursor + + def execute(self, sql, params=()): + # psycopg2 only looks for placeholders when parameters are passed, so a + # statement without any must not be handed an empty tuple. + params = tuple(params) + self.cursor.execute(self.db.sql(sql, bool(params)), self.db.params(params)) + return self + + def fetchone(self): + return self.cursor.fetchone() + + def fetchall(self): + return self.cursor.fetchall() + + def __iter__(self): + return iter(self.cursor) + + +class _Sqlite(_Db): + """The per-machine sqlite3 file the testing has always used.""" + + name = "sqlite3" + + def __init__(self, path): + self.conn = sqlite3.connect(path) + self.path = path + + def sql(self, sql, hasParams=False): + return sql + + def params(self, params): + return params + + def quote(self, ident): + """Quote a table or column name, typically a branch name.""" + return "[%s]" % ident + + def createTables(self, branch): + """The schema migration test.py has always done, plus the branch table.""" + cursor = self.cursor() + user_version = self.userVersion() + if user_version == 0: + # Table to lookup from a run (date, branch) to omcversion used + cursor.execute("CREATE TABLE if not exists [omcversion] (date integer NOT NULL, branch text NOT NULL, omcversion text NOT NULL)") + # Table to lookup from a run (date, branch) which library versions were used + cursor.execute("CREATE TABLE if not exists [libversion] (date integer NOT NULL, branch text NOT NULL, libname text NOT NULL, libversion text NOT NULL, confighash integer NOT NULL)") + elif user_version == 1: + cursor.execute("ALTER TABLE [libversion] ADD COLUMN confighash integer NOT NULL DEFAULT(0)") + elif user_version == 2: + for tbl in [t for t in self.tables() if t not in ["libversion", "omcversion"]]: + cursor.execute("ALTER TABLE [%s] ADD COLUMN parsing real NOT NULL DEFAULT(0.0)" % tbl) + elif user_version != 3: + raise SystemExit("Unknown schema user_version=%d" % user_version) + + cols = ", ".join("%s %s NOT NULL" % (c, SQLITE_TYPES[t]) for c, t in BRANCH_COLUMNS) + cursor.execute("CREATE TABLE if not exists %s (%s)" % (self.quote(branch), cols)) + # The indexes only slow the run's inserts down; the report scripts add them back. + cursor.execute("DROP INDEX IF EXISTS [idx_%s_date]" % branch) + cursor.execute("DROP INDEX IF EXISTS idx_omcversion_date") + cursor.execute("DROP INDEX IF EXISTS idx_libversion_date") + self.setUserVersion(3) + + def tables(self): + return [t for (t,) in self.conn.execute("SELECT name FROM sqlite_master WHERE type='table'")] + + def tableExists(self, name): + return self.conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)).fetchone() is not None + + def groupConcat(self, expr, orderBy=None): + # sqlite keeps the order of the subquery that feeds the aggregate. + return "GROUP_CONCAT(%s)" % expr + + def countIf(self, cond): + return "COUNT(%s or null)" % cond + + def likeNoCase(self, column): + """A case insensitive comparison against a "?" placeholder.""" + return "%s LIKE ? COLLATE NOCASE" % column + + def userVersion(self): + return self.conn.execute("PRAGMA user_version").fetchone()[0] + + def setUserVersion(self, v): + self.conn.execute("PRAGMA user_version=%d" % v) + + +class _Postgres(_Db): + """The shared database several test machines write to at the same time.""" + + name = "postgresql" + + def __init__(self, url): + try: + import psycopg2 + except ImportError: + raise SystemExit("PostgreSQL support needs psycopg2: pip install psycopg2-binary") + # The password belongs in PGPASSWORD or ~/.pgpass, not in the URL. + _fixPgpassPermissions() + self.url = url + self.conn = psycopg2.connect(url) + self.conn.autocommit = False + self.host = socket.gethostname() + self.claims = [] + self.heartbeatThread = None + self.execute(JOB_CLAIM) + self.commit() + + def sql(self, sql, hasParams=False): + """sqlite spells the placeholder "?" and psycopg2 spells it "%s". + + psycopg2 also reads "%" itself, so a literal one has to be doubled - but + only in a statement that has parameters at all. + """ + if not hasParams: + return sql + return sql.replace("%", "%%").replace("?", "%s") + + def params(self, params): + # psycopg2 only looks for placeholders when parameters are passed, so a + # statement without any must not be handed an empty tuple. + return params or None + + def quote(self, ident): + return '"%s"' % ident.replace('"', '""') + + def createTables(self, branch): + """Create the shared tables, with the keys that keep the machines apart. + + Unlike sqlite there is no schema migration and the indexes stay: other + machines are reading the table while this one writes its few thousand rows. + """ + cursor = self.cursor() + cursor.execute("""CREATE TABLE IF NOT EXISTS omcversion ( + date bigint NOT NULL, branch text NOT NULL, omcversion text)""") + cursor.execute("""CREATE TABLE IF NOT EXISTS libversion ( + date bigint NOT NULL, branch text NOT NULL, libname text NOT NULL, + libversion text, confighash bigint NOT NULL)""") + cols = ", ".join("%s %s%s" % (c, POSTGRES_TYPES[t], " NOT NULL" if c in BRANCH_KEY else "") + for c, t in BRANCH_COLUMNS) + cursor.execute("CREATE TABLE IF NOT EXISTS %s (%s)" % (self.quote(branch), cols)) + for tbl in ["omcversion", "libversion", branch]: + key = KEYS.get(tbl, BRANCH_KEY) + cursor.execute("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)" + % (self.quote(("uq_%s_%s" % (tbl, "_".join(key)))[:63]), + self.quote(tbl), ",".join(key))) + cursor.execute("CREATE INDEX IF NOT EXISTS %s ON %s (libname, date)" + % (self.quote(("idx_%s_libname_date" % branch)[:63]), self.quote(branch))) + self.commit() + + def tables(self): + return [t for (t,) in self.execute( + "SELECT tablename FROM pg_tables WHERE schemaname=current_schema()")] + + def tableExists(self, name): + return self.execute("SELECT 1 FROM pg_tables WHERE schemaname=current_schema() AND tablename=?", + (name,)).fetchone() is not None + + def groupConcat(self, expr, orderBy=None): + if orderBy: + return "string_agg(%s::text, ',' ORDER BY %s)" % (expr, orderBy) + return "string_agg(%s::text, ',')" % expr + + def countIf(self, cond): + return "COUNT(*) FILTER (WHERE %s)" % cond + + def likeNoCase(self, column): + return "%s ILIKE ?" % column + + def insertIgnore(self): + # Another machine may have written the same run already; its row wins. + return " ON CONFLICT DO NOTHING" + + def createDateIndex(self, branch): + """Nothing to do: date is the first column of the table's unique key.""" + + def userVersion(self): + return 3 + + def setUserVersion(self, v): + pass + + def vacuum(self): + """Nothing to do: PostgreSQL has autovacuum, and a manual VACUUM of a + 50 GB database after every test run would be a waste of the machine.""" + + def claim(self, branch, libname, libversion, omcversion, confighash): + """Take the job unless another machine is running it right now. + + The claim is one row per (branch, library version, omc version, config). + A machine that dies stops sending its heartbeat, and after + STALE_CLAIM_MINUTES its jobs are up for grabs again. + """ + key = (branch, libname, libversion, omcversion, confighash) + got = self.execute("""INSERT INTO job_claim + (branch, libname, libversion, omcversion, confighash, host, state) + VALUES (?,?,?,?,?,?,'running') + ON CONFLICT (branch, libname, libversion, omcversion, confighash) DO UPDATE + SET host = EXCLUDED.host, state = 'running', claimed_at = now(), heartbeat = now() + WHERE job_claim.state <> 'running' + OR job_claim.heartbeat < now() - interval '%d minutes' + RETURNING host""" % STALE_CLAIM_MINUTES, + key + (self.host,)).fetchone() + self.commit() + if got is None: + return False + self.claims.append(key) + self._startHeartbeat() + return True + + def claimedBy(self, branch, libname, libversion, omcversion, confighash): + """The machine holding that job, for the message telling the user why we skip.""" + row = self.execute("""SELECT host, claimed_at FROM job_claim WHERE branch=? AND libname=? + AND libversion=? AND omcversion=? AND confighash=?""", + (branch, libname, libversion, omcversion, confighash)).fetchone() + return row or ("unknown", None) + + def _startHeartbeat(self): + if self.heartbeatThread is not None: + return + # A daemon thread: the run must not wait for it on the way out. + def beat(): + while self.claims: + time.sleep(HEARTBEAT_SECONDS) + try: + self._heartbeat() + except Exception as e: + print("Failed to update the job claims: %s" % e) + self.heartbeatThread = threading.Thread(target=beat, daemon=True) + self.heartbeatThread.start() + + def _heartbeat(self): + # A separate connection: the main one is in the middle of the run's work. + import psycopg2 + with psycopg2.connect(self.url) as conn: + with conn.cursor() as cur: + for (branch, libname, libversion, omcversion, confighash) in list(self.claims): + cur.execute("""UPDATE job_claim SET heartbeat = now() WHERE branch=%s AND libname=%s + AND libversion=%s AND omcversion=%s AND confighash=%s AND host=%s""", + (branch, libname, libversion, omcversion, confighash, self.host)) + + def release(self): + for (b, libname, libversion, omcversion, confighash) in self.claims: + self.execute("""UPDATE job_claim SET state='done', heartbeat=now() + WHERE branch=? AND libname=? AND libversion=? AND omcversion=? AND confighash=? + AND host=?""", + (b, libname, libversion, omcversion, confighash, self.host)) + self.claims = [] + self.commit() diff --git a/single-model.py b/single-model.py index d72e769..3d9af2b 100755 --- a/single-model.py +++ b/single-model.py @@ -3,11 +3,12 @@ import sys, argparse import simplejson as json -import shared +import shared, resultsdb parser = argparse.ArgumentParser(description='OpenModelica model testing report generation tool') parser.add_argument('models', nargs='*') parser.add_argument('--branch', default='') +resultsdb.addArgument(parser) args = parser.parse_args() @@ -21,28 +22,28 @@ libs = {} -import cgi, sqlite3, time, datetime +import cgi, time, datetime from omcommon import friendlyStr, multiple_replace -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() try: - cursor.execute("SELECT name FROM [sqlite_master] WHERE type='table' AND name=?", (branch,)) - v = cursor.fetchone()[0] + if not db.tableExists(branch): + raise Exception("no table") except: raise Exception("No such table '%s'; specify it using --branch=XXX" % branch) for model in models: lines=[] c=0 - libnames = [libname for (libname,) in cursor.execute("SELECT DISTINCT libname FROM [%s] WHERE model=? ORDER BY libname ASC" % (branch), (model,))] + libnames = [libname for (libname,) in cursor.execute("SELECT DISTINCT libname FROM %s WHERE model=? ORDER BY libname ASC" % (db.quote(branch)), (model,))] for libname in libnames: - for (finalphase,dint,libversion) in cursor.execute("SELECT finalphase,date,libversion FROM [%s] NATURAL JOIN [libversion] WHERE model=? AND libname=? ORDER BY date ASC" % (branch), (model,libname)): + for (finalphase,dint,libversion) in cursor.execute("SELECT finalphase,date,libversion FROM %s NATURAL JOIN libversion WHERE model=? AND libname=? ORDER BY date ASC" % (db.quote(branch)), (model,libname)): c+=1 dstr = str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S')) - cursor2 = conn.cursor() - omcversion = cursor2.execute("SELECT omcversion FROM [omcversion] WHERE date=? AND branch=?", (dint,branch)).fetchone()[0] + cursor2 = db.cursor() + omcversion = cursor2.execute("SELECT omcversion FROM omcversion WHERE date=? AND branch=?", (dint,branch)).fetchone()[0] omcversion = omcversion.replace("OpenModelica ","").replace("OMCompiler ","") lines.insert(0, "%s %s %s %s" % (dstr,shared.finalphaseName(finalphase),omcversion,libversion.strip())) if c==0: diff --git a/test.py b/test.py index affc5e3..b6ca625 100755 --- a/test.py +++ b/test.py @@ -8,7 +8,7 @@ if (sys.version_info < (3, 0)): raise Exception("Python2 is no longer supported") -import html, shutil, os, re, glob, time, argparse, sqlite3, datetime, math, platform +import html, shutil, os, re, glob, time, argparse, datetime, math, platform from joblib import Parallel, delayed import simplejson as json import psutil, subprocess, threading, hashlib @@ -18,7 +18,7 @@ from natsort import natsorted from shared import readConfig, getReferenceFileName, simulationAcceptsFlag, isFMPy from platform import processor -import shared +import shared, resultsdb import signal @@ -47,6 +47,7 @@ parser.add_argument('--msysEnvironment', help = 'MSYS2 environment used by OpenModelica on Windows.', default = 'ucrt64') parser.add_argument('--debug', action="store_true", help="turn on the DEBUG mode", default=False) parser.add_argument('--addmsl', action="store_true", help="add the MSL path to the OPENMODELICAPATH if the MSL is not detected in the libraries path", default=False) +resultsdb.addArgument(parser) args = parser.parse_args() configs = args.configs @@ -557,43 +558,10 @@ def testHelloWorld(cmd): # Create mos-files -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() - -user_version = cursor.execute("PRAGMA user_version").fetchone()[0] - -if user_version==0: - # BOOLEAN NOT NULL CHECK (verify IN (0,1) AND builds IN (0,1) AND simulates IN (0,1)) - # Table to lookup from a run (date, branch) to omcversion used - cursor.execute("CREATE TABLE if not exists [omcversion] (date integer NOT NULL, branch text NOT NULL, omcversion text NOT NULL)") - # Table to lookup from a run (date, branch) which library versions were used - cursor.execute("CREATE TABLE if not exists [libversion] (date integer NOT NULL, branch text NOT NULL, libname text NOT NULL, libversion text NOT NULL, confighash integer NOT NULL)") -elif user_version==1: - cursor.execute("ALTER TABLE [libversion] ADD COLUMN confighash integer NOT NULL DEFAULT(0)") -elif user_version==2: - tables = [tbl for (tbl,) in cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") if tbl not in ["libversion","omcversion"]] - for tbl in tables: - cursor.execute("ALTER TABLE [%s] ADD COLUMN parsing real NOT NULL DEFAULT(0.0)" % tbl) -elif user_version in [3]: - pass -else: - print("Unknown schema user_version=%d" % user_version) - sys.exit(1) - -def createBranchTable(branch): - """A run fills one table per FMI simulator, so this happens more than once.""" - cursor.execute('''CREATE TABLE if not exists [%s] - (date integer NOT NULL, libname text NOT NULL, model text NOT NULL, exectime real NOT NULL, - frontend real NOT NULL, backend real NOT NULL, simcode real NOT NULL, templates real NOT NULL, compile real NOT NULL, simulate real NOT NULL, - verify real NOT NULL, verifyfail integer NOT NULL, verifytotal integer NOT NULL, finalphase integer NOT NULL, parsing real NOT NULL)''' % branch) - cursor.execute('''DROP INDEX IF EXISTS [idx_%s_date]''' % branch) - -createBranchTable(primaryBranch) -cursor.execute('''DROP INDEX IF EXISTS idx_omcversion_date''') -cursor.execute('''DROP INDEX IF EXISTS idx_libversion_date''') - -# Set user_version to the current schema -cursor.execute("PRAGMA user_version=3") +db = resultsdb.connect(args.db) +cursor = db.cursor() +# One table per simulator, so this happens once per branch the run fills. +db.createTables(primaryBranch) def strToHashInt(s): return int(hashlib.sha1((s+"fixCorruptBuilds-2017-03-26").encode("utf-8")).hexdigest()[0:8],16) @@ -778,11 +746,18 @@ def hashReferenceFiles(s): prefix = conf["ignoreModelPrefix"] res=list(filter(lambda x: not x.startswith(prefix), res)) libName=shared.libname(library, conf) - v = cursor.execute("""SELECT date,libversion,libname,branch,omcversion FROM [libversion] NATURAL JOIN [omcversion] + v = cursor.execute("""SELECT date,libversion,libname,branch,omcversion FROM libversion NATURAL JOIN omcversion WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1""", (conf["libraryLastChange"],libName,primaryBranch,omc_version,confighash)).fetchone() if libName in stats_by_libname or libName in skipped_libs: raise Exception("Duplicate libName found: %s" % libName) if v is None or execAllTests: + # On the shared database another machine may already be running this exact + # job; claiming it is what keeps the two from testing the same thing. + if not db.claim(primaryBranch, libName, conf["libraryLastChange"], omc_version, confighash): + (host, since) = db.claimedBy(primaryBranch, libName, conf["libraryLastChange"], omc_version, confighash) + print("Skipping %s as %s has been testing it since %s" % (libName, host, since)) + skipped_libs[libName] = None + continue stats_by_libname[libName] = {"conf":conf, "stats":[]} tests = tests + [(r,library,libName,libName+"_"+r,conf) for r in res] print("Running library %s (%d tests)" % (libName, len(res))) @@ -899,7 +874,7 @@ def expectedExec(c): (model,lib,libName,name,data) = c if "expectedExec" in data: return data["expectedExec"] - cursor.execute("SELECT exectime FROM [%s] WHERE libname = ? AND model = ? ORDER BY date DESC LIMIT 1" % primaryBranch, (libName,model)) + cursor.execute("SELECT exectime FROM %s WHERE libname = ? AND model = ? ORDER BY date DESC LIMIT 1" % db.quote(primaryBranch), (libName,model)) v = cursor.fetchone() data["expectedExec"] = (v or (0.0,))[0] return data["expectedExec"] @@ -1003,17 +978,17 @@ def resultValues(model, libname, data, simulator=None): ) for (resultBranch, simulator) in resultBranches: - createBranchTable(resultBranch) + db.createTables(resultBranch) for key in stats.keys(): (name,model,libname,data)=stats[key] if simulator is None: stats_by_libname[libname]["stats"].append(stats[key]) - cursor.execute("INSERT INTO [%s] VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" % resultBranch, + cursor.execute("INSERT INTO %s VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)%s" % (db.quote(resultBranch), db.insertIgnore()), resultValues(model, libname, data, simulator)) for libname in stats_by_libname.keys(): confighash = stats_by_libname[libname]["conf"]["confighash"] - cursor.execute("INSERT INTO [libversion] VALUES (?,?,?,?,?)", (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash)) - cursor.execute("INSERT INTO [omcversion] VALUES (?,?,?)", (testRunStartTimeAsEpoch, resultBranch, omc_version)) + cursor.execute("INSERT INTO libversion VALUES (?,?,?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash)) + cursor.execute("INSERT INTO omcversion VALUES (?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, omc_version)) """ # Not really a good thing to do; was just done to make generation of the report simpler for libname in skipped_libs.keys(): @@ -1364,7 +1339,8 @@ def artifactSuffix(simulator): print("-- problem during removing of ./files dir") # Do not commit until we have generated and uploaded the reports -conn.commit() -conn.close() +db.commit() +db.release() +db.close() print("all tests done ...")
%s%s%s%s%s%s%s%s%s%s
%s%s%s%s%d