diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index 5d99a79..1619e07 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -3,6 +3,8 @@ pipeline { parameters { booleanParam(name: 'OLDLIBS', defaultValue: false, description: 'Also test some outdated libraries') + booleanParam(name: 'postgres', defaultValue: true, description: 'Store the results in the shared PostgreSQL database (omdb on openmodelica.org) rather than in the per-machine sqlite3 file. Machines coordinate through it, so two of them no longer overwrite each other. Untick to go back to the sqlite3 files.') + booleanParam(name: 'v1_26', defaultValue: false, description: 'maintenance/v1.26 branch (ryzen-5950x-1)') booleanParam(name: 'v1_27', defaultValue: false, description: 'maintenance/v1.27 branch (ryzen-5950x-1)') booleanParam(name: 'master', defaultValue: false, description: 'master branch (ryzen-5950x-1)') @@ -40,6 +42,12 @@ pipeline { } environment { LC_ALL = 'C.UTF-8' + // Where the results go. The scripts take it from here instead of a --db + // option on every single invocation. + LIBTEST_DB = "${params.postgres ? 'postgresql://om@openmodelica.org/omdb' : 'sqlite3.db'}" + // A secret file holding one pgpass line; libpq reads the password from it, + // so it never reaches a command line or the build log. + PGPASSFILE = credentials('omdb-pgpass') } stages { stage('test') { parallel { @@ -455,7 +463,12 @@ pipeline { cd OpenModelica git fetch ''' - sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper1/sqlite3.db' + // The reports read the shared database directly when it is in use. + script { + if (!params.postgres) { + sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper1/sqlite3.db' + } + } sh './clean-empty-omcversion-dates.py' sh "./all-reports.py --email --omcgitdir=OpenModelica ${env.GITBRANCHES} conversion heavy_tests" @@ -521,7 +534,11 @@ pipeline { cd OpenModelica git fetch ''' - sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper2/sqlite3.db' + script { + if (!params.postgres) { + sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper2/sqlite3.db' + } + } sh './clean-empty-omcversion-dates.py' sh "./all-reports.py --email --omcgitdir=OpenModelica ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_NEWBACKEND_DAE} ${env.GITBRANCHES_CPP} gbode cvode ida" @@ -978,14 +995,17 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om sh "test -d '${libraryPath}/.openmodelica/libraries/Modelica trunk'" - sh """ - if ! test -f ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db; then - wget -O ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp -q https://libraries.openmodelica.org/sqlite3/${dbPrefix}/sqlite3.db - mv ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db - fi - cp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db OpenModelicaLibraryTesting/sqlite3.db - test -s OpenModelicaLibraryTesting/sqlite3.db - """ + // The shared database needs none of this: the results go straight into it. + if (!params.postgres) { + sh """ + if ! test -f ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db; then + wget -O ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp -q https://libraries.openmodelica.org/sqlite3/${dbPrefix}/sqlite3.db + mv ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db + fi + cp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db OpenModelicaLibraryTesting/sqlite3.db + test -s OpenModelicaLibraryTesting/sqlite3.db + """ + } sh 'date' @@ -1008,11 +1028,16 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/' --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1 """) sh 'date' - sh "rm -f OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp" - sh "ln OpenModelicaLibraryTesting/sqlite3.db OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp" sh "cd OpenModelicaLibraryTesting/ && ./clean-empty-omcversion-dates.py" - sh "cp OpenModelicaLibraryTesting/sqlite3.db ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db" - sh "rm -f ~/TEST_LIBS_BACKUP/${dbPrefix}-`date +sqlite3.%Y%m%d.db`" - sshPublisher(publishers: [sshPublisherDesc(configName: sshConfig, transfers: [sshTransfer(removePrefix: 'OpenModelicaLibraryTesting', sourceFiles: 'OpenModelicaLibraryTesting/sqlite3.db')])], failOnError: true) + // Copying the file back is what made two machines overwrite each other's + // results, so it only happens while a job still writes its own sqlite3 file. + if (!params.postgres) { + sh "rm -f OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp" + sh "ln OpenModelicaLibraryTesting/sqlite3.db OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp" + sh "cp OpenModelicaLibraryTesting/sqlite3.db ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db" + sh "rm -f ~/TEST_LIBS_BACKUP/${dbPrefix}-`date +sqlite3.%Y%m%d.db`" + + sshPublisher(publishers: [sshPublisherDesc(configName: sshConfig, transfers: [sshTransfer(removePrefix: 'OpenModelicaLibraryTesting', sourceFiles: 'OpenModelicaLibraryTesting/sqlite3.db')])], failOnError: true) + } } diff --git a/.CI/build-dep/Dockerfile b/.CI/build-dep/Dockerfile index 78dbab8..c0f8abe 100644 --- a/.CI/build-dep/Dockerfile +++ b/.CI/build-dep/Dockerfile @@ -1,4 +1,4 @@ FROM docker.openmodelica.org/build-deps:v1.16.3 RUN apt-get update && apt-get install libxml2 libxslt1.1 libxml2-dev libxslt1-dev -RUN pip3 install matplotlib FMPy +RUN pip3 install matplotlib FMPy psycopg2-binary diff --git a/all-plots.py b/all-plots.py index 6397707..e7c8026 100755 --- a/all-plots.py +++ b/all-plots.py @@ -3,7 +3,7 @@ import sys, argparse, subprocess, os import simplejson as json -import shared +import shared, resultsdb import re, time, math from omcommon import friendlyStr @@ -18,6 +18,7 @@ parser = argparse.ArgumentParser(description='OpenModelica model testing report generation tool') parser.add_argument('branches', nargs='*') parser.add_argument('--historypath', default="history") +resultsdb.addArgument(parser) args = parser.parse_args() branches = [branch.split("/")[-1] for branch in args.branches] @@ -25,11 +26,11 @@ libs = {} -import cgi, sqlite3, time, datetime +import cgi, time, datetime from omcommon import friendlyStr, multiple_replace -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() def dateStr(dint): return str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S')) @@ -98,8 +99,7 @@ def plotLibrary(branch, libname, xs, total, frontend,backend,simcode,template,co for branch in branches: try: - cursor.execute("SELECT name FROM [sqlite_master] WHERE type='table' AND name=?", (branch,)) - one = cursor.fetchone() + one = (branch,) if db.tableExists(branch) else None if one == None: print("No such table '%s'; specify it using --branch=XXX when running test.py" % branch) # ignore this table and continue @@ -112,13 +112,13 @@ def plotLibrary(branch, libname, xs, total, frontend,backend,simcode,template,co # ignore this table and continue continue - cursor.execute('''CREATE INDEX IF NOT EXISTS [idx_%s_date] ON [%s](date)''' % (branch,branch)) + db.createDateIndex(branch) libs = {} - for (date,libname,total,frontend,backend,simcode,template,compile,simulate,verify) in cursor.execute("""SELECT date,libname,COUNT(finalphase),COUNT(finalphase>=1 or null),COUNT(finalphase>=2 or null),COUNT(finalphase>=3 or null),COUNT(finalphase>=4 or null),COUNT(finalphase>=5 or null),COUNT(finalphase>=6 or null),COUNT(finalphase>=7 or null) - FROM [%s] + for (date,libname,total,frontend,backend,simcode,template,compile,simulate,verify) in cursor.execute("""SELECT date,libname,COUNT(finalphase),%s + FROM %s GROUP BY date,libname ORDER BY libname,date ASC -""" % (branch)): +""" % (",".join(db.countIf("finalphase>=%d" % i) for i in range(1,8)), db.quote(branch))): if libname not in libs: libs[libname] = ([],[],[],[],[],[],[],[],[]) libs[libname][0].append(datetime.datetime.fromtimestamp(date)) diff --git a/all-reports.py b/all-reports.py index 07f0c6e..dbd1a5b 100755 --- a/all-reports.py +++ b/all-reports.py @@ -5,7 +5,7 @@ import codecs import sys, argparse, subprocess, os, time import simplejson as json -import shared +import shared, resultsdb import re from omcommon import friendlyStr @@ -17,6 +17,7 @@ parser.add_argument('--githuburltesting', default="https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit") parser.add_argument('--omcgitdir', default="../OpenModelica/OpenModelica") parser.add_argument('--email', default=False, action='store_true') +resultsdb.addArgument(parser) args = parser.parse_args() os.environ['TZ'] = 'Europe/Stockholm' @@ -44,11 +45,11 @@ libs = {} -import cgi, sqlite3, time, datetime +import cgi, time, datetime from omcommon import friendlyStr, multiple_replace -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() def dateStr(dint): return str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S')) @@ -70,8 +71,7 @@ def modelLink(libname, modelname, extension, text): emails_to_send = {} for branch in branches: try: - cursor.execute("SELECT name FROM [sqlite_master] WHERE type='table' AND name=?", (branch,)) - one = cursor.fetchone() + one = (branch,) if db.tableExists(branch) else None if one == None: print("No such table '%s'; specify it using --branch=XXX when running test.py" % branch) # ignore this table and continue @@ -86,8 +86,8 @@ def modelLink(libname, modelname, extension, text): missing_branches.append(branch) continue - cursor.execute('''CREATE INDEX IF NOT EXISTS [idx_%s_date] ON [%s](date)''' % (branch,branch)) - cursor.execute("SELECT date,omcversion FROM [omcversion] WHERE branch LIKE ? COLLATE NOCASE ORDER BY date ASC", (branch,)) + db.createDateIndex(branch) + cursor.execute("SELECT date,omcversion FROM omcversion WHERE %s ORDER BY date ASC" % db.likeNoCase("branch"), (branch,)) entries = cursor.fetchall() n = len(entries) urlToOpen = "%s/%s/00_history.html" % (historyurl, branch) @@ -152,11 +152,11 @@ def modelLink(libname, modelname, extension, text): gitloglibrarytesting = "could not get the git log for OpenModelicaLibraryTesting" tpl = tpl.replace("#OMCGITLOG#",gitlog).replace("#NUMCOMMITS#",str(gitlog.count(""))).replace("#3rdParty#",thirdPartyChanged).replace("#OMCLIBRARYTESTINGGITLOG#",gitloglibrarytesting) - libnames = [libname for (libname,) in cursor.execute("""SELECT libname FROM [%s] WHERE date=? GROUP BY libname""" % branch, (d2,))] + libnames = [libname for (libname,) in cursor.execute("""SELECT libname FROM %s WHERE date=? GROUP BY libname""" % db.quote(branch), (d2,))] startdates = {} # Get previous date of each library run and group them together for fast queries later for libname in libnames: - ds = cursor.execute("""SELECT date FROM [%s] WHERE date MAX(finalphase)) OR ((MIN(finalphase) >= ?) AND (MAX(frontend) > ?*MIN(frontend) AND MAX(frontend) > ?) OR @@ -179,7 +183,7 @@ def modelLink(libname, modelname, extension, text): (MAX(compile) > ?*MIN(compile) AND MAX(compile) > ?) OR (MAX(simulate) > ?*MIN(simulate) AND MAX(simulate) > ?) ) - """ % (branch,",".join(["'%s'" % libname for libname in startdates[d1lib]])) + """) % (db.quote(branch),",".join(["'%s'" % libname for libname in startdates[d1lib]])) cursor.execute(query, (d1lib,d2,timeMinPhase,timeRel,timeAbs,timeRel,timeAbs,timeRel,timeAbs,timeRel,timeAbs,timeRel,2*timeAbs,timeRel,timeAbs)) regressions += cursor.fetchall() regressions = sorted(regressions, key = lambda x: (x[1],x[0])) @@ -228,10 +232,10 @@ def modelLink(libname, modelname, extension, text): libstrs = [] for libname in sorted(list(libs)): - cursor.execute("SELECT libversion,confighash FROM [libversion] WHERE branch LIKE ? COLLATE NOCASE AND date<=? AND libname=? ORDER BY date DESC LIMIT 1", (branch,d1,libname)) + cursor.execute("SELECT libversion,confighash FROM libversion WHERE %s AND date<=? AND libname=? ORDER BY date DESC LIMIT 1" % db.likeNoCase("branch"), (branch,d1,libname)) (lv1,lh1) = cursor.fetchone() lv1 = lv1.strip() - cursor.execute("SELECT libversion,confighash FROM [libversion] WHERE branch LIKE ? COLLATE NOCASE AND date<=? AND libname=? ORDER BY date DESC LIMIT 1", (branch,d2,libname)) + cursor.execute("SELECT libversion,confighash FROM libversion WHERE %s AND date<=? AND libname=? ORDER BY date DESC LIMIT 1" % db.likeNoCase("branch"), (branch,d2,libname)) (lv2,lh2) = cursor.fetchone() lv2 = lv2.strip() if lv1 != lv2: diff --git a/clean-dates.py b/clean-dates.py index ddafe63..09b831e 100755 --- a/clean-dates.py +++ b/clean-dates.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 -import argparse, sqlite3, sys +import argparse, sys +import resultsdb from datetime import datetime parser = argparse.ArgumentParser(description='OpenModelica library testing tool') parser.add_argument('startDate') parser.add_argument('stopDate') +resultsdb.addArgument(parser) args = parser.parse_args() @@ -29,12 +31,12 @@ sys.stdout.write("Please respond with 'yes' or 'no'") sys.exit(1) -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() -tables = [tbl for (tbl,) in cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")] +tables = db.tables() for tbl in tables: - cursor.execute("DELETE FROM [%s] WHERE date?" % tbl, (stopTime.timestamp(),startTime.timestamp())) -conn.commit() -conn.execute("VACUUM") -conn.close() + cursor.execute("DELETE FROM %s WHERE date?" % db.quote(tbl), (stopTime.timestamp(),startTime.timestamp())) +db.commit() +db.vacuum() +db.close() diff --git a/clean-empty-omcversion-dates.py b/clean-empty-omcversion-dates.py index 1b0fef4..63aad13 100755 --- a/clean-empty-omcversion-dates.py +++ b/clean-empty-omcversion-dates.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -import argparse, sqlite3, sys +import argparse, sys +import resultsdb from datetime import datetime parser = argparse.ArgumentParser(description='OpenModelica library testing tool') +resultsdb.addArgument(parser) args = parser.parse_args() -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() entries = cursor.execute("SELECT date,branch FROM omcversion").fetchall() dropped=0 @@ -20,7 +22,11 @@ branchDates[branch] = set() branchDates[branch].add(date) for branch in branches: - data=cursor.execute("SELECT DISTINCT date FROM [%s]" % branch).fetchall() + # The shared database holds the branches of every machine, including ones + # this one never created a result table for. + if not db.tableExists(branch): + continue + data=cursor.execute("SELECT DISTINCT date FROM %s" % db.quote(branch)).fetchall() for (date,) in data: try: branchDates[branch].remove(date) @@ -28,10 +34,10 @@ pass for date in branchDates[branch]: print("Dropping empty omcversion entry (%d,%s)" % (date,branch)) - cursor.execute("DELETE FROM [omcversion] WHERE date=? AND branch=?", (date,branch)) + cursor.execute("DELETE FROM omcversion WHERE date=? AND branch=?", (date,branch)) dropped += 1 -conn.commit() +db.commit() if dropped>0: - conn.execute("VACUUM") -conn.close() + db.vacuum() +db.close() diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..5d908a6 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,432 @@ +# The OpenModelicaLibraryTesting result database + +This document describes how the library testing results are stored: the current +sqlite3 layout, what every table and column means, how the scripts query it, and +the PostgreSQL layout the data is being migrated to (issue +[#295](https://github.com/OpenModelica/OpenModelicaLibraryTesting/issues/295)). + +## Where the databases live + +Each test machine keeps a single sqlite3 file called `sqlite3.db` in the working +directory of the test run. The files are published at: + +| machine | URL | on omod-r630-2 | size | tables | rows | +| --- | --- | --- | --- | --- | --- | +| ripper1 | | `/var/www/libraries.openmodelica.org/sqlite3/ripper1/sqlite3.db` | 11.8 GB | 34 | ~69 million | +| ripper2 | | `/var/www/libraries.openmodelica.org/sqlite3/ripper2/sqlite3.db` | 15.5 GB | 54 | ~91 million | + +(sizes as of 2026-08-10; both are at `PRAGMA user_version` 3) + +Every branch/configuration tested on a machine ends up in that machine's file, +which is why the files are large and why two machines cannot test the same job +without overwriting each other's results. The tables are the branches currently +tested (`master`, `newInst-newBackend`, `cpp`, `master-fmi`, `gbode`, `cvode`, +`ida`, `daemode`, ...) plus one per historical release (`v1.9` ... `v1.27`, +`v1.11-fmi` ...). `master` alone holds ~43 million rows on ripper1. + +Six table names exist on both machines - `master`, `newInst`, `heavy_tests`, +`v1.17`, `libversion` and `omcversion` - so a shared database mixes rows from +both. They are distinguished by their `date`, which is unique per run and +machine. + +The scripts always open the file by the hardcoded relative name `sqlite3.db`: +`test.py` writes it, `report.py`, `all-reports.py`, `all-plots.py`, +`single-model.py`, `clean-dates.py` and `clean-empty-omcversion-dates.py` read +it. + +## Schema version + +`test.py` keeps the layout version in sqlite's `PRAGMA user_version` and +migrates on startup (`test.py`, around the `CREATE TABLE` block): + +| user_version | meaning | +| --- | --- | +| 0 | empty/new database; `omcversion` and `libversion` are created | +| 1 | `libversion.confighash` added | +| 2 | `parsing` added to every per-branch table | +| 3 | current layout | + +A database with a higher `user_version` makes `test.py` exit rather than guess. + +## Tables + +### Per-branch result tables + +There is **one table per tested branch/configuration**, named after the branch +(`master`, `gbode`, `ida`, `cvode`, `master-fmi`, `newInst-newBackend`, +`heavy_tests`, ...), created on demand by `test.py`: + +```sql +CREATE TABLE if not exists [] ( + date integer NOT NULL, -- unix epoch: start of the test run + libname text NOT NULL, -- library incl. version suffix, e.g. Buildings_9.1.0 + model text NOT NULL, -- full Modelica class name of the tested model + exectime real NOT NULL, -- wall clock for the whole test of this model [s] + frontend real NOT NULL, -- time in the front end [s] + backend real NOT NULL, -- time in the back end [s] + simcode real NOT NULL, -- time generating SimCode [s] + templates real NOT NULL, -- time running the code generation templates [s] + compile real NOT NULL, -- time compiling the generated code (or building the FMU) [s] + simulate real NOT NULL, -- time simulating [s] + verify real NOT NULL, -- time spent in diffSimulationResults [s] + verifyfail integer NOT NULL, -- number of variables that differ from the reference + verifytotal integer NOT NULL, -- number of variables compared against the reference + finalphase integer NOT NULL, -- how far the model got, see below + parsing real NOT NULL -- time loading/parsing the library [s] +) +``` + +Notes on the values, which are produced by `testmodel.py` and written by +`test.py`: + +- `date` is `int(time.time())` taken **once per test run** (`testRunStartTimeAsEpoch`), + so all rows of one run share the same date. It is the join key to + `omcversion`/`libversion` and the x-axis of every history plot. +- The phase times are exclusive, computed by subtracting the nested OMC timers + from each other (`frontend = frontend - backend`, `backend = backend - simcode`, ...). + A phase that was never reached is stored as `0.0`. +- `compile` is the `build` measurement: `make -f .makefile` for the C + runtime, the FMU build for FMI configurations, and the JIT compile time for + wasm-jit. +- `exectime` is the total wall clock of the model's test process. `test.py` + reads back the most recent value (`SELECT exectime ... ORDER BY date DESC LIMIT 1`) + to sort the queue longest-job-first. +- `verifyfail`/`verifytotal` are `len(diff.vars)` and `diff.numCompared`; a model + without reference variables stores `0`/`0` and still reaches phase 7. + +`finalphase` is the last phase completed; `shared.finalphaseName` maps it to: + +| value | name | meaning | +| --- | --- | --- | +| 0 | Failed | the front end did not finish | +| 1 | FrontEnd | front end ok, back end failed | +| 2 | BackEnd | back end ok, SimCode failed | +| 3 | SimCode | SimCode ok, templates/translation failed | +| 4 | Templates | translated, but compilation/build failed | +| 5 | Compile | built, but the simulation failed | +| 6 | Simulate | simulated, but the result does not verify (or was not compared) | +| 7 | Verify | the result matches the reference file | + +Reports count models per phase with `WHERE finalphase >= i`, so the columns of +the HTML tables are cumulative. + +### `omcversion` + +Maps a test run to the compiler that produced it: + +```sql +CREATE TABLE if not exists [omcversion] ( + date integer NOT NULL, -- same epoch as the result rows of that run + branch text NOT NULL, -- branch/configuration name = result table name + omcversion text NOT NULL -- output of getVersion(), e.g. "OMCompiler v1.26.0-dev.42+g0123abc" +) +``` + +One row per run. `report.py` and `all-reports.py` use it to label a run, and +`all-reports.py` walks it in date order to pair consecutive runs when generating +the regression reports. + +### `libversion` + +Maps a test run to the library versions and configuration used: + +```sql +CREATE TABLE if not exists [libversion] ( + date integer NOT NULL, -- same epoch as the result rows of that run + branch text NOT NULL, + libname text NOT NULL, -- as in the result table + libversion text NOT NULL, -- conf["libraryLastChange"]: version + git revision/zip hash + confighash integer NOT NULL -- hash of the configuration and the reference files +) +``` + +One row per (run, library). `confighash` is `strToHashInt()` over the +configuration dictionary plus the hashes of all reference files, so any change +to the config or to a reference file yields a different value. + +This drives the "do we need to test this at all" decision in `test.py`: before +testing a library it looks for + +```sql +SELECT date,libversion,libname,branch,omcversion FROM [libversion] NATURAL JOIN [omcversion] +WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1 +``` + +and skips the library when the exact same combination of library version, OMC +version and configuration was already tested. + +### `datelookup_` (obsolete) + +`datelookup_(date, runDate, libname, branch)` was a cache mapping every +omcversion date to the latest run date of a library. The code that fills it in +`all-plots.py` sits inside a triple-quoted block and is no longer executed; +neither ripper1 nor ripper2 still has such a table. The migration skips them. + +### Indexes + +No index is stored permanently. `test.py` drops `idx__date`, +`idx_omcversion_date` and `idx_libversion_date` on startup (they slow the bulk +insert down), and `report.py`/`all-reports.py`/`all-plots.py` recreate +`idx__date` when they need it. + +## How a test run writes the database + +1. Open `sqlite3.db`, apply the `user_version` migration, `CREATE TABLE IF NOT EXISTS []`. +2. Compute `confighash` per library, skip libraries already covered (query above). +3. Run the tests; each model writes `files/.stat.json`. +4. At the end, in one transaction: one `INSERT` per model into `[]`, one + `INSERT` per library into `[libversion]`, one `INSERT` into `[omcversion]`, + then `conn.commit()`. + +Nothing is written while the tests run, so an aborted run leaves no rows behind, +and two machines running the same job produce two full sets of rows in two +separate files - whichever file is copied back last wins. + +## Housekeeping scripts + +- `clean-dates.py --start --stop`: `DELETE FROM [] WHERE date?` + over every table, then `VACUUM`. Removes a range of bad runs. +- `clean-empty-omcversion-dates.py`: drops `omcversion` rows whose date has no + result rows in the corresponding branch table. + +## PostgreSQL layout + +The PostgreSQL database is a **mirror** of the sqlite3 one: the same tables with +the same names and columns, one table per branch plus `omcversion` and +`libversion`. That way the test scripts can push new results to the network +database with the same statements they use today, and the report scripts need no +query rewriting beyond the sqlite `[name]` / PostgreSQL `"name"` quoting. + +Only the types are adapted: + +| sqlite3 | PostgreSQL | +| --- | --- | +| `integer` | `bigint` (`integer` for `verifyfail`, `verifytotal`, `finalphase`) | +| `real` | `double precision` | +| `text` | `text` | +| `NOT NULL` on every column | only on the key columns, see below | +| `PRAGMA user_version` | not used; the `parsing` column always exists | +| `datelookup_*` | not migrated (derived data, no longer generated) | + +### Keys + +Each table gets a unique key, which sqlite3 never had: + +| table | key | +| --- | --- | +| `` | `(date, libname, model)` - a run tests every model of a library once | +| `omcversion` | `(date, branch)` - one row per run | +| `libversion` | `(date, branch, libname, confighash)` - one row per run and library | + +This is what makes a shared database possible: results from a second test +machine can be merged into a table that already holds another machine's rows, +and a run that is pushed twice cannot produce duplicates. + +Only those key columns are `NOT NULL`. The sqlite3 tables declare every column +`NOT NULL`, but `CREATE TABLE if not exists` means tables created by an older +`test.py` keep their old, laxer declaration, so the historical data does not +necessarily hold up. `libversion.libversion` for instance stores empty strings +for some old runs. + +So a branch table becomes: + +```sql +CREATE TABLE "master" ( + date bigint NOT NULL, + libname text NOT NULL, + model text NOT NULL, + exectime double precision NOT NULL, + frontend double precision NOT NULL, + backend double precision NOT NULL, + simcode double precision NOT NULL, + templates double precision NOT NULL, + compile double precision NOT NULL, + simulate double precision NOT NULL, + verify double precision NOT NULL, + verifyfail integer NOT NULL, + verifytotal integer NOT NULL, + finalphase integer NOT NULL, + parsing double precision NOT NULL +); +``` + +Two things to keep in mind when querying it: + +- Identifiers must be **double quoted**, not bracketed: branch names such as + `newInst-newBackend` contain upper case letters and dashes, which PostgreSQL + would otherwise fold to lower case or reject. +- All test machines write into the same tables, so a run is identified by + `date` (plus `branch`) exactly as before. Use `--pgschema ripper1` if a + machine should be mirrored into a schema of its own instead. + +Indexes are created by `sqlite2postgres.py --index` rather than on the fly: +`(date)` on every table, `(branch, date)` on `omcversion`, +`(branch, libname, date)` on `libversion` and `(libname, date)` on the branch +tables. + +## Using the shared database + +Every script takes `--db`, which is a path to a local sqlite3 file (the default, +`sqlite3.db`) or a `postgresql://` URL: + +```bash +./test.py --branch=master --db=postgresql://om@openmodelica.org/omdb configs/conf.json +./report.py --branches=master --db=postgresql://om@openmodelica.org/omdb configs/conf.json +``` + +The password comes from `PGPASSWORD` or `~/.pgpass`, never from the URL or the +command line. `resultsdb.py` holds the two backends behind one interface; the +scripts write the same statements for both, with `?` as the placeholder, and ask +the connection where the dialects genuinely differ (`quote()`, `tableExists()`, +`groupConcat()`, `countIf()`, `likeNoCase()`, `insertIgnore()`). + +### Job claiming + +The point of the shared database is that two machines can test at the same time +without overwriting each other. Before testing a library, `test.py` claims the +job in + +```sql +CREATE TABLE job_claim ( + branch text, libname text, libversion text, omcversion text, confighash bigint, + host text NOT NULL, state text NOT NULL, + claimed_at timestamptz NOT NULL DEFAULT now(), + heartbeat timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (branch, libname, libversion, omcversion, confighash) +); +``` + +The key is exactly the question "which library, in which version, against which +compiler and configuration": the same combination the run already uses to decide +whether results exist. A claim is taken with `INSERT ... ON CONFLICT DO UPDATE +... WHERE` so that only one machine can win it, and a machine that loses prints + +``` +Skipping Buildings_9.1.0 as ripper2 has been testing it since 2026-08-10 22:14:03 +``` + +and moves on to the next library instead of repeating the work. The winner +refreshes `heartbeat` every minute from a background thread and sets +`state='done'` when the results are written. A machine that dies stops sending +its heartbeat, and after 30 minutes (`STALE_CLAIM_MINUTES`) another machine may +take its jobs over, so a crash does not park a library forever. + +Nothing of this applies to a local sqlite3 file: it has a single writer, and +`claim()` always says yes. + +### In Jenkins + +The pipeline has a `postgres` parameter, on by default, and sets two variables +for every stage: + +```groovy +environment { + LIBTEST_DB = "${params.postgres ? 'postgresql://om@openmodelica.org/omdb' : 'sqlite3.db'}" + PGPASSFILE = credentials('omdb-pgpass') +} +``` + +`LIBTEST_DB` is where `--db` defaults to, so no invocation has to spell it out, +and `omdb-pgpass` is a Jenkins *secret file* credential holding a single line: + +``` +openmodelica.org:5432:omdb:om: +``` + +libpq reads the password from that file, so it never appears on a command line +or in the build log. `resultsdb.py` takes a private copy of the file when its +permissions let anyone else read it, because libpq silently ignores such a file +and then fails with `fe_sendauth: no password supplied`. + +With `postgres` ticked, a job no longer downloads the machine's `sqlite3.db` +before the run nor publishes it back afterwards - the step that made two +machines overwrite each other. Untick the parameter and the old behaviour is +back, unchanged. + +The test machines need `psycopg2`: `pip3 install psycopg2-binary`, or a rebuild +of the images, since it is in `requirements.txt` and in `.CI/build-dep`. + +## Migrating + +Run this **on omod-r630-2 (openmodelica.org)**: the sqlite3 files and the +PostgreSQL server are on the same machine, so the data never goes over the +network. Pushing it from a developer machine works too, but a home uplink does +0.3-1.8 MB/s, which means hours for ~27 GB. + +```bash +export PGPASSFILE=~/.pgpass # never put the password on the command line +DB=/var/www/libraries.openmodelica.org/sqlite3 +./sqlite2postgres.py --host localhost --sqlite $DB/ripper1/sqlite3.db --source ripper1 +./sqlite2postgres.py --host localhost --index +./sqlite2postgres.py --host localhost --sqlite $DB/ripper2/sqlite3.db --source ripper2 --skip-existing +./sqlite2postgres.py --host localhost --index +./sqlite2postgres.py --host localhost --sqlite $DB/ripper1/sqlite3.db --verify +``` + +The order matters. ripper1 goes in first and without an index, which is the +fast path. `--index` then creates the unique keys, so that ripper2, loaded with +`--skip-existing`, keeps whatever is already there whenever a key collides - +ripper1 wins. The second `--index` covers the tables that only exist on ripper2. + +On the databases as of 2026-08-10 the priority never actually fires: not one +`(branch, date)` is shared between the two machines, not even for the four +branches both of them test, so the merge is a plain union. The rule matters for +re-runs and for two machines pushing results later on. + +### Disk space + +A branch table row measures **212 bytes** in PostgreSQL (measured on `v1.10`, +average `model` length 57, `libname` 17). The whole migration is therefore about + +- 34 GB of table data for the ~160 million rows, plus +- 10-12 GB for the `(date)` and `(libname, date)` indexes. + +so **plan for ~50 GB**. On omod-r630-2 the cluster lives in +`/var/lib/postgresql/16` on the root LV, which has 14 GB free, while the `data` +ZFS pool has 2.9 TB free. Put the database on the pool before loading, e.g. + +```bash +sudo zfs create -o mountpoint=/data/postgres -o compression=lz4 -o recordsize=16k data/postgres +sudo install -d -o postgres -g postgres /data/postgres/omdb +sudo -u postgres psql -c "CREATE TABLESPACE omdb_ts LOCATION '/data/postgres/omdb'" +sudo -u postgres psql -c "ALTER DATABASE omdb SET TABLESPACE omdb_ts" # needs no open connections +``` + +`lz4` on the dataset typically cuts this data to well under half, since the +model names repeat in every run. + +### Catching up with the jobs that are still running + +The migration is a snapshot: a test run that was started before it, or any run +that still writes its own sqlite3 file, adds rows the shared database has never +seen. `--catch-up` copies them over: + +```bash +./sqlite2postgres.py --host localhost --sqlite $DB/ripper1/sqlite3.db --source ripper1 --catch-up +./sqlite2postgres.py --host localhost --sqlite $DB/ripper2/sqlite3.db --source ripper2 --catch-up +``` + +It reads the database from the start and keeps the runs the shared database +does not have, which takes two to three minutes per machine. Not "everything +past the rowid the migration stopped at", tempting as that is: `VACUUM` +renumbers the rowids of these tables and `clean-empty-omcversion-dates.py` runs +one after every test, so that number does not survive a test run. Picking the +runs by date is also what makes it correct for `master`, `newInst`, +`heavy_tests` and `v1.17`, where both machines write into the same table. + +Repeating it costs nothing but the reading: the keys reject anything already +there. Run it once more right after the jobs are switched to the shared +database; from then on nothing writes the sqlite3 files any more and there is +nothing left to catch up with. + +`--source` only names the machine for the bookkeeping table +`migration_progress(source, tbl, last_rowid, rows_read, done)`, which records how +far each sqlite table has been copied. Each batch is committed together with its +progress row, so an interrupted migration continues from the last rowid that +made it in and never copies a batch twice; the command can simply be run again. +Rows are streamed in batches of `--batch` (200000 by default) through +`COPY ... FROM STDIN`, so memory use does not depend on the size of the database. + +`COPY` is used in its text format rather than CSV on purpose: an empty CSV field +reads back as NULL, which would silently turn the empty `libversion` strings in +the old data into NULLs. diff --git a/report.py b/report.py index fe8eafe..97cbd80 100755 --- a/report.py +++ b/report.py @@ -3,11 +3,12 @@ import sys, argparse import simplejson as json -import shared +import shared, resultsdb parser = argparse.ArgumentParser(description='OpenModelica library testing report generation tool') parser.add_argument('configs', nargs='*') parser.add_argument('--branches', default='master') +resultsdb.addArgument(parser) args = parser.parse_args() configs = args.configs @@ -23,7 +24,7 @@ libs = {} -import html, sqlite3, time, datetime +import html, time, datetime from omcommon import friendlyStr, multiple_replace configs_lst = [shared.readConfig(c) for c in configs] @@ -32,8 +33,8 @@ configs = configs + c libnames = set(shared.libname(library,conf) for (library,conf) in configs) -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() nmodels = {} nsimulate = {} @@ -42,7 +43,7 @@ missing_branches = [] for branch in branches: try: - cursor.execute("SELECT date FROM [%s] ORDER BY date DESC LIMIT 1" % branch) + cursor.execute("SELECT date FROM %s ORDER BY date DESC LIMIT 1" % db.quote(branch)) one = cursor.fetchone() if one == None: print("No such table '%s'; specify it using --branch=XXX when running test.py" % branch) @@ -58,18 +59,18 @@ continue dates_str[branch] = str(datetime.datetime.fromtimestamp(v).strftime('%Y-%m-%d %H:%M:%S')) - cursor.execute('''CREATE INDEX IF NOT EXISTS [idx_%s_date] ON [%s](date)''' % (branch,branch)) + db.createDateIndex(branch) dates[branch] = {} branch_nmodels = 0 for libname in libnames: - cursor.execute("SELECT date FROM [%s] WHERE libname=? ORDER BY date DESC LIMIT 1" % branch, (libname,)) + cursor.execute("SELECT date FROM %s WHERE libname=? ORDER BY date DESC LIMIT 1" % db.quote(branch), (libname,)) v = cursor.fetchone() if v is None: dates[branch][libname] = 0 continue dates[branch][libname] = v[0] - for x in cursor.execute("SELECT model FROM [%s] WHERE libname=? AND date=?" % branch, (libname,v[0])): + for x in cursor.execute("SELECT model FROM %s WHERE libname=? AND date=?" % db.quote(branch), (libname,v[0])): if libname not in libs: libs[libname] = set() libs[libname].add(x[0]) @@ -89,7 +90,7 @@ def checkEqual(iterator): for lib in sorted(libs.keys()): models = libs[lib] entries += "

%s

\n" % lib - branches_versions = [(cursor.execute("SELECT libversion FROM [libversion] WHERE libname=? AND branch=? ORDER BY date DESC LIMIT 1", (lib,branch)).fetchone() or ["unknown"])[0] for branch in branches] + branches_versions = [(cursor.execute("SELECT libversion FROM libversion WHERE libname=? AND branch=? ORDER BY date DESC LIMIT 1", (lib,branch)).fetchone() or ["unknown"])[0] for branch in branches] all_equal = checkEqual(branches_versions) if not all_equal: entries += "\n" @@ -109,12 +110,12 @@ def checkEqual(iterator): master_models = [] for i in range(0,8): i_models = set() - for v in cursor.execute("SELECT model FROM [%s] WHERE date=? AND finalphase>=? AND libname=?" % (branch), (dates[branch][lib],i,lib)): + for v in cursor.execute("SELECT model FROM %s WHERE date=? AND finalphase>=? AND libname=?" % (db.quote(branch)), (dates[branch][lib],i,lib)): i_models.add(v[0]) master_models.append(i_models) models[branch] = master_models for branch in branches: - vs = [cursor.execute("SELECT COUNT(*) FROM [%s] WHERE date=? AND finalphase>=? AND libname=?" % (branch), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] + vs = [cursor.execute("SELECT COUNT(*) FROM %s WHERE date=? AND finalphase>=? AND libname=?" % (db.quote(branch)), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] warnings = [] entries += '' % (branch,lib,lib,branch) for i in [0]+list(range(0,len(vs))): @@ -138,8 +139,8 @@ def checkEqual(iterator): entries += "
%s
\n" entries += entryhead for branch in branches: - vs = [cursor.execute("SELECT COUNT(*) FROM [%s] WHERE date=? AND finalphase>=? AND libname=?" % (branch), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] - sums = [cursor.execute("SELECT SUM(%s) FROM [%s] WHERE date=? AND libname=?" % (fields[i],branch), (dates[branch][lib],lib)).fetchone()[0] or 0 for i in range(0,9)] + vs = [cursor.execute("SELECT COUNT(*) FROM %s WHERE date=? AND finalphase>=? AND libname=?" % (db.quote(branch)), (dates[branch][lib],i,lib)).fetchone()[0] for i in range(0,8)] + sums = [cursor.execute("SELECT SUM(%s) FROM %s WHERE date=? AND libname=?" % (fields[i],db.quote(branch)), (dates[branch][lib],lib)).fetchone()[0] or 0 for i in range(0,9)] entries += '' % (branch,lib,lib,branch) entries += ("\n" % (friendlyStr(sums[0]),friendlyStr(sums[1]),friendlyStr(sums[2]),friendlyStr(sums[3]),friendlyStr(sums[4]),friendlyStr(sums[5]),friendlyStr(sums[6]),friendlyStr(sums[7]),friendlyStr(sums[8]))) exectime[branch] += sums[0] @@ -148,7 +149,7 @@ def checkEqual(iterator): nummodels = sum(len(l) for l in libs.values()) branches_lines = [("%d\n" % (html.escape(branch), html.escape( - (cursor.execute("SELECT omcversion FROM [omcversion] WHERE date=? AND branch=?", (max(dates[branch][lib] for lib in libnames),branch)).fetchone() or ["unknown"])[0] + (cursor.execute("SELECT omcversion FROM omcversion WHERE date=? AND branch=?", (max(dates[branch][lib] for lib in libnames),branch)).fetchone() or ["unknown"])[0] ), html.escape(dates_str[branch]), friendlyStr(exectime[branch]), " class=\"warning\"" if nummodels!=nmodels[branch] else "", nsimulate[branch], diff --git a/requirements.txt b/requirements.txt index bf3dd41..c24fbe9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ monotonic natsort ompython==3.6 psutil +psycopg2-binary simplejson diff --git a/resultsdb.py b/resultsdb.py new file mode 100644 index 0000000..81f817e --- /dev/null +++ b/resultsdb.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +""" +The result database of the library testing, either local or shared. + +Historically every test machine wrote its results to its own sqlite3 file, +copied in before a run and copied back afterwards, so two machines running the +same job overwrote each other. The scripts can now talk to a shared PostgreSQL +database instead, where the machines coordinate through [job_claim] and nobody +overwrites anybody. + + db = connect("sqlite3.db") # the old local file + db = connect("postgresql://om@openmodelica.org/omdb") # the shared database + +Both backends take the same statements, with "?" as the placeholder. Where the +two dialects genuinely differ - quoting a branch name, testing whether a table +exists, concatenating a group, counting a condition - ask the connection +instead of writing it out, see quote()/tableExists()/groupConcat()/countIf(). + +PostgreSQL needs psycopg2 (pip install psycopg2-binary) and reads the password +from PGPASSWORD or ~/.pgpass, never from the URL. +""" + +import os +import re +import socket +import sqlite3 +import threading +import time + +# A claim older than this without a heartbeat belongs to a machine that died, +# and another machine may take the job over. +STALE_CLAIM_MINUTES = 30 +HEARTBEAT_SECONDS = 60 + +# The columns of a per-branch result table, with the type spelled per backend. +BRANCH_COLUMNS = [ + ("date", "bigint"), ("libname", "text"), ("model", "text"), ("exectime", "real"), + ("frontend", "real"), ("backend", "real"), ("simcode", "real"), ("templates", "real"), + ("compile", "real"), ("simulate", "real"), ("verify", "real"), + ("verifyfail", "int"), ("verifytotal", "int"), ("finalphase", "int"), ("parsing", "real"), +] +SQLITE_TYPES = {"bigint": "integer", "int": "integer", "real": "real", "text": "text"} +POSTGRES_TYPES = {"bigint": "bigint", "int": "integer", "real": "double precision", "text": "text"} + +# What identifies a row, so that two machines writing the same shared table +# cannot store the same result twice. Mirrors sqlite2postgres.py. +KEYS = { + "omcversion": ["date", "branch"], + "libversion": ["date", "branch", "libname", "confighash"], +} +BRANCH_KEY = ["date", "libname", "model"] + +JOB_CLAIM = """ +CREATE TABLE IF NOT EXISTS job_claim ( + branch text NOT NULL, + libname text NOT NULL, + libversion text NOT NULL, + omcversion text NOT NULL, + confighash bigint NOT NULL, + host text NOT NULL, + state text NOT NULL, + claimed_at timestamptz NOT NULL DEFAULT now(), + heartbeat timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (branch, libname, libversion, omcversion, confighash) +) +""" + + +# Jenkins sets LIBTEST_DB once for the whole pipeline rather than passing --db +# to every script invocation. +DEFAULT_DB = os.environ.get("LIBTEST_DB") or "sqlite3.db" +DB_HELP = ("Result database: a local sqlite3 file, or a postgresql://user@host/database URL for " + "the shared one, taken from the LIBTEST_DB environment variable when not given. " + "Several machines can write to the shared database at the same time; a library " + "another machine is already testing is skipped instead of tested twice. The password " + "is read from PGPASSWORD or ~/.pgpass, never from the URL.") + + +def addArgument(parser): + """Give a script the --db option, spelled the same way everywhere.""" + parser.add_argument("--db", default=DEFAULT_DB, help=DB_HELP) + + +def _fixPgpassPermissions(): + """libpq ignores a password file others can read, and then fails to connect. + + Jenkins hands the credential to the build as a file it created itself, so + rather than have every job remember to chmod it, take a private copy. + """ + path = os.environ.get("PGPASSFILE") + if not path or not os.path.isfile(path): + return + if not (os.stat(path).st_mode & 0o077): + return + import shutil, tempfile + fd, private = tempfile.mkstemp(prefix="pgpass.") + os.close(fd) + os.chmod(private, 0o600) + shutil.copyfile(path, private) + os.environ["PGPASSFILE"] = private + + +def connect(url): + """Open the result database named by url: a path or a postgresql:// URL.""" + if url.startswith("postgres://") or url.startswith("postgresql://"): + return _Postgres(url) + return _Sqlite(url) + + +class _Db: + """What the testing and report scripts use; the backends fill in the rest.""" + + def cursor(self): + return _Cursor(self, self.conn.cursor()) + + def execute(self, sql, params=()): + return self.cursor().execute(sql, params) + + def commit(self): + self.conn.commit() + + def close(self): + self.conn.close() + + def insertIgnore(self): + """The clause that makes an INSERT skip a row that is already there.""" + return "" + + def createDateIndex(self, branch): + """The index the report queries need; test.py drops it before a run.""" + self.execute("CREATE INDEX IF NOT EXISTS %s ON %s (date)" + % (self.quote("idx_%s_date" % branch), self.quote(branch))) + + def claim(self, branch, libname, libversion, omcversion, confighash): + """True when this machine may test that library, False when another one is. + + Only the shared database can say no; a local file has a single writer. + """ + return True + + def claimedBy(self, branch, libname, libversion, omcversion, confighash): + return ("this machine", None) + + def release(self): + """Mark the claims of this run as finished.""" + + def vacuum(self): + self.conn.execute("VACUUM") + + +class _Cursor: + """A cursor that takes "?" placeholders whatever the backend wants.""" + + def __init__(self, db, cursor): + self.db = db + self.cursor = cursor + + def execute(self, sql, params=()): + # psycopg2 only looks for placeholders when parameters are passed, so a + # statement without any must not be handed an empty tuple. + params = tuple(params) + self.cursor.execute(self.db.sql(sql, bool(params)), self.db.params(params)) + return self + + def fetchone(self): + return self.cursor.fetchone() + + def fetchall(self): + return self.cursor.fetchall() + + def __iter__(self): + return iter(self.cursor) + + +class _Sqlite(_Db): + """The per-machine sqlite3 file the testing has always used.""" + + name = "sqlite3" + + def __init__(self, path): + self.conn = sqlite3.connect(path) + self.path = path + + def sql(self, sql, hasParams=False): + return sql + + def params(self, params): + return params + + def quote(self, ident): + """Quote a table or column name, typically a branch name.""" + return "[%s]" % ident + + def createTables(self, branch): + """The schema migration test.py has always done, plus the branch table.""" + cursor = self.cursor() + user_version = self.userVersion() + if user_version == 0: + # Table to lookup from a run (date, branch) to omcversion used + cursor.execute("CREATE TABLE if not exists [omcversion] (date integer NOT NULL, branch text NOT NULL, omcversion text NOT NULL)") + # Table to lookup from a run (date, branch) which library versions were used + cursor.execute("CREATE TABLE if not exists [libversion] (date integer NOT NULL, branch text NOT NULL, libname text NOT NULL, libversion text NOT NULL, confighash integer NOT NULL)") + elif user_version == 1: + cursor.execute("ALTER TABLE [libversion] ADD COLUMN confighash integer NOT NULL DEFAULT(0)") + elif user_version == 2: + for tbl in [t for t in self.tables() if t not in ["libversion", "omcversion"]]: + cursor.execute("ALTER TABLE [%s] ADD COLUMN parsing real NOT NULL DEFAULT(0.0)" % tbl) + elif user_version != 3: + raise SystemExit("Unknown schema user_version=%d" % user_version) + + cols = ", ".join("%s %s NOT NULL" % (c, SQLITE_TYPES[t]) for c, t in BRANCH_COLUMNS) + cursor.execute("CREATE TABLE if not exists %s (%s)" % (self.quote(branch), cols)) + # The indexes only slow the run's inserts down; the report scripts add them back. + cursor.execute("DROP INDEX IF EXISTS [idx_%s_date]" % branch) + cursor.execute("DROP INDEX IF EXISTS idx_omcversion_date") + cursor.execute("DROP INDEX IF EXISTS idx_libversion_date") + self.setUserVersion(3) + + def tables(self): + return [t for (t,) in self.conn.execute("SELECT name FROM sqlite_master WHERE type='table'")] + + def tableExists(self, name): + return self.conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)).fetchone() is not None + + def groupConcat(self, expr, orderBy=None): + # sqlite keeps the order of the subquery that feeds the aggregate. + return "GROUP_CONCAT(%s)" % expr + + def countIf(self, cond): + return "COUNT(%s or null)" % cond + + def likeNoCase(self, column): + """A case insensitive comparison against a "?" placeholder.""" + return "%s LIKE ? COLLATE NOCASE" % column + + def userVersion(self): + return self.conn.execute("PRAGMA user_version").fetchone()[0] + + def setUserVersion(self, v): + self.conn.execute("PRAGMA user_version=%d" % v) + + +class _Postgres(_Db): + """The shared database several test machines write to at the same time.""" + + name = "postgresql" + + def __init__(self, url): + try: + import psycopg2 + except ImportError: + raise SystemExit("PostgreSQL support needs psycopg2: pip install psycopg2-binary") + # The password belongs in PGPASSWORD or ~/.pgpass, not in the URL. + _fixPgpassPermissions() + self.url = url + self.conn = psycopg2.connect(url) + self.conn.autocommit = False + self.host = socket.gethostname() + self.claims = [] + self.heartbeatThread = None + self.execute(JOB_CLAIM) + self.commit() + + def sql(self, sql, hasParams=False): + """sqlite spells the placeholder "?" and psycopg2 spells it "%s". + + psycopg2 also reads "%" itself, so a literal one has to be doubled - but + only in a statement that has parameters at all. + """ + if not hasParams: + return sql + return sql.replace("%", "%%").replace("?", "%s") + + def params(self, params): + # psycopg2 only looks for placeholders when parameters are passed, so a + # statement without any must not be handed an empty tuple. + return params or None + + def quote(self, ident): + return '"%s"' % ident.replace('"', '""') + + def createTables(self, branch): + """Create the shared tables, with the keys that keep the machines apart. + + Unlike sqlite there is no schema migration and the indexes stay: other + machines are reading the table while this one writes its few thousand rows. + """ + cursor = self.cursor() + cursor.execute("""CREATE TABLE IF NOT EXISTS omcversion ( + date bigint NOT NULL, branch text NOT NULL, omcversion text)""") + cursor.execute("""CREATE TABLE IF NOT EXISTS libversion ( + date bigint NOT NULL, branch text NOT NULL, libname text NOT NULL, + libversion text, confighash bigint NOT NULL)""") + cols = ", ".join("%s %s%s" % (c, POSTGRES_TYPES[t], " NOT NULL" if c in BRANCH_KEY else "") + for c, t in BRANCH_COLUMNS) + cursor.execute("CREATE TABLE IF NOT EXISTS %s (%s)" % (self.quote(branch), cols)) + for tbl in ["omcversion", "libversion", branch]: + key = KEYS.get(tbl, BRANCH_KEY) + cursor.execute("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)" + % (self.quote(("uq_%s_%s" % (tbl, "_".join(key)))[:63]), + self.quote(tbl), ",".join(key))) + cursor.execute("CREATE INDEX IF NOT EXISTS %s ON %s (libname, date)" + % (self.quote(("idx_%s_libname_date" % branch)[:63]), self.quote(branch))) + self.commit() + + def tables(self): + return [t for (t,) in self.execute( + "SELECT tablename FROM pg_tables WHERE schemaname=current_schema()")] + + def tableExists(self, name): + return self.execute("SELECT 1 FROM pg_tables WHERE schemaname=current_schema() AND tablename=?", + (name,)).fetchone() is not None + + def groupConcat(self, expr, orderBy=None): + if orderBy: + return "string_agg(%s::text, ',' ORDER BY %s)" % (expr, orderBy) + return "string_agg(%s::text, ',')" % expr + + def countIf(self, cond): + return "COUNT(*) FILTER (WHERE %s)" % cond + + def likeNoCase(self, column): + return "%s ILIKE ?" % column + + def insertIgnore(self): + # Another machine may have written the same run already; its row wins. + return " ON CONFLICT DO NOTHING" + + def createDateIndex(self, branch): + """Nothing to do: date is the first column of the table's unique key.""" + + def userVersion(self): + return 3 + + def setUserVersion(self, v): + pass + + def vacuum(self): + """Nothing to do: PostgreSQL has autovacuum, and a manual VACUUM of a + 50 GB database after every test run would be a waste of the machine.""" + + def claim(self, branch, libname, libversion, omcversion, confighash): + """Take the job unless another machine is running it right now. + + The claim is one row per (branch, library version, omc version, config). + A machine that dies stops sending its heartbeat, and after + STALE_CLAIM_MINUTES its jobs are up for grabs again. + """ + key = (branch, libname, libversion, omcversion, confighash) + got = self.execute("""INSERT INTO job_claim + (branch, libname, libversion, omcversion, confighash, host, state) + VALUES (?,?,?,?,?,?,'running') + ON CONFLICT (branch, libname, libversion, omcversion, confighash) DO UPDATE + SET host = EXCLUDED.host, state = 'running', claimed_at = now(), heartbeat = now() + WHERE job_claim.state <> 'running' + OR job_claim.heartbeat < now() - interval '%d minutes' + RETURNING host""" % STALE_CLAIM_MINUTES, + key + (self.host,)).fetchone() + self.commit() + if got is None: + return False + self.claims.append(key) + self._startHeartbeat() + return True + + def claimedBy(self, branch, libname, libversion, omcversion, confighash): + """The machine holding that job, for the message telling the user why we skip.""" + row = self.execute("""SELECT host, claimed_at FROM job_claim WHERE branch=? AND libname=? + AND libversion=? AND omcversion=? AND confighash=?""", + (branch, libname, libversion, omcversion, confighash)).fetchone() + return row or ("unknown", None) + + def _startHeartbeat(self): + if self.heartbeatThread is not None: + return + # A daemon thread: the run must not wait for it on the way out. + def beat(): + while self.claims: + time.sleep(HEARTBEAT_SECONDS) + try: + self._heartbeat() + except Exception as e: + print("Failed to update the job claims: %s" % e) + self.heartbeatThread = threading.Thread(target=beat, daemon=True) + self.heartbeatThread.start() + + def _heartbeat(self): + # A separate connection: the main one is in the middle of the run's work. + import psycopg2 + with psycopg2.connect(self.url) as conn: + with conn.cursor() as cur: + for (branch, libname, libversion, omcversion, confighash) in list(self.claims): + cur.execute("""UPDATE job_claim SET heartbeat = now() WHERE branch=%s AND libname=%s + AND libversion=%s AND omcversion=%s AND confighash=%s AND host=%s""", + (branch, libname, libversion, omcversion, confighash, self.host)) + + def release(self): + for (b, libname, libversion, omcversion, confighash) in self.claims: + self.execute("""UPDATE job_claim SET state='done', heartbeat=now() + WHERE branch=? AND libname=? AND libversion=? AND omcversion=? AND confighash=? + AND host=?""", + (b, libname, libversion, omcversion, confighash, self.host)) + self.claims = [] + self.commit() diff --git a/single-model.py b/single-model.py index d72e769..3d9af2b 100755 --- a/single-model.py +++ b/single-model.py @@ -3,11 +3,12 @@ import sys, argparse import simplejson as json -import shared +import shared, resultsdb parser = argparse.ArgumentParser(description='OpenModelica model testing report generation tool') parser.add_argument('models', nargs='*') parser.add_argument('--branch', default='') +resultsdb.addArgument(parser) args = parser.parse_args() @@ -21,28 +22,28 @@ libs = {} -import cgi, sqlite3, time, datetime +import cgi, time, datetime from omcommon import friendlyStr, multiple_replace -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() +db = resultsdb.connect(args.db) +cursor = db.cursor() try: - cursor.execute("SELECT name FROM [sqlite_master] WHERE type='table' AND name=?", (branch,)) - v = cursor.fetchone()[0] + if not db.tableExists(branch): + raise Exception("no table") except: raise Exception("No such table '%s'; specify it using --branch=XXX" % branch) for model in models: lines=[] c=0 - libnames = [libname for (libname,) in cursor.execute("SELECT DISTINCT libname FROM [%s] WHERE model=? ORDER BY libname ASC" % (branch), (model,))] + libnames = [libname for (libname,) in cursor.execute("SELECT DISTINCT libname FROM %s WHERE model=? ORDER BY libname ASC" % (db.quote(branch)), (model,))] for libname in libnames: - for (finalphase,dint,libversion) in cursor.execute("SELECT finalphase,date,libversion FROM [%s] NATURAL JOIN [libversion] WHERE model=? AND libname=? ORDER BY date ASC" % (branch), (model,libname)): + for (finalphase,dint,libversion) in cursor.execute("SELECT finalphase,date,libversion FROM %s NATURAL JOIN libversion WHERE model=? AND libname=? ORDER BY date ASC" % (db.quote(branch)), (model,libname)): c+=1 dstr = str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S')) - cursor2 = conn.cursor() - omcversion = cursor2.execute("SELECT omcversion FROM [omcversion] WHERE date=? AND branch=?", (dint,branch)).fetchone()[0] + cursor2 = db.cursor() + omcversion = cursor2.execute("SELECT omcversion FROM omcversion WHERE date=? AND branch=?", (dint,branch)).fetchone()[0] omcversion = omcversion.replace("OpenModelica ","").replace("OMCompiler ","") lines.insert(0, "%s %s %s %s" % (dstr,shared.finalphaseName(finalphase),omcversion,libversion.strip())) if c==0: diff --git a/sqlite2postgres.py b/sqlite2postgres.py new file mode 100644 index 0000000..850d445 --- /dev/null +++ b/sqlite2postgres.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +""" +Migrate OpenModelicaLibraryTesting sqlite3 result databases into PostgreSQL. + +The PostgreSQL layout is a 1:1 mirror of the sqlite3 one: one table per branch +with the same columns, plus the [omcversion] and [libversion] lookup tables. +Only the types are adapted (integer -> bigint, real -> double precision). + +Only the python standard library is used; the psql client binary does the +talking, so nothing has to be installed on the test machines. The password is +never passed on the command line - use PGPASSFILE or PGPASSWORD, as understood +by psql. + +Typical use: + + export PGPASSFILE=~/.pgpass + ./sqlite2postgres.py --sqlite dbs/ripper1.db --source ripper1 + ./sqlite2postgres.py --index + ./sqlite2postgres.py --sqlite dbs/ripper2.db --source ripper2 --skip-existing + ./sqlite2postgres.py --index + +The machines share the tables of the branches they both test, so they are merged +in that order: ripper1 first, then ripper2 with --skip-existing, which keeps the +row already in the database whenever a key collides. --index in between creates +the unique keys that decide what a collision is - (date, libname, model) for a +branch table - so it has to run before the second machine. + +--source only names the machine for the resume bookkeeping; it is not stored in +the data. Use --pgschema ripper1 to keep a machine in a schema of its own +instead of merging. + +The migration is resumable and safe to re-run: how far each table has been +copied is recorded in [migration_progress], and a restart continues from the +last sqlite rowid that made it in. +""" + +import argparse +import io +import os +import sqlite3 +import subprocess +import sys +import time + +# Columns of a per-branch table, in the order used by test.py. Databases with +# PRAGMA user_version < 3 have no "parsing" column. +# +# The sqlite3 tables declare every column NOT NULL, but tables created before +# that declaration still hold NULLs (libversion.libversion has some), so only +# the key columns are NOT NULL here. +BRANCH_COLUMNS = [ + ("date", "bigint"), + ("libname", "text"), + ("model", "text"), + ("exectime", "double precision"), + ("frontend", "double precision"), + ("backend", "double precision"), + ("simcode", "double precision"), + ("templates", "double precision"), + ("compile", "double precision"), + ("simulate", "double precision"), + ("verify", "double precision"), + ("verifyfail", "integer"), + ("verifytotal", "integer"), + ("finalphase", "integer"), + ("parsing", "double precision"), +] + +LOOKUP_COLUMNS = { + "omcversion": [("date", "bigint"), ("branch", "text"), ("omcversion", "text")], + "libversion": [("date", "bigint"), ("branch", "text"), ("libname", "text"), + ("libversion", "text"), ("confighash", "bigint")], +} + +# Derived data, no longer generated by all-plots.py; not worth migrating. +SKIP_PREFIXES = ("datelookup_", "sqlite_") + + +# What identifies a row. A test run writes one row per model, so a model +# appears once per (date, libname); the lookup tables have one row per run and +# per (run, library). --index enforces these, which is what lets a second test +# machine be merged into a table that already holds another machine's results. +KEYS = { + "omcversion": ["date", "branch"], + "libversion": ["date", "branch", "libname", "confighash"], +} +BRANCH_KEY = ["date", "libname", "model"] + +PROGRESS_TABLE = """ +CREATE TABLE IF NOT EXISTS "migration_progress" ( + source text NOT NULL, + tbl text NOT NULL, + last_rowid bigint NOT NULL, + rows_read bigint NOT NULL, + done boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (source, tbl) +) +""" + + +def lit(s): + """Quote a python value as an SQL string literal.""" + return "'" + str(s).replace("'", "''") + "'" + + +def ident(s): + """Quote an SQL identifier. Branch names contain '-' and upper case.""" + return '"' + s.replace('"', '""') + '"' + + +def copy_text(v): + """Encode one value for COPY's text format. + + Not CSV: an empty CSV field reads back as NULL, and libversion.libversion + does contain empty strings that have to stay empty strings. The text format + spells NULL \\N instead, so both survive. + """ + if v is None: + return "\\N" + return (str(v).replace("\\", "\\\\").replace("\n", "\\n") + .replace("\r", "\\r").replace("\t", "\\t")) + + +class Psql: + """Runs SQL through the psql client, which keeps this script dependency-free.""" + + def __init__(self, args): + self.base = ["psql", "--no-psqlrc", "-w", "-v", "ON_ERROR_STOP=1", + "-h", args.host, "-p", str(args.port), "-U", args.user, "-d", args.dbname] + self.schema = args.pgschema + + def _run(self, argv, stdin=None): + p = subprocess.run(self.base + argv, input=stdin, capture_output=True, text=True) + if p.returncode != 0: + raise RuntimeError("psql failed: %s" % (p.stderr.strip() or p.stdout.strip())) + return p.stdout + + def query(self, sql): + """Execute SQL and return the output unaligned and without headers.""" + return self._run(["-tAq", "-c", "SET search_path TO %s; %s" % (ident(self.schema), sql)]).strip() + + def script(self, sql): + return self._run(["-q", "-f", "-"], stdin="SET search_path TO %s;\n%s" % (ident(self.schema), sql)) + + def copy_rows(self, table, columns, rows, also="", skip_existing=None): + """COPY an iterable of tuples into a table. + + The statements in "also" are committed together with the batch, so that a + migration killed halfway through never copies the same rows twice. + + With skip_existing set to the key columns of the table, the batch goes + through a temporary table first and rows that are already there are + dropped, so the machine migrated first keeps its results. + """ + cols = ",".join(ident(c) for c in columns) + target = ident(table) + buf = io.StringIO() + buf.write("SET search_path TO %s;\nBEGIN;\n" % ident(self.schema)) + if skip_existing: + buf.write("CREATE TEMP TABLE batch (LIKE %s) ON COMMIT DROP;\n" % target) + target = "batch" + buf.write("COPY %s (%s) FROM STDIN;\n" % (target, cols)) + for r in rows: + buf.write("\t".join(copy_text(v) for v in r)) + buf.write("\n") + buf.write("\\.\n") + if skip_existing: + key = ",".join(ident(c) for c in skip_existing) + # DISTINCT ON also drops rows the batch itself holds twice, which the + # unique index would otherwise reject. + buf.write("INSERT INTO %s (%s) SELECT DISTINCT ON (%s) %s FROM batch ORDER BY %s" + " ON CONFLICT DO NOTHING;\n" % (ident(table), cols, key, cols, key)) + if also: + buf.write(also.rstrip().rstrip(";") + ";\n") + buf.write("COMMIT;\n") + if not skip_existing: + self._run(["-q", "-f", "-"], stdin=buf.getvalue()) + return None + # Without -q psql reports every statement, so the INSERT says how many rows + # were new - the interesting number when catching up with a running test. + out = self._run(["-f", "-"], stdin=buf.getvalue()) + for line in out.split("\n"): + if line.startswith("INSERT "): + return int(line.split()[-1]) + return 0 + + +def sqlite_tables(conn): + """The tables of a testing database, split into (branch tables, lookup tables).""" + names = [n for (n,) in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")] + branches, lookups = [], [] + for n in names: + if any(n.startswith(p) for p in SKIP_PREFIXES): + continue + (lookups if n in LOOKUP_COLUMNS else branches).append(n) + return branches, lookups + + +def sqlite_columns(conn, tbl): + return [row[1] for row in conn.execute("PRAGMA table_info(%s)" % ident(tbl))] + + +def create_table(pg, tbl, columns): + key = KEYS.get(tbl, BRANCH_KEY) + cols = ",\n ".join("%s %s%s" % (ident(c), t, " NOT NULL" if c in key else "") + for c, t in columns) + pg.script("CREATE TABLE IF NOT EXISTS %s (\n %s\n);" % (ident(tbl), cols)) + + +def read_progress(pg, source, tbl): + out = pg.query("SELECT last_rowid, rows_read, done FROM migration_progress " + "WHERE source=%s AND tbl=%s" % (lit(source), lit(tbl))) + if not out: + return 0, 0, False + last, rows, done = out.split("|") + return int(last), int(rows), done == "t" + + +def progress_sql(source, tbl, last_rowid, rows_read, done): + return ("""INSERT INTO migration_progress (source, tbl, last_rowid, rows_read, done, updated_at) + VALUES (%s, %s, %d, %d, %s, now()) + ON CONFLICT (source, tbl) DO UPDATE SET + last_rowid = EXCLUDED.last_rowid, rows_read = EXCLUDED.rows_read, + done = EXCLUDED.done, updated_at = now()""" + % (lit(source), lit(tbl), last_rowid, rows_read, "true" if done else "false")) + + +def migrate_table(pg, sconn, source, tbl, columns, batch, quiet, skip_existing=False, + catch_up=False): + """Copy one sqlite table into its PostgreSQL twin, in resumable batches.""" + names = [c for c, _ in columns] + have = sqlite_columns(sconn, tbl) + missing = [c for c in names if c not in have] + # Older databases lack "parsing"; test.py defaults it to 0.0 as well. + select = ",".join(ident(c) if c in have else "0" for c in names) + + create_table(pg, tbl, columns) + last_rowid, rows_read, done = read_progress(pg, source, tbl) + if done and not catch_up: + if not quiet: + print(" %-30s done earlier (%d rows)" % (tbl, rows_read)) + return rows_read, 0 + + stored = None + if catch_up: + # Read the table from the start and keep the runs the database has never + # seen. Not "everything past the rowid we stopped at": VACUUM renumbers + # the rowids of these tables, and clean-empty-omcversion-dates.py runs one + # after every test, so that number cannot be trusted between two runs. + stored = set(int(d) for d in pg.query("SELECT DISTINCT date FROM %s" % ident(tbl)).split("\n") if d) + last_rowid, rows_read = 0, 0 + skip_existing = True + key = KEYS.get(tbl, BRANCH_KEY) if skip_existing else None + if not quiet: + extra = "" + if missing: + extra = ", %s defaulted to 0" % ",".join(missing) + if catch_up: + extra += ", %d runs already stored" % len(stored) + elif last_rowid: + extra += ", resuming after rowid %d" % last_rowid + print(" %-30s starting%s" % (tbl, extra)) + + t0 = time.time() + inserted = 0 + date = names.index("date") + while True: + chunk = sconn.execute( + "SELECT rowid,%s FROM %s WHERE rowid > ? ORDER BY rowid LIMIT %d" + % (select, ident(tbl), batch), (last_rowid,)).fetchall() + if not chunk: + break + last_rowid = chunk[-1][0] + rows_read += len(chunk) + rows = [row[1:] for row in chunk] + if stored is not None: + rows = [r for r in rows if r[date] not in stored] + if rows: + n = pg.copy_rows(tbl, names, rows, + also=progress_sql(source, tbl, last_rowid, rows_read, False), + skip_existing=key) + inserted += n if n is not None else len(rows) + if not quiet: + sys.stdout.write("\r %-30s %d rows read, %d new (%.0f rows/s)" + % (tbl, rows_read, inserted, rows_read / max(time.time() - t0, 1e-9))) + sys.stdout.flush() + + pg.query(progress_sql(source, tbl, last_rowid, rows_read, True)) + if not quiet and (inserted or not catch_up): + print("\r %-30s %d rows read, %d new%s" % (tbl, rows_read, inserted, " " * 30)) + return rows_read, inserted + + +def migrate_database(pg, args): + sconn = sqlite3.connect("file:%s?mode=ro" % os.path.abspath(args.sqlite), uri=True) + branches, lookups = sqlite_tables(sconn) + if args.only: + branches = [b for b in branches if b in args.only] + lookups = [l for l in lookups if l in args.only] + print("%s: %d branch tables, lookup tables: %s" + % (args.sqlite, len(branches), ",".join(lookups) or "none")) + + total = new = 0 + for tbl in lookups + branches: + cols = LOOKUP_COLUMNS.get(tbl, BRANCH_COLUMNS) + (r, i) = migrate_table(pg, sconn, args.source, tbl, cols, args.batch, args.quiet, + args.skip_existing, args.catch_up) + total += r + new += i + sconn.close() + print("%s: %d rows read, %d new" % (args.sqlite, total, new)) + + +def create_indexes(pg): + """The unique key of each table, plus the index the reports need. + + The unique index has to exist before a second test machine is migrated with + --skip-existing. It also covers the [date] index the report scripts create + on the fly in sqlite, since date is its first column. + """ + tables = [t for t in pg.query( + "SELECT tablename FROM pg_tables WHERE schemaname=current_schema() ORDER BY tablename").split("\n") if t] + + def index(tbl, cols, unique=False): + name = ("%s_%s_%s" % ("uq" if unique else "idx", tbl, "_".join(cols)))[:63] + pg.script("CREATE %sINDEX IF NOT EXISTS %s ON %s (%s);" + % ("UNIQUE " if unique else "", ident(name), ident(tbl), + ",".join(ident(c) for c in cols))) + + for tbl in tables: + if tbl == "migration_progress": + continue + index(tbl, KEYS.get(tbl, BRANCH_KEY), unique=True) + if tbl == "omcversion": + index(tbl, ["branch", "date"]) + elif tbl == "libversion": + index(tbl, ["branch", "libname", "date"]) + else: + index(tbl, ["libname", "date"]) + print(" indexed %s" % tbl) + + +def verify(pg, args): + """Compare the sqlite and PostgreSQL row counts table by table.""" + sconn = sqlite3.connect("file:%s?mode=ro" % os.path.abspath(args.sqlite), uri=True) + branches, lookups = sqlite_tables(sconn) + bad = 0 + for tbl in lookups + branches: + (n,) = sconn.execute("SELECT COUNT(*) FROM %s" % ident(tbl)).fetchone() + m = int(pg.query("SELECT COUNT(*) FROM %s" % ident(tbl)) or 0) + flag = "ok" if m >= n else "MISSING" + if m < n: + bad += 1 + print(" %-30s sqlite %10d postgres %10d %s" % (tbl, n, m, flag)) + sconn.close() + print("%d of %d tables incomplete" % (bad, len(lookups) + len(branches))) + return bad + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--sqlite", help="sqlite3 database to migrate") + parser.add_argument("--source", help="machine the database comes from, e.g. ripper1; used for resuming") + parser.add_argument("--host", default=os.environ.get("PGHOST", "openmodelica.org")) + parser.add_argument("--port", type=int, default=int(os.environ.get("PGPORT", 5432))) + parser.add_argument("--user", default=os.environ.get("PGUSER", "om")) + parser.add_argument("--dbname", default=os.environ.get("PGDATABASE", "omdb")) + parser.add_argument("--pgschema", default="public", help="PostgreSQL schema to write to (default public)") + parser.add_argument("--batch", type=int, default=200000, help="rows per COPY batch (default 200000)") + parser.add_argument("--only", action="append", help="migrate only this table (repeatable)") + parser.add_argument("--catch-up", action="store_true", + help="copy the runs written since the last migration, for a test that was " + "still using the sqlite3 database. Reads the whole database but only " + "writes the runs it does not have; safe to repeat") + parser.add_argument("--skip-existing", action="store_true", + help="keep the rows already in the database when a key collides; " + "use it for every machine after the first one, and run --index before") + parser.add_argument("--index", action="store_true", help="create the indexes; do this after loading") + parser.add_argument("--verify", action="store_true", help="compare row counts with --sqlite") + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args() + + pg = Psql(args) + pg.script("CREATE SCHEMA IF NOT EXISTS %s;" % ident(args.pgschema)) + pg.script(PROGRESS_TABLE) + + if args.sqlite and not args.verify: + if not args.source: + parser.error("--sqlite needs --source, the machine name, e.g. ripper1") + migrate_database(pg, args) + + if args.index: + print("creating indexes") + create_indexes(pg) + + if args.verify: + if not args.sqlite: + parser.error("--verify needs --sqlite") + sys.exit(1 if verify(pg, args) else 0) + + +if __name__ == "__main__": + main() diff --git a/test.py b/test.py index affc5e3..b6ca625 100755 --- a/test.py +++ b/test.py @@ -8,7 +8,7 @@ if (sys.version_info < (3, 0)): raise Exception("Python2 is no longer supported") -import html, shutil, os, re, glob, time, argparse, sqlite3, datetime, math, platform +import html, shutil, os, re, glob, time, argparse, datetime, math, platform from joblib import Parallel, delayed import simplejson as json import psutil, subprocess, threading, hashlib @@ -18,7 +18,7 @@ from natsort import natsorted from shared import readConfig, getReferenceFileName, simulationAcceptsFlag, isFMPy from platform import processor -import shared +import shared, resultsdb import signal @@ -47,6 +47,7 @@ parser.add_argument('--msysEnvironment', help = 'MSYS2 environment used by OpenModelica on Windows.', default = 'ucrt64') parser.add_argument('--debug', action="store_true", help="turn on the DEBUG mode", default=False) parser.add_argument('--addmsl', action="store_true", help="add the MSL path to the OPENMODELICAPATH if the MSL is not detected in the libraries path", default=False) +resultsdb.addArgument(parser) args = parser.parse_args() configs = args.configs @@ -557,43 +558,10 @@ def testHelloWorld(cmd): # Create mos-files -conn = sqlite3.connect('sqlite3.db') -cursor = conn.cursor() - -user_version = cursor.execute("PRAGMA user_version").fetchone()[0] - -if user_version==0: - # BOOLEAN NOT NULL CHECK (verify IN (0,1) AND builds IN (0,1) AND simulates IN (0,1)) - # Table to lookup from a run (date, branch) to omcversion used - cursor.execute("CREATE TABLE if not exists [omcversion] (date integer NOT NULL, branch text NOT NULL, omcversion text NOT NULL)") - # Table to lookup from a run (date, branch) which library versions were used - cursor.execute("CREATE TABLE if not exists [libversion] (date integer NOT NULL, branch text NOT NULL, libname text NOT NULL, libversion text NOT NULL, confighash integer NOT NULL)") -elif user_version==1: - cursor.execute("ALTER TABLE [libversion] ADD COLUMN confighash integer NOT NULL DEFAULT(0)") -elif user_version==2: - tables = [tbl for (tbl,) in cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") if tbl not in ["libversion","omcversion"]] - for tbl in tables: - cursor.execute("ALTER TABLE [%s] ADD COLUMN parsing real NOT NULL DEFAULT(0.0)" % tbl) -elif user_version in [3]: - pass -else: - print("Unknown schema user_version=%d" % user_version) - sys.exit(1) - -def createBranchTable(branch): - """A run fills one table per FMI simulator, so this happens more than once.""" - cursor.execute('''CREATE TABLE if not exists [%s] - (date integer NOT NULL, libname text NOT NULL, model text NOT NULL, exectime real NOT NULL, - frontend real NOT NULL, backend real NOT NULL, simcode real NOT NULL, templates real NOT NULL, compile real NOT NULL, simulate real NOT NULL, - verify real NOT NULL, verifyfail integer NOT NULL, verifytotal integer NOT NULL, finalphase integer NOT NULL, parsing real NOT NULL)''' % branch) - cursor.execute('''DROP INDEX IF EXISTS [idx_%s_date]''' % branch) - -createBranchTable(primaryBranch) -cursor.execute('''DROP INDEX IF EXISTS idx_omcversion_date''') -cursor.execute('''DROP INDEX IF EXISTS idx_libversion_date''') - -# Set user_version to the current schema -cursor.execute("PRAGMA user_version=3") +db = resultsdb.connect(args.db) +cursor = db.cursor() +# One table per simulator, so this happens once per branch the run fills. +db.createTables(primaryBranch) def strToHashInt(s): return int(hashlib.sha1((s+"fixCorruptBuilds-2017-03-26").encode("utf-8")).hexdigest()[0:8],16) @@ -778,11 +746,18 @@ def hashReferenceFiles(s): prefix = conf["ignoreModelPrefix"] res=list(filter(lambda x: not x.startswith(prefix), res)) libName=shared.libname(library, conf) - v = cursor.execute("""SELECT date,libversion,libname,branch,omcversion FROM [libversion] NATURAL JOIN [omcversion] + v = cursor.execute("""SELECT date,libversion,libname,branch,omcversion FROM libversion NATURAL JOIN omcversion WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1""", (conf["libraryLastChange"],libName,primaryBranch,omc_version,confighash)).fetchone() if libName in stats_by_libname or libName in skipped_libs: raise Exception("Duplicate libName found: %s" % libName) if v is None or execAllTests: + # On the shared database another machine may already be running this exact + # job; claiming it is what keeps the two from testing the same thing. + if not db.claim(primaryBranch, libName, conf["libraryLastChange"], omc_version, confighash): + (host, since) = db.claimedBy(primaryBranch, libName, conf["libraryLastChange"], omc_version, confighash) + print("Skipping %s as %s has been testing it since %s" % (libName, host, since)) + skipped_libs[libName] = None + continue stats_by_libname[libName] = {"conf":conf, "stats":[]} tests = tests + [(r,library,libName,libName+"_"+r,conf) for r in res] print("Running library %s (%d tests)" % (libName, len(res))) @@ -899,7 +874,7 @@ def expectedExec(c): (model,lib,libName,name,data) = c if "expectedExec" in data: return data["expectedExec"] - cursor.execute("SELECT exectime FROM [%s] WHERE libname = ? AND model = ? ORDER BY date DESC LIMIT 1" % primaryBranch, (libName,model)) + cursor.execute("SELECT exectime FROM %s WHERE libname = ? AND model = ? ORDER BY date DESC LIMIT 1" % db.quote(primaryBranch), (libName,model)) v = cursor.fetchone() data["expectedExec"] = (v or (0.0,))[0] return data["expectedExec"] @@ -1003,17 +978,17 @@ def resultValues(model, libname, data, simulator=None): ) for (resultBranch, simulator) in resultBranches: - createBranchTable(resultBranch) + db.createTables(resultBranch) for key in stats.keys(): (name,model,libname,data)=stats[key] if simulator is None: stats_by_libname[libname]["stats"].append(stats[key]) - cursor.execute("INSERT INTO [%s] VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" % resultBranch, + cursor.execute("INSERT INTO %s VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)%s" % (db.quote(resultBranch), db.insertIgnore()), resultValues(model, libname, data, simulator)) for libname in stats_by_libname.keys(): confighash = stats_by_libname[libname]["conf"]["confighash"] - cursor.execute("INSERT INTO [libversion] VALUES (?,?,?,?,?)", (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash)) - cursor.execute("INSERT INTO [omcversion] VALUES (?,?,?)", (testRunStartTimeAsEpoch, resultBranch, omc_version)) + cursor.execute("INSERT INTO libversion VALUES (?,?,?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash)) + cursor.execute("INSERT INTO omcversion VALUES (?,?,?)%s" % db.insertIgnore(), (testRunStartTimeAsEpoch, resultBranch, omc_version)) """ # Not really a good thing to do; was just done to make generation of the report simpler for libname in skipped_libs.keys(): @@ -1364,7 +1339,8 @@ def artifactSuffix(simulator): print("-- problem during removing of ./files dir") # Do not commit until we have generated and uploaded the reports -conn.commit() -conn.close() +db.commit() +db.release() +db.close() print("all tests done ...")
%s%s%s%s%s%s%s%s%s%s
%s%s%s%s%d