sqlite3 to postgresql - migration

.CI/Jenkinsfile

@@ -3,6 +3,8 @@ pipeline {
   parameters {
     booleanParam(name: 'OLDLIBS', defaultValue: false, description: 'Also test some outdated libraries')
 
+    booleanParam(name: 'postgres', defaultValue: true, description: 'Store the results in the shared PostgreSQL database (omdb on openmodelica.org) rather than in the per-machine sqlite3 file. Machines coordinate through it, so two of them no longer overwrite each other. Untick to go back to the sqlite3 files.')
+
     booleanParam(name: 'v1_26', defaultValue: false, description: 'maintenance/v1.26 branch (ryzen-5950x-1)')
     booleanParam(name: 'v1_27', defaultValue: false, description: 'maintenance/v1.27 branch (ryzen-5950x-1)')
     booleanParam(name: 'master', defaultValue: false, description: 'master branch (ryzen-5950x-1)')
@@ -40,6 +42,12 @@ pipeline {
   }
   environment {
     LC_ALL = 'C.UTF-8'
+    // Where the results go. The scripts take it from here instead of a --db
+    // option on every single invocation.
+    LIBTEST_DB = "${params.postgres ? 'postgresql://om@openmodelica.org/omdb' : 'sqlite3.db'}"
+    // A secret file holding one pgpass line; libpq reads the password from it,
+    // so it never reaches a command line or the build log.
+    PGPASSFILE = credentials('omdb-pgpass')
   }
   stages {
     stage('test') { parallel {
@@ -455,7 +463,12 @@ pipeline {
           cd OpenModelica
           git fetch
           '''
-          sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper1/sqlite3.db'
+          // The reports read the shared database directly when it is in use.
+          script {
+            if (!params.postgres) {
+              sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper1/sqlite3.db'
+            }
+          }
           sh './clean-empty-omcversion-dates.py'
 
           sh "./all-reports.py --email --omcgitdir=OpenModelica ${env.GITBRANCHES} conversion heavy_tests"
@@ -521,7 +534,11 @@ pipeline {
           cd OpenModelica
           git fetch
           '''
-          sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper2/sqlite3.db'
+          script {
+            if (!params.postgres) {
+              sh 'wget -q https://libraries.openmodelica.org/sqlite3/ripper2/sqlite3.db'
+            }
+          }
           sh './clean-empty-omcversion-dates.py'
 
           sh "./all-reports.py --email --omcgitdir=OpenModelica ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_NEWBACKEND_DAE} ${env.GITBRANCHES_CPP} gbode cvode ida"
@@ -978,14 +995,17 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om
 
   sh "test -d '${libraryPath}/.openmodelica/libraries/Modelica trunk'"
 
-  sh """
-  if ! test -f ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db; then
-    wget -O ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp -q https://libraries.openmodelica.org/sqlite3/${dbPrefix}/sqlite3.db
-	mv ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db
-  fi
-  cp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db OpenModelicaLibraryTesting/sqlite3.db
-  test -s OpenModelicaLibraryTesting/sqlite3.db
-  """
+  // The shared database needs none of this: the results go straight into it.
+  if (!params.postgres) {
+    sh """
+    if ! test -f ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db; then
+      wget -O ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp -q https://libraries.openmodelica.org/sqlite3/${dbPrefix}/sqlite3.db
+	  mv ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db.tmp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db
+    fi
+    cp ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db OpenModelicaLibraryTesting/sqlite3.db
+    test -s OpenModelicaLibraryTesting/sqlite3.db
+    """
+  }
 
   sh 'date'
 
@@ -1008,11 +1028,16 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om
   stdbuf -oL -eL time ./test.py --ompython_omhome=/usr ${FMI_TESTING_FLAG} --extraflags='${extraFlags}' --extrasimflags='${extrasimflags}' ${testFlags} --branch="${name}" --output="libraries.openmodelica.org:/var/www/libraries.openmodelica.org/branches/${name}/" --libraries='${libraryPath}/.openmodelica/libraries/'  --jobs=${jobs} ${libs_config_file} ${params.OLDLIBS ? "configs/conf-old.json configs/conf-nonstandard.json" : ""} || (killall omc ; false) || exit 1
   """)
   sh 'date'
-  sh "rm -f OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp"
-  sh "ln OpenModelicaLibraryTesting/sqlite3.db OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp"
   sh "cd OpenModelicaLibraryTesting/ && ./clean-empty-omcversion-dates.py"
-  sh "cp OpenModelicaLibraryTesting/sqlite3.db ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db"
-  sh "rm -f ~/TEST_LIBS_BACKUP/${dbPrefix}-`date +sqlite3.%Y%m%d.db`"
 
-  sshPublisher(publishers: [sshPublisherDesc(configName: sshConfig, transfers: [sshTransfer(removePrefix: 'OpenModelicaLibraryTesting', sourceFiles: 'OpenModelicaLibraryTesting/sqlite3.db')])], failOnError: true)
+  // Copying the file back is what made two machines overwrite each other's
+  // results, so it only happens while a job still writes its own sqlite3 file.
+  if (!params.postgres) {
+    sh "rm -f OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp"
+    sh "ln OpenModelicaLibraryTesting/sqlite3.db OpenModelicaLibraryTesting/${dbPrefix}-sqlite3.db.tmp"
+    sh "cp OpenModelicaLibraryTesting/sqlite3.db ~/TEST_LIBS_BACKUP/${dbPrefix}-sqlite3.db"
+    sh "rm -f ~/TEST_LIBS_BACKUP/${dbPrefix}-`date +sqlite3.%Y%m%d.db`"
+
+    sshPublisher(publishers: [sshPublisherDesc(configName: sshConfig, transfers: [sshTransfer(removePrefix: 'OpenModelicaLibraryTesting', sourceFiles: 'OpenModelicaLibraryTesting/sqlite3.db')])], failOnError: true)
+  }
 }

.CI/build-dep/Dockerfile

@@ -1,4 +1,4 @@
 FROM docker.openmodelica.org/build-deps:v1.16.3
 
 RUN apt-get update && apt-get install libxml2 libxslt1.1 libxml2-dev libxslt1-dev
-RUN pip3 install matplotlib FMPy
+RUN pip3 install matplotlib FMPy psycopg2-binary

all-plots.py

@@ -3,7 +3,7 @@
 
 import sys, argparse, subprocess, os
 import simplejson as json
-import shared
+import shared, resultsdb
 import re, time, math
 from omcommon import friendlyStr
 
@@ -18,18 +18,19 @@
 parser = argparse.ArgumentParser(description='OpenModelica model testing report generation tool')
 parser.add_argument('branches', nargs='*')
 parser.add_argument('--historypath', default="history")
+resultsdb.addArgument(parser)
 args = parser.parse_args()
 
 branches = [branch.split("/")[-1] for branch in args.branches]
 fnameprefix = args.historypath
 
 libs = {}
 
-import cgi, sqlite3, time, datetime
+import cgi, time, datetime
 from omcommon import friendlyStr, multiple_replace
 
-conn = sqlite3.connect('sqlite3.db')
-cursor = conn.cursor()
+db = resultsdb.connect(args.db)
+cursor = db.cursor()
 
 def dateStr(dint):
   return str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S'))
@@ -98,8 +99,7 @@ def plotLibrary(branch, libname, xs, total, frontend,backend,simcode,template,co
 
 for branch in branches:
   try:
-    cursor.execute("SELECT name FROM [sqlite_master] WHERE type='table' AND name=?", (branch,))
-    one = cursor.fetchone()
+    one = (branch,) if db.tableExists(branch) else None
     if one == None:
       print("No such table '%s'; specify it using --branch=XXX when running test.py" % branch)
       # ignore this table and continue
@@ -112,13 +112,13 @@ def plotLibrary(branch, libname, xs, total, frontend,backend,simcode,template,co
     # ignore this table and continue
     continue
 
-  cursor.execute('''CREATE INDEX IF NOT EXISTS [idx_%s_date] ON [%s](date)''' % (branch,branch))
+  db.createDateIndex(branch)
   libs = {}
-  for (date,libname,total,frontend,backend,simcode,template,compile,simulate,verify) in cursor.execute("""SELECT date,libname,COUNT(finalphase),COUNT(finalphase>=1 or null),COUNT(finalphase>=2 or null),COUNT(finalphase>=3 or null),COUNT(finalphase>=4 or null),COUNT(finalphase>=5 or null),COUNT(finalphase>=6 or null),COUNT(finalphase>=7 or null)
-    FROM [%s]
+  for (date,libname,total,frontend,backend,simcode,template,compile,simulate,verify) in cursor.execute("""SELECT date,libname,COUNT(finalphase),%s
+    FROM %s
     GROUP BY date,libname
     ORDER BY libname,date ASC
-""" % (branch)):
+""" % (",".join(db.countIf("finalphase>=%d" % i) for i in range(1,8)), db.quote(branch))):
     if libname not in libs:
       libs[libname] = ([],[],[],[],[],[],[],[],[])
     libs[libname][0].append(datetime.datetime.fromtimestamp(date))

all-reports.py

@@ -5,7 +5,7 @@
 import codecs
 import sys, argparse, subprocess, os, time
 import simplejson as json
-import shared
+import shared, resultsdb
 import re
 from omcommon import friendlyStr
 
@@ -17,6 +17,7 @@
 parser.add_argument('--githuburltesting', default="https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit")
 parser.add_argument('--omcgitdir', default="../OpenModelica/OpenModelica")
 parser.add_argument('--email', default=False, action='store_true')
+resultsdb.addArgument(parser)
 args = parser.parse_args()
 
 os.environ['TZ'] = 'Europe/Stockholm'
@@ -44,11 +45,11 @@
 
 libs = {}
 
-import cgi, sqlite3, time, datetime
+import cgi, time, datetime
 from omcommon import friendlyStr, multiple_replace
 
-conn = sqlite3.connect('sqlite3.db')
-cursor = conn.cursor()
+db = resultsdb.connect(args.db)
+cursor = db.cursor()
 
 def dateStr(dint):
   return str(datetime.datetime.fromtimestamp(dint).strftime('%Y-%m-%d %H:%M:%S'))
@@ -70,8 +71,7 @@ def modelLink(libname, modelname, extension, text):
 emails_to_send = {}
 for branch in branches:
   try:
-    cursor.execute("SELECT name FROM [sqlite_master] WHERE type='table' AND name=?", (branch,))
-    one = cursor.fetchone()
+    one = (branch,) if db.tableExists(branch) else None
     if one == None:
       print("No such table '%s'; specify it using --branch=XXX when running test.py" % branch)
       # ignore this table and continue
@@ -86,8 +86,8 @@ def modelLink(libname, modelname, extension, text):
     missing_branches.append(branch)
     continue
 
-  cursor.execute('''CREATE INDEX IF NOT EXISTS [idx_%s_date] ON [%s](date)''' % (branch,branch))
-  cursor.execute("SELECT date,omcversion FROM [omcversion] WHERE branch LIKE ? COLLATE NOCASE ORDER BY date ASC", (branch,))
+  db.createDateIndex(branch)
+  cursor.execute("SELECT date,omcversion FROM omcversion WHERE %s ORDER BY date ASC" % db.likeNoCase("branch"), (branch,))
   entries = cursor.fetchall()
   n = len(entries)
   urlToOpen = "%s/%s/00_history.html" % (historyurl, branch)
@@ -152,11 +152,11 @@ def modelLink(libname, modelname, extension, text):
       gitloglibrarytesting = "<tr><td>could not get the git log for OpenModelicaLibraryTesting</td></tr>"
 
     tpl = tpl.replace("#OMCGITLOG#",gitlog).replace("#NUMCOMMITS#",str(gitlog.count("<tr>"))).replace("#3rdParty#",thirdPartyChanged).replace("#OMCLIBRARYTESTINGGITLOG#",gitloglibrarytesting)
-    libnames = [libname for (libname,) in cursor.execute("""SELECT libname FROM [%s] WHERE date=? GROUP BY libname""" % branch, (d2,))]
+    libnames = [libname for (libname,) in cursor.execute("""SELECT libname FROM %s WHERE date=? GROUP BY libname""" % db.quote(branch), (d2,))]
     startdates = {}
     # Get previous date of each library run and group them together for fast queries later
     for libname in libnames:
-      ds = cursor.execute("""SELECT date FROM [%s] WHERE date<? AND libname=? ORDER BY date DESC LIMIT 1""" % branch, (d2,libname)).fetchall()
+      ds = cursor.execute("""SELECT date FROM %s WHERE date<? AND libname=? ORDER BY date DESC LIMIT 1""" % db.quote(branch), (d2,libname)).fetchall()
       if len(ds)==0:
         continue
       ((d1lib,),) = ds
@@ -166,10 +166,14 @@ def modelLink(libname, modelname, extension, text):
     regressions = []
     for d1lib in startdates.keys():
     # Order by date so we can select and know which is the older and which is the newer value... for finalphase, and the execution times
-    # Note: GROUP_CONCAT returns both values as a string... So you need to split it later
-      query = """SELECT model,libname,GROUP_CONCAT(finalphase),GROUP_CONCAT(frontend),GROUP_CONCAT(backend),GROUP_CONCAT(simcode),GROUP_CONCAT(templates),GROUP_CONCAT(compile),GROUP_CONCAT(simulate) FROM
-    (SELECT model,libname,finalphase,frontend,backend,simcode,templates,compile,simulate FROM [%s] WHERE date IN (?,?) AND libname IN (%s) ORDER BY date)
-  GROUP BY model,libname HAVING
+    # Note: the group concatenation returns both values as a string... So you need to split it
+    # later. The order is the one of the dates, which PostgreSQL only guarantees when the
+    # aggregate says so, hence the date column in the inner query.
+      concat = ",".join(db.groupConcat(c, "date") for c in
+                        ["finalphase","frontend","backend","simcode","templates","compile","simulate"])
+      query = ("""SELECT model,libname,%s FROM
+    (SELECT model,libname,date,finalphase,frontend,backend,simcode,templates,compile,simulate FROM %%s WHERE date IN (?,?) AND libname IN (%%s) ORDER BY date) AS phases
+  GROUP BY model,libname HAVING""" % concat + """
     (MIN(finalphase) <> MAX(finalphase)) OR
     ((MIN(finalphase) >= ?) AND
       (MAX(frontend) > ?*MIN(frontend) AND MAX(frontend) > ?) OR
@@ -179,7 +183,7 @@ def modelLink(libname, modelname, extension, text):
       (MAX(compile) > ?*MIN(compile) AND MAX(compile) > ?) OR
       (MAX(simulate) > ?*MIN(simulate) AND MAX(simulate) > ?)
     )
-  """ % (branch,",".join(["'%s'" % libname for libname in startdates[d1lib]]))
+  """) % (db.quote(branch),",".join(["'%s'" % libname for libname in startdates[d1lib]]))
       cursor.execute(query, (d1lib,d2,timeMinPhase,timeRel,timeAbs,timeRel,timeAbs,timeRel,timeAbs,timeRel,timeAbs,timeRel,2*timeAbs,timeRel,timeAbs))
       regressions += cursor.fetchall()
     regressions = sorted(regressions, key = lambda x: (x[1],x[0]))
@@ -228,10 +232,10 @@ def modelLink(libname, modelname, extension, text):
 
     libstrs = []
     for libname in sorted(list(libs)):
-      cursor.execute("SELECT libversion,confighash FROM [libversion] WHERE branch LIKE ? COLLATE NOCASE AND date<=? AND libname=? ORDER BY date DESC LIMIT 1", (branch,d1,libname))
+      cursor.execute("SELECT libversion,confighash FROM libversion WHERE %s AND date<=? AND libname=? ORDER BY date DESC LIMIT 1" % db.likeNoCase("branch"), (branch,d1,libname))
       (lv1,lh1) = cursor.fetchone()
       lv1 = lv1.strip()
-      cursor.execute("SELECT libversion,confighash FROM [libversion] WHERE branch LIKE ? COLLATE NOCASE AND date<=? AND libname=? ORDER BY date DESC LIMIT 1", (branch,d2,libname))
+      cursor.execute("SELECT libversion,confighash FROM libversion WHERE %s AND date<=? AND libname=? ORDER BY date DESC LIMIT 1" % db.likeNoCase("branch"), (branch,d2,libname))
       (lv2,lh2) = cursor.fetchone()
       lv2 = lv2.strip()
       if lv1 != lv2:

clean-dates.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
-import argparse, sqlite3, sys
+import argparse, sys
+import resultsdb
 from datetime import datetime
 
 parser = argparse.ArgumentParser(description='OpenModelica library testing tool')
 parser.add_argument('startDate')
 parser.add_argument('stopDate')
+resultsdb.addArgument(parser)
 
 args = parser.parse_args()
 
@@ -29,12 +31,12 @@
    sys.stdout.write("Please respond with 'yes' or 'no'")
    sys.exit(1)
 
-conn = sqlite3.connect('sqlite3.db')
-cursor = conn.cursor()
+db = resultsdb.connect(args.db)
+cursor = db.cursor()
 
-tables = [tbl for (tbl,) in cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")]
+tables = db.tables()
 for tbl in tables:
-  cursor.execute("DELETE FROM [%s] WHERE date<? AND date>?" % tbl, (stopTime.timestamp(),startTime.timestamp()))
-conn.commit()
-conn.execute("VACUUM")
-conn.close()
+  cursor.execute("DELETE FROM %s WHERE date<? AND date>?" % db.quote(tbl), (stopTime.timestamp(),startTime.timestamp()))
+db.commit()
+db.vacuum()
+db.close()

clean-empty-omcversion-dates.py

@@ -1,14 +1,16 @@
 #!/usr/bin/env python3
 
-import argparse, sqlite3, sys
+import argparse, sys
+import resultsdb
 from datetime import datetime
 
 parser = argparse.ArgumentParser(description='OpenModelica library testing tool')
+resultsdb.addArgument(parser)
 
 args = parser.parse_args()
 
-conn = sqlite3.connect('sqlite3.db')
-cursor = conn.cursor()
+db = resultsdb.connect(args.db)
+cursor = db.cursor()
 
 entries = cursor.execute("SELECT date,branch FROM omcversion").fetchall()
 dropped=0
@@ -20,18 +22,22 @@
     branchDates[branch] = set()
   branchDates[branch].add(date)
 for branch in branches:
-  data=cursor.execute("SELECT DISTINCT date FROM [%s]" % branch).fetchall()
+  # The shared database holds the branches of every machine, including ones
+  # this one never created a result table for.
+  if not db.tableExists(branch):
+    continue
+  data=cursor.execute("SELECT DISTINCT date FROM %s" % db.quote(branch)).fetchall()
   for (date,) in data:
     try:
       branchDates[branch].remove(date)
     except KeyError:
       pass
   for date in branchDates[branch]:
     print("Dropping empty omcversion entry (%d,%s)" % (date,branch))
-    cursor.execute("DELETE FROM [omcversion] WHERE date=? AND branch=?", (date,branch))
+    cursor.execute("DELETE FROM omcversion WHERE date=? AND branch=?", (date,branch))
     dropped += 1
 
-conn.commit()
+db.commit()
 if dropped>0:
-  conn.execute("VACUUM")
-conn.close()
+  db.vacuum()
+db.close()