From c257beba10f44acc8e4eb7f2c9c3791cbdd22f28 Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Tue, 11 Aug 2026 13:18:46 +0200 Subject: [PATCH 1/6] [FMI] Let --fmisimulator be given more than once First step of #78: the FMI jobs come in pairs - v1.26-fmi and v1.26-fmi-fmpy, v1.27-fmi and v1.27-fmi-fmpy, master-fmi and master-fmi-fmpy - and both jobs of a pair build the same Model Exchange FMUs, some 19500 of them, only to simulate them with a different tool. Measured on ExternData here: 164 seconds building the FMUs and 0.6 simulating them with OMSimulator, then another 167 seconds building the very same FMUs to simulate them for 3 seconds with FMPy. --fmisimulator can now be repeated, as "name=command" or as the bare command it has always been, so that one job can simulate every FMU with several tools after building it once. The FMU can only be shared when everything that goes into it is the same - the same compiler, the same library and the same build flags - so what a merged job varies is the simulator and nothing else. That is why the three cs-fmu-cvode jobs stay as they are: their FMUs are Co-Simulation ones built with --fmiFlags=s:cvode, not the same artifact at all. --branch now names the job rather than the table, and every simulator derives its own from it: OMSimulator keeps the plain v1.27-fmi it has always had, FMPy fills v1.27-fmi-fmpy, PyFMI would fill v1.27-fmi-pyfmi. The mapping is keyed by the simulator and not by the order it was given in, so that a job asked for FMPy alone still writes to v1.27-fmi-fmpy instead of taking over v1.27-fmi. No report and no history has to move. A job that passes a single simulator behaves as it always did, and the version of every tool is now read rather than only the first one's. This only adds the option and the naming rules; testmodel.py still simulates with one tool, which the next commit changes. --- Generated by Claude Code. Signed-off-by: Adrian Pop --- shared.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ test.py | 30 ++++++++++++++++++++---------- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/shared.py b/shared.py index fa8d328..e0c2455 100644 --- a/shared.py +++ b/shared.py @@ -118,4 +118,49 @@ def isFMPy(fmisimulator): else: return False +def fmiSimulatorName(command): + """The short name of an FMI simulator, used to name its branch and its files.""" + if isFMPy(command): + return "fmpy" + if "pyfmi" in command.lower(): + return "pyfmi" + return "OMSimulator" + +def parseFmiSimulators(fmisimulators): + """The --fmisimulator values as an ordered list of (name, command). + + A value is either "name=command" or just the command, whose name is then + taken from the command itself. The order matters: the first simulator keeps + the branch the job was started with and the others get one of their own, see + branchForSimulator. + """ + res = [] + for s in fmisimulators or []: + if not s: + continue + name, sep, command = s.partition("=") + if not sep or "/" in name or " " in name: + (name, command) = (fmiSimulatorName(s), s) + res.append((name, command)) + names = [n for (n, _) in res] + if len(set(names)) != len(names): + raise Exception("The same FMI simulator name is used twice: %s" % ", ".join(names)) + return res + +# The branch a simulator stores its results in is its name appended to the +# branch of the job, except for OMSimulator, which has always had the plain +# -fmi table to itself. Keyed by the simulator rather than by the order it was +# given in, so that a job running only FMPy still fills v1.27-fmi-fmpy and not +# v1.27-fmi. +BRANCH_SUFFIX = {"OMSimulator": ""} + +def branchForSimulator(branch, name): + """Where the results of one FMI simulator of a run are stored. + + --branch names the job, v1.27-fmi, and every simulator derives its own from + it: OMSimulator fills v1.27-fmi, FMPy v1.27-fmi-fmpy, whether they run + together or on their own. + """ + return branch + BRANCH_SUFFIX.get(name, "-%s" % name) + diff --git a/test.py b/test.py index c97573d..6ffc16f 100755 --- a/test.py +++ b/test.py @@ -36,7 +36,7 @@ parser.add_argument('--noclean', action="store_true", default=False) parser.add_argument('--nobuildmodel', action="store_true", help="Translate, build and simulate in a single simulate() call instead of translateModel() followed by simulate(resimulateExecutable=...), so the JIT compile is reported as build time rather than simulation time. Only used by simCodeTarget=wasm-jit.", default=False) parser.add_argument('--coldhot', action="store_true", help="Simulate each model twice in the same omc; the second run reuses the compiled module. Both times are printed, but only the hot one is stored. Only used by simCodeTarget=wasm-jit.", default=False) -parser.add_argument('--fmisimulator', default='') +parser.add_argument('--fmisimulator', action='append', default=[], help="FMI simulator to run the FMUs with, as 'name=command' or just the command. Repeat it to simulate every FMU with several tools without building it more than once; the first one stores its results in --branch and each further one in -, so --branch=master-fmi with OMSimulator and fmpy fills master-fmi and master-fmi-fmpy." ) parser.add_argument('--ulimitvmem', help="Virtual memory limit (in kB) (linux only)", type=int, default=8*1024*1024) parser.add_argument('--default', action='append', help="Add a default value for some configuration key, such as --default=ulimitExe=60. The equals sign is mandatory.", default=[]) parser.add_argument('-j', '--jobs', default=0, help="Ignored and deprecated, use procOMC:0 or procOMC:1 in the config") @@ -67,7 +67,9 @@ extraflags = args.extraflags extrasimflags = args.extrasimflags ompython_omhome = args.ompython_omhome -fmisimulator = args.fmisimulator or None +fmisimulators = shared.parseFmiSimulators(args.fmisimulator) +# The first simulator is the one the single-simulator code paths use. +fmisimulator = fmisimulators[0][1] if fmisimulators else None allTestsFmi = args.fmi fmuType = args.fmuType ulimitMemory = args.ulimitvmem @@ -299,17 +301,24 @@ def timeSeconds(f): sys.stdout.flush() -fmisimulatorversion = None -if fmisimulator: +def fmiSimulatorVersion(command): try: - if not isFMPy(fmisimulator): - fmisimulatorversion = check_output_log([fmisimulator, "-v"], stderr=subprocess.STDOUT).strip() - else: - fmisimulatorversion = subprocess.getoutput(fmisimulator + " --version" ).strip().encode('ascii') + if not isFMPy(command): + return check_output_log([command, "-v"], stderr=subprocess.STDOUT).strip() + return subprocess.getoutput(command + " --version").strip().encode('ascii') except subprocess.CalledProcessError as e: - print("Failure to run %s:\n%s" %(fmisimulator, e.output)) + print("Failure to run %s:\n%s" % (command, e.output)) raise e - print(fmisimulatorversion) + +fmisimulatorversions = {} +fmisimulatorversion = None +if fmisimulator: + for (name, command) in fmisimulators: + fmisimulatorversions[name] = fmiSimulatorVersion(command) + print("%s: %s" % (name, fmisimulatorversions[name])) + # The version of the first one goes into the library version of every branch, + # as it did when a job ran a single simulator. + fmisimulatorversion = fmisimulatorversions[fmisimulators[0][0]] else: if allTestsFmi: raise Exception("No OMSimulator; trying to simulate using FMI") @@ -617,6 +626,7 @@ def hashReferenceFiles(s): conf["haveFMI"] = fmiOK_C conf["haveFMICpp"] = fmiOK_Cpp conf["fmisimulator"] = fmisimulator + conf["fmisimulators"] = ["%s=%s" % (n, c) for (n, c) in fmisimulators] conf["fmuType"] = fmuType if (not canChangeOptLevel) and "optlevel" in conf: print("Deleting optlevel") From 9c0c528681c9b2816fb0b361cf7cbb3a7cb709fe Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Tue, 11 Aug 2026 13:50:41 +0200 Subject: [PATCH 2/6] [FMI] Describe the FMI simulators in a configuration file Adding a simulator to the testing was a case in testmodel.py, one in the version check and one in the choice of result file. It is now an entry in configs/fmi-simulators.json and nothing else: "fmusim": { "untested": true, "resultExtension": "csv", "versionArgument": "--version", "stepSizeArgument": " --output-interval {stepSize:g}", "arguments": "--interface-type ModelExchange --output-file {result} ..." } An entry says how the tool is invoked, what it writes, how it prints its version, how it spells a step size and which table it fills. The arguments are a template formatted with the model and the experiment, so a tool with different flags needs no code, and a Python package without a command line needs a driver script that the entry then points "command" at. stepSizeArgument is a key of its own because the tools disagree about an experiment that has no step size: OMSimulator leaves the flag out, while FMPy passes --output-interval 0 all the same. A tool of the second kind writes {stepSize:g} into its arguments and needs no stepSizeArgument at all. The entry for fmusim is there because it is the next one we expect to want. It is marked untested, and asking for a simulator nobody has run yet says so rather than failing every model with a puzzling error. Checked against the format strings testmodel.py uses today: both tools are given exactly the command line they were given before, with a step size and without one. --- Generated by Claude Code. Signed-off-by: Adrian Pop --- configs/fmi-simulators.json | 35 ++++++++++++++++++++ shared.py | 66 +++++++++++++++++++++++++++++-------- 2 files changed, 87 insertions(+), 14 deletions(-) create mode 100644 configs/fmi-simulators.json diff --git a/configs/fmi-simulators.json b/configs/fmi-simulators.json new file mode 100644 index 0000000..89a3fea --- /dev/null +++ b/configs/fmi-simulators.json @@ -0,0 +1,35 @@ +{ + "_comment": [ + "The FMI simulators the testing knows how to run. Adding one is an entry here and nothing else.", + "", + "arguments the command line, formatted with: simulator, fmu, result, tempDir,", + " startTime, stopTime, tolerance, timeout, stepSize and stepSizeArgument.", + "command how the tool is invoked, {simulator} by default; FMPy needs a subcommand.", + "resultExtension what the tool writes, so the verification knows what to compare.", + "versionArgument the flag that prints the version, recorded with the results.", + "stepSizeArgument the step size flag for a tool that leaves it out when the experiment has no step size; a tool that always wants one puts {stepSize:g} in arguments instead.", + "branchSuffix appended to --branch to get the table of this simulator; the default is", + " -, so OMSimulator is the only one that needs to say anything here.", + "untested set it while nobody has run the entry yet; the run then says so." + ], + "OMSimulator": { + "branchSuffix": "", + "resultExtension": "mat", + "versionArgument": "-v", + "stepSizeArgument": " --stepSize={stepSize:g}", + "arguments": "-r={result} --tempDir={tempDir} --startTime={startTime:g} --stopTime={stopTime:g}{stepSizeArgument} --timeout={timeout:g} --tolerance={tolerance:g} {fmu}" + }, + "fmpy": { + "resultExtension": "csv", + "versionArgument": "--version", + "command": "{simulator} simulate", + "arguments": "--output-file {result} --start-time {startTime:g} --stop-time {stopTime:g} --timeout {timeout:g} --relative-tolerance {tolerance:g} --interface-type ModelExchange --solver CVode --output-interval {stepSize:g} {fmu}" + }, + "fmusim": { + "untested": true, + "resultExtension": "csv", + "versionArgument": "--version", + "stepSizeArgument": " --output-interval {stepSize:g}", + "arguments": "--interface-type ModelExchange --output-file {result} --start-time {startTime:g} --stop-time {stopTime:g}{stepSizeArgument} {fmu}" + } +} diff --git a/shared.py b/shared.py index e0c2455..2105886 100644 --- a/shared.py +++ b/shared.py @@ -119,19 +119,23 @@ def isFMPy(fmisimulator): return False def fmiSimulatorName(command): - """The short name of an FMI simulator, used to name its branch and its files.""" - if isFMPy(command): - return "fmpy" - if "pyfmi" in command.lower(): - return "pyfmi" + """The name of the simulator a bare --fmisimulator runs. + + Any of the known names appearing in the command wins; a command that names + none of them is OMSimulator, which is how --fmisimulator was used before it + could name its simulator. + """ + for name in sorted(fmiSimulators(), key=len, reverse=True): + if name.lower() in command.lower(): + return name return "OMSimulator" def parseFmiSimulators(fmisimulators): """The --fmisimulator values as an ordered list of (name, command). A value is either "name=command" or just the command, whose name is then - taken from the command itself. The order matters: the first simulator keeps - the branch the job was started with and the others get one of their own, see + taken from the command itself. The name decides which simulator of + configs/fmi-simulators.json is run and which branch its results go to, see branchForSimulator. """ res = [] @@ -145,14 +149,48 @@ def parseFmiSimulators(fmisimulators): names = [n for (n, _) in res] if len(set(names)) != len(names): raise Exception("The same FMI simulator name is used twice: %s" % ", ".join(names)) + for name in names: + if fmiSimulator(name).get("untested"): + print("Warning: nobody has run %s through the testing yet; if its flags in %s are wrong, " + "every model will fail to simulate." % (name, FMI_SIMULATORS_FILE)) return res -# The branch a simulator stores its results in is its name appended to the -# branch of the job, except for OMSimulator, which has always had the plain -# -fmi table to itself. Keyed by the simulator rather than by the order it was -# given in, so that a job running only FMPy still fills v1.27-fmi-fmpy and not -# v1.27-fmi. -BRANCH_SUFFIX = {"OMSimulator": ""} +# The FMI simulators live in configs/fmi-simulators.json, so that adding one is +# an entry in a file rather than a change to the scripts. A simulator stores +# its results in the branch of the job with its name appended, except +# OMSimulator, which has always had the plain -fmi table to itself; the mapping +# is keyed by the simulator and not by the order it was given in, so that a job +# running only FMPy still fills v1.27-fmi-fmpy and not v1.27-fmi. +FMI_SIMULATORS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "configs", "fmi-simulators.json") +_fmiSimulators = None + +def fmiSimulators(path=None): + """Everything the testing knows about the FMI simulators.""" + global _fmiSimulators + if _fmiSimulators is None or path: + with open(path or FMI_SIMULATORS_FILE) as fin: + _fmiSimulators = dict((k, v) for (k, v) in json.load(fin).items() if not k.startswith("_")) + return _fmiSimulators + +def fmiSimulator(name): + """What is known about an FMI simulator, by the name --fmisimulator gave it.""" + known = fmiSimulators() + if name not in known: + raise Exception("Unknown FMI simulator %s; known are %s. Adding one is an entry in %s." + % (name, ", ".join(sorted(known)), FMI_SIMULATORS_FILE)) + return known[name] + +def fmiSimulatorCommand(name, command, **values): + """The command line that runs one FMU with one simulator.""" + spec = fmiSimulator(name) + values["simulator"] = command + # Only a tool that leaves the flag out when there is no step size has one; + # the others put the step size in their arguments and always pass it. + values["stepSizeArgument"] = (spec["stepSizeArgument"].format(**values) + if values.get("stepSize") and "stepSizeArgument" in spec else "") + return "%s %s" % (spec.get("command", "{simulator}").format(**values), + spec["arguments"].format(**values)) def branchForSimulator(branch, name): """Where the results of one FMI simulator of a run are stored. @@ -161,6 +199,6 @@ def branchForSimulator(branch, name): it: OMSimulator fills v1.27-fmi, FMPy v1.27-fmi-fmpy, whether they run together or on their own. """ - return branch + BRANCH_SUFFIX.get(name, "-%s" % name) + return branch + fmiSimulator(name).get("branchSuffix", "-%s" % name) From 9db9fe35bbf80724e90f8ae4b26aeb31a834a178 Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Tue, 11 Aug 2026 14:16:48 +0200 Subject: [PATCH 3/6] [FMI] Build each FMU once and simulate it with every tool asked for The heart of #78. testmodel.py builds the FMU as before and then runs every simulator the job was given on it, and test.py writes one row per simulator into the branch of that simulator. A job given OMSimulator and FMPy on --branch=v1.27-fmi therefore fills v1.27-fmi and v1.27-fmi-fmpy from a single build, which is what the two jobs of a pair used to do twice. Measured on ExternData, 12 models of which 9 build an FMU: separate 164s building + 0.60s simulating OMSimulator 167s building + 3.09s simulating FMPy merged 178s building + 0.74s + 2.58s both so the FMUs are built once instead of twice and the run costs about half. The results are the same either way: every model's phase, verification time, number of variables compared and number that differ is identical to what the two separate runs produced, for both tables. The verification of a result against its reference file became a function that returns rather than one that ends the run, so that the next simulator of the same FMU can be verified too, and omc is now quit once all of them are done rather than after the first. A simulator that fails or times out records its own failure and leaves the others alone; the first one keeps ending the model as it always has, so put the tool you trust most first. A model that fails before its FMU exists reports the phase the build stopped at for every simulator, rather than pretending each of them failed. --- Generated by Claude Code. Signed-off-by: Adrian Pop --- configs/fmi-simulators.json | 37 +++-- shared.py | 19 ++- test.py | 61 ++++++-- testmodel.py | 302 ++++++++++++++++++++++-------------- 4 files changed, 265 insertions(+), 154 deletions(-) diff --git a/configs/fmi-simulators.json b/configs/fmi-simulators.json index 89a3fea..ee98939 100644 --- a/configs/fmi-simulators.json +++ b/configs/fmi-simulators.json @@ -2,22 +2,31 @@ "_comment": [ "The FMI simulators the testing knows how to run. Adding one is an entry here and nothing else.", "", - "arguments the command line, formatted with: simulator, fmu, result, tempDir,", - " startTime, stopTime, tolerance, timeout, stepSize and stepSizeArgument.", - "command how the tool is invoked, {simulator} by default; FMPy needs a subcommand.", - "resultExtension what the tool writes, so the verification knows what to compare.", - "versionArgument the flag that prints the version, recorded with the results.", - "stepSizeArgument the step size flag for a tool that leaves it out when the experiment has no step size; a tool that always wants one puts {stepSize:g} in arguments instead.", - "branchSuffix appended to --branch to get the table of this simulator; the default is", - " -, so OMSimulator is the only one that needs to say anything here.", - "untested set it while nobody has run the entry yet; the run then says so." + "arguments the command line, a template over: simulator, fmu, result, tempDir,", + " startTime, stopTime, tolerance, timeout, stepSize, and every name", + " defined in optionalArguments.", + "optionalArguments flags that have to disappear when there is nothing to put in them, as a", + " result is always the file the tool should write; requestedResult is empty", + " when the run wants no result file at all, which only OMSimulator acts on.", + " name and its own template. OMSimulator crashes on --stepSize=0 instead of", + " ignoring it, so its step size flag lives here; FMPy wants --output-interval 0", + " all the same and writes the value straight into arguments.", + "command how the tool is invoked, {simulator} by default; FMPy needs a subcommand.", + "resultExtension what the tool writes, so the verification knows what to compare.", + "versionArgument the flag that prints the version, recorded with the results.", + "branchSuffix appended to --branch to get the table of this simulator; the default is", + " -, so OMSimulator is the only one that needs to say anything here.", + "untested set it while nobody has run the entry yet; the run then says so." ], "OMSimulator": { "branchSuffix": "", "resultExtension": "mat", "versionArgument": "-v", - "stepSizeArgument": " --stepSize={stepSize:g}", - "arguments": "-r={result} --tempDir={tempDir} --startTime={startTime:g} --stopTime={stopTime:g}{stepSizeArgument} --timeout={timeout:g} --tolerance={tolerance:g} {fmu}" + "arguments": "{resultArgument} --tempDir={tempDir} --startTime={startTime:g} --stopTime={stopTime:g}{stepSizeArgument} --timeout={timeout:g} --tolerance={tolerance:g} {fmu}", + "optionalArguments": { + "resultArgument": "-r={requestedResult}", + "stepSizeArgument": " --stepSize={stepSize:g}" + } }, "fmpy": { "resultExtension": "csv", @@ -29,7 +38,9 @@ "untested": true, "resultExtension": "csv", "versionArgument": "--version", - "stepSizeArgument": " --output-interval {stepSize:g}", - "arguments": "--interface-type ModelExchange --output-file {result} --start-time {startTime:g} --stop-time {stopTime:g}{stepSizeArgument} {fmu}" + "arguments": "--interface-type ModelExchange --output-file {result} --start-time {startTime:g} --stop-time {stopTime:g}{stepSizeArgument} {fmu}", + "optionalArguments": { + "stepSizeArgument": " --output-interval {stepSize:g}" + } } } diff --git a/shared.py b/shared.py index 2105886..a5afa80 100644 --- a/shared.py +++ b/shared.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import re, os, subprocess +import re, os, string, subprocess import simplejson as json simCodeTargetRe = re.compile('--simCodeTarget=([^"\'\\s,;)]+)') @@ -182,13 +182,20 @@ def fmiSimulator(name): return known[name] def fmiSimulatorCommand(name, command, **values): - """The command line that runs one FMU with one simulator.""" + """The command line that runs one FMU with one simulator. + + arguments is a template over the values below plus anything the entry defines + in optionalArguments, which are the flags that have to disappear when there + is nothing to put in them: OMSimulator crashes on --stepSize=0 rather than + ignoring it, while FMPy wants --output-interval 0 all the same and therefore + writes the value straight into its arguments. + """ spec = fmiSimulator(name) values["simulator"] = command - # Only a tool that leaves the flag out when there is no step size has one; - # the others put the step size in their arguments and always pass it. - values["stepSizeArgument"] = (spec["stepSizeArgument"].format(**values) - if values.get("stepSize") and "stepSizeArgument" in spec else "") + for (key, template) in (spec.get("optionalArguments") or {}).items(): + used = [f.split(":")[0].split(".")[0].split("[")[0] + for (_, f, _, _) in string.Formatter().parse(template) if f] + values[key] = template.format(**values) if all(values.get(u) for u in used) else "" return "%s %s" % (spec.get("command", "{simulator}").format(**values), spec["arguments"].format(**values)) diff --git a/test.py b/test.py index 6ffc16f..ce7ac43 100755 --- a/test.py +++ b/test.py @@ -542,11 +542,15 @@ def testHelloWorld(cmd): print("Unknown schema user_version=%d" % user_version) sys.exit(1) -cursor.execute('''CREATE TABLE if not exists [%s] +def createBranchTable(branch): + """A run fills one table per FMI simulator, so this happens more than once.""" + cursor.execute('''CREATE TABLE if not exists [%s] (date integer NOT NULL, libname text NOT NULL, model text NOT NULL, exectime real NOT NULL, frontend real NOT NULL, backend real NOT NULL, simcode real NOT NULL, templates real NOT NULL, compile real NOT NULL, simulate real NOT NULL, verify real NOT NULL, verifyfail integer NOT NULL, verifytotal integer NOT NULL, finalphase integer NOT NULL, parsing real NOT NULL)''' % branch) -cursor.execute('''DROP INDEX IF EXISTS [idx_%s_date]''' % branch) + cursor.execute('''DROP INDEX IF EXISTS [idx_%s_date]''' % branch) + +createBranchTable(branch) cursor.execute('''DROP INDEX IF EXISTS idx_omcversion_date''') cursor.execute('''DROP INDEX IF EXISTS idx_libversion_date''') @@ -930,10 +934,20 @@ def loadJsonOrEmptySet(f): print(" %-70s cold %8.4f hot %8.4f" % (model, data["simcold"], data.get("sim") or 0.0)) sys.stdout.flush() -for key in stats.keys(): - (name,model,libname,data)=stats[key] - stats_by_libname[libname]["stats"].append(stats[key]) - values = (testRunStartTimeAsEpoch, +def resultValues(model, libname, data, simulator=None): + """One row of a branch table. + + Everything up to the build is what the model cost whatever simulates it, so + the simulators of one FMU share it and differ only in the simulation, the + verification and how far they got. + """ + simulated = data if simulator is None else (data.get("simulators") or {}).get(simulator) + if simulated is None: + # The model never got as far as being simulated, so every simulator of it + # reports the phase the build stopped at rather than a failure of its own. + simulated = {"phase": data.get("phase") or 0} + diff = simulated.get("diff") or {} + return (testRunStartTimeAsEpoch, libname, model, data.get("exectime") or 0.0, @@ -942,19 +956,32 @@ def loadJsonOrEmptySet(f): data.get("simcode") or 0.0, data.get("templates") or 0.0, data.get("build") or 0.0, - data.get("sim") or 0.0, - (data.get("diff") or {}).get("time") or 0.0, - len((data.get("diff") or {}).get("vars") or []), - (data.get("diff") or {}).get("numCompared") or 0, - data.get("phase") or 0, + simulated.get("sim") or 0.0, + diff.get("time") or 0.0, + len(diff.get("vars") or []), + diff.get("numCompared") or 0, + simulated.get("phase") or 0, data.get("parsing") or 0.0 ) - # print values - cursor.execute("INSERT INTO [%s] VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" % branch, values) -for libname in stats_by_libname.keys(): - confighash = stats_by_libname[libname]["conf"]["confighash"] - cursor.execute("INSERT INTO [libversion] VALUES (?,?,?,?,?)", (testRunStartTimeAsEpoch, branch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash)) -cursor.execute("INSERT INTO [omcversion] VALUES (?,?,?)", (testRunStartTimeAsEpoch, branch, omc_version)) + +# One branch per FMI simulator: the first one reports itself in the results of +# the model, the others under their own name, see testmodel.py. +resultBranches = [(branch, None)] +for (simulatorName, _) in fmisimulators[1:]: + resultBranches.append((shared.branchForSimulator(branch, simulatorName), simulatorName)) + +for (resultBranch, simulator) in resultBranches: + createBranchTable(resultBranch) + for key in stats.keys(): + (name,model,libname,data)=stats[key] + if simulator is None: + stats_by_libname[libname]["stats"].append(stats[key]) + cursor.execute("INSERT INTO [%s] VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" % resultBranch, + resultValues(model, libname, data, simulator)) + for libname in stats_by_libname.keys(): + confighash = stats_by_libname[libname]["conf"]["confighash"] + cursor.execute("INSERT INTO [libversion] VALUES (?,?,?,?,?)", (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash)) + cursor.execute("INSERT INTO [omcversion] VALUES (?,?,?)", (testRunStartTimeAsEpoch, resultBranch, omc_version)) """ # Not really a good thing to do; was just done to make generation of the report simpler for libname in skipped_libs.keys(): diff --git a/testmodel.py b/testmodel.py index 00230f1..dba4b3b 100755 --- a/testmodel.py +++ b/testmodel.py @@ -216,8 +216,12 @@ def target(res): "sim":None, "simcold":None, "diff":None, - "phase":0 + "phase":0, + # One entry per FMI simulator beyond the first, which reports itself in the + # keys above; see configs/fmi-simulators.json. + "simulators":{} } +simulators = execstat["simulators"] with open(config) as fp: conf = json.load(fp) @@ -555,9 +559,54 @@ def simulateCmd(resimulate): writeResult() # Do the simulation -# FMPy generates csv, OMSimulator generates mat (outputFormat) +# The FMU is built once and simulated with every tool the job asked for, so +# that testing FMPy no longer means building the same FMU a second time. The +# tools are described in configs/fmi-simulators.json. fmisimulator = conf.get("fmisimulator") -resFile = "%s_res.%s" % (conf["fileName"], outputFormat if not shared.isFMPy(fmisimulator) else 'csv') +fmisimulators = shared.parseFmiSimulators(conf.get("fmisimulators")) if conf.get("fmi") else [] +if conf.get("fmi") and not fmisimulators and fmisimulator: + fmisimulators = shared.parseFmiSimulators([fmisimulator]) + +def resultFile(name=None): + """Where a simulator writes its results. + + Every tool writes what its entry says, and they only need to be told apart + when more than one of them runs on the same FMU. + """ + if not name: + return "%s_res.%s" % (conf["fileName"], outputFormat) + extension = shared.fmiSimulator(name)["resultExtension"] + if len(fmisimulators) < 2: + return "%s_res.%s" % (conf["fileName"], extension) + return "%s_%s_res.%s" % (conf["fileName"], name, extension) + +def artifactPrefix(name=None): + """The files a simulator's results are written to, under files/.""" + if not name or len(fmisimulators) < 2: + return os.path.abspath("../files/%s" % conf["fileName"]).replace('\\','/') + return os.path.abspath("../files/%s_%s" % (conf["fileName"], name)).replace('\\','/') + +resFile = resultFile(fmisimulators[0][0]) if fmisimulators else resultFile() + +def simulateFmu(name, command, resFile, simFile): + """Run the FMU with one simulator, writing what it says to simFile.""" + # Only tell the runs apart when more than one of them shares the directory. + suffix = "_%s" % name if len(fmisimulators) > 1 else "" + fmitmpdir = "temp_%s%s_fmu" % (conf["fileName"].replace(".","_"), suffix) + with open("%s.tmpfiles" % conf["fileName"], "a+") as fp: + fp.write("%s\n" % fmitmpdir) + cmd = shared.fmiSimulatorCommand(name, command, + fmu="%s.fmu" % conf["fileName"].replace(".","_"), + result=resFile, + requestedResult=resFile if outputFormat != "empty" else "", + tempDir=fmitmpdir, startTime=startTime, stopTime=stopTime, + tolerance=tolerance, timeout=conf["ulimitExe"], + stepSize=stepSize) + with open(simFile,"w") as fp: + fp.write("%s\n" % cmd) + pipe = "%s%s" % (conf["fileName"], suffix) + return checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s > %s.pipe 2>&1)" + % (pipe,pipe,pipe,simFile,cmd,pipe), 1.05*conf["ulimitExe"], conf) def simElapsed(): # omc's own time: the wall clock here covers the wrong run for both flags @@ -569,24 +618,12 @@ def simElapsed(): try: # TODO: Timeout more reliably... if conf.get("fmi"): - if not conf.get("fmisimulator"): + if not fmisimulators: with open(simFile,"w") as fp: - fp.write("OMSimulator not available\n") + fp.write("No FMI simulator available\n") writeResultAndExit(0, False, omc, omc_new) - fmitmpdir = "temp_%s_fmu" % conf["fileName"].replace(".","_") - with open("%s.tmpfiles" % conf["fileName"], "a+") as fp: - fp.write("%s\n" % fmitmpdir) - if shared.isFMPy(fmisimulator): - fmisimulator = "%s simulate " % fmisimulator - cmd = "%s --start-time %g --stop-time %g --timeout %g --relative-tolerance %g --interface-type ModelExchange --solver CVode --output-interval %g %s.fmu" % (("--output-file %s" % resFile),startTime,stopTime,conf["ulimitExe"],tolerance,stepSize,conf["fileName"].replace(".","_")) - else: # OMSimulator - stepSizeStr = "" - if stepSize != 0.0: - stepSizeStr = " --stepSize=%g" % stepSize - cmd = "%s --tempDir=%s --startTime=%g --stopTime=%g%s --timeout=%g --tolerance=%g %s.fmu" % (("-r=%s" % resFile) if outputFormat != "empty" else "",fmitmpdir,startTime,stopTime,stepSizeStr,conf["ulimitExe"],tolerance,conf["fileName"].replace(".","_")) - with open(simFile,"w") as fp: - fp.write("%s %s\n" % (fmisimulator, cmd)) - res = checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s %s > %s.pipe 2>&1)" % (conf["fileName"],conf["fileName"],conf["fileName"],simFile,fmisimulator,cmd,conf["fileName"]), 1.05*conf["ulimitExe"], conf) + (name, command) = fmisimulators[0] + res = simulateFmu(name, command, resFile, simFile) elif isWasmJit: if not useSimulate: cmd = simulateCmd(resimulate=True) @@ -643,112 +680,141 @@ def simElapsed(): execstat["sim"] = monotonic()-start writeResultAndExit(0, True, omc, omc_new) -if referenceFile=="": - writeResultAndExit(0, False, omc, omc_new) -if len(referenceVars)==0: - execstat["diff"] = {"time":0.0, "vars":[], "numCompared":0} - execstat["phase"]=7 - writeResultAndExit(0, False, omc, omc_new) +def verifyAgainstReference(resFile, prefix, stat): + """Compare one simulation result against the reference file. -# Check the reference file... + Fills stat["diff"] and stat["phase"] exactly as the single simulator code + did, but returns instead of ending the run, so that the other simulators + of the same FMU can be verified too. + """ + if referenceFile=="": + return + if len(referenceVars)==0: + stat["diff"] = {"time":0.0, "vars":[], "numCompared":0} + stat["phase"]=7 + return -prefix = os.path.abspath("../files/%s.diff" % conf["fileName"]).replace('\\','/') -if not os.path.exists(os.path.normpath(resFile)): - with open(errFile, 'a+') as fp: - fp.write("TODO: How the !@#!# did the simulation report success but simulation result %s does not exist to compare? outputFormat=%s" % (resFile,outputFormat)) - writeResultAndExit(0) + if not os.path.exists(os.path.normpath(resFile)): + with open(errFile, 'a+') as fp: + fp.write("TODO: How the !@#!# did the simulation report success but simulation result %s does not exist to compare? outputFormat=%s" % (resFile,outputFormat)) + return -start=monotonic() -if False and conf["simCodeTarget"] in ["Cpp"]: # This is a work-around for older C++ runtime not supporting variable filters. We don't really need it for master, so let's no use it. - if not sendExpressionTimeout(omc_new, 'filterSimulationResults("%s", "updated%s", vars={%s}, removeDescription=false, hintReadAllVars=false)' % (resFile, resFile, ", ".join(['"%s"' % s for s in referenceVars])), conf["ulimitOmc"]): + start=monotonic() + if False and conf["simCodeTarget"] in ["Cpp"]: # This is a work-around for older C++ runtime not supporting variable filters. We don't really need it for master, so let's no use it. + if not sendExpressionTimeout(omc_new, 'filterSimulationResults("%s", "updated%s", vars={%s}, removeDescription=false, hintReadAllVars=false)' % (resFile, resFile, ", ".join(['"%s"' % s for s in referenceVars])), conf["ulimitOmc"]): + with open(errFile, 'a+') as fp: + fp.write("Failed to filter simulation results. Took time: %.2f\n" % (monotonic()-start)) + return + os.remove(resFile) + os.rename("updated" + resFile, resFile) with open(errFile, 'a+') as fp: - fp.write("Failed to filter simulation results. Took time: %.2f\n" % (monotonic()-start)) - writeResultAndExit(0, False, omc, omc_new) - os.remove(resFile) - os.rename("updated" + resFile, resFile) - with open(errFile, 'a+') as fp: - fp.write("Filtered simulation results in time: %.2f\n" % (monotonic()-start)) -start=monotonic() -try: - (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","%s",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' % - (resFile, referenceFile, prefix, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"]) -except TimeoutError as e: - with open(errFile, 'a+') as fp: - fp.write("Timeout error for diffSimulationResults") - writeResultAndExit(0, False, omc, omc_new) + fp.write("Filtered simulation results in time: %.2f\n" % (monotonic()-start)) + start=monotonic() + try: + (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","%s",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' % + (resFile, referenceFile, prefix, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"]) + except TimeoutError as e: + with open(errFile, 'a+') as fp: + fp.write("Timeout error for diffSimulationResults") + return -execstat["diff"] = {"time":monotonic()-start, "vars":[], "numCompared":len(referenceVars)} -if len(diffVars)==0 and referenceOK: - execstat["phase"]=7 - with open(errFile, 'a+') as fp: - fp.write("Reference file matches\n") -else: - with open(errFile, 'a+') as fp: - fp.write(omc_new.sendExpression('OpenModelica.Scripting.getErrorString()', parsed = False)) - fp.write("\nVariables in the reference:" ) - fp.write(",".join(referenceVars)+"\n") - resVars=omc_new.sendExpression('readSimulationResultVars("%s", readParameters=true, openmodelicaStyle=true)' % resFile) - fp.write("\nVariables in the result:" ) - fp.write(",".join(resVars)+"\n") - diffFiles = [prefix + "." + var for var in diffVars] - execstat["diff"]["vars"]=diffVars - - # Create a file containing only the calibrated variables, for easy display - lstfiles = "\n".join(['
  • %s (javascript) (csv)
    • ' % (str.split(str(f),".diff.",1)[1],str(os.path.basename(f)),str(os.path.basename(f))) for f in diffFiles]) - with open(prefix+".html", 'w') as fp: - fp.write('

    %s differences from the reference file

    startTime: %g

    stopTime: %g

    Simulated using tolerance: %g

      %s
    ' % (conf["modelName"], startTime, stopTime, tolerance, lstfiles)) - for var in diffVars: - if "/" in var: - continue # Quoted identifier, or possibly an error message... Either way, avoid crapping out below - with open(prefix+"."+var+".html", 'w') as fp: - fp.write(""" - - - - - -
    -

    - - - - - - - - - - - -Parameters used for the comparison: Relative tolerance %g (local), %g (relative to max-min). Range delta %g.

    - - -""" % (tolerance, conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], os.path.basename(prefix + "." + var + ".csv"), var)) + stat["diff"] = {"time":monotonic()-start, "vars":[], "numCompared":len(referenceVars)} + if len(diffVars)==0 and referenceOK: + stat["phase"]=7 + with open(errFile, 'a+') as fp: + fp.write("Reference file matches\n") + else: + with open(errFile, 'a+') as fp: + fp.write(omc_new.sendExpression('OpenModelica.Scripting.getErrorString()', parsed = False)) + fp.write("\nVariables in the reference:" ) + fp.write(",".join(referenceVars)+"\n") + resVars=omc_new.sendExpression('readSimulationResultVars("%s", readParameters=true, openmodelicaStyle=true)' % resFile) + fp.write("\nVariables in the result:" ) + fp.write(",".join(resVars)+"\n") + diffFiles = [prefix + "." + var for var in diffVars] + stat["diff"]["vars"]=diffVars + + # Create a file containing only the calibrated variables, for easy display + lstfiles = "\n".join(['
  • %s (javascript) (csv)
    • ' % (str.split(str(f),".diff.",1)[1],str(os.path.basename(f)),str(os.path.basename(f))) for f in diffFiles]) + with open(prefix+".html", 'w') as fp: + fp.write('

    %s differences from the reference file

    startTime: %g

    stopTime: %g

    Simulated using tolerance: %g

      %s
    ' % (conf["modelName"], startTime, stopTime, tolerance, lstfiles)) + for var in diffVars: + if "/" in var: + continue # Quoted identifier, or possibly an error message... Either way, avoid crapping out below + with open(prefix+"."+var+".html", 'w') as fp: + fp.write(""" + + + + + +
    +

    + + + + + + + + + + + + Parameters used for the comparison: Relative tolerance %g (local), %g (relative to max-min). Range delta %g.

    + + + """ % (tolerance, conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], os.path.basename(prefix + "." + var + ".csv"), var)) + + +# The first simulator's results are the ones every non-FMI code path expects. +verifyAgainstReference(resFile, artifactPrefix(fmisimulators[0][0] if fmisimulators else None) + ".diff", execstat) + +# The FMU is built; every other simulator the job asked for is now only a +# simulation and a comparison. A tool that times out or fails takes its own +# results down with it and leaves the others alone - unlike the first one, +# whose timeout ends the model as it always has. +for (name, command) in fmisimulators[1:]: + stat = {"sim": None, "diff": None, "phase": 5} + simulators[name] = stat + simFileOther = os.path.abspath("../files/%s_%s.sim" % (conf["fileName"], name)).replace('\\','/') + other = resultFile(name) + start = monotonic() + try: + simulateFmu(name, command, other, simFileOther) + stat["sim"] = monotonic()-start + stat["phase"] = 6 + verifyAgainstReference(other, artifactPrefix(name) + ".diff", stat) + except TimeoutError as e: + stat["sim"] = monotonic()-start + with open(errFile, 'a+') as fp: + fp.write("%s timed out simulating the FMU\n" % name) + writeResult() -# quit omc_new +# quit omc_new: every verification needed it omc_new = quit_omc(omc_new) writeResultAndExit(0) From 2d80b2dcd468b75a37dbbd04a76ca8fd2bdd1565 Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Tue, 11 Aug 2026 14:23:31 +0200 Subject: [PATCH 4/6] [FMI] Publish the results of every simulator to its own branch directory The .err and .sim files of a run are collected per library and rsynced to a directory named after the branch under libraries.openmodelica.org/branches. A job that runs several simulators has one set of results per simulator, so it now does that once per branch: v1.27-fmi and v1.27-fmi-fmpy each get their own directory, their own library page and their own files, as they did when two jobs produced them. A branch directory looks exactly as it did before, down to the file names. A simulator writes _.sim in the workspace so that the tools of one job do not overwrite each other, but each branch is published from a directory of its own where those appear under the plain .sim that has always been there. The links are hard, not symbolic, so rsync sees ordinary files and nothing is copied twice on disk; where that is not possible, across file systems, the file is copied. The .err is written by the build and is therefore the same for every simulator of a model, and each branch gets a copy rather than a link into another branch: a job asked for FMPy alone publishes no v1.27-fmi to link into, and a branch that is re-run must not break the pages of another. It costs about 66 MB of 3.4 kB files against the 58 GB such a directory already holds. The report of a library is generated from the results of the simulator whose branch it belongs to, so its phases, times and links describe that simulator and not the first one. The two ways of publishing disagreed about --output and still do: the rsync path is given the directory of the branch, the --noSync path the directory the branches live in. Both now derive the directory of each simulator the way they already derived the one of the job. --- Generated by Claude Code. Signed-off-by: Adrian Pop --- test.py | 491 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 295 insertions(+), 196 deletions(-) diff --git a/test.py b/test.py index ce7ac43..63806b3 100755 --- a/test.py +++ b/test.py @@ -1025,213 +1025,312 @@ def cpu_name(): sysInfo = "%s, %d GB RAM, %s%s" % (cpu_name(), int(math.ceil(psutil.virtual_memory().total / (1024.0**3))), ("Docker " + docker + " ") if docker else "", lsb_release) # create target dir to move results without sync operations (win or when --noSync is used) -if result_location != "" and (isWin or noSync): - resRootPath = os.path.join(result_location, branch) - if os.path.exists(resRootPath): - rmtree(resRootPath) - os.mkdir(resRootPath) - -htmltpl=open("library.html.tpl").read() -for libname in stats_by_libname.keys(): - if libname in skipped_libs: - continue - s = None # Make sure I don't use this - filesList = open(libname + ".files", "w") - filesList.write("/\n") - filesList.write("/%s.html\n" % libname) - filesList.write("/files/\n") - conf = stats_by_libname[libname]["conf"] - stats = stats_by_libname[libname]["stats"] - for s in stats: - filename_prefix = "files/%s_%s" % (s[2],s[1]) - filesList.write("/%s*diff*csv\n" % filename_prefix) - filesList.write("/%s*diff*html\n" % filename_prefix) - if is_non_zero_file(filename_prefix+".sim"): - filesList.write("/%s.sim\n" % filename_prefix) - if is_non_zero_file(filename_prefix+".err"): - filesList.write("/%s.err\n" % filename_prefix) - variables = (s[3].get("diff") or {}).get("vars") or [] - if len(variables)>0: - filesList.write("/%s.diff.html\n" % filename_prefix) - for v in variables: - filesList.write("/%s.diff.%s.csv\n" % (filename_prefix, v)) - filesList.write("/%s.diff.%s.html\n" % (filename_prefix, v)) - filesList.close() - testsHTML = "\n".join(['%s%s%s%s%s%s%s%s%s%s%s%s\n' % - (lambda filename_prefix, diff: - ( - ('%s' % (filename_prefix + ".err", html.escape(s[1]))) if is_non_zero_file(filename_prefix + ".err") else html.escape(s[1]), - (' (sim)' % (filename_prefix + ".sim")) if is_non_zero_file(filename_prefix + ".sim") else "", - checkPhase(s[3]["phase"], 7) if s[3]["phase"]>=6 else "#FFFFFF", - ("%s (%d verified)" % (timeSeconds(diff.get("time")), diff.get("numCompared"))) if s[3]["phase"]>=7 else (" " if diff is None else - ('%s (%d/%d failed)' % (timeSeconds(diff.get("time")), filename_prefix, len(diff.get("vars")), diff.get("numCompared")))), - checkPhase(s[3]["phase"], 6), - timeSeconds(s[3].get("sim") or 0), - checkPhase(s[3]["phase"], 5), - timeSeconds(sum(s[3].get(x) or 0.0 for x in ["frontend","backend","simcode","templates","build"])), - timeSeconds(s[3].get("parsing") or 0), - checkPhase(s[3]["phase"], 1), - timeSeconds(s[3].get("frontend") or 0), - checkPhase(s[3]["phase"], 2), - timeSeconds(s[3].get("backend") or 0), - checkPhase(s[3]["phase"], 3), - timeSeconds(s[3].get("simcode") or 0), - checkPhase(s[3]["phase"], 4), - timeSeconds(s[3].get("templates") or 0), - checkPhase(s[3]["phase"], 5), - timeSeconds(s[3].get("build") or 0), - timeSeconds(s[3].get("exectime") or 0) - ))(filename_prefix="files/%s_%s" % (s[2], s[1]), diff=s[3].get("diff")) - for s in natsorted(stats, key=lambda s: s[1])]) - numSucceeded = [len(stats)] + [sum(1 if s[3]["phase"]>=i else 0 for s in stats) for i in range(1,8)] +def stageRootFor(simulator, suffix): + """The directory a branch is published from. - try: - githuburltesting = "https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit/" - gitloglibrarytesting = check_output_log(["git", "log", '--pretty=
    CommitDateAuthorSummary
    %%h%%ai%%an%%s
    ' % (githuburltesting), "-1"], cwd="./").decode("utf-8") - except subprocess.CalledProcessError as e: - print(str(e)) - gitloglibrarytesting = "
    could not get the git log for OpenModelicaLibraryTesting
    " - - # adrpo: attempt to get the revision of the reference files if possible - if conf.get("referenceFiles"): - c = conf.get("referenceFiles") - if DEBUG: - print("referenceFiles git ... attempting to retrieve info from directory: %s" % c) - sys.stdout.flush() - gitReferenceFiles = c - # see if we have a commit file - f = os.path.join(c, "commit") - if os.path.exists(f): - with open(f) as fin: - gitReferenceFilesVersion = fin.read() - print("referenceFiles git ... read from file %s" % f) - sys.stdout.flush() - else: - try: - if isinstance(c, (str, bytes)): - if DEBUG: - print("referenceFiles git ... see if directory has an evironment variable") - sys.stdout.flush() - m = re.search("^[$][A-Z_]+", c) - if m: - k = m.group(0)[1:] - if k not in os.environ: - if DEBUG: - print("referenceFiles git ... environment variable used in the directory cannot be found in the environment: %s" % k) - sys.stdout.flush() - raise Exception("Environment variable %s not defined, but used in JSON config for reference files" % k) - gitReferenceFiles = c.replace(m.group(0), os.environ[k]) - if DEBUG: - print("referenceFiles git ... directory after replacing the environment variable: %s" % gitReferenceFiles) - sys.stdout.flush() - sys.stdout.flush() - try: - gitReferenceFilesURL = check_output_log(["git", "config", "--get", "remote.origin.url"], cwd=gitReferenceFiles).decode("utf-8") - except subprocess.CalledProcessError as e: - print(e) - gitReferenceFilesURL = gitReferenceFiles - gitReferenceFilesVersion = check_output_log(["git", "log", '--pretty=
    CommitDateAuthorSummary
    %%h%%ai%%an%%s
    ' % (gitReferenceFilesURL), "-1"], cwd=gitReferenceFiles).decode("utf-8") - print("referenceFiles git ... got version information: %s" % gitReferenceFilesVersion) - sys.stdout.flush() - except subprocess.CalledProcessError as e: - print("referenceFiles git ... something went wrong with getting the git info for directory: %s" % c) - print(str(e)) - sys.stdout.flush() - gitReferenceFilesVersion = "" - else: - gitReferenceFilesVersion = "" - - replacements = ( - (u"#sysInfo#", html.escape(sysInfo)), - (u"#omcVersion#", html.escape(omc_version)), - (u"#fmiToolVersion#", ("

    "+html.escape("FMI tool: %s" % fmisimulatorversion)+"

    ") if fmisimulatorversion else ""), - (u"#fmi#", ("

    "+html.escape("FMI version: %s" % conf.get("fmi"))+"

    ") if conf.get("fmi") else ""), - (u"#optlevel#", html.escape(conf.get("optlevel")) if (canChangeOptLevel and conf.get("optlevel")) else "Tool default"), - (u"#timeStart#", html.escape(time.strftime('%Y-%m-%d %H:%M:%S', start_as_time))), - (u"#fileName#", html.escape(libname)), - (u"#customCommands#", html.escape("\n".join(conf["customCommands"]))), - (u"#libraryVersionRevision#", html.escape(conf["libraryVersionRevision"])), - (u"#OpenModelicaLibraryTesting#", gitloglibrarytesting), - (u"#metadata#", html.escape(conf["metadata"])), - (u"#ulimitOmc#", html.escape(str(conf["ulimitOmc"]))), - (u"#ulimitExe#", html.escape(str(conf["ulimitExe"]))), - (u"#defaultTolerance#", html.escape(str(conf["defaultTolerance"]))), - (u"#defaultNumberOfIntervals#", html.escape(str(conf["defaultNumberOfIntervals"]))), - (u"#simFlags#", html.escape(conf.get("simFlags") or "")), - (u"#referenceFiles#", ('

    Reference Files: %s

    %s' % ((conf["referenceFilesURL"].replace(os.path.dirname(os.path.realpath(__file__)),"")), gitReferenceFilesVersion)) if ((conf.get("referenceFilesURL") or "") != "") else ""), - (u"#referenceTool#", ('

    Verified using: %s (diffSimulationResults)

    ' % html.escape(ompython_omc_version)) if ((conf.get("referenceFiles") or "") != "") else ""), - (u"#Total#", html.escape(str(numSucceeded[0]))), - (u"#FrontendColor#", checkNumSucceeded(numSucceeded, 1)), - (u"#BackendColor#", checkNumSucceeded(numSucceeded, 2)), - (u"#SimCodeColor#", checkNumSucceeded(numSucceeded, 3)), - (u"#TemplatesColor#", checkNumSucceeded(numSucceeded, 4)), - (u"#CompilationColor#", checkNumSucceeded(numSucceeded, 5)), - (u"#SimulationColor#", checkNumSucceeded(numSucceeded, 6)), - (u"#VerificationColor#", checkNumSucceeded(numSucceeded, 7)), - (u"#Frontend#", html.escape(str(numSucceeded[1]))), - (u"#Backend#", html.escape(str(numSucceeded[2]))), - (u"#SimCode#", html.escape(str(numSucceeded[3]))), - (u"#Templates#", html.escape(str(numSucceeded[4]))), - (u"#Compilation#", html.escape(str(numSucceeded[5]))), - (u"#Simulation#", html.escape(str(numSucceeded[6]))), - (u"#Verification#", html.escape(str(numSucceeded[7]))), - (u"#totalTime#", html.escape(str(datetime.timedelta(seconds=int(sum(s[3].get("exectime") or 0.0 for s in stats)))))), - (u"#config#", html.escape(json.dumps(conf["configFromFile"], indent=1, sort_keys=True))), - (u"#testsHTML#", testsHTML) - ) - open("%s.html" % libname, "w").write(multiple_replace(htmltpl, *replacements)) - - # move results by sync operations (not available under win) - if result_location != "" and not isWin and not noSync: - result_location_libname = "%s/%s" % (result_location, libname) + The first simulator publishes the workspace itself, as a job always has. The + others write _.sim there so as not to overwrite it, and are + published from a directory of their own where the same files appear under the + plain .sim a branch directory has always held. + """ + if not suffix: + return "." + root = "publish_%s" % simulator + for d in (root, os.path.join(root, "files")): try: - os.mkdir("emptydir") - except: + os.mkdir(d) + except OSError: pass - check_output_log(["rsync", "-aR", "emptydir/", result_location]) - check_output_log(["rsync", "-aR", "emptydir/", result_location_libname]) - check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"]) + return root + +def stagePublished(stageRoot, workspacePrefix, publishedPrefix, suffix): + """Link one model's results into the directory its branch is published from.""" + if not suffix: + return + for f in glob.glob(glob.escape(workspacePrefix) + "*"): + published = os.path.join(stageRoot, publishedPrefix + f[len(workspacePrefix):]) try: - check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname]) - except: - check_output_log(["rsync", "-aR", "emptydir/", result_location]) - check_output_log(["rsync", "-aR", "emptydir/", result_location_libname]) - check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"]) - check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname]) - if (conf.get("referenceFiles") or "") != "" and dygraphs: - check_output_log(["rsync", "-a", dygraphs, result_location_libname+"/files"]) - else: - print("No Sync: result_location [%s] != "" and not isWin [%s] and not noSync [%s] : library: %s" % (result_location, isWin, noSync, libname)) + if os.path.exists(published): + os.unlink(published) + os.link(f, published) + except OSError: + shutil.copy2(f, published) + +def stageShared(stageRoot, prefix, suffix): + """The .err of the build belongs to every simulator of the model.""" + if not suffix: + return + for f in glob.glob(glob.escape(prefix) + ".err"): + published = os.path.join(stageRoot, f) + try: + if os.path.exists(published): + os.unlink(published) + os.link(f, published) + except OSError: + shutil.copy2(f, published) + +def dataForSimulator(data, simulator): + """The results of one model as one simulator saw them. - # move results without sync operations (win or when --noSync is used) + Everything up to the build is shared, so only the simulation, the comparison + against the reference file and the phase come from the simulator itself. + """ + if simulator is None: + return data + simulated = (data.get("simulators") or {}).get(simulator) + merged = dict(data) + merged["sim"] = (simulated or {}).get("sim") + merged["diff"] = (simulated or {}).get("diff") + # A model that never got as far as being simulated stopped in the build, + # which every simulator of it shares. + merged["phase"] = simulated.get("phase") if simulated else data.get("phase") + return merged + +def artifactSuffix(simulator): + """What tells the files of one simulator from those of another.""" + return "_%s" % simulator if simulator and len(fmisimulators) > 1 else "" + +jobOutput = result_location + +def outputFor(resultBranch): + """Where a branch publishes; --output names the directory of the job. + + rsync runs from the directory a branch is staged in, so a local destination + has to be absolute; a remote one, host:path, already is. + """ + location = jobOutput + if jobOutput and resultBranch != branch: + base = jobOutput.rstrip("/") + if base.endswith("/" + branch): + location = "%s%s" % (base[:-len(branch)], resultBranch) + else: + location = "%s/%s" % (base, resultBranch) + if location and ":" not in location.split("/")[0]: + location = os.path.abspath(location) + return location + +# Every simulator publishes its own results - .sim and diff files included - to +# the directory of its own branch; the .err of the build is shared, so each of +# them gets a copy of it. +for (resultBranch, simulator) in resultBranches: + result_location = outputFor(resultBranch) + suffix = artifactSuffix(simulator) if result_location != "" and (isWin or noSync): - print("--> copy res file of library: " + libname) - libPath = os.path.join(resRootPath, libname) - if not os.path.exists(libPath): - os.makedirs(libPath) - libFilesPath = os.path.join(libPath, 'files') - if os.path.exists(libFilesPath): - rmtree(libFilesPath) - os.makedirs(libFilesPath) + # Unlike the rsync path, this one is given the directory the branches live + # in and appends the branch itself. + resRootPath = os.path.join(jobOutput, resultBranch) + if os.path.exists(resRootPath): + rmtree(resRootPath) + os.makedirs(resRootPath) + + htmltpl=open("library.html.tpl").read() + for libname in stats_by_libname.keys(): + if libname in skipped_libs: + continue + s = None # Make sure I don't use this + stageRoot = stageRootFor(simulator, suffix) + filesList = open(os.path.join(stageRoot, libname + ".files"), "w") + filesList.write("/\n") + filesList.write("/%s.html\n" % libname) + filesList.write("/files/\n") + conf = stats_by_libname[libname]["conf"] + stats = [(n, m, l, dataForSimulator(d, simulator)) for (n, m, l, d) in stats_by_libname[libname]["stats"]] + for s in stats: + # What the simulator wrote in the workspace, and what it is published as: + # the branch of a simulator holds its results under the plain name. + workspace_prefix = "files/%s_%s%s" % (s[2],s[1],suffix) + filename_prefix = "files/%s_%s" % (s[2],s[1]) + stagePublished(stageRoot, workspace_prefix, filename_prefix, suffix) + stageShared(stageRoot, "files/%s_%s" % (s[2],s[1]), suffix) + filesList.write("/%s*diff*csv\n" % filename_prefix) + filesList.write("/%s*diff*html\n" % filename_prefix) + if is_non_zero_file(workspace_prefix+".sim"): + filesList.write("/%s.sim\n" % filename_prefix) + errPrefix = "files/%s_%s" % (s[2],s[1]) + if is_non_zero_file(errPrefix+".err"): + filesList.write("/%s.err\n" % errPrefix) + variables = (s[3].get("diff") or {}).get("vars") or [] + if len(variables)>0: + filesList.write("/%s.diff.html\n" % filename_prefix) + for v in variables: + filesList.write("/%s.diff.%s.csv\n" % (filename_prefix, v)) + filesList.write("/%s.diff.%s.html\n" % (filename_prefix, v)) + filesList.close() + testsHTML = "\n".join(['%s%s%s%s%s%s%s%s%s%s%s%s\n' % + (lambda filename_prefix, errPrefix, diff: + ( + ('%s' % (errPrefix + ".err", html.escape(s[1]))) if is_non_zero_file(errPrefix + ".err") else html.escape(s[1]), + (' (sim)' % (filename_prefix + ".sim")) if is_non_zero_file(filename_prefix + suffix + ".sim") else "", + checkPhase(s[3]["phase"], 7) if s[3]["phase"]>=6 else "#FFFFFF", + ("%s (%d verified)" % (timeSeconds(diff.get("time")), diff.get("numCompared"))) if s[3]["phase"]>=7 else (" " if diff is None else + ('%s (%d/%d failed)' % (timeSeconds(diff.get("time")), filename_prefix, len(diff.get("vars")), diff.get("numCompared")))), + checkPhase(s[3]["phase"], 6), + timeSeconds(s[3].get("sim") or 0), + checkPhase(s[3]["phase"], 5), + timeSeconds(sum(s[3].get(x) or 0.0 for x in ["frontend","backend","simcode","templates","build"])), + timeSeconds(s[3].get("parsing") or 0), + checkPhase(s[3]["phase"], 1), + timeSeconds(s[3].get("frontend") or 0), + checkPhase(s[3]["phase"], 2), + timeSeconds(s[3].get("backend") or 0), + checkPhase(s[3]["phase"], 3), + timeSeconds(s[3].get("simcode") or 0), + checkPhase(s[3]["phase"], 4), + timeSeconds(s[3].get("templates") or 0), + checkPhase(s[3]["phase"], 5), + timeSeconds(s[3].get("build") or 0), + timeSeconds(s[3].get("exectime") or 0) + ))(filename_prefix="files/%s_%s" % (s[2], s[1]), errPrefix="files/%s_%s" % (s[2], s[1]), diff=s[3].get("diff")) + for s in natsorted(stats, key=lambda s: s[1])]) + numSucceeded = [len(stats)] + [sum(1 if s[3]["phase"]>=i else 0 for s in stats) for i in range(1,8)] + try: - if clean: - shutil.move("./" + libname + ".html", libPath) + githuburltesting = "https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit/" + gitloglibrarytesting = check_output_log(["git", "log", '--pretty=
    CommitDateAuthorSummary
    %%h%%ai%%an%%s
    ' % (githuburltesting), "-1"], cwd="./").decode("utf-8") + except subprocess.CalledProcessError as e: + print(str(e)) + gitloglibrarytesting = "
    could not get the git log for OpenModelicaLibraryTesting
    " + + # adrpo: attempt to get the revision of the reference files if possible + if conf.get("referenceFiles"): + c = conf.get("referenceFiles") + if DEBUG: + print("referenceFiles git ... attempting to retrieve info from directory: %s" % c) + sys.stdout.flush() + gitReferenceFiles = c + # see if we have a commit file + f = os.path.join(c, "commit") + if os.path.exists(f): + with open(f) as fin: + gitReferenceFilesVersion = fin.read() + print("referenceFiles git ... read from file %s" % f) + sys.stdout.flush() else: - shutil.copy2("./" + libname + ".html", libPath) - except: - print("-- problem durin copy/move of html file of lib: " + libname) - - for file in glob.glob("./files/" + libname + '*.err') \ - + glob.glob("./files/" + libname + '*.sim') \ - + glob.glob("./files/" + libname + '*.csv') \ - + glob.glob("./files/" + libname + '*.json') \ - + glob.glob("./files/" + libname + '*.html'): + try: + if isinstance(c, (str, bytes)): + if DEBUG: + print("referenceFiles git ... see if directory has an evironment variable") + sys.stdout.flush() + m = re.search("^[$][A-Z_]+", c) + if m: + k = m.group(0)[1:] + if k not in os.environ: + if DEBUG: + print("referenceFiles git ... environment variable used in the directory cannot be found in the environment: %s" % k) + sys.stdout.flush() + raise Exception("Environment variable %s not defined, but used in JSON config for reference files" % k) + gitReferenceFiles = c.replace(m.group(0), os.environ[k]) + if DEBUG: + print("referenceFiles git ... directory after replacing the environment variable: %s" % gitReferenceFiles) + sys.stdout.flush() + sys.stdout.flush() + try: + gitReferenceFilesURL = check_output_log(["git", "config", "--get", "remote.origin.url"], cwd=gitReferenceFiles).decode("utf-8") + except subprocess.CalledProcessError as e: + print(e) + gitReferenceFilesURL = gitReferenceFiles + gitReferenceFilesVersion = check_output_log(["git", "log", '--pretty=
    CommitDateAuthorSummary
    %%h%%ai%%an%%s
    ' % (gitReferenceFilesURL), "-1"], cwd=gitReferenceFiles).decode("utf-8") + print("referenceFiles git ... got version information: %s" % gitReferenceFilesVersion) + sys.stdout.flush() + except subprocess.CalledProcessError as e: + print("referenceFiles git ... something went wrong with getting the git info for directory: %s" % c) + print(str(e)) + sys.stdout.flush() + gitReferenceFilesVersion = "" + else: + gitReferenceFilesVersion = "" + + replacements = ( + (u"#sysInfo#", html.escape(sysInfo)), + (u"#omcVersion#", html.escape(omc_version)), + (u"#fmiToolVersion#", ("

    "+html.escape("FMI tool: %s" % fmisimulatorversion)+"

    ") if fmisimulatorversion else ""), + (u"#fmi#", ("

    "+html.escape("FMI version: %s" % conf.get("fmi"))+"

    ") if conf.get("fmi") else ""), + (u"#optlevel#", html.escape(conf.get("optlevel")) if (canChangeOptLevel and conf.get("optlevel")) else "Tool default"), + (u"#timeStart#", html.escape(time.strftime('%Y-%m-%d %H:%M:%S', start_as_time))), + (u"#fileName#", html.escape(libname)), + (u"#customCommands#", html.escape("\n".join(conf["customCommands"]))), + (u"#libraryVersionRevision#", html.escape(conf["libraryVersionRevision"])), + (u"#OpenModelicaLibraryTesting#", gitloglibrarytesting), + (u"#metadata#", html.escape(conf["metadata"])), + (u"#ulimitOmc#", html.escape(str(conf["ulimitOmc"]))), + (u"#ulimitExe#", html.escape(str(conf["ulimitExe"]))), + (u"#defaultTolerance#", html.escape(str(conf["defaultTolerance"]))), + (u"#defaultNumberOfIntervals#", html.escape(str(conf["defaultNumberOfIntervals"]))), + (u"#simFlags#", html.escape(conf.get("simFlags") or "")), + (u"#referenceFiles#", ('

    Reference Files: %s

    %s' % ((conf["referenceFilesURL"].replace(os.path.dirname(os.path.realpath(__file__)),"")), gitReferenceFilesVersion)) if ((conf.get("referenceFilesURL") or "") != "") else ""), + (u"#referenceTool#", ('

    Verified using: %s (diffSimulationResults)

    ' % html.escape(ompython_omc_version)) if ((conf.get("referenceFiles") or "") != "") else ""), + (u"#Total#", html.escape(str(numSucceeded[0]))), + (u"#FrontendColor#", checkNumSucceeded(numSucceeded, 1)), + (u"#BackendColor#", checkNumSucceeded(numSucceeded, 2)), + (u"#SimCodeColor#", checkNumSucceeded(numSucceeded, 3)), + (u"#TemplatesColor#", checkNumSucceeded(numSucceeded, 4)), + (u"#CompilationColor#", checkNumSucceeded(numSucceeded, 5)), + (u"#SimulationColor#", checkNumSucceeded(numSucceeded, 6)), + (u"#VerificationColor#", checkNumSucceeded(numSucceeded, 7)), + (u"#Frontend#", html.escape(str(numSucceeded[1]))), + (u"#Backend#", html.escape(str(numSucceeded[2]))), + (u"#SimCode#", html.escape(str(numSucceeded[3]))), + (u"#Templates#", html.escape(str(numSucceeded[4]))), + (u"#Compilation#", html.escape(str(numSucceeded[5]))), + (u"#Simulation#", html.escape(str(numSucceeded[6]))), + (u"#Verification#", html.escape(str(numSucceeded[7]))), + (u"#totalTime#", html.escape(str(datetime.timedelta(seconds=int(sum(s[3].get("exectime") or 0.0 for s in stats)))))), + (u"#config#", html.escape(json.dumps(conf["configFromFile"], indent=1, sort_keys=True))), + (u"#testsHTML#", testsHTML) + ) + open(os.path.join(stageRoot, "%s.html" % libname), "w").write(multiple_replace(htmltpl, *replacements)) + + # move results by sync operations (not available under win) + if result_location != "" and not isWin and not noSync: + result_location_libname = "%s/%s" % (result_location, libname) try: - print("copy: " + file) - shutil.copy2(file, libFilesPath) + os.mkdir(os.path.join(stageRoot, "emptydir")) except: - print("-- problem during file copy... maybe the file is still hooked by a process... :" + file) pass + check_output_log(["rsync", "-aR", "emptydir/", result_location], cwd=stageRoot) + check_output_log(["rsync", "-aR", "emptydir/", result_location_libname], cwd=stageRoot) + check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"], cwd=stageRoot) + try: + check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname], cwd=stageRoot) + except: + check_output_log(["rsync", "-aR", "emptydir/", result_location], cwd=stageRoot) + check_output_log(["rsync", "-aR", "emptydir/", result_location_libname], cwd=stageRoot) + check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"], cwd=stageRoot) + check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname], cwd=stageRoot) + if (conf.get("referenceFiles") or "") != "" and dygraphs: + check_output_log(["rsync", "-a", dygraphs, result_location_libname+"/files"]) + else: + print("No Sync: result_location [%s] != "" and not isWin [%s] and not noSync [%s] : library: %s" % (result_location, isWin, noSync, libname)) + + # move results without sync operations (win or when --noSync is used) + if result_location != "" and (isWin or noSync): + print("--> copy res file of library: " + libname) + libPath = os.path.join(resRootPath, libname) + if not os.path.exists(libPath): + os.makedirs(libPath) + libFilesPath = os.path.join(libPath, 'files') + if os.path.exists(libFilesPath): + rmtree(libFilesPath) + os.makedirs(libFilesPath) + try: + if clean: + shutil.move("./" + libname + ".html", libPath) + else: + shutil.copy2("./" + libname + ".html", libPath) + except: + print("-- problem durin copy/move of html file of lib: " + libname) + + for file in glob.glob("./files/" + libname + '*.err') \ + + glob.glob("./files/" + libname + '*.sim') \ + + glob.glob("./files/" + libname + '*.csv') \ + + glob.glob("./files/" + libname + '*.json') \ + + glob.glob("./files/" + libname + '*.html'): + try: + print("copy: " + file) + shutil.copy2(file, libFilesPath) + except: + print("-- problem during file copy... maybe the file is still hooked by a process... :" + file) + pass if clean: for g in ["*.o","*.so","*.h","*.c","*.cpp","*.simsuccess","*.conf.json","*.tmpfiles","*.log","*.libs","OMCpp*","*.fmu*","temp_*", "*.exe", "HelloWorld.bat", "*.makefile", "*.mat","*.xml", "*.bin", "*.json"]: From df848088547e700585277f1e5a85f3da0d548002 Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Tue, 11 Aug 2026 14:41:33 +0200 Subject: [PATCH 5/6] [FMI] Run each pair of FMI jobs as one job The last part of #78. The three pairs of FMI jobs - v1.26-fmi and v1.26-fmi-fmpy, v1.27-fmi and v1.27-fmi-fmpy, master-fmi and master-fmi-fmpy - are one job each now. Six stages become three, and the FMUs of a release are built once instead of twice. The parameters keep their meaning: fmi_v1_27 asks for OMSimulator, fmpy_fmi_v1_27 for FMPy, and ticking both runs one job that builds every FMU once and simulates it with both, filling v1.27-fmi and v1.27-fmi-fmpy as the two jobs did. Ticking one runs that tool alone and fills only its table, and its results and files still go where they always went: a job given FMPy alone stores them in v1.27-fmi-fmpy and publishes them to branches/v1.27-fmi-fmpy, not to the branch it was started with. Which simulators a job runs is now something it is told rather than something guessed from its name, but a job that says nothing still gets the tool its name implies, so the cs-fmu-cvode jobs and every job that does not test FMI are untouched. OMSimulator is only cloned and built when a job actually asks for it. Checked here on ExternData, for all three ways of asking: OMSimulator alone fills v1.27-fmi, FMPy alone fills v1.27-fmi-fmpy, both fill the two of them from a single build, and the results of a merged run are the same as those of the two separate runs. --- Generated by Claude Code. Signed-off-by: Adrian Pop --- .CI/Jenkinsfile | 118 ++++++++++++++++++------------------------------ test.py | 73 ++++++++++++++++++------------ 2 files changed, 86 insertions(+), 105 deletions(-) diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index 7734e5e..5d99a79 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -7,17 +7,17 @@ pipeline { booleanParam(name: 'v1_27', defaultValue: false, description: 'maintenance/v1.27 branch (ryzen-5950x-1)') booleanParam(name: 'master', defaultValue: false, description: 'master branch (ryzen-5950x-1)') - booleanParam(name: 'fmi_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with FMI (ryzen-5950x-2)') - booleanParam(name: 'fmi_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with FMI (ryzen-5950x-2)') - booleanParam(name: 'fmi_master', defaultValue: false, description: 'master branch with FMI running OMSimulator (ryzen-5950x-2)') + booleanParam(name: 'fmi_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with FMI, simulated by OMSimulator, filling v1.26-fmi (ryzen-5950x-2). Ticking it together with fmpy_fmi_v1_26 runs one job that builds every FMU once and simulates it with both.') + booleanParam(name: 'fmi_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with FMI, simulated by OMSimulator, filling v1.27-fmi (ryzen-5950x-2). Ticking it together with fmpy_fmi_v1_27 runs one job that builds every FMU once and simulates it with both.') + booleanParam(name: 'fmi_master', defaultValue: false, description: 'master branch with FMI, simulated by OMSimulator, filling master-fmi (ryzen-5950x-2). Ticking it together with fmpy_fmi_master runs one job that builds every FMU once and simulates it with both.') booleanParam(name: 'cs_fmu_cvode_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with CVODE CS FMUs running OMSimulator (ryzen-5950x-2)') booleanParam(name: 'cs_fmu_cvode_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with CVODE CS FMUs running OMSimulator (ryzen-5950x-2)') booleanParam(name: 'cs_fmu_cvode_master', defaultValue: false, description: 'master branch with CVODE CS FMUs running OMSimulator (ryzen-5950x-2)') - booleanParam(name: 'fmpy_fmi_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with FMI (ryzen-5950x-2)') - booleanParam(name: 'fmpy_fmi_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with FMI (ryzen-5950x-2)') - booleanParam(name: 'fmpy_fmi_master', defaultValue: false, description: 'master branch with FMI running FMPy (ryzen-5950x-2)') + booleanParam(name: 'fmpy_fmi_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with FMI, simulated by FMPy, filling v1.26-fmi-fmpy (ryzen-5950x-2). Ticking it together with fmi_v1_26 runs one job that builds every FMU once and simulates it with both.') + booleanParam(name: 'fmpy_fmi_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with FMI, simulated by FMPy, filling v1.27-fmi-fmpy (ryzen-5950x-2). Ticking it together with fmi_v1_27 runs one job that builds every FMU once and simulates it with both.') + booleanParam(name: 'fmpy_fmi_master', defaultValue: false, description: 'master branch with FMI, simulated by FMPy, filling master-fmi-fmpy (ryzen-5950x-2). Ticking it together with fmi_master runs one job that builds every FMU once and simulates it with both.') booleanParam(name: 'newInst_daeMode', defaultValue: false, description: 'master branch, --daeMode with -d=newInst (ryzen-5950x-2)') booleanParam(name: 'newInst_newBackend', defaultValue: false, description: 'master branch, -d=newInst --newBackend, (ryzen-5950x-1)') @@ -129,7 +129,7 @@ pipeline { } } - stage('v1.26 FMI with OMSimulator') { + stage('v1.26 FMI') { agent { node { label 'ryzen-5950x-2-1' @@ -139,13 +139,13 @@ pipeline { options { skipDefaultCheckout() } when { beforeAgent true - expression { params.fmi_v1_26 } + expression { params.fmi_v1_26 || params.fmpy_fmi_v1_26 } } steps { - runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) + runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false, 0, 'configs/conf.json', '', '', fmiSimulators(params.fmi_v1_26, params.fmpy_fmi_v1_26)) } } - stage('v1.27 FMI with OMSimulator') { + stage('v1.27 FMI') { agent { node { label 'ryzen-5950x-2-1' @@ -155,13 +155,13 @@ pipeline { options { skipDefaultCheckout() } when { beforeAgent true - expression { params.fmi_v1_27 } + expression { params.fmi_v1_27 || params.fmpy_fmi_v1_27 } } steps { - runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) + runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false, 0, 'configs/conf.json', '', '', fmiSimulators(params.fmi_v1_27, params.fmpy_fmi_v1_27)) } } - stage('master FMI with OMSimulator') { + stage('master FMI') { agent { node { label 'ryzen-5950x-2-1' @@ -171,10 +171,10 @@ pipeline { options { skipDefaultCheckout() } when { beforeAgent true - expression { params.fmi_master } + expression { params.fmi_master || params.fmpy_fmi_master } } steps { - runRegressiontest('master', 'master-fmi', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) + runRegressiontest('master', 'master-fmi', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false, 0, 'configs/conf.json', '', '', fmiSimulators(params.fmi_master, params.fmpy_fmi_master)) } } @@ -227,57 +227,6 @@ pipeline { } } - stage('v1.26 FMI with FMPy') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.fmpy_fmi_v1_26 } - } - steps { - runRegressiontest('maintenance/v1.26', 'v1.26-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) - } - } - - stage('v1.27 FMI with FMPy') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.fmpy_fmi_v1_27 } - } - steps { - runRegressiontest('maintenance/v1.27', 'v1.27-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) - } - } - - stage('master FMI with FMPy') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.fmpy_fmi_master } - } - steps { - runRegressiontest('master', 'master-fmi-fmpy', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) - } - } - stage('newInst-daeMode') { agent { node { @@ -763,7 +712,17 @@ def sccachePreamble() { * `.CI/wasm-jit`. If non-empty, the omc build and test.py run in that image * instead of on the node; everything using the node's own omc stays outside. */ -def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json', cmakeFlags = '', dockerfile = '') { +/* The FMI simulators a job runs, from the parameters that used to start one job each. + * Both ticked is one job that builds every FMU once and simulates it with both, + * filling -fmi and -fmi-fmpy exactly as the two jobs did. */ +def fmiSimulators(boolean omsimulator, boolean fmpy) { + def simulators = [] + if (omsimulator) simulators << 'OMSimulator' + if (fmpy) simulators << 'fmpy' + return simulators +} + +def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json', cmakeFlags = '', dockerfile = '', fmiSimulators = null) { sh ''' find /tmp -name "*openmodelica.hudson*" -exec rm {} ";" || true mkdir -p ~/TEST_LIBS_BACKUP @@ -814,8 +773,13 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om } } - FMI_TESTING_FLAG="" - if (!name.contains('fmpy') && omsHash) { + // A job that does not say which simulators it wants gets the one its name implies. + def simulators = fmiSimulators + if (simulators == null) { + simulators = name.contains('fmpy') ? ['fmpy'] : (omsHash ? ['OMSimulator'] : []) + } + FMI_TESTING_FLAG = "" + if (simulators.contains('OMSimulator') && omsHash) { sh """ if ! test -d OMSimulator; then git clone --recursive https://openmodelica.org/git-readonly/OMSimulator.git || exit 1 @@ -848,19 +812,23 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om echo OMSimulator version: ${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator --version """ - FMI_TESTING_FLAG="--fmi=true --fmisimulator=${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator --default=ulimitExe=50" - if (name.contains('cvode')) { - FMI_TESTING_FLAG += " --fmuType=cs" - } + FMI_TESTING_FLAG = " --fmisimulator=${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator" } - if (name.contains('fmpy')) { + if (simulators.contains('fmpy')) { sh """ # update fmpy pip install FMPy || true python3 -m fmpy -h || exit 1 """ - FMI_TESTING_FLAG="--fmi=true --fmisimulator='python3 -m fmpy' --default=ulimitExe=50" + FMI_TESTING_FLAG += " --fmisimulator='python3 -m fmpy'" + } + + if (FMI_TESTING_FLAG) { + FMI_TESTING_FLAG = "--fmi=true${FMI_TESTING_FLAG} --default=ulimitExe=50" + if (name.contains('cvode')) { + FMI_TESTING_FLAG += " --fmuType=cs" + } } OMCPATH = "${omcompiler ? '../' : './'}OMCompiler" diff --git a/test.py b/test.py index 63806b3..affc5e3 100755 --- a/test.py +++ b/test.py @@ -70,6 +70,44 @@ fmisimulators = shared.parseFmiSimulators(args.fmisimulator) # The first simulator is the one the single-simulator code paths use. fmisimulator = fmisimulators[0][1] if fmisimulators else None + +# One branch per simulator. The first reports itself in the results of the +# model and the others under their own name, but every one of them stores its +# results where its own simulator belongs: a job given only FMPy on +# --branch=v1.27-fmi fills v1.27-fmi-fmpy, not v1.27-fmi. +resultBranches = [(branch, None)] +if fmisimulators: + resultBranches = [(shared.branchForSimulator(branch, fmisimulators[0][0]), None)] + for (simulatorName, _) in fmisimulators[1:]: + resultBranches.append((shared.branchForSimulator(branch, simulatorName), simulatorName)) +# What the run asks about when it looks for results it already has. +primaryBranch = resultBranches[0][0] + +jobOutput = result_location + +def outputFor(resultBranch): + """Where a branch publishes. + + --output names the directory of the job, which ends in the branch the job was + started with, so every simulator gets the directory of its own branch beside + it: a job given only FMPy on --branch=v1.27-fmi publishes v1.27-fmi-fmpy, the + same branch its results are stored under. + + rsync runs from the directory a branch is staged in, so a local destination + has to be absolute; a remote one, host:path, already is. + """ + location = jobOutput + if jobOutput: + base = jobOutput.rstrip("/") + if base.endswith("/" + branch): + location = "%s%s" % (base[:-len(branch)], resultBranch) + elif resultBranch != primaryBranch: + location = "%s/%s" % (base, resultBranch) + if location and ":" not in location.split("/")[0]: + location = os.path.abspath(location) + return location + + allTestsFmi = args.fmi fmuType = args.fmuType ulimitMemory = args.ulimitvmem @@ -252,8 +290,8 @@ def target(): omc_cmd = [os.path.normpath(os.path.join(os.environ.get("OPENMODELICAHOME"), 'bin', 'omc'))] else: omc_cmd = ["omc"] -if result_location != "" and not os.path.exists(result_location): - os.makedirs(result_location) +if result_location != "" and not os.path.exists(outputFor(primaryBranch)): + os.makedirs(outputFor(primaryBranch)) if configs == []: print("Error: Expected at least one configuration file to start the library test") @@ -550,7 +588,7 @@ def createBranchTable(branch): verify real NOT NULL, verifyfail integer NOT NULL, verifytotal integer NOT NULL, finalphase integer NOT NULL, parsing real NOT NULL)''' % branch) cursor.execute('''DROP INDEX IF EXISTS [idx_%s_date]''' % branch) -createBranchTable(branch) +createBranchTable(primaryBranch) cursor.execute('''DROP INDEX IF EXISTS idx_omcversion_date''') cursor.execute('''DROP INDEX IF EXISTS idx_libversion_date''') @@ -741,7 +779,7 @@ def hashReferenceFiles(s): res=list(filter(lambda x: not x.startswith(prefix), res)) libName=shared.libname(library, conf) v = cursor.execute("""SELECT date,libversion,libname,branch,omcversion FROM [libversion] NATURAL JOIN [omcversion] - WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1""", (conf["libraryLastChange"],libName,branch,omc_version,confighash)).fetchone() + WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1""", (conf["libraryLastChange"],libName,primaryBranch,omc_version,confighash)).fetchone() if libName in stats_by_libname or libName in skipped_libs: raise Exception("Duplicate libName found: %s" % libName) if v is None or execAllTests: @@ -861,7 +899,7 @@ def expectedExec(c): (model,lib,libName,name,data) = c if "expectedExec" in data: return data["expectedExec"] - cursor.execute("SELECT exectime FROM [%s] WHERE libname = ? AND model = ? ORDER BY date DESC LIMIT 1" % branch, (libName,model)) + cursor.execute("SELECT exectime FROM [%s] WHERE libname = ? AND model = ? ORDER BY date DESC LIMIT 1" % primaryBranch, (libName,model)) v = cursor.fetchone() data["expectedExec"] = (v or (0.0,))[0] return data["expectedExec"] @@ -964,12 +1002,6 @@ def resultValues(model, libname, data, simulator=None): data.get("parsing") or 0.0 ) -# One branch per FMI simulator: the first one reports itself in the results of -# the model, the others under their own name, see testmodel.py. -resultBranches = [(branch, None)] -for (simulatorName, _) in fmisimulators[1:]: - resultBranches.append((shared.branchForSimulator(branch, simulatorName), simulatorName)) - for (resultBranch, simulator) in resultBranches: createBranchTable(resultBranch) for key in stats.keys(): @@ -1090,25 +1122,6 @@ def artifactSuffix(simulator): """What tells the files of one simulator from those of another.""" return "_%s" % simulator if simulator and len(fmisimulators) > 1 else "" -jobOutput = result_location - -def outputFor(resultBranch): - """Where a branch publishes; --output names the directory of the job. - - rsync runs from the directory a branch is staged in, so a local destination - has to be absolute; a remote one, host:path, already is. - """ - location = jobOutput - if jobOutput and resultBranch != branch: - base = jobOutput.rstrip("/") - if base.endswith("/" + branch): - location = "%s%s" % (base[:-len(branch)], resultBranch) - else: - location = "%s/%s" % (base, resultBranch) - if location and ":" not in location.split("/")[0]: - location = os.path.abspath(location) - return location - # Every simulator publishes its own results - .sim and diff files included - to # the directory of its own branch; the .err of the build is shared, so each of # them gets a copy of it. From 1f8b4588e92a0aac782341bb5065b494894bd22d Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Tue, 11 Aug 2026 14:50:39 +0200 Subject: [PATCH 6/6] [FMI] Document testing one FMU with several simulators Two sections in the README: how to run a job with more than one FMI simulator, what each of them fills and where it is published, and what may differ between results that share an FMU; and how to add a simulator, which is an entry in configs/fmi-simulators.json and no change to any script. The description of every key of an entry is there, including why a step size flag may have to disappear rather than be passed empty, and what a Python package without a command line needs. --- Generated by Claude Code. Signed-off-by: Adrian Pop --- README.md | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b3a94b4..51faab0 100644 --- a/README.md +++ b/README.md @@ -158,8 +158,12 @@ Options: - `--ompython_omhome=''`: Path to OpenModelica for OMPython (can be different to the OM running the tests) - `--noclean=False`: Clean (most) generated files. -- `--fmisimulator=''`: The default is nothing but you can use the path to - OMSimulator executable or 'fmpy' +- `--fmisimulator=''`: The FMI simulator to run the FMUs with, as `name=command` + or just the command, e.g. the path to the OMSimulator + executable or `'python3 -m fmpy'`. Repeat the option to + simulate every FMU with several tools without building it + more than once, see [Testing FMI with several + simulators](#testing-fmi-with-several-simulators) - `--ulimitvmem=8388608`: Virtual memory limit (in kB) - `--default=[]`: Add a default value for some configuration key, such as `--default=ulimitExe=60`. The equals sign is mandatory @@ -168,6 +172,89 @@ Options: `procCCompile` above for more insight into individual test parallelization. +### Testing FMI with several simulators + +Building an FMU costs far more than simulating it. Measured on the twelve +models of ExternData: 178 seconds building the FMUs, 0.7 simulating them with +OMSimulator and 2.6 with FMPy. Testing the same FMUs with a second tool +therefore used to cost almost twice as much as testing them with one, because +each job built its own copy of them. + +Give `--fmisimulator` once per tool and the FMUs are built once and simulated +with each of them: + +```bash +./test.py --branch=v1.27-fmi --fmi=true \ + --fmisimulator=/path/to/OMSimulator \ + --fmisimulator='python3 -m fmpy' \ + configs/myConf.json +``` + +`--branch` names the job; every simulator stores its results in a branch of its +own derived from it, so the run above fills + +| simulator | branch | published to | +| --- | --- | --- | +| OMSimulator | `v1.27-fmi` | `branches/v1.27-fmi` | +| FMPy | `v1.27-fmi-fmpy` | `branches/v1.27-fmi-fmpy` | + +which is where those results have always been. OMSimulator keeps the plain +`-fmi` branch; every other tool adds its name. The branch a tool fills depends +on the tool and not on the order, so asking for FMPy alone still fills +`v1.27-fmi-fmpy` and leaves `v1.27-fmi` alone. + +A branch directory looks the same as it always did, file names included. The +`.err` of a model is written by the build, so every simulator of it publishes +the same one; the `.sim` and the difference files are the ones that simulator +produced. + +Only the simulator may differ between the results that share an FMU. Anything +that changes the FMU itself - a different compiler, a different library, a +different `--fmuType` or `--fmiFlags` - is a different job, which is why the +Co-Simulation jobs with CVODE are not merged with the Model Exchange ones. + +In Jenkins the parameters keep their meaning: `fmi_v1_27` asks for OMSimulator +and `fmpy_fmi_v1_27` for FMPy. Ticking both runs one job that builds every FMU +once and simulates it with both; ticking one runs that tool alone. + +### Adding an FMI simulator + +The simulators live in [configs/fmi-simulators.json](configs/fmi-simulators.json). +Adding one is an entry there and no change to any script: + +```json +"fmusim": { + "resultExtension": "csv", + "versionArgument": "--version", + "optionalArguments": { "stepSizeArgument": " --output-interval {stepSize:g}" }, + "arguments": "--interface-type ModelExchange --output-file {result} --start-time {startTime:g} --stop-time {stopTime:g}{stepSizeArgument} {fmu}" +} +``` + +- `arguments` is the command line, a template over `simulator`, `fmu`, + `result`, `requestedResult`, `tempDir`, `startTime`, `stopTime`, `tolerance`, + `timeout`, `stepSize` and anything named in `optionalArguments`. +- `optionalArguments` are the flags that have to disappear when there is nothing + to put in them. OMSimulator hangs on `--stepSize=0` rather than ignoring it, + so its step size flag lives here, while FMPy wants `--output-interval 0` all + the same and writes the value straight into its `arguments`. +- `command` is how the tool is invoked, `{simulator}` by default. FMPy needs a + subcommand, `{simulator} simulate`. +- `resultExtension` is what the tool writes, so that the comparison against the + reference file knows what to read. +- `versionArgument` prints the version, which is recorded with the results. +- `branchSuffix` overrides the `-` a tool adds to the branch. Only + OMSimulator needs it, with `""`. +- `untested` marks an entry nobody has run yet; the run then says so instead of + failing every model with a puzzling error. Remove it once it works. + +Then run it with `--fmisimulator=fmusim=/path/to/fmusim`, or just +`--fmisimulator=/path/to/fmusim` if the command contains the name. + +A tool that is a Python package rather than a command line needs a small driver +script that takes the arguments its entry passes, simulates, writes the result +file and exits non-zero when it fails; the entry then points `command` at it. + ### Generate HTML results ```bash