diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index 7734e5e..5d99a79 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -7,17 +7,17 @@ pipeline { booleanParam(name: 'v1_27', defaultValue: false, description: 'maintenance/v1.27 branch (ryzen-5950x-1)') booleanParam(name: 'master', defaultValue: false, description: 'master branch (ryzen-5950x-1)') - booleanParam(name: 'fmi_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with FMI (ryzen-5950x-2)') - booleanParam(name: 'fmi_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with FMI (ryzen-5950x-2)') - booleanParam(name: 'fmi_master', defaultValue: false, description: 'master branch with FMI running OMSimulator (ryzen-5950x-2)') + booleanParam(name: 'fmi_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with FMI, simulated by OMSimulator, filling v1.26-fmi (ryzen-5950x-2). Ticking it together with fmpy_fmi_v1_26 runs one job that builds every FMU once and simulates it with both.') + booleanParam(name: 'fmi_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with FMI, simulated by OMSimulator, filling v1.27-fmi (ryzen-5950x-2). Ticking it together with fmpy_fmi_v1_27 runs one job that builds every FMU once and simulates it with both.') + booleanParam(name: 'fmi_master', defaultValue: false, description: 'master branch with FMI, simulated by OMSimulator, filling master-fmi (ryzen-5950x-2). Ticking it together with fmpy_fmi_master runs one job that builds every FMU once and simulates it with both.') booleanParam(name: 'cs_fmu_cvode_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with CVODE CS FMUs running OMSimulator (ryzen-5950x-2)') booleanParam(name: 'cs_fmu_cvode_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with CVODE CS FMUs running OMSimulator (ryzen-5950x-2)') booleanParam(name: 'cs_fmu_cvode_master', defaultValue: false, description: 'master branch with CVODE CS FMUs running OMSimulator (ryzen-5950x-2)') - booleanParam(name: 'fmpy_fmi_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with FMI (ryzen-5950x-2)') - booleanParam(name: 'fmpy_fmi_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with FMI (ryzen-5950x-2)') - booleanParam(name: 'fmpy_fmi_master', defaultValue: false, description: 'master branch with FMI running FMPy (ryzen-5950x-2)') + booleanParam(name: 'fmpy_fmi_v1_26', defaultValue: false, description: 'maintenance/v1.26 branch with FMI, simulated by FMPy, filling v1.26-fmi-fmpy (ryzen-5950x-2). Ticking it together with fmi_v1_26 runs one job that builds every FMU once and simulates it with both.') + booleanParam(name: 'fmpy_fmi_v1_27', defaultValue: false, description: 'maintenance/v1.27 branch with FMI, simulated by FMPy, filling v1.27-fmi-fmpy (ryzen-5950x-2). Ticking it together with fmi_v1_27 runs one job that builds every FMU once and simulates it with both.') + booleanParam(name: 'fmpy_fmi_master', defaultValue: false, description: 'master branch with FMI, simulated by FMPy, filling master-fmi-fmpy (ryzen-5950x-2). Ticking it together with fmi_master runs one job that builds every FMU once and simulates it with both.') booleanParam(name: 'newInst_daeMode', defaultValue: false, description: 'master branch, --daeMode with -d=newInst (ryzen-5950x-2)') booleanParam(name: 'newInst_newBackend', defaultValue: false, description: 'master branch, -d=newInst --newBackend, (ryzen-5950x-1)') @@ -129,7 +129,7 @@ pipeline { } } - stage('v1.26 FMI with OMSimulator') { + stage('v1.26 FMI') { agent { node { label 'ryzen-5950x-2-1' @@ -139,13 +139,13 @@ pipeline { options { skipDefaultCheckout() } when { beforeAgent true - expression { params.fmi_v1_26 } + expression { params.fmi_v1_26 || params.fmpy_fmi_v1_26 } } steps { - runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) + runRegressiontest('maintenance/v1.26', 'v1.26-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false, 0, 'configs/conf.json', '', '', fmiSimulators(params.fmi_v1_26, params.fmpy_fmi_v1_26)) } } - stage('v1.27 FMI with OMSimulator') { + stage('v1.27 FMI') { agent { node { label 'ryzen-5950x-2-1' @@ -155,13 +155,13 @@ pipeline { options { skipDefaultCheckout() } when { beforeAgent true - expression { params.fmi_v1_27 } + expression { params.fmi_v1_27 || params.fmpy_fmi_v1_27 } } steps { - runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) + runRegressiontest('maintenance/v1.27', 'v1.27-fmi', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false, 0, 'configs/conf.json', '', '', fmiSimulators(params.fmi_v1_27, params.fmpy_fmi_v1_27)) } } - stage('master FMI with OMSimulator') { + stage('master FMI') { agent { node { label 'ryzen-5950x-2-1' @@ -171,10 +171,10 @@ pipeline { options { skipDefaultCheckout() } when { beforeAgent true - expression { params.fmi_master } + expression { params.fmi_master || params.fmpy_fmi_master } } steps { - runRegressiontest('master', 'master-fmi', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) + runRegressiontest('master', 'master-fmi', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false, 0, 'configs/conf.json', '', '', fmiSimulators(params.fmi_master, params.fmpy_fmi_master)) } } @@ -227,57 +227,6 @@ pipeline { } } - stage('v1.26 FMI with FMPy') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.fmpy_fmi_v1_26 } - } - steps { - runRegressiontest('maintenance/v1.26', 'v1.26-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) - } - } - - stage('v1.27 FMI with FMPy') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.fmpy_fmi_v1_27 } - } - steps { - runRegressiontest('maintenance/v1.27', 'v1.27-fmi-fmpy', '', omsimulatorHash(), 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) - } - } - - stage('master FMI with FMPy') { - agent { - node { - label 'ryzen-5950x-2-1' - customWorkspace 'ws/OpenModelicaLibraryTestingWork' - } - } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { params.fmpy_fmi_master } - } - steps { - runRegressiontest('master', 'master-fmi-fmpy', '', 'origin/master', 'ripper2', 'LibraryTestingRipper2DB', false, '', '', false, false) - } - } - stage('newInst-daeMode') { agent { node { @@ -763,7 +712,17 @@ def sccachePreamble() { * `.CI/wasm-jit`. If non-empty, the omc build and test.py run in that image * instead of on the node; everything using the node's own omc stays outside. */ -def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json', cmakeFlags = '', dockerfile = '') { +/* The FMI simulators a job runs, from the parameters that used to start one job each. + * Both ticked is one job that builds every FMU once and simulates it with both, + * filling -fmi and -fmi-fmpy exactly as the two jobs did. */ +def fmiSimulators(boolean omsimulator, boolean fmpy) { + def simulators = [] + if (omsimulator) simulators << 'OMSimulator' + if (fmpy) simulators << 'fmpy' + return simulators +} + +def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, omcompiler, extrasimflags, testFlags, boolean removePackageOrder, boolean conversionScript, int jobs=0, libs_config_file = 'configs/conf.json', cmakeFlags = '', dockerfile = '', fmiSimulators = null) { sh ''' find /tmp -name "*openmodelica.hudson*" -exec rm {} ";" || true mkdir -p ~/TEST_LIBS_BACKUP @@ -814,8 +773,13 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om } } - FMI_TESTING_FLAG="" - if (!name.contains('fmpy') && omsHash) { + // A job that does not say which simulators it wants gets the one its name implies. + def simulators = fmiSimulators + if (simulators == null) { + simulators = name.contains('fmpy') ? ['fmpy'] : (omsHash ? ['OMSimulator'] : []) + } + FMI_TESTING_FLAG = "" + if (simulators.contains('OMSimulator') && omsHash) { sh """ if ! test -d OMSimulator; then git clone --recursive https://openmodelica.org/git-readonly/OMSimulator.git || exit 1 @@ -848,19 +812,23 @@ def runRegressiontest(branch, name, extraFlags, omsHash, dbPrefix, sshConfig, om echo OMSimulator version: ${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator --version """ - FMI_TESTING_FLAG="--fmi=true --fmisimulator=${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator --default=ulimitExe=50" - if (name.contains('cvode')) { - FMI_TESTING_FLAG += " --fmuType=cs" - } + FMI_TESTING_FLAG = " --fmisimulator=${env.HOME}/saved_omc/OMSimulator/install/bin/OMSimulator" } - if (name.contains('fmpy')) { + if (simulators.contains('fmpy')) { sh """ # update fmpy pip install FMPy || true python3 -m fmpy -h || exit 1 """ - FMI_TESTING_FLAG="--fmi=true --fmisimulator='python3 -m fmpy' --default=ulimitExe=50" + FMI_TESTING_FLAG += " --fmisimulator='python3 -m fmpy'" + } + + if (FMI_TESTING_FLAG) { + FMI_TESTING_FLAG = "--fmi=true${FMI_TESTING_FLAG} --default=ulimitExe=50" + if (name.contains('cvode')) { + FMI_TESTING_FLAG += " --fmuType=cs" + } } OMCPATH = "${omcompiler ? '../' : './'}OMCompiler" diff --git a/README.md b/README.md index b3a94b4..51faab0 100644 --- a/README.md +++ b/README.md @@ -158,8 +158,12 @@ Options: - `--ompython_omhome=''`: Path to OpenModelica for OMPython (can be different to the OM running the tests) - `--noclean=False`: Clean (most) generated files. -- `--fmisimulator=''`: The default is nothing but you can use the path to - OMSimulator executable or 'fmpy' +- `--fmisimulator=''`: The FMI simulator to run the FMUs with, as `name=command` + or just the command, e.g. the path to the OMSimulator + executable or `'python3 -m fmpy'`. Repeat the option to + simulate every FMU with several tools without building it + more than once, see [Testing FMI with several + simulators](#testing-fmi-with-several-simulators) - `--ulimitvmem=8388608`: Virtual memory limit (in kB) - `--default=[]`: Add a default value for some configuration key, such as `--default=ulimitExe=60`. The equals sign is mandatory @@ -168,6 +172,89 @@ Options: `procCCompile` above for more insight into individual test parallelization. +### Testing FMI with several simulators + +Building an FMU costs far more than simulating it. Measured on the twelve +models of ExternData: 178 seconds building the FMUs, 0.7 simulating them with +OMSimulator and 2.6 with FMPy. Testing the same FMUs with a second tool +therefore used to cost almost twice as much as testing them with one, because +each job built its own copy of them. + +Give `--fmisimulator` once per tool and the FMUs are built once and simulated +with each of them: + +```bash +./test.py --branch=v1.27-fmi --fmi=true \ + --fmisimulator=/path/to/OMSimulator \ + --fmisimulator='python3 -m fmpy' \ + configs/myConf.json +``` + +`--branch` names the job; every simulator stores its results in a branch of its +own derived from it, so the run above fills + +| simulator | branch | published to | +| --- | --- | --- | +| OMSimulator | `v1.27-fmi` | `branches/v1.27-fmi` | +| FMPy | `v1.27-fmi-fmpy` | `branches/v1.27-fmi-fmpy` | + +which is where those results have always been. OMSimulator keeps the plain +`-fmi` branch; every other tool adds its name. The branch a tool fills depends +on the tool and not on the order, so asking for FMPy alone still fills +`v1.27-fmi-fmpy` and leaves `v1.27-fmi` alone. + +A branch directory looks the same as it always did, file names included. The +`.err` of a model is written by the build, so every simulator of it publishes +the same one; the `.sim` and the difference files are the ones that simulator +produced. + +Only the simulator may differ between the results that share an FMU. Anything +that changes the FMU itself - a different compiler, a different library, a +different `--fmuType` or `--fmiFlags` - is a different job, which is why the +Co-Simulation jobs with CVODE are not merged with the Model Exchange ones. + +In Jenkins the parameters keep their meaning: `fmi_v1_27` asks for OMSimulator +and `fmpy_fmi_v1_27` for FMPy. Ticking both runs one job that builds every FMU +once and simulates it with both; ticking one runs that tool alone. + +### Adding an FMI simulator + +The simulators live in [configs/fmi-simulators.json](configs/fmi-simulators.json). +Adding one is an entry there and no change to any script: + +```json +"fmusim": { + "resultExtension": "csv", + "versionArgument": "--version", + "optionalArguments": { "stepSizeArgument": " --output-interval {stepSize:g}" }, + "arguments": "--interface-type ModelExchange --output-file {result} --start-time {startTime:g} --stop-time {stopTime:g}{stepSizeArgument} {fmu}" +} +``` + +- `arguments` is the command line, a template over `simulator`, `fmu`, + `result`, `requestedResult`, `tempDir`, `startTime`, `stopTime`, `tolerance`, + `timeout`, `stepSize` and anything named in `optionalArguments`. +- `optionalArguments` are the flags that have to disappear when there is nothing + to put in them. OMSimulator hangs on `--stepSize=0` rather than ignoring it, + so its step size flag lives here, while FMPy wants `--output-interval 0` all + the same and writes the value straight into its `arguments`. +- `command` is how the tool is invoked, `{simulator}` by default. FMPy needs a + subcommand, `{simulator} simulate`. +- `resultExtension` is what the tool writes, so that the comparison against the + reference file knows what to read. +- `versionArgument` prints the version, which is recorded with the results. +- `branchSuffix` overrides the `-` a tool adds to the branch. Only + OMSimulator needs it, with `""`. +- `untested` marks an entry nobody has run yet; the run then says so instead of + failing every model with a puzzling error. Remove it once it works. + +Then run it with `--fmisimulator=fmusim=/path/to/fmusim`, or just +`--fmisimulator=/path/to/fmusim` if the command contains the name. + +A tool that is a Python package rather than a command line needs a small driver +script that takes the arguments its entry passes, simulates, writes the result +file and exits non-zero when it fails; the entry then points `command` at it. + ### Generate HTML results ```bash diff --git a/configs/fmi-simulators.json b/configs/fmi-simulators.json new file mode 100644 index 0000000..ee98939 --- /dev/null +++ b/configs/fmi-simulators.json @@ -0,0 +1,46 @@ +{ + "_comment": [ + "The FMI simulators the testing knows how to run. Adding one is an entry here and nothing else.", + "", + "arguments the command line, a template over: simulator, fmu, result, tempDir,", + " startTime, stopTime, tolerance, timeout, stepSize, and every name", + " defined in optionalArguments.", + "optionalArguments flags that have to disappear when there is nothing to put in them, as a", + " result is always the file the tool should write; requestedResult is empty", + " when the run wants no result file at all, which only OMSimulator acts on.", + " name and its own template. OMSimulator crashes on --stepSize=0 instead of", + " ignoring it, so its step size flag lives here; FMPy wants --output-interval 0", + " all the same and writes the value straight into arguments.", + "command how the tool is invoked, {simulator} by default; FMPy needs a subcommand.", + "resultExtension what the tool writes, so the verification knows what to compare.", + "versionArgument the flag that prints the version, recorded with the results.", + "branchSuffix appended to --branch to get the table of this simulator; the default is", + " -, so OMSimulator is the only one that needs to say anything here.", + "untested set it while nobody has run the entry yet; the run then says so." + ], + "OMSimulator": { + "branchSuffix": "", + "resultExtension": "mat", + "versionArgument": "-v", + "arguments": "{resultArgument} --tempDir={tempDir} --startTime={startTime:g} --stopTime={stopTime:g}{stepSizeArgument} --timeout={timeout:g} --tolerance={tolerance:g} {fmu}", + "optionalArguments": { + "resultArgument": "-r={requestedResult}", + "stepSizeArgument": " --stepSize={stepSize:g}" + } + }, + "fmpy": { + "resultExtension": "csv", + "versionArgument": "--version", + "command": "{simulator} simulate", + "arguments": "--output-file {result} --start-time {startTime:g} --stop-time {stopTime:g} --timeout {timeout:g} --relative-tolerance {tolerance:g} --interface-type ModelExchange --solver CVode --output-interval {stepSize:g} {fmu}" + }, + "fmusim": { + "untested": true, + "resultExtension": "csv", + "versionArgument": "--version", + "arguments": "--interface-type ModelExchange --output-file {result} --start-time {startTime:g} --stop-time {stopTime:g}{stepSizeArgument} {fmu}", + "optionalArguments": { + "stepSizeArgument": " --output-interval {stepSize:g}" + } + } +} diff --git a/shared.py b/shared.py index fa8d328..a5afa80 100644 --- a/shared.py +++ b/shared.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import re, os, subprocess +import re, os, string, subprocess import simplejson as json simCodeTargetRe = re.compile('--simCodeTarget=([^"\'\\s,;)]+)') @@ -118,4 +118,94 @@ def isFMPy(fmisimulator): else: return False +def fmiSimulatorName(command): + """The name of the simulator a bare --fmisimulator runs. + + Any of the known names appearing in the command wins; a command that names + none of them is OMSimulator, which is how --fmisimulator was used before it + could name its simulator. + """ + for name in sorted(fmiSimulators(), key=len, reverse=True): + if name.lower() in command.lower(): + return name + return "OMSimulator" + +def parseFmiSimulators(fmisimulators): + """The --fmisimulator values as an ordered list of (name, command). + + A value is either "name=command" or just the command, whose name is then + taken from the command itself. The name decides which simulator of + configs/fmi-simulators.json is run and which branch its results go to, see + branchForSimulator. + """ + res = [] + for s in fmisimulators or []: + if not s: + continue + name, sep, command = s.partition("=") + if not sep or "/" in name or " " in name: + (name, command) = (fmiSimulatorName(s), s) + res.append((name, command)) + names = [n for (n, _) in res] + if len(set(names)) != len(names): + raise Exception("The same FMI simulator name is used twice: %s" % ", ".join(names)) + for name in names: + if fmiSimulator(name).get("untested"): + print("Warning: nobody has run %s through the testing yet; if its flags in %s are wrong, " + "every model will fail to simulate." % (name, FMI_SIMULATORS_FILE)) + return res + +# The FMI simulators live in configs/fmi-simulators.json, so that adding one is +# an entry in a file rather than a change to the scripts. A simulator stores +# its results in the branch of the job with its name appended, except +# OMSimulator, which has always had the plain -fmi table to itself; the mapping +# is keyed by the simulator and not by the order it was given in, so that a job +# running only FMPy still fills v1.27-fmi-fmpy and not v1.27-fmi. +FMI_SIMULATORS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "configs", "fmi-simulators.json") +_fmiSimulators = None + +def fmiSimulators(path=None): + """Everything the testing knows about the FMI simulators.""" + global _fmiSimulators + if _fmiSimulators is None or path: + with open(path or FMI_SIMULATORS_FILE) as fin: + _fmiSimulators = dict((k, v) for (k, v) in json.load(fin).items() if not k.startswith("_")) + return _fmiSimulators + +def fmiSimulator(name): + """What is known about an FMI simulator, by the name --fmisimulator gave it.""" + known = fmiSimulators() + if name not in known: + raise Exception("Unknown FMI simulator %s; known are %s. Adding one is an entry in %s." + % (name, ", ".join(sorted(known)), FMI_SIMULATORS_FILE)) + return known[name] + +def fmiSimulatorCommand(name, command, **values): + """The command line that runs one FMU with one simulator. + + arguments is a template over the values below plus anything the entry defines + in optionalArguments, which are the flags that have to disappear when there + is nothing to put in them: OMSimulator crashes on --stepSize=0 rather than + ignoring it, while FMPy wants --output-interval 0 all the same and therefore + writes the value straight into its arguments. + """ + spec = fmiSimulator(name) + values["simulator"] = command + for (key, template) in (spec.get("optionalArguments") or {}).items(): + used = [f.split(":")[0].split(".")[0].split("[")[0] + for (_, f, _, _) in string.Formatter().parse(template) if f] + values[key] = template.format(**values) if all(values.get(u) for u in used) else "" + return "%s %s" % (spec.get("command", "{simulator}").format(**values), + spec["arguments"].format(**values)) + +def branchForSimulator(branch, name): + """Where the results of one FMI simulator of a run are stored. + + --branch names the job, v1.27-fmi, and every simulator derives its own from + it: OMSimulator fills v1.27-fmi, FMPy v1.27-fmi-fmpy, whether they run + together or on their own. + """ + return branch + fmiSimulator(name).get("branchSuffix", "-%s" % name) + diff --git a/test.py b/test.py index c97573d..affc5e3 100755 --- a/test.py +++ b/test.py @@ -36,7 +36,7 @@ parser.add_argument('--noclean', action="store_true", default=False) parser.add_argument('--nobuildmodel', action="store_true", help="Translate, build and simulate in a single simulate() call instead of translateModel() followed by simulate(resimulateExecutable=...), so the JIT compile is reported as build time rather than simulation time. Only used by simCodeTarget=wasm-jit.", default=False) parser.add_argument('--coldhot', action="store_true", help="Simulate each model twice in the same omc; the second run reuses the compiled module. Both times are printed, but only the hot one is stored. Only used by simCodeTarget=wasm-jit.", default=False) -parser.add_argument('--fmisimulator', default='') +parser.add_argument('--fmisimulator', action='append', default=[], help="FMI simulator to run the FMUs with, as 'name=command' or just the command. Repeat it to simulate every FMU with several tools without building it more than once; the first one stores its results in --branch and each further one in -, so --branch=master-fmi with OMSimulator and fmpy fills master-fmi and master-fmi-fmpy." ) parser.add_argument('--ulimitvmem', help="Virtual memory limit (in kB) (linux only)", type=int, default=8*1024*1024) parser.add_argument('--default', action='append', help="Add a default value for some configuration key, such as --default=ulimitExe=60. The equals sign is mandatory.", default=[]) parser.add_argument('-j', '--jobs', default=0, help="Ignored and deprecated, use procOMC:0 or procOMC:1 in the config") @@ -67,7 +67,47 @@ extraflags = args.extraflags extrasimflags = args.extrasimflags ompython_omhome = args.ompython_omhome -fmisimulator = args.fmisimulator or None +fmisimulators = shared.parseFmiSimulators(args.fmisimulator) +# The first simulator is the one the single-simulator code paths use. +fmisimulator = fmisimulators[0][1] if fmisimulators else None + +# One branch per simulator. The first reports itself in the results of the +# model and the others under their own name, but every one of them stores its +# results where its own simulator belongs: a job given only FMPy on +# --branch=v1.27-fmi fills v1.27-fmi-fmpy, not v1.27-fmi. +resultBranches = [(branch, None)] +if fmisimulators: + resultBranches = [(shared.branchForSimulator(branch, fmisimulators[0][0]), None)] + for (simulatorName, _) in fmisimulators[1:]: + resultBranches.append((shared.branchForSimulator(branch, simulatorName), simulatorName)) +# What the run asks about when it looks for results it already has. +primaryBranch = resultBranches[0][0] + +jobOutput = result_location + +def outputFor(resultBranch): + """Where a branch publishes. + + --output names the directory of the job, which ends in the branch the job was + started with, so every simulator gets the directory of its own branch beside + it: a job given only FMPy on --branch=v1.27-fmi publishes v1.27-fmi-fmpy, the + same branch its results are stored under. + + rsync runs from the directory a branch is staged in, so a local destination + has to be absolute; a remote one, host:path, already is. + """ + location = jobOutput + if jobOutput: + base = jobOutput.rstrip("/") + if base.endswith("/" + branch): + location = "%s%s" % (base[:-len(branch)], resultBranch) + elif resultBranch != primaryBranch: + location = "%s/%s" % (base, resultBranch) + if location and ":" not in location.split("/")[0]: + location = os.path.abspath(location) + return location + + allTestsFmi = args.fmi fmuType = args.fmuType ulimitMemory = args.ulimitvmem @@ -250,8 +290,8 @@ def target(): omc_cmd = [os.path.normpath(os.path.join(os.environ.get("OPENMODELICAHOME"), 'bin', 'omc'))] else: omc_cmd = ["omc"] -if result_location != "" and not os.path.exists(result_location): - os.makedirs(result_location) +if result_location != "" and not os.path.exists(outputFor(primaryBranch)): + os.makedirs(outputFor(primaryBranch)) if configs == []: print("Error: Expected at least one configuration file to start the library test") @@ -299,17 +339,24 @@ def timeSeconds(f): sys.stdout.flush() -fmisimulatorversion = None -if fmisimulator: +def fmiSimulatorVersion(command): try: - if not isFMPy(fmisimulator): - fmisimulatorversion = check_output_log([fmisimulator, "-v"], stderr=subprocess.STDOUT).strip() - else: - fmisimulatorversion = subprocess.getoutput(fmisimulator + " --version" ).strip().encode('ascii') + if not isFMPy(command): + return check_output_log([command, "-v"], stderr=subprocess.STDOUT).strip() + return subprocess.getoutput(command + " --version").strip().encode('ascii') except subprocess.CalledProcessError as e: - print("Failure to run %s:\n%s" %(fmisimulator, e.output)) + print("Failure to run %s:\n%s" % (command, e.output)) raise e - print(fmisimulatorversion) + +fmisimulatorversions = {} +fmisimulatorversion = None +if fmisimulator: + for (name, command) in fmisimulators: + fmisimulatorversions[name] = fmiSimulatorVersion(command) + print("%s: %s" % (name, fmisimulatorversions[name])) + # The version of the first one goes into the library version of every branch, + # as it did when a job ran a single simulator. + fmisimulatorversion = fmisimulatorversions[fmisimulators[0][0]] else: if allTestsFmi: raise Exception("No OMSimulator; trying to simulate using FMI") @@ -533,11 +580,15 @@ def testHelloWorld(cmd): print("Unknown schema user_version=%d" % user_version) sys.exit(1) -cursor.execute('''CREATE TABLE if not exists [%s] +def createBranchTable(branch): + """A run fills one table per FMI simulator, so this happens more than once.""" + cursor.execute('''CREATE TABLE if not exists [%s] (date integer NOT NULL, libname text NOT NULL, model text NOT NULL, exectime real NOT NULL, frontend real NOT NULL, backend real NOT NULL, simcode real NOT NULL, templates real NOT NULL, compile real NOT NULL, simulate real NOT NULL, verify real NOT NULL, verifyfail integer NOT NULL, verifytotal integer NOT NULL, finalphase integer NOT NULL, parsing real NOT NULL)''' % branch) -cursor.execute('''DROP INDEX IF EXISTS [idx_%s_date]''' % branch) + cursor.execute('''DROP INDEX IF EXISTS [idx_%s_date]''' % branch) + +createBranchTable(primaryBranch) cursor.execute('''DROP INDEX IF EXISTS idx_omcversion_date''') cursor.execute('''DROP INDEX IF EXISTS idx_libversion_date''') @@ -617,6 +668,7 @@ def hashReferenceFiles(s): conf["haveFMI"] = fmiOK_C conf["haveFMICpp"] = fmiOK_Cpp conf["fmisimulator"] = fmisimulator + conf["fmisimulators"] = ["%s=%s" % (n, c) for (n, c) in fmisimulators] conf["fmuType"] = fmuType if (not canChangeOptLevel) and "optlevel" in conf: print("Deleting optlevel") @@ -727,7 +779,7 @@ def hashReferenceFiles(s): res=list(filter(lambda x: not x.startswith(prefix), res)) libName=shared.libname(library, conf) v = cursor.execute("""SELECT date,libversion,libname,branch,omcversion FROM [libversion] NATURAL JOIN [omcversion] - WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1""", (conf["libraryLastChange"],libName,branch,omc_version,confighash)).fetchone() + WHERE libversion=? AND libname=? AND branch=? AND omcversion=? AND confighash=? ORDER BY date DESC LIMIT 1""", (conf["libraryLastChange"],libName,primaryBranch,omc_version,confighash)).fetchone() if libName in stats_by_libname or libName in skipped_libs: raise Exception("Duplicate libName found: %s" % libName) if v is None or execAllTests: @@ -847,7 +899,7 @@ def expectedExec(c): (model,lib,libName,name,data) = c if "expectedExec" in data: return data["expectedExec"] - cursor.execute("SELECT exectime FROM [%s] WHERE libname = ? AND model = ? ORDER BY date DESC LIMIT 1" % branch, (libName,model)) + cursor.execute("SELECT exectime FROM [%s] WHERE libname = ? AND model = ? ORDER BY date DESC LIMIT 1" % primaryBranch, (libName,model)) v = cursor.fetchone() data["expectedExec"] = (v or (0.0,))[0] return data["expectedExec"] @@ -920,10 +972,20 @@ def loadJsonOrEmptySet(f): print(" %-70s cold %8.4f hot %8.4f" % (model, data["simcold"], data.get("sim") or 0.0)) sys.stdout.flush() -for key in stats.keys(): - (name,model,libname,data)=stats[key] - stats_by_libname[libname]["stats"].append(stats[key]) - values = (testRunStartTimeAsEpoch, +def resultValues(model, libname, data, simulator=None): + """One row of a branch table. + + Everything up to the build is what the model cost whatever simulates it, so + the simulators of one FMU share it and differ only in the simulation, the + verification and how far they got. + """ + simulated = data if simulator is None else (data.get("simulators") or {}).get(simulator) + if simulated is None: + # The model never got as far as being simulated, so every simulator of it + # reports the phase the build stopped at rather than a failure of its own. + simulated = {"phase": data.get("phase") or 0} + diff = simulated.get("diff") or {} + return (testRunStartTimeAsEpoch, libname, model, data.get("exectime") or 0.0, @@ -932,19 +994,26 @@ def loadJsonOrEmptySet(f): data.get("simcode") or 0.0, data.get("templates") or 0.0, data.get("build") or 0.0, - data.get("sim") or 0.0, - (data.get("diff") or {}).get("time") or 0.0, - len((data.get("diff") or {}).get("vars") or []), - (data.get("diff") or {}).get("numCompared") or 0, - data.get("phase") or 0, + simulated.get("sim") or 0.0, + diff.get("time") or 0.0, + len(diff.get("vars") or []), + diff.get("numCompared") or 0, + simulated.get("phase") or 0, data.get("parsing") or 0.0 ) - # print values - cursor.execute("INSERT INTO [%s] VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" % branch, values) -for libname in stats_by_libname.keys(): - confighash = stats_by_libname[libname]["conf"]["confighash"] - cursor.execute("INSERT INTO [libversion] VALUES (?,?,?,?,?)", (testRunStartTimeAsEpoch, branch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash)) -cursor.execute("INSERT INTO [omcversion] VALUES (?,?,?)", (testRunStartTimeAsEpoch, branch, omc_version)) + +for (resultBranch, simulator) in resultBranches: + createBranchTable(resultBranch) + for key in stats.keys(): + (name,model,libname,data)=stats[key] + if simulator is None: + stats_by_libname[libname]["stats"].append(stats[key]) + cursor.execute("INSERT INTO [%s] VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" % resultBranch, + resultValues(model, libname, data, simulator)) + for libname in stats_by_libname.keys(): + confighash = stats_by_libname[libname]["conf"]["confighash"] + cursor.execute("INSERT INTO [libversion] VALUES (?,?,?,?,?)", (testRunStartTimeAsEpoch, resultBranch, libname, stats_by_libname[libname]["conf"]["libraryLastChange"], confighash)) + cursor.execute("INSERT INTO [omcversion] VALUES (?,?,?)", (testRunStartTimeAsEpoch, resultBranch, omc_version)) """ # Not really a good thing to do; was just done to make generation of the report simpler for libname in skipped_libs.keys(): @@ -988,213 +1057,293 @@ def cpu_name(): sysInfo = "%s, %d GB RAM, %s%s" % (cpu_name(), int(math.ceil(psutil.virtual_memory().total / (1024.0**3))), ("Docker " + docker + " ") if docker else "", lsb_release) # create target dir to move results without sync operations (win or when --noSync is used) -if result_location != "" and (isWin or noSync): - resRootPath = os.path.join(result_location, branch) - if os.path.exists(resRootPath): - rmtree(resRootPath) - os.mkdir(resRootPath) - -htmltpl=open("library.html.tpl").read() -for libname in stats_by_libname.keys(): - if libname in skipped_libs: - continue - s = None # Make sure I don't use this - filesList = open(libname + ".files", "w") - filesList.write("/\n") - filesList.write("/%s.html\n" % libname) - filesList.write("/files/\n") - conf = stats_by_libname[libname]["conf"] - stats = stats_by_libname[libname]["stats"] - for s in stats: - filename_prefix = "files/%s_%s" % (s[2],s[1]) - filesList.write("/%s*diff*csv\n" % filename_prefix) - filesList.write("/%s*diff*html\n" % filename_prefix) - if is_non_zero_file(filename_prefix+".sim"): - filesList.write("/%s.sim\n" % filename_prefix) - if is_non_zero_file(filename_prefix+".err"): - filesList.write("/%s.err\n" % filename_prefix) - variables = (s[3].get("diff") or {}).get("vars") or [] - if len(variables)>0: - filesList.write("/%s.diff.html\n" % filename_prefix) - for v in variables: - filesList.write("/%s.diff.%s.csv\n" % (filename_prefix, v)) - filesList.write("/%s.diff.%s.html\n" % (filename_prefix, v)) - filesList.close() - testsHTML = "\n".join(['%s%s%s%s%s%s%s%s%s%s%s%s\n' % - (lambda filename_prefix, diff: - ( - ('%s' % (filename_prefix + ".err", html.escape(s[1]))) if is_non_zero_file(filename_prefix + ".err") else html.escape(s[1]), - (' (sim)' % (filename_prefix + ".sim")) if is_non_zero_file(filename_prefix + ".sim") else "", - checkPhase(s[3]["phase"], 7) if s[3]["phase"]>=6 else "#FFFFFF", - ("%s (%d verified)" % (timeSeconds(diff.get("time")), diff.get("numCompared"))) if s[3]["phase"]>=7 else (" " if diff is None else - ('%s (%d/%d failed)' % (timeSeconds(diff.get("time")), filename_prefix, len(diff.get("vars")), diff.get("numCompared")))), - checkPhase(s[3]["phase"], 6), - timeSeconds(s[3].get("sim") or 0), - checkPhase(s[3]["phase"], 5), - timeSeconds(sum(s[3].get(x) or 0.0 for x in ["frontend","backend","simcode","templates","build"])), - timeSeconds(s[3].get("parsing") or 0), - checkPhase(s[3]["phase"], 1), - timeSeconds(s[3].get("frontend") or 0), - checkPhase(s[3]["phase"], 2), - timeSeconds(s[3].get("backend") or 0), - checkPhase(s[3]["phase"], 3), - timeSeconds(s[3].get("simcode") or 0), - checkPhase(s[3]["phase"], 4), - timeSeconds(s[3].get("templates") or 0), - checkPhase(s[3]["phase"], 5), - timeSeconds(s[3].get("build") or 0), - timeSeconds(s[3].get("exectime") or 0) - ))(filename_prefix="files/%s_%s" % (s[2], s[1]), diff=s[3].get("diff")) - for s in natsorted(stats, key=lambda s: s[1])]) - numSucceeded = [len(stats)] + [sum(1 if s[3]["phase"]>=i else 0 for s in stats) for i in range(1,8)] - - try: - githuburltesting = "https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit/" - gitloglibrarytesting = check_output_log(["git", "log", '--pretty=
CommitDateAuthorSummary
%%h%%ai%%an%%s
' % (githuburltesting), "-1"], cwd="./").decode("utf-8") - except subprocess.CalledProcessError as e: - print(str(e)) - gitloglibrarytesting = "
could not get the git log for OpenModelicaLibraryTesting
" - - # adrpo: attempt to get the revision of the reference files if possible - if conf.get("referenceFiles"): - c = conf.get("referenceFiles") - if DEBUG: - print("referenceFiles git ... attempting to retrieve info from directory: %s" % c) - sys.stdout.flush() - gitReferenceFiles = c - # see if we have a commit file - f = os.path.join(c, "commit") - if os.path.exists(f): - with open(f) as fin: - gitReferenceFilesVersion = fin.read() - print("referenceFiles git ... read from file %s" % f) - sys.stdout.flush() - else: - try: - if isinstance(c, (str, bytes)): - if DEBUG: - print("referenceFiles git ... see if directory has an evironment variable") - sys.stdout.flush() - m = re.search("^[$][A-Z_]+", c) - if m: - k = m.group(0)[1:] - if k not in os.environ: - if DEBUG: - print("referenceFiles git ... environment variable used in the directory cannot be found in the environment: %s" % k) - sys.stdout.flush() - raise Exception("Environment variable %s not defined, but used in JSON config for reference files" % k) - gitReferenceFiles = c.replace(m.group(0), os.environ[k]) - if DEBUG: - print("referenceFiles git ... directory after replacing the environment variable: %s" % gitReferenceFiles) - sys.stdout.flush() - sys.stdout.flush() - try: - gitReferenceFilesURL = check_output_log(["git", "config", "--get", "remote.origin.url"], cwd=gitReferenceFiles).decode("utf-8") - except subprocess.CalledProcessError as e: - print(e) - gitReferenceFilesURL = gitReferenceFiles - gitReferenceFilesVersion = check_output_log(["git", "log", '--pretty=
CommitDateAuthorSummary
%%h%%ai%%an%%s
' % (gitReferenceFilesURL), "-1"], cwd=gitReferenceFiles).decode("utf-8") - print("referenceFiles git ... got version information: %s" % gitReferenceFilesVersion) - sys.stdout.flush() - except subprocess.CalledProcessError as e: - print("referenceFiles git ... something went wrong with getting the git info for directory: %s" % c) - print(str(e)) - sys.stdout.flush() - gitReferenceFilesVersion = "" - else: - gitReferenceFilesVersion = "" - - replacements = ( - (u"#sysInfo#", html.escape(sysInfo)), - (u"#omcVersion#", html.escape(omc_version)), - (u"#fmiToolVersion#", ("

"+html.escape("FMI tool: %s" % fmisimulatorversion)+"

") if fmisimulatorversion else ""), - (u"#fmi#", ("

"+html.escape("FMI version: %s" % conf.get("fmi"))+"

") if conf.get("fmi") else ""), - (u"#optlevel#", html.escape(conf.get("optlevel")) if (canChangeOptLevel and conf.get("optlevel")) else "Tool default"), - (u"#timeStart#", html.escape(time.strftime('%Y-%m-%d %H:%M:%S', start_as_time))), - (u"#fileName#", html.escape(libname)), - (u"#customCommands#", html.escape("\n".join(conf["customCommands"]))), - (u"#libraryVersionRevision#", html.escape(conf["libraryVersionRevision"])), - (u"#OpenModelicaLibraryTesting#", gitloglibrarytesting), - (u"#metadata#", html.escape(conf["metadata"])), - (u"#ulimitOmc#", html.escape(str(conf["ulimitOmc"]))), - (u"#ulimitExe#", html.escape(str(conf["ulimitExe"]))), - (u"#defaultTolerance#", html.escape(str(conf["defaultTolerance"]))), - (u"#defaultNumberOfIntervals#", html.escape(str(conf["defaultNumberOfIntervals"]))), - (u"#simFlags#", html.escape(conf.get("simFlags") or "")), - (u"#referenceFiles#", ('

Reference Files: %s

%s' % ((conf["referenceFilesURL"].replace(os.path.dirname(os.path.realpath(__file__)),"")), gitReferenceFilesVersion)) if ((conf.get("referenceFilesURL") or "") != "") else ""), - (u"#referenceTool#", ('

Verified using: %s (diffSimulationResults)

' % html.escape(ompython_omc_version)) if ((conf.get("referenceFiles") or "") != "") else ""), - (u"#Total#", html.escape(str(numSucceeded[0]))), - (u"#FrontendColor#", checkNumSucceeded(numSucceeded, 1)), - (u"#BackendColor#", checkNumSucceeded(numSucceeded, 2)), - (u"#SimCodeColor#", checkNumSucceeded(numSucceeded, 3)), - (u"#TemplatesColor#", checkNumSucceeded(numSucceeded, 4)), - (u"#CompilationColor#", checkNumSucceeded(numSucceeded, 5)), - (u"#SimulationColor#", checkNumSucceeded(numSucceeded, 6)), - (u"#VerificationColor#", checkNumSucceeded(numSucceeded, 7)), - (u"#Frontend#", html.escape(str(numSucceeded[1]))), - (u"#Backend#", html.escape(str(numSucceeded[2]))), - (u"#SimCode#", html.escape(str(numSucceeded[3]))), - (u"#Templates#", html.escape(str(numSucceeded[4]))), - (u"#Compilation#", html.escape(str(numSucceeded[5]))), - (u"#Simulation#", html.escape(str(numSucceeded[6]))), - (u"#Verification#", html.escape(str(numSucceeded[7]))), - (u"#totalTime#", html.escape(str(datetime.timedelta(seconds=int(sum(s[3].get("exectime") or 0.0 for s in stats)))))), - (u"#config#", html.escape(json.dumps(conf["configFromFile"], indent=1, sort_keys=True))), - (u"#testsHTML#", testsHTML) - ) - open("%s.html" % libname, "w").write(multiple_replace(htmltpl, *replacements)) - - # move results by sync operations (not available under win) - if result_location != "" and not isWin and not noSync: - result_location_libname = "%s/%s" % (result_location, libname) +def stageRootFor(simulator, suffix): + """The directory a branch is published from. + + The first simulator publishes the workspace itself, as a job always has. The + others write _.sim there so as not to overwrite it, and are + published from a directory of their own where the same files appear under the + plain .sim a branch directory has always held. + """ + if not suffix: + return "." + root = "publish_%s" % simulator + for d in (root, os.path.join(root, "files")): try: - os.mkdir("emptydir") - except: + os.mkdir(d) + except OSError: pass - check_output_log(["rsync", "-aR", "emptydir/", result_location]) - check_output_log(["rsync", "-aR", "emptydir/", result_location_libname]) - check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"]) + return root + +def stagePublished(stageRoot, workspacePrefix, publishedPrefix, suffix): + """Link one model's results into the directory its branch is published from.""" + if not suffix: + return + for f in glob.glob(glob.escape(workspacePrefix) + "*"): + published = os.path.join(stageRoot, publishedPrefix + f[len(workspacePrefix):]) try: - check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname]) - except: - check_output_log(["rsync", "-aR", "emptydir/", result_location]) - check_output_log(["rsync", "-aR", "emptydir/", result_location_libname]) - check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"]) - check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname]) - if (conf.get("referenceFiles") or "") != "" and dygraphs: - check_output_log(["rsync", "-a", dygraphs, result_location_libname+"/files"]) - else: - print("No Sync: result_location [%s] != "" and not isWin [%s] and not noSync [%s] : library: %s" % (result_location, isWin, noSync, libname)) - - # move results without sync operations (win or when --noSync is used) + if os.path.exists(published): + os.unlink(published) + os.link(f, published) + except OSError: + shutil.copy2(f, published) + +def stageShared(stageRoot, prefix, suffix): + """The .err of the build belongs to every simulator of the model.""" + if not suffix: + return + for f in glob.glob(glob.escape(prefix) + ".err"): + published = os.path.join(stageRoot, f) + try: + if os.path.exists(published): + os.unlink(published) + os.link(f, published) + except OSError: + shutil.copy2(f, published) + +def dataForSimulator(data, simulator): + """The results of one model as one simulator saw them. + + Everything up to the build is shared, so only the simulation, the comparison + against the reference file and the phase come from the simulator itself. + """ + if simulator is None: + return data + simulated = (data.get("simulators") or {}).get(simulator) + merged = dict(data) + merged["sim"] = (simulated or {}).get("sim") + merged["diff"] = (simulated or {}).get("diff") + # A model that never got as far as being simulated stopped in the build, + # which every simulator of it shares. + merged["phase"] = simulated.get("phase") if simulated else data.get("phase") + return merged + +def artifactSuffix(simulator): + """What tells the files of one simulator from those of another.""" + return "_%s" % simulator if simulator and len(fmisimulators) > 1 else "" + +# Every simulator publishes its own results - .sim and diff files included - to +# the directory of its own branch; the .err of the build is shared, so each of +# them gets a copy of it. +for (resultBranch, simulator) in resultBranches: + result_location = outputFor(resultBranch) + suffix = artifactSuffix(simulator) if result_location != "" and (isWin or noSync): - print("--> copy res file of library: " + libname) - libPath = os.path.join(resRootPath, libname) - if not os.path.exists(libPath): - os.makedirs(libPath) - libFilesPath = os.path.join(libPath, 'files') - if os.path.exists(libFilesPath): - rmtree(libFilesPath) - os.makedirs(libFilesPath) + # Unlike the rsync path, this one is given the directory the branches live + # in and appends the branch itself. + resRootPath = os.path.join(jobOutput, resultBranch) + if os.path.exists(resRootPath): + rmtree(resRootPath) + os.makedirs(resRootPath) + + htmltpl=open("library.html.tpl").read() + for libname in stats_by_libname.keys(): + if libname in skipped_libs: + continue + s = None # Make sure I don't use this + stageRoot = stageRootFor(simulator, suffix) + filesList = open(os.path.join(stageRoot, libname + ".files"), "w") + filesList.write("/\n") + filesList.write("/%s.html\n" % libname) + filesList.write("/files/\n") + conf = stats_by_libname[libname]["conf"] + stats = [(n, m, l, dataForSimulator(d, simulator)) for (n, m, l, d) in stats_by_libname[libname]["stats"]] + for s in stats: + # What the simulator wrote in the workspace, and what it is published as: + # the branch of a simulator holds its results under the plain name. + workspace_prefix = "files/%s_%s%s" % (s[2],s[1],suffix) + filename_prefix = "files/%s_%s" % (s[2],s[1]) + stagePublished(stageRoot, workspace_prefix, filename_prefix, suffix) + stageShared(stageRoot, "files/%s_%s" % (s[2],s[1]), suffix) + filesList.write("/%s*diff*csv\n" % filename_prefix) + filesList.write("/%s*diff*html\n" % filename_prefix) + if is_non_zero_file(workspace_prefix+".sim"): + filesList.write("/%s.sim\n" % filename_prefix) + errPrefix = "files/%s_%s" % (s[2],s[1]) + if is_non_zero_file(errPrefix+".err"): + filesList.write("/%s.err\n" % errPrefix) + variables = (s[3].get("diff") or {}).get("vars") or [] + if len(variables)>0: + filesList.write("/%s.diff.html\n" % filename_prefix) + for v in variables: + filesList.write("/%s.diff.%s.csv\n" % (filename_prefix, v)) + filesList.write("/%s.diff.%s.html\n" % (filename_prefix, v)) + filesList.close() + testsHTML = "\n".join(['%s%s%s%s%s%s%s%s%s%s%s%s\n' % + (lambda filename_prefix, errPrefix, diff: + ( + ('%s' % (errPrefix + ".err", html.escape(s[1]))) if is_non_zero_file(errPrefix + ".err") else html.escape(s[1]), + (' (sim)' % (filename_prefix + ".sim")) if is_non_zero_file(filename_prefix + suffix + ".sim") else "", + checkPhase(s[3]["phase"], 7) if s[3]["phase"]>=6 else "#FFFFFF", + ("%s (%d verified)" % (timeSeconds(diff.get("time")), diff.get("numCompared"))) if s[3]["phase"]>=7 else (" " if diff is None else + ('%s (%d/%d failed)' % (timeSeconds(diff.get("time")), filename_prefix, len(diff.get("vars")), diff.get("numCompared")))), + checkPhase(s[3]["phase"], 6), + timeSeconds(s[3].get("sim") or 0), + checkPhase(s[3]["phase"], 5), + timeSeconds(sum(s[3].get(x) or 0.0 for x in ["frontend","backend","simcode","templates","build"])), + timeSeconds(s[3].get("parsing") or 0), + checkPhase(s[3]["phase"], 1), + timeSeconds(s[3].get("frontend") or 0), + checkPhase(s[3]["phase"], 2), + timeSeconds(s[3].get("backend") or 0), + checkPhase(s[3]["phase"], 3), + timeSeconds(s[3].get("simcode") or 0), + checkPhase(s[3]["phase"], 4), + timeSeconds(s[3].get("templates") or 0), + checkPhase(s[3]["phase"], 5), + timeSeconds(s[3].get("build") or 0), + timeSeconds(s[3].get("exectime") or 0) + ))(filename_prefix="files/%s_%s" % (s[2], s[1]), errPrefix="files/%s_%s" % (s[2], s[1]), diff=s[3].get("diff")) + for s in natsorted(stats, key=lambda s: s[1])]) + numSucceeded = [len(stats)] + [sum(1 if s[3]["phase"]>=i else 0 for s in stats) for i in range(1,8)] + try: - if clean: - shutil.move("./" + libname + ".html", libPath) + githuburltesting = "https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit/" + gitloglibrarytesting = check_output_log(["git", "log", '--pretty=
CommitDateAuthorSummary
%%h%%ai%%an%%s
' % (githuburltesting), "-1"], cwd="./").decode("utf-8") + except subprocess.CalledProcessError as e: + print(str(e)) + gitloglibrarytesting = "
could not get the git log for OpenModelicaLibraryTesting
" + + # adrpo: attempt to get the revision of the reference files if possible + if conf.get("referenceFiles"): + c = conf.get("referenceFiles") + if DEBUG: + print("referenceFiles git ... attempting to retrieve info from directory: %s" % c) + sys.stdout.flush() + gitReferenceFiles = c + # see if we have a commit file + f = os.path.join(c, "commit") + if os.path.exists(f): + with open(f) as fin: + gitReferenceFilesVersion = fin.read() + print("referenceFiles git ... read from file %s" % f) + sys.stdout.flush() else: - shutil.copy2("./" + libname + ".html", libPath) - except: - print("-- problem durin copy/move of html file of lib: " + libname) - - for file in glob.glob("./files/" + libname + '*.err') \ - + glob.glob("./files/" + libname + '*.sim') \ - + glob.glob("./files/" + libname + '*.csv') \ - + glob.glob("./files/" + libname + '*.json') \ - + glob.glob("./files/" + libname + '*.html'): + try: + if isinstance(c, (str, bytes)): + if DEBUG: + print("referenceFiles git ... see if directory has an evironment variable") + sys.stdout.flush() + m = re.search("^[$][A-Z_]+", c) + if m: + k = m.group(0)[1:] + if k not in os.environ: + if DEBUG: + print("referenceFiles git ... environment variable used in the directory cannot be found in the environment: %s" % k) + sys.stdout.flush() + raise Exception("Environment variable %s not defined, but used in JSON config for reference files" % k) + gitReferenceFiles = c.replace(m.group(0), os.environ[k]) + if DEBUG: + print("referenceFiles git ... directory after replacing the environment variable: %s" % gitReferenceFiles) + sys.stdout.flush() + sys.stdout.flush() + try: + gitReferenceFilesURL = check_output_log(["git", "config", "--get", "remote.origin.url"], cwd=gitReferenceFiles).decode("utf-8") + except subprocess.CalledProcessError as e: + print(e) + gitReferenceFilesURL = gitReferenceFiles + gitReferenceFilesVersion = check_output_log(["git", "log", '--pretty=
CommitDateAuthorSummary
%%h%%ai%%an%%s
' % (gitReferenceFilesURL), "-1"], cwd=gitReferenceFiles).decode("utf-8") + print("referenceFiles git ... got version information: %s" % gitReferenceFilesVersion) + sys.stdout.flush() + except subprocess.CalledProcessError as e: + print("referenceFiles git ... something went wrong with getting the git info for directory: %s" % c) + print(str(e)) + sys.stdout.flush() + gitReferenceFilesVersion = "" + else: + gitReferenceFilesVersion = "" + + replacements = ( + (u"#sysInfo#", html.escape(sysInfo)), + (u"#omcVersion#", html.escape(omc_version)), + (u"#fmiToolVersion#", ("

"+html.escape("FMI tool: %s" % fmisimulatorversion)+"

") if fmisimulatorversion else ""), + (u"#fmi#", ("

"+html.escape("FMI version: %s" % conf.get("fmi"))+"

") if conf.get("fmi") else ""), + (u"#optlevel#", html.escape(conf.get("optlevel")) if (canChangeOptLevel and conf.get("optlevel")) else "Tool default"), + (u"#timeStart#", html.escape(time.strftime('%Y-%m-%d %H:%M:%S', start_as_time))), + (u"#fileName#", html.escape(libname)), + (u"#customCommands#", html.escape("\n".join(conf["customCommands"]))), + (u"#libraryVersionRevision#", html.escape(conf["libraryVersionRevision"])), + (u"#OpenModelicaLibraryTesting#", gitloglibrarytesting), + (u"#metadata#", html.escape(conf["metadata"])), + (u"#ulimitOmc#", html.escape(str(conf["ulimitOmc"]))), + (u"#ulimitExe#", html.escape(str(conf["ulimitExe"]))), + (u"#defaultTolerance#", html.escape(str(conf["defaultTolerance"]))), + (u"#defaultNumberOfIntervals#", html.escape(str(conf["defaultNumberOfIntervals"]))), + (u"#simFlags#", html.escape(conf.get("simFlags") or "")), + (u"#referenceFiles#", ('

Reference Files: %s

%s' % ((conf["referenceFilesURL"].replace(os.path.dirname(os.path.realpath(__file__)),"")), gitReferenceFilesVersion)) if ((conf.get("referenceFilesURL") or "") != "") else ""), + (u"#referenceTool#", ('

Verified using: %s (diffSimulationResults)

' % html.escape(ompython_omc_version)) if ((conf.get("referenceFiles") or "") != "") else ""), + (u"#Total#", html.escape(str(numSucceeded[0]))), + (u"#FrontendColor#", checkNumSucceeded(numSucceeded, 1)), + (u"#BackendColor#", checkNumSucceeded(numSucceeded, 2)), + (u"#SimCodeColor#", checkNumSucceeded(numSucceeded, 3)), + (u"#TemplatesColor#", checkNumSucceeded(numSucceeded, 4)), + (u"#CompilationColor#", checkNumSucceeded(numSucceeded, 5)), + (u"#SimulationColor#", checkNumSucceeded(numSucceeded, 6)), + (u"#VerificationColor#", checkNumSucceeded(numSucceeded, 7)), + (u"#Frontend#", html.escape(str(numSucceeded[1]))), + (u"#Backend#", html.escape(str(numSucceeded[2]))), + (u"#SimCode#", html.escape(str(numSucceeded[3]))), + (u"#Templates#", html.escape(str(numSucceeded[4]))), + (u"#Compilation#", html.escape(str(numSucceeded[5]))), + (u"#Simulation#", html.escape(str(numSucceeded[6]))), + (u"#Verification#", html.escape(str(numSucceeded[7]))), + (u"#totalTime#", html.escape(str(datetime.timedelta(seconds=int(sum(s[3].get("exectime") or 0.0 for s in stats)))))), + (u"#config#", html.escape(json.dumps(conf["configFromFile"], indent=1, sort_keys=True))), + (u"#testsHTML#", testsHTML) + ) + open(os.path.join(stageRoot, "%s.html" % libname), "w").write(multiple_replace(htmltpl, *replacements)) + + # move results by sync operations (not available under win) + if result_location != "" and not isWin and not noSync: + result_location_libname = "%s/%s" % (result_location, libname) try: - print("copy: " + file) - shutil.copy2(file, libFilesPath) + os.mkdir(os.path.join(stageRoot, "emptydir")) except: - print("-- problem during file copy... maybe the file is still hooked by a process... :" + file) pass + check_output_log(["rsync", "-aR", "emptydir/", result_location], cwd=stageRoot) + check_output_log(["rsync", "-aR", "emptydir/", result_location_libname], cwd=stageRoot) + check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"], cwd=stageRoot) + try: + check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname], cwd=stageRoot) + except: + check_output_log(["rsync", "-aR", "emptydir/", result_location], cwd=stageRoot) + check_output_log(["rsync", "-aR", "emptydir/", result_location_libname], cwd=stageRoot) + check_output_log(["rsync", "-aR", "emptydir/", result_location_libname+"/files"], cwd=stageRoot) + check_output_log(["rsync", "-aR", "--delete-excluded", "--include-from=%s.files" % libname, "--exclude=*", "./", result_location_libname], cwd=stageRoot) + if (conf.get("referenceFiles") or "") != "" and dygraphs: + check_output_log(["rsync", "-a", dygraphs, result_location_libname+"/files"]) + else: + print("No Sync: result_location [%s] != "" and not isWin [%s] and not noSync [%s] : library: %s" % (result_location, isWin, noSync, libname)) + + # move results without sync operations (win or when --noSync is used) + if result_location != "" and (isWin or noSync): + print("--> copy res file of library: " + libname) + libPath = os.path.join(resRootPath, libname) + if not os.path.exists(libPath): + os.makedirs(libPath) + libFilesPath = os.path.join(libPath, 'files') + if os.path.exists(libFilesPath): + rmtree(libFilesPath) + os.makedirs(libFilesPath) + try: + if clean: + shutil.move("./" + libname + ".html", libPath) + else: + shutil.copy2("./" + libname + ".html", libPath) + except: + print("-- problem durin copy/move of html file of lib: " + libname) + + for file in glob.glob("./files/" + libname + '*.err') \ + + glob.glob("./files/" + libname + '*.sim') \ + + glob.glob("./files/" + libname + '*.csv') \ + + glob.glob("./files/" + libname + '*.json') \ + + glob.glob("./files/" + libname + '*.html'): + try: + print("copy: " + file) + shutil.copy2(file, libFilesPath) + except: + print("-- problem during file copy... maybe the file is still hooked by a process... :" + file) + pass if clean: for g in ["*.o","*.so","*.h","*.c","*.cpp","*.simsuccess","*.conf.json","*.tmpfiles","*.log","*.libs","OMCpp*","*.fmu*","temp_*", "*.exe", "HelloWorld.bat", "*.makefile", "*.mat","*.xml", "*.bin", "*.json"]: diff --git a/testmodel.py b/testmodel.py index 00230f1..dba4b3b 100755 --- a/testmodel.py +++ b/testmodel.py @@ -216,8 +216,12 @@ def target(res): "sim":None, "simcold":None, "diff":None, - "phase":0 + "phase":0, + # One entry per FMI simulator beyond the first, which reports itself in the + # keys above; see configs/fmi-simulators.json. + "simulators":{} } +simulators = execstat["simulators"] with open(config) as fp: conf = json.load(fp) @@ -555,9 +559,54 @@ def simulateCmd(resimulate): writeResult() # Do the simulation -# FMPy generates csv, OMSimulator generates mat (outputFormat) +# The FMU is built once and simulated with every tool the job asked for, so +# that testing FMPy no longer means building the same FMU a second time. The +# tools are described in configs/fmi-simulators.json. fmisimulator = conf.get("fmisimulator") -resFile = "%s_res.%s" % (conf["fileName"], outputFormat if not shared.isFMPy(fmisimulator) else 'csv') +fmisimulators = shared.parseFmiSimulators(conf.get("fmisimulators")) if conf.get("fmi") else [] +if conf.get("fmi") and not fmisimulators and fmisimulator: + fmisimulators = shared.parseFmiSimulators([fmisimulator]) + +def resultFile(name=None): + """Where a simulator writes its results. + + Every tool writes what its entry says, and they only need to be told apart + when more than one of them runs on the same FMU. + """ + if not name: + return "%s_res.%s" % (conf["fileName"], outputFormat) + extension = shared.fmiSimulator(name)["resultExtension"] + if len(fmisimulators) < 2: + return "%s_res.%s" % (conf["fileName"], extension) + return "%s_%s_res.%s" % (conf["fileName"], name, extension) + +def artifactPrefix(name=None): + """The files a simulator's results are written to, under files/.""" + if not name or len(fmisimulators) < 2: + return os.path.abspath("../files/%s" % conf["fileName"]).replace('\\','/') + return os.path.abspath("../files/%s_%s" % (conf["fileName"], name)).replace('\\','/') + +resFile = resultFile(fmisimulators[0][0]) if fmisimulators else resultFile() + +def simulateFmu(name, command, resFile, simFile): + """Run the FMU with one simulator, writing what it says to simFile.""" + # Only tell the runs apart when more than one of them shares the directory. + suffix = "_%s" % name if len(fmisimulators) > 1 else "" + fmitmpdir = "temp_%s%s_fmu" % (conf["fileName"].replace(".","_"), suffix) + with open("%s.tmpfiles" % conf["fileName"], "a+") as fp: + fp.write("%s\n" % fmitmpdir) + cmd = shared.fmiSimulatorCommand(name, command, + fmu="%s.fmu" % conf["fileName"].replace(".","_"), + result=resFile, + requestedResult=resFile if outputFormat != "empty" else "", + tempDir=fmitmpdir, startTime=startTime, stopTime=stopTime, + tolerance=tolerance, timeout=conf["ulimitExe"], + stepSize=stepSize) + with open(simFile,"w") as fp: + fp.write("%s\n" % cmd) + pipe = "%s%s" % (conf["fileName"], suffix) + return checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s > %s.pipe 2>&1)" + % (pipe,pipe,pipe,simFile,cmd,pipe), 1.05*conf["ulimitExe"], conf) def simElapsed(): # omc's own time: the wall clock here covers the wrong run for both flags @@ -569,24 +618,12 @@ def simElapsed(): try: # TODO: Timeout more reliably... if conf.get("fmi"): - if not conf.get("fmisimulator"): + if not fmisimulators: with open(simFile,"w") as fp: - fp.write("OMSimulator not available\n") + fp.write("No FMI simulator available\n") writeResultAndExit(0, False, omc, omc_new) - fmitmpdir = "temp_%s_fmu" % conf["fileName"].replace(".","_") - with open("%s.tmpfiles" % conf["fileName"], "a+") as fp: - fp.write("%s\n" % fmitmpdir) - if shared.isFMPy(fmisimulator): - fmisimulator = "%s simulate " % fmisimulator - cmd = "%s --start-time %g --stop-time %g --timeout %g --relative-tolerance %g --interface-type ModelExchange --solver CVode --output-interval %g %s.fmu" % (("--output-file %s" % resFile),startTime,stopTime,conf["ulimitExe"],tolerance,stepSize,conf["fileName"].replace(".","_")) - else: # OMSimulator - stepSizeStr = "" - if stepSize != 0.0: - stepSizeStr = " --stepSize=%g" % stepSize - cmd = "%s --tempDir=%s --startTime=%g --stopTime=%g%s --timeout=%g --tolerance=%g %s.fmu" % (("-r=%s" % resFile) if outputFormat != "empty" else "",fmitmpdir,startTime,stopTime,stepSizeStr,conf["ulimitExe"],tolerance,conf["fileName"].replace(".","_")) - with open(simFile,"w") as fp: - fp.write("%s %s\n" % (fmisimulator, cmd)) - res = checkOutputTimeout("(rm -f %s.pipe ; mkfifo %s.pipe ; head -c 1048576 < %s.pipe >> %s & %s %s > %s.pipe 2>&1)" % (conf["fileName"],conf["fileName"],conf["fileName"],simFile,fmisimulator,cmd,conf["fileName"]), 1.05*conf["ulimitExe"], conf) + (name, command) = fmisimulators[0] + res = simulateFmu(name, command, resFile, simFile) elif isWasmJit: if not useSimulate: cmd = simulateCmd(resimulate=True) @@ -643,112 +680,141 @@ def simElapsed(): execstat["sim"] = monotonic()-start writeResultAndExit(0, True, omc, omc_new) -if referenceFile=="": - writeResultAndExit(0, False, omc, omc_new) -if len(referenceVars)==0: - execstat["diff"] = {"time":0.0, "vars":[], "numCompared":0} - execstat["phase"]=7 - writeResultAndExit(0, False, omc, omc_new) +def verifyAgainstReference(resFile, prefix, stat): + """Compare one simulation result against the reference file. -# Check the reference file... + Fills stat["diff"] and stat["phase"] exactly as the single simulator code + did, but returns instead of ending the run, so that the other simulators + of the same FMU can be verified too. + """ + if referenceFile=="": + return + if len(referenceVars)==0: + stat["diff"] = {"time":0.0, "vars":[], "numCompared":0} + stat["phase"]=7 + return -prefix = os.path.abspath("../files/%s.diff" % conf["fileName"]).replace('\\','/') -if not os.path.exists(os.path.normpath(resFile)): - with open(errFile, 'a+') as fp: - fp.write("TODO: How the !@#!# did the simulation report success but simulation result %s does not exist to compare? outputFormat=%s" % (resFile,outputFormat)) - writeResultAndExit(0) + if not os.path.exists(os.path.normpath(resFile)): + with open(errFile, 'a+') as fp: + fp.write("TODO: How the !@#!# did the simulation report success but simulation result %s does not exist to compare? outputFormat=%s" % (resFile,outputFormat)) + return -start=monotonic() -if False and conf["simCodeTarget"] in ["Cpp"]: # This is a work-around for older C++ runtime not supporting variable filters. We don't really need it for master, so let's no use it. - if not sendExpressionTimeout(omc_new, 'filterSimulationResults("%s", "updated%s", vars={%s}, removeDescription=false, hintReadAllVars=false)' % (resFile, resFile, ", ".join(['"%s"' % s for s in referenceVars])), conf["ulimitOmc"]): + start=monotonic() + if False and conf["simCodeTarget"] in ["Cpp"]: # This is a work-around for older C++ runtime not supporting variable filters. We don't really need it for master, so let's no use it. + if not sendExpressionTimeout(omc_new, 'filterSimulationResults("%s", "updated%s", vars={%s}, removeDescription=false, hintReadAllVars=false)' % (resFile, resFile, ", ".join(['"%s"' % s for s in referenceVars])), conf["ulimitOmc"]): + with open(errFile, 'a+') as fp: + fp.write("Failed to filter simulation results. Took time: %.2f\n" % (monotonic()-start)) + return + os.remove(resFile) + os.rename("updated" + resFile, resFile) with open(errFile, 'a+') as fp: - fp.write("Failed to filter simulation results. Took time: %.2f\n" % (monotonic()-start)) - writeResultAndExit(0, False, omc, omc_new) - os.remove(resFile) - os.rename("updated" + resFile, resFile) - with open(errFile, 'a+') as fp: - fp.write("Filtered simulation results in time: %.2f\n" % (monotonic()-start)) -start=monotonic() -try: - (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","%s",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' % - (resFile, referenceFile, prefix, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"]) -except TimeoutError as e: - with open(errFile, 'a+') as fp: - fp.write("Timeout error for diffSimulationResults") - writeResultAndExit(0, False, omc, omc_new) + fp.write("Filtered simulation results in time: %.2f\n" % (monotonic()-start)) + start=monotonic() + try: + (referenceOK,diffVars) = sendExpressionTimeout(omc_new, 'diffSimulationResults("%s","%s","%s",relTol=%g,relTolDiffMinMax=%g,rangeDelta=%g)' % + (resFile, referenceFile, prefix, conf["reference_reltol"],conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"]), conf["ulimitOmc"]) + except TimeoutError as e: + with open(errFile, 'a+') as fp: + fp.write("Timeout error for diffSimulationResults") + return -execstat["diff"] = {"time":monotonic()-start, "vars":[], "numCompared":len(referenceVars)} -if len(diffVars)==0 and referenceOK: - execstat["phase"]=7 - with open(errFile, 'a+') as fp: - fp.write("Reference file matches\n") -else: - with open(errFile, 'a+') as fp: - fp.write(omc_new.sendExpression('OpenModelica.Scripting.getErrorString()', parsed = False)) - fp.write("\nVariables in the reference:" ) - fp.write(",".join(referenceVars)+"\n") - resVars=omc_new.sendExpression('readSimulationResultVars("%s", readParameters=true, openmodelicaStyle=true)' % resFile) - fp.write("\nVariables in the result:" ) - fp.write(",".join(resVars)+"\n") - diffFiles = [prefix + "." + var for var in diffVars] - execstat["diff"]["vars"]=diffVars - - # Create a file containing only the calibrated variables, for easy display - lstfiles = "\n".join(['
  • %s (javascript) (csv)
    • ' % (str.split(str(f),".diff.",1)[1],str(os.path.basename(f)),str(os.path.basename(f))) for f in diffFiles]) - with open(prefix+".html", 'w') as fp: - fp.write('

    %s differences from the reference file

    startTime: %g

    stopTime: %g

    Simulated using tolerance: %g

      %s
    ' % (conf["modelName"], startTime, stopTime, tolerance, lstfiles)) - for var in diffVars: - if "/" in var: - continue # Quoted identifier, or possibly an error message... Either way, avoid crapping out below - with open(prefix+"."+var+".html", 'w') as fp: - fp.write(""" - - - - - -
    -

    - - - - - - - - - - - -Parameters used for the comparison: Relative tolerance %g (local), %g (relative to max-min). Range delta %g.

    - - -""" % (tolerance, conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], os.path.basename(prefix + "." + var + ".csv"), var)) + stat["diff"] = {"time":monotonic()-start, "vars":[], "numCompared":len(referenceVars)} + if len(diffVars)==0 and referenceOK: + stat["phase"]=7 + with open(errFile, 'a+') as fp: + fp.write("Reference file matches\n") + else: + with open(errFile, 'a+') as fp: + fp.write(omc_new.sendExpression('OpenModelica.Scripting.getErrorString()', parsed = False)) + fp.write("\nVariables in the reference:" ) + fp.write(",".join(referenceVars)+"\n") + resVars=omc_new.sendExpression('readSimulationResultVars("%s", readParameters=true, openmodelicaStyle=true)' % resFile) + fp.write("\nVariables in the result:" ) + fp.write(",".join(resVars)+"\n") + diffFiles = [prefix + "." + var for var in diffVars] + stat["diff"]["vars"]=diffVars + + # Create a file containing only the calibrated variables, for easy display + lstfiles = "\n".join(['
  • %s (javascript) (csv)
    • ' % (str.split(str(f),".diff.",1)[1],str(os.path.basename(f)),str(os.path.basename(f))) for f in diffFiles]) + with open(prefix+".html", 'w') as fp: + fp.write('

    %s differences from the reference file

    startTime: %g

    stopTime: %g

    Simulated using tolerance: %g

      %s
    ' % (conf["modelName"], startTime, stopTime, tolerance, lstfiles)) + for var in diffVars: + if "/" in var: + continue # Quoted identifier, or possibly an error message... Either way, avoid crapping out below + with open(prefix+"."+var+".html", 'w') as fp: + fp.write(""" + + + + + +
    +

    + + + + + + + + + + + + Parameters used for the comparison: Relative tolerance %g (local), %g (relative to max-min). Range delta %g.

    + + + """ % (tolerance, conf["reference_reltolDiffMinMax"], conf["reference_rangeDelta"], os.path.basename(prefix + "." + var + ".csv"), var)) + + +# The first simulator's results are the ones every non-FMI code path expects. +verifyAgainstReference(resFile, artifactPrefix(fmisimulators[0][0] if fmisimulators else None) + ".diff", execstat) + +# The FMU is built; every other simulator the job asked for is now only a +# simulation and a comparison. A tool that times out or fails takes its own +# results down with it and leaves the others alone - unlike the first one, +# whose timeout ends the model as it always has. +for (name, command) in fmisimulators[1:]: + stat = {"sim": None, "diff": None, "phase": 5} + simulators[name] = stat + simFileOther = os.path.abspath("../files/%s_%s.sim" % (conf["fileName"], name)).replace('\\','/') + other = resultFile(name) + start = monotonic() + try: + simulateFmu(name, command, other, simFileOther) + stat["sim"] = monotonic()-start + stat["phase"] = 6 + verifyAgainstReference(other, artifactPrefix(name) + ".diff", stat) + except TimeoutError as e: + stat["sim"] = monotonic()-start + with open(errFile, 'a+') as fp: + fp.write("%s timed out simulating the FMU\n" % name) + writeResult() -# quit omc_new +# quit omc_new: every verification needed it omc_new = quit_omc(omc_new) writeResultAndExit(0)