From 3a6eedf3ba84ed4571f6f5ea6ab5de3ce4a70d15 Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Wed, 12 Aug 2026 22:00:06 +0200 Subject: [PATCH] Let the other simulators run when the first one fails An FMI job that runs several simulators over one FMU (#297, #309) gives the first of them a veto over the rest. When it fails, the TimeoutError handler calls writeResultAndExit, the process is gone, and the loop that runs fmisimulators[1:] never starts -- but a row is still written for every simulator, recording the phase the first one stopped at. The other tools are marked as having failed at a model they were never given. checkOutputTimeout raises TimeoutError for a command that exits non-zero as well as for one that runs out of time, so this covers an ordinary failure and not just a slow one. In the master-fmi run of 2026-08-12 this cost 433 models. They had never simulated under OMSimulator -- phase 5 in every master-fmi run back to 2026-08-08 -- but FMPy ran them, and the previous master-fmi-fmpy run has them at phase 6 or 7. In the first combined run they are phase 5 in both tables, with FMPy's simulation time recorded as exactly 0: OMSimulator failed in under a second for 344 of them, and FMPy never started. They show up as 433 of the 653 regressions on the comparison page, next to the 213 that PR #309 really did find. It is also why that page reports OMSimulator's version, and its JSON parse warning, in place of "FMPy version 0.3.29": FMPy never ran to say what it was. The first simulator now records its failure and falls through to the others, which is what every simulator after it already did. Nothing changes when only one simulator is configured: that case still ends the model where it always has, so the non-FMI paths and the single-simulator FMI jobs are untouched. Verified on MyLibrary.Blocks.Examples.PID_Controller with the first simulator replaced by /bin/false: before phase 5, "simulators": {} after phase 5, "simulators": {"fmpy": {"sim": 0.64, "phase": 7}} and, unchanged in both: one failing simulator phase 5, os._exit as before both simulators working OMSimulator phase 7, fmpy phase 7 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MYvMaAotMy425H7nvKzWX7 --- testmodel.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/testmodel.py b/testmodel.py index bd89869..da5e1c7 100755 --- a/testmodel.py +++ b/testmodel.py @@ -646,6 +646,9 @@ def simElapsed(): return monotonic()-start start=monotonic() +# Set when the first FMI simulator fails and there are others waiting for the +# same FMU, so that its result file is not compared against the reference. +firstSimulatorFailed = False try: # TODO: Timeout more reliably... if conf.get("fmi"): @@ -709,7 +712,20 @@ def simElapsed(): execstat["phase"] = 6 except TimeoutError as e: execstat["sim"] = monotonic()-start - writeResultAndExit(0, True, omc, omc_new) + # checkOutputTimeout raises TimeoutError for a command that fails as well as + # for one that runs out of time, so this covers both. + if len(fmisimulators) > 1: + # The FMU is built and the other simulators are about to run it. What the + # first one did says nothing about them, so record the failure and let them + # have their turn. Ending the model here would write them all down as having + # failed at this phase without ever being started, which is what the loop + # below already avoids for a failure in any simulator but the first. + firstSimulatorFailed = True + with open(errFile, 'a+') as fp: + fp.write("%s failed or timed out simulating the FMU; the other simulators still get to run it\n" + % fmisimulators[0][0]) + else: + writeResultAndExit(0, True, omc, omc_new) def verifyAgainstReference(resFile, prefix, stat): """Compare one simulation result against the reference file. @@ -822,12 +838,14 @@ def verifyAgainstReference(resFile, prefix, stat): # The first simulator's results are the ones every non-FMI code path expects. -verifyAgainstReference(resFile, artifactPrefix(fmisimulators[0][0] if fmisimulators else None) + ".diff", execstat) +# There is nothing to compare when it never produced any. +if not firstSimulatorFailed: + verifyAgainstReference(resFile, artifactPrefix(fmisimulators[0][0] if fmisimulators else None) + ".diff", execstat) # The FMU is built; every other simulator the job asked for is now only a # simulation and a comparison. A tool that times out or fails takes its own -# results down with it and leaves the others alone - unlike the first one, -# whose timeout ends the model as it always has. +# results down with it and leaves the others alone - the first one included, +# see the TimeoutError handler above. for (name, command) in fmisimulators[1:]: stat = {"sim": None, "diff": None, "phase": 5} simulators[name] = stat