From 93a5d32d5155da4b77c5b9a376fb135ce0c985b9 Mon Sep 17 00:00:00 2001 From: Adrian Pop Date: Wed, 12 Aug 2026 11:59:21 +0200 Subject: [PATCH] Test every library of an FMI job with FMI A library whose reference files are already prepared by another library took a `continue` that skipped the rest of the configuration loop - and the only thing after it is the line that turns FMI on: if allTestsFmi: c["fmi"] = "2.0" So four libraries of configs/conf.json have been running as ordinary native models inside the FMI jobs: PowerGrids_symb_jac, PowerGrids_dev, ClaRa_dev and ScalableTestSuite_noopt, each of them the second library to want a reference directory that PowerGrids, ClaRa and ScalableTestSuite had already cloned. They verify identically in master and in master-fmi - 66, 19, 72 and 244 models - while every library that prepares its own directory drops as an FMI job should (242 to 176 for ScalableTestSuite, 65 to 31 for PowerGrids). The reuse also looked the directory up under a key it had not computed yet, so it took whatever `destination` the previous library left behind. Both are fixed by computing the normalised destination first and putting the preparation in an else branch, which leaves nothing after the loop's body unreachable. Since #297 this crashed the run rather than quietly mistesting it. Such a model has no per-simulator results, so the second simulator's branch reported the phase the *first* simulator reached - 7, verified - with no comparison to go with it, and building the report died on it after seven hours: ("%s (%d verified)" % (timeSeconds(diff.get("time")), ...)) if s[3]["phase"]>=7 AttributeError: 'NoneType' object has no attribute 'get' A simulator that never ran a model now reports the phase the shared build reached and never a phase another simulator went on to reach with the same FMU, in the report and in the database row alike; and a model with nothing to compare renders as an empty cell whatever phase it claims, so a mismatch can never again throw away a finished run. Fixes the master-fmi failure of build 11369. --- test.py | 131 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 72 insertions(+), 59 deletions(-) diff --git a/test.py b/test.py index fb24cc8..4b61852 100755 --- a/test.py +++ b/test.py @@ -490,65 +490,65 @@ def testHelloWorld(cmd): refFilesGitTag = c["referenceFiles"]["git-ref"].strip() if not refFilesGitTag.startswith("origin/"): refFilesGitTag = 'origin/%s' % refFilesGitTag - if c["referenceFiles"]["destination"] in preparedReferenceDirs: + # Several libraries share one set of reference files. The first of them + # prepares the directory and the rest take what it recorded - but the rest + # of this loop still has to run for every one of them. + destination = os.path.normpath(c["referenceFiles"]["destination"]) + if destination in preparedReferenceDirs: (c["referenceFiles"],c["referenceFilesURL"]) = preparedReferenceDirs[destination] - continue - giturl = c["referenceFiles"]["giturl"] - destination = c["referenceFiles"]["destination"] - if DEBUG: - print("destination %s" % destination) - destination = os.path.normpath(destination) - if DEBUG: - print("normalized destination %s" % destination) - print("not os.path.isdir(destination): %s" % (not os.path.isdir(destination))) - destinationReal = os.path.realpath(destination) - if DEBUG: - print("destinationReal %s" % destinationReal) - destinationReal = os.path.normpath(destinationReal) - if DEBUG: - print("normalized destinationReal %s" % destinationReal) - print("referenceFiles:", c["referenceFiles"]) + else: + giturl = c["referenceFiles"]["giturl"] + if DEBUG: + print("destination %s" % destination) + print("not os.path.isdir(destination): %s" % (not os.path.isdir(destination))) + destinationReal = os.path.realpath(destination) + if DEBUG: + print("destinationReal %s" % destinationReal) + destinationReal = os.path.normpath(destinationReal) + if DEBUG: + print("normalized destinationReal %s" % destinationReal) + print("referenceFiles:", c["referenceFiles"]) + + if not os.path.isdir(destination): + if "git-directory" in c["referenceFiles"]: + # Sparse clone + os.makedirs(destination) + check_call_log(["git", "init"], stderr=subprocess.STDOUT, cwd=destinationReal) + check_call_log(["git", "remote", "add", "-f", "origin", giturl], stderr=subprocess.STDOUT, cwd=destinationReal) + check_call_log(["git", "config", "core.sparseCheckout", "true"], stderr=subprocess.STDOUT, cwd=destinationReal) + file = open(os.path.join(destinationReal,".git", "info", "sparse-checkout"), "a") + file.write(c["referenceFiles"]["git-directory"].strip()) + file.close() + else: + # Clone + check_call_log(["git", "clone", giturl, destination], stderr=subprocess.STDOUT) + + check_call_log(["git", "clean", "-fdx", "--exclude=*.hash"], stderr=subprocess.STDOUT, cwd=destinationReal) + check_call_log(["git", "fetch", "origin"], stderr=subprocess.STDOUT, cwd=destination) + # do not fail if the branch we were given doesn't exist + try: + check_call_log(["git", "reset", "--hard", refFilesGitTag], stderr=subprocess.STDOUT, cwd=destinationReal) + except subprocess.CalledProcessError as e: + print(e.output) + check_call_log(["git", "clean", "-fdx", "--exclude=*.hash"], stderr=subprocess.STDOUT, cwd=destinationReal) + if glob.glob(destinationReal + "/*.mat.xz"): + check_call_log(["find", ".", "-name", "*.mat.xz", "-exec", "xz", "--decompress", "--keep", "{}", ";"], stderr=subprocess.STDOUT, cwd=destinationReal) + try: + githash = check_output_log(["git", "rev-parse", "--verify", "HEAD"], stderr=subprocess.STDOUT, cwd=destinationReal, encoding='utf8') + except subprocess.CalledProcessError as e: + print(e.output) - if not os.path.isdir(destination): if "git-directory" in c["referenceFiles"]: - # Sparse clone - os.makedirs(destination) - check_call_log(["git", "init"], stderr=subprocess.STDOUT, cwd=destinationReal) - check_call_log(["git", "remote", "add", "-f", "origin", giturl], stderr=subprocess.STDOUT, cwd=destinationReal) - check_call_log(["git", "config", "core.sparseCheckout", "true"], stderr=subprocess.STDOUT, cwd=destinationReal) - file = open(os.path.join(destinationReal,".git", "info", "sparse-checkout"), "a") - file.write(c["referenceFiles"]["git-directory"].strip()) - file.close() + c["referenceFiles"] = os.path.join(destinationReal, c["referenceFiles"]['git-directory']) + print(c["referenceFiles"]) else: - # Clone - check_call_log(["git", "clone", giturl, destination], stderr=subprocess.STDOUT) - - check_call_log(["git", "clean", "-fdx", "--exclude=*.hash"], stderr=subprocess.STDOUT, cwd=destinationReal) - check_call_log(["git", "fetch", "origin"], stderr=subprocess.STDOUT, cwd=destination) - # do not fail if the branch we were given doesn't exist - try: - check_call_log(["git", "reset", "--hard", refFilesGitTag], stderr=subprocess.STDOUT, cwd=destinationReal) - except subprocess.CalledProcessError as e: - print(e.output) - check_call_log(["git", "clean", "-fdx", "--exclude=*.hash"], stderr=subprocess.STDOUT, cwd=destinationReal) - if glob.glob(destinationReal + "/*.mat.xz"): - check_call_log(["find", ".", "-name", "*.mat.xz", "-exec", "xz", "--decompress", "--keep", "{}", ";"], stderr=subprocess.STDOUT, cwd=destinationReal) - try: - githash = check_output_log(["git", "rev-parse", "--verify", "HEAD"], stderr=subprocess.STDOUT, cwd=destinationReal, encoding='utf8') - except subprocess.CalledProcessError as e: - print(e.output) + c["referenceFiles"] = destinationReal - if "git-directory" in c["referenceFiles"]: - c["referenceFiles"] = os.path.join(destinationReal, c["referenceFiles"]['git-directory']) - print(c["referenceFiles"]) - else: - c["referenceFiles"] = destinationReal - - if giturl.startswith("https://github.com"): - c["referenceFilesURL"] = '%s (%s)' % (giturl, githash.strip(), giturl, githash.strip()) - else: - c["referenceFilesURL"] = "%s (%s)" % (giturl, githash.strip()) - preparedReferenceDirs[destination] = (c["referenceFiles"],c["referenceFilesURL"]) + if giturl.startswith("https://github.com"): + c["referenceFilesURL"] = '%s (%s)' % (giturl, githash.strip(), giturl, githash.strip()) + else: + c["referenceFilesURL"] = "%s (%s)" % (giturl, githash.strip()) + preparedReferenceDirs[destination] = (c["referenceFiles"],c["referenceFilesURL"]) else: raise Exception("Unknown referenceFiles in config: %s" % (str(c["referenceFiles"]))) @@ -952,6 +952,19 @@ def loadJsonOrEmptySet(f): print(" %-70s cold %8.4f hot %8.4f" % (model, data["simcold"], data.get("sim") or 0.0)) sys.stdout.flush() +BUILD_PHASE = 5 +"""The last phase a model shares with every simulator of its FMU.""" + +def phaseWithoutSimulator(data): + """How far a model got for a simulator that never ran it. + + Everything up to the build is shared, so such a model reports the phase the + build reached and never the phase another simulator went on to reach with the + same FMU. Reporting that one would claim a simulation, and a verification, + that this simulator never performed. + """ + return min(data.get("phase") or 0, BUILD_PHASE) + def resultValues(model, libname, data, simulator=None): """One row of a branch table. @@ -961,9 +974,7 @@ def resultValues(model, libname, data, simulator=None): """ simulated = data if simulator is None else (data.get("simulators") or {}).get(simulator) if simulated is None: - # The model never got as far as being simulated, so every simulator of it - # reports the phase the build stopped at rather than a failure of its own. - simulated = {"phase": data.get("phase") or 0} + simulated = {"phase": phaseWithoutSimulator(data)} diff = simulated.get("diff") or {} return (testRunStartTimeAsEpoch, libname, @@ -1095,7 +1106,7 @@ def dataForSimulator(data, simulator): merged["diff"] = (simulated or {}).get("diff") # A model that never got as far as being simulated stopped in the build, # which every simulator of it shares. - merged["phase"] = simulated.get("phase") if simulated else data.get("phase") + merged["phase"] = simulated.get("phase") if simulated is not None else phaseWithoutSimulator(data) return merged def artifactSuffix(simulator): @@ -1155,7 +1166,9 @@ def artifactSuffix(simulator): ('%s' % (errPrefix + ".err", html.escape(s[1]))) if is_non_zero_file(errPrefix + ".err") else html.escape(s[1]), (' (sim)' % (filename_prefix + ".sim")) if is_non_zero_file(filename_prefix + suffix + ".sim") else "", checkPhase(s[3]["phase"], 7) if s[3]["phase"]>=6 else "#FFFFFF", - ("%s (%d verified)" % (timeSeconds(diff.get("time")), diff.get("numCompared"))) if s[3]["phase"]>=7 else (" " if diff is None else + # Nothing was compared, whatever phase the model reports. + " " if diff is None else + (("%s (%d verified)" % (timeSeconds(diff.get("time")), diff.get("numCompared"))) if s[3]["phase"]>=7 else ('%s (%d/%d failed)' % (timeSeconds(diff.get("time")), filename_prefix, len(diff.get("vars")), diff.get("numCompared")))), checkPhase(s[3]["phase"], 6), timeSeconds(s[3].get("sim") or 0),