From 6fba921165763d87e7a226bd077383ee7f89e205 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Wed, 1 Jul 2026 16:51:08 -0500 Subject: [PATCH 01/24] yaml config: add YAML config front-end for parametric fabrics Introduce a config compiler that reads the friendly YAML/JSON topology and component format and lowers it to the same ConfigVTable the .conf text parser produces, so codes_mapping and the models consume it unchanged through the configuration_get_* accessors. The front-end is three layers so the parsing/validation logic is unit-testable and never leaks ROSS: - config_compiler: a pure C++ core, compile(text) -> compiled_config, an ordered plain-data image of the LPGROUPS/PARAMS tree. It does all parsing and validation with no dependency on ROSS, MPI, or abort: on any invalid input it throws config_error, and ryml's own parse errors are routed to throw as well. - config_emitter: a dumb pass that turns the compiled_config into a ConfigVTable; no validation, its coverage comes from the equivalence tests. - yaml_configfile: a thin extern "C" shim, the only place ROSS lives -- it runs compile() + emit() and translates a config_error into tw_error. Real-run behavior (abort + diagnostic) is unchanged from the .conf path; "abort" is now a boundary policy rather than being baked into every validation site. Every rank reads identical bytes and runs the identical deterministic compile, so malformed input still aborts everywhere at once. configuration_load dispatches by file extension: a .yaml/.yml/.json path goes through the front-end, everything else through the existing text parser. The global config stays a ConfigVTable* either way, so no call site or model changes. ryml is vendored and always built, so there is no feature guard. Validation the throwing core enforces now: schema_version is required, integer, and any value this build doesn't know is a hard error (a newer config can't be read safely); unknown top-level keys, unknown topology keys, and nested blocks a component/fabric doesn't consume are errors rather than silent drops. This first cut compiles the parametric-fabric source for the regular dragonfly: the group/repetition and per-router LP counts are derived from the fabric shape (the same num_routers-driven math the model does internally), per-link-class bandwidth/vc_size and routing map to the model's PARAMS, and modelnet_order is derived from the fabric model. Values are carried through as their raw scalar text so a compiled config is byte-comparable to the .conf it replaces. Verified against modelnet-synthetic-dragonfly.conf: model-net-synthetic produces byte-identical per-LP lp-io output from the .yaml twin. --- src/CMakeLists.txt | 23 + src/modelconfig/config_compiler.cxx | 426 ++++++++++++++++++ src/modelconfig/config_compiler.h | 90 ++++ src/modelconfig/config_emitter.cxx | 47 ++ src/modelconfig/config_emitter.h | 40 ++ src/modelconfig/configuration.c | 37 +- src/modelconfig/yaml_configfile.cxx | 38 ++ src/modelconfig/yaml_configfile.h | 34 ++ .../conf/modelnet-synthetic-dragonfly.yaml | 30 ++ 9 files changed, 755 insertions(+), 10 deletions(-) create mode 100644 src/modelconfig/config_compiler.cxx create mode 100644 src/modelconfig/config_compiler.h create mode 100644 src/modelconfig/config_emitter.cxx create mode 100644 src/modelconfig/config_emitter.h create mode 100644 src/modelconfig/yaml_configfile.cxx create mode 100644 src/modelconfig/yaml_configfile.h create mode 100644 src/network-workloads/conf/modelnet-synthetic-dragonfly.yaml diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 56e0ddd8..0278e0ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -92,6 +92,19 @@ list(APPEND SRCS modelconfig/configuration.c modelconfig/txt_configfile.h modelconfig/txt_configfile.c + + # YAML/JSON config front-end: a pure C++ core that compiles the friendly + # topology/component format to the compiled_config IR (config_compiler), a + # dumb emitter that lowers that IR to the same ConfigVTable the .conf parser + # yields (config_emitter), and the thin extern "C" shim configuration.c + # dispatches to (yaml_configfile). Always built -- ryml is vendored, so the + # front-end is core, not an optional feature. + modelconfig/config_compiler.h + modelconfig/config_compiler.cxx + modelconfig/config_emitter.h + modelconfig/config_emitter.cxx + modelconfig/yaml_configfile.h + modelconfig/yaml_configfile.cxx ) list(APPEND LIBS_TO_LINK ROSS::ROSS) @@ -196,6 +209,16 @@ target_include_directories(codes PUBLIC target_link_libraries(codes PUBLIC ${LIBS_TO_LINK}) +# ryml backs the YAML config front-end and is an implementation detail (no public +# codes header includes it). codes is a STATIC library in an install/export set, +# so instead of linking a separate in-tree lib (which the build-tree export() +# would drag into the export set), compile the vendored amalgamation's single +# implementation TU straight into codes and put its headers on codes' PRIVATE +# include path. The ryml objects then live inside codes.a, invisible to the +# installed interface. See thirdparty/rapidyaml/CMakeLists.txt. +target_sources(codes PRIVATE ${CODES_RYML_IMPL_TU}) +target_include_directories(codes SYSTEM PRIVATE ${CODES_RYML_INCLUDE_DIRS}) + add_executable(topology-test networks/model-net/topology-test.c) add_executable(model-net-mpi-replay network-workloads/model-net-mpi-replay.c network-workloads/model-net-mpi-replay-main.c) if(USE_DUMPI) diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx new file mode 100644 index 00000000..f1f78cbb --- /dev/null +++ b/src/modelconfig/config_compiler.cxx @@ -0,0 +1,426 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#include "config_compiler.h" + +#include + +#include +#include +#include +#include +#include + +namespace codes { +namespace config { + +/* ------------------------------------------------------------------------- + * compiled_config IR builders + * ---------------------------------------------------------------------- */ + +void compiled_section::add_key(std::string key, std::string value) { + keys.push_back(compiled_key{std::move(key), {std::move(value)}}); +} + +void compiled_section::add_key(std::string key, std::vector values) { + keys.push_back(compiled_key{std::move(key), std::move(values)}); +} + +compiled_section& compiled_section::add_subsection(std::string subname) { + subsections.push_back(compiled_section{std::move(subname), {}, {}}); + return subsections.back(); +} + +compiled_section& compiled_config::add_section(std::string name) { + sections.push_back(compiled_section{std::move(name), {}, {}}); + return sections.back(); +} + +namespace { + +/* ------------------------------------------------------------------------- + * ryml plumbing -- route parser errors to config_error, never to ROSS. The + * core throws; the extern "C" shim is the only place tw_error lives. + * ---------------------------------------------------------------------- */ + +[[noreturn]] void ryml_throw(ryml::csubstr msg, ryml::ErrorDataBasic const& ed, void*) { + std::string where = + ed.location.line ? (" at line " + std::to_string(ed.location.line)) : std::string(); + throw config_error("config error: YAML error" + where + ": " + std::string(msg.str, msg.len)); +} + +/* ryml raises *parse* (syntax) errors through the separate parse-error callback, + * whose default implementation prints + aborts; install our own so a malformed + * document throws config_error like everything else. The message ryml builds + * already carries the location, so it is passed through as-is. */ +[[noreturn]] void ryml_throw_parse(ryml::csubstr msg, ryml::ErrorDataParse const& ed, void*) { + std::string where = + ed.ymlloc.line ? (" at line " + std::to_string(ed.ymlloc.line)) : std::string(); + throw config_error("config error: malformed YAML" + where + ": " + + std::string(msg.str, msg.len)); +} + +/* raw scalar text of a node, preserving the exact source spelling */ +std::string scalar(ryml::ConstNodeRef n) { + ryml::csubstr v = n.val(); + return std::string(v.str, v.len); +} + +std::string key_of(ryml::ConstNodeRef n) { + ryml::csubstr k = n.key(); + return std::string(k.str, k.len); +} + +bool has(ryml::ConstNodeRef n, const char* key) { + return n.readable() && n.is_map() && n.has_child(ryml::to_csubstr(key)); +} + +/* Parse a scalar strictly as a base-10 integer (whole string consumed). */ +long parse_int_strict(const std::string& s, const char* what) { + errno = 0; + char* end = nullptr; + long v = std::strtol(s.c_str(), &end, 10); + if (s.empty() || end == s.c_str() || *end != '\0' || errno != 0) + throw config_error(std::string("config error: ") + what + " must be an integer, got \"" + + s + "\""); + return v; +} + +/* ------------------------------------------------------------------------- + * Friendly-format intermediate representation + * ---------------------------------------------------------------------- */ + +using kv_list = std::vector>; + +/* A custom component: a model paired with configured parameters. */ +struct component { + std::string key; /* the components: key referenced by a topology */ + std::string model; /* ComponentModel name (nw-lp, ...) */ + std::string network; /* enumerated flat models: the NIC model a compute node + runs its workload over (added with the flat path) */ + kv_list params; /* scalar model params, raw text, in source order */ +}; + +/* A per-link-class parameter block (e.g. dragonfly local/global/cn). */ +struct link_class { + std::string name; /* class name; combines with each param as _ */ + kv_list params; +}; + +/* A parametric fabric: an HPC topology described by shape parameters. */ +struct fabric { + std::string model; /* network model, e.g. "dragonfly" */ + kv_list shape; /* shape parameters (also drive count derivation) */ + std::vector links; /* per-link-class bandwidth / vc_size */ + kv_list routing; /* routing.* (algorithm maps to PARAMS "routing") */ + kv_list connections; /* connections.{intra,inter}: file-enumerated wiring */ + kv_list extra; /* other scalar fabric keys -> PARAMS verbatim */ + std::string hosts_component; /* hosts.component: the per-terminal workload */ +}; + +struct friendly_config { + std::vector components; + bool parametric = false; + fabric fab; /* parametric topology */ + + const component* find_component(const std::string& k) const { + for (const component& c : components) + if (c.key == k) + return &c; + return nullptr; + } +}; + +/* ------------------------------------------------------------------------- + * Model registry -- maps a friendly fabric model name to its LP-type names, + * modelnet_order method names, and shape->counts derivation. + * ---------------------------------------------------------------------- */ + +/* The LP layout of one repetition. */ +struct layout { + long repetitions; + long terminals_per_rep; /* workload + NIC LP count per repetition */ + long routers_per_rep; /* router/switch LP count per repetition */ +}; + +struct fabric_model { + const char* name; /* friendly name used in fabric.model */ + const char* terminal_lp; /* LPGROUPS lp-type name for the NIC/terminal */ + const char* router_lp; /* LPGROUPS lp-type name for the router/switch */ + const char* term_method; /* modelnet_order method name for the terminal */ + const char* router_method; /* modelnet_order method for the router, or nullptr + if the router is not a separate model-net method */ + layout (*derive)(const kv_list& shape); /* shape -> LP layout */ +}; + +/* Look up a shape value by name, throwing if absent. */ +long shape_int(const kv_list& shape, const char* key) { + for (const auto& kv : shape) + if (kv.first == key) + return std::strtol(kv.second.c_str(), nullptr, 10); + throw config_error(std::string("config error: fabric shape is missing required key \"") + key + + "\""); +} + +/* Regular (Kim-Dally) dragonfly: every count follows from num_routers, the + * routers per group -- the same derivation the model does internally + * (num_cn = num_routers/2, num_groups = num_routers*num_cn + 1). */ +layout derive_dragonfly(const kv_list& shape) { + long num_routers = shape_int(shape, "num_routers"); + if (num_routers <= 0) + throw config_error("config error: dragonfly num_routers must be positive"); + long num_cn = num_routers / 2; + long num_groups = num_routers * num_cn + 1; + return {num_groups * num_routers, num_cn, 1}; +} + +const fabric_model fabric_models[] = { + {"dragonfly", "modelnet_dragonfly", "modelnet_dragonfly_router", "dragonfly", + "dragonfly_router", derive_dragonfly}, +}; + +const fabric_model* find_fabric_model(const std::string& name) { + for (const fabric_model& m : fabric_models) + if (name == m.name) + return &m; + return nullptr; +} + +/* ------------------------------------------------------------------------- + * Parse: ryml tree -> friendly IR (validating as it goes -- unknown / + * unconsumed keys are errors, not silent drops). + * ---------------------------------------------------------------------- */ + +void parse_components(ryml::ConstNodeRef root, friendly_config& cfg) { + if (!has(root, "components")) + return; + ryml::ConstNodeRef comps = root["components"]; + if (!comps.is_map()) + throw config_error("config error: \"components\" must be a map of name -> component"); + for (ryml::ConstNodeRef cnode : comps.children()) { + component c; + c.key = key_of(cnode); + for (ryml::ConstNodeRef f : cnode.children()) { + std::string k = key_of(f); + if (k == "model") + c.model = scalar(f); + else if (k == "network") + c.network = scalar(f); + else if (k == "type") + ; /* inferred from the model; not needed for the compiled config */ + else if (f.is_keyval()) + c.params.emplace_back(k, scalar(f)); + else + throw config_error("config error: component \"" + c.key + + "\": unexpected block \"" + k + + "\"; a component takes a model, an optional network, and scalar " + "params (per-node data, edges and inline workloads are not " + "supported)"); + } + cfg.components.push_back(std::move(c)); + } +} + +void parse_fabric(ryml::ConstNodeRef fnode, fabric& fab) { + for (ryml::ConstNodeRef c : fnode.children()) { + std::string k = key_of(c); + if (k == "model") { + fab.model = scalar(c); + } else if (k == "shape") { + for (ryml::ConstNodeRef s : c.children()) + fab.shape.emplace_back(key_of(s), scalar(s)); + } else if (k == "links") { + for (ryml::ConstNodeRef lc : c.children()) { + link_class cls; + cls.name = key_of(lc); + for (ryml::ConstNodeRef p : lc.children()) + cls.params.emplace_back(key_of(p), scalar(p)); + fab.links.push_back(std::move(cls)); + } + } else if (k == "routing") { + for (ryml::ConstNodeRef r : c.children()) + fab.routing.emplace_back(key_of(r), scalar(r)); + } else if (k == "connections") { + /* file-enumerated dragonflies reference the binary connection files + * by path; the compiler maps intra/inter to the model's key names. */ + for (ryml::ConstNodeRef cn : c.children()) + fab.connections.emplace_back(key_of(cn), scalar(cn)); + } else if (c.is_keyval()) { + fab.extra.emplace_back(k, scalar(c)); + } else { + throw config_error("config error: fabric: unexpected block \"" + k + "\""); + } + } +} + +void parse_topology(ryml::ConstNodeRef root, friendly_config& cfg) { + if (!has(root, "topology")) + throw config_error("config error: missing required \"topology\" block"); + ryml::ConstNodeRef topo = root["topology"]; + + std::string format = has(topo, "format") ? scalar(topo["format"]) : std::string(); + + if (format == "parametric") { + cfg.parametric = true; + /* only these keys are consumed for a parametric topology. */ + for (ryml::ConstNodeRef c : topo.children()) { + std::string k = key_of(c); + if (k != "format" && k != "fabric" && k != "hosts") + throw config_error("config error: topology: unexpected key \"" + k + + "\" for a parametric topology"); + } + if (!has(topo, "fabric")) + throw config_error("config error: parametric topology needs a \"fabric\" block"); + parse_fabric(topo["fabric"], cfg.fab); + if (has(topo, "hosts") && has(topo["hosts"], "component")) + cfg.fab.hosts_component = scalar(topo["hosts"]["component"]); + else + throw config_error("config error: parametric topology needs hosts.component naming the " + "per-terminal workload"); + } else if (format.empty()) { + throw config_error("config error: topology needs a \"format\" (e.g. parametric)"); + } else { + throw config_error("config error: unknown topology format \"" + format + "\""); + } +} + +friendly_config parse_friendly(ryml::ConstNodeRef root) { + /* reject unknown top-level keys rather than silently ignoring them. */ + for (ryml::ConstNodeRef c : root.children()) { + std::string k = key_of(c); + if (k != "schema_version" && k != "components" && k != "topology") + throw config_error("config error: unexpected top-level key \"" + k + "\""); + } + friendly_config cfg; + parse_components(root, cfg); + parse_topology(root, cfg); + return cfg; +} + +/* schema_version is required, integer, and any value this build doesn't + * know is a hard error -- a newer config can't be interpreted safely. */ +void require_schema_version(ryml::ConstNodeRef root) { + if (!has(root, "schema_version")) + throw config_error( + "config error: missing required top-level \"schema_version\" (this build understands " + "version 1)"); + long v = parse_int_strict(scalar(root["schema_version"]), "schema_version"); + if (v != 1) + throw config_error("config error: unsupported schema_version " + std::to_string(v) + + "; this build understands version 1"); +} + +/* ------------------------------------------------------------------------- + * Compile: friendly IR -> compiled_config + * ---------------------------------------------------------------------- */ + +/* Compile a parametric fabric into LPGROUPS + PARAMS. */ +void compile_fabric(const friendly_config& cfg, compiled_config& out) { + const fabric& fab = cfg.fab; + + const fabric_model* model = find_fabric_model(fab.model); + if (!model) + throw config_error("config error: unknown fabric model \"" + fab.model + "\""); + + const component* host = cfg.find_component(fab.hosts_component); + if (!host) + throw config_error("config error: hosts.component \"" + fab.hosts_component + + "\" is not defined under components:"); + + layout lay = model->derive(fab.shape); + + /* --- LPGROUPS: one group of `repetitions` slices, each with the + * per-terminal workload + NIC LPs and the router/switch LPs, emitted in + * [workload, terminal, router] order to match the layout the model + * expects. --- */ + compiled_section& grp = out.add_section("LPGROUPS").add_subsection("MODELNET_GRP"); + grp.add_key("repetitions", std::to_string(lay.repetitions)); + grp.add_key(host->model, std::to_string(lay.terminals_per_rep)); + grp.add_key(model->terminal_lp, std::to_string(lay.terminals_per_rep)); + grp.add_key(model->router_lp, std::to_string(lay.routers_per_rep)); + + /* --- PARAMS --- */ + compiled_section& params = out.add_section("PARAMS"); + + /* modelnet_order is derived from the fabric model: the terminal, plus the + * router when it is a distinct model-net method. */ + if (model->router_method) + params.add_key("modelnet_order", + std::vector{model->term_method, model->router_method}); + else + params.add_key("modelnet_order", std::vector{model->term_method}); + + /* shape parameters pass straight through (num_routers etc.). */ + for (const auto& kv : fab.shape) + params.add_key(kv.first, kv.second); + + /* per-link-class params become _ (local_bandwidth, ...). */ + for (const link_class& cls : fab.links) + for (const auto& kv : cls.params) + params.add_key(cls.name + "_" + kv.first, kv.second); + + /* routing.algorithm -> "routing"; any other routing.* passes through. */ + for (const auto& kv : fab.routing) + params.add_key(kv.first == "algorithm" ? std::string("routing") : kv.first, kv.second); + + /* connections.{intra,inter} -> the file-enumerated model's connection-file + * keys; paths pass through verbatim (the model reads them relative to the + * working directory). */ + for (const auto& kv : fab.connections) { + if (kv.first == "intra") + params.add_key("intra-group-connections", kv.second); + else if (kv.first == "inter") + params.add_key("inter-group-connections", kv.second); + else + params.add_key(kv.first, kv.second); + } + + /* remaining scalar fabric keys (packet_size, chunk_size, parity pass-through + * knobs) map to PARAMS verbatim. */ + for (const auto& kv : fab.extra) + params.add_key(kv.first, kv.second); + + /* the workload component's own params (if any) also land in PARAMS. */ + for (const auto& kv : host->params) + params.add_key(kv.first, kv.second); +} + +} // namespace + +compiled_config compile(std::string_view text) { + /* Construct the parser with our throwing error handler so that ryml's own + * parse errors route through config_error -> tw_error at the shim, exactly + * like our validation errors. The no-parser parse_in_arena builds an event + * handler with ryml's default callbacks, which print + abort directly and + * bypass the shim, so we build the handler (and thus its callbacks) + * explicitly here. */ + ryml::Callbacks cb(nullptr, nullptr, nullptr, ryml_throw); + /* The Callbacks ctor installs ryml's default parse-error handler (which + * prints + aborts); replace it with ours so parse errors reach the shim's + * tw_error like every other error. */ + cb.set_error_parse(ryml_throw_parse); + ryml::EventHandlerTree evt_handler(cb); + ryml::Parser parser(&evt_handler); + ryml::Tree tree = ryml::parse_in_arena(&parser, ryml::to_csubstr(""), + ryml::csubstr(text.data(), text.size())); + ryml::ConstNodeRef root = tree.rootref(); + + if (!root.readable() || !root.is_map()) + throw config_error("config error: config must be a YAML/JSON mapping at the top level"); + + require_schema_version(root); + friendly_config cfg = parse_friendly(root); + + compiled_config out; + if (cfg.parametric) + compile_fabric(cfg, out); + else + throw config_error("config error: no supported topology found"); + return out; +} + +} // namespace config +} // namespace codes diff --git a/src/modelconfig/config_compiler.h b/src/modelconfig/config_compiler.h new file mode 100644 index 00000000..f9197c72 --- /dev/null +++ b/src/modelconfig/config_compiler.h @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#ifndef SRC_MODELCONFIG_CONFIG_COMPILER_H +#define SRC_MODELCONFIG_CONFIG_COMPILER_H + +/** + * @file config_compiler.h + * + * Pure C++ core of the YAML/JSON configuration front-end. + * + * codes::config::compile() turns the user-friendly topology + component text + * into a @ref codes::config::compiled_config -- an ordered, plain-data image of + * the LPGROUPS/PARAMS configuration the legacy `.conf` parser produces. It does + * all parsing and validation and has **no** dependency on ROSS, MPI, or + * `abort`: on any invalid input it throws @ref codes::config::config_error. + * + * The dumb emitter (config_emitter.h) turns a compiled_config into a + * ConfigVTable, and a thin `extern "C"` shim (yaml_configfile.h) is the *only* + * place that translates a config_error into ROSS's `tw_error`. Keeping the core + * free of ROSS makes it unit-testable in isolation: tests drive compile() and + * assert on the returned compiled_config, never on simulator behavior. This + * "pure core that throws + boundary shim that owns the tw_error translation" is + * the template for future C++ subsystems in codes. + */ + +#include +#include +#include +#include + +namespace codes { +namespace config { + +/** + * Thrown by compile() on any syntax or validation failure. The message is a + * finished, user-facing diagnostic; the shim hands it verbatim to tw_error. + */ +struct config_error : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +/** One key and its value(s): a scalar carries one value, a list-valued key + * (e.g. `modelnet_order`) several, emitted as `("a","b",...)`. */ +struct compiled_key { + std::string name; + std::vector values; +}; + +/** One configuration section: named keys plus nested subsections. LPGROUPS + * holds a MODELNET_GRP subsection; PARAMS is flat. Insertion order is + * preserved throughout, so a compiled config is byte-comparable to the `.conf` + * it replaces. */ +struct compiled_section { + std::string name; + std::vector keys; + std::vector subsections; + + /** Append a scalar key. */ + void add_key(std::string key, std::string value); + /** Append a list-valued key. */ + void add_key(std::string key, std::vector values); + /** Append and return a nested subsection. */ + compiled_section& add_subsection(std::string subname); +}; + +/** The whole compiled config: the top-level (ROOT) sections, in order. */ +struct compiled_config { + std::vector sections; + + /** Append and return a top-level section. */ + compiled_section& add_section(std::string name); +}; + +/** + * Compile YAML/JSON configuration text into the compiled_config IR. + * + * @param text the raw config bytes (JSON is a subset of YAML, so one parser + * handles both). + * @throws config_error on malformed YAML or any schema violation. + */ +compiled_config compile(std::string_view text); + +} // namespace config +} // namespace codes + +#endif diff --git a/src/modelconfig/config_emitter.cxx b/src/modelconfig/config_emitter.cxx new file mode 100644 index 00000000..6da6fd05 --- /dev/null +++ b/src/modelconfig/config_emitter.cxx @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#include "config_emitter.h" + +#include "configstoreadapter.h" +#include + +#include + +namespace codes { +namespace config { + +namespace { + +void emit_keys(ConfigVTable* cf, SectionHandle sec, const std::vector& keys) { + for (const compiled_key& k : keys) { + std::vector vals; + vals.reserve(k.values.size()); + for (const std::string& v : k.values) + vals.push_back(v.c_str()); + cf_createKey(cf, sec, k.name.c_str(), vals.data(), (unsigned int)vals.size()); + } +} + +void emit_section(ConfigVTable* cf, SectionHandle parent, const compiled_section& s) { + SectionHandle sec; + cf_createSection(cf, parent, s.name.c_str(), &sec); + emit_keys(cf, sec, s.keys); + for (const compiled_section& sub : s.subsections) + emit_section(cf, sec, sub); +} + +} // namespace + +ConfigVTable* emit(const compiled_config& cfg) { + ConfigVTable* cf = cfsa_create_empty(); + for (const compiled_section& s : cfg.sections) + emit_section(cf, ROOT_SECTION, s); + return cf; +} + +} // namespace config +} // namespace codes diff --git a/src/modelconfig/config_emitter.h b/src/modelconfig/config_emitter.h new file mode 100644 index 00000000..535d9561 --- /dev/null +++ b/src/modelconfig/config_emitter.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#ifndef SRC_MODELCONFIG_CONFIG_EMITTER_H +#define SRC_MODELCONFIG_CONFIG_EMITTER_H + +/** + * @file config_emitter.h + * + * The dumb second stage of the YAML config front-end: it turns the plain-data + * @ref codes::config::compiled_config the core produced into a ConfigVTable -- + * the same structure the legacy `.conf` text parser yields, so codes_mapping and + * every model read it unchanged through the configuration_get_* accessors. + * + * It is pure data-to-data: it does no validation (compile() already did) and no + * ROSS/tw_error. Its coverage comes from the config-equivalence tests, which + * check a compiled `.yaml` against its golden `.conf`. + */ + +#include "config_compiler.h" + +struct ConfigVTable; + +namespace codes { +namespace config { + +/** + * Emit a compiled_config as a freshly-allocated ConfigVTable (free with + * cf_free). Sections, subsections, keys, and list values are emitted in the + * order the core recorded them. + */ +struct ::ConfigVTable* emit(const compiled_config& cfg); + +} // namespace config +} // namespace codes + +#endif diff --git a/src/modelconfig/configuration.c b/src/modelconfig/configuration.c index 377fe446..90865772 100644 --- a/src/modelconfig/configuration.c +++ b/src/modelconfig/configuration.c @@ -15,6 +15,16 @@ #include #include "txt_configfile.h" +#include "yaml_configfile.h" + +/* A .yaml/.yml/.json path selects the YAML/JSON config front-end; anything else + * is parsed by the legacy .conf text parser. */ +static int config_is_yaml(const char* path) { + const char* dot = strrchr(path, '.'); + if (!dot) + return 0; + return strcmp(dot, ".yaml") == 0 || strcmp(dot, ".yml") == 0 || strcmp(dot, ".json") == 0; +} /* * Global to hold configuration in memory @@ -49,20 +59,27 @@ int configuration_load(const char* filepath, MPI_Comm comm, ConfigHandle* handle if (rc != MPI_SUCCESS) goto finalize; + if (config_is_yaml(filepath)) { + /* the YAML front-end compiles the friendly format into the same + * ConfigVTable the text parser yields, reading the bytes already pulled + * in by the collective read above. */ + *handle = yaml_configfile_load(txtdata, txtsize); + } else { #ifdef __APPLE__ - f = fopen(filepath, "r"); + f = fopen(filepath, "r"); #else - f = fmemopen(txtdata, txtsize, "rb"); + f = fmemopen(txtdata, txtsize, "rb"); #endif - if (!f) { - rc = 1; - goto finalize; - } + if (!f) { + rc = 1; + goto finalize; + } - *handle = txtfile_openStream(f, &error); - if (error) { - rc = 1; - goto finalize; + *handle = txtfile_openStream(f, &error); + if (error) { + rc = 1; + goto finalize; + } } /* NOTE: posix version overwrites argument :(. */ diff --git a/src/modelconfig/yaml_configfile.cxx b/src/modelconfig/yaml_configfile.cxx new file mode 100644 index 00000000..a8d5d343 --- /dev/null +++ b/src/modelconfig/yaml_configfile.cxx @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +/* + * Thin extern "C" boundary of the YAML/JSON configuration front-end. It wires + * the pure C++ core (config_compiler.h -- parse + validate, throwing on error) + * to the dumb emitter (config_emitter.h -- IR -> ConfigVTable), and it is the + * ONLY place in the front-end that touches ROSS: a config_error thrown by the + * core is translated here into tw_error. Real-run behavior (abort + diagnostic) + * is therefore unchanged from the .conf path, while "abort" stays a boundary + * policy rather than being baked into every validation site in the core. + * + * Determinism: every rank reads identical bytes and runs the identical + * deterministic compile, so malformed input throws on every rank and this + * tw_error fires everywhere at once -- no new divergence versus the .conf path. + */ + +#include "yaml_configfile.h" + +#include "config_compiler.h" +#include "config_emitter.h" + +#include + +#include + +struct ConfigVTable* yaml_configfile_load(const char* data, size_t len) { + try { + codes::config::compiled_config cfg = codes::config::compile(std::string_view(data, len)); + return codes::config::emit(cfg); + } catch (const codes::config::config_error& e) { + tw_error(TW_LOC, "%s", e.what()); + return nullptr; /* unreachable: tw_error aborts */ + } +} diff --git a/src/modelconfig/yaml_configfile.h b/src/modelconfig/yaml_configfile.h new file mode 100644 index 00000000..ce4901c7 --- /dev/null +++ b/src/modelconfig/yaml_configfile.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2013 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#ifndef SRC_MODELCONFIG_YAML_CONFIGFILE_H +#define SRC_MODELCONFIG_YAML_CONFIGFILE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Compile a YAML/JSON config (the user-friendly topology + component format) + * into the same ConfigVTable the legacy .conf text parser produces, so that + * codes_mapping and every model read it unchanged through configuration_get_*. + * + * This is the thin C boundary of the front-end: it runs the pure C++ core + * (compile) and the emitter (emit), and it is the only place that translates a + * compile-time config_error into a ROSS tw_error. data/len are the raw file + * bytes (already read, e.g. via the MPI collective read in configuration_load). + * On success returns a newly-allocated ConfigVTable (free with cf_free); on a + * syntax or validation error it aborts through tw_error with a diagnostic, + * matching how the rest of the configuration front-end reports malformed input. */ +struct ConfigVTable* yaml_configfile_load(const char* data, size_t len); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/src/network-workloads/conf/modelnet-synthetic-dragonfly.yaml b/src/network-workloads/conf/modelnet-synthetic-dragonfly.yaml new file mode 100644 index 00000000..63cac09a --- /dev/null +++ b/src/network-workloads/conf/modelnet-synthetic-dragonfly.yaml @@ -0,0 +1,30 @@ +schema_version: 1 + +# YAML twin of modelnet-synthetic-dragonfly.conf: the same 8-router regular +# dragonfly, expressed in the friendly parametric-fabric form. The compiler +# derives the group/repetition counts from the shape and emits the PARAMS the +# dragonfly model reads. + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 8 + links: + local: { bandwidth: 5.25, vc_size: 4096 } + global: { bandwidth: 4.7, vc_size: 8192 } + cn: { bandwidth: 5.25, vc_size: 4096 } + routing: + algorithm: adaptive + packet_size: 512 + chunk_size: 32 + num_vcs: 1 + modelnet_scheduler: fcfs + message_size: 512 + hosts: + component: compute_host From c4672d78e75f3df92b710d534b32732c436baeb5 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Wed, 1 Jul 2026 16:54:39 -0500 Subject: [PATCH 02/24] tests: check dragonfly YAML against its .conf twin Point the synthetic-dragonfly lp-io equivalence test at the YAML config instead of running the .conf twice, so it now asserts the YAML front-end and the legacy text parser drive the model to byte-identical per-LP output. --- tests/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 294d5c44..d8c91cfa 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -393,13 +393,13 @@ codes_add_run_test(NAME modelnet-test-slimfly-synthetic BINARY model-net-synthet ARGS --sync=1 -- "${_nwconf}/modelnet-synthetic-slimfly-min.conf") # Config-equivalence harness: run two configs, diff per-LP lp-io output. -# Proof-of-concept uses the same config twice (must be identical -> also an lp-io -# determinism check). When the YAML config adapter lands, CONFIG_B becomes the -# compiled .yaml, giving the old-.conf vs new-.yaml equivalence check. +# The legacy .conf vs its YAML twin -- proves the YAML front-end compiles to a +# config the dragonfly model runs byte-identically to the .conf. ryml is always +# built, so this is unconditional. codes_add_lpio_equivalence_test( NAME synthetic-dragonfly-lpio-equivalence BINARY model-net-synthetic NP 1 ARGS --sync=1 --num_messages=1 CONFIG_A "${_nwconf}/modelnet-synthetic-dragonfly.conf" - CONFIG_B "${_nwconf}/modelnet-synthetic-dragonfly.conf") + CONFIG_B "${_nwconf}/modelnet-synthetic-dragonfly.yaml") From ce0cc9404bfec23b304053d9d22a399ce987b8f8 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Wed, 1 Jul 2026 17:14:35 -0500 Subject: [PATCH 03/24] yaml config: compile flat networks from a component + node count Extend the YAML front-end to the flat all-to-all network models, simplenet and simplep2p. A flat topology is expressed as one component -- the workload model (nw-lp) bundled with the NIC model it runs over (network: simplenet|simplep2p) plus that model's params -- and a scalar node count: topology: format: flat component: compute_node nodes: 16 The compiler lays out one repetition per node (the workload LP and its NIC LP), derives modelnet_order from the network model, and passes the component params (message_size, packet_size, and simplep2p's matrix-file references) straight through to PARAMS. simplep2p's link table stays referenced by path to its existing matrix files; the friendly form supplies only the node count. This is deliberately not a node/edge graph: the Cytoscape element form (per-node data, per-edge bandwidth/latency) is a new representation with no consuming model yet, so it waits for the WAN model. Adds a marker-based config-equivalence helper (the flat-model test binaries write lp-io to a fixed path, so committed-event counts are compared instead of lp-io dirs) and registers both twins. Verified: modelnet-simplep2p-test and modelnet-test produce identical event counts from the .yaml and the .conf. --- src/modelconfig/config_compiler.cxx | 80 ++++++++++++++++++++++++- tests/CMakeLists.txt | 48 +++++++++++++++ tests/conf/modelnet-test-simplenet.yaml | 22 +++++++ tests/conf/modelnet-test-simplep2p.yaml | 21 +++++++ 4 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 tests/conf/modelnet-test-simplenet.yaml create mode 100644 tests/conf/modelnet-test-simplep2p.yaml diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx index f1f78cbb..4adce2eb 100644 --- a/src/modelconfig/config_compiler.cxx +++ b/src/modelconfig/config_compiler.cxx @@ -126,6 +126,10 @@ struct friendly_config { bool parametric = false; fabric fab; /* parametric topology */ + bool flat = false; /* flat enumerated topology */ + std::string flat_component; /* the component every node runs */ + long node_count = 0; /* number of nodes = repetitions */ + const component* find_component(const std::string& k) const { for (const component& c : components) if (c.key == k) @@ -189,6 +193,27 @@ const fabric_model* find_fabric_model(const std::string& name) { return nullptr; } +/* A flat (enumerated) network model: one NIC LP per compute node, all peers. + * Maps a friendly network name to the LPGROUPS lp-type name and the + * modelnet_order method the model registers. */ +struct network_model { + const char* name; /* friendly name used in a component's network: field */ + const char* nic_lp; /* LPGROUPS lp-type name for the NIC */ + const char* method; /* modelnet_order method name */ +}; + +const network_model network_models[] = { + {"simplenet", "modelnet_simplenet", "simplenet"}, + {"simplep2p", "modelnet_simplep2p", "simplep2p"}, +}; + +const network_model* find_network_model(const std::string& name) { + for (const network_model& m : network_models) + if (name == m.name) + return &m; + return nullptr; +} + /* ------------------------------------------------------------------------- * Parse: ryml tree -> friendly IR (validating as it goes -- unknown / * unconsumed keys are errors, not silent drops). @@ -280,8 +305,26 @@ void parse_topology(ryml::ConstNodeRef root, friendly_config& cfg) { else throw config_error("config error: parametric topology needs hosts.component naming the " "per-terminal workload"); + } else if (format == "flat") { + cfg.flat = true; + /* only these keys are consumed for a flat topology. */ + for (ryml::ConstNodeRef c : topo.children()) { + std::string k = key_of(c); + if (k != "format" && k != "component" && k != "nodes") + throw config_error("config error: topology: unexpected key \"" + k + + "\" for a flat topology"); + } + if (!has(topo, "component")) + throw config_error("config error: flat topology needs a \"component\" naming the " + "compute node (workload + its NIC model)"); + cfg.flat_component = scalar(topo["component"]); + if (!has(topo, "nodes")) + throw config_error("config error: flat topology needs a \"nodes\" count"); + cfg.node_count = parse_int_strict(scalar(topo["nodes"]), "topology.nodes"); + if (cfg.node_count <= 0) + throw config_error("config error: topology.nodes must be positive"); } else if (format.empty()) { - throw config_error("config error: topology needs a \"format\" (e.g. parametric)"); + throw config_error("config error: topology needs a \"format\" (flat or parametric)"); } else { throw config_error("config error: unknown topology format \"" + format + "\""); } @@ -388,6 +431,39 @@ void compile_fabric(const friendly_config& cfg, compiled_config& out) { params.add_key(kv.first, kv.second); } +/* Compile a flat (enumerated) network into LPGROUPS + PARAMS: `node_count` + * peer compute nodes, each one repetition running the component's workload LP + * over its NIC LP. simplep2p's link table stays referenced by path in the + * component params; the friendly form supplies only the node count. */ +void compile_flat(const friendly_config& cfg, compiled_config& out) { + const component* comp = cfg.find_component(cfg.flat_component); + if (!comp) + throw config_error("config error: topology.component \"" + cfg.flat_component + + "\" is not defined under components:"); + if (comp->network.empty()) + throw config_error("config error: component \"" + cfg.flat_component + + "\" needs a network: field naming its NIC model"); + + const network_model* net = find_network_model(comp->network); + if (!net) + throw config_error("config error: unknown network model \"" + comp->network + "\""); + + /* --- LPGROUPS: one repetition per node, each a workload LP + its NIC LP, + * emitted in [workload, NIC] order to match the model's layout. --- */ + compiled_section& grp = out.add_section("LPGROUPS").add_subsection("MODELNET_GRP"); + grp.add_key("repetitions", std::to_string(cfg.node_count)); + grp.add_key(comp->model, "1"); + grp.add_key(net->nic_lp, "1"); + + /* --- PARAMS: modelnet_order from the network model; the component's params + * (message_size, packet_size, simplep2p's matrix-file references, ...) pass + * straight through. --- */ + compiled_section& params = out.add_section("PARAMS"); + params.add_key("modelnet_order", std::vector{net->method}); + for (const auto& kv : comp->params) + params.add_key(kv.first, kv.second); +} + } // namespace compiled_config compile(std::string_view text) { @@ -417,6 +493,8 @@ compiled_config compile(std::string_view text) { compiled_config out; if (cfg.parametric) compile_fabric(cfg, out); + else if (cfg.flat) + compile_flat(cfg, out); else throw config_error("config error: no supported topology found"); return out; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d8c91cfa..c7ba98f4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -206,6 +206,38 @@ function(codes_add_lpio_equivalence_test) WORKING_DIRECTORY "${CODES_BINARY_DIR}") endfunction() +# codes_add_config_equivalence_test — assert two configs drive a model to the +# same committed work, comparing a marker line rather than lp-io. For models +# whose test binary writes lp-io to a fixed (non-per-run) path, so two lp-io dirs +# can't coexist: the old .conf vs new .yaml check for those. +# +# codes_add_config_equivalence_test( +# NAME BINARY [NP ] +# CONFIG_A # e.g. legacy .conf +# CONFIG_B # e.g. new .yaml +# [MARKER ] [ARGS ...]) +function(codes_add_config_equivalence_test) + cmake_parse_arguments(T "" "NAME;BINARY;NP;CONFIG_A;CONFIG_B;MARKER" "ARGS" ${ARGN}) + if(NOT T_NAME OR NOT T_BINARY OR NOT T_CONFIG_A OR NOT T_CONFIG_B) + message(FATAL_ERROR "codes_add_config_equivalence_test: NAME, BINARY, CONFIG_A and CONFIG_B are required") + endif() + if(NOT T_NP) + set(T_NP 1) + endif() + if(NOT T_MARKER) + set(T_MARKER "Net Events Processed") + endif() + set(_bin "$") + set(_launch ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${T_NP} ${MPIEXEC_PREFLAGS}) + add_test(NAME ${T_NAME} + COMMAND "${CMAKE_CURRENT_BINARY_DIR}/run-test.sh" + bash "${CMAKE_CURRENT_SOURCE_DIR}/equivalence-run.sh" + --marker "${T_MARKER}" + @@ ${_launch} "${_bin}" ${T_ARGS} ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_A}" + @@ ${_launch} "${_bin}" ${T_ARGS} ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_B}" + WORKING_DIRECTORY "${CODES_BINARY_DIR}") +endfunction() + # Unfortunatelly, CMake doesn't support iteration of a key-pair structure, # otherwise the following lists could be easily compressed into a single # list/dictionary/structure. Instead each C file name **MUST** match each @@ -403,3 +435,19 @@ codes_add_lpio_equivalence_test( ARGS --sync=1 --num_messages=1 CONFIG_A "${_nwconf}/modelnet-synthetic-dragonfly.conf" CONFIG_B "${_nwconf}/modelnet-synthetic-dragonfly.yaml") + +# Flat networks (component + node count). These test binaries write lp-io to a +# fixed path, so compare committed-event counts between the .conf and its YAML +# twin rather than lp-io dirs. +codes_add_config_equivalence_test( + NAME simplep2p-config-equivalence + BINARY modelnet-simplep2p-test + ARGS --sync=1 + CONFIG_A "${_tconf}/modelnet-test-simplep2p.conf" + CONFIG_B "${_tconf}/modelnet-test-simplep2p.yaml") +codes_add_config_equivalence_test( + NAME simplenet-config-equivalence + BINARY modelnet-test + ARGS --sync=1 + CONFIG_A "${_tconf}/modelnet-test.conf" + CONFIG_B "${_tconf}/modelnet-test-simplenet.yaml") diff --git a/tests/conf/modelnet-test-simplenet.yaml b/tests/conf/modelnet-test-simplenet.yaml new file mode 100644 index 00000000..e1cc8aa7 --- /dev/null +++ b/tests/conf/modelnet-test-simplenet.yaml @@ -0,0 +1,22 @@ +schema_version: 1 + +# YAML twin of modelnet-test.conf: sixteen peer compute nodes running a workload +# over a simplenet NIC. A flat topology is just a component (the workload plus +# its NIC model) and a node count -- simplenet is a uniform all-to-all model, so +# there is no matrix file and no explicit node graph, and the net params sit +# directly on the component. + +components: + compute_node: + model: nw-lp + network: simplenet + packet_size: 512 + message_size: 464 + modelnet_scheduler: fcfs + net_startup_ns: 1.5 + net_bw_mbps: 20000 + +topology: + format: flat + component: compute_node + nodes: 16 diff --git a/tests/conf/modelnet-test-simplep2p.yaml b/tests/conf/modelnet-test-simplep2p.yaml new file mode 100644 index 00000000..b31b05e6 --- /dev/null +++ b/tests/conf/modelnet-test-simplep2p.yaml @@ -0,0 +1,21 @@ +schema_version: 1 + +# YAML twin of modelnet-test-simplep2p.conf: three peer compute nodes running a +# workload over a simplep2p NIC. simplep2p's link table comes from the existing +# latency/bandwidth matrix files, referenced by path on the component; the flat +# topology just supplies the node count. + +components: + compute_node: + model: nw-lp + network: simplep2p + message_size: 464 + packet_size: 1024 + modelnet_scheduler: fcfs + net_latency_ns_file: modelnet-test-latency.conf + net_bw_mbps_file: modelnet-test-bw.conf + +topology: + format: flat + component: compute_node + nodes: 3 From 202750cda5537df0a21e4e25ee7645ecd8964184 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Wed, 1 Jul 2026 17:16:49 -0500 Subject: [PATCH 04/24] yaml config: add fattree and dragonfly-dally fabric families The fabric model descriptor already varies the per-repetition router/switch LP count and whether the router is a distinct model-net method, so add the two families as registry entries plus their shape->counts derivations: - fattree (internally generated): one repetition per edge switch, switch_radix/2 terminals each, one switch LP per level, and only the terminal in modelnet_order. - dragonfly-dally (file enumerated): repetitions = num_groups * num_routers with the shape counts passed through as genuine inputs, and connections.{intra,inter} mapped to the model's connection-file keys and referenced by path. Verified: model-net-synthetic-dragonfly-all produces byte-identical per-LP lp-io from the dally YAML and its .conf; model-net-synthetic-fattree produces matching committed-event counts from the fattree YAML and its .conf (fattree's lp-io switch stats are not reproducible run to run, so event counts are the signal). The dally test configures both configs with an absolute path to the shared connection files so the runner's per-run subdirs resolve them. --- src/modelconfig/config_compiler.cxx | 33 ++++++++++++++++ .../conf/modelnet-synthetic-fattree.yaml | 38 +++++++++++++++++++ tests/CMakeLists.txt | 23 +++++++++++ tests/conf/dragonfly-dally/dfdally-72.conf.in | 37 ++++++++++++++++++ tests/conf/dragonfly-dally/dfdally-72.yaml.in | 38 +++++++++++++++++++ 5 files changed, 169 insertions(+) create mode 100644 src/network-workloads/conf/modelnet-synthetic-fattree.yaml create mode 100644 tests/conf/dragonfly-dally/dfdally-72.conf.in create mode 100644 tests/conf/dragonfly-dally/dfdally-72.yaml.in diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx index 4adce2eb..577754b7 100644 --- a/src/modelconfig/config_compiler.cxx +++ b/src/modelconfig/config_compiler.cxx @@ -169,6 +169,14 @@ long shape_int(const kv_list& shape, const char* key) { "\""); } +/* Look up a shape value by name, returning a default when absent. */ +long shape_int_default(const kv_list& shape, const char* key, long dflt) { + for (const auto& kv : shape) + if (kv.first == key) + return std::strtol(kv.second.c_str(), nullptr, 10); + return dflt; +} + /* Regular (Kim-Dally) dragonfly: every count follows from num_routers, the * routers per group -- the same derivation the model does internally * (num_cn = num_routers/2, num_groups = num_routers*num_cn + 1). */ @@ -181,9 +189,34 @@ layout derive_dragonfly(const kv_list& shape) { return {num_groups * num_routers, num_cn, 1}; } +/* Dragonfly-dally (file-enumerated): the shape counts are genuine inputs that + * must match the connection files. total routers = num_groups * num_planes * + * num_routers; each router hosts num_cns_per_router terminals. */ +layout derive_dragonfly_dally(const kv_list& shape) { + long num_routers = shape_int(shape, "num_routers"); + long num_groups = shape_int(shape, "num_groups"); + long num_cns = shape_int(shape, "num_cns_per_router"); + long num_planes = shape_int_default(shape, "num_planes", 1); + return {num_groups * num_planes * num_routers, num_cns, 1}; +} + +/* Fat-tree (internally-generated): one repetition per edge switch, each hosting + * switch_radix/2 terminals, with one switch LP per level. The fabric's switch is + * not a separate model-net method, so only the terminal appears in + * modelnet_order. */ +layout derive_fattree(const kv_list& shape) { + long switch_count = shape_int(shape, "switch_count"); + long switch_radix = shape_int(shape, "switch_radix"); + long num_levels = shape_int(shape, "num_levels"); + return {switch_count, switch_radix / 2, num_levels}; +} + const fabric_model fabric_models[] = { {"dragonfly", "modelnet_dragonfly", "modelnet_dragonfly_router", "dragonfly", "dragonfly_router", derive_dragonfly}, + {"dragonfly-dally", "modelnet_dragonfly_dally", "modelnet_dragonfly_dally_router", + "dragonfly_dally", "dragonfly_dally_router", derive_dragonfly_dally}, + {"fattree", "modelnet_fattree", "fattree_switch", "fattree", nullptr, derive_fattree}, }; const fabric_model* find_fabric_model(const std::string& name) { diff --git a/src/network-workloads/conf/modelnet-synthetic-fattree.yaml b/src/network-workloads/conf/modelnet-synthetic-fattree.yaml new file mode 100644 index 00000000..54ac1a75 --- /dev/null +++ b/src/network-workloads/conf/modelnet-synthetic-fattree.yaml @@ -0,0 +1,38 @@ +schema_version: 1 + +# YAML twin of modelnet-synthetic-fattree.conf: a 3-level fat-tree, expressed in +# the parametric-fabric form. The compiler derives the per-edge-switch layout +# from the shape (switch_count edge switches, switch_radix/2 terminals each, one +# switch per level) and emits the PARAMS the fattree model reads. The fattree +# bandwidth/vc parameters are named flat rather than per link-class, so they are +# carried directly. + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: fattree + shape: + num_levels: 3 + switch_count: 32 + switch_radix: 8 + routing: + algorithm: adaptive + ft_type: 0 + packet_size: 512 + message_size: 512 + chunk_size: 512 + modelnet_scheduler: fcfs + router_delay: 90 + terminal_radix: 1 + soft_delay: 1000 + vc_size: 65536 + cn_vc_size: 65536 + link_bandwidth: 12.5 + cn_bandwidth: 12.5 + rail_routing: adaptive + hosts: + component: compute_host diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c7ba98f4..34adce20 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -451,3 +451,26 @@ codes_add_config_equivalence_test( ARGS --sync=1 CONFIG_A "${_tconf}/modelnet-test.conf" CONFIG_B "${_tconf}/modelnet-test-simplenet.yaml") + +# fattree's lp-io switch stats are not reproducible run to run, so compare +# committed-event counts rather than lp-io. +codes_add_config_equivalence_test( + NAME fattree-config-equivalence + BINARY model-net-synthetic-fattree + ARGS --sync=1 + CONFIG_A "${_nwconf}/modelnet-synthetic-fattree.conf" + CONFIG_B "${_nwconf}/modelnet-synthetic-fattree.yaml") + +# dragonfly-dally is file-enumerated: it reads its binary connection files by a +# working-directory-relative path. The equivalence runner executes each run in +# its own subdir, so configure both configs with an absolute path to the +# (shared, in-tree) connection files. +configure_file(conf/dragonfly-dally/dfdally-72.conf.in conf/dragonfly-dally/dfdally-72.conf @ONLY) +configure_file(conf/dragonfly-dally/dfdally-72.yaml.in conf/dragonfly-dally/dfdally-72.yaml @ONLY) +codes_add_lpio_equivalence_test( + NAME dragonfly-dally-lpio-equivalence + BINARY model-net-synthetic-dragonfly-all + NP 1 + ARGS --sync=1 --num_messages=1 + CONFIG_A "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-dally/dfdally-72.conf" + CONFIG_B "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-dally/dfdally-72.yaml") diff --git a/tests/conf/dragonfly-dally/dfdally-72.conf.in b/tests/conf/dragonfly-dally/dfdally-72.conf.in new file mode 100644 index 00000000..5a953221 --- /dev/null +++ b/tests/conf/dragonfly-dally/dfdally-72.conf.in @@ -0,0 +1,37 @@ +# Golden dragonfly-dally config for the YAML-equivalence test. Identical to +# src/network-workloads/conf/dragonfly-dally/dfdally_72.conf except the +# connection-file paths are absolute (@CMAKE_SOURCE_DIR@ substituted by CMake) so +# the model resolves them regardless of the run directory. +LPGROUPS +{ + MODELNET_GRP + { + repetitions="36"; + nw-lp="2"; + modelnet_dragonfly_dally="2"; + modelnet_dragonfly_dally_router="1"; + } +} +PARAMS +{ + packet_size="4096"; + modelnet_order=( "dragonfly_dally","dragonfly_dally_router" ); + modelnet_scheduler="fcfs"; + chunk_size="4096"; + num_routers="4"; + num_groups="9"; + local_vc_size="16384"; + global_vc_size="16384"; + cn_vc_size="32768"; + local_bandwidth="2.0"; + global_bandwidth="2.0"; + cn_bandwidth="2.0"; + message_size="736"; + num_cns_per_router="2"; + num_global_channels="2"; + intra-group-connections="@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-dally/dfdally-72-intra"; + inter-group-connections="@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-dally/dfdally-72-inter"; + routing="minimal"; + minimal-bias="1"; + df-dally-vc = "1"; +} diff --git a/tests/conf/dragonfly-dally/dfdally-72.yaml.in b/tests/conf/dragonfly-dally/dfdally-72.yaml.in new file mode 100644 index 00000000..40898586 --- /dev/null +++ b/tests/conf/dragonfly-dally/dfdally-72.yaml.in @@ -0,0 +1,38 @@ +schema_version: 1 + +# YAML twin of dfdally_72.conf: a 72-terminal dragonfly-dally. It is file- +# enumerated, so the shape counts are genuine inputs that must match the binary +# connection files, referenced by path (absolute here, @CMAKE_SOURCE_DIR@ +# substituted by CMake, so the model resolves them from any run directory). The +# compiler derives repetitions = num_groups * num_routers and lays out the LPs. + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly-dally + shape: + num_routers: 4 + num_groups: 9 + num_cns_per_router: 2 + num_global_channels: 2 + links: + local: { bandwidth: 2.0, vc_size: 16384 } + global: { bandwidth: 2.0, vc_size: 16384 } + cn: { bandwidth: 2.0, vc_size: 32768 } + routing: + algorithm: minimal + connections: + intra: "@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-dally/dfdally-72-intra" + inter: "@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-dally/dfdally-72-inter" + packet_size: 4096 + chunk_size: 4096 + message_size: 736 + modelnet_scheduler: fcfs + minimal-bias: 1 + df-dally-vc: 1 + hosts: + component: compute_host From 0fae7fc7837fa933cb2a19ee8022f5969dc830b2 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Wed, 1 Jul 2026 17:17:52 -0500 Subject: [PATCH 05/24] docs: document running with a YAML config --- README.md | 7 +++ doc/dev/yaml-config.md | 133 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 doc/dev/yaml-config.md diff --git a/README.md b/README.md index 13ca5c5e..28cddcf7 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,13 @@ bash run-experiment.sh path-to-experiment/script.sh A folder will be created under `path-to-experiment/results` containing the result of running the experiment. +### Configuration format + +A model's configuration can be written as a `.conf` (the legacy format) or as a +YAML/JSON file — both are supported, chosen per file by extension. See +[Running with a YAML config](doc/dev/yaml-config.md) for the YAML format and +worked examples. + ## Features CODES provides comprehensive simulation capabilities for: diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md new file mode 100644 index 00000000..99f21bce --- /dev/null +++ b/doc/dev/yaml-config.md @@ -0,0 +1,133 @@ +# Running with a YAML config + +CODES accepts a YAML (or JSON) configuration anywhere it accepts a legacy +`.conf` today. Both formats are supported side by side — the format is chosen per +file, by extension: + +- `.yaml`, `.yml`, `.json` → the YAML front-end +- anything else → the legacy `.conf` text parser + +Point an existing executable at a YAML file exactly as you would a `.conf` — the +config path is the trailing positional argument, after ROSS's `--`: + +```bash +./model-net-synthetic --sync=1 --num_messages=1 -- my-network.yaml +``` + +Nothing else changes: the YAML front-end compiles the friendly format down to the +same internal configuration the `.conf` parser produces, so every model reads it +unchanged. A YAML config and its `.conf` twin drive a model to identical results. + +The YAML front-end is always available — RapidYAML is vendored in-tree +(`thirdparty/rapidyaml/`) and built unconditionally, so there is nothing to +enable and no configure-time option to set. + +## Shape of a config + +A config has a few top-level blocks: + +```yaml +schema_version: 1 # required; this build understands version 1 +components: # named component configs referenced by the topology +topology: # a flat network, or a parametric fabric +``` + +A **component** pairs a model with its parameters and is referenced by name from +the topology. `schema_version` is required and must be a version this build +knows; unknown top-level keys, unknown topology keys, and blocks a component or +fabric doesn't consume are errors, not silent drops. + +## Flat networks + +The flat all-to-all network models — `simplenet` and `simplep2p` — are described +as a single component plus a node count. The component bundles the workload model +with the NIC model it runs over: + +```yaml +schema_version: 1 + +components: + compute_node: + model: nw-lp # the workload + network: simplenet # the NIC model the workload runs over + packet_size: 512 + message_size: 464 + modelnet_scheduler: fcfs + net_startup_ns: 1.5 + net_bw_mbps: 20000 + +topology: + format: flat + component: compute_node + nodes: 16 +``` + +`nodes` becomes the number of compute-node slots. `simplep2p` takes its per-link +latency/bandwidth from the existing matrix files, referenced by path from the +component (`net_latency_ns_file`, `net_bw_mbps_file`); the flat form supplies +only the node count. (A node/edge graph form — per-node overrides, per-edge link +rates — is a separate representation that lands with the model that consumes it.) + +## Parametric fabric (HPC networks) + +Regular HPC fabrics are not drawn node by node — their connectivity follows from +a handful of shape parameters. They use `format: parametric` and a `fabric` +block instead of a node count: + +```yaml +schema_version: 1 + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 8 # routers per group; the rest of the layout follows + links: + local: { bandwidth: 5.25, vc_size: 4096 } + global: { bandwidth: 4.7, vc_size: 8192 } + cn: { bandwidth: 5.25, vc_size: 4096 } + routing: + algorithm: adaptive + packet_size: 512 + chunk_size: 32 + num_vcs: 1 + modelnet_scheduler: fcfs + message_size: 512 + hosts: + component: compute_host # the workload on every terminal +``` + +The compiler derives the group, repetition, and per-router counts from the +`shape` (the same math the model does internally), maps the per-link-class +`links` and `routing` onto the model's parameters, and runs the fabric's +connectivity generation exactly as today. + +Supported fabric `model`s are `dragonfly`, `dragonfly-dally`, and `fattree`. +`dragonfly-dally` is *file-enumerated*: its wiring comes from binary connection +files produced by the existing generator scripts, referenced by path so the +model reads them unchanged: + +```yaml + connections: + intra: conf/dragonfly-dally/dfdally-72-intra + inter: conf/dragonfly-dally/dfdally-72-inter +``` + +For a file-enumerated fabric the `shape` counts are inputs that must stay +consistent with the connection files. + +## Worked examples in the tree + +Each of these YAML files is a twin of the `.conf` beside it, checked in CI to +produce identical results: + +- `tests/conf/modelnet-test-simplenet.yaml` +- `tests/conf/modelnet-test-simplep2p.yaml` +- `src/network-workloads/conf/modelnet-synthetic-dragonfly.yaml` +- `src/network-workloads/conf/modelnet-synthetic-fattree.yaml` +- `tests/conf/dragonfly-dally/dfdally-72.yaml.in` (dragonfly-dally) From d0941d0a9978f3a2b81d8083dc2600ec7028dea3 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Fri, 3 Jul 2026 13:59:56 -0500 Subject: [PATCH 06/24] yaml config: extend YAML config front-end to more network models Add YAML config support for five more model-net networks, each with a .conf-equivalence test proving the compiled config drives the model to identical results: - loggp (flat/p2p; one-line registry entry) - torus (internally-generated; no separate router LP) - express-mesh (internally-generated) - slimfly (internally-generated; list-valued generator sets) - dragonfly-plus (file-enumerated; per-LP lp-io equivalence) Two small additions to the compiler core support these: - null router LP: torus folds routing into the terminal node, so the LPGROUPS router line is skipped when a fabric has no router method. - list-valued fabric params: slimfly's generator_set_X / _X_prime are written as YAML sequences and emitted as multi-value PARAMS (the emitter already handled multi-value keys). This brings YAML coverage to 10 of 11 network models. dragonfly-custom remains. --- doc/dev/yaml-config.md | 36 +++++-- src/modelconfig/config_compiler.cxx | 105 ++++++++++++++++++++- tests/CMakeLists.txt | 40 ++++++++ tests/conf/dragonfly-plus/dfp-test.conf.in | 38 ++++++++ tests/conf/dragonfly-plus/dfp-test.yaml.in | 40 ++++++++ tests/conf/modelnet-test-em.yaml | 35 +++++++ tests/conf/modelnet-test-loggp.yaml | 21 +++++ tests/conf/modelnet-test-slimfly.yaml | 38 ++++++++ tests/conf/modelnet-test-torus.yaml | 28 ++++++ 9 files changed, 370 insertions(+), 11 deletions(-) create mode 100644 tests/conf/dragonfly-plus/dfp-test.conf.in create mode 100644 tests/conf/dragonfly-plus/dfp-test.yaml.in create mode 100644 tests/conf/modelnet-test-em.yaml create mode 100644 tests/conf/modelnet-test-loggp.yaml create mode 100644 tests/conf/modelnet-test-slimfly.yaml create mode 100644 tests/conf/modelnet-test-torus.yaml diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md index 99f21bce..3b0c8a68 100644 --- a/doc/dev/yaml-config.md +++ b/doc/dev/yaml-config.md @@ -39,9 +39,9 @@ fabric doesn't consume are errors, not silent drops. ## Flat networks -The flat all-to-all network models — `simplenet` and `simplep2p` — are described -as a single component plus a node count. The component bundles the workload model -with the NIC model it runs over: +The flat all-to-all network models — `simplenet`, `simplep2p`, and `loggp` — are +described as a single component plus a node count. The component bundles the +workload model with the NIC model it runs over: ```yaml schema_version: 1 @@ -64,8 +64,9 @@ topology: `nodes` becomes the number of compute-node slots. `simplep2p` takes its per-link latency/bandwidth from the existing matrix files, referenced by path from the -component (`net_latency_ns_file`, `net_bw_mbps_file`); the flat form supplies -only the node count. (A node/edge graph form — per-node overrides, per-edge link +component (`net_latency_ns_file`, `net_bw_mbps_file`); `loggp` likewise references +its LogGP parameter table by path (`net_config_file`); the flat form supplies only +the node count. (A node/edge graph form — per-node overrides, per-edge link rates — is a separate representation that lands with the model that consumes it.) ## Parametric fabric (HPC networks) @@ -107,10 +108,22 @@ The compiler derives the group, repetition, and per-router counts from the `links` and `routing` onto the model's parameters, and runs the fabric's connectivity generation exactly as today. -Supported fabric `model`s are `dragonfly`, `dragonfly-dally`, and `fattree`. -`dragonfly-dally` is *file-enumerated*: its wiring comes from binary connection -files produced by the existing generator scripts, referenced by path so the -model reads them unchanged: +Supported fabric `model`s are `dragonfly`, `torus`, `slimfly`, `express-mesh`, +`fattree`, `dragonfly-dally`, and `dragonfly-plus`. The internally-generated +fabrics (`dragonfly`, `torus`, `slimfly`, `express-mesh`, `fattree`) take only +their shape parameters and generate connectivity as today; `torus`'s node count +is the product of its `dim_length`, and `slimfly`'s list-valued generator sets are +written as YAML sequences: + +```yaml + shape: + dim_length: "4,2,2" # torus: node count = 4*2*2 = 16 + generator_set_X: [1, 4] # slimfly: emitted as generator_set_X=("1","4") +``` + +`dragonfly-dally` and `dragonfly-plus` are *file-enumerated*: their wiring comes +from binary connection files produced by the existing generator scripts, +referenced by path so the model reads them unchanged: ```yaml connections: @@ -128,6 +141,11 @@ produce identical results: - `tests/conf/modelnet-test-simplenet.yaml` - `tests/conf/modelnet-test-simplep2p.yaml` +- `tests/conf/modelnet-test-loggp.yaml` +- `tests/conf/modelnet-test-torus.yaml` +- `tests/conf/modelnet-test-slimfly.yaml` +- `tests/conf/modelnet-test-em.yaml` (express-mesh) - `src/network-workloads/conf/modelnet-synthetic-dragonfly.yaml` - `src/network-workloads/conf/modelnet-synthetic-fattree.yaml` - `tests/conf/dragonfly-dally/dfdally-72.yaml.in` (dragonfly-dally) +- `tests/conf/dragonfly-plus/dfp-test.yaml.in` (dragonfly-plus) diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx index 577754b7..86c611bd 100644 --- a/src/modelconfig/config_compiler.cxx +++ b/src/modelconfig/config_compiler.cxx @@ -118,7 +118,9 @@ struct fabric { kv_list routing; /* routing.* (algorithm maps to PARAMS "routing") */ kv_list connections; /* connections.{intra,inter}: file-enumerated wiring */ kv_list extra; /* other scalar fabric keys -> PARAMS verbatim */ - std::string hosts_component; /* hosts.component: the per-terminal workload */ + /* list-valued fabric keys (e.g. slimfly generator_set_X) -> multi-value PARAMS */ + std::vector>> extra_lists; + std::string hosts_component; /* hosts.component: the per-terminal workload */ }; struct friendly_config { @@ -177,6 +179,42 @@ long shape_int_default(const kv_list& shape, const char* key, long dflt) { return dflt; } +/* Look up a shape value by name as its raw string, throwing if absent. */ +const std::string& shape_str(const kv_list& shape, const char* key) { + for (const auto& kv : shape) + if (kv.first == key) + return kv.second; + throw config_error(std::string("config error: fabric shape is missing required key \"") + key + + "\""); +} + +/* Product of a comma-separated dimension list ("4,2,2" -> 16). Used by the + * mesh-style fabrics (torus, express_mesh) whose repetition count is the number + * of mesh nodes/routers implied by dim_length. */ +long dim_product(const kv_list& shape, const char* key) { + const std::string& s = shape_str(shape, key); + long prod = 1; + const char* p = s.c_str(); + bool saw_digit = false; + while (*p) { + char* end = nullptr; + errno = 0; + long d = std::strtol(p, &end, 10); + if (end == p || errno != 0 || d <= 0) + throw config_error(std::string("config error: fabric shape \"") + key + + "\" must be a comma-separated list of positive integers, got \"" + + s + "\""); + prod *= d; + saw_digit = true; + p = end; + while (*p == ',' || *p == ' ') + ++p; + } + if (!saw_digit) + throw config_error(std::string("config error: fabric shape \"") + key + "\" is empty"); + return prod; +} + /* Regular (Kim-Dally) dragonfly: every count follows from num_routers, the * routers per group -- the same derivation the model does internally * (num_cn = num_routers/2, num_groups = num_routers*num_cn + 1). */ @@ -211,12 +249,59 @@ layout derive_fattree(const kv_list& shape) { return {switch_count, switch_radix / 2, num_levels}; } +/* Torus (internally-generated): one repetition per torus node, each a single + * terminal, and no separate router LP (the torus node combines routing and the + * terminal). The node count is the product of the per-dimension lengths. */ +layout derive_torus(const kv_list& shape) { + long nodes = dim_product(shape, "dim_length"); + return {nodes, 1, 0}; +} + +/* Express mesh (internally-generated): one repetition per mesh router, each + * hosting num_cn terminals plus one router LP. The router count is the product + * of the per-dimension lengths. */ +layout derive_express_mesh(const kv_list& shape) { + long routers = dim_product(shape, "dim_length"); + long num_cn = shape_int(shape, "num_cn"); + return {routers, num_cn, 1}; +} + +/* Slimfly (internally-generated, MMS topology): the two Cayley subgraphs give + * 2 * num_routers^2 routers total, one repetition each, hosting num_terminals + * terminals plus one router LP. */ +layout derive_slimfly(const kv_list& shape) { + long num_routers = shape_int(shape, "num_routers"); + long num_terminals = shape_int(shape, "num_terminals"); + if (num_routers <= 0) + throw config_error("config error: slimfly num_routers must be positive"); + return {2 * num_routers * num_routers, num_terminals, 1}; +} + +/* Dragonfly-plus (file-enumerated): one repetition per group. Each group's + * routers split into a spine and a leaf level (num_router_spine + num_router_leaf + * router LPs); only the leaf routers host terminals, num_cns_per_router each. The + * shape counts are genuine inputs that must match the connection files. */ +layout derive_dragonfly_plus(const kv_list& shape) { + long num_groups = shape_int(shape, "num_groups"); + long spine = shape_int(shape, "num_router_spine"); + long leaf = shape_int(shape, "num_router_leaf"); + long num_cns = shape_int(shape, "num_cns_per_router"); + return {num_groups, leaf * num_cns, spine + leaf}; +} + const fabric_model fabric_models[] = { {"dragonfly", "modelnet_dragonfly", "modelnet_dragonfly_router", "dragonfly", "dragonfly_router", derive_dragonfly}, {"dragonfly-dally", "modelnet_dragonfly_dally", "modelnet_dragonfly_dally_router", "dragonfly_dally", "dragonfly_dally_router", derive_dragonfly_dally}, {"fattree", "modelnet_fattree", "fattree_switch", "fattree", nullptr, derive_fattree}, + {"torus", "modelnet_torus", nullptr, "torus", nullptr, derive_torus}, + {"express-mesh", "modelnet_express_mesh", "modelnet_express_mesh_router", "express_mesh", + "express_mesh_router", derive_express_mesh}, + {"slimfly", "modelnet_slimfly", "modelnet_slimfly_router", "slimfly", "slimfly_router", + derive_slimfly}, + {"dragonfly-plus", "modelnet_dragonfly_plus", "modelnet_dragonfly_plus_router", + "dragonfly_plus", "dragonfly_plus_router", derive_dragonfly_plus}, }; const fabric_model* find_fabric_model(const std::string& name) { @@ -238,6 +323,7 @@ struct network_model { const network_model network_models[] = { {"simplenet", "modelnet_simplenet", "simplenet"}, {"simplep2p", "modelnet_simplep2p", "simplep2p"}, + {"loggp", "modelnet_loggp", "loggp"}, }; const network_model* find_network_model(const std::string& name) { @@ -306,6 +392,13 @@ void parse_fabric(ryml::ConstNodeRef fnode, fabric& fab) { * by path; the compiler maps intra/inter to the model's key names. */ for (ryml::ConstNodeRef cn : c.children()) fab.connections.emplace_back(key_of(cn), scalar(cn)); + } else if (c.is_seq()) { + /* a list-valued fabric param (e.g. slimfly generator_set_X: [1, 4]) + * becomes a multi-value PARAMS key. */ + std::vector vals; + for (ryml::ConstNodeRef v : c.children()) + vals.push_back(scalar(v)); + fab.extra_lists.emplace_back(k, std::move(vals)); } else if (c.is_keyval()) { fab.extra.emplace_back(k, scalar(c)); } else { @@ -416,7 +509,10 @@ void compile_fabric(const friendly_config& cfg, compiled_config& out) { grp.add_key("repetitions", std::to_string(lay.repetitions)); grp.add_key(host->model, std::to_string(lay.terminals_per_rep)); grp.add_key(model->terminal_lp, std::to_string(lay.terminals_per_rep)); - grp.add_key(model->router_lp, std::to_string(lay.routers_per_rep)); + /* Mesh-style fabrics (torus) fold routing into the terminal node and have no + * separate router LP; skip the router line for them. */ + if (model->router_lp) + grp.add_key(model->router_lp, std::to_string(lay.routers_per_rep)); /* --- PARAMS --- */ compiled_section& params = out.add_section("PARAMS"); @@ -459,6 +555,11 @@ void compile_fabric(const friendly_config& cfg, compiled_config& out) { for (const auto& kv : fab.extra) params.add_key(kv.first, kv.second); + /* list-valued fabric keys (slimfly's generator_set_X / _X_prime) emit as + * multi-value PARAMS, e.g. generator_set_X=("1","4"). */ + for (const auto& kv : fab.extra_lists) + params.add_key(kv.first, kv.second); + /* the workload component's own params (if any) also land in PARAMS. */ for (const auto& kv : host->params) params.add_key(kv.first, kv.second); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 34adce20..1e7465f4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -451,6 +451,34 @@ codes_add_config_equivalence_test( ARGS --sync=1 CONFIG_A "${_tconf}/modelnet-test.conf" CONFIG_B "${_tconf}/modelnet-test-simplenet.yaml") +codes_add_config_equivalence_test( + NAME loggp-config-equivalence + BINARY modelnet-test + ARGS --sync=1 + CONFIG_A "${_tconf}/modelnet-test-loggp.conf" + CONFIG_B "${_tconf}/modelnet-test-loggp.yaml") + +# Internally-generated fabrics (torus / express-mesh / slimfly), each run through +# the modelnet-test binary (fixed-path lp-io), so compare committed-event counts +# between the .conf and its parametric-fabric YAML twin. +codes_add_config_equivalence_test( + NAME torus-config-equivalence + BINARY modelnet-test + ARGS --sync=1 + CONFIG_A "${_tconf}/modelnet-test-torus.conf" + CONFIG_B "${_tconf}/modelnet-test-torus.yaml") +codes_add_config_equivalence_test( + NAME express-mesh-config-equivalence + BINARY modelnet-test + ARGS --sync=1 + CONFIG_A "${_tconf}/modelnet-test-em.conf" + CONFIG_B "${_tconf}/modelnet-test-em.yaml") +codes_add_config_equivalence_test( + NAME slimfly-config-equivalence + BINARY modelnet-test + ARGS --sync=1 + CONFIG_A "${_tconf}/modelnet-test-slimfly.conf" + CONFIG_B "${_tconf}/modelnet-test-slimfly.yaml") # fattree's lp-io switch stats are not reproducible run to run, so compare # committed-event counts rather than lp-io. @@ -474,3 +502,15 @@ codes_add_lpio_equivalence_test( ARGS --sync=1 --num_messages=1 CONFIG_A "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-dally/dfdally-72.conf" CONFIG_B "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-dally/dfdally-72.yaml") + +# dragonfly-plus is likewise file-enumerated; same absolute-path treatment for +# its binary connection files. +configure_file(conf/dragonfly-plus/dfp-test.conf.in conf/dragonfly-plus/dfp-test.conf @ONLY) +configure_file(conf/dragonfly-plus/dfp-test.yaml.in conf/dragonfly-plus/dfp-test.yaml @ONLY) +codes_add_lpio_equivalence_test( + NAME dragonfly-plus-lpio-equivalence + BINARY model-net-synthetic-dragonfly-all + NP 1 + ARGS --sync=1 --num_messages=1 + CONFIG_A "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-plus/dfp-test.conf" + CONFIG_B "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-plus/dfp-test.yaml") diff --git a/tests/conf/dragonfly-plus/dfp-test.conf.in b/tests/conf/dragonfly-plus/dfp-test.conf.in new file mode 100644 index 00000000..ec43f4c2 --- /dev/null +++ b/tests/conf/dragonfly-plus/dfp-test.conf.in @@ -0,0 +1,38 @@ +# Golden dragonfly-plus config for the YAML-equivalence test. Identical to +# src/network-workloads/conf/dragonfly-plus/dfp-test.conf except the +# connection-file paths are absolute (@CMAKE_SOURCE_DIR@ substituted by CMake) so +# the model resolves them regardless of the run directory. +LPGROUPS +{ + MODELNET_GRP + { + repetitions="5"; + nw-lp="16"; + modelnet_dragonfly_plus="16"; + modelnet_dragonfly_plus_router="8"; + } +} +PARAMS +{ + packet_size="1024"; + modelnet_order=( "dragonfly_plus","dragonfly_plus_router" ); + modelnet_scheduler="fcfs"; + chunk_size="1024"; + num_router_spine="4"; + num_router_leaf="4"; + num_level_chans="1"; + num_groups="5"; + local_vc_size="8192"; + global_vc_size="16384"; + cn_vc_size="8192"; + local_bandwidth="5.25"; + global_bandwidth="1.5"; + cn_bandwidth="8.0"; + message_size="608"; + num_cns_per_router="4"; + num_global_connections="4"; + intra-group-connections="@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-plus/dfp-test-intra"; + inter-group-connections="@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-plus/dfp-test-inter"; + routing="prog-adaptive"; + route_scoring_metric="delta"; +} diff --git a/tests/conf/dragonfly-plus/dfp-test.yaml.in b/tests/conf/dragonfly-plus/dfp-test.yaml.in new file mode 100644 index 00000000..aa7b4e4c --- /dev/null +++ b/tests/conf/dragonfly-plus/dfp-test.yaml.in @@ -0,0 +1,40 @@ +schema_version: 1 + +# YAML twin of dfp-test.conf: a 5-group dragonfly-plus. It is file-enumerated, so +# the shape counts are genuine inputs that must match the binary connection +# files, referenced by path (@CMAKE_SOURCE_DIR@ substituted by CMake so the model +# resolves them from any run directory). The compiler derives repetitions = +# num_groups, terminals per group = num_router_leaf * num_cns_per_router, and +# routers per group = num_router_spine + num_router_leaf. + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly-plus + shape: + num_router_spine: 4 + num_router_leaf: 4 + num_groups: 5 + num_cns_per_router: 4 + links: + local: { bandwidth: 5.25, vc_size: 8192 } + global: { bandwidth: 1.5, vc_size: 16384 } + cn: { bandwidth: 8.0, vc_size: 8192 } + routing: + algorithm: prog-adaptive + connections: + intra: "@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-plus/dfp-test-intra" + inter: "@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-plus/dfp-test-inter" + packet_size: 1024 + chunk_size: 1024 + message_size: 608 + modelnet_scheduler: fcfs + num_level_chans: 1 + num_global_connections: 4 + route_scoring_metric: delta + hosts: + component: compute_host diff --git a/tests/conf/modelnet-test-em.yaml b/tests/conf/modelnet-test-em.yaml new file mode 100644 index 00000000..9cf76c33 --- /dev/null +++ b/tests/conf/modelnet-test-em.yaml @@ -0,0 +1,35 @@ +schema_version: 1 + +# YAML twin of modelnet-test-em.conf: a 3-dimensional 4x4x4 express mesh. It is +# internally generated with a separate router LP -- the compiler derives +# repetitions = product(dim_length) (64 routers), each hosting num_cn terminals +# plus one router LP. dim_length is a comma-separated string carried verbatim. + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: express-mesh + shape: + n_dims: 3 + dim_length: "4,4,4" + num_cn: 3 + routing: + algorithm: static + message_size: 512 + packet_size: 4096 + chunk_size: 4096 + modelnet_scheduler: round-robin + gap: 1 + num_vcs: 1 + link_bandwidth: 12.5 + cn_bandwidth: 12.5 + vc_size: 65536 + cn_vc_size: 65536 + soft_delay: 0 + router_delay: 90 + hosts: + component: compute_host diff --git a/tests/conf/modelnet-test-loggp.yaml b/tests/conf/modelnet-test-loggp.yaml new file mode 100644 index 00000000..718b7e8e --- /dev/null +++ b/tests/conf/modelnet-test-loggp.yaml @@ -0,0 +1,21 @@ +schema_version: 1 + +# YAML twin of modelnet-test-loggp.conf: 16 peer compute nodes running a +# workload over a loggp NIC. Like simplenet/simplep2p, loggp is a flat +# point-to-point model with no router, so the friendly form just names the +# component (workload + NIC model) and a node count. loggp's LogGP parameter +# table is referenced by path (net_config_file); the model resolves it relative +# to this config's directory, so a bare name finds tests/conf/ng-mpi-tukey.dat. + +components: + compute_node: + model: nw-lp + network: loggp + message_size: 464 + modelnet_scheduler: fcfs-full + net_config_file: ng-mpi-tukey.dat + +topology: + format: flat + component: compute_node + nodes: 16 diff --git a/tests/conf/modelnet-test-slimfly.yaml b/tests/conf/modelnet-test-slimfly.yaml new file mode 100644 index 00000000..ee860dac --- /dev/null +++ b/tests/conf/modelnet-test-slimfly.yaml @@ -0,0 +1,38 @@ +schema_version: 1 + +# YAML twin of modelnet-test-slimfly.conf: an MMS slimfly built on num_routers=5. +# It is internally generated with a separate router LP -- the compiler derives +# repetitions = 2 * num_routers^2 (50 routers), each hosting num_terminals +# terminals plus one router LP. The generator sets are list-valued PARAMS, so +# they are written as YAML sequences (emitted as generator_set_X=("1","4")). The +# local/global/cn bandwidth + vc_size trios use the per-link-class links block. + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: slimfly + shape: + num_routers: 5 + num_terminals: 3 + links: + local: { bandwidth: 9.0, vc_size: 25600 } + global: { bandwidth: 9.0, vc_size: 25600 } + cn: { bandwidth: 9.0, vc_size: 25600 } + routing: + algorithm: minimal + generator_set_X: [1, 4] + generator_set_X_prime: [2, 3] + packet_size: 512 + chunk_size: 256 + message_size: 464 + modelnet_scheduler: fcfs + num_vcs: 4 + global_channels: 5 + local_channels: 2 + link_delay: 0 + hosts: + component: compute_host diff --git a/tests/conf/modelnet-test-torus.yaml b/tests/conf/modelnet-test-torus.yaml new file mode 100644 index 00000000..05a1a2f3 --- /dev/null +++ b/tests/conf/modelnet-test-torus.yaml @@ -0,0 +1,28 @@ +schema_version: 1 + +# YAML twin of modelnet-test-torus.conf: a 3-dimensional 4x2x2 torus. torus is +# internally generated and folds routing into the terminal node, so there is no +# separate router LP -- the compiler derives repetitions = product(dim_length) +# (16 nodes here) with one terminal each. dim_length is a comma-separated string +# the model parses itself, so it is carried verbatim (not a YAML list). + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: torus + shape: + n_dims: 3 + dim_length: "4,2,2" + packet_size: 512 + chunk_size: 256 + message_size: 464 + modelnet_scheduler: fcfs + link_bandwidth: 2.0 + buffer_size: 4096 + num_vc: 1 + hosts: + component: compute_host From f6fd4c079557845c793e0ac0a4f3d093447c45db Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Fri, 3 Jul 2026 14:11:11 -0500 Subject: [PATCH 07/24] yaml config: add dragonfly-custom to the YAML config front-end Complete model-net YAML parity: dragonfly-custom is the last topology to land. Like dragonfly-dally/plus it is file-enumerated, so the shape counts (num_router_rows x num_router_cols routers per group, num_groups groups, num_cns_per_router terminals per router) are genuine inputs the compiler derives the LP layout from and that must match the binary connection files. The golden fixture is the 8-group Theta-style custom dragonfly from scripts/dragonfly-custom/example (6x16 router mesh -> 768 routers). The older in-tree custom configs were unusable as a baseline (dead hard-coded paths, and a transposed rows/cols shape that segfaulted the model against the available connection files); the equivalence test uses the corrected shape and passes at per-LP lp-io. Every model-net topology can now be configured from YAML. --- doc/dev/yaml-config.md | 17 ++++---- src/modelconfig/config_compiler.cxx | 15 +++++++ tests/CMakeLists.txt | 12 ++++++ .../dragonfly-custom/dfcustom-8group.conf.in | 39 +++++++++++++++++++ .../dragonfly-custom/dfcustom-8group.yaml.in | 39 +++++++++++++++++++ 5 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 tests/conf/dragonfly-custom/dfcustom-8group.conf.in create mode 100644 tests/conf/dragonfly-custom/dfcustom-8group.yaml.in diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md index 3b0c8a68..374c450e 100644 --- a/doc/dev/yaml-config.md +++ b/doc/dev/yaml-config.md @@ -109,11 +109,11 @@ The compiler derives the group, repetition, and per-router counts from the connectivity generation exactly as today. Supported fabric `model`s are `dragonfly`, `torus`, `slimfly`, `express-mesh`, -`fattree`, `dragonfly-dally`, and `dragonfly-plus`. The internally-generated -fabrics (`dragonfly`, `torus`, `slimfly`, `express-mesh`, `fattree`) take only -their shape parameters and generate connectivity as today; `torus`'s node count -is the product of its `dim_length`, and `slimfly`'s list-valued generator sets are -written as YAML sequences: +`fattree`, `dragonfly-dally`, `dragonfly-plus`, and `dragonfly-custom`. The +internally-generated fabrics (`dragonfly`, `torus`, `slimfly`, `express-mesh`, +`fattree`) take only their shape parameters and generate connectivity as today; +`torus`'s node count is the product of its `dim_length`, and `slimfly`'s +list-valued generator sets are written as YAML sequences: ```yaml shape: @@ -121,9 +121,9 @@ written as YAML sequences: generator_set_X: [1, 4] # slimfly: emitted as generator_set_X=("1","4") ``` -`dragonfly-dally` and `dragonfly-plus` are *file-enumerated*: their wiring comes -from binary connection files produced by the existing generator scripts, -referenced by path so the model reads them unchanged: +`dragonfly-dally`, `dragonfly-plus`, and `dragonfly-custom` are *file-enumerated*: +their wiring comes from binary connection files produced by the existing generator +scripts, referenced by path so the model reads them unchanged: ```yaml connections: @@ -149,3 +149,4 @@ produce identical results: - `src/network-workloads/conf/modelnet-synthetic-fattree.yaml` - `tests/conf/dragonfly-dally/dfdally-72.yaml.in` (dragonfly-dally) - `tests/conf/dragonfly-plus/dfp-test.yaml.in` (dragonfly-plus) +- `tests/conf/dragonfly-custom/dfcustom-8group.yaml.in` (dragonfly-custom) diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx index 86c611bd..07478dfb 100644 --- a/src/modelconfig/config_compiler.cxx +++ b/src/modelconfig/config_compiler.cxx @@ -289,6 +289,19 @@ layout derive_dragonfly_plus(const kv_list& shape) { return {num_groups, leaf * num_cns, spine + leaf}; } +/* Dragonfly-custom (file-enumerated): one repetition per router. Each group is a + * num_router_rows x num_router_cols mesh of routers, so total routers = + * num_groups * num_router_rows * num_router_cols; each router hosts + * num_cns_per_router terminals plus one router LP. The shape counts are genuine + * inputs that must match the connection files. */ +layout derive_dragonfly_custom(const kv_list& shape) { + long num_groups = shape_int(shape, "num_groups"); + long rows = shape_int(shape, "num_router_rows"); + long cols = shape_int(shape, "num_router_cols"); + long num_cns = shape_int(shape, "num_cns_per_router"); + return {num_groups * rows * cols, num_cns, 1}; +} + const fabric_model fabric_models[] = { {"dragonfly", "modelnet_dragonfly", "modelnet_dragonfly_router", "dragonfly", "dragonfly_router", derive_dragonfly}, @@ -302,6 +315,8 @@ const fabric_model fabric_models[] = { derive_slimfly}, {"dragonfly-plus", "modelnet_dragonfly_plus", "modelnet_dragonfly_plus_router", "dragonfly_plus", "dragonfly_plus_router", derive_dragonfly_plus}, + {"dragonfly-custom", "modelnet_dragonfly_custom", "modelnet_dragonfly_custom_router", + "dragonfly_custom", "dragonfly_custom_router", derive_dragonfly_custom}, }; const fabric_model* find_fabric_model(const std::string& name) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1e7465f4..7c390049 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -514,3 +514,15 @@ codes_add_lpio_equivalence_test( ARGS --sync=1 --num_messages=1 CONFIG_A "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-plus/dfp-test.conf" CONFIG_B "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-plus/dfp-test.yaml") + +# dragonfly-custom is file-enumerated too; its connection files live under +# scripts/dragonfly-custom/example (referenced by absolute path). +configure_file(conf/dragonfly-custom/dfcustom-8group.conf.in conf/dragonfly-custom/dfcustom-8group.conf @ONLY) +configure_file(conf/dragonfly-custom/dfcustom-8group.yaml.in conf/dragonfly-custom/dfcustom-8group.yaml @ONLY) +codes_add_lpio_equivalence_test( + NAME dragonfly-custom-lpio-equivalence + BINARY model-net-synthetic-dragonfly-all + NP 1 + ARGS --sync=1 --num_messages=1 + CONFIG_A "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-custom/dfcustom-8group.conf" + CONFIG_B "${CMAKE_CURRENT_BINARY_DIR}/conf/dragonfly-custom/dfcustom-8group.yaml") diff --git a/tests/conf/dragonfly-custom/dfcustom-8group.conf.in b/tests/conf/dragonfly-custom/dfcustom-8group.conf.in new file mode 100644 index 00000000..c3d8c6f3 --- /dev/null +++ b/tests/conf/dragonfly-custom/dfcustom-8group.conf.in @@ -0,0 +1,39 @@ +# Golden dragonfly-custom config for the YAML-equivalence test: an 8-group Theta- +# style custom dragonfly (6x16 router mesh per group -> 768 routers). Derived from +# scripts/dragonfly-custom/example/network-model.conf, with the connection-file +# paths made absolute (@CMAKE_SOURCE_DIR@ substituted by CMake) so the model +# resolves them from any run directory. nw-lp matches modelnet_dragonfly_custom +# (num_cns_per_router = 4); the example conf's nw-lp=8 was an inconsistency. +LPGROUPS +{ + MODELNET_GRP + { + repetitions="768"; + nw-lp="4"; + modelnet_dragonfly_custom="4"; + modelnet_dragonfly_custom_router="1"; + } +} +PARAMS +{ + packet_size="4096"; + message_size="656"; + chunk_size="4096"; + modelnet_scheduler="fcfs"; + modelnet_order=( "dragonfly_custom","dragonfly_custom_router" ); + num_router_rows="6"; + num_router_cols="16"; + num_groups="8"; + num_cns_per_router="4"; + num_global_channels="4"; + local_vc_size="65536"; + global_vc_size="65536"; + cn_vc_size="65536"; + local_bandwidth="12.5"; + global_bandwidth="12.5"; + cn_bandwidth="12.5"; + router_delay="90"; + intra-group-connections="@CMAKE_SOURCE_DIR@/scripts/dragonfly-custom/example/intra-theta-8group"; + inter-group-connections="@CMAKE_SOURCE_DIR@/scripts/dragonfly-custom/example/inter-theta-8group"; + routing="adaptive"; +} diff --git a/tests/conf/dragonfly-custom/dfcustom-8group.yaml.in b/tests/conf/dragonfly-custom/dfcustom-8group.yaml.in new file mode 100644 index 00000000..f621e99b --- /dev/null +++ b/tests/conf/dragonfly-custom/dfcustom-8group.yaml.in @@ -0,0 +1,39 @@ +schema_version: 1 + +# YAML twin of dfcustom-8group.conf: an 8-group Theta-style custom dragonfly. It +# is file-enumerated, so the shape counts are genuine inputs that must match the +# binary connection files, referenced by path (@CMAKE_SOURCE_DIR@ substituted by +# CMake so the model resolves them from any run directory). The compiler derives +# repetitions = num_groups * num_router_rows * num_router_cols (768 routers), with +# num_cns_per_router terminals plus one router LP per repetition. + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly-custom + shape: + num_router_rows: 6 + num_router_cols: 16 + num_groups: 8 + num_cns_per_router: 4 + num_global_channels: 4 + links: + local: { bandwidth: 12.5, vc_size: 65536 } + global: { bandwidth: 12.5, vc_size: 65536 } + cn: { bandwidth: 12.5, vc_size: 65536 } + routing: + algorithm: adaptive + connections: + intra: "@CMAKE_SOURCE_DIR@/scripts/dragonfly-custom/example/intra-theta-8group" + inter: "@CMAKE_SOURCE_DIR@/scripts/dragonfly-custom/example/inter-theta-8group" + packet_size: 4096 + message_size: 656 + chunk_size: 4096 + modelnet_scheduler: fcfs + router_delay: 90 + hosts: + component: compute_host From 3df88bae918ace7cb3584044dfbd42c195c671c7 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Fri, 3 Jul 2026 15:23:44 -0500 Subject: [PATCH 08/24] docs: make yaml-config.md a complete standalone reference --- doc/dev/yaml-config.md | 532 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 481 insertions(+), 51 deletions(-) diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md index 374c450e..f65b7265 100644 --- a/doc/dev/yaml-config.md +++ b/doc/dev/yaml-config.md @@ -32,16 +32,63 @@ components: # named component configs referenced by the topology topology: # a flat network, or a parametric fabric ``` -A **component** pairs a model with its parameters and is referenced by name from -the topology. `schema_version` is required and must be a version this build -knows; unknown top-level keys, unknown topology keys, and blocks a component or -fabric doesn't consume are errors, not silent drops. +- **`schema_version`** is required, an integer, and must be a version this build + knows (currently `1`). A newer, unknown version is a hard error rather than a + best-effort guess. +- **`components`** is a map of name → component. A **component** pairs a model + (`model:`) with its parameters; a flat-network component also names the NIC + model it runs over (`network:`). Components are referenced by name from the + topology. +- **`topology`** selects the network. `format: flat` is an all-to-all point-to- + point network described by a component and a node count; `format: parametric` + is an HPC fabric described by shape parameters. -## Flat networks +Validation is strict: unknown top-level keys, unknown topology keys, a block a +component or fabric doesn't consume, and (for flat topologies) an unexpected +`edges`/graph block are all **errors, not silent drops**. A malformed value +(e.g. a non-integer node count) is likewise rejected with a diagnostic. -The flat all-to-all network models — `simplenet`, `simplep2p`, and `loggp` — are -described as a single component plus a node count. The component bundles the -workload model with the NIC model it runs over: +### How it compiles + +The compiler walks the friendly form and emits the `LPGROUPS` (LP layout) and +`PARAMS` (model knobs) the models already read: + +- a flat network becomes one repetition per node — a `[workload, NIC]` LP pair; +- a parametric fabric derives its repetition / per-router / per-group counts from + the `shape` (the same arithmetic the model does internally) and lays out the + `[workload, terminal, router]` LPs; +- any scalar key the compiler doesn't special-case is **passed through verbatim** + to `PARAMS`, so advanced model knobs need no compiler change. + +--- + +# Flat networks + +The flat point-to-point network models — `simplenet`, `simplep2p`, and `loggp` — +are described as a single **component** plus a **node count**. The component +bundles the workload model (`model:`) with the NIC model it runs over +(`network:`), and carries the NIC's parameters directly. The topology supplies +only how many peer nodes to instantiate: + +```yaml +topology: + format: flat + component: + nodes: +``` + +`nodes` becomes the number of compute-node slots (repetitions). All flat models +are uniform all-to-all — there is no per-node or per-edge form in a flat config; +a model whose links vary per pair reads them from its own file (see `simplep2p` +and `loggp` below). + +## simplenet + +A uniform all-to-all network with a single startup latency and bandwidth. No +external file — the rates sit directly on the component. + +Component keys: `net_startup_ns`, `net_bw_mbps`, plus the usual `packet_size`, +`message_size`, `modelnet_scheduler`. ```yaml schema_version: 1 @@ -62,18 +109,109 @@ topology: nodes: 16 ``` -`nodes` becomes the number of compute-node slots. `simplep2p` takes its per-link -latency/bandwidth from the existing matrix files, referenced by path from the -component (`net_latency_ns_file`, `net_bw_mbps_file`); `loggp` likewise references -its LogGP parameter table by path (`net_config_file`); the flat form supplies only -the node count. (A node/edge graph form — per-node overrides, per-edge link -rates — is a separate representation that lands with the model that consumes it.) +## simplep2p + +Point-to-point with a per-pair latency/bandwidth matrix. The matrices come from +the existing files, referenced by path on the component (`net_latency_ns_file`, +`net_bw_mbps_file`); the model resolves them relative to the config file's +directory, so a bare name sits next to the config. + +```yaml +schema_version: 1 + +components: + compute_node: + model: nw-lp + network: simplep2p + message_size: 464 + packet_size: 1024 + modelnet_scheduler: fcfs + net_latency_ns_file: modelnet-test-latency.conf + net_bw_mbps_file: modelnet-test-bw.conf + +topology: + format: flat + component: compute_node + nodes: 3 +``` -## Parametric fabric (HPC networks) +## loggp -Regular HPC fabrics are not drawn node by node — their connectivity follows from -a handful of shape parameters. They use `format: parametric` and a `fabric` -block instead of a node count: +The LogGP point-to-point model. Its parameter table (`G`, latency, etc.) comes +from a network-config file referenced by path (`net_config_file`), resolved +relative to the config file's directory like simplep2p's matrices. + +```yaml +schema_version: 1 + +components: + compute_node: + model: nw-lp + network: loggp + message_size: 464 + modelnet_scheduler: fcfs-full + net_config_file: ng-mpi-tukey.dat + +topology: + format: flat + component: compute_node + nodes: 16 +``` + +--- + +# Parametric fabrics (HPC networks) + +HPC fabrics are not drawn node by node — their connectivity follows from a +handful of shape parameters. They use `format: parametric` and a `fabric` block, +with a `hosts.component` naming the per-terminal workload: + +```yaml +topology: + format: parametric + fabric: + model: + shape: { ... } # size knobs; the compiler derives LP counts from these + links: { ... } # optional per-link-class bandwidth / vc_size sugar + routing: { ... } # routing.algorithm -> the model's "routing" param + connections:{ ... } # file-enumerated fabrics only: intra/inter wiring files + # any other scalar key here passes straight through to PARAMS + hosts: + component: # the workload on every terminal +``` + +Building blocks shared by all fabrics: + +- **`shape`** — the size knobs specific to each model (below). The compiler runs + the same shape→counts arithmetic the model does, so you set the small, + meaningful numbers and the repetition/group counts follow. +- **`links`** — optional sugar for the `local` / `global` / `cn` link-class + pattern: `local: { bandwidth: 5.25, vc_size: 4096 }` expands to + `local_bandwidth=5.25` and `local_vc_size=4096`. Models that name their link + parameters flat (e.g. `torus`'s `link_bandwidth`) just set those keys directly + instead — anything not recognized is passed through verbatim. +- **`routing`** — `routing.algorithm: minimal` becomes the model's `routing` + param; any other `routing.*` key passes through under its own name. +- **`connections`** — file-enumerated fabrics only: `intra`/`inter` name the + binary wiring files (`intra-group-connections` / `inter-group-connections`). +- **pass-through** — any remaining scalar key on the `fabric` (e.g. + `packet_size`, `chunk_size`, `message_size`, `modelnet_scheduler`) lands in + `PARAMS` unchanged. A list value (e.g. `slimfly`'s generator sets) is emitted + as a multi-value key. + +Supported fabric `model`s split into **internally-generated** (the compiler emits +only shape params and the model generates its wiring) and **file-enumerated** (the +wiring is read from binary connection files produced by the generator scripts). + +## Internally-generated fabrics + +### dragonfly + +Regular (Kim–Dally) dragonfly. The whole layout follows from one number, +`num_routers` (routers per group): the model uses `num_cn = num_routers / 2` +terminals per router and `num_groups = num_routers * num_cn + 1`. + +Shape: `num_routers`. ```yaml schema_version: 1 @@ -100,53 +238,345 @@ topology: modelnet_scheduler: fcfs message_size: 512 hosts: - component: compute_host # the workload on every terminal + component: compute_host +``` + +### torus + +An n-dimensional torus. It folds routing into the terminal node, so there is +**no separate router LP**; the node count is the product of the per-dimension +lengths. `dim_length` is a comma-separated string the model parses itself, so it +is written as a quoted scalar (not a YAML list). Bandwidth is a single flat +`link_bandwidth`, not a per-link-class block. + +Shape: `n_dims`, `dim_length` (repetitions = product of `dim_length`). + +```yaml +schema_version: 1 + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: torus + shape: + n_dims: 3 + dim_length: "4,2,2" # node count = 4*2*2 = 16 + packet_size: 512 + chunk_size: 256 + message_size: 464 + modelnet_scheduler: fcfs + link_bandwidth: 2.0 + buffer_size: 4096 + num_vc: 1 + hosts: + component: compute_host +``` + +### slimfly + +An MMS (McKay–Miller–Širáň) slimfly built on `num_routers`: the two Cayley +subgraphs give `2 * num_routers^2` routers total, each hosting `num_terminals` +terminals. The generator sets are list-valued, written as YAML sequences and +emitted as multi-value params (`generator_set_X=("1","4")`). + +Shape: `num_routers`, `num_terminals` (repetitions = `2 * num_routers^2`). + +```yaml +schema_version: 1 + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: slimfly + shape: + num_routers: 5 # -> 2 * 5^2 = 50 routers + num_terminals: 3 + links: + local: { bandwidth: 9.0, vc_size: 25600 } + global: { bandwidth: 9.0, vc_size: 25600 } + cn: { bandwidth: 9.0, vc_size: 25600 } + routing: + algorithm: minimal + generator_set_X: [1, 4] + generator_set_X_prime: [2, 3] + packet_size: 512 + chunk_size: 256 + message_size: 464 + modelnet_scheduler: fcfs + num_vcs: 4 + global_channels: 5 + local_channels: 2 + link_delay: 0 + hosts: + component: compute_host +``` + +### express-mesh + +An n-dimensional express mesh with a separate router LP. The router count is the +product of the per-dimension lengths, and each router hosts `num_cn` terminals. +Like torus it uses flat bandwidth keys (`link_bandwidth`, `cn_bandwidth`, ...) +rather than a `links` block, and `dim_length` is a quoted scalar. + +Shape: `n_dims`, `dim_length` (router count = product of `dim_length`), `num_cn`. + +```yaml +schema_version: 1 + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: express-mesh + shape: + n_dims: 3 + dim_length: "4,4,4" # router count = 4*4*4 = 64 + num_cn: 3 # terminals per router + routing: + algorithm: static + message_size: 512 + packet_size: 4096 + chunk_size: 4096 + modelnet_scheduler: round-robin + gap: 1 + num_vcs: 1 + link_bandwidth: 12.5 + cn_bandwidth: 12.5 + vc_size: 65536 + cn_vc_size: 65536 + soft_delay: 0 + router_delay: 90 + hosts: + component: compute_host ``` -The compiler derives the group, repetition, and per-router counts from the -`shape` (the same math the model does internally), maps the per-link-class -`links` and `routing` onto the model's parameters, and runs the fabric's -connectivity generation exactly as today. +### fattree + +A multi-level fat-tree. One repetition per edge switch, each hosting +`switch_radix / 2` terminals, with one switch LP per level. The fattree switch is +not a separate model-net method, so it does not appear in `modelnet_order`. +Bandwidth is set with flat keys (`link_bandwidth`, `cn_bandwidth`). -Supported fabric `model`s are `dragonfly`, `torus`, `slimfly`, `express-mesh`, -`fattree`, `dragonfly-dally`, `dragonfly-plus`, and `dragonfly-custom`. The -internally-generated fabrics (`dragonfly`, `torus`, `slimfly`, `express-mesh`, -`fattree`) take only their shape parameters and generate connectivity as today; -`torus`'s node count is the product of its `dim_length`, and `slimfly`'s -list-valued generator sets are written as YAML sequences: +Shape: `num_levels`, `switch_count`, `switch_radix` +(repetitions = `switch_count`, terminals per switch = `switch_radix / 2`). ```yaml +schema_version: 1 + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: fattree shape: - dim_length: "4,2,2" # torus: node count = 4*2*2 = 16 - generator_set_X: [1, 4] # slimfly: emitted as generator_set_X=("1","4") + num_levels: 3 + switch_count: 32 + switch_radix: 8 + routing: + algorithm: adaptive + ft_type: 0 + packet_size: 512 + message_size: 512 + chunk_size: 512 + modelnet_scheduler: fcfs + router_delay: 90 + terminal_radix: 1 + soft_delay: 1000 + vc_size: 65536 + cn_vc_size: 65536 + link_bandwidth: 12.5 + cn_bandwidth: 12.5 + rail_routing: adaptive + hosts: + component: compute_host ``` -`dragonfly-dally`, `dragonfly-plus`, and `dragonfly-custom` are *file-enumerated*: -their wiring comes from binary connection files produced by the existing generator -scripts, referenced by path so the model reads them unchanged: +## File-enumerated fabrics + +These fabrics read their wiring from binary connection files produced by the +existing generator scripts. Add a `connections` block naming the `intra`/`inter` +files by path; the model reads them unchanged. For a file-enumerated fabric the +`shape` counts are **inputs that must stay consistent with the connection +files** — they are not free to choose independently of the wiring. ```yaml connections: - intra: conf/dragonfly-dally/dfdally-72-intra - inter: conf/dragonfly-dally/dfdally-72-inter + intra: /path/to/-intra + inter: /path/to/-inter ``` -For a file-enumerated fabric the `shape` counts are inputs that must stay -consistent with the connection files. +Paths are resolved by the model as given (relative to the run directory), so +tests use absolute paths (`@CMAKE_SOURCE_DIR@` substituted by CMake) to stay +independent of where the run happens; the `.yaml.in` twins in the tree show the +pattern. + +### dragonfly-dally + +Total routers = `num_groups * num_planes * num_routers` (with `num_planes` +defaulting to 1); each router hosts `num_cns_per_router` terminals. + +Shape: `num_routers`, `num_groups`, `num_cns_per_router`, `num_global_channels`, +optional `num_planes`. + +```yaml +schema_version: 1 + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly-dally + shape: + num_routers: 4 + num_groups: 9 + num_cns_per_router: 2 + num_global_channels: 2 + links: + local: { bandwidth: 2.0, vc_size: 16384 } + global: { bandwidth: 2.0, vc_size: 16384 } + cn: { bandwidth: 2.0, vc_size: 32768 } + routing: + algorithm: minimal + connections: + intra: /abs/path/conf/dragonfly-dally/dfdally-72-intra + inter: /abs/path/conf/dragonfly-dally/dfdally-72-inter + packet_size: 4096 + chunk_size: 4096 + message_size: 736 + modelnet_scheduler: fcfs + minimal-bias: 1 + df-dally-vc: 1 + hosts: + component: compute_host +``` + +### dragonfly-plus + +One repetition per group. Each group's routers split into a spine and a leaf +level (`num_router_spine + num_router_leaf` router LPs), and only the leaf +routers host terminals (`num_cns_per_router` each), so terminals per group = +`num_router_leaf * num_cns_per_router`. + +Shape: `num_router_spine`, `num_router_leaf`, `num_groups`, `num_cns_per_router`. + +```yaml +schema_version: 1 + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly-plus + shape: + num_router_spine: 4 + num_router_leaf: 4 + num_groups: 5 + num_cns_per_router: 4 + links: + local: { bandwidth: 5.25, vc_size: 8192 } + global: { bandwidth: 1.5, vc_size: 16384 } + cn: { bandwidth: 8.0, vc_size: 8192 } + routing: + algorithm: prog-adaptive + connections: + intra: /abs/path/conf/dragonfly-plus/dfp-test-intra + inter: /abs/path/conf/dragonfly-plus/dfp-test-inter + packet_size: 1024 + chunk_size: 1024 + message_size: 608 + modelnet_scheduler: fcfs + num_level_chans: 1 + num_global_connections: 4 + route_scoring_metric: delta + hosts: + component: compute_host +``` + +### dragonfly-custom + +Each group is a `num_router_rows × num_router_cols` mesh of routers, so total +routers = `num_groups * num_router_rows * num_router_cols`; each router hosts +`num_cns_per_router` terminals. + +Shape: `num_router_rows`, `num_router_cols`, `num_groups`, `num_cns_per_router`, +`num_global_channels`. + +```yaml +schema_version: 1 + +components: + compute_host: + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly-custom + shape: + num_router_rows: 6 + num_router_cols: 16 + num_groups: 8 + num_cns_per_router: 4 + num_global_channels: 4 + links: + local: { bandwidth: 12.5, vc_size: 65536 } + global: { bandwidth: 12.5, vc_size: 65536 } + cn: { bandwidth: 12.5, vc_size: 65536 } + routing: + algorithm: adaptive + connections: + intra: /abs/path/scripts/dragonfly-custom/example/intra-theta-8group + inter: /abs/path/scripts/dragonfly-custom/example/inter-theta-8group + packet_size: 4096 + message_size: 656 + chunk_size: 4096 + modelnet_scheduler: fcfs + router_delay: 90 + hosts: + component: compute_host +``` + +--- ## Worked examples in the tree Each of these YAML files is a twin of the `.conf` beside it, checked in CI to -produce identical results: - -- `tests/conf/modelnet-test-simplenet.yaml` -- `tests/conf/modelnet-test-simplep2p.yaml` -- `tests/conf/modelnet-test-loggp.yaml` -- `tests/conf/modelnet-test-torus.yaml` -- `tests/conf/modelnet-test-slimfly.yaml` -- `tests/conf/modelnet-test-em.yaml` (express-mesh) -- `src/network-workloads/conf/modelnet-synthetic-dragonfly.yaml` -- `src/network-workloads/conf/modelnet-synthetic-fattree.yaml` -- `tests/conf/dragonfly-dally/dfdally-72.yaml.in` (dragonfly-dally) -- `tests/conf/dragonfly-plus/dfp-test.yaml.in` (dragonfly-plus) -- `tests/conf/dragonfly-custom/dfcustom-8group.yaml.in` (dragonfly-custom) +produce identical results — the authoritative, runnable reference for each model: + +| Model | Example | +|-------|---------| +| simplenet | `tests/conf/modelnet-test-simplenet.yaml` | +| simplep2p | `tests/conf/modelnet-test-simplep2p.yaml` | +| loggp | `tests/conf/modelnet-test-loggp.yaml` | +| dragonfly | `src/network-workloads/conf/modelnet-synthetic-dragonfly.yaml` | +| torus | `tests/conf/modelnet-test-torus.yaml` | +| slimfly | `tests/conf/modelnet-test-slimfly.yaml` | +| express-mesh | `tests/conf/modelnet-test-em.yaml` | +| fattree | `src/network-workloads/conf/modelnet-synthetic-fattree.yaml` | +| dragonfly-dally | `tests/conf/dragonfly-dally/dfdally-72.yaml.in` | +| dragonfly-plus | `tests/conf/dragonfly-plus/dfp-test.yaml.in` | +| dragonfly-custom| `tests/conf/dragonfly-custom/dfcustom-8group.yaml.in` | + +The `.yaml.in` files are CMake templates (the `@CMAKE_SOURCE_DIR@` in their +connection paths is substituted at configure time); the plain `.yaml` files are +ready to run as-is. From 23244537ad5c09a5e22a4a2f8368233283c4ba5b Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Fri, 3 Jul 2026 17:10:56 -0500 Subject: [PATCH 09/24] yaml config: pass-through model sections + case-insensitive section names Add a top-level `sections:` block to the YAML config front-end for config a model reads directly by name (DIRECTOR, NETWORK_SURROGATE, resource, storage, ...). Each entry is emitted verbatim as a top-level section: scalars become single-value keys, lists multi-value keys, nested maps subsections. This restores the .conf ergonomics for such config -- a new feature adds its section with no compiler change -- while topology stays friendly and strictly validated. A section may optionally register an *open* schema (section_schemas[]): required keys are enforced while any other key still passes through, which suits a feature whose config is still in flux. `resource` is registered (its `available` key is mandatory -- resource-lp.c aborts without it), so a missing key is now a clear compile-time error. LPGROUPS/PARAMS are reserved (emitted from the topology). Section names are case-insensitive: the all-caps convention (PARAMS, DIRECTOR, ...) is a historical carryover, so mcs_findsubsection matches case-insensitively. Keys stay case-sensitive -- models read them by exact name. Verified end to end: a model reads `params` written lowercase. Also lands the first real unit test of the ROSS-free compiler core (tests/codes-config-compiler-test.cxx), asserting on compiled_config for the pass-through, open-schema, reserved-name, and case-preservation behavior -- with no codes/ROSS/MPI link, as the layered design intends. Documents the feature and a two-tier "adding a config section" process in doc/dev/yaml-config.md. --- doc/dev/yaml-config.md | 77 ++++++++++++ src/modelconfig/config_compiler.cxx | 118 +++++++++++++++++- src/modelconfig/configstore.c | 20 ++- tests/CMakeLists.txt | 13 ++ tests/codes-config-compiler-test.cxx | 178 +++++++++++++++++++++++++++ 5 files changed, 399 insertions(+), 7 deletions(-) create mode 100644 tests/codes-config-compiler-test.cxx diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md index f65b7265..97c7bc65 100644 --- a/doc/dev/yaml-config.md +++ b/doc/dev/yaml-config.md @@ -558,6 +558,83 @@ topology: --- +# Config a model reads directly: `sections:` + +Not every subsystem's config is topology the compiler derives. Many read their +own named section straight from the config — the storage model reads `resource`, +the surrogate/director stack reads `NETWORK_SURROGATE`, `APPLICATION_SURROGATE`, +and `DIRECTOR`, and so on. Those keys aren't transformed, just read, so the YAML +front-end carries them through **verbatim** under a top-level `sections:` block: + +```yaml +schema_version: 1 +topology: { ... } # friendly, derived, strictly validated + +sections: # emitted verbatim as top-level config sections + resource: + available: 8192 + network_surrogate: + enable: 1 + fixed_switch_timestamps: [25.0e6, 400.0e6] # a list -> a multi-value key + torch_jit: # a nested block -> a subsection + mode: single-static-model-for-all-terminals +``` + +Inside a section, a scalar becomes a single-value key, a **list** becomes a +multi-value key (`("a","b")`), and a **nested map** becomes a subsection. There +is no fixed schema — a section can carry whatever keys the model reads. + +**Section names are case-insensitive.** The all-caps convention (`PARAMS`, +`DIRECTOR`, `NETWORK_SURROGATE`) is a historical carryover; write `resource` or +`network_surrogate` (or any case) and the model finds it. **Keys inside a section +stay case-sensitive** — a model reads them by exact name, so `available` and +`Available` are different keys. `LPGROUPS` and `PARAMS` are reserved: the compiler +emits them from the topology, so they can't appear under `sections:` (put model +parameters on the component or fabric instead). + +## Adding a config section + +When a new feature needs configuration, decide which of two tiers it belongs to. +The test is one question: **does the compiler need to derive, rename, or +cross-check anything?** + +**Tier 1 — the model reads the keys as-is (most sections).** No compiler change: + +1. read your keys in the model with `configuration_get_value(&config, + "my_section", ...)`; +2. put them under `sections:` in the YAML; +3. document the section and its keys here. + +Optionally, register an **open schema** to enforce required keys while still +allowing anything else through — handy for a section that's still in flux, where +you know a couple of keys are mandatory but the rest are unsettled. Add a row to +`section_schemas[]` in `src/modelconfig/config_compiler.cxx`: + +```cpp +const char* const my_required[] = {"must_have", nullptr}; +const section_schema section_schemas[] = { + {"resource", resource_required}, + {"my_section", my_required}, // required keys checked; unlisted keys pass through +}; +``` + +A missing required key becomes a `config_error` up front (a clearer, earlier +message than a mid-run abort). A section with no registered schema passes through +with no validation at all. + +**Tier 2 — the compiler derives or transforms the config (topology, and later +jobs).** This earns a first-class module: a parser, a compiler that emits the +derived sections, a registry entry, and a `.conf`-equivalence test — the pattern +the `fabric_models[]` table and `compile_fabric()` already follow. Reach for this +only when the compiler is doing real work (shape→counts, friendly→internal names, +cross-section consistency); otherwise Tier 1 is the answer. + +The compiler core is ROSS-free and unit-tested in isolation +(`tests/codes-config-compiler-test.cxx`); add cases there for whatever a new +section or tier introduces. + +--- + ## Worked examples in the tree Each of these YAML files is a twin of the `.conf` beside it, checked in CI to diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx index 07478dfb..a6ed5ffb 100644 --- a/src/modelconfig/config_compiler.cxx +++ b/src/modelconfig/config_compiler.cxx @@ -10,6 +10,7 @@ #include #include +#include /* strcasecmp: section names are case-insensitive */ #include #include #include @@ -132,6 +133,11 @@ struct friendly_config { std::string flat_component; /* the component every node runs */ long node_count = 0; /* number of nodes = repetitions */ + /* verbatim `sections:` blocks -- config a model reads directly (DIRECTOR, + * surrogate, storage, ...), passed straight through to the compiled output. + * Emitted after the topology sections. */ + std::vector passthrough; + const component* find_component(const std::string& k) const { for (const component& c : components) if (c.key == k) @@ -471,16 +477,122 @@ void parse_topology(ryml::ConstNodeRef root, friendly_config& cfg) { } } +/* ------------------------------------------------------------------------- + * Pass-through sections (`sections:`) + * + * Config a model reads directly by name (DIRECTOR, NETWORK_SURROGATE, storage, + * resource, ...) is not something the compiler derives or transforms, so it is + * carried through verbatim rather than modeled key-by-key. A new feature can + * add its section here with no compiler change: write the block under + * `sections:` and read it in the model. Section names are case-insensitive + * (matched so at lookup time); a section may optionally register a schema below + * to enforce required keys while still allowing any other key through. + * ---------------------------------------------------------------------- */ + +bool iequals(const std::string& a, const char* b) { + return strcasecmp(a.c_str(), b) == 0; +} + +/* Recursively turn a ryml map node into a compiled_section: scalars become + * single-value keys, sequences multi-value keys, nested maps subsections. */ +compiled_section build_passthrough_section(const std::string& name, ryml::ConstNodeRef node) { + compiled_section sec{name, {}, {}}; + for (ryml::ConstNodeRef c : node.children()) { + std::string k = key_of(c); + if (c.is_seq()) { + std::vector vals; + for (ryml::ConstNodeRef v : c.children()) + vals.push_back(scalar(v)); + sec.add_key(std::move(k), std::move(vals)); + } else if (c.is_map()) { + sec.subsections.push_back(build_passthrough_section(k, c)); + } else if (c.is_keyval()) { + sec.add_key(std::move(k), scalar(c)); + } else { + throw config_error("config error: sections: \"" + name + "\": \"" + k + + "\" must be a scalar, a list, or a nested block"); + } + } + return sec; +} + +/* Optional, open schema for a pass-through section: it enforces required keys + * but does NOT restrict the rest, so a section can carry keys not listed here + * (useful while a feature's config is still in flux). A section with no entry is + * passed through entirely unvalidated. To register one, add a row -- see + * doc/dev/yaml-config.md ("Adding a config section"). */ +struct section_schema { + const char* name; /* section name, matched case-insensitively */ + const char* const* required; /* nullptr-terminated required key names */ +}; + +/* The resource LP aborts at runtime if its `resource` section lacks `available` + * (src/util/resource-lp.c); catch it here with a clearer, earlier diagnostic. */ +const char* const resource_required[] = {"available", nullptr}; + +const section_schema section_schemas[] = { + {"resource", resource_required}, +}; + +const section_schema* find_section_schema(const std::string& name) { + for (const section_schema& s : section_schemas) + if (iequals(name, s.name)) + return &s; + return nullptr; +} + +/* Required-key presence is checked case-sensitively: a model reads its keys by + * exact name, so the required key must be spelled as the model reads it. */ +bool section_has_key(const compiled_section& sec, const char* key) { + for (const compiled_key& k : sec.keys) + if (k.name == key) + return true; + return false; +} + +void validate_section_schema(const compiled_section& sec) { + const section_schema* schema = find_section_schema(sec.name); + if (!schema || !schema->required) + return; + for (const char* const* r = schema->required; *r; ++r) + if (!section_has_key(sec, *r)) + throw config_error("config error: section \"" + sec.name + + "\" is missing required key \"" + std::string(*r) + "\""); +} + +void parse_sections(ryml::ConstNodeRef root, friendly_config& cfg) { + if (!has(root, "sections")) + return; + ryml::ConstNodeRef secs = root["sections"]; + if (!secs.is_map()) + throw config_error("config error: \"sections\" must be a map of section-name -> keys"); + for (ryml::ConstNodeRef s : secs.children()) { + std::string name = key_of(s); + if (!s.is_map()) + throw config_error("config error: sections: \"" + name + "\" must be a block of keys"); + /* the compiler emits LPGROUPS and PARAMS from the topology; a pass-through + * section must not shadow them. */ + if (iequals(name, "LPGROUPS") || iequals(name, "PARAMS")) + throw config_error("config error: sections: \"" + name + + "\" is reserved -- the compiler emits it from the topology; put " + "model parameters on the component or fabric instead"); + compiled_section sec = build_passthrough_section(name, s); + validate_section_schema(sec); + cfg.passthrough.push_back(std::move(sec)); + } +} + friendly_config parse_friendly(ryml::ConstNodeRef root) { /* reject unknown top-level keys rather than silently ignoring them. */ for (ryml::ConstNodeRef c : root.children()) { std::string k = key_of(c); - if (k != "schema_version" && k != "components" && k != "topology") + if (k != "schema_version" && k != "components" && k != "topology" && k != "sections") throw config_error("config error: unexpected top-level key \"" + k + "\""); } friendly_config cfg; parse_components(root, cfg); parse_topology(root, cfg); + parse_sections(root, cfg); return cfg; } @@ -646,6 +758,10 @@ compiled_config compile(std::string_view text) { compile_flat(cfg, out); else throw config_error("config error: no supported topology found"); + + /* verbatim `sections:` blocks follow the compiler-derived topology sections. */ + for (compiled_section& s : cfg.passthrough) + out.sections.push_back(std::move(s)); return out; } diff --git a/src/modelconfig/configstore.c b/src/modelconfig/configstore.c index 4af9b9e9..06081be8 100644 --- a/src/modelconfig/configstore.c +++ b/src/modelconfig/configstore.c @@ -7,6 +7,7 @@ #include "codes_config.h" #include #include +#include /* strcasecmp: case-insensitive section-name matching */ #include #ifdef HAVE_MALLOC_H #include @@ -273,13 +274,20 @@ static mcs_entry* mcs_findchild(const mcs_entry* e, const char* name) { } +/* Section names are case-insensitive. The all-caps convention (PARAMS, + * LPGROUPS, DIRECTOR, ...) is a historical carryover; the YAML front-end lets a + * config use any case, so section lookups must not depend on it. Keys stay + * case-sensitive -- models read them by exact name -- so only this section-only + * lookup is insensitive, not mcs_findkey. */ mcs_entry* mcs_findsubsection(const mcs_entry* e, const char* name) { - mcs_entry* ret = mcs_findchild(e, name); - if (!ret) - return 0; - if (!ret->is_section) - return 0; - return ret; + mcs_entry* curchild = e->child; + + while (curchild) { + if (curchild->is_section && curchild->name && !strcasecmp(curchild->name, name)) + return curchild; + curchild = curchild->next; + } + return 0; } mcs_entry* mcs_findkey(const mcs_entry* e, const char* name) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7c390049..82b67059 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,6 +15,19 @@ option(CODES_ENABLE_ZMQML_HYBRID_TESTS add_executable(codes-unit-smoke codes-unit-smoke.cxx) target_link_libraries(codes-unit-smoke PRIVATE GTest::gtest_main) add_test(NAME codes-unit-smoke COMMAND codes-unit-smoke) + +# Unit tests for the YAML config compiler core. The core (config_compiler.cxx) +# is ROSS-free by design, so it compiles straight into the test with ryml -- no +# codes/ROSS/MPI link -- and the tests assert on the returned compiled_config. +# This is the first real unit-test consumer of the GoogleTest framework. +add_executable(codes-config-compiler-test + codes-config-compiler-test.cxx + ${CMAKE_SOURCE_DIR}/src/modelconfig/config_compiler.cxx + ${CODES_RYML_IMPL_TU}) +target_include_directories(codes-config-compiler-test SYSTEM PRIVATE ${CODES_RYML_INCLUDE_DIRS}) +target_include_directories(codes-config-compiler-test PRIVATE ${CMAKE_SOURCE_DIR}/src/modelconfig) +target_link_libraries(codes-config-compiler-test PRIVATE GTest::gtest_main) +add_test(NAME codes-config-compiler-test COMMAND codes-config-compiler-test) # --------------------------------------------------------------------------- # MPI launch preflight. Fails loudly if mpiexec can't form a real multi-rank # MPI_COMM_WORLD, guarding against a broken launcher (e.g. Ubuntu 24.04's mpich diff --git a/tests/codes-config-compiler-test.cxx b/tests/codes-config-compiler-test.cxx new file mode 100644 index 00000000..11964da4 --- /dev/null +++ b/tests/codes-config-compiler-test.cxx @@ -0,0 +1,178 @@ +// Unit tests for the YAML config compiler core (codes::config::compile). +// +// The core is ROSS-free by design (see config_compiler.h), so it compiles +// straight into this test alongside ryml -- no codes/ROSS/MPI link -- and the +// tests assert on the returned compiled_config plain data rather than on any +// simulator behavior. This is the first real unit-test consumer of the vendored +// GoogleTest framework; the model-net equivalence tests cover the emit path. +#include "config_compiler.h" + +#include + +using codes::config::compile; +using codes::config::compiled_config; +using codes::config::compiled_key; +using codes::config::compiled_section; +using codes::config::config_error; + +namespace { + +const compiled_section* find_section(const compiled_config& c, const char* name) { + for (const compiled_section& s : c.sections) + if (s.name == name) + return &s; + return nullptr; +} + +const compiled_section* find_subsection(const compiled_section& s, const char* name) { + for (const compiled_section& sub : s.subsections) + if (sub.name == name) + return ⊂ + return nullptr; +} + +const compiled_key* find_key(const compiled_section& s, const char* name) { + for (const compiled_key& k : s.keys) + if (k.name == name) + return &k; + return nullptr; +} + +// A minimal, valid flat-network config; tests append a `sections:` block to it. +const char* kFlatBase = R"( +schema_version: 1 +components: + cn: + model: nw-lp + network: simplenet + message_size: 464 +topology: + format: flat + component: cn + nodes: 4 +)"; + +} // namespace + +// --- topology sanity (the compiler still does its normal job) --------------- + +TEST(ConfigCompiler, FlatTopologyEmitsLpgroupsAndParams) { + compiled_config c = compile(kFlatBase); + const compiled_section* lpg = find_section(c, "LPGROUPS"); + ASSERT_NE(lpg, nullptr); + const compiled_section* grp = find_subsection(*lpg, "MODELNET_GRP"); + ASSERT_NE(grp, nullptr); + const compiled_key* reps = find_key(*grp, "repetitions"); + ASSERT_NE(reps, nullptr); + EXPECT_EQ(reps->values.at(0), "4"); + ASSERT_NE(find_section(c, "PARAMS"), nullptr); +} + +TEST(ConfigCompiler, RejectsUnknownTopLevelKey) { + EXPECT_THROW(compile(std::string(kFlatBase) + "bogus: 1\n"), config_error); +} + +// --- pass-through `sections:` ----------------------------------------------- + +TEST(ConfigCompiler, PassthroughSectionScalarsListsAndNesting) { + std::string yaml = std::string(kFlatBase) + R"( +sections: + director: + start_iter: 100 + fixed_switch_timestamps: [25.0e6, 400.0e6] + nested: + foo: bar +)"; + compiled_config c = compile(yaml); + + const compiled_section* dir = find_section(c, "director"); + ASSERT_NE(dir, nullptr) << "pass-through section should appear verbatim"; + + const compiled_key* start = find_key(*dir, "start_iter"); + ASSERT_NE(start, nullptr); + EXPECT_EQ(start->values.at(0), "100"); + + const compiled_key* ts = find_key(*dir, "fixed_switch_timestamps"); + ASSERT_NE(ts, nullptr); + ASSERT_EQ(ts->values.size(), 2u) << "a YAML sequence becomes a multi-value key"; + EXPECT_EQ(ts->values.at(0), "25.0e6"); + EXPECT_EQ(ts->values.at(1), "400.0e6"); + + const compiled_section* nested = find_subsection(*dir, "nested"); + ASSERT_NE(nested, nullptr) << "a nested map becomes a subsection"; + const compiled_key* foo = find_key(*nested, "foo"); + ASSERT_NE(foo, nullptr); + EXPECT_EQ(foo->values.at(0), "bar"); +} + +TEST(ConfigCompiler, PassthroughPreservesSectionNameCase) { + // The compiler emits the name as written; case-insensitive matching happens + // later at lookup time, so it must not normalize the case here. + std::string yaml = std::string(kFlatBase) + R"( +sections: + NetworkSurrogate: + enable: 1 +)"; + compiled_config c = compile(yaml); + EXPECT_NE(find_section(c, "NetworkSurrogate"), nullptr); + EXPECT_EQ(find_section(c, "networksurrogate"), nullptr); +} + +TEST(ConfigCompiler, PassthroughSectionsFollowTopology) { + std::string yaml = std::string(kFlatBase) + R"( +sections: + director: + start_iter: 1 +)"; + compiled_config c = compile(yaml); + ASSERT_GE(c.sections.size(), 3u); // LPGROUPS, PARAMS, then director + EXPECT_EQ(c.sections.back().name, "director"); +} + +// --- reserved names --------------------------------------------------------- + +TEST(ConfigCompiler, RejectsReservedSectionNames) { + // LPGROUPS / PARAMS are compiler-owned (emitted from the topology), matched + // case-insensitively. + EXPECT_THROW(compile(std::string(kFlatBase) + "sections:\n PARAMS:\n x: 1\n"), + config_error); + EXPECT_THROW(compile(std::string(kFlatBase) + "sections:\n lpgroups:\n x: 1\n"), + config_error); +} + +// --- optional open schema (required keys enforced, extras allowed) ---------- + +TEST(ConfigCompiler, RegisteredSchemaAcceptsRequiredKeyPlusExtras) { + std::string yaml = std::string(kFlatBase) + R"( +sections: + resource: + available: 8192 + experimental_knob: 7 +)"; + compiled_config c = compile(yaml); + const compiled_section* res = find_section(c, "resource"); + ASSERT_NE(res, nullptr); + EXPECT_NE(find_key(*res, "available"), nullptr); + EXPECT_NE(find_key(*res, "experimental_knob"), nullptr) + << "an open schema still passes through keys it doesn't list"; +} + +TEST(ConfigCompiler, RegisteredSchemaRejectsMissingRequiredKey) { + std::string yaml = std::string(kFlatBase) + R"( +sections: + resource: + not_available: 1 +)"; + EXPECT_THROW(compile(yaml), config_error); +} + +// An unregistered section is passed through with no validation at all. +TEST(ConfigCompiler, UnregisteredSectionIsUnvalidated) { + std::string yaml = std::string(kFlatBase) + R"( +sections: + my_new_feature: + anything_goes: 1 +)"; + compiled_config c = compile(yaml); + EXPECT_NE(find_section(c, "my_new_feature"), nullptr); +} From 1726c4407c875f2ede11504ba2c36d29e686d28b Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Fri, 3 Jul 2026 17:59:23 -0500 Subject: [PATCH 10/24] yaml config: explicit LP-groups topology form (`format: groups`) Add an explicit LP-layout form to the YAML front-end for configs that are not a single friendly network -- storage clusters, resource/buffer tests, multi- partition and mapping layouts -- where the compiler derives nothing and the user lays out the groups directly: topology: format: groups params: { message_size: 512 } groups: TRITON_GRP: repetitions: 1 lps: { nw-lp: 1, lsm: 1 } Each group's `repetitions` and `lps` (lp-type -> count) transcribe a .conf LPGROUPS directly, validated (positive counts, known keys). An LP-type key may carry an annotation as `type@annotation` -- the spelling codes_mapping splits on '@' -- so the same type can appear more than once per group. `params` becomes PARAMS (scalars/lists/nested pass through). Composes with the `sections:` pass-through for a model that also reads its own section. Verified end to end: lsm-test (TRITON_GRP + lsm section) and resource-test (BUF + resource section) run byte-identically from their .conf and their new explicit-groups .yaml twins. The config-equivalence harness gains a CONFIG_FLAG option so binaries that take the config as a ROSS option (--conf=, --codes-config=) rather than a trailing positional can be equivalence-tested. Unit tests (codes-config-compiler-test) cover the groups/params structure, annotations, composition with sections, and the validation errors. Documented in doc/dev/yaml-config.md. --- doc/dev/yaml-config.md | 47 ++++++++++++ src/modelconfig/config_compiler.cxx | 86 +++++++++++++++++++++- tests/CMakeLists.txt | 38 +++++++++- tests/codes-config-compiler-test.cxx | 105 +++++++++++++++++++++++++++ tests/conf/buffer_test.yaml | 21 ++++++ tests/conf/lsm-test.yaml | 28 +++++++ 6 files changed, 319 insertions(+), 6 deletions(-) create mode 100644 tests/conf/buffer_test.yaml create mode 100644 tests/conf/lsm-test.yaml diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md index 97c7bc65..c84e2980 100644 --- a/doc/dev/yaml-config.md +++ b/doc/dev/yaml-config.md @@ -558,6 +558,51 @@ topology: --- +# Explicit LP groups: `format: groups` + +The flat and parametric forms above each describe a single network. Some configs +aren't a single network at all — a storage cluster, a mapping test, several +partitions side by side — and there is nothing for the compiler to derive. For +those, lay the LP groups out directly with `format: groups`: + +```yaml +schema_version: 1 + +topology: + format: groups + params: # -> PARAMS, written out verbatim + message_size: 512 + groups: # -> LPGROUPS, one entry per group + TRITON_GRP: + repetitions: 1 + lps: # LP type -> count within each repetition + nw-lp: 1 + lsm: 1 +``` + +Each group names its `repetitions` and, under `lps`, the LP types with their +per-repetition counts — a direct, validated transcription of a `.conf` +`LPGROUPS`. There is no network to derive from, so `modelnet_order` and any other +knobs come from whatever you put in `params`. Group and LP-type names are +free-form (they match what each model registers). + +**Annotations.** `codes_mapping` lets the same LP type appear more than once in a +group under different annotations. Write the annotation on the LP-type key as +`type@annotation`: + +```yaml + lps: + a: 1 + a@foo: 1 # LP type "a", annotation "foo" -- a distinct entry from "a" +``` + +Combine this with `sections:` (below) for a model that also reads its own config +section — e.g. a storage model's `lsm` or `resource` block. +`tests/conf/lsm-test.yaml` and `tests/conf/buffer_test.yaml` are full twins doing +exactly that. + +--- + # Config a model reads directly: `sections:` Not every subsystem's config is topology the compiler derives. Many read their @@ -653,6 +698,8 @@ produce identical results — the authoritative, runnable reference for each mod | dragonfly-dally | `tests/conf/dragonfly-dally/dfdally-72.yaml.in` | | dragonfly-plus | `tests/conf/dragonfly-plus/dfp-test.yaml.in` | | dragonfly-custom| `tests/conf/dragonfly-custom/dfcustom-8group.yaml.in` | +| explicit groups (storage/lsm) | `tests/conf/lsm-test.yaml` | +| explicit groups (resource) | `tests/conf/buffer_test.yaml` | The `.yaml.in` files are CMake templates (the `@CMAKE_SOURCE_DIR@` in their connection paths is substituted at configure time); the plain `.yaml` files are diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx index a6ed5ffb..196d9442 100644 --- a/src/modelconfig/config_compiler.cxx +++ b/src/modelconfig/config_compiler.cxx @@ -138,6 +138,13 @@ struct friendly_config { * Emitted after the topology sections. */ std::vector passthrough; + /* explicit LP-groups topology (`format: groups`): the user lays out groups, + * LP types, counts and annotations directly. The escape hatch for configs + * that are not a single friendly network. Built during parse. */ + bool explicit_groups = false; + compiled_section explicit_lpgroups{"LPGROUPS", {}, {}}; + compiled_section explicit_params{"PARAMS", {}, {}}; + const component* find_component(const std::string& k) const { for (const component& c : components) if (c.key == k) @@ -428,6 +435,57 @@ void parse_fabric(ryml::ConstNodeRef fnode, fabric& fab) { } } +/* defined with the other pass-through helpers below; used here to build PARAMS + * for the explicit-groups form. */ +compiled_section build_passthrough_section(const std::string& name, ryml::ConstNodeRef node); + +/* Build the LPGROUPS section from an explicit `groups` map. Each group names its + * repetitions and its LP types with counts; an LP-type key may carry an + * annotation as `type@annotation` (the .conf spelling codes_mapping splits on + * '@'). This is the general layout escape hatch for configs that are not a + * single friendly network -- storage clusters, multi-partition, mapping tests -- + * where the compiler derives nothing and the user lays out the LPs directly. */ +void parse_explicit_groups(ryml::ConstNodeRef groups, friendly_config& cfg) { + if (!groups.is_map() || groups.num_children() == 0) + throw config_error("config error: topology.groups must be a non-empty map of " + "group-name -> { repetitions, lps }"); + for (ryml::ConstNodeRef g : groups.children()) { + std::string gname = key_of(g); + if (!g.is_map()) + throw config_error("config error: topology.groups: \"" + gname + + "\" must be a block with repetitions and lps"); + for (ryml::ConstNodeRef c : g.children()) { + std::string k = key_of(c); + if (k != "repetitions" && k != "lps") + throw config_error("config error: topology.groups: \"" + gname + + "\": unexpected key \"" + k + "\" (only repetitions and lps)"); + } + if (!has(g, "repetitions")) + throw config_error("config error: topology.groups: \"" + gname + + "\" needs a repetitions count"); + std::string reps_what = "topology.groups \"" + gname + "\" repetitions"; + long reps = parse_int_strict(scalar(g["repetitions"]), reps_what.c_str()); + if (reps <= 0) + throw config_error("config error: topology.groups: \"" + gname + + "\" repetitions must be positive"); + if (!has(g, "lps") || !g["lps"].is_map() || g["lps"].num_children() == 0) + throw config_error("config error: topology.groups: \"" + gname + + "\" needs a non-empty lps map of lp-type -> count"); + compiled_section& grp = cfg.explicit_lpgroups.add_subsection(gname); + grp.add_key("repetitions", std::to_string(reps)); + for (ryml::ConstNodeRef lp : g["lps"].children()) { + std::string lptype = key_of(lp); + std::string count_what = + "topology.groups \"" + gname + "\" lp \"" + lptype + "\" count"; + long count = parse_int_strict(scalar(lp), count_what.c_str()); + if (count <= 0) + throw config_error("config error: topology.groups: \"" + gname + "\": lp \"" + + lptype + "\" count must be positive"); + grp.add_key(lptype, std::to_string(count)); + } + } +} + void parse_topology(ryml::ConstNodeRef root, friendly_config& cfg) { if (!has(root, "topology")) throw config_error("config error: missing required \"topology\" block"); @@ -470,8 +528,28 @@ void parse_topology(ryml::ConstNodeRef root, friendly_config& cfg) { cfg.node_count = parse_int_strict(scalar(topo["nodes"]), "topology.nodes"); if (cfg.node_count <= 0) throw config_error("config error: topology.nodes must be positive"); + } else if (format == "groups") { + cfg.explicit_groups = true; + /* only these keys are consumed for an explicit-groups topology. */ + for (ryml::ConstNodeRef c : topo.children()) { + std::string k = key_of(c); + if (k != "format" && k != "groups" && k != "params") + throw config_error("config error: topology: unexpected key \"" + k + + "\" for an explicit-groups topology"); + } + if (!has(topo, "groups")) + throw config_error("config error: explicit-groups topology needs a \"groups\" block"); + parse_explicit_groups(topo["groups"], cfg); + /* PARAMS is written out directly here (the compiler derives nothing for + * this form); a scalar/list/nested map passes through like any section. */ + if (has(topo, "params")) { + if (!topo["params"].is_map()) + throw config_error("config error: topology.params must be a map of key -> value"); + cfg.explicit_params = build_passthrough_section("PARAMS", topo["params"]); + } } else if (format.empty()) { - throw config_error("config error: topology needs a \"format\" (flat or parametric)"); + throw config_error( + "config error: topology needs a \"format\" (flat, parametric, or groups)"); } else { throw config_error("config error: unknown topology format \"" + format + "\""); } @@ -752,7 +830,11 @@ compiled_config compile(std::string_view text) { friendly_config cfg = parse_friendly(root); compiled_config out; - if (cfg.parametric) + if (cfg.explicit_groups) { + /* explicit form: LPGROUPS and PARAMS were built verbatim during parse. */ + out.sections.push_back(std::move(cfg.explicit_lpgroups)); + out.sections.push_back(std::move(cfg.explicit_params)); + } else if (cfg.parametric) compile_fabric(cfg, out); else if (cfg.flat) compile_flat(cfg, out); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 82b67059..7d1e6922 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -228,9 +228,12 @@ endfunction() # NAME BINARY [NP ] # CONFIG_A # e.g. legacy .conf # CONFIG_B # e.g. new .yaml -# [MARKER ] [ARGS ...]) +# [MARKER ] [ARGS ...] +# [CONFIG_FLAG ]) # e.g. "--conf=" for binaries that take +# # the config as a ROSS option instead +# # of the trailing `-- ` positional function(codes_add_config_equivalence_test) - cmake_parse_arguments(T "" "NAME;BINARY;NP;CONFIG_A;CONFIG_B;MARKER" "ARGS" ${ARGN}) + cmake_parse_arguments(T "" "NAME;BINARY;NP;CONFIG_A;CONFIG_B;MARKER;CONFIG_FLAG" "ARGS" ${ARGN}) if(NOT T_NAME OR NOT T_BINARY OR NOT T_CONFIG_A OR NOT T_CONFIG_B) message(FATAL_ERROR "codes_add_config_equivalence_test: NAME, BINARY, CONFIG_A and CONFIG_B are required") endif() @@ -242,12 +245,21 @@ function(codes_add_config_equivalence_test) endif() set(_bin "$") set(_launch ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${T_NP} ${MPIEXEC_PREFLAGS}) + if(T_CONFIG_FLAG) + # config is a ROSS option (e.g. --conf=): glue it to the prefix and + # pass it among the ROSS args, with no trailing `-- ` positional. + set(_run_a ${_launch} "${_bin}" ${T_ARGS} "${T_CONFIG_FLAG}${T_CONFIG_A}" ${MPIEXEC_POSTFLAGS}) + set(_run_b ${_launch} "${_bin}" ${T_ARGS} "${T_CONFIG_FLAG}${T_CONFIG_B}" ${MPIEXEC_POSTFLAGS}) + else() + set(_run_a ${_launch} "${_bin}" ${T_ARGS} ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_A}") + set(_run_b ${_launch} "${_bin}" ${T_ARGS} ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_B}") + endif() add_test(NAME ${T_NAME} COMMAND "${CMAKE_CURRENT_BINARY_DIR}/run-test.sh" bash "${CMAKE_CURRENT_SOURCE_DIR}/equivalence-run.sh" --marker "${T_MARKER}" - @@ ${_launch} "${_bin}" ${T_ARGS} ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_A}" - @@ ${_launch} "${_bin}" ${T_ARGS} ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_B}" + @@ ${_run_a} + @@ ${_run_b} WORKING_DIRECTORY "${CODES_BINARY_DIR}") endfunction() @@ -493,6 +505,24 @@ codes_add_config_equivalence_test( CONFIG_A "${_tconf}/modelnet-test-slimfly.conf" CONFIG_B "${_tconf}/modelnet-test-slimfly.yaml") +# Explicit-groups form (non-network layouts): a custom group with a workload LP +# co-located with a storage/resource LP, plus that model's own pass-through +# section. These binaries take the config as a ROSS option, not a positional. +codes_add_config_equivalence_test( + NAME lsm-config-equivalence + BINARY lsm-test + CONFIG_FLAG "--conf=" + ARGS --sync=1 + CONFIG_A "${_tconf}/lsm-test.conf" + CONFIG_B "${_tconf}/lsm-test.yaml") +codes_add_config_equivalence_test( + NAME resource-config-equivalence + BINARY resource-test + CONFIG_FLAG "--codes-config=" + ARGS --sync=1 + CONFIG_A "${_tconf}/buffer_test.conf" + CONFIG_B "${_tconf}/buffer_test.yaml") + # fattree's lp-io switch stats are not reproducible run to run, so compare # committed-event counts rather than lp-io. codes_add_config_equivalence_test( diff --git a/tests/codes-config-compiler-test.cxx b/tests/codes-config-compiler-test.cxx index 11964da4..a40745a2 100644 --- a/tests/codes-config-compiler-test.cxx +++ b/tests/codes-config-compiler-test.cxx @@ -52,6 +52,27 @@ schema_version: 1 nodes: 4 )"; +// An explicit LP-groups config: two groups, custom LP types, an annotation, and +// a PARAMS block -- the layout escape hatch for non-network configs. +const char* kGroupsBase = R"( +schema_version: 1 +topology: + format: groups + params: + message_size: 256 + groups: + GRP1: + repetitions: 2 + lps: + a: 1 + b: 2 + a@foo: 1 + GRP2: + repetitions: 3 + lps: + c: 2 +)"; + } // namespace // --- topology sanity (the compiler still does its normal job) --------------- @@ -176,3 +197,87 @@ TEST(ConfigCompiler, UnregisteredSectionIsUnvalidated) { compiled_config c = compile(yaml); EXPECT_NE(find_section(c, "my_new_feature"), nullptr); } + +// --- explicit LP-groups form (`format: groups`) ----------------------------- + +TEST(ConfigCompiler, ExplicitGroupsEmitsLpgroupsAndParams) { + compiled_config c = compile(kGroupsBase); + + const compiled_section* lpg = find_section(c, "LPGROUPS"); + ASSERT_NE(lpg, nullptr); + + const compiled_section* g1 = find_subsection(*lpg, "GRP1"); + ASSERT_NE(g1, nullptr); + ASSERT_NE(find_key(*g1, "repetitions"), nullptr); + EXPECT_EQ(find_key(*g1, "repetitions")->values.at(0), "2"); + ASSERT_NE(find_key(*g1, "b"), nullptr); + EXPECT_EQ(find_key(*g1, "b")->values.at(0), "2"); + + const compiled_section* g2 = find_subsection(*lpg, "GRP2"); + ASSERT_NE(g2, nullptr); + EXPECT_EQ(find_key(*g2, "repetitions")->values.at(0), "3"); + + const compiled_section* params = find_section(c, "PARAMS"); + ASSERT_NE(params, nullptr); + ASSERT_NE(find_key(*params, "message_size"), nullptr); + EXPECT_EQ(find_key(*params, "message_size")->values.at(0), "256"); +} + +TEST(ConfigCompiler, ExplicitGroupsCarriesAnnotationVerbatim) { + compiled_config c = compile(kGroupsBase); + const compiled_section* g1 = find_subsection(*find_section(c, "LPGROUPS"), "GRP1"); + ASSERT_NE(g1, nullptr); + // the annotated LP type keeps its `type@annotation` spelling (codes_mapping + // splits on '@') and is distinct from the un-annotated `a`. + const compiled_key* annotated = find_key(*g1, "a@foo"); + ASSERT_NE(annotated, nullptr); + EXPECT_EQ(annotated->values.at(0), "1"); + EXPECT_NE(find_key(*g1, "a"), nullptr); +} + +TEST(ConfigCompiler, ExplicitGroupsComposeWithPassthroughSections) { + std::string yaml = std::string(kGroupsBase) + R"( +sections: + lsm: + request_sizes: [0] + write_rates: [12000.0] +)"; + compiled_config c = compile(yaml); + EXPECT_NE(find_section(c, "LPGROUPS"), nullptr); + const compiled_section* lsm = find_section(c, "lsm"); + ASSERT_NE(lsm, nullptr); + EXPECT_NE(find_key(*lsm, "request_sizes"), nullptr); +} + +TEST(ConfigCompiler, ExplicitGroupsRejectsMissingRepetitions) { + EXPECT_THROW(compile(R"( +schema_version: 1 +topology: + format: groups + groups: + G: { lps: { a: 1 } } +)"), + config_error); +} + +TEST(ConfigCompiler, ExplicitGroupsRejectsNonPositiveCount) { + EXPECT_THROW(compile(R"( +schema_version: 1 +topology: + format: groups + groups: + G: { repetitions: 1, lps: { a: 0 } } +)"), + config_error); +} + +TEST(ConfigCompiler, ExplicitGroupsRejectsUnknownGroupKey) { + EXPECT_THROW(compile(R"( +schema_version: 1 +topology: + format: groups + groups: + G: { repetitions: 1, lps: { a: 1 }, bogus: 2 } +)"), + config_error); +} diff --git a/tests/conf/buffer_test.yaml b/tests/conf/buffer_test.yaml new file mode 100644 index 00000000..900dc6fb --- /dev/null +++ b/tests/conf/buffer_test.yaml @@ -0,0 +1,21 @@ +schema_version: 1 + +# YAML twin of buffer_test.conf: a BUF group with a workload LP co-located with a +# resource LP. Not a network topology, so it uses the explicit-groups form. The +# resource model reads its own `resource` section (carried through verbatim); the +# compiler enforces that section's required `available` key up front. + +topology: + format: groups + params: + message_size: 300 + groups: + BUF: + repetitions: 2 + lps: + nw-lp: 2 + resource: 1 + +sections: + resource: + available: 8192 diff --git a/tests/conf/lsm-test.yaml b/tests/conf/lsm-test.yaml new file mode 100644 index 00000000..e6d95ac6 --- /dev/null +++ b/tests/conf/lsm-test.yaml @@ -0,0 +1,28 @@ +schema_version: 1 + +# YAML twin of lsm-test.conf: a single TRITON_GRP with a workload LP co-located +# with a local-storage-model (lsm) LP. This is not a network topology, so it uses +# the explicit-groups form -- the group name, LP types and counts are laid out +# directly. The lsm model reads its own `lsm` section, carried through verbatim. + +topology: + format: groups + params: + message_size: 512 + groups: + TRITON_GRP: + repetitions: 1 + lps: + nw-lp: 1 + lsm: 1 + +sections: + lsm: + use_scheduler: 1 + request_sizes: [0] + write_rates: [12000.0] + read_rates: [12000.0] + write_seeks: [2500.0] + read_seeks: [2500.0] + write_overheads: [20.0] + read_overheads: [20.0] From 048f3c6f16e7b35c90859af737dfc5b48c31c77b Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Sat, 4 Jul 2026 21:29:02 -0500 Subject: [PATCH 11/24] yaml config: config includes (`include:`) for reusing config across files A YAML config can now pull in other files with a top-level `include:` (a filename or a list), so a shared fragment -- a network, a set of components, a model section -- can be factored out and reused: define it once, vary the rest (e.g. the same section across different layouts). Included files are the base; the including file overrides them. `components` and `sections` merge by name (local wins on a clash); `topology`/`schema_version` are singular (local replaces). Multiple includes apply in list order, local last. One level deep for now -- an included file may not itself `include:`. The compiler core stays I/O-free: include resolution is a loader-boundary concern. compile() now takes the main document plus already-read base documents and merges them; a new parse_includes() extracts a document's include list. The extern "C" shim (yaml_configfile_load) gains the config path, reads the referenced files relative to its directory, and hands their contents to the core. configuration_load threads the path through. Included files must be readable by every rank (documented). Tests: unit tests for the multi-document merge (component/section merge and override, topology-from-base, nested-include rejection, parse_includes) and an end-to-end equivalence test -- lsm-test-include.yaml includes a shared lsm-workload fragment and runs byte-identical to lsm-test.conf. Documented in doc/dev/yaml-config.md. --- doc/dev/yaml-config.md | 31 ++++++ src/modelconfig/config_compiler.cxx | 146 +++++++++++++++++++++------ src/modelconfig/config_compiler.h | 20 +++- src/modelconfig/configuration.c | 5 +- src/modelconfig/yaml_configfile.cxx | 49 ++++++++- src/modelconfig/yaml_configfile.h | 7 +- tests/CMakeLists.txt | 9 ++ tests/codes-config-compiler-test.cxx | 80 +++++++++++++++ tests/conf/lsm-test-include.yaml | 19 ++++ tests/conf/lsm-workload.yaml | 17 ++++ 10 files changed, 341 insertions(+), 42 deletions(-) create mode 100644 tests/conf/lsm-test-include.yaml create mode 100644 tests/conf/lsm-workload.yaml diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md index c84e2980..3ccca20e 100644 --- a/doc/dev/yaml-config.md +++ b/doc/dev/yaml-config.md @@ -680,6 +680,36 @@ section or tier introduces. --- +# Reusing config across files: `include:` + +A config can pull in other files with a top-level `include:` — a filename, or a +list of them, resolved relative to the including file's directory: + +```yaml +schema_version: 1 +include: [ common.yaml, sections/storage.yaml ] +topology: { ... } # this file's own keys +``` + +Included files are the **base**; the including file **overrides** them. So you +can factor out a shared piece and vary the rest — define a network once and run +it under different configs, or (as in `lsm-test-include.yaml`) share a model +section across layouts. Merge rules: + +- `components:` and `sections:` merge **by name** — an included set plus your own + additions, with a local entry of the same name winning. +- `topology:` and `schema_version:` are singular — a local one replaces the + included one. +- Multiple includes apply in list order; the local file is applied last. + +Includes are resolved when the config is loaded, *before* compilation, so the +referenced files must be readable by every rank. An included file may not itself +use `include:` (one level deep, for now). The pure compiler core does no file +I/O — the loader reads the fragments and hands them in — so `include:` is a +loader feature a model never sees. + +--- + ## Worked examples in the tree Each of these YAML files is a twin of the `.conf` beside it, checked in CI to @@ -700,6 +730,7 @@ produce identical results — the authoritative, runnable reference for each mod | dragonfly-custom| `tests/conf/dragonfly-custom/dfcustom-8group.yaml.in` | | explicit groups (storage/lsm) | `tests/conf/lsm-test.yaml` | | explicit groups (resource) | `tests/conf/buffer_test.yaml` | +| `include:` composition | `tests/conf/lsm-test-include.yaml` (+ `lsm-workload.yaml`) | The `.yaml.in` files are CMake templates (the `@CMAKE_SOURCE_DIR@` in their connection paths is substituted at configure time); the plain `.yaml` files are diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx index 196d9442..773c64c9 100644 --- a/src/modelconfig/config_compiler.cxx +++ b/src/modelconfig/config_compiler.cxx @@ -8,6 +8,7 @@ #include +#include #include #include #include /* strcasecmp: section names are case-insensitive */ @@ -392,6 +393,11 @@ void parse_components(ryml::ConstNodeRef root, friendly_config& cfg) { "params (per-node data, edges and inline workloads are not " "supported)"); } + /* include-merge: a later document's component overrides an earlier one of + * the same name (within one document, keys are already unique). */ + cfg.components.erase(std::remove_if(cfg.components.begin(), cfg.components.end(), + [&](const component& e) { return e.key == c.key; }), + cfg.components.end()); cfg.components.push_back(std::move(c)); } } @@ -656,26 +662,53 @@ void parse_sections(ryml::ConstNodeRef root, friendly_config& cfg) { "model parameters on the component or fabric instead"); compiled_section sec = build_passthrough_section(name, s); validate_section_schema(sec); + /* include-merge: a later document's section overrides an earlier one of + * the same (case-insensitive) name. */ + cfg.passthrough.erase(std::remove_if(cfg.passthrough.begin(), cfg.passthrough.end(), + [&](const compiled_section& e) { + return iequals(e.name, sec.name.c_str()); + }), + cfg.passthrough.end()); cfg.passthrough.push_back(std::move(sec)); } } -friendly_config parse_friendly(ryml::ConstNodeRef root) { - /* reject unknown top-level keys rather than silently ignoring them. */ +/* Parse one document with our throwing error handler installed (so ryml's own + * parse errors route through config_error, exactly like our validation errors), + * check it is a top-level map, and hand its root to `fn`. The parser and tree + * live for the duration of `fn`; the friendly IR copies out owned strings, so + * nothing references the tree afterward. */ +template void with_parsed_document(std::string_view text, F&& fn) { + ryml::Callbacks cb(nullptr, nullptr, nullptr, ryml_throw); + cb.set_error_parse(ryml_throw_parse); + ryml::EventHandlerTree evt_handler(cb); + ryml::Parser parser(&evt_handler); + ryml::Tree tree = ryml::parse_in_arena(&parser, ryml::to_csubstr(""), + ryml::csubstr(text.data(), text.size())); + ryml::ConstNodeRef root = tree.rootref(); + if (!root.readable() || !root.is_map()) + throw config_error("config error: config must be a YAML/JSON mapping at the top level"); + fn(root); +} + +/* reject unknown top-level keys rather than silently ignoring them. An + * included (base) document may not itself use `include` -- nested includes are + * not supported. */ +void validate_toplevel_keys(ryml::ConstNodeRef root, bool is_base) { for (ryml::ConstNodeRef c : root.children()) { std::string k = key_of(c); - if (k != "schema_version" && k != "components" && k != "topology" && k != "sections") + if (k != "schema_version" && k != "components" && k != "topology" && k != "sections" && + k != "include") throw config_error("config error: unexpected top-level key \"" + k + "\""); + if (k == "include" && is_base) + throw config_error("config error: an included file cannot itself use \"include\" " + "(nested includes are not supported)"); } - friendly_config cfg; - parse_components(root, cfg); - parse_topology(root, cfg); - parse_sections(root, cfg); - return cfg; } -/* schema_version is required, integer, and any value this build doesn't - * know is a hard error -- a newer config can't be interpreted safely. */ +/* schema_version is required (on the main document), integer, and any value + * this build doesn't know is a hard error -- a newer config can't be interpreted + * safely. */ void require_schema_version(ryml::ConstNodeRef root) { if (!has(root, "schema_version")) throw config_error( @@ -687,6 +720,37 @@ void require_schema_version(ryml::ConstNodeRef root) { "; this build understands version 1"); } +/* An included document need not restate schema_version, but if it does it must + * agree with this build. */ +void validate_schema_version_if_present(ryml::ConstNodeRef root) { + if (has(root, "schema_version")) + require_schema_version(root); +} + +/* Clear any topology state so a later document's topology fully replaces an + * earlier one (local-overrides-included). */ +void reset_topology(friendly_config& cfg) { + cfg.parametric = false; + cfg.flat = false; + cfg.explicit_groups = false; + cfg.fab = fabric{}; + cfg.flat_component.clear(); + cfg.node_count = 0; + cfg.explicit_lpgroups = compiled_section{"LPGROUPS", {}, {}}; + cfg.explicit_params = compiled_section{"PARAMS", {}, {}}; +} + +/* Merge one document into the accumulating friendly config. Components and + * sections override by name; a topology block replaces any earlier one. */ +void merge_document(ryml::ConstNodeRef root, friendly_config& cfg) { + parse_components(root, cfg); + if (has(root, "topology")) { + reset_topology(cfg); + parse_topology(root, cfg); + } + parse_sections(root, cfg); +} + /* ------------------------------------------------------------------------- * Compile: friendly IR -> compiled_config * ---------------------------------------------------------------------- */ @@ -805,29 +869,47 @@ void compile_flat(const friendly_config& cfg, compiled_config& out) { } // namespace -compiled_config compile(std::string_view text) { - /* Construct the parser with our throwing error handler so that ryml's own - * parse errors route through config_error -> tw_error at the shim, exactly - * like our validation errors. The no-parser parse_in_arena builds an event - * handler with ryml's default callbacks, which print + abort directly and - * bypass the shim, so we build the handler (and thus its callbacks) - * explicitly here. */ - ryml::Callbacks cb(nullptr, nullptr, nullptr, ryml_throw); - /* The Callbacks ctor installs ryml's default parse-error handler (which - * prints + aborts); replace it with ours so parse errors reach the shim's - * tw_error like every other error. */ - cb.set_error_parse(ryml_throw_parse); - ryml::EventHandlerTree evt_handler(cb); - ryml::Parser parser(&evt_handler); - ryml::Tree tree = ryml::parse_in_arena(&parser, ryml::to_csubstr(""), - ryml::csubstr(text.data(), text.size())); - ryml::ConstNodeRef root = tree.rootref(); +std::vector parse_includes(std::string_view doc) { + std::vector out; + with_parsed_document(doc, [&](ryml::ConstNodeRef root) { + if (!has(root, "include")) + return; + ryml::ConstNodeRef inc = root["include"]; + if (inc.is_seq()) { + for (ryml::ConstNodeRef c : inc.children()) { + if (!c.has_val()) + throw config_error("config error: \"include\" list must contain filenames"); + out.push_back(scalar(c)); + } + } else if (inc.has_val()) { + out.push_back(scalar(inc)); + } else { + throw config_error( + "config error: \"include\" must be a filename or a list of filenames"); + } + }); + return out; +} - if (!root.readable() || !root.is_map()) - throw config_error("config error: config must be a YAML/JSON mapping at the top level"); +compiled_config compile(std::string_view main_doc, const std::vector& base_docs) { + friendly_config cfg; - require_schema_version(root); - friendly_config cfg = parse_friendly(root); + /* Included documents are the base; the main document overrides them. Merge + * the bases first, in listed order, then the main document last. */ + for (const std::string& doc : base_docs) + with_parsed_document(doc, [&](ryml::ConstNodeRef root) { + validate_toplevel_keys(root, /*is_base=*/true); + validate_schema_version_if_present(root); + merge_document(root, cfg); + }); + with_parsed_document(main_doc, [&](ryml::ConstNodeRef root) { + validate_toplevel_keys(root, /*is_base=*/false); + require_schema_version(root); + merge_document(root, cfg); + }); + + if (!cfg.parametric && !cfg.flat && !cfg.explicit_groups) + throw config_error("config error: missing required \"topology\" block"); compiled_config out; if (cfg.explicit_groups) { @@ -838,8 +920,6 @@ compiled_config compile(std::string_view text) { compile_fabric(cfg, out); else if (cfg.flat) compile_flat(cfg, out); - else - throw config_error("config error: no supported topology found"); /* verbatim `sections:` blocks follow the compiler-derived topology sections. */ for (compiled_section& s : cfg.passthrough) diff --git a/src/modelconfig/config_compiler.h b/src/modelconfig/config_compiler.h index f9197c72..9b2ab876 100644 --- a/src/modelconfig/config_compiler.h +++ b/src/modelconfig/config_compiler.h @@ -78,11 +78,25 @@ struct compiled_config { /** * Compile YAML/JSON configuration text into the compiled_config IR. * - * @param text the raw config bytes (JSON is a subset of YAML, so one parser - * handles both). + * @param main_doc the raw config bytes of the top-level file (JSON is a subset + * of YAML, so one parser handles both). + * @param base_docs the contents of any files named by `main_doc`'s top-level + * `include:` list, already read (the pure core does no file + * I/O), in listed order. They are merged as the base; + * `main_doc` overrides them (components and sections merge by + * name, a topology block replaces any earlier one). * @throws config_error on malformed YAML or any schema violation. */ -compiled_config compile(std::string_view text); +compiled_config compile(std::string_view main_doc, const std::vector& base_docs = {}); + +/** + * Extract a document's top-level `include:` list (filenames), or an empty vector + * if it has none. The loader boundary uses this to read the referenced files and + * pass them to compile() as base documents, keeping this core free of file I/O. + * + * @throws config_error on malformed YAML or a malformed `include:` value. + */ +std::vector parse_includes(std::string_view doc); } // namespace config } // namespace codes diff --git a/src/modelconfig/configuration.c b/src/modelconfig/configuration.c index 90865772..75fb06e2 100644 --- a/src/modelconfig/configuration.c +++ b/src/modelconfig/configuration.c @@ -62,8 +62,9 @@ int configuration_load(const char* filepath, MPI_Comm comm, ConfigHandle* handle if (config_is_yaml(filepath)) { /* the YAML front-end compiles the friendly format into the same * ConfigVTable the text parser yields, reading the bytes already pulled - * in by the collective read above. */ - *handle = yaml_configfile_load(txtdata, txtsize); + * in by the collective read above. filepath lets it resolve a top-level + * `include:` list relative to the config's directory. */ + *handle = yaml_configfile_load(txtdata, txtsize, filepath); } else { #ifdef __APPLE__ f = fopen(filepath, "r"); diff --git a/src/modelconfig/yaml_configfile.cxx b/src/modelconfig/yaml_configfile.cxx index a8d5d343..1e79472b 100644 --- a/src/modelconfig/yaml_configfile.cxx +++ b/src/modelconfig/yaml_configfile.cxx @@ -25,11 +25,56 @@ #include +#include +#include +#include #include +#include -struct ConfigVTable* yaml_configfile_load(const char* data, size_t len) { +namespace { + +/* Directory portion of a path ("a/b/c.yaml" -> "a/b"), or "." if none. */ +std::string dir_of(const char* path) { + std::string p = path ? path : ""; + std::string::size_type slash = p.find_last_of('/'); + return slash == std::string::npos ? std::string(".") : p.substr(0, slash); +} + +/* Resolve an include path: absolute as-is, otherwise relative to base_dir. */ +std::string resolve_path(const std::string& base_dir, const std::string& rel) { + if (!rel.empty() && rel[0] == '/') + return rel; + return base_dir + "/" + rel; +} + +/* Read a whole file into a string, throwing config_error if it can't be read. */ +std::string read_file(const std::string& path) { + std::ifstream in(path, std::ios::binary); + if (!in) + throw codes::config::config_error("config error: cannot read included file \"" + path + + "\""); + std::ostringstream ss; + ss << in.rdbuf(); + return ss.str(); +} + +/* Resolve the main document's top-level `include:` list into the contents of the + * referenced files (in listed order), read relative to the config's directory. */ +std::vector read_includes(std::string_view main_doc, const char* path) { + std::vector docs; + std::string base_dir = dir_of(path); + for (const std::string& rel : codes::config::parse_includes(main_doc)) + docs.push_back(read_file(resolve_path(base_dir, rel))); + return docs; +} + +} // namespace + +struct ConfigVTable* yaml_configfile_load(const char* data, size_t len, const char* path) { try { - codes::config::compiled_config cfg = codes::config::compile(std::string_view(data, len)); + std::string_view main_doc(data, len); + std::vector includes = read_includes(main_doc, path); + codes::config::compiled_config cfg = codes::config::compile(main_doc, includes); return codes::config::emit(cfg); } catch (const codes::config::config_error& e) { tw_error(TW_LOC, "%s", e.what()); diff --git a/src/modelconfig/yaml_configfile.h b/src/modelconfig/yaml_configfile.h index ce4901c7..2f10bc1a 100644 --- a/src/modelconfig/yaml_configfile.h +++ b/src/modelconfig/yaml_configfile.h @@ -21,11 +21,14 @@ extern "C" { * This is the thin C boundary of the front-end: it runs the pure C++ core * (compile) and the emitter (emit), and it is the only place that translates a * compile-time config_error into a ROSS tw_error. data/len are the raw file - * bytes (already read, e.g. via the MPI collective read in configuration_load). + * bytes (already read, e.g. via the MPI collective read in configuration_load); + * path is the config file's path, used to resolve a top-level `include:` list + * relative to its directory (the boundary reads the referenced files -- the pure + * core does no file I/O -- so included files must be readable by every rank). * On success returns a newly-allocated ConfigVTable (free with cf_free); on a * syntax or validation error it aborts through tw_error with a diagnostic, * matching how the rest of the configuration front-end reports malformed input. */ -struct ConfigVTable* yaml_configfile_load(const char* data, size_t len); +struct ConfigVTable* yaml_configfile_load(const char* data, size_t len, const char* path); #ifdef __cplusplus } /* extern "C" */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7d1e6922..bd0443e2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -515,6 +515,15 @@ codes_add_config_equivalence_test( ARGS --sync=1 CONFIG_A "${_tconf}/lsm-test.conf" CONFIG_B "${_tconf}/lsm-test.yaml") +# Same run, but the lsm section is factored into a shared fragment pulled in with +# `include:` -- exercises the loader-boundary include resolution end to end. +codes_add_config_equivalence_test( + NAME lsm-include-equivalence + BINARY lsm-test + CONFIG_FLAG "--conf=" + ARGS --sync=1 + CONFIG_A "${_tconf}/lsm-test.conf" + CONFIG_B "${_tconf}/lsm-test-include.yaml") codes_add_config_equivalence_test( NAME resource-config-equivalence BINARY resource-test diff --git a/tests/codes-config-compiler-test.cxx b/tests/codes-config-compiler-test.cxx index a40745a2..08db3e4d 100644 --- a/tests/codes-config-compiler-test.cxx +++ b/tests/codes-config-compiler-test.cxx @@ -14,6 +14,7 @@ using codes::config::compiled_config; using codes::config::compiled_key; using codes::config::compiled_section; using codes::config::config_error; +using codes::config::parse_includes; namespace { @@ -281,3 +282,82 @@ schema_version: 1 )"), config_error); } + +// --- includes / multi-document merge ---------------------------------------- + +TEST(ConfigCompiler, IncludeMergesComponentsFromBaseDoc) { + // the base defines the component; the main references it in its topology. + std::string base = R"( +schema_version: 1 +components: + cn: { model: nw-lp, network: simplenet, message_size: 464 } +)"; + std::string main = R"( +schema_version: 1 +topology: { format: flat, component: cn, nodes: 4 } +)"; + compiled_config c = compile(main, {base}); + const compiled_section* grp = find_subsection(*find_section(c, "LPGROUPS"), "MODELNET_GRP"); + ASSERT_NE(grp, nullptr); + EXPECT_NE(find_key(*grp, "modelnet_simplenet"), nullptr); // component from base resolved +} + +TEST(ConfigCompiler, IncludeLocalOverridesBaseComponent) { + std::string base = R"( +schema_version: 1 +components: + cn: { model: nw-lp, network: simplenet, message_size: 111 } +)"; + std::string main = R"( +schema_version: 1 +components: + cn: { model: nw-lp, network: simplenet, message_size: 999 } +topology: { format: flat, component: cn, nodes: 2 } +)"; + compiled_config c = compile(main, {base}); + const compiled_section* params = find_section(c, "PARAMS"); + ASSERT_NE(params, nullptr); + ASSERT_NE(find_key(*params, "message_size"), nullptr); + EXPECT_EQ(find_key(*params, "message_size")->values.at(0), "999"); // local wins +} + +TEST(ConfigCompiler, IncludeMergesSectionsAndTopologyFromBase) { + // "same network, vary the rest": the base provides topology; main adds a section. + std::string base = kFlatBase; + std::string main = R"( +schema_version: 1 +sections: + director: { start_iter: 1 } +)"; + compiled_config c = compile(main, {base}); + EXPECT_NE(find_section(c, "LPGROUPS"), nullptr); // topology from base + EXPECT_NE(find_section(c, "director"), nullptr); // section from main +} + +TEST(ConfigCompiler, NestedIncludeRejected) { + std::string base = R"( +schema_version: 1 +include: [other.yaml] +components: + cn: { model: nw-lp, network: simplenet } +)"; + std::string main = R"( +schema_version: 1 +topology: { format: flat, component: cn, nodes: 2 } +)"; + EXPECT_THROW(compile(main, {base}), config_error); +} + +TEST(ConfigCompiler, ParseIncludesExtractsList) { + EXPECT_TRUE(parse_includes("schema_version: 1\ntopology: {}\n").empty()); + + std::vector one = parse_includes("include: common.yaml\nschema_version: 1\n"); + ASSERT_EQ(one.size(), 1u); + EXPECT_EQ(one.at(0), "common.yaml"); + + std::vector many = + parse_includes("include: [a.yaml, b.yaml]\nschema_version: 1\n"); + ASSERT_EQ(many.size(), 2u); + EXPECT_EQ(many.at(0), "a.yaml"); + EXPECT_EQ(many.at(1), "b.yaml"); +} diff --git a/tests/conf/lsm-test-include.yaml b/tests/conf/lsm-test-include.yaml new file mode 100644 index 00000000..6b5ef2e5 --- /dev/null +++ b/tests/conf/lsm-test-include.yaml @@ -0,0 +1,19 @@ +schema_version: 1 + +# Same run as lsm-test.yaml, but the lsm section is factored into a shared +# fragment and pulled in with `include:`. Demonstrates config composition: the +# storage-model config lives in one file (lsm-workload.yaml), the layout here. +# Included files are the base; keys in this file override them. + +include: [ lsm-workload.yaml ] + +topology: + format: groups + params: + message_size: 512 + groups: + TRITON_GRP: + repetitions: 1 + lps: + nw-lp: 1 + lsm: 1 diff --git a/tests/conf/lsm-workload.yaml b/tests/conf/lsm-workload.yaml new file mode 100644 index 00000000..aa5dc223 --- /dev/null +++ b/tests/conf/lsm-workload.yaml @@ -0,0 +1,17 @@ +schema_version: 1 + +# A reusable fragment: just the local-storage-model (lsm) section. A run file +# pulls this in with `include:` and supplies its own topology -- so the same +# storage-model config can be shared across different layouts. See +# lsm-test-include.yaml. + +sections: + lsm: + use_scheduler: 1 + request_sizes: [0] + write_rates: [12000.0] + read_rates: [12000.0] + write_seeks: [2500.0] + read_seeks: [2500.0] + write_overheads: [20.0] + read_overheads: [20.0] From 9c0d6ee2e847ea2fa9cb461905439c2053ef15fb Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Mon, 6 Jul 2026 13:28:45 -0500 Subject: [PATCH 12/24] fix: strict validation and boundary hardening for the YAML front-end Close the validation gaps found in review of the config compiler core: - shape values parse strictly: a non-integer (num_groups: abc) or trailing garbage (9x) is a diagnostic naming the key, not a silent 0/9; a backstop after every fabric derivation rejects degenerate layouts (repetitions/terminals <= 0) before they reach codes_mapping - dim_length: reject empty segments, trailing commas, and space-separated lists; guard product overflow; cross-check the entry count against n_dims (torus, express-mesh) - reject an empty component model: (was an empty LP-type key in LPGROUPS), unexpected keys under hosts:, the component type: key (reserved; would otherwise leak into PARAMS), network: on a parametric host component, and an odd fattree switch_radix - replace the deprecated 4-arg ryml::Callbacks constructor with the default constructor + setters (silences the only warning in new code) - catch std::exception at the extern "C" boundary so bad_alloc & co. reach tw_error with a message instead of std::terminate - match the config extension case-insensitively (.YAML, .Yml, .JSON) Adds 13 unit tests (first parametric-path coverage plus each new error path) and the matching yaml-config.md updates. --- doc/dev/yaml-config.md | 44 +++-- src/modelconfig/config_compiler.cxx | 153 +++++++++++---- src/modelconfig/configuration.c | 8 +- src/modelconfig/yaml_configfile.cxx | 7 + tests/codes-config-compiler-test.cxx | 266 +++++++++++++++++++++++++++ 5 files changed, 433 insertions(+), 45 deletions(-) diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md index 3ccca20e..92c8618e 100644 --- a/doc/dev/yaml-config.md +++ b/doc/dev/yaml-config.md @@ -24,12 +24,14 @@ enable and no configure-time option to set. ## Shape of a config -A config has a few top-level blocks: +A config has up to five top-level blocks: ```yaml schema_version: 1 # required; this build understands version 1 components: # named component configs referenced by the topology -topology: # a flat network, or a parametric fabric +topology: # flat network, parametric fabric, or explicit LP groups +sections: # config a model reads directly, carried through verbatim +include: # other config files to reuse (base; this file overrides) ``` - **`schema_version`** is required, an integer, and must be a version this build @@ -39,14 +41,21 @@ topology: # a flat network, or a parametric fabric (`model:`) with its parameters; a flat-network component also names the NIC model it runs over (`network:`). Components are referenced by name from the topology. -- **`topology`** selects the network. `format: flat` is an all-to-all point-to- - point network described by a component and a node count; `format: parametric` - is an HPC fabric described by shape parameters. +- **`topology`** selects the layout, via one of three `format`s: `flat` is an + all-to-all point-to-point network described by a component and a node count; + `parametric` is an HPC fabric described by shape parameters; `groups` lays the + LP groups out directly (the escape hatch for configs that aren't a single + network). Each is documented in its own section below. +- **`sections`** carries config a model reads directly by name (DIRECTOR, + NETWORK_SURROGATE, resource, …) through verbatim — see `sections:` below. +- **`include`** reuses other config files — see `include:` below. Validation is strict: unknown top-level keys, unknown topology keys, a block a component or fabric doesn't consume, and (for flat topologies) an unexpected `edges`/graph block are all **errors, not silent drops**. A malformed value -(e.g. a non-integer node count) is likewise rejected with a diagnostic. +(e.g. a non-integer node count) is likewise rejected with a diagnostic. On any +such error the run aborts at config load with that diagnostic — on every rank, +since every rank compiles the same bytes deterministically. ### How it compiles @@ -199,6 +208,10 @@ Building blocks shared by all fabrics: `PARAMS` unchanged. A list value (e.g. `slimfly`'s generator sets) is emitted as a multi-value key. +The `hosts.component`'s own scalar params, if it declares any, are also appended +to `PARAMS` (after the fabric's keys). A parametric host component must not set +`network:` — the fabric defines the network itself, so that key is rejected here. + Supported fabric `model`s split into **internally-generated** (the compiler emits only shape params and the model generates its wiring) and **file-enumerated** (the wiring is read from binary connection files produced by the generator scripts). @@ -246,10 +259,13 @@ topology: An n-dimensional torus. It folds routing into the terminal node, so there is **no separate router LP**; the node count is the product of the per-dimension lengths. `dim_length` is a comma-separated string the model parses itself, so it -is written as a quoted scalar (not a YAML list). Bandwidth is a single flat +is written as a quoted scalar (not a YAML list) — a strictly comma-separated list +of positive integers whose count must equal `n_dims` (a mismatch, an empty +segment, or a trailing comma is a config error). Bandwidth is a single flat `link_bandwidth`, not a per-link-class block. -Shape: `n_dims`, `dim_length` (repetitions = product of `dim_length`). +Shape: `n_dims`, `dim_length` (repetitions = product of `dim_length`, one entry +per `n_dims`). ```yaml schema_version: 1 @@ -324,9 +340,11 @@ topology: An n-dimensional express mesh with a separate router LP. The router count is the product of the per-dimension lengths, and each router hosts `num_cn` terminals. Like torus it uses flat bandwidth keys (`link_bandwidth`, `cn_bandwidth`, ...) -rather than a `links` block, and `dim_length` is a quoted scalar. +rather than a `links` block, and `dim_length` is a quoted scalar — a strictly +comma-separated list of positive integers whose count must equal `n_dims`. -Shape: `n_dims`, `dim_length` (router count = product of `dim_length`), `num_cn`. +Shape: `n_dims`, `dim_length` (router count = product of `dim_length`, one entry +per `n_dims`), `num_cn`. ```yaml schema_version: 1 @@ -583,8 +601,10 @@ topology: Each group names its `repetitions` and, under `lps`, the LP types with their per-repetition counts — a direct, validated transcription of a `.conf` `LPGROUPS`. There is no network to derive from, so `modelnet_order` and any other -knobs come from whatever you put in `params`. Group and LP-type names are -free-form (they match what each model registers). +knobs come from whatever you put in `params`. `params:` follows the same +scalar/list/nested-map rules as a `sections:` block (below): a scalar becomes a +single-value key, a list a multi-value key, and a nested map a subsection. Group +and LP-type names are free-form (they match what each model registers). **Annotations.** `codes_mapping` lets the same LP type appear more than once in a group under different annotations. Write the annotation on the LP-type key as diff --git a/src/modelconfig/config_compiler.cxx b/src/modelconfig/config_compiler.cxx index 773c64c9..2d7ea105 100644 --- a/src/modelconfig/config_compiler.cxx +++ b/src/modelconfig/config_compiler.cxx @@ -10,6 +10,7 @@ #include #include +#include /* LONG_MAX: guard dimension-product overflow */ #include #include /* strcasecmp: section names are case-insensitive */ #include @@ -176,20 +177,29 @@ struct fabric_model { layout (*derive)(const kv_list& shape); /* shape -> LP layout */ }; -/* Look up a shape value by name, throwing if absent. */ +/* Look up a shape value by name, throwing if absent. The value is parsed + * strictly, so a non-integer (num_groups: abc) or trailing garbage (9x) is a + * diagnostic naming the offending key rather than a silent 0/9. */ long shape_int(const kv_list& shape, const char* key) { - for (const auto& kv : shape) - if (kv.first == key) - return std::strtol(kv.second.c_str(), nullptr, 10); + for (const auto& kv : shape) { + if (kv.first == key) { + std::string what = std::string("fabric shape \"") + key + "\""; + return parse_int_strict(kv.second, what.c_str()); + } + } throw config_error(std::string("config error: fabric shape is missing required key \"") + key + "\""); } -/* Look up a shape value by name, returning a default when absent. */ +/* Look up a shape value by name, returning a default when absent. Present values + * are parsed strictly (see shape_int). */ long shape_int_default(const kv_list& shape, const char* key, long dflt) { - for (const auto& kv : shape) - if (kv.first == key) - return std::strtol(kv.second.c_str(), nullptr, 10); + for (const auto& kv : shape) { + if (kv.first == key) { + std::string what = std::string("fabric shape \"") + key + "\""; + return parse_int_strict(kv.second, what.c_str()); + } + } return dflt; } @@ -202,30 +212,57 @@ const std::string& shape_str(const kv_list& shape, const char* key) { "\""); } -/* Product of a comma-separated dimension list ("4,2,2" -> 16). Used by the - * mesh-style fabrics (torus, express_mesh) whose repetition count is the number - * of mesh nodes/routers implied by dim_length. */ -long dim_product(const kv_list& shape, const char* key) { +/* Parse a comma-separated dimension list ("4,2,2" -> {4,2,2}) strictly: a + * non-empty, strictly comma-separated list of positive integers (optional spaces + * around a value), rejecting empty segments ("4,,2"), a trailing comma ("4,2,"), + * and space-separated lists ("4 2"). */ +std::vector parse_dim_lengths(const kv_list& shape, const char* key) { const std::string& s = shape_str(shape, key); - long prod = 1; + auto bad = [&]() -> config_error { + return config_error(std::string("config error: fabric shape \"") + key + + "\" must be a comma-separated list of positive integers, got \"" + s + + "\""); + }; + std::vector dims; const char* p = s.c_str(); - bool saw_digit = false; - while (*p) { + for (;;) { char* end = nullptr; errno = 0; - long d = std::strtol(p, &end, 10); + long d = std::strtol(p, &end, 10); /* strtol skips any leading spaces */ if (end == p || errno != 0 || d <= 0) - throw config_error(std::string("config error: fabric shape \"") + key + - "\" must be a comma-separated list of positive integers, got \"" + - s + "\""); - prod *= d; - saw_digit = true; + throw bad(); + dims.push_back(d); p = end; - while (*p == ',' || *p == ' ') + while (*p == ' ') /* trailing spaces after this value */ ++p; + if (*p == '\0') + break; + if (*p != ',') /* only a comma may separate values (rejects "4 2") */ + throw bad(); + ++p; /* consume the single comma; the next value is now required */ + } + return dims; +} + +/* Node/router count of a mesh-style fabric (torus, express_mesh): the product of + * the per-dimension lengths in dim_length. The entry count is cross-checked + * against the n_dims shape value, and the running product is guarded against + * signed overflow. `model_name` names the offending fabric in diagnostics. */ +long mesh_node_count(const kv_list& shape, const char* model_name) { + long n_dims = shape_int(shape, "n_dims"); + std::vector dims = parse_dim_lengths(shape, "dim_length"); + if (static_cast(dims.size()) != n_dims) + throw config_error(std::string("config error: ") + model_name + " n_dims (" + + std::to_string(n_dims) + + ") does not match the number of dim_length entries (" + + std::to_string(dims.size()) + ")"); + long prod = 1; + for (long d : dims) { + if (prod > LONG_MAX / d) /* d >= 1, so the divide is safe */ + throw config_error(std::string("config error: ") + model_name + + " dim_length product overflows"); + prod *= d; } - if (!saw_digit) - throw config_error(std::string("config error: fabric shape \"") + key + "\" is empty"); return prod; } @@ -260,6 +297,12 @@ layout derive_fattree(const kv_list& shape) { long switch_count = shape_int(shape, "switch_count"); long switch_radix = shape_int(shape, "switch_radix"); long num_levels = shape_int(shape, "num_levels"); + /* Each edge switch hosts switch_radix/2 terminals; an odd radix would + * silently truncate that split, so reject it rather than lose a terminal. */ + if (switch_radix % 2 != 0) + throw config_error("config error: fattree switch_radix must be even (each edge switch " + "hosts switch_radix/2 terminals), got " + + std::to_string(switch_radix)); return {switch_count, switch_radix / 2, num_levels}; } @@ -267,7 +310,7 @@ layout derive_fattree(const kv_list& shape) { * terminal, and no separate router LP (the torus node combines routing and the * terminal). The node count is the product of the per-dimension lengths. */ layout derive_torus(const kv_list& shape) { - long nodes = dim_product(shape, "dim_length"); + long nodes = mesh_node_count(shape, "torus"); return {nodes, 1, 0}; } @@ -275,7 +318,7 @@ layout derive_torus(const kv_list& shape) { * hosting num_cn terminals plus one router LP. The router count is the product * of the per-dimension lengths. */ layout derive_express_mesh(const kv_list& shape) { - long routers = dim_product(shape, "dim_length"); + long routers = mesh_node_count(shape, "express-mesh"); long num_cn = shape_int(shape, "num_cn"); return {routers, num_cn, 1}; } @@ -383,7 +426,13 @@ void parse_components(ryml::ConstNodeRef root, friendly_config& cfg) { else if (k == "network") c.network = scalar(f); else if (k == "type") - ; /* inferred from the model; not needed for the compiled config */ + /* Reserved for a future schema version. Reject explicitly: a bare + * `type:` scalar would otherwise fall through to the is_keyval() + * branch below and silently become a model param in PARAMS. */ + throw config_error("config error: component \"" + c.key + + "\": key \"type\" is reserved for a future schema version and " + "is not accepted yet; remove it (the model is inferred from " + "\"model:\")"); else if (f.is_keyval()) c.params.emplace_back(k, scalar(f)); else @@ -511,11 +560,22 @@ void parse_topology(ryml::ConstNodeRef root, friendly_config& cfg) { if (!has(topo, "fabric")) throw config_error("config error: parametric topology needs a \"fabric\" block"); parse_fabric(topo["fabric"], cfg.fab); - if (has(topo, "hosts") && has(topo["hosts"], "component")) - cfg.fab.hosts_component = scalar(topo["hosts"]["component"]); - else + if (!has(topo, "hosts")) + throw config_error("config error: parametric topology needs hosts.component naming the " + "per-terminal workload"); + ryml::ConstNodeRef hosts = topo["hosts"]; + /* only `component` is consumed under hosts; anything else is an error + * rather than a silent drop. */ + for (ryml::ConstNodeRef h : hosts.children()) { + std::string k = key_of(h); + if (k != "component") + throw config_error("config error: topology.hosts: unexpected key \"" + k + + "\" (only \"component\" is supported)"); + } + if (!has(hosts, "component")) throw config_error("config error: parametric topology needs hosts.component naming the " "per-terminal workload"); + cfg.fab.hosts_component = scalar(hosts["component"]); } else if (format == "flat") { cfg.flat = true; /* only these keys are consumed for a flat topology. */ @@ -679,7 +739,8 @@ void parse_sections(ryml::ConstNodeRef root, friendly_config& cfg) { * live for the duration of `fn`; the friendly IR copies out owned strings, so * nothing references the tree afterward. */ template void with_parsed_document(std::string_view text, F&& fn) { - ryml::Callbacks cb(nullptr, nullptr, nullptr, ryml_throw); + ryml::Callbacks cb; + cb.set_error_basic(ryml_throw); cb.set_error_parse(ryml_throw_parse); ryml::EventHandlerTree evt_handler(cb); ryml::Parser parser(&evt_handler); @@ -767,9 +828,33 @@ void compile_fabric(const friendly_config& cfg, compiled_config& out) { if (!host) throw config_error("config error: hosts.component \"" + fab.hosts_component + "\" is not defined under components:"); + /* An empty model: would flow into LPGROUPS as an empty LP-type key and fail + * confusingly in codes_mapping later; reject it here with a clear message. */ + if (host->model.empty()) + throw config_error("config error: hosts.component \"" + fab.hosts_component + + "\" has an empty \"model:\"; a component needs a model naming its " + "workload LP type"); + /* A parametric fabric defines the network itself, so a network: on the host + * component is meaningless here (it only applies to flat-topology components) + * and would otherwise be silently ignored. */ + if (!host->network.empty()) + throw config_error("config error: hosts.component \"" + fab.hosts_component + + "\" sets \"network:\", which is only meaningful for a flat-topology " + "component; a parametric fabric defines the network itself"); layout lay = model->derive(fab.shape); + /* A backstop over every model's derivation: a shape that produces a + * degenerate layout (e.g. num_groups: 0) must not reach codes_mapping. Each + * repetition needs at least one terminal; routers may legitimately be 0 for + * mesh-style fabrics that fold routing into the terminal. */ + if (lay.repetitions <= 0 || lay.terminals_per_rep <= 0 || lay.routers_per_rep < 0) + throw config_error( + "config error: fabric model \"" + fab.model + + "\" derived a degenerate layout (repetitions=" + std::to_string(lay.repetitions) + + ", terminals_per_rep=" + std::to_string(lay.terminals_per_rep) + ", routers_per_rep=" + + std::to_string(lay.routers_per_rep) + "); check the shape values"); + /* --- LPGROUPS: one group of `repetitions` slices, each with the * per-terminal workload + NIC LPs and the router/switch LPs, emitted in * [workload, terminal, router] order to match the layout the model @@ -843,6 +928,12 @@ void compile_flat(const friendly_config& cfg, compiled_config& out) { if (!comp) throw config_error("config error: topology.component \"" + cfg.flat_component + "\" is not defined under components:"); + /* An empty model: would flow into LPGROUPS as an empty LP-type key and fail + * confusingly in codes_mapping later; reject it here with a clear message. */ + if (comp->model.empty()) + throw config_error("config error: topology.component \"" + cfg.flat_component + + "\" has an empty \"model:\"; a component needs a model naming its " + "workload LP type"); if (comp->network.empty()) throw config_error("config error: component \"" + cfg.flat_component + "\" needs a network: field naming its NIC model"); diff --git a/src/modelconfig/configuration.c b/src/modelconfig/configuration.c index 75fb06e2..c267b6a4 100644 --- a/src/modelconfig/configuration.c +++ b/src/modelconfig/configuration.c @@ -7,6 +7,7 @@ #include #include #include +#include /* strcasecmp: the extension match is case-insensitive */ #include #include #include @@ -18,12 +19,15 @@ #include "yaml_configfile.h" /* A .yaml/.yml/.json path selects the YAML/JSON config front-end; anything else - * is parsed by the legacy .conf text parser. */ + * is parsed by the legacy .conf text parser. The extension is matched + * case-insensitively so .YAML/.Yml/.JSON select the front-end too rather than + * falling through and dying with an unrelated .conf syntax error. */ static int config_is_yaml(const char* path) { const char* dot = strrchr(path, '.'); if (!dot) return 0; - return strcmp(dot, ".yaml") == 0 || strcmp(dot, ".yml") == 0 || strcmp(dot, ".json") == 0; + return strcasecmp(dot, ".yaml") == 0 || strcasecmp(dot, ".yml") == 0 || + strcasecmp(dot, ".json") == 0; } /* diff --git a/src/modelconfig/yaml_configfile.cxx b/src/modelconfig/yaml_configfile.cxx index 1e79472b..5b47c9f4 100644 --- a/src/modelconfig/yaml_configfile.cxx +++ b/src/modelconfig/yaml_configfile.cxx @@ -25,6 +25,7 @@ #include +#include #include #include #include @@ -79,5 +80,11 @@ struct ConfigVTable* yaml_configfile_load(const char* data, size_t len, const ch } catch (const codes::config::config_error& e) { tw_error(TW_LOC, "%s", e.what()); return nullptr; /* unreachable: tw_error aborts */ + } catch (const std::exception& e) { + /* Any non-config_error exception (std::bad_alloc, a ryml internal, ...) + * must not escape this extern "C" frame -- that would call + * std::terminate with no diagnostic. Route it to tw_error too. */ + tw_error(TW_LOC, "config error (internal): %s", e.what()); + return nullptr; /* unreachable: tw_error aborts */ } } diff --git a/tests/codes-config-compiler-test.cxx b/tests/codes-config-compiler-test.cxx index 08db3e4d..dddb9080 100644 --- a/tests/codes-config-compiler-test.cxx +++ b/tests/codes-config-compiler-test.cxx @@ -53,6 +53,24 @@ schema_version: 1 nodes: 4 )"; +// A minimal, valid parametric dragonfly config. With num_routers: 4 the +// derivation is num_cn = 4/2 = 2, num_groups = 4*2+1 = 9, so repetitions = +// num_groups*num_routers = 36, terminals per rep = 2, one router per rep. +const char* kParametricDragonfly = R"( +schema_version: 1 +components: + compute_host: + model: nw-lp +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + hosts: + component: compute_host +)"; + // An explicit LP-groups config: two groups, custom LP types, an annotation, and // a PARAMS block -- the layout escape hatch for non-network configs. const char* kGroupsBase = R"( @@ -283,6 +301,254 @@ schema_version: 1 config_error); } +// --- parametric fabrics ----------------------------------------------------- + +TEST(ConfigCompiler, ParametricDragonflyDerivesLayoutAndOrder) { + compiled_config c = compile(kParametricDragonfly); + + const compiled_section* lpg = find_section(c, "LPGROUPS"); + ASSERT_NE(lpg, nullptr); + const compiled_section* grp = find_subsection(*lpg, "MODELNET_GRP"); + ASSERT_NE(grp, nullptr); + + // num_routers: 4 -> num_cn 2, num_groups 9, repetitions 9*4 = 36. + ASSERT_NE(find_key(*grp, "repetitions"), nullptr); + EXPECT_EQ(find_key(*grp, "repetitions")->values.at(0), "36"); + // [workload, terminal, router] LPs: 2 terminals + 1 router per repetition. + ASSERT_NE(find_key(*grp, "nw-lp"), nullptr); + EXPECT_EQ(find_key(*grp, "nw-lp")->values.at(0), "2"); + ASSERT_NE(find_key(*grp, "modelnet_dragonfly"), nullptr); + EXPECT_EQ(find_key(*grp, "modelnet_dragonfly")->values.at(0), "2"); + ASSERT_NE(find_key(*grp, "modelnet_dragonfly_router"), nullptr); + EXPECT_EQ(find_key(*grp, "modelnet_dragonfly_router")->values.at(0), "1"); + + const compiled_section* params = find_section(c, "PARAMS"); + ASSERT_NE(params, nullptr); + const compiled_key* order = find_key(*params, "modelnet_order"); + ASSERT_NE(order, nullptr); + ASSERT_EQ(order->values.size(), 2u) << "terminal + router are distinct model-net methods"; + EXPECT_EQ(order->values.at(0), "dragonfly"); + EXPECT_EQ(order->values.at(1), "dragonfly_router"); + // shape values pass straight through to PARAMS. + ASSERT_NE(find_key(*params, "num_routers"), nullptr); + EXPECT_EQ(find_key(*params, "num_routers")->values.at(0), "4"); +} + +TEST(ConfigCompiler, ParametricRejectsMissingShapeKey) { + // dragonfly-dally needs num_groups etc.; drop a required shape key. + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: dragonfly-dally + shape: + num_routers: 4 + num_cns_per_router: 2 + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, ParametricRejectsNonIntegerShapeValue) { + // num_routers: abc previously parsed as 0; it is now a diagnostic. + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: abc + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, ParametricRejectsDegenerateDerivedLayout) { + // num_groups: 0 -> repetitions 0; the backstop rejects it before codes_mapping. + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: dragonfly-dally + shape: + num_routers: 4 + num_groups: 0 + num_cns_per_router: 2 + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, ParametricRejectsEmptyComponentModel) { + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: "" } +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, ParametricRejectsExtraHostsKey) { + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + hosts: + component: compute_host + bogus: 1 +)"), + config_error); +} + +TEST(ConfigCompiler, ComponentTypeKeyRejected) { + // `type:` is reserved for a future schema version, not silently dropped nor + // (worse) folded into PARAMS as a model param. + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp, type: router } +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, ParametricRejectsNetworkOnHostComponent) { + // network: is a flat-topology concept; the fabric defines the network itself. + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp, network: simplenet } +topology: + format: parametric + fabric: + model: dragonfly + shape: + num_routers: 4 + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, FattreeRejectsOddSwitchRadix) { + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: fattree + shape: + num_levels: 3 + switch_count: 32 + switch_radix: 7 + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, TorusRejectsMalformedDimLength) { + // A helper that swaps in a given dim_length value and compiles a torus. + auto with_dim_length = [](const char* dim) { + return std::string(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: torus + shape: + n_dims: 3 + dim_length: ")") + + dim + R"(" + hosts: + component: compute_host +)"; + }; + EXPECT_THROW(compile(with_dim_length("4,,2")), config_error); // empty segment + EXPECT_THROW(compile(with_dim_length("abc")), config_error); // non-integer + EXPECT_THROW(compile(with_dim_length("4,2,")), config_error); // trailing comma + EXPECT_THROW(compile(with_dim_length("4 2 2")), config_error); // space-separated +} + +TEST(ConfigCompiler, TorusRejectsNdimsDimLengthMismatch) { + // n_dims 3 but only two dim_length entries. + EXPECT_THROW(compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: torus + shape: + n_dims: 3 + dim_length: "4,2" + hosts: + component: compute_host +)"), + config_error); +} + +TEST(ConfigCompiler, TorusHappyPathMatchingNdims) { + // n_dims 3 with three entries: node count = 4*2*2 = 16, no separate router LP. + compiled_config c = compile(R"( +schema_version: 1 +components: + compute_host: { model: nw-lp } +topology: + format: parametric + fabric: + model: torus + shape: + n_dims: 3 + dim_length: "4,2,2" + hosts: + component: compute_host +)"); + const compiled_section* grp = find_subsection(*find_section(c, "LPGROUPS"), "MODELNET_GRP"); + ASSERT_NE(grp, nullptr); + EXPECT_EQ(find_key(*grp, "repetitions")->values.at(0), "16"); + // torus folds routing into the terminal node -> no router LP line. + EXPECT_EQ(find_key(*grp, "modelnet_torus_router"), nullptr); +} + // --- includes / multi-document merge ---------------------------------------- TEST(ConfigCompiler, IncludeMergesComponentsFromBaseDoc) { From c2bde19d54d756e33ae1b808c2198f2fe39bd2c7 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Mon, 6 Jul 2026 13:30:21 -0500 Subject: [PATCH 13/24] fix: read `include:` files with collective MPI I/O, not per-rank POSIX The YAML shim used to read include: fragments itself with std::ifstream -- one POSIX open per rank per include (a metadata storm on parallel filesystems at scale) and a hang risk: a file readable on some nodes but not others makes those ranks tw_error while the rest proceed into the next collective. Includes are now read in configuration_load with the same collective MPI_File_read_all pattern as the main config -- one collective read per file across the job. The shim gains yaml_configfile_list_includes() (resolves include paths from the main document's bytes, no file I/O) plus a matching free helper, and yaml_configfile_load() now consumes the include bytes directly, making it the pure bytes -> ConfigVTable function its header always described. A missing or unreadable include aborts the job with a diagnostic naming the resolved path. --- doc/dev/yaml-config.md | 14 +++-- src/modelconfig/configuration.c | 60 ++++++++++++++++++++-- src/modelconfig/yaml_configfile.cxx | 79 +++++++++++++++++++---------- src/modelconfig/yaml_configfile.h | 41 +++++++++++---- 4 files changed, 150 insertions(+), 44 deletions(-) diff --git a/doc/dev/yaml-config.md b/doc/dev/yaml-config.md index 92c8618e..6ad12871 100644 --- a/doc/dev/yaml-config.md +++ b/doc/dev/yaml-config.md @@ -722,11 +722,15 @@ section across layouts. Merge rules: included one. - Multiple includes apply in list order; the local file is applied last. -Includes are resolved when the config is loaded, *before* compilation, so the -referenced files must be readable by every rank. An included file may not itself -use `include:` (one level deep, for now). The pure compiler core does no file -I/O — the loader reads the fragments and hands them in — so `include:` is a -loader feature a model never sees. +Includes are resolved when the config is loaded, *before* compilation. The +loader reads each included file with the **same collective MPI read as the main +config** — one collective read per file across the job, not one open per rank — +so the referenced files must be reachable at the same path from every node, and +a missing or unreadable include aborts the whole job with a diagnostic naming +the resolved path. An included file may not itself use `include:` (one level +deep, for now). The pure compiler core does no file I/O — the loader reads the +fragments and hands their bytes in — so `include:` is a loader feature a model +never sees. --- diff --git a/src/modelconfig/configuration.c b/src/modelconfig/configuration.c index c267b6a4..5e057a7e 100644 --- a/src/modelconfig/configuration.c +++ b/src/modelconfig/configuration.c @@ -47,6 +47,12 @@ int configuration_load(const char* filepath, MPI_Comm comm, ConfigHandle* handle char* error = NULL; int rc = 0; char* tmp_path = NULL; + /* Top-level `include:` files: resolved paths, per-file byte buffers, and + * their lengths. All read collectively below and freed at finalize. */ + char** inc_paths = NULL; + char** inc_data = NULL; + size_t* inc_lens = NULL; + size_t n_inc = 0; rc = MPI_File_open(comm, (char*)filepath, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh); if (rc != MPI_SUCCESS) @@ -65,10 +71,49 @@ int configuration_load(const char* filepath, MPI_Comm comm, ConfigHandle* handle if (config_is_yaml(filepath)) { /* the YAML front-end compiles the friendly format into the same - * ConfigVTable the text parser yields, reading the bytes already pulled - * in by the collective read above. filepath lets it resolve a top-level - * `include:` list relative to the config's directory. */ - *handle = yaml_configfile_load(txtdata, txtsize, filepath); + * ConfigVTable the text parser yields, consuming the bytes already + * pulled in by the collective read above. A top-level `include:` list + * names extra config fragments; resolve those paths (relative to the + * config's directory) and read EACH with the same collective MPI pattern + * as the main file, so at scale we do one collective read per file + * rather than one POSIX open per rank per include. */ + inc_paths = yaml_configfile_list_includes(txtdata, txtsize, filepath, &n_inc); + if (n_inc > 0) { + inc_data = (char**)calloc(n_inc, sizeof(*inc_data)); + inc_lens = (size_t*)malloc(n_inc * sizeof(*inc_lens)); + assert(inc_data && inc_lens); + for (size_t i = 0; i < n_inc; i++) { + MPI_File inc_fh = MPI_FILE_NULL; + MPI_Offset inc_size = 0; + + /* A collective failure here (missing/unreadable include) is seen + * by every rank -- the whole job holds identical config bytes -- + * so aborting via tw_error is safe and, unlike the main-file + * open's bare rc, names the resolved path so it's debuggable. */ + rc = MPI_File_open(comm, inc_paths[i], MPI_MODE_RDONLY, MPI_INFO_NULL, &inc_fh); + if (rc != MPI_SUCCESS) + tw_error(TW_LOC, "config error: cannot open included file \"%s\"", + inc_paths[i]); + + rc = MPI_File_get_size(inc_fh, &inc_size); + if (rc != MPI_SUCCESS) + tw_error(TW_LOC, "config error: cannot stat included file \"%s\"", + inc_paths[i]); + + inc_data[i] = (char*)malloc(inc_size ? (size_t)inc_size : 1); + assert(inc_data[i]); + + rc = MPI_File_read_all(inc_fh, inc_data[i], inc_size, MPI_BYTE, &status); + if (rc != MPI_SUCCESS) + tw_error(TW_LOC, "config error: cannot read included file \"%s\"", + inc_paths[i]); + + inc_lens[i] = (size_t)inc_size; + MPI_File_close(&inc_fh); + } + } + *handle = + yaml_configfile_load(txtdata, txtsize, (const char* const*)inc_data, inc_lens, n_inc); } else { #ifdef __APPLE__ f = fopen(filepath, "r"); @@ -102,6 +147,13 @@ int configuration_load(const char* filepath, MPI_Comm comm, ConfigHandle* handle fclose(f); free(txtdata); free(tmp_path); + if (inc_data) { + for (size_t i = 0; i < n_inc; i++) + free(inc_data[i]); + free(inc_data); + } + free(inc_lens); + yaml_configfile_free_includes(inc_paths, n_inc); if (error) { fprintf(stderr, "config error: %s\n", error); free(error); diff --git a/src/modelconfig/yaml_configfile.cxx b/src/modelconfig/yaml_configfile.cxx index 5b47c9f4..56efd4e7 100644 --- a/src/modelconfig/yaml_configfile.cxx +++ b/src/modelconfig/yaml_configfile.cxx @@ -13,9 +13,13 @@ * is therefore unchanged from the .conf path, while "abort" stays a boundary * policy rather than being baked into every validation site in the core. * - * Determinism: every rank reads identical bytes and runs the identical - * deterministic compile, so malformed input throws on every rank and this - * tw_error fires everywhere at once -- no new divergence versus the .conf path. + * Determinism: every rank runs the identical deterministic compile over + * identical bytes -- the top-level config and every included file are read + * collectively by the loader (configuration_load), so malformed input throws on + * every rank and this tw_error fires everywhere at once. The shim itself does no + * file I/O: it consumes the bytes it is handed. It only computes the *names* of + * the include files (yaml_configfile_list_includes), leaving the reads to the + * loader's collective path. */ #include "yaml_configfile.h" @@ -25,9 +29,9 @@ #include +#include +#include #include -#include -#include #include #include #include @@ -48,34 +52,57 @@ std::string resolve_path(const std::string& base_dir, const std::string& rel) { return base_dir + "/" + rel; } -/* Read a whole file into a string, throwing config_error if it can't be read. */ -std::string read_file(const std::string& path) { - std::ifstream in(path, std::ios::binary); - if (!in) - throw codes::config::config_error("config error: cannot read included file \"" + path + - "\""); - std::ostringstream ss; - ss << in.rdbuf(); - return ss.str(); +/* malloc a NUL-terminated copy of s, for handoff to a C caller that frees it. */ +char* dup_cstr(const std::string& s) { + char* out = static_cast(std::malloc(s.size() + 1)); + if (!out) + throw std::bad_alloc(); + std::memcpy(out, s.c_str(), s.size() + 1); + return out; } -/* Resolve the main document's top-level `include:` list into the contents of the - * referenced files (in listed order), read relative to the config's directory. */ -std::vector read_includes(std::string_view main_doc, const char* path) { - std::vector docs; - std::string base_dir = dir_of(path); - for (const std::string& rel : codes::config::parse_includes(main_doc)) - docs.push_back(read_file(resolve_path(base_dir, rel))); - return docs; +} // namespace + +char** yaml_configfile_list_includes(const char* data, size_t len, const char* path, + size_t* count) { + try { + std::vector rels = codes::config::parse_includes(std::string_view(data, len)); + *count = rels.size(); + if (rels.empty()) + return nullptr; + std::string base_dir = dir_of(path); + char** paths = static_cast(std::malloc(rels.size() * sizeof(char*))); + if (!paths) + throw std::bad_alloc(); + for (size_t i = 0; i < rels.size(); i++) + paths[i] = dup_cstr(resolve_path(base_dir, rels[i])); + return paths; + } catch (const codes::config::config_error& e) { + tw_error(TW_LOC, "%s", e.what()); + return nullptr; /* unreachable: tw_error aborts */ + } catch (const std::exception& e) { + tw_error(TW_LOC, "config error (internal): %s", e.what()); + return nullptr; /* unreachable: tw_error aborts */ + } } -} // namespace +void yaml_configfile_free_includes(char** paths, size_t count) { + if (!paths) + return; + for (size_t i = 0; i < count; i++) + std::free(paths[i]); + std::free(paths); +} -struct ConfigVTable* yaml_configfile_load(const char* data, size_t len, const char* path) { +struct ConfigVTable* yaml_configfile_load(const char* data, size_t len, const char* const* inc_data, + const size_t* inc_lens, size_t n_inc) { try { std::string_view main_doc(data, len); - std::vector includes = read_includes(main_doc, path); - codes::config::compiled_config cfg = codes::config::compile(main_doc, includes); + std::vector base_docs; + base_docs.reserve(n_inc); + for (size_t i = 0; i < n_inc; i++) + base_docs.emplace_back(inc_data[i], inc_lens[i]); + codes::config::compiled_config cfg = codes::config::compile(main_doc, base_docs); return codes::config::emit(cfg); } catch (const codes::config::config_error& e) { tw_error(TW_LOC, "%s", e.what()); diff --git a/src/modelconfig/yaml_configfile.h b/src/modelconfig/yaml_configfile.h index 2f10bc1a..c4476f48 100644 --- a/src/modelconfig/yaml_configfile.h +++ b/src/modelconfig/yaml_configfile.h @@ -14,21 +14,44 @@ extern "C" { #endif +/* Resolve a config's top-level `include:` list into filesystem paths. + * + * data/len are the raw bytes of the top-level config file; path is its path, + * used to resolve each relative include against the config's directory + * (absolute includes pass through unchanged). Returns a newly-malloc'd array of + * *count newly-malloc'd, NUL-terminated resolved paths, in listed order, and + * writes the element count through count (0, with a NULL return, if the config + * has no `include:`). The caller owns the result and must release it with + * yaml_configfile_free_includes (equivalently: free() each element, then the + * array). This function does NO file I/O -- it only parses `data` to discover + * the names and joins them with the config's directory; the loader performs the + * actual (collective) reads. On malformed YAML it aborts through tw_error, just + * like yaml_configfile_load; because every rank holds the identical config bytes + * from the collective read, that abort fires on every rank at once. */ +char** yaml_configfile_list_includes(const char* data, size_t len, const char* path, size_t* count); + +/* Free an array returned by yaml_configfile_list_includes (frees each element + * and the array; a NULL array with count 0 is a no-op). */ +void yaml_configfile_free_includes(char** paths, size_t count); + /* Compile a YAML/JSON config (the user-friendly topology + component format) * into the same ConfigVTable the legacy .conf text parser produces, so that * codes_mapping and every model read it unchanged through configuration_get_*. * * This is the thin C boundary of the front-end: it runs the pure C++ core * (compile) and the emitter (emit), and it is the only place that translates a - * compile-time config_error into a ROSS tw_error. data/len are the raw file - * bytes (already read, e.g. via the MPI collective read in configuration_load); - * path is the config file's path, used to resolve a top-level `include:` list - * relative to its directory (the boundary reads the referenced files -- the pure - * core does no file I/O -- so included files must be readable by every rank). - * On success returns a newly-allocated ConfigVTable (free with cf_free); on a - * syntax or validation error it aborts through tw_error with a diagnostic, - * matching how the rest of the configuration front-end reports malformed input. */ -struct ConfigVTable* yaml_configfile_load(const char* data, size_t len, const char* path); + * compile-time config_error into a ROSS tw_error. It consumes bytes and does no + * file I/O of its own: data/len are the raw bytes of the top-level config file, + * and inc_data/inc_lens are the raw bytes of the n_inc included files (in the + * order yaml_configfile_list_includes reported them), each already read -- e.g. + * via the MPI collective read in configuration_load -- so included files must be + * readable by every rank. Pass n_inc == 0 (inc_data/inc_lens may be NULL) when + * the config has no `include:`. On success returns a newly-allocated + * ConfigVTable (free with cf_free); on a syntax or validation error it aborts + * through tw_error with a diagnostic, matching how the rest of the configuration + * front-end reports malformed input. */ +struct ConfigVTable* yaml_configfile_load(const char* data, size_t len, const char* const* inc_data, + const size_t* inc_lens, size_t n_inc); #ifdef __cplusplus } /* extern "C" */ From 9f0e6dec844d5a9c30a370d4d7134b7db9b6b933 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Tue, 7 Jul 2026 16:54:18 -0500 Subject: [PATCH 14/24] docs: doxygen-comment the YAML front-end C boundary and IR fields --- src/modelconfig/config_compiler.h | 12 ++-- src/modelconfig/configstore.h | 3 +- src/modelconfig/yaml_configfile.h | 91 ++++++++++++++++++++++--------- 3 files changed, 72 insertions(+), 34 deletions(-) diff --git a/src/modelconfig/config_compiler.h b/src/modelconfig/config_compiler.h index 9b2ab876..f76d6439 100644 --- a/src/modelconfig/config_compiler.h +++ b/src/modelconfig/config_compiler.h @@ -46,8 +46,8 @@ struct config_error : std::runtime_error { /** One key and its value(s): a scalar carries one value, a list-valued key * (e.g. `modelnet_order`) several, emitted as `("a","b",...)`. */ struct compiled_key { - std::string name; - std::vector values; + std::string name; ///< the key name, as it is emitted + std::vector values; ///< one value for a scalar; several for a list-valued key }; /** One configuration section: named keys plus nested subsections. LPGROUPS @@ -55,9 +55,9 @@ struct compiled_key { * preserved throughout, so a compiled config is byte-comparable to the `.conf` * it replaces. */ struct compiled_section { - std::string name; - std::vector keys; - std::vector subsections; + std::string name; ///< section name (e.g. LPGROUPS, PARAMS) + std::vector keys; ///< this section's keys, in insertion order + std::vector subsections; ///< nested subsections, in insertion order /** Append a scalar key. */ void add_key(std::string key, std::string value); @@ -69,7 +69,7 @@ struct compiled_section { /** The whole compiled config: the top-level (ROOT) sections, in order. */ struct compiled_config { - std::vector sections; + std::vector sections; ///< top-level (ROOT) sections, in order /** Append and return a top-level section. */ compiled_section& add_section(std::string name); diff --git a/src/modelconfig/configstore.h b/src/modelconfig/configstore.h index 262836a6..e4f13f8a 100644 --- a/src/modelconfig/configstore.h +++ b/src/modelconfig/configstore.h @@ -59,7 +59,8 @@ int mcs_getvaluemultiple(const mcs_entry* e, char** buf, unsigned int* maxcount) * value. */ int mcs_getvaluesingle(const mcs_entry* e, char* buf, unsigned int bufsize); -/* Lookup the named subsection in section */ +/* Lookup the named subsection in section; the name match is case-insensitive + * (keys, looked up by mcs_findkey, stay case-sensitive). */ mcs_entry* mcs_findsubsection(const mcs_entry* e, const char* name); /* Lookup the named key in section */ diff --git a/src/modelconfig/yaml_configfile.h b/src/modelconfig/yaml_configfile.h index c4476f48..f6dc29c8 100644 --- a/src/modelconfig/yaml_configfile.h +++ b/src/modelconfig/yaml_configfile.h @@ -7,6 +7,17 @@ #ifndef SRC_MODELCONFIG_YAML_CONFIGFILE_H #define SRC_MODELCONFIG_YAML_CONFIGFILE_H +/** + * @file yaml_configfile.h + * + * Thin `extern "C"` boundary of the YAML/JSON configuration front-end: the only + * place that runs the pure C++ core (config_compiler.h) plus the emitter + * (config_emitter.h) and translates a compile-time config_error into ROSS's + * `tw_error`. Everything here consumes bytes and does no file I/O of its own -- + * the loader (configuration_load) performs the collective reads and hands the + * bytes in. + */ + #include #include @@ -14,42 +25,68 @@ extern "C" { #endif -/* Resolve a config's top-level `include:` list into filesystem paths. +/** + * Resolve a config's top-level `include:` list into filesystem paths. * - * data/len are the raw bytes of the top-level config file; path is its path, - * used to resolve each relative include against the config's directory - * (absolute includes pass through unchanged). Returns a newly-malloc'd array of - * *count newly-malloc'd, NUL-terminated resolved paths, in listed order, and - * writes the element count through count (0, with a NULL return, if the config - * has no `include:`). The caller owns the result and must release it with - * yaml_configfile_free_includes (equivalently: free() each element, then the - * array). This function does NO file I/O -- it only parses `data` to discover - * the names and joins them with the config's directory; the loader performs the - * actual (collective) reads. On malformed YAML it aborts through tw_error, just - * like yaml_configfile_load; because every rank holds the identical config bytes - * from the collective read, that abort fires on every rank at once. */ + * Parses @p data only to discover the include names and joins each with the + * config's directory; it does NO file I/O -- the loader performs the actual + * (collective) reads. A relative include is resolved against the directory of + * @p path; an absolute include passes through unchanged. + * + * @param data the raw bytes of the top-level config file. + * @param len length of @p data in bytes. + * @param path path of the top-level config file, used as the base directory + * for resolving relative includes. + * @param count out: the number of resolved paths returned (0, with a NULL + * return, if the config has no `include:`). + * @return a newly-malloc'd array of @p *count newly-malloc'd, NUL-terminated + * resolved paths, in listed order (NULL when there are none). The + * caller owns the result and must release it with + * yaml_configfile_free_includes (equivalently: free() each element, + * then the array). + * @note On malformed YAML it aborts through `tw_error`, just like + * yaml_configfile_load; because every rank holds the identical config + * bytes from the collective read, that abort fires on every rank at once. + */ char** yaml_configfile_list_includes(const char* data, size_t len, const char* path, size_t* count); -/* Free an array returned by yaml_configfile_list_includes (frees each element - * and the array; a NULL array with count 0 is a no-op). */ +/** + * Free an array returned by yaml_configfile_list_includes: frees each element + * and then the array. + * + * @param paths the array to free; a NULL @p paths (as returned for a config + * with no includes) is a no-op. + * @param count the element count that yaml_configfile_list_includes reported + * through its @p count out-parameter. + */ void yaml_configfile_free_includes(char** paths, size_t count); -/* Compile a YAML/JSON config (the user-friendly topology + component format) - * into the same ConfigVTable the legacy .conf text parser produces, so that +/** + * Compile a YAML/JSON config (the user-friendly topology + component format) + * into the same ConfigVTable the legacy `.conf` text parser produces, so that * codes_mapping and every model read it unchanged through configuration_get_*. * * This is the thin C boundary of the front-end: it runs the pure C++ core * (compile) and the emitter (emit), and it is the only place that translates a - * compile-time config_error into a ROSS tw_error. It consumes bytes and does no - * file I/O of its own: data/len are the raw bytes of the top-level config file, - * and inc_data/inc_lens are the raw bytes of the n_inc included files (in the - * order yaml_configfile_list_includes reported them), each already read -- e.g. - * via the MPI collective read in configuration_load -- so included files must be - * readable by every rank. Pass n_inc == 0 (inc_data/inc_lens may be NULL) when - * the config has no `include:`. On success returns a newly-allocated - * ConfigVTable (free with cf_free); on a syntax or validation error it aborts - * through tw_error with a diagnostic, matching how the rest of the configuration - * front-end reports malformed input. */ + * compile-time config_error into a ROSS `tw_error`. It consumes bytes and does + * no file I/O of its own; the included files are passed in already read -- e.g. + * via the MPI collective read in configuration_load -- so they must be readable + * by every rank. + * + * @param data the raw bytes of the top-level config file. + * @param len length of @p data in bytes. + * @param inc_data the raw bytes of each of the @p n_inc included files, in the + * order yaml_configfile_list_includes reported them; may be + * NULL when @p n_inc is 0. + * @param inc_lens length of each buffer in @p inc_data; may be NULL when + * @p n_inc is 0. + * @param n_inc number of included files (0 when the config has no + * `include:`). + * @return a newly-allocated ConfigVTable (free with cf_free). + * @note On a syntax or validation error it aborts through `tw_error` with a + * diagnostic, matching how the rest of the configuration front-end reports + * malformed input. + */ struct ConfigVTable* yaml_configfile_load(const char* data, size_t len, const char* const* inc_data, const size_t* inc_lens, size_t n_inc); From fab707b62af73ba243f50e3507a64fed64202984 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Tue, 7 Jul 2026 17:26:57 -0500 Subject: [PATCH 15/24] tests: diff lp-io in config-equivalence tests where it is reproducible The simplep2p / simplenet / loggp / torus / express-mesh / slimfly and both lsm config-equivalence tests only compared a marker line ("Net Events Processed") -- not because their models cannot be diffed per-LP, but because their test binaries wrote lp-io to a fixed, non-per-run path, so two runs' output could not coexist. Give modelnet-test, modelnet-simplep2p-test and lsm-test the same configurable lp-io options already used by model-net-synthetic (--lp-io-dir / --lp-io-use-suffix; no output when the flag is unset), then upgrade those eight registrations to codes_add_lpio_equivalence_test, which diffs the per-LP lp-io output (send/recv counts, timings, torus link stats, ...) on top of the marker -- a strictly stronger check. lsm-test takes its config via --conf=, so add a CONFIG_FLAG option to codes_add_lpio_equivalence_test mirroring the one already on codes_add_config_equivalence_test. Nothing consumed these binaries' previous fixed-path lp-io output (checked tests/*.sh, tests/expected/ and CI), so emitting no lp-io when the flag is unset is safe; the plain run-tests for the touched binaries still pass. Left marker-based, with corrected comments: - resource-test (buffer_test) emits no lp-io at all. - fattree: its network switch/msg stats ARE reproducible run to run, but model-net-synthetic-fattree also dumps a PARAMS_LOG sim_log.txt into the lp-io dir with wall-clock-derived ROSS columns (running_time, event_rate, ...) that vary every run, and the binary hardcodes its lp-io path, so a whole-dir lp-io diff would spuriously fail. The old comment blaming the switch stats was inaccurate and is fixed. modelnet-p2p-bw likewise hardcodes lp_io_prepare but only backs a single- run smoke test, so no existing comparison would benefit; left unchanged. --- tests/CMakeLists.txt | 68 +++++++++++++++++++++----------- tests/local-storage-model-test.c | 30 ++++++++++---- tests/modelnet-simplep2p-test.c | 21 ++++++++-- tests/modelnet-test.c | 21 ++++++++-- 4 files changed, 101 insertions(+), 39 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bd0443e2..c00bc851 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -196,9 +196,11 @@ endfunction() # CONFIG_B # e.g. new .yaml (or a second .conf) # [NP ] # default 1 # [ARGS ] -# ) +# [CONFIG_FLAG ]) # e.g. "--conf=" for binaries that take +# # the config as a ROSS option instead +# # of the trailing `-- ` positional function(codes_add_lpio_equivalence_test) - cmake_parse_arguments(T "" "NAME;BINARY;NP;CONFIG_A;CONFIG_B" "ARGS" ${ARGN}) + cmake_parse_arguments(T "" "NAME;BINARY;NP;CONFIG_A;CONFIG_B;CONFIG_FLAG" "ARGS" ${ARGN}) if(NOT T_NAME OR NOT T_BINARY OR NOT T_CONFIG_A OR NOT T_CONFIG_B) message(FATAL_ERROR "codes_add_lpio_equivalence_test: NAME, BINARY, CONFIG_A and CONFIG_B are required") endif() @@ -210,19 +212,31 @@ function(codes_add_lpio_equivalence_test) # Each run writes lp-io into a fresh "lp-io" dir inside its own run-N subdir # (lp_io_prepare mkdir's it and aborts if it already exists, so it must be # relative + per-run-isolated, which the runner's subdirs provide). + if(T_CONFIG_FLAG) + # config is a ROSS option (e.g. --conf=): glue it to the prefix and + # pass it among the ROSS args, with no trailing `-- ` positional. + set(_run_a ${_launch} "${_bin}" ${T_ARGS} --lp-io-dir=lp-io "${T_CONFIG_FLAG}${T_CONFIG_A}" ${MPIEXEC_POSTFLAGS}) + set(_run_b ${_launch} "${_bin}" ${T_ARGS} --lp-io-dir=lp-io "${T_CONFIG_FLAG}${T_CONFIG_B}" ${MPIEXEC_POSTFLAGS}) + else() + set(_run_a ${_launch} "${_bin}" ${T_ARGS} --lp-io-dir=lp-io ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_A}") + set(_run_b ${_launch} "${_bin}" ${T_ARGS} --lp-io-dir=lp-io ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_B}") + endif() add_test(NAME ${T_NAME} COMMAND "${CMAKE_CURRENT_BINARY_DIR}/run-test.sh" bash "${CMAKE_CURRENT_SOURCE_DIR}/equivalence-run.sh" --marker "Net Events Processed" --lp-io lp-io - @@ ${_launch} "${_bin}" ${T_ARGS} --lp-io-dir=lp-io ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_A}" - @@ ${_launch} "${_bin}" ${T_ARGS} --lp-io-dir=lp-io ${MPIEXEC_POSTFLAGS} -- "${T_CONFIG_B}" + @@ ${_run_a} + @@ ${_run_b} WORKING_DIRECTORY "${CODES_BINARY_DIR}") endfunction() # codes_add_config_equivalence_test — assert two configs drive a model to the -# same committed work, comparing a marker line rather than lp-io. For models -# whose test binary writes lp-io to a fixed (non-per-run) path, so two lp-io dirs -# can't coexist: the old .conf vs new .yaml check for those. +# same committed work, comparing a marker line rather than lp-io. The fallback +# for the cases where a per-LP lp-io diff can't be used: the binary emits no +# lp-io at all (e.g. resource-test / buffer_test), or its lp-io is not +# reproducible run to run (e.g. fattree's switch stats). The old .conf vs new +# .yaml check for those. Everything whose lp-io *is* per-run reproducible should +# use codes_add_lpio_equivalence_test instead -- a strictly stronger check. # # codes_add_config_equivalence_test( # NAME BINARY [NP ] @@ -461,22 +475,22 @@ codes_add_lpio_equivalence_test( CONFIG_A "${_nwconf}/modelnet-synthetic-dragonfly.conf" CONFIG_B "${_nwconf}/modelnet-synthetic-dragonfly.yaml") -# Flat networks (component + node count). These test binaries write lp-io to a -# fixed path, so compare committed-event counts between the .conf and its YAML -# twin rather than lp-io dirs. -codes_add_config_equivalence_test( +# Flat networks (component + node count). The modelnet test binaries now honor +# --lp-io-dir, so each run writes lp-io to its own per-run dir: diff the per-LP +# lp-io output of the .conf against its YAML twin directly. +codes_add_lpio_equivalence_test( NAME simplep2p-config-equivalence BINARY modelnet-simplep2p-test ARGS --sync=1 CONFIG_A "${_tconf}/modelnet-test-simplep2p.conf" CONFIG_B "${_tconf}/modelnet-test-simplep2p.yaml") -codes_add_config_equivalence_test( +codes_add_lpio_equivalence_test( NAME simplenet-config-equivalence BINARY modelnet-test ARGS --sync=1 CONFIG_A "${_tconf}/modelnet-test.conf" CONFIG_B "${_tconf}/modelnet-test-simplenet.yaml") -codes_add_config_equivalence_test( +codes_add_lpio_equivalence_test( NAME loggp-config-equivalence BINARY modelnet-test ARGS --sync=1 @@ -484,21 +498,21 @@ codes_add_config_equivalence_test( CONFIG_B "${_tconf}/modelnet-test-loggp.yaml") # Internally-generated fabrics (torus / express-mesh / slimfly), each run through -# the modelnet-test binary (fixed-path lp-io), so compare committed-event counts -# between the .conf and its parametric-fabric YAML twin. -codes_add_config_equivalence_test( +# the modelnet-test binary (per-run lp-io via --lp-io-dir), so diff the per-LP +# lp-io output between the .conf and its parametric-fabric YAML twin. +codes_add_lpio_equivalence_test( NAME torus-config-equivalence BINARY modelnet-test ARGS --sync=1 CONFIG_A "${_tconf}/modelnet-test-torus.conf" CONFIG_B "${_tconf}/modelnet-test-torus.yaml") -codes_add_config_equivalence_test( +codes_add_lpio_equivalence_test( NAME express-mesh-config-equivalence BINARY modelnet-test ARGS --sync=1 CONFIG_A "${_tconf}/modelnet-test-em.conf" CONFIG_B "${_tconf}/modelnet-test-em.yaml") -codes_add_config_equivalence_test( +codes_add_lpio_equivalence_test( NAME slimfly-config-equivalence BINARY modelnet-test ARGS --sync=1 @@ -507,8 +521,9 @@ codes_add_config_equivalence_test( # Explicit-groups form (non-network layouts): a custom group with a workload LP # co-located with a storage/resource LP, plus that model's own pass-through -# section. These binaries take the config as a ROSS option, not a positional. -codes_add_config_equivalence_test( +# section. lsm-test now honors --lp-io-dir (config still via the --conf= ROSS +# option, not a positional), so diff its per-LP lp-io output directly. +codes_add_lpio_equivalence_test( NAME lsm-config-equivalence BINARY lsm-test CONFIG_FLAG "--conf=" @@ -517,13 +532,15 @@ codes_add_config_equivalence_test( CONFIG_B "${_tconf}/lsm-test.yaml") # Same run, but the lsm section is factored into a shared fragment pulled in with # `include:` -- exercises the loader-boundary include resolution end to end. -codes_add_config_equivalence_test( +codes_add_lpio_equivalence_test( NAME lsm-include-equivalence BINARY lsm-test CONFIG_FLAG "--conf=" ARGS --sync=1 CONFIG_A "${_tconf}/lsm-test.conf" CONFIG_B "${_tconf}/lsm-test-include.yaml") +# resource-test (buffer_test) writes no lp-io at all, so there is nothing to diff: +# fall back to comparing the marker line. Config is passed as a ROSS option. codes_add_config_equivalence_test( NAME resource-config-equivalence BINARY resource-test @@ -532,8 +549,13 @@ codes_add_config_equivalence_test( CONFIG_A "${_tconf}/buffer_test.conf" CONFIG_B "${_tconf}/buffer_test.yaml") -# fattree's lp-io switch stats are not reproducible run to run, so compare -# committed-event counts rather than lp-io. +# The fat-tree *network* lp-io (switch / msg stats, model-net categories) is in +# fact reproducible run to run -- but the model-net-synthetic-fattree binary also +# dumps a PARAMS_LOG "sim_log.txt" into the same lp-io dir carrying wall-clock- +# derived ROSS runtime columns (running_time, event_rate, ...) that differ every +# run. A per-LP lp-io diff compares every file in the dir, so it would spuriously +# fail on sim_log.txt; the binary also hardcodes its lp-io path (no --lp-io-dir). +# So this one falls back to comparing the marker line. codes_add_config_equivalence_test( NAME fattree-config-equivalence BINARY model-net-synthetic-fattree diff --git a/tests/local-storage-model-test.c b/tests/local-storage-model-test.c index a7afef29..f5ad3dec 100644 --- a/tests/local-storage-model-test.c +++ b/tests/local-storage-model-test.c @@ -50,9 +50,17 @@ static struct codes_cb_info cb_info; char conf_file_name[256] = {0}; -const tw_optdef app_opt[] = {TWOPT_GROUP("Simple Network Test Model"), - TWOPT_CHAR("conf", conf_file_name, "Name of configuration file"), - TWOPT_END()}; +static char lp_io_dir[256] = {'\0'}; +static unsigned int lp_io_use_suffix = 0; +static int do_lp_io = 0; + +const tw_optdef app_opt[] = { + TWOPT_GROUP("Simple Network Test Model"), + TWOPT_CHAR("conf", conf_file_name, "Name of configuration file"), + TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output)"), + TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, + "Whether to append uniq suffix to lp-io directory (default 0)"), + TWOPT_END()}; static void svr_init(svr_state* ns, tw_lp* lp); @@ -102,17 +110,23 @@ int main(int argc, char** argv) { lsm_configure(); - ret = lp_io_prepare("lsm-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_WORLD); - if (ret < 0) { - return (-1); + if (lp_io_dir[0]) { + do_lp_io = 1; + int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0; + ret = lp_io_prepare(lp_io_dir, flags, &handle, MPI_COMM_WORLD); + if (ret < 0) { + return (-1); + } } INIT_CODES_CB_INFO(&cb_info, svr_msg, h, tag, ret); tw_run(); - ret = lp_io_flush(handle, MPI_COMM_WORLD); - assert(ret == 0); + if (do_lp_io) { + ret = lp_io_flush(handle, MPI_COMM_WORLD); + assert(ret == 0); + } tw_end(); diff --git a/tests/modelnet-simplep2p-test.c b/tests/modelnet-simplep2p-test.c index 740fff48..e6772f63 100644 --- a/tests/modelnet-simplep2p-test.c +++ b/tests/modelnet-simplep2p-test.c @@ -30,6 +30,10 @@ static int net_id = 0; static int num_servers = 0; +static char lp_io_dir[256] = {'\0'}; +static unsigned int lp_io_use_suffix = 0; +static int do_lp_io = 0; + typedef struct svr_msg svr_msg; typedef struct svr_state svr_state; @@ -81,7 +85,12 @@ static void handle_kickoff_rev_event(svr_state* ns, svr_msg* m, tw_lp* lp); static void handle_ack_rev_event(svr_state* ns, svr_msg* m, tw_lp* lp); static void handle_req_rev_event(svr_state* ns, svr_msg* m, tw_lp* lp); -const tw_optdef app_opt[] = {TWOPT_GROUP("Model net test case"), TWOPT_END()}; +const tw_optdef app_opt[] = { + TWOPT_GROUP("Model net test case"), + TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output)"), + TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, + "Whether to append uniq suffix to lp-io directory (default 0)"), + TWOPT_END()}; int main(int argc, char** argv) { int nprocs; @@ -116,14 +125,18 @@ int main(int argc, char** argv) { num_servers = codes_mapping_get_lp_count("MODELNET_GRP", 0, "nw-lp", NULL, 1); assert(num_servers == 3); - if (lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_WORLD) < 0) { - return (-1); + if (lp_io_dir[0]) { + do_lp_io = 1; + int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0; + if (lp_io_prepare(lp_io_dir, flags, &handle, MPI_COMM_WORLD) < 0) { + return (-1); + } } tw_run(); model_net_report_stats(net_id); - if (lp_io_flush(handle, MPI_COMM_WORLD) < 0) { + if (do_lp_io && lp_io_flush(handle, MPI_COMM_WORLD) < 0) { return (-1); } diff --git a/tests/modelnet-test.c b/tests/modelnet-test.c index 96875e58..7cb43ab5 100644 --- a/tests/modelnet-test.c +++ b/tests/modelnet-test.c @@ -32,6 +32,10 @@ static int num_routers = 0; static int num_servers = 0; static int offset = 2; +static char lp_io_dir[256] = {'\0'}; +static unsigned int lp_io_use_suffix = 0; +static int do_lp_io = 0; + /* whether to pull instead of push */ static int do_pull = 0; @@ -94,7 +98,12 @@ static void handle_kickoff_rev_event(svr_state* ns, svr_msg* m, tw_lp* lp); static void handle_ack_rev_event(svr_state* ns, svr_msg* m, tw_lp* lp); static void handle_req_rev_event(svr_state* ns, svr_msg* m, tw_lp* lp); -const tw_optdef app_opt[] = {TWOPT_GROUP("Model net test case"), TWOPT_END()}; +const tw_optdef app_opt[] = { + TWOPT_GROUP("Model net test case"), + TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output)"), + TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, + "Whether to append uniq suffix to lp-io directory (default 0)"), + TWOPT_END()}; int main(int argc, char** argv) { int nprocs; @@ -148,14 +157,18 @@ int main(int argc, char** argv) { offset = 1; } - if (lp_io_prepare("modelnet-test", LP_IO_UNIQ_SUFFIX, &handle, MPI_COMM_WORLD) < 0) { - return (-1); + if (lp_io_dir[0]) { + do_lp_io = 1; + int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0; + if (lp_io_prepare(lp_io_dir, flags, &handle, MPI_COMM_WORLD) < 0) { + return (-1); + } } tw_run(); model_net_report_stats(net_id); - if (lp_io_flush(handle, MPI_COMM_WORLD) < 0) { + if (do_lp_io && lp_io_flush(handle, MPI_COMM_WORLD) < 0) { return (-1); } From 26a5098624694935f9f63b7f3fa10ecbd2f9dea7 Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Tue, 7 Jul 2026 17:47:01 -0500 Subject: [PATCH 16/24] yaml config: tutorial ping-pong YAML twins + conf/yaml equivalence tests Add yaml configs to ping-pong tutorial and tests proving the YAML front-end drives the model identically to the legacy .conf. To cover the surrogate, teach codes_add_config_equivalence_test the SETUP and REQUIRE options codes_add_equivalence_test already has: with SETUP, CONFIG_A/B are the bare names a per-run setup script generates, so one script generating both the .conf and its .yaml twin lets the two formats be compared on the run- time-assembled config; REQUIRE asserts the surrogate actually switched. The new surrogate-config-equivalence-setup.sh generates both from their templates with the same env, so they differ only in format. --- doc/example/CMakeLists.txt | 7 ++ .../tutorial-ping-pong-surrogate.yaml.in | 99 +++++++++++++++++++ doc/example/tutorial-ping-pong.yaml.in | 60 +++++++++++ tests/CMakeLists.txt | 55 ++++++++++- tests/surrogate-config-equivalence-setup.sh | 18 ++++ 5 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 doc/example/tutorial-ping-pong-surrogate.yaml.in create mode 100644 doc/example/tutorial-ping-pong.yaml.in create mode 100644 tests/surrogate-config-equivalence-setup.sh diff --git a/doc/example/CMakeLists.txt b/doc/example/CMakeLists.txt index 1b0c5605..ffea52b3 100644 --- a/doc/example/CMakeLists.txt +++ b/doc/example/CMakeLists.txt @@ -11,6 +11,10 @@ endforeach() # Saving default config files to run experiments with configure_file(tutorial-ping-pong.conf.in tutorial-ping-pong.template.conf.in @ONLY) configure_file(tutorial-ping-pong-surrogate.conf.in tutorial-ping-pong-surrogate.template.conf.in @ONLY) +# YAML twins of the templates above (@ONLY so @CMAKE_SOURCE_DIR@ is substituted but +# the ${VAR} tokens stay for envsubst, exactly like the .conf templates). +configure_file(tutorial-ping-pong.yaml.in tutorial-ping-pong.template.yaml.in @ONLY) +configure_file(tutorial-ping-pong-surrogate.yaml.in tutorial-ping-pong-surrogate.template.yaml.in @ONLY) configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.template.conf.in @ONLY) configure_file(kb.dfdally-72-event-time-director.conf.in kb.dfdally-72-event-time-director.template.conf.in @ONLY) configure_file(kb.dfdally-72-milc-small.workload.conf.in kb.dfdally-72-milc-small.workload.template.conf.in @ONLY) @@ -39,6 +43,9 @@ set(SURROGATE_ENABLED "1") set(TRAINING_ENABLED "1") configure_file(tutorial-ping-pong.conf.in tutorial-ping-pong.conf) configure_file(tutorial-ping-pong-surrogate.conf.in tutorial-ping-pong-surrogate.conf) +# YAML twins, configured for direct build-dir use (all ${VAR} tokens substituted). +configure_file(tutorial-ping-pong.yaml.in tutorial-ping-pong.yaml) +configure_file(tutorial-ping-pong-surrogate.yaml.in tutorial-ping-pong-surrogate.yaml) configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.conf) configure_file(kb.dfdally-72-event-time-director.conf.in kb.dfdally-72-event-time-director.conf) configure_file(kb.dfdally-72-milc-small.workload.conf.in kb.dfdally-72-milc-small.workload.conf) diff --git a/doc/example/tutorial-ping-pong-surrogate.yaml.in b/doc/example/tutorial-ping-pong-surrogate.yaml.in new file mode 100644 index 00000000..1f621040 --- /dev/null +++ b/doc/example/tutorial-ping-pong-surrogate.yaml.in @@ -0,0 +1,99 @@ +# Run this example with: +# > cd path-to-codes/build +# > mpirun -np 3 doc/example/tutorial-synthetic-ping-pong --synch=3 --num_messages=10000 --lp-io-dir=codes-output -- doc/example/tutorial-ping-pong-surrogate.yaml +schema_version: 1 + +# YAML twin of tutorial-ping-pong-surrogate.conf: the same 72-terminal dragonfly- +# dally fabric as tutorial-ping-pong.yaml, plus the NETWORK_SURROGATE section that +# drives the surrogate/high-fidelity director. dragonfly-dally is file-enumerated, +# so the shape counts must match the binary connection files (referenced by +# absolute path; @CMAKE_SOURCE_DIR@ is substituted by CMake). The configure-time +# placeholders are filled in by CMake for the build-dir .yaml, or by envsubst for +# the .template.yaml.in -- exactly as in the .conf. + +components: + compute_host: +# name of this lp changes according to the model + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly-dally + shape: +# number of routers in group + num_routers: 4 +# number of groups in the network + num_groups: 9 +# number of compute nodes connected to router, dictated by dragonfly config file + num_cns_per_router: 2 +# number of global channels per router + num_global_channels: 2 + links: +# bandwidth in GiB/s and virtual-channel buffer size in bytes, per link class: +# local + global channels, and the compute node-router channel + local: { bandwidth: 2.0, vc_size: 16384 } + global: { bandwidth: 2.0, vc_size: 16384 } + cn: { bandwidth: 2.0, vc_size: 32768 } + routing: +# routing protocol to be used + algorithm: prog-adaptive + connections: +# network config files for intra-group and inter-group connections + intra: "@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-dally/dfdally-72-intra" + inter: "@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-dally/dfdally-72-inter" +# packet size in the network + packet_size: ${PACKET_SIZE} +# chunk size in the network (when chunk size = packet size, packets will not be +# divided into chunks) + chunk_size: ${CHUNK_SIZE} +# scheduler options (alternative: round-robin) + modelnet_scheduler: fcfs +# ROSS message size + message_size: 440 +# folder path to store packet latency from terminal to terminal, if no value is given it won't save anything + save_packet_latency_path: "${PACKET_LATENCY_TRACE_PATH}" +# folder path to store router-local timing rows for router queueing-delay training + save_router_timing_trace_path: "${ROUTER_TIMING_TRACE_PATH}" + save_router_timing_trace_stride: "${ROUTER_TIMING_TRACE_STRIDE}" +# router buffer occupancy snapshots + router_buffer_snapshots: [ ${BUFFER_SNAPSHOTS} ] + hosts: + component: compute_host + +sections: +# read verbatim by the surrogate/director stack (section names are case-insensitive, +# so network_surrogate here is the NETWORK_SURROGATE the model reads). + network_surrogate: + enable: 1 # Options: 0 or 1 + +# determines the director switching from surrogate to high-def simulation strategy + director_mode: at-fixed-virtual-times + +# director configuration for: director_mode == "at-fixed-virtual-times" +# timestamps at which to switch to surrogate-mode and back + #fixed_switch_timestamps: [ "100e4", "8900e4" ] # first switch at ~100 ping messages, second at ~9900 pings + #fixed_switch_timestamps: [ "1000e4", "8900e4" ] # first switch at ~1000 ping messages, second at ~9900 pings + fixed_switch_timestamps: [ ${SWITCH_TIMESTAMPS} ] + +# latency predictor to use. Options: average, torch-jit + packet_latency_predictor: "${PREDICTOR_TYPE}" +# some workload models need some time to stabilize, a point where the network behaviour stabilizes. The predictor will ignore all packet latencies that arrive during this period + ignore_until: "${IGNORE_UNTIL}" + +# parameters for torch-jit latency predictor +# accepted modes: +# single-static-model-for-all-terminals: uses torch_jit_model_path, input [1,4], output [1,2] +# lp-aware-single-static-model: uses torch_jit_model_path, input [1,12], output [1,2] +# lp-aware-lp-type-models: uses the terminal/router/default paths below + torch_jit_mode: "${TORCH_JIT_MODE}" + torch_jit_model_path: "${TORCH_JIT_MODEL_PATH}" + torch_jit_terminal_model_path: "${TORCH_JIT_TERMINAL_MODEL_PATH}" + torch_jit_router_timing_model_path: "${TORCH_JIT_ROUTER_TIMING_MODEL_PATH}" + torch_jit_default_model_path: "${TORCH_JIT_DEFAULT_MODEL_PATH}" + +# temporary surrogate debug prints. Options: 0 or 1 + debug_prints: "${SURROGATE_DEBUG_PRINTS}" + +# selecting network treatment on switching to surrogate. Options: freeze, nothing + network_treatment_on_switch: "${NETWORK_TREATMENT}" diff --git a/doc/example/tutorial-ping-pong.yaml.in b/doc/example/tutorial-ping-pong.yaml.in new file mode 100644 index 00000000..ad0dddf9 --- /dev/null +++ b/doc/example/tutorial-ping-pong.yaml.in @@ -0,0 +1,60 @@ +schema_version: 1 + +# YAML twin of tutorial-ping-pong.conf: the 72-terminal dragonfly-dally fabric the +# ping-pong tutorial runs on. dragonfly-dally is file-enumerated, so the shape +# counts are genuine inputs that must match the binary connection files (referenced +# by absolute path; @CMAKE_SOURCE_DIR@ is substituted by CMake so the model resolves +# them from any run directory). The configure-time placeholders are filled in by +# CMake for the build-dir .yaml, or by envsubst for the .template.yaml.in -- exactly +# as in the .conf. The compiler derives repetitions = num_groups * num_routers and +# lays out the [workload, terminal, router] LPs, so nw-lp / modelnet_dragonfly_dally* +# are emitted for you. + +components: + compute_host: +# name of this lp changes according to the model + model: nw-lp + +topology: + format: parametric + fabric: + model: dragonfly-dally + shape: +# number of routers in group + num_routers: 4 +# number of groups in the network + num_groups: 9 +# number of compute nodes connected to router, dictated by dragonfly config file + num_cns_per_router: 2 +# number of global channels per router + num_global_channels: 2 + links: +# bandwidth in GiB/s and virtual-channel buffer size in bytes, per link class: +# local + global channels, and the compute node-router channel + local: { bandwidth: 2.0, vc_size: 16384 } + global: { bandwidth: 2.0, vc_size: 16384 } + cn: { bandwidth: 2.0, vc_size: 32768 } + routing: +# routing protocol to be used + algorithm: prog-adaptive + connections: +# network config files for intra-group and inter-group connections + intra: "@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-dally/dfdally-72-intra" + inter: "@CMAKE_SOURCE_DIR@/src/network-workloads/conf/dragonfly-dally/dfdally-72-inter" +# packet size in the network + packet_size: ${PACKET_SIZE} +# chunk size in the network (when chunk size = packet size, packets will not be +# divided into chunks) + chunk_size: ${CHUNK_SIZE} +# scheduler options (alternative: round-robin) + modelnet_scheduler: fcfs +# ROSS message size + message_size: 440 +# router buffer occupancy snapshots + router_buffer_snapshots: [ ${BUFFER_SNAPSHOTS} ] +# folder path to store packet latency from terminal to terminal, if no value is given it won't save anything + save_packet_latency_path: "${PACKET_LATENCY_TRACE_PATH}" + save_router_timing_trace_path: "${ROUTER_TIMING_TRACE_PATH}" + save_router_timing_trace_stride: "${ROUTER_TIMING_TRACE_STRIDE}" + hosts: + component: compute_host diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c00bc851..f8f228bf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -243,11 +243,20 @@ endfunction() # CONFIG_A # e.g. legacy .conf # CONFIG_B # e.g. new .yaml # [MARKER ] [ARGS ...] -# [CONFIG_FLAG ]) # e.g. "--conf=" for binaries that take +# [CONFIG_FLAG ] # e.g. "--conf=" for binaries that take # # the config as a ROSS option instead # # of the trailing `-- ` positional +# [SETUP