From 9cd0c042187b1a91db6648eb332fec004cf404c4 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Tue, 7 Jul 2026 16:00:40 -0400 Subject: [PATCH 01/20] Add other files required for simplep2p flow model --- doc/example/flow-control-simplep2p-bw.conf | 2 + .../flow-control-simplep2p-latency.conf | 2 + doc/example/flow-control-simplep2p.conf.in | 60 ++ .../model-net-flow-control-simplep2p.cxx | 662 ++++++++++++++++++ src/surrogate/zmqml/model/mlflowcontrol.py | 512 ++++++++++++++ 5 files changed, 1238 insertions(+) create mode 100644 doc/example/flow-control-simplep2p-bw.conf create mode 100644 doc/example/flow-control-simplep2p-latency.conf create mode 100644 doc/example/flow-control-simplep2p.conf.in create mode 100644 src/network-workloads/model-net-flow-control-simplep2p.cxx create mode 100644 src/surrogate/zmqml/model/mlflowcontrol.py diff --git a/doc/example/flow-control-simplep2p-bw.conf b/doc/example/flow-control-simplep2p-bw.conf new file mode 100644 index 00000000..1533b9e0 --- /dev/null +++ b/doc/example/flow-control-simplep2p-bw.conf @@ -0,0 +1,2 @@ +0.0,0.0 200.0,200.0 +100.0,100.0 0.0,0.0 diff --git a/doc/example/flow-control-simplep2p-latency.conf b/doc/example/flow-control-simplep2p-latency.conf new file mode 100644 index 00000000..77ae2f34 --- /dev/null +++ b/doc/example/flow-control-simplep2p-latency.conf @@ -0,0 +1,2 @@ +0.0,0.0 1000.0,1000.0 +1000.0,1000.0 0.0,0.0 diff --git a/doc/example/flow-control-simplep2p.conf.in b/doc/example/flow-control-simplep2p.conf.in new file mode 100644 index 00000000..d30e5cf4 --- /dev/null +++ b/doc/example/flow-control-simplep2p.conf.in @@ -0,0 +1,60 @@ +# ZeroMQ flow-control surrogate example for a two-terminal simplep2p workload. +# +# ML predicts raw egress packets by directed flow. CODES applies source queue +# capacity, per-flow grant capacity, drops, feedback, and then injects granted +# traffic into modelnet_simplep2p. + +LPGROUPS +{ + MODELNET_GRP + { + repetitions="2"; + nw-lp="1"; + modelnet_simplep2p="1"; + } +} + +PARAMS +{ + message_size="464"; + packet_size="${PACKET_SIZE}"; + modelnet_order=("simplep2p"); + modelnet_scheduler="fcfs"; + + # simplep2p reads these with configuration_get_value_relpath(), so keep + # them relative to this generated config file in build/doc/example/. + net_latency_ns_file="flow-control-simplep2p-latency.conf"; + net_bw_mbps_file="flow-control-simplep2p-bw.conf"; +} + +FLOW_CONTROL +{ + # CMake fills only the run mode and log path for generated concrete configs. + inferencing_enabled="${FLOW_CONTROL_INFERENCING_ENABLED}"; + training_enabled="${FLOW_CONTROL_TRAINING_ENABLED}"; + flow_log_path="${FLOW_CONTROL_LOG_PATH}"; + + debug_prints="1"; + + interval_seconds="10"; + num_intervals="50"; + packet_size_bytes="${PACKET_SIZE}"; + + # These source-side grant capacities should match the simplep2p bandwidth + # matrix for this first prototype. + bandwidth_0_to_1_mbps="200"; + bandwidth_1_to_0_mbps="100"; + + queue_capacity_0_to_1_packets="500"; + queue_capacity_1_to_0_packets="500"; + + # Pure-PDES synthetic demand generator defaults. + base_0_to_1_packets="50"; + base_1_to_0_packets="25"; + burst_0_to_1_packets="30"; + burst_1_to_0_packets="15"; + burst_period="5"; + ingress_gain="0.10"; + + mark_threshold_ratio="0.8"; +} diff --git a/src/network-workloads/model-net-flow-control-simplep2p.cxx b/src/network-workloads/model-net-flow-control-simplep2p.cxx new file mode 100644 index 00000000..765da69c --- /dev/null +++ b/src/network-workloads/model-net-flow-control-simplep2p.cxx @@ -0,0 +1,662 @@ +/* + * Standalone interval-flow workload over model-net simplep2p. + * + * This is a normal runnable CODES binary, not a ctest harness. It supports + * the workflow: + * pure PDES record collection -> train/save flow-control ZMQML model -> + * hybrid full-surrogate traffic generation with PDES communication. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "codes/codes.h" +#include "codes/codes_mapping.h" +#include "codes/configuration.h" +#include "codes/lp-type-lookup.h" +#include "codes/model-net.h" +#include "zmqmlrequester.h" + +static int net_id = 0; +static int num_servers = 0; + +struct flow_feedback { + double raw_predicted_packets; + double available_packets; + double granted_packets; + double dropped_packets; + double grant_ratio; + double queue_packets_next; + double queue_occupancy_ratio; + double queue_delay_estimate; + double capacity_packets; + int congestion_mark_flag; + int overflow_flag; +}; + +struct flow_config { + int inferencing_enabled; + int training_enabled; + int debug_prints; + double interval_seconds; + int num_intervals; + int packet_size_bytes; + double bandwidth_0_to_1_mbps; + double bandwidth_1_to_0_mbps; + double queue_capacity_0_to_1_packets; + double queue_capacity_1_to_0_packets; + double base_0_to_1_packets; + double base_1_to_0_packets; + double burst_0_to_1_packets; + double burst_1_to_0_packets; + int burst_period; + double ingress_gain; + double mark_threshold_ratio; + char flow_log_path[1024]; +}; + +static flow_config cfg = { + 0, /* inferencing_enabled */ + 1, /* training_enabled */ + 0, /* debug_prints */ + 10.0, /* interval_seconds */ + 20, /* num_intervals */ + 4096, /* packet_size_bytes */ + 200.0, /* bandwidth 0->1 */ + 100.0, /* bandwidth 1->0 */ + 100000., /* queue cap 0->1 */ + 100000., /* queue cap 1->0 */ + 50000., /* pure base 0->1 */ + 25000., /* pure base 1->0 */ + 30000., /* burst 0->1 */ + 15000., /* burst 1->0 */ + 5, /* burst period */ + 0.10, /* ingress gain */ + 0.80, /* mark threshold */ + ""}; + +struct flow_state { + int rel_id; + int peer_rel_id; + tw_lpid peer_gid; + int interval_id; + double ingress_packets_current; + double source_queue_packets; + flow_feedback previous_feedback; + + double total_raw_packets; + double total_granted_packets; + double total_dropped_packets; + double total_delivered_packets; + +}; + +struct flow_msg { + int event_type; + int src_rel_id; + int dst_rel_id; + int interval_id; + double packets; + model_net_event_return ret; +}; + +enum flow_event_type { + FLOW_INTERVAL = 1, + FLOW_DELIVER = 2, +}; + +static void flow_init(flow_state* ns, tw_lp* lp); +static void flow_event(flow_state* ns, tw_bf* b, flow_msg* m, tw_lp* lp); +static void flow_rev_event(flow_state* ns, tw_bf* b, flow_msg* m, tw_lp* lp); +static void flow_finalize(flow_state* ns, tw_lp* lp); + +static tw_lptype flow_lp = { + (init_f)flow_init, + (pre_run_f)NULL, + (event_f)flow_event, + (revent_f)flow_rev_event, + (commit_f)NULL, + (final_f)flow_finalize, + (map_f)codes_mapping, + sizeof(flow_state), +}; + +static const tw_lptype* flow_get_lp_type(void) { return &flow_lp; } + +static void flow_add_lp_type(void) { lp_type_register("nw-lp", flow_get_lp_type()); } + +static double seconds_to_ns(double seconds) { return seconds * 1000.0 * 1000.0 * 1000.0; } + +static double ns_to_seconds(double ns) { return ns / (1000.0 * 1000.0 * 1000.0); } + +static double clamp_packets(double value) { + if (!std::isfinite(value) || value < 0.0) { + return 0.0; + } + return value; +} + +static flow_feedback default_feedback(void) { + flow_feedback fb; + std::memset(&fb, 0, sizeof(fb)); + fb.grant_ratio = 1.0; + return fb; +} + +static void read_int(const char* key, int* value) { + int tmp = 0; + if (configuration_get_value_int(&config, "FLOW_CONTROL", key, NULL, &tmp) == 0) { + *value = tmp; + } +} + +static void read_double(const char* key, double* value) { + double tmp = 0.0; + if (configuration_get_value_double(&config, "FLOW_CONTROL", key, NULL, &tmp) == 0) { + *value = tmp; + } +} + +static void read_string(const char* key, char* value, size_t len) { + char tmp[1024]; + std::memset(tmp, 0, sizeof(tmp)); + if (configuration_get_value(&config, "FLOW_CONTROL", key, NULL, tmp, sizeof(tmp)) > 0) { + std::snprintf(value, len, "%s", tmp); + } +} + +static void load_flow_config(void) { + read_int("inferencing_enabled", &cfg.inferencing_enabled); + read_int("training_enabled", &cfg.training_enabled); + read_int("debug_prints", &cfg.debug_prints); + read_double("interval_seconds", &cfg.interval_seconds); + read_int("num_intervals", &cfg.num_intervals); + read_int("packet_size_bytes", &cfg.packet_size_bytes); + read_double("bandwidth_0_to_1_mbps", &cfg.bandwidth_0_to_1_mbps); + read_double("bandwidth_1_to_0_mbps", &cfg.bandwidth_1_to_0_mbps); + read_double("queue_capacity_0_to_1_packets", &cfg.queue_capacity_0_to_1_packets); + read_double("queue_capacity_1_to_0_packets", &cfg.queue_capacity_1_to_0_packets); + read_double("base_0_to_1_packets", &cfg.base_0_to_1_packets); + read_double("base_1_to_0_packets", &cfg.base_1_to_0_packets); + read_double("burst_0_to_1_packets", &cfg.burst_0_to_1_packets); + read_double("burst_1_to_0_packets", &cfg.burst_1_to_0_packets); + read_int("burst_period", &cfg.burst_period); + read_double("ingress_gain", &cfg.ingress_gain); + read_double("mark_threshold_ratio", &cfg.mark_threshold_ratio); + read_string("flow_log_path", cfg.flow_log_path, sizeof(cfg.flow_log_path)); + + if (cfg.interval_seconds <= 0.0) { + tw_error(TW_LOC, "FLOW_CONTROL.interval_seconds must be positive"); + } + if (cfg.num_intervals <= 0) { + tw_error(TW_LOC, "FLOW_CONTROL.num_intervals must be positive"); + } + if (cfg.packet_size_bytes <= 0) { + tw_error(TW_LOC, "FLOW_CONTROL.packet_size_bytes must be positive"); + } + if (cfg.burst_period <= 0) { + cfg.burst_period = 1; + } +} + + +static void push_zmqml_debug_setting(void) { + if (!cfg.training_enabled && !cfg.inferencing_enabled) { + return; + } + + std::vector args; + args.push_back("1"); + args.push_back(cfg.debug_prints ? "1" : "0"); + + std::vector reply = zmqml_request("set-debug", args); + if (reply.empty() || reply[0] != "done") { + std::fprintf(stderr, + "[flow-control simplep2p] warning: failed to propagate debug_prints=%d to zmqmlserver\n", + cfg.debug_prints); + } +} + +static double bandwidth_for_flow(int src_rel, int dst_rel) { + if (src_rel == 0 && dst_rel == 1) { + return cfg.bandwidth_0_to_1_mbps; + } + if (src_rel == 1 && dst_rel == 0) { + return cfg.bandwidth_1_to_0_mbps; + } + return 0.0; +} + +static double queue_capacity_for_flow(int src_rel, int dst_rel) { + if (src_rel == 0 && dst_rel == 1) { + return cfg.queue_capacity_0_to_1_packets; + } + if (src_rel == 1 && dst_rel == 0) { + return cfg.queue_capacity_1_to_0_packets; + } + return 0.0; +} + +static double interval_capacity_packets(int src_rel, int dst_rel) { + double bw_mbps = bandwidth_for_flow(src_rel, dst_rel); + double bytes = bw_mbps * 1000.0 * 1000.0 / 8.0 * cfg.interval_seconds; + return clamp_packets(std::floor(bytes / (double)cfg.packet_size_bytes)); +} + +static std::string flow_key(int src_rel, int dst_rel) { + std::ostringstream os; + os << src_rel << "->" << dst_rel; + return os.str(); +} + +static std::string json_escape(const std::string& in) { + std::ostringstream os; + for (char c : in) { + switch (c) { + case '\\': os << "\\\\"; break; + case '"': os << "\\\""; break; + case '\n': os << "\\n"; break; + case '\r': os << "\\r"; break; + case '\t': os << "\\t"; break; + default: os << c; break; + } + } + return os.str(); +} + +static std::string build_flow_control_payload(const flow_state* ns) { + const std::string in_key = flow_key(ns->peer_rel_id, ns->rel_id); + const std::string out_key = flow_key(ns->rel_id, ns->peer_rel_id); + const flow_feedback& fb = ns->previous_feedback; + + std::ostringstream os; + os << std::setprecision(17); + os << "{"; + os << "\"lp_id\":" << ns->rel_id << ","; + os << "\"interval_id\":" << ns->interval_id << ","; + os << "\"interval_seconds\":" << cfg.interval_seconds << ","; + os << "\"incoming_flows\":{"; + os << "\"" << json_escape(in_key) << "\":{\"packets\":" + << ns->ingress_packets_current << "}"; + os << "},"; + os << "\"outgoing_feedback\":{"; + os << "\"" << json_escape(out_key) << "\":{"; + os << "\"raw_predicted_packets\":" << fb.raw_predicted_packets << ","; + os << "\"available_packets\":" << fb.available_packets << ","; + os << "\"granted_packets\":" << fb.granted_packets << ","; + os << "\"dropped_packets\":" << fb.dropped_packets << ","; + os << "\"grant_ratio\":" << fb.grant_ratio << ","; + os << "\"queue_packets_next\":" << fb.queue_packets_next << ","; + os << "\"queue_occupancy_ratio\":" << fb.queue_occupancy_ratio << ","; + os << "\"queue_delay_estimate\":" << fb.queue_delay_estimate << ","; + os << "\"capacity_packets\":" << fb.capacity_packets << ","; + os << "\"congestion_mark_flag\":" << fb.congestion_mark_flag << ","; + os << "\"overflow_flag\":" << fb.overflow_flag; + os << "}"; + os << "},"; + os << "\"outgoing_flow_keys\":[\"" << json_escape(out_key) << "\"]"; + os << "}"; + return os.str(); +} + +static double parse_prediction_for_flow(const std::string& predictions, + const std::string& out_key, + double fallback) { + std::istringstream is(predictions); + std::string token; + const std::string prefix = out_key + ":"; + while (is >> token) { + if (token.compare(0, prefix.size(), prefix) == 0) { + char* end = NULL; + const char* start = token.c_str() + prefix.size(); + double value = std::strtod(start, &end); + if (end != start) { + return clamp_packets(value); + } + } + } + return clamp_packets(fallback); +} + +static double pure_pdes_raw_packets(const flow_state* ns) { + const bool fwd = ns->rel_id == 0 && ns->peer_rel_id == 1; + const double base = fwd ? cfg.base_0_to_1_packets : cfg.base_1_to_0_packets; + const double burst = fwd ? cfg.burst_0_to_1_packets : cfg.burst_1_to_0_packets; + const int phase = fwd ? 0 : 2; + const bool burst_now = ((ns->interval_id + phase) % cfg.burst_period) == 0; + const double ramp = 0.03 * base * (double)(ns->interval_id % cfg.burst_period); + const double response = cfg.ingress_gain * ns->ingress_packets_current; + return clamp_packets(base + ramp + (burst_now ? burst : 0.0) + response); +} + +static flow_feedback apply_flow_control(double raw_predicted_packets, double queued_packets, + int src_rel, int dst_rel) { + flow_feedback fb = default_feedback(); + const double capacity = interval_capacity_packets(src_rel, dst_rel); + const double queue_cap = queue_capacity_for_flow(src_rel, dst_rel); + const double available = queued_packets + raw_predicted_packets; + const double granted = std::min(available, capacity); + const double unsent = std::max(0.0, available - granted); + const double queue_next = std::min(unsent, queue_cap); + const double dropped = std::max(0.0, unsent - queue_cap); + const double capacity_per_second = cfg.interval_seconds > 0.0 ? capacity / cfg.interval_seconds : 0.0; + + fb.raw_predicted_packets = raw_predicted_packets; + fb.available_packets = available; + fb.granted_packets = granted; + fb.dropped_packets = dropped; + fb.grant_ratio = available > 0.0 ? granted / available : 1.0; + fb.queue_packets_next = queue_next; + fb.queue_occupancy_ratio = queue_cap > 0.0 ? queue_next / queue_cap : 0.0; + fb.queue_delay_estimate = capacity_per_second > 0.0 ? queue_next / capacity_per_second : 0.0; + fb.capacity_packets = capacity; + fb.congestion_mark_flag = fb.queue_occupancy_ratio >= cfg.mark_threshold_ratio ? 1 : 0; + fb.overflow_flag = dropped > 0.0 ? 1 : 0; + return fb; +} + +static void send_training_record(flow_state* ns, double raw_packets) { + const std::string in_key = flow_key(ns->peer_rel_id, ns->rel_id); + const std::string out_key = flow_key(ns->rel_id, ns->peer_rel_id); + + std::ostringstream record; + record << "schema_version,lp_id,interval_id,interval_seconds,incoming_flow_key," + << "incoming_packets,outgoing_flow_key,raw_egress_packets,prev_grant_ratio," + << "prev_queue_occupancy_ratio,prev_dropped_packets,prev_capacity_packets\n"; + record << std::setprecision(17) + << 1 << ',' << ns->rel_id << ',' << ns->interval_id << ',' + << cfg.interval_seconds << ',' << in_key << ',' + << ns->ingress_packets_current << ',' << out_key << ',' << raw_packets << ',' + << ns->previous_feedback.grant_ratio << ',' + << ns->previous_feedback.queue_occupancy_ratio << ',' + << ns->previous_feedback.dropped_packets << ',' + << ns->previous_feedback.capacity_packets << '\n'; + + std::vector reply = zmqml_director_request( + "flow-control", "simplep2p", "send-records", std::vector(), record.str()); + if (reply.empty() || reply[0] != "done") { + std::fprintf(stderr, + "[flow-control simplep2p] warning: failed to send training record for lp %d interval %d\n", + ns->rel_id, ns->interval_id); + } +} + +static void append_flow_log(const flow_state* ns, const flow_feedback& fb, double raw_packets, + const char* source, tw_lp* lp) { + if (cfg.flow_log_path[0] == '\0') { + return; + } + + const bool new_file = std::ifstream(cfg.flow_log_path).good() == false; + std::ofstream out(cfg.flow_log_path, std::ios::app); + if (!out) { + if (cfg.debug_prints) { + std::fprintf(stderr, "[flow-control simplep2p] could not write %s\n", cfg.flow_log_path); + } + return; + } + if (new_file) { + out << "lp_gid,lp_rel,peer_rel,interval_id,sim_time_seconds,source,incoming_packets," + << "raw_predicted_packets,queue_before_packets,available_packets,capacity_packets," + << "granted_packets,queue_after_packets,dropped_packets,grant_ratio," + << "queue_occupancy_ratio,queue_delay_estimate,congestion_mark_flag,overflow_flag\n"; + } + out << std::setprecision(17) + << (unsigned long long)lp->gid << ',' << ns->rel_id << ',' << ns->peer_rel_id << ',' + << ns->interval_id << ',' << ns_to_seconds(tw_now(lp)) << ',' << source << ',' + << ns->ingress_packets_current << ',' << raw_packets << ',' << ns->source_queue_packets << ',' + << fb.available_packets << ',' << fb.capacity_packets << ',' << fb.granted_packets << ',' + << fb.queue_packets_next << ',' << fb.dropped_packets << ',' << fb.grant_ratio << ',' + << fb.queue_occupancy_ratio << ',' << fb.queue_delay_estimate << ',' + << fb.congestion_mark_flag << ',' << fb.overflow_flag << '\n'; +} + +static double infer_raw_packets(flow_state* ns) { + const std::string out_key = flow_key(ns->rel_id, ns->peer_rel_id); + const double fallback = pure_pdes_raw_packets(ns); + const std::string payload = build_flow_control_payload(ns); + + std::vector reply = zmqml_director_request( + "flow-control", "simplep2p", "inference", std::vector(), payload); + + if (reply.empty() || reply[0] != "done") { + std::fprintf(stderr, + "[flow-control simplep2p] warning: inference failed for flow %s interval %d; using fallback %f\n", + out_key.c_str(), ns->interval_id, fallback); + return fallback; + } + + std::string predictions; + if (reply.size() >= 4) { + predictions = reply[3]; + } else if (reply.size() >= 3) { + predictions = reply[2]; + } + return parse_prediction_for_flow(predictions, out_key, fallback); +} + +static void send_granted_packets(flow_state* ns, flow_msg* m, tw_lp* lp, double granted_packets) { + if (granted_packets <= 0.0) { + return; + } + + const double bytes_double = granted_packets * (double)cfg.packet_size_bytes; + uint64_t message_size = 0; + if (bytes_double >= (double)std::numeric_limits::max()) { + message_size = std::numeric_limits::max(); + } else { + message_size = (uint64_t)std::ceil(bytes_double); + } + if (message_size == 0) { + return; + } + + flow_msg remote; + std::memset(&remote, 0, sizeof(remote)); + remote.event_type = FLOW_DELIVER; + remote.src_rel_id = ns->rel_id; + remote.dst_rel_id = ns->peer_rel_id; + remote.interval_id = ns->interval_id; + remote.packets = granted_packets; + + m->ret = model_net_event(net_id, "flow-control", ns->peer_gid, message_size, 0.0, + sizeof(remote), &remote, 0, NULL, lp); +} + +static void schedule_next_interval(flow_state* ns, tw_lp* lp) { + if (ns->interval_id >= cfg.num_intervals) { + return; + } + tw_event* e = tw_event_new(lp->gid, seconds_to_ns(cfg.interval_seconds), lp); + flow_msg* m = (flow_msg*)tw_event_data(e); + std::memset(m, 0, sizeof(*m)); + m->event_type = FLOW_INTERVAL; + tw_event_send(e); +} + +static void handle_interval(flow_state* ns, flow_msg* m, tw_lp* lp) { + double raw_packets = 0.0; + const char* source = "pure-pdes"; + + if (cfg.inferencing_enabled) { + raw_packets = infer_raw_packets(ns); + source = "flow-control-ml"; + } else { + raw_packets = pure_pdes_raw_packets(ns); + } + + raw_packets = clamp_packets(raw_packets); + + if (cfg.training_enabled) { + send_training_record(ns, raw_packets); + } + + flow_feedback fb = apply_flow_control(raw_packets, ns->source_queue_packets, + ns->rel_id, ns->peer_rel_id); + append_flow_log(ns, fb, raw_packets, source, lp); + + if (cfg.debug_prints && + (fb.grant_ratio < 0.999999 || fb.dropped_packets > 0.0 || fb.congestion_mark_flag)) { + std::printf("[flow-control throttle] lp=%llu flow=%d->%d interval=%d source=%s " + "raw=%.6f queue_before=%.6f available=%.6f capacity=%.6f " + "granted=%.6f queue_after=%.6f dropped=%.6f grant_ratio=%.6f " + "queue_occ=%.6f mark=%d overflow=%d\n", + (unsigned long long)lp->gid, ns->rel_id, ns->peer_rel_id, + ns->interval_id, source, raw_packets, ns->source_queue_packets, + fb.available_packets, fb.capacity_packets, fb.granted_packets, + fb.queue_packets_next, fb.dropped_packets, fb.grant_ratio, + fb.queue_occupancy_ratio, fb.congestion_mark_flag, fb.overflow_flag); + std::fflush(stdout); + } + + send_granted_packets(ns, m, lp, fb.granted_packets); + + ns->source_queue_packets = fb.queue_packets_next; + ns->previous_feedback = fb; + ns->total_raw_packets += raw_packets; + ns->total_granted_packets += fb.granted_packets; + ns->total_dropped_packets += fb.dropped_packets; + ns->ingress_packets_current = 0.0; + ns->interval_id++; + + schedule_next_interval(ns, lp); +} + +static void handle_deliver(flow_state* ns, const flow_msg* m) { + ns->ingress_packets_current += m->packets; + ns->total_delivered_packets += m->packets; +} + +static void flow_init(flow_state* ns, tw_lp* lp) { + std::memset(ns, 0, sizeof(*ns)); + ns->rel_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); + ns->peer_rel_id = 1 - ns->rel_id; + ns->previous_feedback = default_feedback(); + + codes_mapping_get_lp_id("MODELNET_GRP", "nw-lp", NULL, 1, ns->peer_rel_id, 0, + &ns->peer_gid); + + tw_event* e = tw_event_new(lp->gid, g_tw_lookahead + tw_rand_unif(lp->rng), lp); + flow_msg* m = (flow_msg*)tw_event_data(e); + std::memset(m, 0, sizeof(*m)); + m->event_type = FLOW_INTERVAL; + tw_event_send(e); +} + +static void flow_event(flow_state* ns, tw_bf* b, flow_msg* m, tw_lp* lp) { + (void)b; + switch (m->event_type) { + case FLOW_INTERVAL: + handle_interval(ns, m, lp); + break; + case FLOW_DELIVER: + handle_deliver(ns, m); + break; + default: + tw_error(TW_LOC, "unknown flow-control event type %d", m->event_type); + } +} + +static void flow_rev_event(flow_state* ns, tw_bf* b, flow_msg* m, tw_lp* lp) { + (void)ns; + (void)b; + (void)m; + (void)lp; + tw_error(TW_LOC, "model-net-flow-control-simplep2p is intended for sequential runs first"); +} + +static void flow_finalize(flow_state* ns, tw_lp* lp) { + std::printf("flow-control-simplep2p lp=%llu rel=%d peer=%d intervals=%d raw=%f granted=%f " + "delivered=%f dropped=%f queue=%f\n", + (unsigned long long)lp->gid, ns->rel_id, ns->peer_rel_id, ns->interval_id, + ns->total_raw_packets, ns->total_granted_packets, ns->total_delivered_packets, + ns->total_dropped_packets, ns->source_queue_packets); +} + +const tw_optdef app_opt[] = {TWOPT_GROUP("model-net flow-control simplep2p"), TWOPT_END()}; + +static const char* find_config_arg(int argc, char** argv) { + for (int i = argc - 1; i >= 1; --i) { + if (argv[i] == NULL) { + continue; + } + const char* arg = argv[i]; + const size_t len = std::strlen(arg); + if (len >= 5 && std::strcmp(arg + len - 5, ".conf") == 0) { + return arg; + } + } + return NULL; +} + +int main(int argc, char** argv) { + int rank = 0; + int num_nets = 0; + int* net_ids = NULL; + + g_tw_ts_end = seconds_to_ns(60.0 * 60.0 * 24.0 * 365.0); + + tw_opt_add(app_opt); + tw_init(&argc, &argv); + + const char* config_file = find_config_arg(argc, argv); + if (config_file == NULL) { + std::printf("Usage: mpirun -np %s --sync=1 -- \n", argv[0]); + std::printf(" or: mpirun -np %s --synch=1 -- \n", argv[0]); + MPI_Finalize(); + return 1; + } + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + configuration_load(config_file, MPI_COMM_WORLD, &config); + load_flow_config(); + + if (rank == 0) { + push_zmqml_debug_setting(); + } + MPI_Barrier(MPI_COMM_WORLD); + + flow_add_lp_type(); + model_net_register(); + codes_mapping_setup(); + + net_ids = model_net_configure(&num_nets); + assert(num_nets == 1); + net_id = net_ids[0]; + free(net_ids); + + num_servers = codes_mapping_get_lp_count("MODELNET_GRP", 0, "nw-lp", NULL, 1); + if (num_servers != 2) { + tw_error(TW_LOC, "model-net-flow-control-simplep2p expects exactly 2 nw-lp LPs"); + } + + if (rank == 0 && cfg.flow_log_path[0] != '\0') { + std::remove(cfg.flow_log_path); + } + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == 0 && cfg.debug_prints) { + std::printf("flow-control config: inferencing=%d training=%d intervals=%d interval_seconds=%f\n", + cfg.inferencing_enabled, cfg.training_enabled, cfg.num_intervals, + cfg.interval_seconds); + } + + tw_run(); + model_net_report_stats(net_id); + tw_end(); + return 0; +} diff --git a/src/surrogate/zmqml/model/mlflowcontrol.py b/src/surrogate/zmqml/model/mlflowcontrol.py new file mode 100644 index 00000000..77966d68 --- /dev/null +++ b/src/surrogate/zmqml/model/mlflowcontrol.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 +"""Per-terminal interval-flow surrogate family for the ZeroMQ Director. + +The server exposes one surrogate family name, ``flow-control``, but this module +keeps one trainable model per terminal/LP id. Each terminal model learns only +from records whose ``lp_id`` matches that terminal, and inference for an LP uses +only that LP's model. + +The model predicts raw per-flow offered load. CODES/PDES remains responsible +for source queue capacity, grant capacity, drops, routing, and delivery. +""" + +from __future__ import annotations + +import csv +import io +import json +import math +import os +import pickle +from pathlib import Path +from typing import Any + +import numpy as np + + +FLOW_RECORD_FIELDS = [ + "schema_version", + "lp_id", + "interval_id", + "interval_seconds", + "incoming_flow_key", + "incoming_packets", + "outgoing_flow_key", + "raw_egress_packets", + "prev_grant_ratio", + "prev_queue_occupancy_ratio", + "prev_dropped_packets", + "prev_capacity_packets", +] + + +def _finite_float(value: Any, default: float = 0.0) -> float: + try: + out = float(value) + except Exception: + return default + if not math.isfinite(out): + return default + return out + + +def _lp_key(value: Any) -> str: + """Normalize LP ids so CSV strings, ints, and JSON numbers match.""" + text = str(value).strip() + if text == "": + return "unknown" + try: + num = float(text) + if math.isfinite(num) and num.is_integer(): + return str(int(num)) + except Exception: + pass + return text + + +def _flow_key_reverse(flow_key: str) -> str | None: + parts = str(flow_key).split("->", 1) + if len(parts) != 2: + return None + left = parts[0].strip() + right = parts[1].strip() + if not left or not right: + return None + return f"{right}->{left}" + + +def _packets_from_flow_entry(entry: Any) -> float: + if isinstance(entry, dict): + return max(0.0, _finite_float(entry.get("packets", 0.0), 0.0)) + return max(0.0, _finite_float(entry, 0.0)) + + +class TerminalFlowControlModel: + """One ML model owned by one terminal/LP. + + For this first prototype, each terminal model is a small ridge-regression + model per outgoing flow. This still gives the desired ownership boundary: + records and inference are isolated by LP id. Extending this to one + multi-output model per LP later only changes this class, not the server API. + """ + + def __init__( + self, + lp_id: str, + *, + ridge_alpha: float, + min_rows_per_flow: int, + default_prediction_packets: float, + max_prediction_packets: float, + ) -> None: + self.lp_id = _lp_key(lp_id) + self.ridge_alpha = max(0.0, float(ridge_alpha)) + self.min_rows_per_flow = max(1, int(min_rows_per_flow)) + self.default_prediction_packets = max(0.0, float(default_prediction_packets)) + self.max_prediction_packets = max(0.0, float(max_prediction_packets)) + + self.rows: list[dict[str, Any]] = [] + self.coefficients: dict[str, np.ndarray] = {} + self.flow_means: dict[str, float] = {} + self.num_requests = 0 + self.last_predictions: dict[str, float] = {} + + @staticmethod + def _features( + *, + incoming_packets: float, + grant_ratio: float, + queue_occupancy_ratio: float, + dropped_packets: float, + capacity_packets: float, + interval_id: float, + ) -> np.ndarray: + return np.asarray( + [ + 1.0, + max(0.0, incoming_packets), + min(1.0, max(0.0, grant_ratio)), + min(1.0, max(0.0, queue_occupancy_ratio)), + math.log1p(max(0.0, dropped_packets)), + math.log1p(max(0.0, capacity_packets)), + float(interval_id), + ], + dtype=np.float64, + ) + + @classmethod + def _row_features(cls, row: dict[str, Any]) -> np.ndarray: + return cls._features( + incoming_packets=_finite_float(row.get("incoming_packets"), 0.0), + grant_ratio=_finite_float(row.get("prev_grant_ratio"), 1.0), + queue_occupancy_ratio=_finite_float(row.get("prev_queue_occupancy_ratio"), 0.0), + dropped_packets=_finite_float(row.get("prev_dropped_packets"), 0.0), + capacity_packets=_finite_float(row.get("prev_capacity_packets"), 0.0), + interval_id=_finite_float(row.get("interval_id"), 0.0), + ) + + @staticmethod + def _target(row: dict[str, Any]) -> float: + return max(0.0, _finite_float(row.get("raw_egress_packets"), 0.0)) + + def add_row(self, row: dict[str, Any]) -> None: + self.rows.append(row) + + def train_or_update(self) -> bool: + by_flow: dict[str, list[dict[str, Any]]] = {} + for row in self.rows: + flow = str(row.get("outgoing_flow_key", "")).strip() + if flow: + by_flow.setdefault(flow, []).append(row) + + trained_any = False + for flow, rows in sorted(by_flow.items()): + if len(rows) < self.min_rows_per_flow: + continue + + x = np.vstack([self._row_features(row) for row in rows]) + y = np.asarray([self._target(row) for row in rows], dtype=np.float64) + self.flow_means[flow] = float(np.mean(y)) if len(y) else self.default_prediction_packets + + eye = np.eye(x.shape[1], dtype=np.float64) + eye[0, 0] = 0.0 # Do not penalize intercept. + try: + beta = np.linalg.solve(x.T @ x + self.ridge_alpha * eye, x.T @ y) + except np.linalg.LinAlgError: + beta = np.linalg.pinv(x) @ y + + self.coefficients[flow] = beta.astype(np.float64) + trained_any = True + + return trained_any + + def _predict_one(self, flow_key: str, features: np.ndarray) -> float: + if flow_key in self.coefficients: + raw = float(features @ self.coefficients[flow_key]) + elif flow_key in self.flow_means: + raw = float(self.flow_means[flow_key]) + else: + raw = self.default_prediction_packets + + raw = max(0.0, raw) + if self.max_prediction_packets > 0.0: + raw = min(raw, self.max_prediction_packets) + return raw + + def predict(self, payload: dict[str, Any]) -> dict[str, float]: + incoming = payload.get("incoming_flows", {}) or {} + outgoing_feedback = payload.get("outgoing_feedback", {}) or {} + outgoing_keys = payload.get("outgoing_flow_keys", []) or [] + interval_id = _finite_float(payload.get("interval_id"), 0.0) + + predictions: dict[str, float] = {} + for key in outgoing_keys: + flow_key = str(key).strip() + if not flow_key: + continue + + reverse_key = _flow_key_reverse(flow_key) + incoming_packets = 0.0 + if reverse_key is not None and reverse_key in incoming: + incoming_packets = _packets_from_flow_entry(incoming[reverse_key]) + elif incoming: + incoming_packets = sum(_packets_from_flow_entry(v) for v in incoming.values()) + + feedback = outgoing_feedback.get(flow_key, {}) or {} + if not isinstance(feedback, dict): + feedback = {} + + features = self._features( + incoming_packets=incoming_packets, + grant_ratio=_finite_float(feedback.get("grant_ratio"), 1.0), + queue_occupancy_ratio=_finite_float(feedback.get("queue_occupancy_ratio"), 0.0), + dropped_packets=_finite_float(feedback.get("dropped_packets"), 0.0), + capacity_packets=_finite_float(feedback.get("capacity_packets"), 0.0), + interval_id=interval_id, + ) + predictions[flow_key] = self._predict_one(flow_key, features) + + self.num_requests += 1 + self.last_predictions = dict(predictions) + return predictions + + def to_payload(self) -> dict[str, Any]: + return { + "lp_id": self.lp_id, + "rows": self.rows, + "coefficients": self.coefficients, + "flow_means": self.flow_means, + "num_requests": self.num_requests, + "last_predictions": self.last_predictions, + } + + @classmethod + def from_payload( + cls, + payload: dict[str, Any], + *, + ridge_alpha: float, + min_rows_per_flow: int, + default_prediction_packets: float, + max_prediction_packets: float, + ) -> "TerminalFlowControlModel": + model = cls( + _lp_key(payload.get("lp_id", "unknown")), + ridge_alpha=ridge_alpha, + min_rows_per_flow=min_rows_per_flow, + default_prediction_packets=default_prediction_packets, + max_prediction_packets=max_prediction_packets, + ) + model.rows = list(payload.get("rows", [])) + model.coefficients = { + str(k): np.asarray(v, dtype=np.float64) + for k, v in dict(payload.get("coefficients", {})).items() + } + model.flow_means = { + str(k): float(v) for k, v in dict(payload.get("flow_means", {})).items() + } + model.num_requests = int(payload.get("num_requests", 0)) + model.last_predictions = { + str(k): float(v) for k, v in dict(payload.get("last_predictions", {})).items() + } + return model + + def status(self) -> dict[str, Any]: + flows = sorted(set(self.flow_means) | set(self.coefficients)) + return { + "lp_id": self.lp_id, + "rows": len(self.rows), + "trained": bool(self.coefficients), + "trained_flows": len(self.coefficients), + "flows": flows, + "requests": self.num_requests, + } + + +class FlowControlSurrogateFamily: + """Server-side family registry with one terminal model per LP id.""" + + def __init__( + self, + *, + ridge_alpha: float = 1.0, + min_rows_per_flow: int = 1, + default_prediction_packets: float = 1024.0, + max_prediction_packets: float = 0.0, + ) -> None: + self.ridge_alpha = max(0.0, float(ridge_alpha)) + self.min_rows_per_flow = max(1, int(min_rows_per_flow)) + self.default_prediction_packets = max(0.0, float(default_prediction_packets)) + self.max_prediction_packets = max(0.0, float(max_prediction_packets)) + self.debug = False + + self.terminal_models: dict[str, TerminalFlowControlModel] = {} + self.model_version = 0 + self.num_requests = 0 + + def set_debug(self, enabled: bool) -> None: + self.debug = bool(enabled) + + def _new_terminal_model(self, lp_id: str) -> TerminalFlowControlModel: + return TerminalFlowControlModel( + lp_id, + ridge_alpha=self.ridge_alpha, + min_rows_per_flow=self.min_rows_per_flow, + default_prediction_packets=self.default_prediction_packets, + max_prediction_packets=self.max_prediction_packets, + ) + + def _model_for_lp(self, lp_id: Any) -> TerminalFlowControlModel: + key = _lp_key(lp_id) + model = self.terminal_models.get(key) + if model is None: + model = self._new_terminal_model(key) + self.terminal_models[key] = model + return model + + @staticmethod + def _normalize_record_row(row: dict[str, Any]) -> dict[str, Any] | None: + flow = str(row.get("outgoing_flow_key", "")).strip() + if not flow: + return None + out = {field: row.get(field, "") for field in FLOW_RECORD_FIELDS} + out["lp_id"] = _lp_key(out.get("lp_id", "unknown")) + out["outgoing_flow_key"] = flow + return out + + def add_records_text(self, payload_text: str) -> int: + text = (payload_text or "").strip() + if not text: + return 0 + + loaded = 0 + if text.lstrip().startswith("{"): + for line in text.splitlines(): + line = line.strip() + if not line: + continue + row = json.loads(line) + if not isinstance(row, dict): + continue + normalized = self._normalize_record_row(row) + if normalized is None: + continue + self._model_for_lp(normalized["lp_id"]).add_row(normalized) + loaded += 1 + return loaded + + reader = csv.DictReader(io.StringIO(text)) + for row in reader: + normalized = self._normalize_record_row(row) + if normalized is None: + continue + self._model_for_lp(normalized["lp_id"]).add_row(normalized) + loaded += 1 + return loaded + + def load_records_csv(self, path: str | Path) -> int: + path = Path(path) + loaded = 0 + files = sorted(path.rglob("*.csv")) if path.is_dir() else [path] + for child in files: + if not child.is_file(): + continue + loaded += self.add_records_text(child.read_text()) + return loaded + + def train_or_update(self) -> bool: + trained_any = False + trained_lps: list[str] = [] + for lp_id, model in sorted(self.terminal_models.items()): + if model.train_or_update(): + trained_any = True + trained_lps.append(lp_id) + + if trained_any: + self.model_version += 1 + + if self.debug: + print( + f"[flow-control train] terminal_models={len(self.terminal_models)} " + f"trained_lps={trained_lps}", + flush=True, + ) + return trained_any + + def predict(self, payload: dict[str, Any]) -> dict[str, float]: + lp_id = _lp_key(payload.get("lp_id", "unknown")) + model = self._model_for_lp(lp_id) + predictions = model.predict(payload) + self.num_requests += 1 + + if self.debug: + trained = bool(model.coefficients) + print( + f"[flow-control inference] lp={lp_id} trained={int(trained)} " + f"interval={payload.get('interval_id')} predictions={predictions}", + flush=True, + ) + return predictions + + def predict_from_text(self, payload_text: str) -> dict[str, float]: + payload_text = (payload_text or "").strip() + if not payload_text: + raise ValueError("missing flow-control JSON payload") + payload = json.loads(payload_text) + if not isinstance(payload, dict): + raise ValueError("flow-control payload must be a JSON object") + return self.predict(payload) + + def save(self, path: str | Path) -> None: + path = Path(path) + if path.parent: + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "format": "flow-control-per-terminal-linear-v1", + "ridge_alpha": self.ridge_alpha, + "min_rows_per_flow": self.min_rows_per_flow, + "default_prediction_packets": self.default_prediction_packets, + "max_prediction_packets": self.max_prediction_packets, + "terminal_models": { + lp_id: model.to_payload() for lp_id, model in sorted(self.terminal_models.items()) + }, + "model_version": self.model_version, + "num_requests": self.num_requests, + } + with path.open("wb") as f: + pickle.dump(payload, f) + + def load(self, path: str | Path) -> None: + with Path(path).open("rb") as f: + payload = pickle.load(f) + + self.ridge_alpha = float(payload.get("ridge_alpha", self.ridge_alpha)) + self.min_rows_per_flow = int(payload.get("min_rows_per_flow", self.min_rows_per_flow)) + self.default_prediction_packets = float( + payload.get("default_prediction_packets", self.default_prediction_packets) + ) + self.max_prediction_packets = float(payload.get("max_prediction_packets", self.max_prediction_packets)) + self.model_version = int(payload.get("model_version", self.model_version)) + self.num_requests = int(payload.get("num_requests", self.num_requests)) + + # New format: one serialized model per LP. + if "terminal_models" in payload: + self.terminal_models = { + _lp_key(lp_id): TerminalFlowControlModel.from_payload( + model_payload, + ridge_alpha=self.ridge_alpha, + min_rows_per_flow=self.min_rows_per_flow, + default_prediction_packets=self.default_prediction_packets, + max_prediction_packets=self.max_prediction_packets, + ) + for lp_id, model_payload in dict(payload.get("terminal_models", {})).items() + } + return + + # Backward compatibility with the earlier one-object/per-flow prototype. + legacy_rows = list(payload.get("rows", [])) + self.terminal_models = {} + for row in legacy_rows: + normalized = self._normalize_record_row(row) + if normalized is not None: + self._model_for_lp(normalized["lp_id"]).add_row(normalized) + self.train_or_update() + + def status(self) -> dict[str, str]: + lp_status = {lp_id: model.status() for lp_id, model in sorted(self.terminal_models.items())} + trained_lps = [lp_id for lp_id, st in lp_status.items() if st["trained"]] + trained_flow_labels: list[str] = [] + for lp_id, st in lp_status.items(): + for flow in st["flows"]: + trained_flow_labels.append(f"{lp_id}:{flow}") + + total_rows = sum(int(st["rows"]) for st in lp_status.values()) + total_requests = sum(int(st["requests"]) for st in lp_status.values()) + + return { + "model_type": "per-terminal-linear-flow-control", + "trained": "1" if trained_lps else "0", + "rows": str(total_rows), + "terminal_models": str(len(self.terminal_models)), + "trained_terminal_models": str(len(trained_lps)), + "trained_lps": ";".join(trained_lps), + "trained_flows": str(len(trained_flow_labels)), + "flows": ";".join(sorted(trained_flow_labels)), + "requests": str(total_requests), + "family_requests": str(self.num_requests), + "model_version": str(self.model_version), + "ridge_alpha": str(self.ridge_alpha), + "min_rows_per_flow": str(self.min_rows_per_flow), + "default_prediction_packets": str(self.default_prediction_packets), + } + + +def flow_control_from_env() -> FlowControlSurrogateFamily: + return FlowControlSurrogateFamily( + ridge_alpha=float(os.environ.get("ZMQML_FLOW_CONTROL_RIDGE_ALPHA", "1.0")), + min_rows_per_flow=int(os.environ.get("ZMQML_FLOW_CONTROL_MIN_ROWS_PER_FLOW", "1")), + default_prediction_packets=float( + os.environ.get("ZMQML_FLOW_CONTROL_DEFAULT_PACKETS", "1024") + ), + max_prediction_packets=float(os.environ.get("ZMQML_FLOW_CONTROL_MAX_PACKETS", "0")), + ) From cc0895123918c7b366eb2175ce61b10c9767cb98 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Tue, 7 Jul 2026 16:12:24 -0400 Subject: [PATCH 02/20] Add initial switch and terminal flow model --- doc/example/CMakeLists.txt | 3 + doc/example/fluid-switch-topology.yaml | 24 + doc/example/fluid-switch.conf.in | 39 + src/CMakeLists.txt | 2 + .../model-net-fluid-switch.cxx | 1089 +++++++++++++++++ 5 files changed, 1157 insertions(+) create mode 100644 doc/example/fluid-switch-topology.yaml create mode 100644 doc/example/fluid-switch.conf.in create mode 100644 src/network-workloads/model-net-fluid-switch.cxx diff --git a/doc/example/CMakeLists.txt b/doc/example/CMakeLists.txt index 1b0c5605..550c18e2 100644 --- a/doc/example/CMakeLists.txt +++ b/doc/example/CMakeLists.txt @@ -14,6 +14,7 @@ configure_file(tutorial-ping-pong-surrogate.conf.in tutorial-ping-pong-surrogate configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.template.conf.in @ONLY) configure_file(kb.dfdally-72-event-time-director.conf.in kb.dfdally-72-event-time-director.template.conf.in @ONLY) configure_file(kb.dfdally-72-milc-small.workload.conf.in kb.dfdally-72-milc-small.workload.template.conf.in @ONLY) +configure_file(fluid-switch.conf.in fluid-switch.template.conf.in @ONLY) configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.template.conf.in @ONLY) set(single_quote "'") @@ -42,6 +43,8 @@ configure_file(tutorial-ping-pong-surrogate.conf.in tutorial-ping-pong-surrogate configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.conf) configure_file(kb.dfdally-72-event-time-director.conf.in kb.dfdally-72-event-time-director.conf) configure_file(kb.dfdally-72-milc-small.workload.conf.in kb.dfdally-72-milc-small.workload.conf) +configure_file(fluid-switch.conf.in fluid-switch.conf) configure_file(kb.dfdally-72-milc-small.json kb.dfdally-72-milc-small.json COPYONLY) configure_file(kb.dfdally-72-milc-small.alloc.conf kb.dfdally-72-milc-small.alloc.conf COPYONLY) +configure_file(fluid-switch-topology.yaml fluid-switch-topology.yaml COPYONLY) configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.conf) diff --git a/doc/example/fluid-switch-topology.yaml b/doc/example/fluid-switch-topology.yaml new file mode 100644 index 00000000..b7d21fb7 --- /dev/null +++ b/doc/example/fluid-switch-topology.yaml @@ -0,0 +1,24 @@ +topology: + switches: + A: + terminals: 2 + terminal_bandwidth: "10 Mbps" + switch_buffer: "1000 Mb" + connections: + B: "20 Mbps" + + B: + terminals: 2 + terminal_bandwidth: "5 Mbps" + switch_buffer: "1000 Mb" + connections: + A: "15 Mbps" + C: "25 Mbps" + + C: + terminals: 2 + terminal_bandwidth: "15 Mbps" + switch_buffer: "1000 Mb" + connections: + A: "20 Mbps" + B: "15 Mbps" diff --git a/doc/example/fluid-switch.conf.in b/doc/example/fluid-switch.conf.in new file mode 100644 index 00000000..0347621d --- /dev/null +++ b/doc/example/fluid-switch.conf.in @@ -0,0 +1,39 @@ +# Interval-fluid switch/terminal pure-PDES example. +# Run from the build directory, for example: +# mpirun -np 1 ./src/model-net-fluid-switch --synch=1 -- doc/example/fluid-switch.conf + +LPGROUPS +{ + FLUID_GRP + { + repetitions="1"; + fluid-switch-lp="3"; + fluid-terminal-lp="6"; + } +} + +PARAMS +{ + message_size="256"; + pe_mem_factor="1024"; +} + +FLUID_SWITCH +{ + topology_yaml_file="fluid-switch-topology.yaml"; + + interval_seconds="1"; + num_send_intervals="20"; + num_drain_intervals="20"; + rng_seed="12345"; + + terminal_send_every_n_intervals="1"; + terminal_send_probability="1.0"; + terminal_min_send_mbit="0"; + terminal_max_send_fraction_of_link_capacity="1.0"; + + debug_prints="0"; + + terminal_log_path="../../zmqml_artifacts/fluid-switch/terminal-events.csv"; + switch_log_path="../../zmqml_artifacts/fluid-switch/switch-events.csv"; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 56e0ddd8..21b21fa3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -206,6 +206,7 @@ add_executable(model-net-synthetic network-workloads/model-net-synthetic.c) add_executable(model-net-synthetic-slimfly network-workloads/model-net-synthetic-slimfly.c) add_executable(model-net-synthetic-fattree network-workloads/model-net-synthetic-fattree.c) add_executable(model-net-synthetic-dragonfly-all network-workloads/model-net-synthetic-dragonfly-all.c) +add_executable(model-net-fluid-switch network-workloads/model-net-fluid-switch.cxx) set(CODES_TARGETS topology-test @@ -214,6 +215,7 @@ set(CODES_TARGETS model-net-synthetic-slimfly model-net-synthetic-fattree model-net-synthetic-dragonfly-all + model-net-fluid-switch ) if(USE_DUMPI) diff --git a/src/network-workloads/model-net-fluid-switch.cxx b/src/network-workloads/model-net-fluid-switch.cxx new file mode 100644 index 00000000..01efda5c --- /dev/null +++ b/src/network-workloads/model-net-fluid-switch.cxx @@ -0,0 +1,1089 @@ + +/* + * Standalone interval-fluid switch/terminal workload model. + * + * This is a pure PDES model. It does not use model-net and does not call the + * ZeroMQ Director. Terminal LPs generate stochastic bounded workload flowlets. + * Switch LPs route flowlets, queue them per output link, and send per-flowlet + * fragments subject to interval link capacity. + * + * Intended first-run mode: --synch=1. Reverse computation is intentionally + * disabled until the pure sequential model is validated. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "codes/codes.h" +#include "codes/codes_mapping.h" +#include "codes/configuration.h" +#include "codes/lp-type-lookup.h" + +static const char* GROUP_NAME = "FLUID_GRP"; +static const char* TERMINAL_LP_NAME = "fluid-terminal-lp"; +static const char* SWITCH_LP_NAME = "fluid-switch-lp"; + +static constexpr int MAX_SWITCHES = 64; +static constexpr int MAX_TERMINALS = 4096; +static constexpr int MAX_PORTS_PER_SWITCH = 128; +static constexpr double EPS = 1e-9; + +static constexpr double PHASE_GENERATE = 0.10; +static constexpr double PHASE_ARRIVAL = 0.20; +static constexpr double PHASE_SWITCH_EGRESS = 0.60; + +struct link_info { + int dst_switch; + double bandwidth_mbps; + double buffer_mbit; +}; + +struct switch_info { + std::string name; + int terminal_count = 0; + int terminal_start = 0; + double terminal_bandwidth_mbps = 10.0; + double switch_buffer_mbit = 1024.0; + std::vector links; +}; + +struct terminal_info { + int switch_id = -1; + int local_id = -1; + std::string name; +}; + +struct sim_config { + double interval_seconds = 1.0; + int num_send_intervals = 20; + int num_drain_intervals = 20; + int rng_seed = 12345; + int terminal_send_every_n_intervals = 1; + double terminal_send_probability = 1.0; + double terminal_min_send_mbit = 0.0; + double terminal_max_send_fraction_of_link_capacity = 1.0; + int debug_prints = 0; + char topology_yaml_file[1024] = ""; + char terminal_log_path[1024] = ""; + char switch_log_path[1024] = ""; +}; + +static sim_config cfg; +static std::vector switches; +static std::vector terminals; +static std::map switch_name_to_id; +static std::vector > next_switch_table; +static int total_switch_lps = 0; +static int total_terminal_lps = 0; + +struct queued_flowlet { + int valid; + unsigned long long flowlet_id; + int source_terminal; + int destination_terminal; + int creation_interval; + int enqueue_interval; + int age_intervals; + double remaining_mbit; +}; + +struct port_desc { + int is_terminal; + int target_index; + double capacity_mbit_per_interval; + double buffer_mbit; +}; + +struct terminal_state { + int terminal_id; + int attached_switch; + int local_terminal_id; + unsigned long long next_flowlet_seq; + double generated_mbit; + double sent_to_switch_mbit; + double received_mbit; + int generated_flowlets; + int received_flowlets; +}; + +struct switch_state { + int switch_id; + int num_ports; + port_desc ports[MAX_PORTS_PER_SWITCH]; + std::vector* queues[MAX_PORTS_PER_SWITCH]; + double enqueued_mbit; + double sent_mbit; + double delivered_local_mbit; + double dropped_mbit; + int received_flowlets; + int sent_fragments; +}; + +enum fluid_event_type { + WORKLOAD_GENERATE = 1, + FLOWLET_ARRIVAL = 2, + SWITCH_EGRESS = 3, +}; + +struct fluid_msg { + int event_type; + int interval_id; + int source_terminal; + int destination_terminal; + int source_switch; + int destination_switch; + int port_id; + int creation_interval; + unsigned long long flowlet_id; + double mbit; +}; + +static void terminal_init(terminal_state* ns, tw_lp* lp); +static void terminal_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); +static void terminal_rev_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); +static void terminal_finalize(terminal_state* ns, tw_lp* lp); + +static void switch_init(switch_state* ns, tw_lp* lp); +static void switch_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); +static void switch_rev_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); +static void switch_finalize(switch_state* ns, tw_lp* lp); + +static tw_lptype terminal_lp = { + (init_f)terminal_init, + (pre_run_f)NULL, + (event_f)terminal_event, + (revent_f)terminal_rev_event, + (commit_f)NULL, + (final_f)terminal_finalize, + (map_f)codes_mapping, + sizeof(terminal_state), +}; + +static tw_lptype switch_lp = { + (init_f)switch_init, + (pre_run_f)NULL, + (event_f)switch_event, + (revent_f)switch_rev_event, + (commit_f)NULL, + (final_f)switch_finalize, + (map_f)codes_mapping, + sizeof(switch_state), +}; + +static const tw_lptype* terminal_get_lp_type(void) { return &terminal_lp; } +static const tw_lptype* switch_get_lp_type(void) { return &switch_lp; } + +static void add_lp_types(void) { + lp_type_register(TERMINAL_LP_NAME, terminal_get_lp_type()); + lp_type_register(SWITCH_LP_NAME, switch_get_lp_type()); +} + +static double seconds_to_ns(double seconds) { return seconds * 1000.0 * 1000.0 * 1000.0; } + +static double event_time_ns(int interval_id, double phase_seconds) { + return seconds_to_ns(((double)interval_id * cfg.interval_seconds) + phase_seconds); +} + +static double delay_until_ns(int target_interval, double phase_seconds, tw_lp* lp) { + double target = event_time_ns(target_interval, phase_seconds); + double delta = target - tw_now(lp); + if (delta <= g_tw_lookahead) { + delta = g_tw_lookahead + 1.0; + } + return delta; +} + +static std::string trim(const std::string& s) { + size_t b = 0; + while (b < s.size() && std::isspace((unsigned char)s[b])) { + ++b; + } + size_t e = s.size(); + while (e > b && std::isspace((unsigned char)s[e - 1])) { + --e; + } + return s.substr(b, e - b); +} + +static int leading_spaces(const std::string& s) { + int n = 0; + while (n < (int)s.size() && s[n] == ' ') { + ++n; + } + return n; +} + +static std::string strip_quotes(std::string s) { + s = trim(s); + if (s.size() >= 2 && ((s.front() == '"' && s.back() == '"') || + (s.front() == '\'' && s.back() == '\''))) { + return s.substr(1, s.size() - 2); + } + return s; +} + +static std::string remove_inline_comment(const std::string& s) { + bool in_quote = false; + char quote_char = 0; + for (size_t i = 0; i < s.size(); ++i) { + if ((s[i] == '"' || s[i] == '\'') && (i == 0 || s[i - 1] != '\\')) { + if (!in_quote) { + in_quote = true; + quote_char = s[i]; + } else if (quote_char == s[i]) { + in_quote = false; + quote_char = 0; + } + } + if (!in_quote && s[i] == '#') { + return s.substr(0, i); + } + } + return s; +} + +static bool split_key_value(const std::string& line, std::string* key, std::string* value) { + size_t pos = line.find(':'); + if (pos == std::string::npos) { + return false; + } + *key = trim(line.substr(0, pos)); + *value = trim(line.substr(pos + 1)); + return !key->empty(); +} + +static double parse_mbps(const std::string& raw) { + std::string s = strip_quotes(raw); + std::stringstream ss(s); + double v = 0.0; + ss >> v; + if (!std::isfinite(v) || v < 0.0) { + return 0.0; + } + return v; +} + +static double parse_mbit(const std::string& raw) { + std::string s = strip_quotes(raw); + std::stringstream ss(s); + double v = 0.0; + ss >> v; + if (!std::isfinite(v) || v < 0.0) { + return 0.0; + } + if (s.find("GB") != std::string::npos || s.find("Gb") != std::string::npos) { + return v * 1000.0; + } + if (s.find("MB") != std::string::npos || s.find("Mb") != std::string::npos) { + return v; + } + return v; +} + +static int get_or_add_switch(const std::string& name) { + std::map::iterator it = switch_name_to_id.find(name); + if (it != switch_name_to_id.end()) { + return it->second; + } + int id = (int)switches.size(); + switch_info sw; + sw.name = name; + switches.push_back(sw); + switch_name_to_id[name] = id; + return id; +} + +static void add_or_update_link(int src, int dst, double bw_mbps, double buffer_mbit) { + for (size_t i = 0; i < switches[src].links.size(); ++i) { + if (switches[src].links[i].dst_switch == dst) { + switches[src].links[i].bandwidth_mbps = bw_mbps; + if (buffer_mbit > 0.0) { + switches[src].links[i].buffer_mbit = buffer_mbit; + } + return; + } + } + link_info li; + li.dst_switch = dst; + li.bandwidth_mbps = bw_mbps; + li.buffer_mbit = buffer_mbit > 0.0 ? buffer_mbit : switches[src].switch_buffer_mbit; + switches[src].links.push_back(li); +} + +static void load_topology_yaml(const char* path) { + switches.clear(); + terminals.clear(); + switch_name_to_id.clear(); + + std::ifstream in(path); + if (!in.good()) { + tw_error(TW_LOC, "could not open topology YAML file: %s", path); + } + + std::string section; + bool in_switches = false; + bool in_connections = false; + int current_switch = -1; + int current_conn_dst = -1; + double current_conn_buffer = 0.0; + + std::string raw; + while (std::getline(in, raw)) { + std::string no_comment = remove_inline_comment(raw); + if (trim(no_comment).empty()) { + continue; + } + int indent = leading_spaces(no_comment); + std::string line = trim(no_comment); + if (line == "topology:") { + section = "topology"; + in_switches = false; + in_connections = false; + current_switch = -1; + continue; + } + if (section == "topology" && line == "switches:") { + in_switches = true; + in_connections = false; + current_switch = -1; + continue; + } + + std::string key; + std::string value; + if (!split_key_value(line, &key, &value)) { + continue; + } + + if (section == "topology" && in_switches) { + if (indent == 4 && value.empty()) { + current_switch = get_or_add_switch(key); + in_connections = false; + current_conn_dst = -1; + continue; + } + if (current_switch < 0) { + continue; + } + if (indent == 6 && key == "connections") { + in_connections = true; + current_conn_dst = -1; + current_conn_buffer = switches[current_switch].switch_buffer_mbit; + continue; + } + if (!in_connections && indent >= 6) { + if (key == "terminals") switches[current_switch].terminal_count = atoi(strip_quotes(value).c_str()); + else if (key == "terminal_bandwidth") switches[current_switch].terminal_bandwidth_mbps = parse_mbps(value); + else if (key == "switch_buffer") switches[current_switch].switch_buffer_mbit = parse_mbit(value); + continue; + } + if (in_connections && indent == 8) { + int dst = get_or_add_switch(key); + current_conn_dst = dst; + current_conn_buffer = switches[current_switch].switch_buffer_mbit; + if (!value.empty()) { + add_or_update_link(current_switch, dst, parse_mbps(value), current_conn_buffer); + } + continue; + } + if (in_connections && indent >= 10 && current_conn_dst >= 0) { + if (key == "bandwidth") { + double bw = parse_mbps(value); + add_or_update_link(current_switch, current_conn_dst, bw, current_conn_buffer); + } else if (key == "buffer") { + current_conn_buffer = parse_mbit(value); + for (size_t i = 0; i < switches[current_switch].links.size(); ++i) { + if (switches[current_switch].links[i].dst_switch == current_conn_dst) { + switches[current_switch].links[i].buffer_mbit = current_conn_buffer; + } + } + } + } + } + } + + int terminal_start = 0; + for (int s = 0; s < (int)switches.size(); ++s) { + switches[s].terminal_start = terminal_start; + for (int t = 0; t < switches[s].terminal_count; ++t) { + terminal_info ti; + ti.switch_id = s; + ti.local_id = t; + std::ostringstream name; + name << switches[s].name << "." << t; + ti.name = name.str(); + terminals.push_back(ti); + ++terminal_start; + } + } + + if (switches.empty()) { + tw_error(TW_LOC, "topology YAML defined no switches"); + } + if (terminals.size() < 2) { + tw_error(TW_LOC, "topology YAML must define at least two terminals"); + } + if (cfg.interval_seconds <= 0.0) tw_error(TW_LOC, "interval_seconds must be positive"); + if (cfg.num_send_intervals <= 0) tw_error(TW_LOC, "num_send_intervals must be positive"); + if (cfg.num_drain_intervals < 0) cfg.num_drain_intervals = 0; + if (cfg.terminal_send_every_n_intervals <= 0) cfg.terminal_send_every_n_intervals = 1; + if (cfg.terminal_send_probability < 0.0) cfg.terminal_send_probability = 0.0; + if (cfg.terminal_send_probability > 1.0) cfg.terminal_send_probability = 1.0; + if (cfg.terminal_max_send_fraction_of_link_capacity < 0.0) cfg.terminal_max_send_fraction_of_link_capacity = 0.0; + if (cfg.terminal_max_send_fraction_of_link_capacity > 1.0) cfg.terminal_max_send_fraction_of_link_capacity = 1.0; +} + +static void compute_routes(void) { + int n = (int)switches.size(); + next_switch_table.assign(n, std::vector(n, -1)); + for (int src = 0; src < n; ++src) { + std::vector prev(n, -1); + std::vector seen(n, 0); + std::queue q; + seen[src] = 1; + q.push(src); + while (!q.empty()) { + int u = q.front(); + q.pop(); + for (size_t i = 0; i < switches[u].links.size(); ++i) { + int v = switches[u].links[i].dst_switch; + if (!seen[v]) { + seen[v] = 1; + prev[v] = u; + q.push(v); + } + } + } + for (int dst = 0; dst < n; ++dst) { + if (dst == src) { + next_switch_table[src][dst] = src; + continue; + } + if (!seen[dst]) { + next_switch_table[src][dst] = -1; + continue; + } + int cur = dst; + int parent = prev[cur]; + while (parent >= 0 && parent != src) { + cur = parent; + parent = prev[cur]; + } + next_switch_table[src][dst] = cur; + } + } +} + +static void read_int_param(const char* section, const char* key, int* value) { + int tmp = 0; + if (configuration_get_value_int(&config, section, key, NULL, &tmp) == 0) { + *value = tmp; + } +} + +static void read_double_param(const char* section, const char* key, double* value) { + double tmp = 0.0; + if (configuration_get_value_double(&config, section, key, NULL, &tmp) == 0) { + *value = tmp; + } +} + + +static void read_relpath_param(const char* section, const char* key, char* value, size_t len) { + char tmp[1024]; + memset(tmp, 0, sizeof(tmp)); + if (configuration_get_value_relpath(&config, section, key, NULL, tmp, sizeof(tmp)) > 0) { + snprintf(value, len, "%s", tmp); + } +} + +static void load_config(void) { + read_relpath_param("FLUID_SWITCH", "topology_yaml_file", cfg.topology_yaml_file, + sizeof(cfg.topology_yaml_file)); + read_relpath_param("FLUID_SWITCH", "terminal_log_path", cfg.terminal_log_path, + sizeof(cfg.terminal_log_path)); + read_relpath_param("FLUID_SWITCH", "switch_log_path", cfg.switch_log_path, + sizeof(cfg.switch_log_path)); + read_double_param("FLUID_SWITCH", "interval_seconds", &cfg.interval_seconds); + read_int_param("FLUID_SWITCH", "num_send_intervals", &cfg.num_send_intervals); + read_int_param("FLUID_SWITCH", "num_drain_intervals", &cfg.num_drain_intervals); + read_int_param("FLUID_SWITCH", "rng_seed", &cfg.rng_seed); + read_int_param("FLUID_SWITCH", "terminal_send_every_n_intervals", &cfg.terminal_send_every_n_intervals); + read_double_param("FLUID_SWITCH", "terminal_send_probability", &cfg.terminal_send_probability); + read_double_param("FLUID_SWITCH", "terminal_min_send_mbit", &cfg.terminal_min_send_mbit); + read_double_param("FLUID_SWITCH", "terminal_max_send_fraction_of_link_capacity", + &cfg.terminal_max_send_fraction_of_link_capacity); + read_int_param("FLUID_SWITCH", "debug_prints", &cfg.debug_prints); + + if (cfg.topology_yaml_file[0] == '\0') { + tw_error(TW_LOC, "FLUID_SWITCH.topology_yaml_file is required"); + } + load_topology_yaml(cfg.topology_yaml_file); + compute_routes(); +} + +static tw_lpid get_switch_gid(int switch_id) { + if (switch_id < 0 || switch_id >= (int)switches.size()) { + tw_error(TW_LOC, "switch id %d out of range [0, %zu)", switch_id, switches.size()); + } + + tw_lpid gid = 0; + + /* + * LPGROUPS has one FLUID_GRP repetition containing all switch LPs. + * The switch index is therefore the LP-type offset, not the group repetition. + */ + codes_mapping_get_lp_id(GROUP_NAME, SWITCH_LP_NAME, NULL, 1, 0, switch_id, &gid); + + return gid; +} + +static tw_lpid get_terminal_gid(int terminal_id) { + if (terminal_id < 0 || terminal_id >= (int)terminals.size()) { + tw_error(TW_LOC, "terminal id %d out of range [0, %zu)", terminal_id, terminals.size()); + } + + tw_lpid gid = 0; + + /* + * LPGROUPS has one FLUID_GRP repetition containing all terminal LPs. + * The terminal index is therefore the LP-type offset, not the group repetition. + */ + codes_mapping_get_lp_id(GROUP_NAME, TERMINAL_LP_NAME, NULL, 1, 0, terminal_id, &gid); + + return gid; +} + +static int find_switch_link_port(const switch_state* ns, int dst_switch) { + for (int p = 0; p < ns->num_ports; ++p) { + if (!ns->ports[p].is_terminal && ns->ports[p].target_index == dst_switch) { + return p; + } + } + return -1; +} + +static int find_terminal_port(const switch_state* ns, int dst_terminal) { + for (int p = 0; p < ns->num_ports; ++p) { + if (ns->ports[p].is_terminal && ns->ports[p].target_index == dst_terminal) { + return p; + } + } + return -1; +} + +static int enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m) { + if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { + return 0; + } + std::vector& qv = *ns->queues[port_id]; + + double queued_on_port = 0.0; + for (size_t i = 0; i < qv.size(); ++i) { + queued_on_port += qv[i].remaining_mbit; + } + + double accepted = m->mbit; + if (queued_on_port + accepted > ns->ports[port_id].buffer_mbit + EPS) { + accepted = std::max(0.0, ns->ports[port_id].buffer_mbit - queued_on_port); + ns->dropped_mbit += std::max(0.0, m->mbit - accepted); + } + if (accepted <= EPS) { + return 0; + } + + queued_flowlet q; + memset(&q, 0, sizeof(q)); + q.valid = 1; + q.flowlet_id = m->flowlet_id; + q.source_terminal = m->source_terminal; + q.destination_terminal = m->destination_terminal; + q.creation_interval = m->creation_interval; + q.enqueue_interval = m->interval_id; + q.age_intervals = m->interval_id - m->creation_interval; + q.remaining_mbit = accepted; + qv.push_back(q); + ns->enqueued_mbit += accepted; + return 1; +} + +static double queued_mbit_on_port(const switch_state* ns, int port_id) { + if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { + return 0.0; + } + double total = 0.0; + const std::vector& qv = *ns->queues[port_id]; + for (size_t i = 0; i < qv.size(); ++i) { + total += qv[i].remaining_mbit; + } + return total; +} + +static void compact_port_queue(switch_state* ns, int port_id) { + if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { + return; + } + std::vector& qv = *ns->queues[port_id]; + qv.erase(std::remove_if(qv.begin(), qv.end(), [](const queued_flowlet& q) { + return !q.valid || q.remaining_mbit <= EPS; + }), + qv.end()); +} + +static void append_terminal_log(int interval_id, const char* event_name, int terminal_id, + int peer_id, double mbit) { + if (cfg.terminal_log_path[0] == '\0') { + return; + } + std::ofstream out(cfg.terminal_log_path, std::ios::app); + out << interval_id << ',' << event_name << ',' << terminal_id << ',' + << terminals[terminal_id].name << ',' << terminals[terminal_id].switch_id << ',' + << peer_id << ',' << mbit << '\n'; +} + +static void append_switch_log(int interval_id, const char* event_name, int switch_id, int port_id, + int target_is_terminal, int target_index, double capacity_mbit, + double sent_mbit, double queued_after_mbit, double dropped_mbit, + int active_flowlets) { + if (cfg.switch_log_path[0] == '\0') { + return; + } + std::ofstream out(cfg.switch_log_path, std::ios::app); + out << interval_id << ',' << event_name << ',' << switch_id << ',' + << switches[switch_id].name << ',' << port_id << ',' + << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' + << capacity_mbit << ',' << sent_mbit << ',' << queued_after_mbit << ',' + << dropped_mbit << ',' << active_flowlets << '\n'; +} + +static void schedule_workload_generate(terminal_state* ns, int interval_id, tw_lp* lp) { + if (interval_id >= cfg.num_send_intervals) { + return; + } + tw_event* e = tw_event_new(lp->gid, delay_until_ns(interval_id, PHASE_GENERATE, lp), lp); + fluid_msg* m = (fluid_msg*)tw_event_data(e); + memset(m, 0, sizeof(*m)); + m->event_type = WORKLOAD_GENERATE; + m->interval_id = interval_id; + m->source_terminal = ns->terminal_id; + tw_event_send(e); +} + +static void schedule_switch_egress(int interval_id, int port_id, tw_lp* lp) { + if (interval_id >= cfg.num_send_intervals + cfg.num_drain_intervals) { + return; + } + tw_event* e = tw_event_new(lp->gid, delay_until_ns(interval_id, PHASE_SWITCH_EGRESS, lp), lp); + fluid_msg* m = (fluid_msg*)tw_event_data(e); + memset(m, 0, sizeof(*m)); + m->event_type = SWITCH_EGRESS; + m->interval_id = interval_id; + m->port_id = port_id; + tw_event_send(e); +} + +static void schedule_arrival(int interval_id, tw_lpid dst_gid, const fluid_msg* src_msg, double mbit, + int dst_switch, tw_lp* lp) { + tw_event* e = tw_event_new(dst_gid, delay_until_ns(interval_id, PHASE_ARRIVAL, lp), lp); + fluid_msg* m = (fluid_msg*)tw_event_data(e); + memset(m, 0, sizeof(*m)); + m->event_type = FLOWLET_ARRIVAL; + m->interval_id = interval_id; + m->source_terminal = src_msg->source_terminal; + m->destination_terminal = src_msg->destination_terminal; + m->source_switch = src_msg->source_switch; + m->destination_switch = dst_switch; + m->creation_interval = src_msg->creation_interval; + m->flowlet_id = src_msg->flowlet_id; + m->mbit = mbit; + tw_event_send(e); +} + +static void terminal_init(terminal_state* ns, tw_lp* lp) { + memset(ns, 0, sizeof(*ns)); + ns->terminal_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); + if (ns->terminal_id < 0 || ns->terminal_id >= (int)terminals.size()) { + tw_error(TW_LOC, "terminal LP relative id %d out of range", ns->terminal_id); + } + ns->attached_switch = terminals[ns->terminal_id].switch_id; + ns->local_terminal_id = terminals[ns->terminal_id].local_id; + ns->next_flowlet_seq = 0; + schedule_workload_generate(ns, 0, lp); +} + +static int choose_random_destination(int self, tw_lp* lp) { + int n = (int)terminals.size(); + if (n <= 1) { + return self; + } + int offset = (int)tw_rand_integer(lp->rng, 1, n - 1); + return (self + offset) % n; +} + +static double random_workload_mbit(int terminal_id, tw_lp* lp) { + int sw = terminals[terminal_id].switch_id; + double cap = switches[sw].terminal_bandwidth_mbps * cfg.interval_seconds; + double max_mbit = cap * cfg.terminal_max_send_fraction_of_link_capacity; + double min_mbit = std::min(cfg.terminal_min_send_mbit, max_mbit); + double u = tw_rand_unif(lp->rng); + return min_mbit + u * (max_mbit - min_mbit); +} + +static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp) { + int interval = m->interval_id; + if (interval % cfg.terminal_send_every_n_intervals == 0) { + double p = tw_rand_unif(lp->rng); + if (p <= cfg.terminal_send_probability) { + int dst = choose_random_destination(ns->terminal_id, lp); + double mbit = random_workload_mbit(ns->terminal_id, lp); + if (mbit > EPS) { + fluid_msg out_msg; + memset(&out_msg, 0, sizeof(out_msg)); + out_msg.event_type = FLOWLET_ARRIVAL; + out_msg.interval_id = interval + 1; + out_msg.source_terminal = ns->terminal_id; + out_msg.destination_terminal = dst; + out_msg.source_switch = ns->attached_switch; + out_msg.destination_switch = ns->attached_switch; + out_msg.creation_interval = interval; + out_msg.flowlet_id = ((unsigned long long)ns->terminal_id << 48) | + (unsigned long long)ns->next_flowlet_seq++; + out_msg.mbit = mbit; + + tw_lpid sw_gid = get_switch_gid(ns->attached_switch); + schedule_arrival(interval + 1, sw_gid, &out_msg, mbit, ns->attached_switch, lp); + + ns->generated_mbit += mbit; + ns->sent_to_switch_mbit += mbit; + ns->generated_flowlets++; + append_terminal_log(interval, "generate_send", ns->terminal_id, dst, mbit); + + if (cfg.debug_prints) { + printf("[fluid terminal] interval=%d terminal=%d dst=%d mbit=%.6f\n", + interval, ns->terminal_id, dst, mbit); + } + } + } + } + schedule_workload_generate(ns, interval + 1, lp); +} + +static void handle_terminal_arrival(terminal_state* ns, fluid_msg* m) { + ns->received_mbit += m->mbit; + ns->received_flowlets++; + append_terminal_log(m->interval_id, "receive", ns->terminal_id, m->source_terminal, m->mbit); +} + +static void terminal_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { + (void)b; + switch (m->event_type) { + case WORKLOAD_GENERATE: + handle_workload_generate(ns, m, lp); + break; + case FLOWLET_ARRIVAL: + handle_terminal_arrival(ns, m); + break; + default: + tw_error(TW_LOC, "terminal received unknown event type %d", m->event_type); + } +} + +static void terminal_rev_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { + (void)ns; + (void)b; + (void)m; + (void)lp; + tw_error(TW_LOC, "model-net-fluid-switch currently supports sequential validation runs only"); +} + +static void terminal_finalize(terminal_state* ns, tw_lp* lp) { + printf("fluid-terminal gid=%llu terminal=%d switch=%d generated_mbit=%.6f sent_mbit=%.6f " + "received_mbit=%.6f generated_flowlets=%d received_flowlets=%d\n", + (unsigned long long)lp->gid, ns->terminal_id, ns->attached_switch, + ns->generated_mbit, ns->sent_to_switch_mbit, ns->received_mbit, + ns->generated_flowlets, ns->received_flowlets); +} + +static void switch_init(switch_state* ns, tw_lp* lp) { + memset(ns, 0, sizeof(*ns)); + ns->switch_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); + if (ns->switch_id < 0 || ns->switch_id >= (int)switches.size()) { + tw_error(TW_LOC, "switch LP relative id %d out of range", ns->switch_id); + } + + const switch_info& sw = switches[ns->switch_id]; + for (size_t i = 0; i < sw.links.size(); ++i) { + if (ns->num_ports >= MAX_PORTS_PER_SWITCH) { + tw_error(TW_LOC, "too many ports on switch %d", ns->switch_id); + } + port_desc* p = &ns->ports[ns->num_ports++]; + memset(p, 0, sizeof(*p)); + p->is_terminal = 0; + p->target_index = sw.links[i].dst_switch; + p->capacity_mbit_per_interval = sw.links[i].bandwidth_mbps * cfg.interval_seconds; + p->buffer_mbit = sw.links[i].buffer_mbit; + ns->queues[ns->num_ports - 1] = new std::vector(); + } + for (int t = 0; t < sw.terminal_count; ++t) { + if (ns->num_ports >= MAX_PORTS_PER_SWITCH) { + tw_error(TW_LOC, "too many ports on switch %d", ns->switch_id); + } + port_desc* p = &ns->ports[ns->num_ports++]; + memset(p, 0, sizeof(*p)); + p->is_terminal = 1; + p->target_index = sw.terminal_start + t; + p->capacity_mbit_per_interval = sw.terminal_bandwidth_mbps * cfg.interval_seconds; + p->buffer_mbit = sw.switch_buffer_mbit; + ns->queues[ns->num_ports - 1] = new std::vector(); + } + + for (int p = 0; p < ns->num_ports; ++p) { + schedule_switch_egress(0, p, lp); + } +} + +static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { + ns->received_flowlets++; + int dst_sw = terminals[m->destination_terminal].switch_id; + int port_id = -1; + if (dst_sw == ns->switch_id) { + port_id = find_terminal_port(ns, m->destination_terminal); + } else { + int next_sw = next_switch_table[ns->switch_id][dst_sw]; + if (next_sw >= 0) { + port_id = find_switch_link_port(ns, next_sw); + } + } + + if (port_id < 0) { + ns->dropped_mbit += m->mbit; + append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, + 0.0, 0.0, 0.0, m->mbit, 0); + return; + } + enqueue_flowlet(ns, port_id, m); +} + +static void send_flowlet_fragment(switch_state* ns, int port_id, const queued_flowlet* q, + double send_mbit, int interval_id, tw_lp* lp) { + if (send_mbit <= EPS) { + return; + } + fluid_msg out_msg; + memset(&out_msg, 0, sizeof(out_msg)); + out_msg.event_type = FLOWLET_ARRIVAL; + out_msg.interval_id = interval_id + 1; + out_msg.source_terminal = q->source_terminal; + out_msg.destination_terminal = q->destination_terminal; + out_msg.source_switch = ns->switch_id; + out_msg.creation_interval = q->creation_interval; + out_msg.flowlet_id = q->flowlet_id; + out_msg.mbit = send_mbit; + + const port_desc* p = &ns->ports[port_id]; + if (p->is_terminal) { + out_msg.destination_switch = ns->switch_id; + tw_lpid term_gid = get_terminal_gid(p->target_index); + schedule_arrival(interval_id + 1, term_gid, &out_msg, send_mbit, ns->switch_id, lp); + ns->delivered_local_mbit += send_mbit; + } else { + out_msg.destination_switch = p->target_index; + tw_lpid sw_gid = get_switch_gid(p->target_index); + schedule_arrival(interval_id + 1, sw_gid, &out_msg, send_mbit, p->target_index, lp); + } + ns->sent_mbit += send_mbit; + ns->sent_fragments++; +} + +static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { + int port_id = m->port_id; + if (port_id < 0 || port_id >= ns->num_ports) { + tw_error(TW_LOC, "invalid switch egress port %d", port_id); + } + if (ns->queues[port_id] == NULL) { + schedule_switch_egress(m->interval_id + 1, port_id, lp); + return; + } + + port_desc* p = &ns->ports[port_id]; + std::vector& qv = *ns->queues[port_id]; + double capacity = p->capacity_mbit_per_interval; + double remaining_capacity = capacity; + double sent_total = 0.0; + + std::vector active; + for (int i = 0; i < (int)qv.size(); ++i) { + if (qv[i].valid && qv[i].remaining_mbit > EPS) { + active.push_back(i); + } + } + + /* Max-min fair allocation with redistribution. For 25 Mb and 10 Mb + * flowlets on a 25 Mb link, this sends 15 Mb from the larger flowlet and + * 10 Mb from the smaller one, leaving 10 Mb queued from the larger flowlet. + */ + while (!active.empty() && remaining_capacity > EPS) { + double share = remaining_capacity / (double)active.size(); + std::vector still_active; + bool made_progress = false; + for (size_t ai = 0; ai < active.size(); ++ai) { + int idx = active[ai]; + queued_flowlet before = qv[idx]; + double send = std::min(before.remaining_mbit, share); + if (send > EPS) { + send_flowlet_fragment(ns, port_id, &before, send, m->interval_id, lp); + qv[idx].remaining_mbit -= send; + remaining_capacity -= send; + sent_total += send; + made_progress = true; + } + if (qv[idx].remaining_mbit > EPS) { + still_active.push_back(idx); + } + } + active.swap(still_active); + if (!made_progress) { + break; + } + } + + compact_port_queue(ns, port_id); + double queued_after = queued_mbit_on_port(ns, port_id); + append_switch_log(m->interval_id, "egress", ns->switch_id, port_id, p->is_terminal, + p->target_index, capacity, sent_total, queued_after, 0.0, + (int)qv.size()); + + schedule_switch_egress(m->interval_id + 1, port_id, lp); +} + +static void switch_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { + (void)b; + switch (m->event_type) { + case FLOWLET_ARRIVAL: + handle_switch_arrival(ns, m); + break; + case SWITCH_EGRESS: + handle_switch_egress(ns, m, lp); + break; + default: + tw_error(TW_LOC, "switch received unknown event type %d", m->event_type); + } +} + +static void switch_rev_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { + (void)ns; + (void)b; + (void)m; + (void)lp; + tw_error(TW_LOC, "model-net-fluid-switch currently supports sequential validation runs only"); +} + +static void switch_finalize(switch_state* ns, tw_lp* lp) { + double queued = 0.0; + for (int p = 0; p < ns->num_ports; ++p) { + queued += queued_mbit_on_port(ns, p); + } + printf("fluid-switch gid=%llu switch=%d name=%s ports=%d received_flowlets=%d " + "sent_fragments=%d enqueued_mbit=%.6f sent_mbit=%.6f local_delivery_mbit=%.6f " + "dropped_mbit=%.6f queued_mbit=%.6f\n", + (unsigned long long)lp->gid, ns->switch_id, switches[ns->switch_id].name.c_str(), + ns->num_ports, ns->received_flowlets, ns->sent_fragments, ns->enqueued_mbit, + ns->sent_mbit, ns->delivered_local_mbit, ns->dropped_mbit, queued); +} + +const tw_optdef app_opt[] = {TWOPT_GROUP("model-net interval-fluid switch/terminal workload"), + TWOPT_END()}; + +static const char* find_config_arg(int argc, char** argv) { + for (int i = argc - 1; i >= 1; --i) { + if (argv[i] == NULL) { + continue; + } + const char* arg = argv[i]; + size_t len = strlen(arg); + if (len >= 5 && strcmp(arg + len - 5, ".conf") == 0) { + return arg; + } + } + return NULL; +} + +static void write_log_headers(int rank) { + if (rank != 0) { + return; + } + if (cfg.terminal_log_path[0] != '\0') { + std::remove(cfg.terminal_log_path); + std::ofstream out(cfg.terminal_log_path, std::ios::app); + out << "interval,event,terminal,terminal_name,attached_switch,peer_terminal,mbit\n"; + } + if (cfg.switch_log_path[0] != '\0') { + std::remove(cfg.switch_log_path); + std::ofstream out(cfg.switch_log_path, std::ios::app); + out << "interval,event,switch,switch_name,port,target_type,target_index,capacity_mbit," + "sent_mbit,queued_after_mbit,dropped_mbit,active_flowlets\n"; + } +} + +int main(int argc, char** argv) { + int rank = 0; + + g_tw_ts_end = seconds_to_ns(60.0 * 60.0 * 24.0 * 365.0); + + tw_opt_add(app_opt); + tw_init(&argc, &argv); + + const char* config_file = find_config_arg(argc, argv); + if (config_file == NULL) { + printf("Usage: mpirun -np %s --synch=1 -- \n", argv[0]); + MPI_Finalize(); + return 1; + } + + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + configuration_load(config_file, MPI_COMM_WORLD, &config); + load_config(); + + add_lp_types(); + codes_mapping_setup(); + + total_switch_lps = codes_mapping_get_lp_count(GROUP_NAME, 0, SWITCH_LP_NAME, NULL, 1); + total_terminal_lps = codes_mapping_get_lp_count(GROUP_NAME, 0, TERMINAL_LP_NAME, NULL, 1); + if (total_switch_lps != (int)switches.size()) { + tw_error(TW_LOC, "config defines %d switch LPs but topology YAML defines %zu switches", + total_switch_lps, switches.size()); + } + if (total_terminal_lps != (int)terminals.size()) { + tw_error(TW_LOC, "config defines %d terminal LPs but topology YAML defines %zu terminals", + total_terminal_lps, terminals.size()); + } + + write_log_headers(rank); + MPI_Barrier(MPI_COMM_WORLD); + + if (rank == 0) { + printf("fluid-switch config: switches=%zu terminals=%zu interval_seconds=%.6f " + "num_send_intervals=%d num_drain_intervals=%d\n", + switches.size(), terminals.size(), cfg.interval_seconds, cfg.num_send_intervals, + cfg.num_drain_intervals); + } + + tw_run(); + tw_end(); + return 0; +} From b9f83955f8030a8e03254ad282e178a8763db614 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Tue, 7 Jul 2026 16:15:03 -0400 Subject: [PATCH 03/20] Revert "Add other files required for simplep2p flow model" This reverts commit 9cd0c042187b1a91db6648eb332fec004cf404c4 as the changes for it are in the digital-twin-sbir-develop-flow-control branch. --- doc/example/flow-control-simplep2p-bw.conf | 2 - .../flow-control-simplep2p-latency.conf | 2 - doc/example/flow-control-simplep2p.conf.in | 60 -- .../model-net-flow-control-simplep2p.cxx | 662 ------------------ src/surrogate/zmqml/model/mlflowcontrol.py | 512 -------------- 5 files changed, 1238 deletions(-) delete mode 100644 doc/example/flow-control-simplep2p-bw.conf delete mode 100644 doc/example/flow-control-simplep2p-latency.conf delete mode 100644 doc/example/flow-control-simplep2p.conf.in delete mode 100644 src/network-workloads/model-net-flow-control-simplep2p.cxx delete mode 100644 src/surrogate/zmqml/model/mlflowcontrol.py diff --git a/doc/example/flow-control-simplep2p-bw.conf b/doc/example/flow-control-simplep2p-bw.conf deleted file mode 100644 index 1533b9e0..00000000 --- a/doc/example/flow-control-simplep2p-bw.conf +++ /dev/null @@ -1,2 +0,0 @@ -0.0,0.0 200.0,200.0 -100.0,100.0 0.0,0.0 diff --git a/doc/example/flow-control-simplep2p-latency.conf b/doc/example/flow-control-simplep2p-latency.conf deleted file mode 100644 index 77ae2f34..00000000 --- a/doc/example/flow-control-simplep2p-latency.conf +++ /dev/null @@ -1,2 +0,0 @@ -0.0,0.0 1000.0,1000.0 -1000.0,1000.0 0.0,0.0 diff --git a/doc/example/flow-control-simplep2p.conf.in b/doc/example/flow-control-simplep2p.conf.in deleted file mode 100644 index d30e5cf4..00000000 --- a/doc/example/flow-control-simplep2p.conf.in +++ /dev/null @@ -1,60 +0,0 @@ -# ZeroMQ flow-control surrogate example for a two-terminal simplep2p workload. -# -# ML predicts raw egress packets by directed flow. CODES applies source queue -# capacity, per-flow grant capacity, drops, feedback, and then injects granted -# traffic into modelnet_simplep2p. - -LPGROUPS -{ - MODELNET_GRP - { - repetitions="2"; - nw-lp="1"; - modelnet_simplep2p="1"; - } -} - -PARAMS -{ - message_size="464"; - packet_size="${PACKET_SIZE}"; - modelnet_order=("simplep2p"); - modelnet_scheduler="fcfs"; - - # simplep2p reads these with configuration_get_value_relpath(), so keep - # them relative to this generated config file in build/doc/example/. - net_latency_ns_file="flow-control-simplep2p-latency.conf"; - net_bw_mbps_file="flow-control-simplep2p-bw.conf"; -} - -FLOW_CONTROL -{ - # CMake fills only the run mode and log path for generated concrete configs. - inferencing_enabled="${FLOW_CONTROL_INFERENCING_ENABLED}"; - training_enabled="${FLOW_CONTROL_TRAINING_ENABLED}"; - flow_log_path="${FLOW_CONTROL_LOG_PATH}"; - - debug_prints="1"; - - interval_seconds="10"; - num_intervals="50"; - packet_size_bytes="${PACKET_SIZE}"; - - # These source-side grant capacities should match the simplep2p bandwidth - # matrix for this first prototype. - bandwidth_0_to_1_mbps="200"; - bandwidth_1_to_0_mbps="100"; - - queue_capacity_0_to_1_packets="500"; - queue_capacity_1_to_0_packets="500"; - - # Pure-PDES synthetic demand generator defaults. - base_0_to_1_packets="50"; - base_1_to_0_packets="25"; - burst_0_to_1_packets="30"; - burst_1_to_0_packets="15"; - burst_period="5"; - ingress_gain="0.10"; - - mark_threshold_ratio="0.8"; -} diff --git a/src/network-workloads/model-net-flow-control-simplep2p.cxx b/src/network-workloads/model-net-flow-control-simplep2p.cxx deleted file mode 100644 index 765da69c..00000000 --- a/src/network-workloads/model-net-flow-control-simplep2p.cxx +++ /dev/null @@ -1,662 +0,0 @@ -/* - * Standalone interval-flow workload over model-net simplep2p. - * - * This is a normal runnable CODES binary, not a ctest harness. It supports - * the workflow: - * pure PDES record collection -> train/save flow-control ZMQML model -> - * hybrid full-surrogate traffic generation with PDES communication. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "codes/codes.h" -#include "codes/codes_mapping.h" -#include "codes/configuration.h" -#include "codes/lp-type-lookup.h" -#include "codes/model-net.h" -#include "zmqmlrequester.h" - -static int net_id = 0; -static int num_servers = 0; - -struct flow_feedback { - double raw_predicted_packets; - double available_packets; - double granted_packets; - double dropped_packets; - double grant_ratio; - double queue_packets_next; - double queue_occupancy_ratio; - double queue_delay_estimate; - double capacity_packets; - int congestion_mark_flag; - int overflow_flag; -}; - -struct flow_config { - int inferencing_enabled; - int training_enabled; - int debug_prints; - double interval_seconds; - int num_intervals; - int packet_size_bytes; - double bandwidth_0_to_1_mbps; - double bandwidth_1_to_0_mbps; - double queue_capacity_0_to_1_packets; - double queue_capacity_1_to_0_packets; - double base_0_to_1_packets; - double base_1_to_0_packets; - double burst_0_to_1_packets; - double burst_1_to_0_packets; - int burst_period; - double ingress_gain; - double mark_threshold_ratio; - char flow_log_path[1024]; -}; - -static flow_config cfg = { - 0, /* inferencing_enabled */ - 1, /* training_enabled */ - 0, /* debug_prints */ - 10.0, /* interval_seconds */ - 20, /* num_intervals */ - 4096, /* packet_size_bytes */ - 200.0, /* bandwidth 0->1 */ - 100.0, /* bandwidth 1->0 */ - 100000., /* queue cap 0->1 */ - 100000., /* queue cap 1->0 */ - 50000., /* pure base 0->1 */ - 25000., /* pure base 1->0 */ - 30000., /* burst 0->1 */ - 15000., /* burst 1->0 */ - 5, /* burst period */ - 0.10, /* ingress gain */ - 0.80, /* mark threshold */ - ""}; - -struct flow_state { - int rel_id; - int peer_rel_id; - tw_lpid peer_gid; - int interval_id; - double ingress_packets_current; - double source_queue_packets; - flow_feedback previous_feedback; - - double total_raw_packets; - double total_granted_packets; - double total_dropped_packets; - double total_delivered_packets; - -}; - -struct flow_msg { - int event_type; - int src_rel_id; - int dst_rel_id; - int interval_id; - double packets; - model_net_event_return ret; -}; - -enum flow_event_type { - FLOW_INTERVAL = 1, - FLOW_DELIVER = 2, -}; - -static void flow_init(flow_state* ns, tw_lp* lp); -static void flow_event(flow_state* ns, tw_bf* b, flow_msg* m, tw_lp* lp); -static void flow_rev_event(flow_state* ns, tw_bf* b, flow_msg* m, tw_lp* lp); -static void flow_finalize(flow_state* ns, tw_lp* lp); - -static tw_lptype flow_lp = { - (init_f)flow_init, - (pre_run_f)NULL, - (event_f)flow_event, - (revent_f)flow_rev_event, - (commit_f)NULL, - (final_f)flow_finalize, - (map_f)codes_mapping, - sizeof(flow_state), -}; - -static const tw_lptype* flow_get_lp_type(void) { return &flow_lp; } - -static void flow_add_lp_type(void) { lp_type_register("nw-lp", flow_get_lp_type()); } - -static double seconds_to_ns(double seconds) { return seconds * 1000.0 * 1000.0 * 1000.0; } - -static double ns_to_seconds(double ns) { return ns / (1000.0 * 1000.0 * 1000.0); } - -static double clamp_packets(double value) { - if (!std::isfinite(value) || value < 0.0) { - return 0.0; - } - return value; -} - -static flow_feedback default_feedback(void) { - flow_feedback fb; - std::memset(&fb, 0, sizeof(fb)); - fb.grant_ratio = 1.0; - return fb; -} - -static void read_int(const char* key, int* value) { - int tmp = 0; - if (configuration_get_value_int(&config, "FLOW_CONTROL", key, NULL, &tmp) == 0) { - *value = tmp; - } -} - -static void read_double(const char* key, double* value) { - double tmp = 0.0; - if (configuration_get_value_double(&config, "FLOW_CONTROL", key, NULL, &tmp) == 0) { - *value = tmp; - } -} - -static void read_string(const char* key, char* value, size_t len) { - char tmp[1024]; - std::memset(tmp, 0, sizeof(tmp)); - if (configuration_get_value(&config, "FLOW_CONTROL", key, NULL, tmp, sizeof(tmp)) > 0) { - std::snprintf(value, len, "%s", tmp); - } -} - -static void load_flow_config(void) { - read_int("inferencing_enabled", &cfg.inferencing_enabled); - read_int("training_enabled", &cfg.training_enabled); - read_int("debug_prints", &cfg.debug_prints); - read_double("interval_seconds", &cfg.interval_seconds); - read_int("num_intervals", &cfg.num_intervals); - read_int("packet_size_bytes", &cfg.packet_size_bytes); - read_double("bandwidth_0_to_1_mbps", &cfg.bandwidth_0_to_1_mbps); - read_double("bandwidth_1_to_0_mbps", &cfg.bandwidth_1_to_0_mbps); - read_double("queue_capacity_0_to_1_packets", &cfg.queue_capacity_0_to_1_packets); - read_double("queue_capacity_1_to_0_packets", &cfg.queue_capacity_1_to_0_packets); - read_double("base_0_to_1_packets", &cfg.base_0_to_1_packets); - read_double("base_1_to_0_packets", &cfg.base_1_to_0_packets); - read_double("burst_0_to_1_packets", &cfg.burst_0_to_1_packets); - read_double("burst_1_to_0_packets", &cfg.burst_1_to_0_packets); - read_int("burst_period", &cfg.burst_period); - read_double("ingress_gain", &cfg.ingress_gain); - read_double("mark_threshold_ratio", &cfg.mark_threshold_ratio); - read_string("flow_log_path", cfg.flow_log_path, sizeof(cfg.flow_log_path)); - - if (cfg.interval_seconds <= 0.0) { - tw_error(TW_LOC, "FLOW_CONTROL.interval_seconds must be positive"); - } - if (cfg.num_intervals <= 0) { - tw_error(TW_LOC, "FLOW_CONTROL.num_intervals must be positive"); - } - if (cfg.packet_size_bytes <= 0) { - tw_error(TW_LOC, "FLOW_CONTROL.packet_size_bytes must be positive"); - } - if (cfg.burst_period <= 0) { - cfg.burst_period = 1; - } -} - - -static void push_zmqml_debug_setting(void) { - if (!cfg.training_enabled && !cfg.inferencing_enabled) { - return; - } - - std::vector args; - args.push_back("1"); - args.push_back(cfg.debug_prints ? "1" : "0"); - - std::vector reply = zmqml_request("set-debug", args); - if (reply.empty() || reply[0] != "done") { - std::fprintf(stderr, - "[flow-control simplep2p] warning: failed to propagate debug_prints=%d to zmqmlserver\n", - cfg.debug_prints); - } -} - -static double bandwidth_for_flow(int src_rel, int dst_rel) { - if (src_rel == 0 && dst_rel == 1) { - return cfg.bandwidth_0_to_1_mbps; - } - if (src_rel == 1 && dst_rel == 0) { - return cfg.bandwidth_1_to_0_mbps; - } - return 0.0; -} - -static double queue_capacity_for_flow(int src_rel, int dst_rel) { - if (src_rel == 0 && dst_rel == 1) { - return cfg.queue_capacity_0_to_1_packets; - } - if (src_rel == 1 && dst_rel == 0) { - return cfg.queue_capacity_1_to_0_packets; - } - return 0.0; -} - -static double interval_capacity_packets(int src_rel, int dst_rel) { - double bw_mbps = bandwidth_for_flow(src_rel, dst_rel); - double bytes = bw_mbps * 1000.0 * 1000.0 / 8.0 * cfg.interval_seconds; - return clamp_packets(std::floor(bytes / (double)cfg.packet_size_bytes)); -} - -static std::string flow_key(int src_rel, int dst_rel) { - std::ostringstream os; - os << src_rel << "->" << dst_rel; - return os.str(); -} - -static std::string json_escape(const std::string& in) { - std::ostringstream os; - for (char c : in) { - switch (c) { - case '\\': os << "\\\\"; break; - case '"': os << "\\\""; break; - case '\n': os << "\\n"; break; - case '\r': os << "\\r"; break; - case '\t': os << "\\t"; break; - default: os << c; break; - } - } - return os.str(); -} - -static std::string build_flow_control_payload(const flow_state* ns) { - const std::string in_key = flow_key(ns->peer_rel_id, ns->rel_id); - const std::string out_key = flow_key(ns->rel_id, ns->peer_rel_id); - const flow_feedback& fb = ns->previous_feedback; - - std::ostringstream os; - os << std::setprecision(17); - os << "{"; - os << "\"lp_id\":" << ns->rel_id << ","; - os << "\"interval_id\":" << ns->interval_id << ","; - os << "\"interval_seconds\":" << cfg.interval_seconds << ","; - os << "\"incoming_flows\":{"; - os << "\"" << json_escape(in_key) << "\":{\"packets\":" - << ns->ingress_packets_current << "}"; - os << "},"; - os << "\"outgoing_feedback\":{"; - os << "\"" << json_escape(out_key) << "\":{"; - os << "\"raw_predicted_packets\":" << fb.raw_predicted_packets << ","; - os << "\"available_packets\":" << fb.available_packets << ","; - os << "\"granted_packets\":" << fb.granted_packets << ","; - os << "\"dropped_packets\":" << fb.dropped_packets << ","; - os << "\"grant_ratio\":" << fb.grant_ratio << ","; - os << "\"queue_packets_next\":" << fb.queue_packets_next << ","; - os << "\"queue_occupancy_ratio\":" << fb.queue_occupancy_ratio << ","; - os << "\"queue_delay_estimate\":" << fb.queue_delay_estimate << ","; - os << "\"capacity_packets\":" << fb.capacity_packets << ","; - os << "\"congestion_mark_flag\":" << fb.congestion_mark_flag << ","; - os << "\"overflow_flag\":" << fb.overflow_flag; - os << "}"; - os << "},"; - os << "\"outgoing_flow_keys\":[\"" << json_escape(out_key) << "\"]"; - os << "}"; - return os.str(); -} - -static double parse_prediction_for_flow(const std::string& predictions, - const std::string& out_key, - double fallback) { - std::istringstream is(predictions); - std::string token; - const std::string prefix = out_key + ":"; - while (is >> token) { - if (token.compare(0, prefix.size(), prefix) == 0) { - char* end = NULL; - const char* start = token.c_str() + prefix.size(); - double value = std::strtod(start, &end); - if (end != start) { - return clamp_packets(value); - } - } - } - return clamp_packets(fallback); -} - -static double pure_pdes_raw_packets(const flow_state* ns) { - const bool fwd = ns->rel_id == 0 && ns->peer_rel_id == 1; - const double base = fwd ? cfg.base_0_to_1_packets : cfg.base_1_to_0_packets; - const double burst = fwd ? cfg.burst_0_to_1_packets : cfg.burst_1_to_0_packets; - const int phase = fwd ? 0 : 2; - const bool burst_now = ((ns->interval_id + phase) % cfg.burst_period) == 0; - const double ramp = 0.03 * base * (double)(ns->interval_id % cfg.burst_period); - const double response = cfg.ingress_gain * ns->ingress_packets_current; - return clamp_packets(base + ramp + (burst_now ? burst : 0.0) + response); -} - -static flow_feedback apply_flow_control(double raw_predicted_packets, double queued_packets, - int src_rel, int dst_rel) { - flow_feedback fb = default_feedback(); - const double capacity = interval_capacity_packets(src_rel, dst_rel); - const double queue_cap = queue_capacity_for_flow(src_rel, dst_rel); - const double available = queued_packets + raw_predicted_packets; - const double granted = std::min(available, capacity); - const double unsent = std::max(0.0, available - granted); - const double queue_next = std::min(unsent, queue_cap); - const double dropped = std::max(0.0, unsent - queue_cap); - const double capacity_per_second = cfg.interval_seconds > 0.0 ? capacity / cfg.interval_seconds : 0.0; - - fb.raw_predicted_packets = raw_predicted_packets; - fb.available_packets = available; - fb.granted_packets = granted; - fb.dropped_packets = dropped; - fb.grant_ratio = available > 0.0 ? granted / available : 1.0; - fb.queue_packets_next = queue_next; - fb.queue_occupancy_ratio = queue_cap > 0.0 ? queue_next / queue_cap : 0.0; - fb.queue_delay_estimate = capacity_per_second > 0.0 ? queue_next / capacity_per_second : 0.0; - fb.capacity_packets = capacity; - fb.congestion_mark_flag = fb.queue_occupancy_ratio >= cfg.mark_threshold_ratio ? 1 : 0; - fb.overflow_flag = dropped > 0.0 ? 1 : 0; - return fb; -} - -static void send_training_record(flow_state* ns, double raw_packets) { - const std::string in_key = flow_key(ns->peer_rel_id, ns->rel_id); - const std::string out_key = flow_key(ns->rel_id, ns->peer_rel_id); - - std::ostringstream record; - record << "schema_version,lp_id,interval_id,interval_seconds,incoming_flow_key," - << "incoming_packets,outgoing_flow_key,raw_egress_packets,prev_grant_ratio," - << "prev_queue_occupancy_ratio,prev_dropped_packets,prev_capacity_packets\n"; - record << std::setprecision(17) - << 1 << ',' << ns->rel_id << ',' << ns->interval_id << ',' - << cfg.interval_seconds << ',' << in_key << ',' - << ns->ingress_packets_current << ',' << out_key << ',' << raw_packets << ',' - << ns->previous_feedback.grant_ratio << ',' - << ns->previous_feedback.queue_occupancy_ratio << ',' - << ns->previous_feedback.dropped_packets << ',' - << ns->previous_feedback.capacity_packets << '\n'; - - std::vector reply = zmqml_director_request( - "flow-control", "simplep2p", "send-records", std::vector(), record.str()); - if (reply.empty() || reply[0] != "done") { - std::fprintf(stderr, - "[flow-control simplep2p] warning: failed to send training record for lp %d interval %d\n", - ns->rel_id, ns->interval_id); - } -} - -static void append_flow_log(const flow_state* ns, const flow_feedback& fb, double raw_packets, - const char* source, tw_lp* lp) { - if (cfg.flow_log_path[0] == '\0') { - return; - } - - const bool new_file = std::ifstream(cfg.flow_log_path).good() == false; - std::ofstream out(cfg.flow_log_path, std::ios::app); - if (!out) { - if (cfg.debug_prints) { - std::fprintf(stderr, "[flow-control simplep2p] could not write %s\n", cfg.flow_log_path); - } - return; - } - if (new_file) { - out << "lp_gid,lp_rel,peer_rel,interval_id,sim_time_seconds,source,incoming_packets," - << "raw_predicted_packets,queue_before_packets,available_packets,capacity_packets," - << "granted_packets,queue_after_packets,dropped_packets,grant_ratio," - << "queue_occupancy_ratio,queue_delay_estimate,congestion_mark_flag,overflow_flag\n"; - } - out << std::setprecision(17) - << (unsigned long long)lp->gid << ',' << ns->rel_id << ',' << ns->peer_rel_id << ',' - << ns->interval_id << ',' << ns_to_seconds(tw_now(lp)) << ',' << source << ',' - << ns->ingress_packets_current << ',' << raw_packets << ',' << ns->source_queue_packets << ',' - << fb.available_packets << ',' << fb.capacity_packets << ',' << fb.granted_packets << ',' - << fb.queue_packets_next << ',' << fb.dropped_packets << ',' << fb.grant_ratio << ',' - << fb.queue_occupancy_ratio << ',' << fb.queue_delay_estimate << ',' - << fb.congestion_mark_flag << ',' << fb.overflow_flag << '\n'; -} - -static double infer_raw_packets(flow_state* ns) { - const std::string out_key = flow_key(ns->rel_id, ns->peer_rel_id); - const double fallback = pure_pdes_raw_packets(ns); - const std::string payload = build_flow_control_payload(ns); - - std::vector reply = zmqml_director_request( - "flow-control", "simplep2p", "inference", std::vector(), payload); - - if (reply.empty() || reply[0] != "done") { - std::fprintf(stderr, - "[flow-control simplep2p] warning: inference failed for flow %s interval %d; using fallback %f\n", - out_key.c_str(), ns->interval_id, fallback); - return fallback; - } - - std::string predictions; - if (reply.size() >= 4) { - predictions = reply[3]; - } else if (reply.size() >= 3) { - predictions = reply[2]; - } - return parse_prediction_for_flow(predictions, out_key, fallback); -} - -static void send_granted_packets(flow_state* ns, flow_msg* m, tw_lp* lp, double granted_packets) { - if (granted_packets <= 0.0) { - return; - } - - const double bytes_double = granted_packets * (double)cfg.packet_size_bytes; - uint64_t message_size = 0; - if (bytes_double >= (double)std::numeric_limits::max()) { - message_size = std::numeric_limits::max(); - } else { - message_size = (uint64_t)std::ceil(bytes_double); - } - if (message_size == 0) { - return; - } - - flow_msg remote; - std::memset(&remote, 0, sizeof(remote)); - remote.event_type = FLOW_DELIVER; - remote.src_rel_id = ns->rel_id; - remote.dst_rel_id = ns->peer_rel_id; - remote.interval_id = ns->interval_id; - remote.packets = granted_packets; - - m->ret = model_net_event(net_id, "flow-control", ns->peer_gid, message_size, 0.0, - sizeof(remote), &remote, 0, NULL, lp); -} - -static void schedule_next_interval(flow_state* ns, tw_lp* lp) { - if (ns->interval_id >= cfg.num_intervals) { - return; - } - tw_event* e = tw_event_new(lp->gid, seconds_to_ns(cfg.interval_seconds), lp); - flow_msg* m = (flow_msg*)tw_event_data(e); - std::memset(m, 0, sizeof(*m)); - m->event_type = FLOW_INTERVAL; - tw_event_send(e); -} - -static void handle_interval(flow_state* ns, flow_msg* m, tw_lp* lp) { - double raw_packets = 0.0; - const char* source = "pure-pdes"; - - if (cfg.inferencing_enabled) { - raw_packets = infer_raw_packets(ns); - source = "flow-control-ml"; - } else { - raw_packets = pure_pdes_raw_packets(ns); - } - - raw_packets = clamp_packets(raw_packets); - - if (cfg.training_enabled) { - send_training_record(ns, raw_packets); - } - - flow_feedback fb = apply_flow_control(raw_packets, ns->source_queue_packets, - ns->rel_id, ns->peer_rel_id); - append_flow_log(ns, fb, raw_packets, source, lp); - - if (cfg.debug_prints && - (fb.grant_ratio < 0.999999 || fb.dropped_packets > 0.0 || fb.congestion_mark_flag)) { - std::printf("[flow-control throttle] lp=%llu flow=%d->%d interval=%d source=%s " - "raw=%.6f queue_before=%.6f available=%.6f capacity=%.6f " - "granted=%.6f queue_after=%.6f dropped=%.6f grant_ratio=%.6f " - "queue_occ=%.6f mark=%d overflow=%d\n", - (unsigned long long)lp->gid, ns->rel_id, ns->peer_rel_id, - ns->interval_id, source, raw_packets, ns->source_queue_packets, - fb.available_packets, fb.capacity_packets, fb.granted_packets, - fb.queue_packets_next, fb.dropped_packets, fb.grant_ratio, - fb.queue_occupancy_ratio, fb.congestion_mark_flag, fb.overflow_flag); - std::fflush(stdout); - } - - send_granted_packets(ns, m, lp, fb.granted_packets); - - ns->source_queue_packets = fb.queue_packets_next; - ns->previous_feedback = fb; - ns->total_raw_packets += raw_packets; - ns->total_granted_packets += fb.granted_packets; - ns->total_dropped_packets += fb.dropped_packets; - ns->ingress_packets_current = 0.0; - ns->interval_id++; - - schedule_next_interval(ns, lp); -} - -static void handle_deliver(flow_state* ns, const flow_msg* m) { - ns->ingress_packets_current += m->packets; - ns->total_delivered_packets += m->packets; -} - -static void flow_init(flow_state* ns, tw_lp* lp) { - std::memset(ns, 0, sizeof(*ns)); - ns->rel_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); - ns->peer_rel_id = 1 - ns->rel_id; - ns->previous_feedback = default_feedback(); - - codes_mapping_get_lp_id("MODELNET_GRP", "nw-lp", NULL, 1, ns->peer_rel_id, 0, - &ns->peer_gid); - - tw_event* e = tw_event_new(lp->gid, g_tw_lookahead + tw_rand_unif(lp->rng), lp); - flow_msg* m = (flow_msg*)tw_event_data(e); - std::memset(m, 0, sizeof(*m)); - m->event_type = FLOW_INTERVAL; - tw_event_send(e); -} - -static void flow_event(flow_state* ns, tw_bf* b, flow_msg* m, tw_lp* lp) { - (void)b; - switch (m->event_type) { - case FLOW_INTERVAL: - handle_interval(ns, m, lp); - break; - case FLOW_DELIVER: - handle_deliver(ns, m); - break; - default: - tw_error(TW_LOC, "unknown flow-control event type %d", m->event_type); - } -} - -static void flow_rev_event(flow_state* ns, tw_bf* b, flow_msg* m, tw_lp* lp) { - (void)ns; - (void)b; - (void)m; - (void)lp; - tw_error(TW_LOC, "model-net-flow-control-simplep2p is intended for sequential runs first"); -} - -static void flow_finalize(flow_state* ns, tw_lp* lp) { - std::printf("flow-control-simplep2p lp=%llu rel=%d peer=%d intervals=%d raw=%f granted=%f " - "delivered=%f dropped=%f queue=%f\n", - (unsigned long long)lp->gid, ns->rel_id, ns->peer_rel_id, ns->interval_id, - ns->total_raw_packets, ns->total_granted_packets, ns->total_delivered_packets, - ns->total_dropped_packets, ns->source_queue_packets); -} - -const tw_optdef app_opt[] = {TWOPT_GROUP("model-net flow-control simplep2p"), TWOPT_END()}; - -static const char* find_config_arg(int argc, char** argv) { - for (int i = argc - 1; i >= 1; --i) { - if (argv[i] == NULL) { - continue; - } - const char* arg = argv[i]; - const size_t len = std::strlen(arg); - if (len >= 5 && std::strcmp(arg + len - 5, ".conf") == 0) { - return arg; - } - } - return NULL; -} - -int main(int argc, char** argv) { - int rank = 0; - int num_nets = 0; - int* net_ids = NULL; - - g_tw_ts_end = seconds_to_ns(60.0 * 60.0 * 24.0 * 365.0); - - tw_opt_add(app_opt); - tw_init(&argc, &argv); - - const char* config_file = find_config_arg(argc, argv); - if (config_file == NULL) { - std::printf("Usage: mpirun -np %s --sync=1 -- \n", argv[0]); - std::printf(" or: mpirun -np %s --synch=1 -- \n", argv[0]); - MPI_Finalize(); - return 1; - } - - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - configuration_load(config_file, MPI_COMM_WORLD, &config); - load_flow_config(); - - if (rank == 0) { - push_zmqml_debug_setting(); - } - MPI_Barrier(MPI_COMM_WORLD); - - flow_add_lp_type(); - model_net_register(); - codes_mapping_setup(); - - net_ids = model_net_configure(&num_nets); - assert(num_nets == 1); - net_id = net_ids[0]; - free(net_ids); - - num_servers = codes_mapping_get_lp_count("MODELNET_GRP", 0, "nw-lp", NULL, 1); - if (num_servers != 2) { - tw_error(TW_LOC, "model-net-flow-control-simplep2p expects exactly 2 nw-lp LPs"); - } - - if (rank == 0 && cfg.flow_log_path[0] != '\0') { - std::remove(cfg.flow_log_path); - } - MPI_Barrier(MPI_COMM_WORLD); - - if (rank == 0 && cfg.debug_prints) { - std::printf("flow-control config: inferencing=%d training=%d intervals=%d interval_seconds=%f\n", - cfg.inferencing_enabled, cfg.training_enabled, cfg.num_intervals, - cfg.interval_seconds); - } - - tw_run(); - model_net_report_stats(net_id); - tw_end(); - return 0; -} diff --git a/src/surrogate/zmqml/model/mlflowcontrol.py b/src/surrogate/zmqml/model/mlflowcontrol.py deleted file mode 100644 index 77966d68..00000000 --- a/src/surrogate/zmqml/model/mlflowcontrol.py +++ /dev/null @@ -1,512 +0,0 @@ -#!/usr/bin/env python3 -"""Per-terminal interval-flow surrogate family for the ZeroMQ Director. - -The server exposes one surrogate family name, ``flow-control``, but this module -keeps one trainable model per terminal/LP id. Each terminal model learns only -from records whose ``lp_id`` matches that terminal, and inference for an LP uses -only that LP's model. - -The model predicts raw per-flow offered load. CODES/PDES remains responsible -for source queue capacity, grant capacity, drops, routing, and delivery. -""" - -from __future__ import annotations - -import csv -import io -import json -import math -import os -import pickle -from pathlib import Path -from typing import Any - -import numpy as np - - -FLOW_RECORD_FIELDS = [ - "schema_version", - "lp_id", - "interval_id", - "interval_seconds", - "incoming_flow_key", - "incoming_packets", - "outgoing_flow_key", - "raw_egress_packets", - "prev_grant_ratio", - "prev_queue_occupancy_ratio", - "prev_dropped_packets", - "prev_capacity_packets", -] - - -def _finite_float(value: Any, default: float = 0.0) -> float: - try: - out = float(value) - except Exception: - return default - if not math.isfinite(out): - return default - return out - - -def _lp_key(value: Any) -> str: - """Normalize LP ids so CSV strings, ints, and JSON numbers match.""" - text = str(value).strip() - if text == "": - return "unknown" - try: - num = float(text) - if math.isfinite(num) and num.is_integer(): - return str(int(num)) - except Exception: - pass - return text - - -def _flow_key_reverse(flow_key: str) -> str | None: - parts = str(flow_key).split("->", 1) - if len(parts) != 2: - return None - left = parts[0].strip() - right = parts[1].strip() - if not left or not right: - return None - return f"{right}->{left}" - - -def _packets_from_flow_entry(entry: Any) -> float: - if isinstance(entry, dict): - return max(0.0, _finite_float(entry.get("packets", 0.0), 0.0)) - return max(0.0, _finite_float(entry, 0.0)) - - -class TerminalFlowControlModel: - """One ML model owned by one terminal/LP. - - For this first prototype, each terminal model is a small ridge-regression - model per outgoing flow. This still gives the desired ownership boundary: - records and inference are isolated by LP id. Extending this to one - multi-output model per LP later only changes this class, not the server API. - """ - - def __init__( - self, - lp_id: str, - *, - ridge_alpha: float, - min_rows_per_flow: int, - default_prediction_packets: float, - max_prediction_packets: float, - ) -> None: - self.lp_id = _lp_key(lp_id) - self.ridge_alpha = max(0.0, float(ridge_alpha)) - self.min_rows_per_flow = max(1, int(min_rows_per_flow)) - self.default_prediction_packets = max(0.0, float(default_prediction_packets)) - self.max_prediction_packets = max(0.0, float(max_prediction_packets)) - - self.rows: list[dict[str, Any]] = [] - self.coefficients: dict[str, np.ndarray] = {} - self.flow_means: dict[str, float] = {} - self.num_requests = 0 - self.last_predictions: dict[str, float] = {} - - @staticmethod - def _features( - *, - incoming_packets: float, - grant_ratio: float, - queue_occupancy_ratio: float, - dropped_packets: float, - capacity_packets: float, - interval_id: float, - ) -> np.ndarray: - return np.asarray( - [ - 1.0, - max(0.0, incoming_packets), - min(1.0, max(0.0, grant_ratio)), - min(1.0, max(0.0, queue_occupancy_ratio)), - math.log1p(max(0.0, dropped_packets)), - math.log1p(max(0.0, capacity_packets)), - float(interval_id), - ], - dtype=np.float64, - ) - - @classmethod - def _row_features(cls, row: dict[str, Any]) -> np.ndarray: - return cls._features( - incoming_packets=_finite_float(row.get("incoming_packets"), 0.0), - grant_ratio=_finite_float(row.get("prev_grant_ratio"), 1.0), - queue_occupancy_ratio=_finite_float(row.get("prev_queue_occupancy_ratio"), 0.0), - dropped_packets=_finite_float(row.get("prev_dropped_packets"), 0.0), - capacity_packets=_finite_float(row.get("prev_capacity_packets"), 0.0), - interval_id=_finite_float(row.get("interval_id"), 0.0), - ) - - @staticmethod - def _target(row: dict[str, Any]) -> float: - return max(0.0, _finite_float(row.get("raw_egress_packets"), 0.0)) - - def add_row(self, row: dict[str, Any]) -> None: - self.rows.append(row) - - def train_or_update(self) -> bool: - by_flow: dict[str, list[dict[str, Any]]] = {} - for row in self.rows: - flow = str(row.get("outgoing_flow_key", "")).strip() - if flow: - by_flow.setdefault(flow, []).append(row) - - trained_any = False - for flow, rows in sorted(by_flow.items()): - if len(rows) < self.min_rows_per_flow: - continue - - x = np.vstack([self._row_features(row) for row in rows]) - y = np.asarray([self._target(row) for row in rows], dtype=np.float64) - self.flow_means[flow] = float(np.mean(y)) if len(y) else self.default_prediction_packets - - eye = np.eye(x.shape[1], dtype=np.float64) - eye[0, 0] = 0.0 # Do not penalize intercept. - try: - beta = np.linalg.solve(x.T @ x + self.ridge_alpha * eye, x.T @ y) - except np.linalg.LinAlgError: - beta = np.linalg.pinv(x) @ y - - self.coefficients[flow] = beta.astype(np.float64) - trained_any = True - - return trained_any - - def _predict_one(self, flow_key: str, features: np.ndarray) -> float: - if flow_key in self.coefficients: - raw = float(features @ self.coefficients[flow_key]) - elif flow_key in self.flow_means: - raw = float(self.flow_means[flow_key]) - else: - raw = self.default_prediction_packets - - raw = max(0.0, raw) - if self.max_prediction_packets > 0.0: - raw = min(raw, self.max_prediction_packets) - return raw - - def predict(self, payload: dict[str, Any]) -> dict[str, float]: - incoming = payload.get("incoming_flows", {}) or {} - outgoing_feedback = payload.get("outgoing_feedback", {}) or {} - outgoing_keys = payload.get("outgoing_flow_keys", []) or [] - interval_id = _finite_float(payload.get("interval_id"), 0.0) - - predictions: dict[str, float] = {} - for key in outgoing_keys: - flow_key = str(key).strip() - if not flow_key: - continue - - reverse_key = _flow_key_reverse(flow_key) - incoming_packets = 0.0 - if reverse_key is not None and reverse_key in incoming: - incoming_packets = _packets_from_flow_entry(incoming[reverse_key]) - elif incoming: - incoming_packets = sum(_packets_from_flow_entry(v) for v in incoming.values()) - - feedback = outgoing_feedback.get(flow_key, {}) or {} - if not isinstance(feedback, dict): - feedback = {} - - features = self._features( - incoming_packets=incoming_packets, - grant_ratio=_finite_float(feedback.get("grant_ratio"), 1.0), - queue_occupancy_ratio=_finite_float(feedback.get("queue_occupancy_ratio"), 0.0), - dropped_packets=_finite_float(feedback.get("dropped_packets"), 0.0), - capacity_packets=_finite_float(feedback.get("capacity_packets"), 0.0), - interval_id=interval_id, - ) - predictions[flow_key] = self._predict_one(flow_key, features) - - self.num_requests += 1 - self.last_predictions = dict(predictions) - return predictions - - def to_payload(self) -> dict[str, Any]: - return { - "lp_id": self.lp_id, - "rows": self.rows, - "coefficients": self.coefficients, - "flow_means": self.flow_means, - "num_requests": self.num_requests, - "last_predictions": self.last_predictions, - } - - @classmethod - def from_payload( - cls, - payload: dict[str, Any], - *, - ridge_alpha: float, - min_rows_per_flow: int, - default_prediction_packets: float, - max_prediction_packets: float, - ) -> "TerminalFlowControlModel": - model = cls( - _lp_key(payload.get("lp_id", "unknown")), - ridge_alpha=ridge_alpha, - min_rows_per_flow=min_rows_per_flow, - default_prediction_packets=default_prediction_packets, - max_prediction_packets=max_prediction_packets, - ) - model.rows = list(payload.get("rows", [])) - model.coefficients = { - str(k): np.asarray(v, dtype=np.float64) - for k, v in dict(payload.get("coefficients", {})).items() - } - model.flow_means = { - str(k): float(v) for k, v in dict(payload.get("flow_means", {})).items() - } - model.num_requests = int(payload.get("num_requests", 0)) - model.last_predictions = { - str(k): float(v) for k, v in dict(payload.get("last_predictions", {})).items() - } - return model - - def status(self) -> dict[str, Any]: - flows = sorted(set(self.flow_means) | set(self.coefficients)) - return { - "lp_id": self.lp_id, - "rows": len(self.rows), - "trained": bool(self.coefficients), - "trained_flows": len(self.coefficients), - "flows": flows, - "requests": self.num_requests, - } - - -class FlowControlSurrogateFamily: - """Server-side family registry with one terminal model per LP id.""" - - def __init__( - self, - *, - ridge_alpha: float = 1.0, - min_rows_per_flow: int = 1, - default_prediction_packets: float = 1024.0, - max_prediction_packets: float = 0.0, - ) -> None: - self.ridge_alpha = max(0.0, float(ridge_alpha)) - self.min_rows_per_flow = max(1, int(min_rows_per_flow)) - self.default_prediction_packets = max(0.0, float(default_prediction_packets)) - self.max_prediction_packets = max(0.0, float(max_prediction_packets)) - self.debug = False - - self.terminal_models: dict[str, TerminalFlowControlModel] = {} - self.model_version = 0 - self.num_requests = 0 - - def set_debug(self, enabled: bool) -> None: - self.debug = bool(enabled) - - def _new_terminal_model(self, lp_id: str) -> TerminalFlowControlModel: - return TerminalFlowControlModel( - lp_id, - ridge_alpha=self.ridge_alpha, - min_rows_per_flow=self.min_rows_per_flow, - default_prediction_packets=self.default_prediction_packets, - max_prediction_packets=self.max_prediction_packets, - ) - - def _model_for_lp(self, lp_id: Any) -> TerminalFlowControlModel: - key = _lp_key(lp_id) - model = self.terminal_models.get(key) - if model is None: - model = self._new_terminal_model(key) - self.terminal_models[key] = model - return model - - @staticmethod - def _normalize_record_row(row: dict[str, Any]) -> dict[str, Any] | None: - flow = str(row.get("outgoing_flow_key", "")).strip() - if not flow: - return None - out = {field: row.get(field, "") for field in FLOW_RECORD_FIELDS} - out["lp_id"] = _lp_key(out.get("lp_id", "unknown")) - out["outgoing_flow_key"] = flow - return out - - def add_records_text(self, payload_text: str) -> int: - text = (payload_text or "").strip() - if not text: - return 0 - - loaded = 0 - if text.lstrip().startswith("{"): - for line in text.splitlines(): - line = line.strip() - if not line: - continue - row = json.loads(line) - if not isinstance(row, dict): - continue - normalized = self._normalize_record_row(row) - if normalized is None: - continue - self._model_for_lp(normalized["lp_id"]).add_row(normalized) - loaded += 1 - return loaded - - reader = csv.DictReader(io.StringIO(text)) - for row in reader: - normalized = self._normalize_record_row(row) - if normalized is None: - continue - self._model_for_lp(normalized["lp_id"]).add_row(normalized) - loaded += 1 - return loaded - - def load_records_csv(self, path: str | Path) -> int: - path = Path(path) - loaded = 0 - files = sorted(path.rglob("*.csv")) if path.is_dir() else [path] - for child in files: - if not child.is_file(): - continue - loaded += self.add_records_text(child.read_text()) - return loaded - - def train_or_update(self) -> bool: - trained_any = False - trained_lps: list[str] = [] - for lp_id, model in sorted(self.terminal_models.items()): - if model.train_or_update(): - trained_any = True - trained_lps.append(lp_id) - - if trained_any: - self.model_version += 1 - - if self.debug: - print( - f"[flow-control train] terminal_models={len(self.terminal_models)} " - f"trained_lps={trained_lps}", - flush=True, - ) - return trained_any - - def predict(self, payload: dict[str, Any]) -> dict[str, float]: - lp_id = _lp_key(payload.get("lp_id", "unknown")) - model = self._model_for_lp(lp_id) - predictions = model.predict(payload) - self.num_requests += 1 - - if self.debug: - trained = bool(model.coefficients) - print( - f"[flow-control inference] lp={lp_id} trained={int(trained)} " - f"interval={payload.get('interval_id')} predictions={predictions}", - flush=True, - ) - return predictions - - def predict_from_text(self, payload_text: str) -> dict[str, float]: - payload_text = (payload_text or "").strip() - if not payload_text: - raise ValueError("missing flow-control JSON payload") - payload = json.loads(payload_text) - if not isinstance(payload, dict): - raise ValueError("flow-control payload must be a JSON object") - return self.predict(payload) - - def save(self, path: str | Path) -> None: - path = Path(path) - if path.parent: - path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "format": "flow-control-per-terminal-linear-v1", - "ridge_alpha": self.ridge_alpha, - "min_rows_per_flow": self.min_rows_per_flow, - "default_prediction_packets": self.default_prediction_packets, - "max_prediction_packets": self.max_prediction_packets, - "terminal_models": { - lp_id: model.to_payload() for lp_id, model in sorted(self.terminal_models.items()) - }, - "model_version": self.model_version, - "num_requests": self.num_requests, - } - with path.open("wb") as f: - pickle.dump(payload, f) - - def load(self, path: str | Path) -> None: - with Path(path).open("rb") as f: - payload = pickle.load(f) - - self.ridge_alpha = float(payload.get("ridge_alpha", self.ridge_alpha)) - self.min_rows_per_flow = int(payload.get("min_rows_per_flow", self.min_rows_per_flow)) - self.default_prediction_packets = float( - payload.get("default_prediction_packets", self.default_prediction_packets) - ) - self.max_prediction_packets = float(payload.get("max_prediction_packets", self.max_prediction_packets)) - self.model_version = int(payload.get("model_version", self.model_version)) - self.num_requests = int(payload.get("num_requests", self.num_requests)) - - # New format: one serialized model per LP. - if "terminal_models" in payload: - self.terminal_models = { - _lp_key(lp_id): TerminalFlowControlModel.from_payload( - model_payload, - ridge_alpha=self.ridge_alpha, - min_rows_per_flow=self.min_rows_per_flow, - default_prediction_packets=self.default_prediction_packets, - max_prediction_packets=self.max_prediction_packets, - ) - for lp_id, model_payload in dict(payload.get("terminal_models", {})).items() - } - return - - # Backward compatibility with the earlier one-object/per-flow prototype. - legacy_rows = list(payload.get("rows", [])) - self.terminal_models = {} - for row in legacy_rows: - normalized = self._normalize_record_row(row) - if normalized is not None: - self._model_for_lp(normalized["lp_id"]).add_row(normalized) - self.train_or_update() - - def status(self) -> dict[str, str]: - lp_status = {lp_id: model.status() for lp_id, model in sorted(self.terminal_models.items())} - trained_lps = [lp_id for lp_id, st in lp_status.items() if st["trained"]] - trained_flow_labels: list[str] = [] - for lp_id, st in lp_status.items(): - for flow in st["flows"]: - trained_flow_labels.append(f"{lp_id}:{flow}") - - total_rows = sum(int(st["rows"]) for st in lp_status.values()) - total_requests = sum(int(st["requests"]) for st in lp_status.values()) - - return { - "model_type": "per-terminal-linear-flow-control", - "trained": "1" if trained_lps else "0", - "rows": str(total_rows), - "terminal_models": str(len(self.terminal_models)), - "trained_terminal_models": str(len(trained_lps)), - "trained_lps": ";".join(trained_lps), - "trained_flows": str(len(trained_flow_labels)), - "flows": ";".join(sorted(trained_flow_labels)), - "requests": str(total_requests), - "family_requests": str(self.num_requests), - "model_version": str(self.model_version), - "ridge_alpha": str(self.ridge_alpha), - "min_rows_per_flow": str(self.min_rows_per_flow), - "default_prediction_packets": str(self.default_prediction_packets), - } - - -def flow_control_from_env() -> FlowControlSurrogateFamily: - return FlowControlSurrogateFamily( - ridge_alpha=float(os.environ.get("ZMQML_FLOW_CONTROL_RIDGE_ALPHA", "1.0")), - min_rows_per_flow=int(os.environ.get("ZMQML_FLOW_CONTROL_MIN_ROWS_PER_FLOW", "1")), - default_prediction_packets=float( - os.environ.get("ZMQML_FLOW_CONTROL_DEFAULT_PACKETS", "1024") - ), - max_prediction_packets=float(os.environ.get("ZMQML_FLOW_CONTROL_MAX_PACKETS", "0")), - ) From 3e354dece54349d3f9a19b3716faa910bb756701 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Wed, 8 Jul 2026 11:48:05 -0400 Subject: [PATCH 04/20] Improve fluid switch validation logs Add flowlet and switch-training logs, queued-before/drop fields, configurable switch schedulers, two-phase egress allocation, and fragment coalescing for cleaner rollback and ML training targets. Rename receive counters to received_fragments. --- doc/example/fluid-switch-topology.yaml | 12 +- doc/example/fluid-switch.conf.in | 3 + .../model-net-fluid-switch.cxx | 413 +++++++++++++++--- 3 files changed, 370 insertions(+), 58 deletions(-) diff --git a/doc/example/fluid-switch-topology.yaml b/doc/example/fluid-switch-topology.yaml index b7d21fb7..0e3040be 100644 --- a/doc/example/fluid-switch-topology.yaml +++ b/doc/example/fluid-switch-topology.yaml @@ -2,23 +2,23 @@ topology: switches: A: terminals: 2 - terminal_bandwidth: "10 Mbps" + terminal_bandwidth: "20 Mbps" switch_buffer: "1000 Mb" connections: - B: "20 Mbps" + B: "15 Mbps" B: terminals: 2 - terminal_bandwidth: "5 Mbps" + terminal_bandwidth: "25 Mbps" switch_buffer: "1000 Mb" connections: A: "15 Mbps" - C: "25 Mbps" + C: "10 Mbps" C: terminals: 2 - terminal_bandwidth: "15 Mbps" + terminal_bandwidth: "20 Mbps" switch_buffer: "1000 Mb" connections: - A: "20 Mbps" + A: "10 Mbps" B: "15 Mbps" diff --git a/doc/example/fluid-switch.conf.in b/doc/example/fluid-switch.conf.in index 0347621d..a92bc3d3 100644 --- a/doc/example/fluid-switch.conf.in +++ b/doc/example/fluid-switch.conf.in @@ -33,7 +33,10 @@ FLUID_SWITCH terminal_max_send_fraction_of_link_capacity="1.0"; debug_prints="0"; + switch_scheduler="max_min_fair"; terminal_log_path="../../zmqml_artifacts/fluid-switch/terminal-events.csv"; switch_log_path="../../zmqml_artifacts/fluid-switch/switch-events.csv"; + flowlet_log_path="../../zmqml_artifacts/fluid-switch/flowlet-events.csv"; + switch_training_log_path="../../zmqml_artifacts/fluid-switch/switch-training.csv"; } diff --git a/src/network-workloads/model-net-fluid-switch.cxx b/src/network-workloads/model-net-fluid-switch.cxx index 01efda5c..26505eb6 100644 --- a/src/network-workloads/model-net-fluid-switch.cxx +++ b/src/network-workloads/model-net-fluid-switch.cxx @@ -81,6 +81,9 @@ struct sim_config { char topology_yaml_file[1024] = ""; char terminal_log_path[1024] = ""; char switch_log_path[1024] = ""; + char flowlet_log_path[1024] = ""; + char switch_training_log_path[1024] = ""; + char switch_scheduler[64] = "max_min_fair"; }; static sim_config cfg; @@ -118,7 +121,7 @@ struct terminal_state { double sent_to_switch_mbit; double received_mbit; int generated_flowlets; - int received_flowlets; + int received_fragments; }; struct switch_state { @@ -130,7 +133,7 @@ struct switch_state { double sent_mbit; double delivered_local_mbit; double dropped_mbit; - int received_flowlets; + int received_fragments; int sent_fragments; }; @@ -503,6 +506,13 @@ static void read_double_param(const char* section, const char* key, double* valu } } +static void read_string_param(const char* section, const char* key, char* value, size_t len) { + char tmp[1024]; + memset(tmp, 0, sizeof(tmp)); + if (configuration_get_value(&config, section, key, NULL, tmp, sizeof(tmp)) > 0) { + snprintf(value, len, "%s", tmp); + } +} static void read_relpath_param(const char* section, const char* key, char* value, size_t len) { char tmp[1024]; @@ -519,6 +529,12 @@ static void load_config(void) { sizeof(cfg.terminal_log_path)); read_relpath_param("FLUID_SWITCH", "switch_log_path", cfg.switch_log_path, sizeof(cfg.switch_log_path)); + read_relpath_param("FLUID_SWITCH", "flowlet_log_path", cfg.flowlet_log_path, + sizeof(cfg.flowlet_log_path)); + read_relpath_param("FLUID_SWITCH", "switch_training_log_path", cfg.switch_training_log_path, + sizeof(cfg.switch_training_log_path)); + read_string_param("FLUID_SWITCH", "switch_scheduler", cfg.switch_scheduler, + sizeof(cfg.switch_scheduler)); read_double_param("FLUID_SWITCH", "interval_seconds", &cfg.interval_seconds); read_int_param("FLUID_SWITCH", "num_send_intervals", &cfg.num_send_intervals); read_int_param("FLUID_SWITCH", "num_drain_intervals", &cfg.num_drain_intervals); @@ -530,6 +546,13 @@ static void load_config(void) { &cfg.terminal_max_send_fraction_of_link_capacity); read_int_param("FLUID_SWITCH", "debug_prints", &cfg.debug_prints); + if (strcmp(cfg.switch_scheduler, "max_min_fair") != 0 && + strcmp(cfg.switch_scheduler, "fifo") != 0) { + tw_error(TW_LOC, + "FLUID_SWITCH.switch_scheduler must be one of: max_min_fair, fifo; got '%s'", + cfg.switch_scheduler); + } + if (cfg.topology_yaml_file[0] == '\0') { tw_error(TW_LOC, "FLUID_SWITCH.topology_yaml_file is required"); } @@ -587,10 +610,30 @@ static int find_terminal_port(const switch_state* ns, int dst_terminal) { return -1; } -static int enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m) { +static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, + double* queued_before_out, double* dropped_out, + double* flowlet_remaining_after_out, + int* coalesced_out) { + if (queued_before_out != NULL) { + *queued_before_out = 0.0; + } + if (dropped_out != NULL) { + *dropped_out = 0.0; + } + if (flowlet_remaining_after_out != NULL) { + *flowlet_remaining_after_out = 0.0; + } + if (coalesced_out != NULL) { + *coalesced_out = 0; + } + if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { - return 0; + if (dropped_out != NULL) { + *dropped_out = m->mbit; + } + return 0.0; } + std::vector& qv = *ns->queues[port_id]; double queued_on_port = 0.0; @@ -598,13 +641,61 @@ static int enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m) { queued_on_port += qv[i].remaining_mbit; } + if (queued_before_out != NULL) { + *queued_before_out = queued_on_port; + } + double accepted = m->mbit; + double dropped = 0.0; + if (queued_on_port + accepted > ns->ports[port_id].buffer_mbit + EPS) { accepted = std::max(0.0, ns->ports[port_id].buffer_mbit - queued_on_port); - ns->dropped_mbit += std::max(0.0, m->mbit - accepted); + dropped = std::max(0.0, m->mbit - accepted); + ns->dropped_mbit += dropped; + } + + if (dropped_out != NULL) { + *dropped_out = dropped; } + if (accepted <= EPS) { - return 0; + return 0.0; + } + + /* + * Fragments of the same original interval-fluid flowlet can reconverge at + * the same switch output queue. Coalesce them into one queue entry so that + * each switch-port interval has at most one mutable queue record and one + * allocation target per original flowlet_id. + * + * This makes future reverse computation simpler: + * + * - one bytes_remaining delta per flowlet + * - at most one scheduled send fragment per flowlet per egress event + * - one ML training target per flowlet per switch-port interval + */ + for (size_t i = 0; i < qv.size(); ++i) { + if (!qv[i].valid) { + continue; + } + + if (qv[i].flowlet_id == m->flowlet_id && + qv[i].source_terminal == m->source_terminal && + qv[i].destination_terminal == m->destination_terminal && + qv[i].creation_interval == m->creation_interval) { + qv[i].remaining_mbit += accepted; + qv[i].age_intervals = m->interval_id - m->creation_interval; + ns->enqueued_mbit += accepted; + + if (flowlet_remaining_after_out != NULL) { + *flowlet_remaining_after_out = qv[i].remaining_mbit; + } + if (coalesced_out != NULL) { + *coalesced_out = 1; + } + + return accepted; + } } queued_flowlet q; @@ -617,9 +708,15 @@ static int enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m) { q.enqueue_interval = m->interval_id; q.age_intervals = m->interval_id - m->creation_interval; q.remaining_mbit = accepted; + qv.push_back(q); ns->enqueued_mbit += accepted; - return 1; + + if (flowlet_remaining_after_out != NULL) { + *flowlet_remaining_after_out = accepted; + } + + return accepted; } static double queued_mbit_on_port(const switch_state* ns, int port_id) { @@ -634,6 +731,20 @@ static double queued_mbit_on_port(const switch_state* ns, int port_id) { return total; } +static int active_flowlet_count_on_port(const switch_state* ns, int port_id) { + if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { + return 0; + } + int count = 0; + const std::vector& qv = *ns->queues[port_id]; + for (size_t i = 0; i < qv.size(); ++i) { + if (qv[i].valid && qv[i].remaining_mbit > EPS) { + ++count; + } + } + return count; +} + static void compact_port_queue(switch_state* ns, int port_id) { if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { return; @@ -658,8 +769,9 @@ static void append_terminal_log(int interval_id, const char* event_name, int ter static void append_switch_log(int interval_id, const char* event_name, int switch_id, int port_id, int target_is_terminal, int target_index, double capacity_mbit, - double sent_mbit, double queued_after_mbit, double dropped_mbit, - int active_flowlets) { + double queued_before_mbit, double sent_mbit, + double queued_after_mbit, double dropped_mbit, + int active_queue_entries) { if (cfg.switch_log_path[0] == '\0') { return; } @@ -667,8 +779,53 @@ static void append_switch_log(int interval_id, const char* event_name, int switc out << interval_id << ',' << event_name << ',' << switch_id << ',' << switches[switch_id].name << ',' << port_id << ',' << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' - << capacity_mbit << ',' << sent_mbit << ',' << queued_after_mbit << ',' - << dropped_mbit << ',' << active_flowlets << '\n'; + << capacity_mbit << ',' << queued_before_mbit << ',' << sent_mbit << ',' + << queued_after_mbit << ',' << dropped_mbit << ',' << active_queue_entries << '\n'; +} + +static void append_flowlet_log(int interval_id, const char* event_name, int switch_id, int port_id, + int target_is_terminal, int target_index, + unsigned long long flowlet_id, int source_terminal, + int destination_terminal, int creation_interval, + double capacity_mbit, double queued_before_mbit, + double send_mbit, double remaining_after_mbit, + double dropped_mbit) { + if (cfg.flowlet_log_path[0] == '\0') { + return; + } + std::ofstream out(cfg.flowlet_log_path, std::ios::app); + out << interval_id << ',' << event_name << ',' << switch_id << ',' + << switches[switch_id].name << ',' << port_id << ',' + << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' + << flowlet_id << ',' << source_terminal << ',' << destination_terminal << ',' + << creation_interval << ',' << (interval_id - creation_interval) << ',' + << capacity_mbit << ',' << queued_before_mbit << ',' << send_mbit << ',' + << remaining_after_mbit << ',' << dropped_mbit << '\n'; +} + +static void append_switch_training_log(int interval_id, int switch_id, int port_id, + int target_is_terminal, int target_index, + double capacity_mbit, double queued_before_mbit, + double sent_mbit, double queued_after_mbit, + double dropped_mbit, int active_before, + int active_after, + const std::vector& allocations) { + if (cfg.switch_training_log_path[0] == '\0') { + return; + } + std::ofstream out(cfg.switch_training_log_path, std::ios::app); + out << interval_id << ',' << switch_id << ',' << switches[switch_id].name << ',' + << port_id << ',' << (target_is_terminal ? "terminal" : "switch") << ',' + << target_index << ',' << cfg.switch_scheduler << ',' << capacity_mbit << ',' + << queued_before_mbit << ',' << sent_mbit << ',' << queued_after_mbit << ',' + << dropped_mbit << ',' << active_before << ',' << active_after << ','; + for (size_t i = 0; i < allocations.size(); ++i) { + if (i > 0) { + out << ';'; + } + out << allocations[i]; + } + out << '\n'; } static void schedule_workload_generate(terminal_state* ns, int interval_id, tw_lp* lp) { @@ -785,7 +942,7 @@ static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp static void handle_terminal_arrival(terminal_state* ns, fluid_msg* m) { ns->received_mbit += m->mbit; - ns->received_flowlets++; + ns->received_fragments++; append_terminal_log(m->interval_id, "receive", ns->terminal_id, m->source_terminal, m->mbit); } @@ -813,10 +970,10 @@ static void terminal_rev_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp static void terminal_finalize(terminal_state* ns, tw_lp* lp) { printf("fluid-terminal gid=%llu terminal=%d switch=%d generated_mbit=%.6f sent_mbit=%.6f " - "received_mbit=%.6f generated_flowlets=%d received_flowlets=%d\n", + "received_mbit=%.6f generated_flowlets=%d received_fragments=%d\n", (unsigned long long)lp->gid, ns->terminal_id, ns->attached_switch, ns->generated_mbit, ns->sent_to_switch_mbit, ns->received_mbit, - ns->generated_flowlets, ns->received_flowlets); + ns->generated_flowlets, ns->received_fragments); } static void switch_init(switch_state* ns, tw_lp* lp) { @@ -858,7 +1015,7 @@ static void switch_init(switch_state* ns, tw_lp* lp) { } static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { - ns->received_flowlets++; + ns->received_fragments++; int dst_sw = terminals[m->destination_terminal].switch_id; int port_id = -1; if (dst_sw == ns->switch_id) { @@ -873,10 +1030,42 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { if (port_id < 0) { ns->dropped_mbit += m->mbit; append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, - 0.0, 0.0, 0.0, m->mbit, 0); + 0.0, 0.0, 0.0, 0.0, m->mbit, 0); + append_flowlet_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, + m->flowlet_id, m->source_terminal, m->destination_terminal, + m->creation_interval, 0.0, 0.0, 0.0, 0.0, m->mbit); return; } - enqueue_flowlet(ns, port_id, m); + + port_desc* p = &ns->ports[port_id]; + double queued_before = 0.0; + double dropped = 0.0; + double flowlet_remaining_after = 0.0; + int coalesced = 0; + + double accepted = enqueue_flowlet(ns, port_id, m, &queued_before, &dropped, + &flowlet_remaining_after, &coalesced); + + double queued_after = queued_mbit_on_port(ns, port_id); + int active_after = active_flowlet_count_on_port(ns, port_id); + + if (accepted > EPS) { + append_flowlet_log(m->interval_id, coalesced ? "enqueue_coalesce" : "enqueue", + ns->switch_id, port_id, p->is_terminal, p->target_index, + m->flowlet_id, m->source_terminal, m->destination_terminal, + m->creation_interval, p->capacity_mbit_per_interval, + queued_before, 0.0, flowlet_remaining_after, 0.0); + } + if (dropped > EPS) { + append_switch_log(m->interval_id, "drop_buffer_overflow", ns->switch_id, port_id, + p->is_terminal, p->target_index, p->capacity_mbit_per_interval, + queued_before, 0.0, queued_after, dropped, active_after); + append_flowlet_log(m->interval_id, "drop_buffer_overflow", ns->switch_id, port_id, + p->is_terminal, p->target_index, m->flowlet_id, m->source_terminal, + m->destination_terminal, m->creation_interval, + p->capacity_mbit_per_interval, queued_before, 0.0, queued_after, + dropped); + } } static void send_flowlet_fragment(switch_state* ns, int port_id, const queued_flowlet* q, @@ -922,51 +1111,157 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { port_desc* p = &ns->ports[port_id]; std::vector& qv = *ns->queues[port_id]; + double capacity = p->capacity_mbit_per_interval; double remaining_capacity = capacity; double sent_total = 0.0; + double queued_before = queued_mbit_on_port(ns, port_id); + int active_before = active_flowlet_count_on_port(ns, port_id); - std::vector active; - for (int i = 0; i < (int)qv.size(); ++i) { - if (qv[i].valid && qv[i].remaining_mbit > EPS) { - active.push_back(i); + std::vector allocations; + std::vector send_plan(qv.size(), 0.0); + + auto add_to_plan = [&](int idx, double requested_send) -> double { + if (idx < 0 || idx >= (int)send_plan.size() || requested_send <= EPS) { + return 0.0; } - } - /* Max-min fair allocation with redistribution. For 25 Mb and 10 Mb - * flowlets on a 25 Mb link, this sends 15 Mb from the larger flowlet and - * 10 Mb from the smaller one, leaving 10 Mb queued from the larger flowlet. - */ - while (!active.empty() && remaining_capacity > EPS) { - double share = remaining_capacity / (double)active.size(); - std::vector still_active; - bool made_progress = false; - for (size_t ai = 0; ai < active.size(); ++ai) { - int idx = active[ai]; - queued_flowlet before = qv[idx]; - double send = std::min(before.remaining_mbit, share); - if (send > EPS) { - send_flowlet_fragment(ns, port_id, &before, send, m->interval_id, lp); - qv[idx].remaining_mbit -= send; - remaining_capacity -= send; - sent_total += send; - made_progress = true; + double still_available_for_flowlet = qv[idx].remaining_mbit - send_plan[idx]; + if (still_available_for_flowlet <= EPS) { + return 0.0; + } + + double actual_send = std::min(requested_send, still_available_for_flowlet); + send_plan[idx] += actual_send; + return actual_send; + }; + + auto record_allocation = [&](const queued_flowlet& before, double send, + double remaining_after) { + append_flowlet_log(m->interval_id, "allocate_send", ns->switch_id, port_id, + p->is_terminal, p->target_index, before.flowlet_id, + before.source_terminal, before.destination_terminal, + before.creation_interval, capacity, queued_before, send, + remaining_after, 0.0); + + std::ostringstream ss; + ss << before.flowlet_id << ':' << before.source_terminal << ':' + << before.destination_terminal << ':' << before.creation_interval << ':' + << before.remaining_mbit << ':' << send << ':' << remaining_after; + allocations.push_back(ss.str()); + }; + + if (strcmp(cfg.switch_scheduler, "fifo") == 0) { + /* + * FIFO is also computed as a plan first, then applied once below. + * That keeps the rollback shape identical across schedulers. + */ + for (int i = 0; i < (int)qv.size() && remaining_capacity > EPS; ++i) { + if (!qv[i].valid || qv[i].remaining_mbit <= EPS) { + continue; } - if (qv[idx].remaining_mbit > EPS) { - still_active.push_back(idx); + + double requested_send = std::min(qv[i].remaining_mbit, remaining_capacity); + double planned_send = add_to_plan(i, requested_send); + remaining_capacity -= planned_send; + } + } else { + std::vector virtual_remaining(qv.size(), 0.0); + std::vector active; + + for (int i = 0; i < (int)qv.size(); ++i) { + if (qv[i].valid && qv[i].remaining_mbit > EPS) { + virtual_remaining[i] = qv[i].remaining_mbit; + active.push_back(i); } } - active.swap(still_active); - if (!made_progress) { - break; + + /* + * Max-min fair allocation with redistribution. + * + * This is intentionally two-phase. The redistribution loop computes + * the final per-flowlet allocation in send_plan without mutating queue + * state or scheduling next-hop arrivals. The application loop below + * then applies each affected flowlet exactly once. + * + * This preserves the fair-sharing result while making future optimistic + * reverse computation much simpler: + * + * - one queue byte delta per flowlet + * - at most one scheduled fragment per flowlet per switch-port interval + * - one training/log target per flowlet + */ + while (!active.empty() && remaining_capacity > EPS) { + double share = remaining_capacity / (double)active.size(); + std::vector still_active; + bool made_progress = false; + + for (size_t ai = 0; ai < active.size(); ++ai) { + int idx = active[ai]; + + double planned_send = std::min(virtual_remaining[idx], share); + if (planned_send > EPS) { + send_plan[idx] += planned_send; + virtual_remaining[idx] -= planned_send; + remaining_capacity -= planned_send; + made_progress = true; + } + + if (virtual_remaining[idx] > EPS) { + still_active.push_back(idx); + } + } + + active.swap(still_active); + + if (!made_progress) { + break; + } + } + } + + for (int i = 0; i < (int)qv.size(); ++i) { + double send = send_plan[i]; + + if (!qv[i].valid || send <= EPS) { + continue; + } + + if (send > qv[i].remaining_mbit + EPS) { + tw_error(TW_LOC, + "computed send %.12f exceeds remaining %.12f for flowlet %llu " + "on switch %d port %d", + send, qv[i].remaining_mbit, + (unsigned long long)qv[i].flowlet_id, ns->switch_id, port_id); } + + queued_flowlet before = qv[i]; + + /* + * Clamp only to protect against floating-point roundoff near EPS. A + * larger violation is caught by the tw_error above. + */ + send = std::min(send, qv[i].remaining_mbit); + + send_flowlet_fragment(ns, port_id, &before, send, m->interval_id, lp); + qv[i].remaining_mbit -= send; + sent_total += send; + + record_allocation(before, send, qv[i].remaining_mbit); } compact_port_queue(ns, port_id); + double queued_after = queued_mbit_on_port(ns, port_id); + int active_after = active_flowlet_count_on_port(ns, port_id); + append_switch_log(m->interval_id, "egress", ns->switch_id, port_id, p->is_terminal, - p->target_index, capacity, sent_total, queued_after, 0.0, - (int)qv.size()); + p->target_index, capacity, queued_before, sent_total, queued_after, 0.0, + active_after); + + append_switch_training_log(m->interval_id, ns->switch_id, port_id, p->is_terminal, + p->target_index, capacity, queued_before, sent_total, queued_after, + 0.0, active_before, active_after, allocations); schedule_switch_egress(m->interval_id + 1, port_id, lp); } @@ -998,11 +1293,11 @@ static void switch_finalize(switch_state* ns, tw_lp* lp) { for (int p = 0; p < ns->num_ports; ++p) { queued += queued_mbit_on_port(ns, p); } - printf("fluid-switch gid=%llu switch=%d name=%s ports=%d received_flowlets=%d " + printf("fluid-switch gid=%llu switch=%d name=%s ports=%d received_fragments=%d " "sent_fragments=%d enqueued_mbit=%.6f sent_mbit=%.6f local_delivery_mbit=%.6f " "dropped_mbit=%.6f queued_mbit=%.6f\n", (unsigned long long)lp->gid, ns->switch_id, switches[ns->switch_id].name.c_str(), - ns->num_ports, ns->received_flowlets, ns->sent_fragments, ns->enqueued_mbit, + ns->num_ports, ns->received_fragments, ns->sent_fragments, ns->enqueued_mbit, ns->sent_mbit, ns->delivered_local_mbit, ns->dropped_mbit, queued); } @@ -1036,7 +1331,21 @@ static void write_log_headers(int rank) { std::remove(cfg.switch_log_path); std::ofstream out(cfg.switch_log_path, std::ios::app); out << "interval,event,switch,switch_name,port,target_type,target_index,capacity_mbit," - "sent_mbit,queued_after_mbit,dropped_mbit,active_flowlets\n"; + "queued_before_mbit,sent_mbit,queued_after_mbit,dropped_mbit,active_queue_entries\n"; + } + if (cfg.flowlet_log_path[0] != '\0') { + std::remove(cfg.flowlet_log_path); + std::ofstream out(cfg.flowlet_log_path, std::ios::app); + out << "interval,event,switch,switch_name,port,target_type,target_index,flowlet_id," + "source_terminal,destination_terminal,creation_interval,age_intervals,capacity_mbit," + "queued_before_mbit,send_mbit,remaining_after_mbit,dropped_mbit\n"; + } + if (cfg.switch_training_log_path[0] != '\0') { + std::remove(cfg.switch_training_log_path); + std::ofstream out(cfg.switch_training_log_path, std::ios::app); + out << "interval,switch,switch_name,port,target_type,target_index,scheduler,capacity_mbit," + "queued_before_mbit,sent_mbit,queued_after_mbit,dropped_mbit,active_before," + "active_after,allocations\n"; } } @@ -1078,9 +1387,9 @@ int main(int argc, char** argv) { if (rank == 0) { printf("fluid-switch config: switches=%zu terminals=%zu interval_seconds=%.6f " - "num_send_intervals=%d num_drain_intervals=%d\n", + "num_send_intervals=%d num_drain_intervals=%d switch_scheduler=%s\n", switches.size(), terminals.size(), cfg.interval_seconds, cfg.num_send_intervals, - cfg.num_drain_intervals); + cfg.num_drain_intervals, cfg.switch_scheduler); } tw_run(); From 5836a0bd9c65d2320d6b9d7be10d7ee1125aee8e Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Wed, 8 Jul 2026 12:35:13 -0400 Subject: [PATCH 05/20] Replace per switch queue buffer with shared buffer --- .../model-net-fluid-switch.cxx | 192 +++++++++++------- 1 file changed, 115 insertions(+), 77 deletions(-) diff --git a/src/network-workloads/model-net-fluid-switch.cxx b/src/network-workloads/model-net-fluid-switch.cxx index 26505eb6..71207cf2 100644 --- a/src/network-workloads/model-net-fluid-switch.cxx +++ b/src/network-workloads/model-net-fluid-switch.cxx @@ -4,8 +4,9 @@ * * This is a pure PDES model. It does not use model-net and does not call the * ZeroMQ Director. Terminal LPs generate stochastic bounded workload flowlets. - * Switch LPs route flowlets, queue them per output link, and send per-flowlet - * fragments subject to interval link capacity. + * Switch LPs route flowlets, queue them per output link, enforce a shared + * switch-wide buffer, and send per-flowlet fragments subject to interval link + * capacity. * * Intended first-run mode: --synch=1. Reverse computation is intentionally * disabled until the pure sequential model is validated. @@ -127,6 +128,7 @@ struct terminal_state { struct switch_state { int switch_id; int num_ports; + double shared_buffer_mbit; port_desc ports[MAX_PORTS_PER_SWITCH]; std::vector* queues[MAX_PORTS_PER_SWITCH]; double enqueued_mbit; @@ -610,12 +612,23 @@ static int find_terminal_port(const switch_state* ns, int dst_terminal) { return -1; } +static double queued_mbit_on_switch(const switch_state* ns); + static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, - double* queued_before_out, double* dropped_out, + double* port_queued_before_out, + double* shared_queued_before_out, + double* shared_queued_after_out, + double* dropped_out, double* flowlet_remaining_after_out, int* coalesced_out) { - if (queued_before_out != NULL) { - *queued_before_out = 0.0; + if (port_queued_before_out != NULL) { + *port_queued_before_out = 0.0; + } + if (shared_queued_before_out != NULL) { + *shared_queued_before_out = 0.0; + } + if (shared_queued_after_out != NULL) { + *shared_queued_after_out = 0.0; } if (dropped_out != NULL) { *dropped_out = 0.0; @@ -636,21 +649,29 @@ static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, std::vector& qv = *ns->queues[port_id]; - double queued_on_port = 0.0; + double port_queued_before = 0.0; for (size_t i = 0; i < qv.size(); ++i) { - queued_on_port += qv[i].remaining_mbit; + port_queued_before += qv[i].remaining_mbit; } - if (queued_before_out != NULL) { - *queued_before_out = queued_on_port; + double shared_queued_before = queued_mbit_on_switch(ns); + + if (port_queued_before_out != NULL) { + *port_queued_before_out = port_queued_before; + } + if (shared_queued_before_out != NULL) { + *shared_queued_before_out = shared_queued_before; } - double accepted = m->mbit; - double dropped = 0.0; + /* + * Admission is controlled by the shared switch-wide buffer. Output queues + * remain per port, but they all draw from this one occupancy limit. + */ + double shared_available = std::max(0.0, ns->shared_buffer_mbit - shared_queued_before); + double accepted = std::min(m->mbit, shared_available); + double dropped = std::max(0.0, m->mbit - accepted); - if (queued_on_port + accepted > ns->ports[port_id].buffer_mbit + EPS) { - accepted = std::max(0.0, ns->ports[port_id].buffer_mbit - queued_on_port); - dropped = std::max(0.0, m->mbit - accepted); + if (dropped > EPS) { ns->dropped_mbit += dropped; } @@ -659,21 +680,12 @@ static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, } if (accepted <= EPS) { + if (shared_queued_after_out != NULL) { + *shared_queued_after_out = shared_queued_before; + } return 0.0; } - /* - * Fragments of the same original interval-fluid flowlet can reconverge at - * the same switch output queue. Coalesce them into one queue entry so that - * each switch-port interval has at most one mutable queue record and one - * allocation target per original flowlet_id. - * - * This makes future reverse computation simpler: - * - * - one bytes_remaining delta per flowlet - * - at most one scheduled send fragment per flowlet per egress event - * - one ML training target per flowlet per switch-port interval - */ for (size_t i = 0; i < qv.size(); ++i) { if (!qv[i].valid) { continue; @@ -693,6 +705,9 @@ static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, if (coalesced_out != NULL) { *coalesced_out = 1; } + if (shared_queued_after_out != NULL) { + *shared_queued_after_out = shared_queued_before + accepted; + } return accepted; } @@ -715,10 +730,14 @@ static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, if (flowlet_remaining_after_out != NULL) { *flowlet_remaining_after_out = accepted; } + if (shared_queued_after_out != NULL) { + *shared_queued_after_out = shared_queued_before + accepted; + } return accepted; } + static double queued_mbit_on_port(const switch_state* ns, int port_id) { if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { return 0.0; @@ -731,6 +750,15 @@ static double queued_mbit_on_port(const switch_state* ns, int port_id) { return total; } +static double queued_mbit_on_switch(const switch_state* ns) { + double total = 0.0; + for (int p = 0; p < ns->num_ports; ++p) { + total += queued_mbit_on_port(ns, p); + } + return total; +} + + static int active_flowlet_count_on_port(const switch_state* ns, int port_id) { if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { return 0; @@ -770,7 +798,11 @@ static void append_terminal_log(int interval_id, const char* event_name, int ter static void append_switch_log(int interval_id, const char* event_name, int switch_id, int port_id, int target_is_terminal, int target_index, double capacity_mbit, double queued_before_mbit, double sent_mbit, - double queued_after_mbit, double dropped_mbit, + double queued_after_mbit, + double shared_queued_before_mbit, + double shared_queued_after_mbit, + double shared_buffer_mbit, + double dropped_mbit, int active_queue_entries) { if (cfg.switch_log_path[0] == '\0') { return; @@ -780,9 +812,12 @@ static void append_switch_log(int interval_id, const char* event_name, int switc << switches[switch_id].name << ',' << port_id << ',' << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' << capacity_mbit << ',' << queued_before_mbit << ',' << sent_mbit << ',' - << queued_after_mbit << ',' << dropped_mbit << ',' << active_queue_entries << '\n'; + << queued_after_mbit << ',' << shared_queued_before_mbit << ',' + << shared_queued_after_mbit << ',' << shared_buffer_mbit << ',' + << dropped_mbit << ',' << active_queue_entries << '\n'; } + static void append_flowlet_log(int interval_id, const char* event_name, int switch_id, int port_id, int target_is_terminal, int target_index, unsigned long long flowlet_id, int source_terminal, @@ -984,6 +1019,17 @@ static void switch_init(switch_state* ns, tw_lp* lp) { } const switch_info& sw = switches[ns->switch_id]; + + /* + * switch_buffer is modeled as one shared switch-wide pool. Output queues + * remain per port for scheduling, but admission checks total queued bytes + * across all ports on this switch. + */ + ns->shared_buffer_mbit = sw.switch_buffer_mbit; + if (ns->shared_buffer_mbit < 0.0) { + tw_error(TW_LOC, "negative shared switch buffer on switch %d", ns->switch_id); + } + for (size_t i = 0; i < sw.links.size(); ++i) { if (ns->num_ports >= MAX_PORTS_PER_SWITCH) { tw_error(TW_LOC, "too many ports on switch %d", ns->switch_id); @@ -993,7 +1039,7 @@ static void switch_init(switch_state* ns, tw_lp* lp) { p->is_terminal = 0; p->target_index = sw.links[i].dst_switch; p->capacity_mbit_per_interval = sw.links[i].bandwidth_mbps * cfg.interval_seconds; - p->buffer_mbit = sw.links[i].buffer_mbit; + p->buffer_mbit = 0.0; /* buffer capacity is switch-wide, not per port */ ns->queues[ns->num_ports - 1] = new std::vector(); } for (int t = 0; t < sw.terminal_count; ++t) { @@ -1005,7 +1051,7 @@ static void switch_init(switch_state* ns, tw_lp* lp) { p->is_terminal = 1; p->target_index = sw.terminal_start + t; p->capacity_mbit_per_interval = sw.terminal_bandwidth_mbps * cfg.interval_seconds; - p->buffer_mbit = sw.switch_buffer_mbit; + p->buffer_mbit = 0.0; /* buffer capacity is switch-wide, not per port */ ns->queues[ns->num_ports - 1] = new std::vector(); } @@ -1014,6 +1060,7 @@ static void switch_init(switch_state* ns, tw_lp* lp) { } } + static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { ns->received_fragments++; int dst_sw = terminals[m->destination_terminal].switch_id; @@ -1028,9 +1075,11 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { } if (port_id < 0) { + double shared_before = queued_mbit_on_switch(ns); ns->dropped_mbit += m->mbit; append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, - 0.0, 0.0, 0.0, 0.0, m->mbit, 0); + 0.0, 0.0, 0.0, 0.0, shared_before, shared_before, + ns->shared_buffer_mbit, m->mbit, 0); append_flowlet_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, m->flowlet_id, m->source_terminal, m->destination_terminal, m->creation_interval, 0.0, 0.0, 0.0, 0.0, m->mbit); @@ -1038,15 +1087,18 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { } port_desc* p = &ns->ports[port_id]; - double queued_before = 0.0; + double port_queued_before = 0.0; + double shared_queued_before = 0.0; + double shared_queued_after = 0.0; double dropped = 0.0; double flowlet_remaining_after = 0.0; int coalesced = 0; - double accepted = enqueue_flowlet(ns, port_id, m, &queued_before, &dropped, - &flowlet_remaining_after, &coalesced); + double accepted = enqueue_flowlet(ns, port_id, m, &port_queued_before, + &shared_queued_before, &shared_queued_after, + &dropped, &flowlet_remaining_after, &coalesced); - double queued_after = queued_mbit_on_port(ns, port_id); + double port_queued_after = queued_mbit_on_port(ns, port_id); int active_after = active_flowlet_count_on_port(ns, port_id); if (accepted > EPS) { @@ -1054,20 +1106,23 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { ns->switch_id, port_id, p->is_terminal, p->target_index, m->flowlet_id, m->source_terminal, m->destination_terminal, m->creation_interval, p->capacity_mbit_per_interval, - queued_before, 0.0, flowlet_remaining_after, 0.0); + port_queued_before, 0.0, flowlet_remaining_after, 0.0); } if (dropped > EPS) { - append_switch_log(m->interval_id, "drop_buffer_overflow", ns->switch_id, port_id, - p->is_terminal, p->target_index, p->capacity_mbit_per_interval, - queued_before, 0.0, queued_after, dropped, active_after); - append_flowlet_log(m->interval_id, "drop_buffer_overflow", ns->switch_id, port_id, - p->is_terminal, p->target_index, m->flowlet_id, m->source_terminal, - m->destination_terminal, m->creation_interval, - p->capacity_mbit_per_interval, queued_before, 0.0, queued_after, - dropped); + append_switch_log(m->interval_id, "drop_shared_buffer_overflow", ns->switch_id, + port_id, p->is_terminal, p->target_index, + p->capacity_mbit_per_interval, port_queued_before, 0.0, + port_queued_after, shared_queued_before, shared_queued_after, + ns->shared_buffer_mbit, dropped, active_after); + append_flowlet_log(m->interval_id, "drop_shared_buffer_overflow", ns->switch_id, + port_id, p->is_terminal, p->target_index, m->flowlet_id, + m->source_terminal, m->destination_terminal, m->creation_interval, + p->capacity_mbit_per_interval, port_queued_before, 0.0, + flowlet_remaining_after, dropped); } } + static void send_flowlet_fragment(switch_state* ns, int port_id, const queued_flowlet* q, double send_mbit, int interval_id, tw_lp* lp) { if (send_mbit <= EPS) { @@ -1116,6 +1171,7 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { double remaining_capacity = capacity; double sent_total = 0.0; double queued_before = queued_mbit_on_port(ns, port_id); + double shared_queued_before = queued_mbit_on_switch(ns); int active_before = active_flowlet_count_on_port(ns, port_id); std::vector allocations; @@ -1152,10 +1208,6 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { }; if (strcmp(cfg.switch_scheduler, "fifo") == 0) { - /* - * FIFO is also computed as a plan first, then applied once below. - * That keeps the rollback shape identical across schedulers. - */ for (int i = 0; i < (int)qv.size() && remaining_capacity > EPS; ++i) { if (!qv[i].valid || qv[i].remaining_mbit <= EPS) { continue; @@ -1176,21 +1228,6 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { } } - /* - * Max-min fair allocation with redistribution. - * - * This is intentionally two-phase. The redistribution loop computes - * the final per-flowlet allocation in send_plan without mutating queue - * state or scheduling next-hop arrivals. The application loop below - * then applies each affected flowlet exactly once. - * - * This preserves the fair-sharing result while making future optimistic - * reverse computation much simpler: - * - * - one queue byte delta per flowlet - * - at most one scheduled fragment per flowlet per switch-port interval - * - one training/log target per flowlet - */ while (!active.empty() && remaining_capacity > EPS) { double share = remaining_capacity / (double)active.size(); std::vector still_active; @@ -1236,11 +1273,6 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { } queued_flowlet before = qv[i]; - - /* - * Clamp only to protect against floating-point roundoff near EPS. A - * larger violation is caught by the tw_error above. - */ send = std::min(send, qv[i].remaining_mbit); send_flowlet_fragment(ns, port_id, &before, send, m->interval_id, lp); @@ -1253,11 +1285,13 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { compact_port_queue(ns, port_id); double queued_after = queued_mbit_on_port(ns, port_id); + double shared_queued_after = queued_mbit_on_switch(ns); int active_after = active_flowlet_count_on_port(ns, port_id); append_switch_log(m->interval_id, "egress", ns->switch_id, port_id, p->is_terminal, - p->target_index, capacity, queued_before, sent_total, queued_after, 0.0, - active_after); + p->target_index, capacity, queued_before, sent_total, queued_after, + shared_queued_before, shared_queued_after, ns->shared_buffer_mbit, + 0.0, active_after); append_switch_training_log(m->interval_id, ns->switch_id, port_id, p->is_terminal, p->target_index, capacity, queued_before, sent_total, queued_after, @@ -1266,6 +1300,7 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { schedule_switch_egress(m->interval_id + 1, port_id, lp); } + static void switch_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { (void)b; switch (m->event_type) { @@ -1293,12 +1328,13 @@ static void switch_finalize(switch_state* ns, tw_lp* lp) { for (int p = 0; p < ns->num_ports; ++p) { queued += queued_mbit_on_port(ns, p); } - printf("fluid-switch gid=%llu switch=%d name=%s ports=%d received_fragments=%d " - "sent_fragments=%d enqueued_mbit=%.6f sent_mbit=%.6f local_delivery_mbit=%.6f " - "dropped_mbit=%.6f queued_mbit=%.6f\n", + printf("fluid-switch gid=%llu switch=%d name=%s ports=%d shared_buffer_mbit=%.6f " + "received_fragments=%d sent_fragments=%d enqueued_mbit=%.6f sent_mbit=%.6f " + "local_delivery_mbit=%.6f dropped_mbit=%.6f queued_mbit=%.6f\n", (unsigned long long)lp->gid, ns->switch_id, switches[ns->switch_id].name.c_str(), - ns->num_ports, ns->received_fragments, ns->sent_fragments, ns->enqueued_mbit, - ns->sent_mbit, ns->delivered_local_mbit, ns->dropped_mbit, queued); + ns->num_ports, ns->shared_buffer_mbit, ns->received_fragments, ns->sent_fragments, + ns->enqueued_mbit, ns->sent_mbit, ns->delivered_local_mbit, ns->dropped_mbit, + queued); } const tw_optdef app_opt[] = {TWOPT_GROUP("model-net interval-fluid switch/terminal workload"), @@ -1331,7 +1367,8 @@ static void write_log_headers(int rank) { std::remove(cfg.switch_log_path); std::ofstream out(cfg.switch_log_path, std::ios::app); out << "interval,event,switch,switch_name,port,target_type,target_index,capacity_mbit," - "queued_before_mbit,sent_mbit,queued_after_mbit,dropped_mbit,active_queue_entries\n"; + "queued_before_mbit,sent_mbit,queued_after_mbit,shared_queued_before_mbit," + "shared_queued_after_mbit,shared_buffer_mbit,dropped_mbit,active_queue_entries\n"; } if (cfg.flowlet_log_path[0] != '\0') { std::remove(cfg.flowlet_log_path); @@ -1387,7 +1424,8 @@ int main(int argc, char** argv) { if (rank == 0) { printf("fluid-switch config: switches=%zu terminals=%zu interval_seconds=%.6f " - "num_send_intervals=%d num_drain_intervals=%d switch_scheduler=%s\n", + "num_send_intervals=%d num_drain_intervals=%d switch_scheduler=%s " + "buffer_mode=shared\n", switches.size(), terminals.size(), cfg.interval_seconds, cfg.num_send_intervals, cfg.num_drain_intervals, cfg.switch_scheduler); } From fb49e94719942dbf6806624cf2ab3222e741b483 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Wed, 8 Jul 2026 13:19:17 -0400 Subject: [PATCH 06/20] Add reverse handler for switch-terminal fluid flow model --- doc/example/fluid-switch.conf.in | 2 +- .../model-net-fluid-switch.cxx | 345 +++++++++++++++++- 2 files changed, 328 insertions(+), 19 deletions(-) diff --git a/doc/example/fluid-switch.conf.in b/doc/example/fluid-switch.conf.in index a92bc3d3..5339fbd5 100644 --- a/doc/example/fluid-switch.conf.in +++ b/doc/example/fluid-switch.conf.in @@ -14,7 +14,7 @@ LPGROUPS PARAMS { - message_size="256"; + message_size="32768"; pe_mem_factor="1024"; } diff --git a/src/network-workloads/model-net-fluid-switch.cxx b/src/network-workloads/model-net-fluid-switch.cxx index 71207cf2..aeb28e79 100644 --- a/src/network-workloads/model-net-fluid-switch.cxx +++ b/src/network-workloads/model-net-fluid-switch.cxx @@ -8,8 +8,9 @@ * switch-wide buffer, and send per-flowlet fragments subject to interval link * capacity. * - * Intended first-run mode: --synch=1. Reverse computation is intentionally - * disabled until the pure sequential model is validated. + * Supports sequential validation and optimistic execution. Optimistic mode uses + * event-local reverse metadata to undo terminal generation, switch arrivals, + * queue mutations, egress byte deltas, and RNG draws. */ #include @@ -42,6 +43,7 @@ static const char* SWITCH_LP_NAME = "fluid-switch-lp"; static constexpr int MAX_SWITCHES = 64; static constexpr int MAX_TERMINALS = 4096; static constexpr int MAX_PORTS_PER_SWITCH = 128; +static constexpr int MAX_RC_ALLOCATIONS = 256; static constexpr double EPS = 1e-9; static constexpr double PHASE_GENERATE = 0.10; @@ -145,6 +147,13 @@ enum fluid_event_type { SWITCH_EGRESS = 3, }; +struct rc_alloc_record { + int valid; + int queue_index; + queued_flowlet before; + double send_mbit; +}; + struct fluid_msg { int event_type; int interval_id; @@ -156,6 +165,21 @@ struct fluid_msg { int creation_interval; unsigned long long flowlet_id; double mbit; + + /* + * Reverse-computation metadata. These fields are written by the forward + * event handler and consumed by the reverse handler if the event rolls back. + */ + int rc_rng_count; + int rc_generated; + int rc_no_route; + int rc_port_id; + int rc_queue_index; + int rc_coalesced; + double rc_accepted_mbit; + double rc_dropped_mbit; + int rc_alloc_count; + rc_alloc_record rc_allocs[MAX_RC_ALLOCATIONS]; }; static void terminal_init(terminal_state* ns, tw_lp* lp); @@ -198,6 +222,32 @@ static void add_lp_types(void) { lp_type_register(SWITCH_LP_NAME, switch_get_lp_type()); } +static int get_configured_message_size_bytes(void) { + int configured_message_size = 0; + if (configuration_get_value_int(&config, "PARAMS", "message_size", NULL, + &configured_message_size) != 0) { + return 0; + } + return configured_message_size; +} + +static void validate_ross_message_size_or_abort(int rank) { + int configured_message_size = get_configured_message_size_bytes(); + size_t required_message_size = sizeof(fluid_msg); + + if (configured_message_size <= 0 || + (size_t)configured_message_size < required_message_size) { + if (rank == 0) { + fprintf(stderr, + "fluid-switch error: PARAMS.message_size=%d is too small for " + "sizeof(fluid_msg)=%zu. Increase PARAMS.message_size in the " + "CODES config before calling codes_mapping_setup().\n", + configured_message_size, required_message_size); + } + MPI_Abort(MPI_COMM_WORLD, 1); + } +} + static double seconds_to_ns(double seconds) { return seconds * 1000.0 * 1000.0 * 1000.0; } static double event_time_ns(int interval_id, double phase_seconds) { @@ -620,7 +670,8 @@ static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, double* shared_queued_after_out, double* dropped_out, double* flowlet_remaining_after_out, - int* coalesced_out) { + int* coalesced_out, + int* queue_index_out) { if (port_queued_before_out != NULL) { *port_queued_before_out = 0.0; } @@ -639,6 +690,9 @@ static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, if (coalesced_out != NULL) { *coalesced_out = 0; } + if (queue_index_out != NULL) { + *queue_index_out = -1; + } if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { if (dropped_out != NULL) { @@ -705,6 +759,9 @@ static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, if (coalesced_out != NULL) { *coalesced_out = 1; } + if (queue_index_out != NULL) { + *queue_index_out = (int)i; + } if (shared_queued_after_out != NULL) { *shared_queued_after_out = shared_queued_before + accepted; } @@ -725,6 +782,9 @@ static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, q.remaining_mbit = accepted; qv.push_back(q); + if (queue_index_out != NULL) { + *queue_index_out = (int)qv.size() - 1; + } ns->enqueued_mbit += accepted; if (flowlet_remaining_after_out != NULL) { @@ -773,6 +833,53 @@ static int active_flowlet_count_on_port(const switch_state* ns, int port_id) { return count; } +static int find_queue_index_for_flowlet(const switch_state* ns, int port_id, + const queued_flowlet& needle) { + if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { + return -1; + } + + const std::vector& qv = *ns->queues[port_id]; + + for (int i = 0; i < (int)qv.size(); ++i) { + if (!qv[i].valid) { + continue; + } + + if (qv[i].flowlet_id == needle.flowlet_id && + qv[i].source_terminal == needle.source_terminal && + qv[i].destination_terminal == needle.destination_terminal && + qv[i].creation_interval == needle.creation_interval) { + return i; + } + } + + return -1; +} + +static int find_queue_index_for_msg(const switch_state* ns, int port_id, const fluid_msg* m) { + if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { + return -1; + } + + const std::vector& qv = *ns->queues[port_id]; + + for (int i = 0; i < (int)qv.size(); ++i) { + if (!qv[i].valid) { + continue; + } + + if (qv[i].flowlet_id == m->flowlet_id && + qv[i].source_terminal == m->source_terminal && + qv[i].destination_terminal == m->destination_terminal && + qv[i].creation_interval == m->creation_interval) { + return i; + } + } + + return -1; +} + static void compact_port_queue(switch_state* ns, int port_id) { if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { return; @@ -784,9 +891,19 @@ static void compact_port_queue(switch_state* ns, int port_id) { qv.end()); } +static bool fluid_csv_logs_enabled(void) { + /* + * CSV logging is intentionally sequential-only. Optimistic execution may + * roll events back after a forward handler has already appended a row. + * Until we add commit-time logging, skip CSV output under optimistic modes + * and use final LP summaries for optimistic validation. + */ + return g_tw_synchronization_protocol == SEQUENTIAL; +} + static void append_terminal_log(int interval_id, const char* event_name, int terminal_id, int peer_id, double mbit) { - if (cfg.terminal_log_path[0] == '\0') { + if (cfg.terminal_log_path[0] == '\0' || !fluid_csv_logs_enabled()) { return; } std::ofstream out(cfg.terminal_log_path, std::ios::app); @@ -804,7 +921,7 @@ static void append_switch_log(int interval_id, const char* event_name, int switc double shared_buffer_mbit, double dropped_mbit, int active_queue_entries) { - if (cfg.switch_log_path[0] == '\0') { + if (cfg.switch_log_path[0] == '\0' || !fluid_csv_logs_enabled()) { return; } std::ofstream out(cfg.switch_log_path, std::ios::app); @@ -825,7 +942,7 @@ static void append_flowlet_log(int interval_id, const char* event_name, int swit double capacity_mbit, double queued_before_mbit, double send_mbit, double remaining_after_mbit, double dropped_mbit) { - if (cfg.flowlet_log_path[0] == '\0') { + if (cfg.flowlet_log_path[0] == '\0' || !fluid_csv_logs_enabled()) { return; } std::ofstream out(cfg.flowlet_log_path, std::ios::app); @@ -845,7 +962,7 @@ static void append_switch_training_log(int interval_id, int switch_id, int port_ double dropped_mbit, int active_before, int active_after, const std::vector& allocations) { - if (cfg.switch_training_log_path[0] == '\0') { + if (cfg.switch_training_log_path[0] == '\0' || !fluid_csv_logs_enabled()) { return; } std::ofstream out(cfg.switch_training_log_path, std::ios::app); @@ -938,11 +1055,24 @@ static double random_workload_mbit(int terminal_id, tw_lp* lp) { static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp) { int interval = m->interval_id; + + m->rc_rng_count = 0; + m->rc_generated = 0; + m->mbit = 0.0; + m->destination_terminal = -1; + m->flowlet_id = 0; + if (interval % cfg.terminal_send_every_n_intervals == 0) { double p = tw_rand_unif(lp->rng); + m->rc_rng_count++; + if (p <= cfg.terminal_send_probability) { int dst = choose_random_destination(ns->terminal_id, lp); + m->rc_rng_count++; + double mbit = random_workload_mbit(ns->terminal_id, lp); + m->rc_rng_count++; + if (mbit > EPS) { fluid_msg out_msg; memset(&out_msg, 0, sizeof(out_msg)); @@ -963,6 +1093,12 @@ static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp ns->generated_mbit += mbit; ns->sent_to_switch_mbit += mbit; ns->generated_flowlets++; + + m->rc_generated = 1; + m->destination_terminal = dst; + m->flowlet_id = out_msg.flowlet_id; + m->mbit = mbit; + append_terminal_log(interval, "generate_send", ns->terminal_id, dst, mbit); if (cfg.debug_prints) { @@ -972,9 +1108,11 @@ static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp } } } + schedule_workload_generate(ns, interval + 1, lp); } + static void handle_terminal_arrival(terminal_state* ns, fluid_msg* m) { ns->received_mbit += m->mbit; ns->received_fragments++; @@ -996,11 +1134,37 @@ static void terminal_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp } static void terminal_rev_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { - (void)ns; (void)b; - (void)m; - (void)lp; - tw_error(TW_LOC, "model-net-fluid-switch currently supports sequential validation runs only"); + + switch (m->event_type) { + case WORKLOAD_GENERATE: + if (m->rc_generated) { + ns->generated_mbit -= m->mbit; + ns->sent_to_switch_mbit -= m->mbit; + ns->generated_flowlets--; + + if (ns->next_flowlet_seq == 0) { + tw_error(TW_LOC, "terminal %d next_flowlet_seq underflow during rollback", + ns->terminal_id); + } + + ns->next_flowlet_seq--; + } + + for (int i = 0; i < m->rc_rng_count; ++i) { + tw_rand_reverse_unif(lp->rng); + } + + break; + + case FLOWLET_ARRIVAL: + ns->received_mbit -= m->mbit; + ns->received_fragments--; + break; + + default: + tw_error(TW_LOC, "terminal reverse received unknown event type %d", m->event_type); + } } static void terminal_finalize(terminal_state* ns, tw_lp* lp) { @@ -1076,6 +1240,10 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { if (port_id < 0) { double shared_before = queued_mbit_on_switch(ns); + m->rc_no_route = 1; + m->rc_port_id = -1; + m->rc_accepted_mbit = 0.0; + m->rc_dropped_mbit = m->mbit; ns->dropped_mbit += m->mbit; append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, 0.0, 0.0, 0.0, 0.0, shared_before, shared_before, @@ -1093,10 +1261,24 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { double dropped = 0.0; double flowlet_remaining_after = 0.0; int coalesced = 0; + int queue_index = -1; + + m->rc_no_route = 0; + m->rc_port_id = port_id; + m->rc_queue_index = -1; + m->rc_coalesced = 0; + m->rc_accepted_mbit = 0.0; + m->rc_dropped_mbit = 0.0; double accepted = enqueue_flowlet(ns, port_id, m, &port_queued_before, &shared_queued_before, &shared_queued_after, - &dropped, &flowlet_remaining_after, &coalesced); + &dropped, &flowlet_remaining_after, &coalesced, + &queue_index); + + m->rc_queue_index = queue_index; + m->rc_coalesced = coalesced; + m->rc_accepted_mbit = accepted; + m->rc_dropped_mbit = dropped; double port_queued_after = queued_mbit_on_port(ns, port_id); int active_after = active_flowlet_count_on_port(ns, port_id); @@ -1176,6 +1358,7 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { std::vector allocations; std::vector send_plan(qv.size(), 0.0); + m->rc_alloc_count = 0; auto add_to_plan = [&](int idx, double requested_send) -> double { if (idx < 0 || idx >= (int)send_plan.size() || requested_send <= EPS) { @@ -1275,6 +1458,21 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { queued_flowlet before = qv[i]; send = std::min(send, qv[i].remaining_mbit); + if (m->rc_alloc_count >= MAX_RC_ALLOCATIONS) { + tw_error(TW_LOC, + "too many egress allocations for reverse metadata: switch %d port %d " + "interval %d count %d max %d", + ns->switch_id, port_id, m->interval_id, m->rc_alloc_count, + MAX_RC_ALLOCATIONS); + } + + rc_alloc_record* rc = &m->rc_allocs[m->rc_alloc_count++]; + memset(rc, 0, sizeof(*rc)); + rc->valid = 1; + rc->queue_index = i; + rc->before = before; + rc->send_mbit = send; + send_flowlet_fragment(ns, port_id, &before, send, m->interval_id, lp); qv[i].remaining_mbit -= send; sent_total += send; @@ -1315,12 +1513,120 @@ static void switch_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { } } +static void rollback_switch_arrival(switch_state* ns, fluid_msg* m) { + ns->received_fragments--; + + if (m->rc_no_route) { + ns->dropped_mbit -= m->rc_dropped_mbit; + return; + } + + int port_id = m->rc_port_id; + + if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { + tw_error(TW_LOC, "invalid rollback arrival port %d on switch %d", port_id, + ns->switch_id); + } + + ns->dropped_mbit -= m->rc_dropped_mbit; + + if (m->rc_accepted_mbit <= EPS) { + return; + } + + std::vector& qv = *ns->queues[port_id]; + + int idx = m->rc_queue_index; + if (idx < 0 || idx >= (int)qv.size() || qv[idx].flowlet_id != m->flowlet_id) { + idx = find_queue_index_for_msg(ns, port_id, m); + } + + if (idx < 0) { + tw_error(TW_LOC, + "could not find flowlet %llu on switch %d port %d during arrival rollback", + (unsigned long long)m->flowlet_id, ns->switch_id, port_id); + } + + if (m->rc_coalesced) { + qv[idx].remaining_mbit -= m->rc_accepted_mbit; + + if (qv[idx].remaining_mbit <= EPS) { + tw_error(TW_LOC, + "coalesced flowlet %llu became empty during arrival rollback on switch %d port %d", + (unsigned long long)m->flowlet_id, ns->switch_id, port_id); + } + } else { + qv.erase(qv.begin() + idx); + } + + ns->enqueued_mbit -= m->rc_accepted_mbit; +} + +static void rollback_switch_egress(switch_state* ns, fluid_msg* m) { + int port_id = m->port_id; + + if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { + tw_error(TW_LOC, "invalid rollback egress port %d on switch %d", port_id, + ns->switch_id); + } + + std::vector& qv = *ns->queues[port_id]; + const port_desc* p = &ns->ports[port_id]; + + for (int r = m->rc_alloc_count - 1; r >= 0; --r) { + rc_alloc_record* rc = &m->rc_allocs[r]; + + if (!rc->valid || rc->send_mbit <= EPS) { + continue; + } + + int idx = rc->queue_index; + + if (idx < 0 || idx >= (int)qv.size() || + qv[idx].flowlet_id != rc->before.flowlet_id) { + idx = find_queue_index_for_flowlet(ns, port_id, rc->before); + } + + if (idx >= 0) { + qv[idx] = rc->before; + } else { + int insert_idx = rc->queue_index; + + if (insert_idx < 0) { + insert_idx = 0; + } + if (insert_idx > (int)qv.size()) { + insert_idx = (int)qv.size(); + } + + qv.insert(qv.begin() + insert_idx, rc->before); + } + + ns->sent_mbit -= rc->send_mbit; + ns->sent_fragments--; + + if (p->is_terminal) { + ns->delivered_local_mbit -= rc->send_mbit; + } + } +} + static void switch_rev_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { - (void)ns; (void)b; - (void)m; (void)lp; - tw_error(TW_LOC, "model-net-fluid-switch currently supports sequential validation runs only"); + + switch (m->event_type) { + case FLOWLET_ARRIVAL: + rollback_switch_arrival(ns, m); + break; + + case SWITCH_EGRESS: + rollback_switch_egress(ns, m); + break; + + default: + tw_error(TW_LOC, "switch reverse received unknown event type %d", m->event_type); + } } static void switch_finalize(switch_state* ns, tw_lp* lp) { @@ -1355,7 +1661,7 @@ static const char* find_config_arg(int argc, char** argv) { } static void write_log_headers(int rank) { - if (rank != 0) { + if (rank != 0 || !fluid_csv_logs_enabled()) { return; } if (cfg.terminal_log_path[0] != '\0') { @@ -1404,6 +1710,7 @@ int main(int argc, char** argv) { MPI_Comm_rank(MPI_COMM_WORLD, &rank); configuration_load(config_file, MPI_COMM_WORLD, &config); load_config(); + validate_ross_message_size_or_abort(rank); add_lp_types(); codes_mapping_setup(); @@ -1425,9 +1732,11 @@ int main(int argc, char** argv) { if (rank == 0) { printf("fluid-switch config: switches=%zu terminals=%zu interval_seconds=%.6f " "num_send_intervals=%d num_drain_intervals=%d switch_scheduler=%s " - "buffer_mode=shared\n", + "buffer_mode=shared csv_logs=%s ross_message_size=%d fluid_msg_size=%zu\n", switches.size(), terminals.size(), cfg.interval_seconds, cfg.num_send_intervals, - cfg.num_drain_intervals, cfg.switch_scheduler); + cfg.num_drain_intervals, cfg.switch_scheduler, + fluid_csv_logs_enabled() ? "enabled" : "disabled_for_rollback_safety", + get_configured_message_size_bytes(), sizeof(fluid_msg)); } tw_run(); From 684a2bf2a24d9c0833d3d69117ceb00646938b9b Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Wed, 8 Jul 2026 14:16:30 -0400 Subject: [PATCH 07/20] Optimizations for fluid flow simn efficiency --- .../model-net-fluid-switch.cxx | 234 ++++++++++++++---- 1 file changed, 184 insertions(+), 50 deletions(-) diff --git a/src/network-workloads/model-net-fluid-switch.cxx b/src/network-workloads/model-net-fluid-switch.cxx index aeb28e79..914f0248 100644 --- a/src/network-workloads/model-net-fluid-switch.cxx +++ b/src/network-workloads/model-net-fluid-switch.cxx @@ -133,6 +133,14 @@ struct switch_state { double shared_buffer_mbit; port_desc ports[MAX_PORTS_PER_SWITCH]; std::vector* queues[MAX_PORTS_PER_SWITCH]; + + /* + * Demand-driven egress scheduling. Each port may have at most one + * outstanding SWITCH_EGRESS event. -1 means no egress event is currently + * outstanding for that port. + */ + int scheduled_egress_interval[MAX_PORTS_PER_SWITCH]; + double enqueued_mbit; double sent_mbit; double delivered_local_mbit; @@ -179,6 +187,10 @@ struct fluid_msg { double rc_accepted_mbit; double rc_dropped_mbit; int rc_alloc_count; + + int rc_scheduled_egress; + int rc_prev_scheduled_egress_interval; + rc_alloc_record rc_allocs[MAX_RC_ALLOCATIONS]; }; @@ -980,10 +992,53 @@ static void append_switch_training_log(int interval_id, int switch_id, int port_ out << '\n'; } +static int next_terminal_generate_interval_after(int interval_id) { + int period = cfg.terminal_send_every_n_intervals; + if (period <= 0) { + period = 1; + } + + if (interval_id < 0) { + return 0; + } + + return interval_id + period; +} + +static int first_terminal_generate_interval(void) { + /* + * The first eligible terminal workload interval is always 0. Later events + * advance by terminal_send_every_n_intervals, so terminals do not carry a + * speculative self-event chain through intervals where no workload can be + * generated. + */ + return 0; +} + static void schedule_workload_generate(terminal_state* ns, int interval_id, tw_lp* lp) { if (interval_id >= cfg.num_send_intervals) { return; } + + int period = cfg.terminal_send_every_n_intervals; + if (period <= 0) { + period = 1; + } + + /* + * Defensive alignment: callers should already pass eligible send intervals, + * but if a future caller passes an arbitrary interval, round up to the next + * eligible workload-generation interval instead of scheduling a no-op event. + */ + int rem = interval_id % period; + if (rem != 0) { + interval_id += period - rem; + } + + if (interval_id >= cfg.num_send_intervals) { + return; + } + tw_event* e = tw_event_new(lp->gid, delay_until_ns(interval_id, PHASE_GENERATE, lp), lp); fluid_msg* m = (fluid_msg*)tw_event_data(e); memset(m, 0, sizeof(*m)); @@ -1006,6 +1061,51 @@ static void schedule_switch_egress(int interval_id, int port_id, tw_lp* lp) { tw_event_send(e); } +static void request_switch_egress(switch_state* ns, int interval_id, int port_id, tw_lp* lp, + fluid_msg* cause_msg) { + if (port_id < 0 || port_id >= ns->num_ports) { + tw_error(TW_LOC, "invalid request_switch_egress port %d on switch %d", + port_id, ns->switch_id); + } + + int prev = ns->scheduled_egress_interval[port_id]; + + if (cause_msg != NULL) { + cause_msg->rc_scheduled_egress = 0; + cause_msg->rc_prev_scheduled_egress_interval = prev; + } + + if (interval_id >= cfg.num_send_intervals + cfg.num_drain_intervals) { + return; + } + + if (prev == interval_id) { + return; + } + + if (prev >= 0 && prev < interval_id) { + /* + * An earlier egress is already outstanding. Let that earlier event + * decide whether residual bytes require a later egress. + */ + return; + } + + if (prev >= 0 && prev > interval_id) { + tw_error(TW_LOC, + "switch %d port %d has future egress interval %d while trying " + "to schedule earlier interval %d", + ns->switch_id, port_id, prev, interval_id); + } + + schedule_switch_egress(interval_id, port_id, lp); + ns->scheduled_egress_interval[port_id] = interval_id; + + if (cause_msg != NULL) { + cause_msg->rc_scheduled_egress = 1; + } +} + static void schedule_arrival(int interval_id, tw_lpid dst_gid, const fluid_msg* src_msg, double mbit, int dst_switch, tw_lp* lp) { tw_event* e = tw_event_new(dst_gid, delay_until_ns(interval_id, PHASE_ARRIVAL, lp), lp); @@ -1032,7 +1132,7 @@ static void terminal_init(terminal_state* ns, tw_lp* lp) { ns->attached_switch = terminals[ns->terminal_id].switch_id; ns->local_terminal_id = terminals[ns->terminal_id].local_id; ns->next_flowlet_seq = 0; - schedule_workload_generate(ns, 0, lp); + schedule_workload_generate(ns, first_terminal_generate_interval(), lp); } static int choose_random_destination(int self, tw_lp* lp) { @@ -1062,54 +1162,52 @@ static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp m->destination_terminal = -1; m->flowlet_id = 0; - if (interval % cfg.terminal_send_every_n_intervals == 0) { - double p = tw_rand_unif(lp->rng); + double p = tw_rand_unif(lp->rng); + m->rc_rng_count++; + + if (p <= cfg.terminal_send_probability) { + int dst = choose_random_destination(ns->terminal_id, lp); m->rc_rng_count++; - if (p <= cfg.terminal_send_probability) { - int dst = choose_random_destination(ns->terminal_id, lp); - m->rc_rng_count++; - - double mbit = random_workload_mbit(ns->terminal_id, lp); - m->rc_rng_count++; - - if (mbit > EPS) { - fluid_msg out_msg; - memset(&out_msg, 0, sizeof(out_msg)); - out_msg.event_type = FLOWLET_ARRIVAL; - out_msg.interval_id = interval + 1; - out_msg.source_terminal = ns->terminal_id; - out_msg.destination_terminal = dst; - out_msg.source_switch = ns->attached_switch; - out_msg.destination_switch = ns->attached_switch; - out_msg.creation_interval = interval; - out_msg.flowlet_id = ((unsigned long long)ns->terminal_id << 48) | - (unsigned long long)ns->next_flowlet_seq++; - out_msg.mbit = mbit; - - tw_lpid sw_gid = get_switch_gid(ns->attached_switch); - schedule_arrival(interval + 1, sw_gid, &out_msg, mbit, ns->attached_switch, lp); - - ns->generated_mbit += mbit; - ns->sent_to_switch_mbit += mbit; - ns->generated_flowlets++; - - m->rc_generated = 1; - m->destination_terminal = dst; - m->flowlet_id = out_msg.flowlet_id; - m->mbit = mbit; - - append_terminal_log(interval, "generate_send", ns->terminal_id, dst, mbit); - - if (cfg.debug_prints) { - printf("[fluid terminal] interval=%d terminal=%d dst=%d mbit=%.6f\n", - interval, ns->terminal_id, dst, mbit); - } + double mbit = random_workload_mbit(ns->terminal_id, lp); + m->rc_rng_count++; + + if (mbit > EPS) { + fluid_msg out_msg; + memset(&out_msg, 0, sizeof(out_msg)); + out_msg.event_type = FLOWLET_ARRIVAL; + out_msg.interval_id = interval + 1; + out_msg.source_terminal = ns->terminal_id; + out_msg.destination_terminal = dst; + out_msg.source_switch = ns->attached_switch; + out_msg.destination_switch = ns->attached_switch; + out_msg.creation_interval = interval; + out_msg.flowlet_id = ((unsigned long long)ns->terminal_id << 48) | + (unsigned long long)ns->next_flowlet_seq++; + out_msg.mbit = mbit; + + tw_lpid sw_gid = get_switch_gid(ns->attached_switch); + schedule_arrival(interval + 1, sw_gid, &out_msg, mbit, ns->attached_switch, lp); + + ns->generated_mbit += mbit; + ns->sent_to_switch_mbit += mbit; + ns->generated_flowlets++; + + m->rc_generated = 1; + m->destination_terminal = dst; + m->flowlet_id = out_msg.flowlet_id; + m->mbit = mbit; + + append_terminal_log(interval, "generate_send", ns->terminal_id, dst, mbit); + + if (cfg.debug_prints) { + printf("[fluid terminal] interval=%d terminal=%d dst=%d mbit=%.6f\n", + interval, ns->terminal_id, dst, mbit); } } } - schedule_workload_generate(ns, interval + 1, lp); + schedule_workload_generate(ns, next_terminal_generate_interval_after(interval), lp); } @@ -1182,6 +1280,10 @@ static void switch_init(switch_state* ns, tw_lp* lp) { tw_error(TW_LOC, "switch LP relative id %d out of range", ns->switch_id); } + for (int p = 0; p < MAX_PORTS_PER_SWITCH; ++p) { + ns->scheduled_egress_interval[p] = -1; + } + const switch_info& sw = switches[ns->switch_id]; /* @@ -1219,13 +1321,16 @@ static void switch_init(switch_state* ns, tw_lp* lp) { ns->queues[ns->num_ports - 1] = new std::vector(); } - for (int p = 0; p < ns->num_ports; ++p) { - schedule_switch_egress(0, p, lp); - } + /* + * Egress is demand-driven. We no longer pre-schedule periodic empty + * egress events for every port. Arrivals schedule the first egress, and + * egress events reschedule themselves only while residual queued bytes + * remain. + */ } -static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { +static void handle_switch_arrival(switch_state* ns, fluid_msg* m, tw_lp* lp) { ns->received_fragments++; int dst_sw = terminals[m->destination_terminal].switch_id; int port_id = -1; @@ -1244,6 +1349,8 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { m->rc_port_id = -1; m->rc_accepted_mbit = 0.0; m->rc_dropped_mbit = m->mbit; + m->rc_scheduled_egress = 0; + m->rc_prev_scheduled_egress_interval = -1; ns->dropped_mbit += m->mbit; append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, 0.0, 0.0, 0.0, 0.0, shared_before, shared_before, @@ -1269,6 +1376,8 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { m->rc_coalesced = 0; m->rc_accepted_mbit = 0.0; m->rc_dropped_mbit = 0.0; + m->rc_scheduled_egress = 0; + m->rc_prev_scheduled_egress_interval = ns->scheduled_egress_interval[port_id]; double accepted = enqueue_flowlet(ns, port_id, m, &port_queued_before, &shared_queued_before, &shared_queued_after, @@ -1302,6 +1411,10 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m) { p->capacity_mbit_per_interval, port_queued_before, 0.0, flowlet_remaining_after, dropped); } + + if (port_queued_after > EPS) { + request_switch_egress(ns, m->interval_id, port_id, lp, m); + } } @@ -1342,13 +1455,20 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { tw_error(TW_LOC, "invalid switch egress port %d", port_id); } if (ns->queues[port_id] == NULL) { - schedule_switch_egress(m->interval_id + 1, port_id, lp); + m->rc_prev_scheduled_egress_interval = -1; + m->rc_scheduled_egress = 0; return; } port_desc* p = &ns->ports[port_id]; std::vector& qv = *ns->queues[port_id]; + m->rc_prev_scheduled_egress_interval = ns->scheduled_egress_interval[port_id]; + m->rc_scheduled_egress = 0; + if (ns->scheduled_egress_interval[port_id] == m->interval_id) { + ns->scheduled_egress_interval[port_id] = -1; + } + double capacity = p->capacity_mbit_per_interval; double remaining_capacity = capacity; double sent_total = 0.0; @@ -1495,7 +1615,14 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { p->target_index, capacity, queued_before, sent_total, queued_after, 0.0, active_before, active_after, allocations); - schedule_switch_egress(m->interval_id + 1, port_id, lp); + if (queued_after > EPS) { + /* + * This event's reverse handler restores scheduled_egress_interval from + * rc_prev_scheduled_egress_interval. Pass NULL here so the next-event + * scheduling does not overwrite this event's saved pre-state. + */ + request_switch_egress(ns, m->interval_id + 1, port_id, lp, NULL); + } } @@ -1503,7 +1630,7 @@ static void switch_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { (void)b; switch (m->event_type) { case FLOWLET_ARRIVAL: - handle_switch_arrival(ns, m); + handle_switch_arrival(ns, m, lp); break; case SWITCH_EGRESS: handle_switch_egress(ns, m, lp); @@ -1516,6 +1643,11 @@ static void switch_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { static void rollback_switch_arrival(switch_state* ns, fluid_msg* m) { ns->received_fragments--; + if (!m->rc_no_route && m->rc_port_id >= 0 && m->rc_port_id < ns->num_ports) { + ns->scheduled_egress_interval[m->rc_port_id] = + m->rc_prev_scheduled_egress_interval; + } + if (m->rc_no_route) { ns->dropped_mbit -= m->rc_dropped_mbit; return; @@ -1570,6 +1702,8 @@ static void rollback_switch_egress(switch_state* ns, fluid_msg* m) { ns->switch_id); } + ns->scheduled_egress_interval[port_id] = m->rc_prev_scheduled_egress_interval; + std::vector& qv = *ns->queues[port_id]; const port_desc* p = &ns->ports[port_id]; From 26d6ea1c9395da6dadec1abe3717c83f13f4ed8a Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Wed, 8 Jul 2026 14:54:52 -0400 Subject: [PATCH 08/20] Add ROSS commit handler and csv logging for fluid simn --- .../model-net-fluid-switch.cxx | 280 +++++++++++++++++- 1 file changed, 271 insertions(+), 9 deletions(-) diff --git a/src/network-workloads/model-net-fluid-switch.cxx b/src/network-workloads/model-net-fluid-switch.cxx index 914f0248..732377cc 100644 --- a/src/network-workloads/model-net-fluid-switch.cxx +++ b/src/network-workloads/model-net-fluid-switch.cxx @@ -191,17 +191,31 @@ struct fluid_msg { int rc_scheduled_egress; int rc_prev_scheduled_egress_interval; + int rc_log_target_is_terminal; + int rc_log_target_index; + double rc_log_capacity_mbit; + double rc_log_port_queued_before_mbit; + double rc_log_port_queued_after_mbit; + double rc_log_shared_queued_before_mbit; + double rc_log_shared_queued_after_mbit; + double rc_log_flowlet_remaining_after_mbit; + double rc_log_sent_total_mbit; + int rc_log_active_before_entries; + int rc_log_active_after_entries; + rc_alloc_record rc_allocs[MAX_RC_ALLOCATIONS]; }; static void terminal_init(terminal_state* ns, tw_lp* lp); static void terminal_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); static void terminal_rev_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); +static void terminal_commit_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); static void terminal_finalize(terminal_state* ns, tw_lp* lp); static void switch_init(switch_state* ns, tw_lp* lp); static void switch_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); static void switch_rev_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); +static void switch_commit_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp); static void switch_finalize(switch_state* ns, tw_lp* lp); static tw_lptype terminal_lp = { @@ -209,7 +223,7 @@ static tw_lptype terminal_lp = { (pre_run_f)NULL, (event_f)terminal_event, (revent_f)terminal_rev_event, - (commit_f)NULL, + (commit_f)terminal_commit_event, (final_f)terminal_finalize, (map_f)codes_mapping, sizeof(terminal_state), @@ -220,7 +234,7 @@ static tw_lptype switch_lp = { (pre_run_f)NULL, (event_f)switch_event, (revent_f)switch_rev_event, - (commit_f)NULL, + (commit_f)switch_commit_event, (final_f)switch_finalize, (map_f)codes_mapping, sizeof(switch_state), @@ -903,14 +917,24 @@ static void compact_port_queue(switch_state* ns, int port_id) { qv.end()); } +static bool fluid_commit_logging_active = false; + +static bool fluid_csv_forward_logs_enabled(void) { + return g_tw_synchronization_protocol == SEQUENTIAL; +} + +static bool fluid_csv_commit_logs_enabled(void) { + return g_tw_synchronization_protocol != SEQUENTIAL; +} + static bool fluid_csv_logs_enabled(void) { /* - * CSV logging is intentionally sequential-only. Optimistic execution may - * roll events back after a forward handler has already appended a row. - * Until we add commit-time logging, skip CSV output under optimistic modes - * and use final LP summaries for optimistic validation. + * Sequential mode logs directly from forward handlers. Optimistic mode logs + * only from commit handlers, after ROSS guarantees the event will not roll + * back. The process-local flag lets the same append_* helpers be reused by + * both paths without permitting speculative forward-time appends. */ - return g_tw_synchronization_protocol == SEQUENTIAL; + return fluid_csv_forward_logs_enabled() || fluid_commit_logging_active; } static void append_terminal_log(int interval_id, const char* event_name, int terminal_id, @@ -1265,6 +1289,65 @@ static void terminal_rev_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp } } +static void fluid_commit_logging_begin(void) { fluid_commit_logging_active = true; } + +static void fluid_commit_logging_end(void) { fluid_commit_logging_active = false; } + +static void terminal_commit_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { + (void)b; + (void)lp; + + if (!fluid_csv_commit_logs_enabled()) { + return; + } + + fluid_commit_logging_begin(); + + switch (m->event_type) { + case WORKLOAD_GENERATE: + if (m->rc_generated) { + append_terminal_log(m->interval_id, "generate_send", ns->terminal_id, + m->destination_terminal, m->mbit); + } + break; + + case FLOWLET_ARRIVAL: + append_terminal_log(m->interval_id, "receive", ns->terminal_id, + m->source_terminal, m->mbit); + break; + + default: + break; + } + + fluid_commit_logging_end(); +} + +static void build_allocation_strings_from_rc(const fluid_msg* m, + std::vector& allocations) { + allocations.clear(); + + for (int i = 0; i < m->rc_alloc_count; ++i) { + const rc_alloc_record* rc = &m->rc_allocs[i]; + + if (!rc->valid || rc->send_mbit <= EPS) { + continue; + } + + double remaining_after = rc->before.remaining_mbit - rc->send_mbit; + if (remaining_after < 0.0 && remaining_after > -EPS) { + remaining_after = 0.0; + } + + std::ostringstream ss; + ss << rc->before.flowlet_id << ':' << rc->before.source_terminal << ':' + << rc->before.destination_terminal << ':' << rc->before.creation_interval << ':' + << rc->before.remaining_mbit << ':' << rc->send_mbit << ':' << remaining_after; + allocations.push_back(ss.str()); + } +} + + static void terminal_finalize(terminal_state* ns, tw_lp* lp) { printf("fluid-terminal gid=%llu terminal=%d switch=%d generated_mbit=%.6f sent_mbit=%.6f " "received_mbit=%.6f generated_flowlets=%d received_fragments=%d\n", @@ -1351,6 +1434,17 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m, tw_lp* lp) { m->rc_dropped_mbit = m->mbit; m->rc_scheduled_egress = 0; m->rc_prev_scheduled_egress_interval = -1; + m->rc_log_target_is_terminal = 0; + m->rc_log_target_index = -1; + m->rc_log_capacity_mbit = 0.0; + m->rc_log_port_queued_before_mbit = 0.0; + m->rc_log_port_queued_after_mbit = 0.0; + m->rc_log_shared_queued_before_mbit = shared_before; + m->rc_log_shared_queued_after_mbit = shared_before; + m->rc_log_flowlet_remaining_after_mbit = 0.0; + m->rc_log_sent_total_mbit = 0.0; + m->rc_log_active_before_entries = 0; + m->rc_log_active_after_entries = 0; ns->dropped_mbit += m->mbit; append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, 0.0, 0.0, 0.0, 0.0, shared_before, shared_before, @@ -1378,6 +1472,17 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m, tw_lp* lp) { m->rc_dropped_mbit = 0.0; m->rc_scheduled_egress = 0; m->rc_prev_scheduled_egress_interval = ns->scheduled_egress_interval[port_id]; + m->rc_log_target_is_terminal = p->is_terminal; + m->rc_log_target_index = p->target_index; + m->rc_log_capacity_mbit = p->capacity_mbit_per_interval; + m->rc_log_port_queued_before_mbit = 0.0; + m->rc_log_port_queued_after_mbit = 0.0; + m->rc_log_shared_queued_before_mbit = 0.0; + m->rc_log_shared_queued_after_mbit = 0.0; + m->rc_log_flowlet_remaining_after_mbit = 0.0; + m->rc_log_sent_total_mbit = 0.0; + m->rc_log_active_before_entries = 0; + m->rc_log_active_after_entries = 0; double accepted = enqueue_flowlet(ns, port_id, m, &port_queued_before, &shared_queued_before, &shared_queued_after, @@ -1392,6 +1497,13 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m, tw_lp* lp) { double port_queued_after = queued_mbit_on_port(ns, port_id); int active_after = active_flowlet_count_on_port(ns, port_id); + m->rc_log_port_queued_before_mbit = port_queued_before; + m->rc_log_port_queued_after_mbit = port_queued_after; + m->rc_log_shared_queued_before_mbit = shared_queued_before; + m->rc_log_shared_queued_after_mbit = shared_queued_after; + m->rc_log_flowlet_remaining_after_mbit = flowlet_remaining_after; + m->rc_log_active_after_entries = active_after; + if (accepted > EPS) { append_flowlet_log(m->interval_id, coalesced ? "enqueue_coalesce" : "enqueue", ns->switch_id, port_id, p->is_terminal, p->target_index, @@ -1479,6 +1591,17 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { std::vector allocations; std::vector send_plan(qv.size(), 0.0); m->rc_alloc_count = 0; + m->rc_log_target_is_terminal = p->is_terminal; + m->rc_log_target_index = p->target_index; + m->rc_log_capacity_mbit = capacity; + m->rc_log_port_queued_before_mbit = queued_before; + m->rc_log_port_queued_after_mbit = queued_before; + m->rc_log_shared_queued_before_mbit = shared_queued_before; + m->rc_log_shared_queued_after_mbit = shared_queued_before; + m->rc_log_flowlet_remaining_after_mbit = 0.0; + m->rc_log_sent_total_mbit = 0.0; + m->rc_log_active_before_entries = active_before; + m->rc_log_active_after_entries = active_before; auto add_to_plan = [&](int idx, double requested_send) -> double { if (idx < 0 || idx >= (int)send_plan.size() || requested_send <= EPS) { @@ -1606,6 +1729,11 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { double shared_queued_after = queued_mbit_on_switch(ns); int active_after = active_flowlet_count_on_port(ns, port_id); + m->rc_log_port_queued_after_mbit = queued_after; + m->rc_log_shared_queued_after_mbit = shared_queued_after; + m->rc_log_sent_total_mbit = sent_total; + m->rc_log_active_after_entries = active_after; + append_switch_log(m->interval_id, "egress", ns->switch_id, port_id, p->is_terminal, p->target_index, capacity, queued_before, sent_total, queued_after, shared_queued_before, shared_queued_after, ns->shared_buffer_mbit, @@ -1763,6 +1891,140 @@ static void switch_rev_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp } } +static void switch_commit_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { + (void)b; + (void)lp; + + if (!fluid_csv_commit_logs_enabled()) { + return; + } + + fluid_commit_logging_begin(); + + switch (m->event_type) { + case FLOWLET_ARRIVAL: + if (m->rc_no_route) { + append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, + 0.0, 0.0, 0.0, 0.0, + m->rc_log_shared_queued_before_mbit, + m->rc_log_shared_queued_after_mbit, + ns->shared_buffer_mbit, m->rc_dropped_mbit, 0); + append_flowlet_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, + m->flowlet_id, m->source_terminal, m->destination_terminal, + m->creation_interval, 0.0, 0.0, 0.0, 0.0, + m->rc_dropped_mbit); + } else { + if (m->rc_accepted_mbit > EPS) { + append_flowlet_log(m->interval_id, + m->rc_coalesced ? "enqueue_coalesce" : "enqueue", + ns->switch_id, m->rc_port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->flowlet_id, m->source_terminal, + m->destination_terminal, m->creation_interval, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + 0.0, + m->rc_log_flowlet_remaining_after_mbit, + 0.0); + } + + if (m->rc_dropped_mbit > EPS) { + append_switch_log(m->interval_id, "drop_shared_buffer_overflow", + ns->switch_id, m->rc_port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + 0.0, + m->rc_log_port_queued_after_mbit, + m->rc_log_shared_queued_before_mbit, + m->rc_log_shared_queued_after_mbit, + ns->shared_buffer_mbit, + m->rc_dropped_mbit, + m->rc_log_active_after_entries); + + append_flowlet_log(m->interval_id, "drop_shared_buffer_overflow", + ns->switch_id, m->rc_port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->flowlet_id, m->source_terminal, + m->destination_terminal, m->creation_interval, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + 0.0, + m->rc_log_flowlet_remaining_after_mbit, + m->rc_dropped_mbit); + } + } + break; + + case SWITCH_EGRESS: { + std::vector allocations; + + for (int i = 0; i < m->rc_alloc_count; ++i) { + const rc_alloc_record* rc = &m->rc_allocs[i]; + + if (!rc->valid || rc->send_mbit <= EPS) { + continue; + } + + double remaining_after = rc->before.remaining_mbit - rc->send_mbit; + if (remaining_after < 0.0 && remaining_after > -EPS) { + remaining_after = 0.0; + } + + append_flowlet_log(m->interval_id, "allocate_send", + ns->switch_id, m->port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + rc->before.flowlet_id, + rc->before.source_terminal, + rc->before.destination_terminal, + rc->before.creation_interval, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + rc->send_mbit, + remaining_after, + 0.0); + } + + append_switch_log(m->interval_id, "egress", ns->switch_id, m->port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + m->rc_log_sent_total_mbit, + m->rc_log_port_queued_after_mbit, + m->rc_log_shared_queued_before_mbit, + m->rc_log_shared_queued_after_mbit, + ns->shared_buffer_mbit, + 0.0, + m->rc_log_active_after_entries); + + build_allocation_strings_from_rc(m, allocations); + append_switch_training_log(m->interval_id, ns->switch_id, m->port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + m->rc_log_sent_total_mbit, + m->rc_log_port_queued_after_mbit, + 0.0, + m->rc_log_active_before_entries, + m->rc_log_active_after_entries, + allocations); + break; + } + + default: + break; + } + + fluid_commit_logging_end(); +} + + static void switch_finalize(switch_state* ns, tw_lp* lp) { double queued = 0.0; for (int p = 0; p < ns->num_ports; ++p) { @@ -1795,7 +2057,7 @@ static const char* find_config_arg(int argc, char** argv) { } static void write_log_headers(int rank) { - if (rank != 0 || !fluid_csv_logs_enabled()) { + if (rank != 0) { return; } if (cfg.terminal_log_path[0] != '\0') { @@ -1869,7 +2131,7 @@ int main(int argc, char** argv) { "buffer_mode=shared csv_logs=%s ross_message_size=%d fluid_msg_size=%zu\n", switches.size(), terminals.size(), cfg.interval_seconds, cfg.num_send_intervals, cfg.num_drain_intervals, cfg.switch_scheduler, - fluid_csv_logs_enabled() ? "enabled" : "disabled_for_rollback_safety", + fluid_csv_forward_logs_enabled() ? "forward" : "commit", get_configured_message_size_bytes(), sizeof(fluid_msg)); } From 31956fd638061a3bfc2c3ad186741b1b6a4fd4bb Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 10:57:57 -0400 Subject: [PATCH 09/20] Clean up fluid switch logging Buffer CSV rows in memory for both sequential and optimistic runs, then merge logs after simulation completion to avoid event-path file I/O and unsafe parallel appends. Refactor common terminal and switch logging paths, add CSV file-open checks, and keep optimistic logging rollback-safe through commit handlers. Use MPI_COMM_CODES consistently, remove dead event/config fields, document the large reverse-allocation event payload, clean up comments, and release switch queue storage during finalization. --- .../model-net-fluid-switch.cxx | 606 ++++++++++-------- 1 file changed, 326 insertions(+), 280 deletions(-) diff --git a/src/network-workloads/model-net-fluid-switch.cxx b/src/network-workloads/model-net-fluid-switch.cxx index 732377cc..faa1cf97 100644 --- a/src/network-workloads/model-net-fluid-switch.cxx +++ b/src/network-workloads/model-net-fluid-switch.cxx @@ -2,13 +2,11 @@ /* * Standalone interval-fluid switch/terminal workload model. * - * This is a pure PDES model. It does not use model-net and does not call the - * ZeroMQ Director. Terminal LPs generate stochastic bounded workload flowlets. - * Switch LPs route flowlets, queue them per output link, enforce a shared - * switch-wide buffer, and send per-flowlet fragments subject to interval link - * capacity. + * Terminal LPs generate stochastic bounded workload flowlets. Switch LPs route + * flowlets, queue them per output link, enforce a shared switch-wide buffer, + * and send per-flowlet fragments subject to interval link capacity. * - * Supports sequential validation and optimistic execution. Optimistic mode uses + * Supports sequential validation and optimistic execution. Optimistic mode uses * event-local reverse metadata to undo terminal generation, switch arrivals, * queue mutations, egress byte deltas, and RNG draws. */ @@ -40,9 +38,15 @@ static const char* GROUP_NAME = "FLUID_GRP"; static const char* TERMINAL_LP_NAME = "fluid-terminal-lp"; static const char* SWITCH_LP_NAME = "fluid-switch-lp"; -static constexpr int MAX_SWITCHES = 64; -static constexpr int MAX_TERMINALS = 4096; static constexpr int MAX_PORTS_PER_SWITCH = 128; +/* + * This upper bound is intentionally part of the event-message footprint: every + * fluid_msg carries rc_allocs[MAX_RC_ALLOCATIONS] so optimistic execution can + * reverse a switch-egress allocation without external state. Keep + * PARAMS.message_size >= sizeof(fluid_msg), and reduce this bound only after + * measuring the maximum allocations per switch-egress event for the target + * topology/workload. + */ static constexpr int MAX_RC_ALLOCATIONS = 256; static constexpr double EPS = 1e-9; @@ -112,7 +116,6 @@ struct port_desc { int is_terminal; int target_index; double capacity_mbit_per_interval; - double buffer_mbit; }; struct terminal_state { @@ -188,7 +191,6 @@ struct fluid_msg { double rc_dropped_mbit; int rc_alloc_count; - int rc_scheduled_egress; int rc_prev_scheduled_egress_interval; int rc_log_target_is_terminal; @@ -267,10 +269,12 @@ static void validate_ross_message_size_or_abort(int rank) { fprintf(stderr, "fluid-switch error: PARAMS.message_size=%d is too small for " "sizeof(fluid_msg)=%zu. Increase PARAMS.message_size in the " - "CODES config before calling codes_mapping_setup().\n", - configured_message_size, required_message_size); + "CODES config before calling codes_mapping_setup(). " + "fluid_msg is large because MAX_RC_ALLOCATIONS=%d reverse " + "allocation records are carried in each event.\n", + configured_message_size, required_message_size, MAX_RC_ALLOCATIONS); } - MPI_Abort(MPI_COMM_WORLD, 1); + MPI_Abort(MPI_COMM_CODES, 1); } } @@ -918,34 +922,49 @@ static void compact_port_queue(switch_state* ns, int port_id) { } static bool fluid_commit_logging_active = false; +static std::ostringstream terminal_log_buffer; +static std::ostringstream switch_log_buffer; +static std::ostringstream flowlet_log_buffer; +static std::ostringstream switch_training_log_buffer; + +static bool fluid_optimistic_mode(void) { + return g_tw_synchronization_protocol == OPTIMISTIC || + g_tw_synchronization_protocol == OPTIMISTIC_DEBUG || + g_tw_synchronization_protocol == OPTIMISTIC_REALTIME; +} static bool fluid_csv_forward_logs_enabled(void) { - return g_tw_synchronization_protocol == SEQUENTIAL; + return !fluid_optimistic_mode(); } static bool fluid_csv_commit_logs_enabled(void) { - return g_tw_synchronization_protocol != SEQUENTIAL; + return fluid_optimistic_mode(); } static bool fluid_csv_logs_enabled(void) { - /* - * Sequential mode logs directly from forward handlers. Optimistic mode logs - * only from commit handlers, after ROSS guarantees the event will not roll - * back. The process-local flag lets the same append_* helpers be reused by - * both paths without permitting speculative forward-time appends. - */ return fluid_csv_forward_logs_enabled() || fluid_commit_logging_active; } -static void append_terminal_log(int interval_id, const char* event_name, int terminal_id, - int peer_id, double mbit) { - if (cfg.terminal_log_path[0] == '\0' || !fluid_csv_logs_enabled()) { +static void fluid_commit_logging_begin(void) { fluid_commit_logging_active = true; } + +static void fluid_commit_logging_end(void) { fluid_commit_logging_active = false; } + +static void append_log_row(const char* path, std::ostringstream* log_buffer, + const std::string& row) { + if (path[0] == '\0' || !fluid_csv_logs_enabled()) { return; } - std::ofstream out(cfg.terminal_log_path, std::ios::app); - out << interval_id << ',' << event_name << ',' << terminal_id << ',' + + (*log_buffer) << row; +} + +static void append_terminal_log(int interval_id, const char* event_name, int terminal_id, + int peer_id, double mbit) { + std::ostringstream row; + row << interval_id << ',' << event_name << ',' << terminal_id << ',' << terminals[terminal_id].name << ',' << terminals[terminal_id].switch_id << ',' << peer_id << ',' << mbit << '\n'; + append_log_row(cfg.terminal_log_path, &terminal_log_buffer, row.str()); } static void append_switch_log(int interval_id, const char* event_name, int switch_id, int port_id, @@ -957,20 +976,17 @@ static void append_switch_log(int interval_id, const char* event_name, int switc double shared_buffer_mbit, double dropped_mbit, int active_queue_entries) { - if (cfg.switch_log_path[0] == '\0' || !fluid_csv_logs_enabled()) { - return; - } - std::ofstream out(cfg.switch_log_path, std::ios::app); - out << interval_id << ',' << event_name << ',' << switch_id << ',' + std::ostringstream row; + row << interval_id << ',' << event_name << ',' << switch_id << ',' << switches[switch_id].name << ',' << port_id << ',' << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' << capacity_mbit << ',' << queued_before_mbit << ',' << sent_mbit << ',' << queued_after_mbit << ',' << shared_queued_before_mbit << ',' << shared_queued_after_mbit << ',' << shared_buffer_mbit << ',' << dropped_mbit << ',' << active_queue_entries << '\n'; + append_log_row(cfg.switch_log_path, &switch_log_buffer, row.str()); } - static void append_flowlet_log(int interval_id, const char* event_name, int switch_id, int port_id, int target_is_terminal, int target_index, unsigned long long flowlet_id, int source_terminal, @@ -978,17 +994,15 @@ static void append_flowlet_log(int interval_id, const char* event_name, int swit double capacity_mbit, double queued_before_mbit, double send_mbit, double remaining_after_mbit, double dropped_mbit) { - if (cfg.flowlet_log_path[0] == '\0' || !fluid_csv_logs_enabled()) { - return; - } - std::ofstream out(cfg.flowlet_log_path, std::ios::app); - out << interval_id << ',' << event_name << ',' << switch_id << ',' + std::ostringstream row; + row << interval_id << ',' << event_name << ',' << switch_id << ',' << switches[switch_id].name << ',' << port_id << ',' << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' << flowlet_id << ',' << source_terminal << ',' << destination_terminal << ',' << creation_interval << ',' << (interval_id - creation_interval) << ',' << capacity_mbit << ',' << queued_before_mbit << ',' << send_mbit << ',' << remaining_after_mbit << ',' << dropped_mbit << '\n'; + append_log_row(cfg.flowlet_log_path, &flowlet_log_buffer, row.str()); } static void append_switch_training_log(int interval_id, int switch_id, int port_id, @@ -998,22 +1012,20 @@ static void append_switch_training_log(int interval_id, int switch_id, int port_ double dropped_mbit, int active_before, int active_after, const std::vector& allocations) { - if (cfg.switch_training_log_path[0] == '\0' || !fluid_csv_logs_enabled()) { - return; - } - std::ofstream out(cfg.switch_training_log_path, std::ios::app); - out << interval_id << ',' << switch_id << ',' << switches[switch_id].name << ',' + std::ostringstream row; + row << interval_id << ',' << switch_id << ',' << switches[switch_id].name << ',' << port_id << ',' << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' << cfg.switch_scheduler << ',' << capacity_mbit << ',' << queued_before_mbit << ',' << sent_mbit << ',' << queued_after_mbit << ',' << dropped_mbit << ',' << active_before << ',' << active_after << ','; for (size_t i = 0; i < allocations.size(); ++i) { if (i > 0) { - out << ';'; + row << ';'; } - out << allocations[i]; + row << allocations[i]; } - out << '\n'; + row << '\n'; + append_log_row(cfg.switch_training_log_path, &switch_training_log_buffer, row.str()); } static int next_terminal_generate_interval_after(int interval_id) { @@ -1095,7 +1107,6 @@ static void request_switch_egress(switch_state* ns, int interval_id, int port_id int prev = ns->scheduled_egress_interval[port_id]; if (cause_msg != NULL) { - cause_msg->rc_scheduled_egress = 0; cause_msg->rc_prev_scheduled_egress_interval = prev; } @@ -1126,7 +1137,6 @@ static void request_switch_egress(switch_state* ns, int interval_id, int port_id ns->scheduled_egress_interval[port_id] = interval_id; if (cause_msg != NULL) { - cause_msg->rc_scheduled_egress = 1; } } @@ -1177,6 +1187,156 @@ static double random_workload_mbit(int terminal_id, tw_lp* lp) { return min_mbit + u * (max_mbit - min_mbit); } +static void build_allocation_strings_from_rc(const fluid_msg* m, + std::vector& allocations) { + allocations.clear(); + + for (int i = 0; i < m->rc_alloc_count; ++i) { + const rc_alloc_record* rc = &m->rc_allocs[i]; + + if (!rc->valid || rc->send_mbit <= EPS) { + continue; + } + + double remaining_after = rc->before.remaining_mbit - rc->send_mbit; + if (remaining_after < 0.0 && remaining_after > -EPS) { + remaining_after = 0.0; + } + + std::ostringstream ss; + ss << rc->before.flowlet_id << ':' << rc->before.source_terminal << ':' + << rc->before.destination_terminal << ':' << rc->before.creation_interval << ':' + << rc->before.remaining_mbit << ':' << rc->send_mbit << ':' << remaining_after; + allocations.push_back(ss.str()); + } +} + +static void log_terminal_generate_event(const terminal_state* ns, const fluid_msg* m) { + if (m->rc_generated) { + append_terminal_log(m->interval_id, "generate_send", ns->terminal_id, + m->destination_terminal, m->mbit); + } +} + +static void log_terminal_receive_event(const terminal_state* ns, const fluid_msg* m) { + append_terminal_log(m->interval_id, "receive", ns->terminal_id, + m->source_terminal, m->mbit); +} + +static void log_switch_arrival_event(const switch_state* ns, const fluid_msg* m) { + if (m->rc_no_route) { + append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, + 0.0, 0.0, 0.0, 0.0, + m->rc_log_shared_queued_before_mbit, + m->rc_log_shared_queued_after_mbit, + ns->shared_buffer_mbit, m->rc_dropped_mbit, 0); + append_flowlet_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, + m->flowlet_id, m->source_terminal, m->destination_terminal, + m->creation_interval, 0.0, 0.0, 0.0, 0.0, + m->rc_dropped_mbit); + return; + } + + if (m->rc_accepted_mbit > EPS) { + append_flowlet_log(m->interval_id, + m->rc_coalesced ? "enqueue_coalesce" : "enqueue", + ns->switch_id, m->rc_port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->flowlet_id, m->source_terminal, + m->destination_terminal, m->creation_interval, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + 0.0, + m->rc_log_flowlet_remaining_after_mbit, + 0.0); + } + + if (m->rc_dropped_mbit > EPS) { + append_switch_log(m->interval_id, "drop_shared_buffer_overflow", + ns->switch_id, m->rc_port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + 0.0, + m->rc_log_port_queued_after_mbit, + m->rc_log_shared_queued_before_mbit, + m->rc_log_shared_queued_after_mbit, + ns->shared_buffer_mbit, + m->rc_dropped_mbit, + m->rc_log_active_after_entries); + + append_flowlet_log(m->interval_id, "drop_shared_buffer_overflow", + ns->switch_id, m->rc_port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->flowlet_id, m->source_terminal, + m->destination_terminal, m->creation_interval, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + 0.0, + m->rc_log_flowlet_remaining_after_mbit, + m->rc_dropped_mbit); + } +} + +static void log_switch_egress_event(const switch_state* ns, const fluid_msg* m) { + for (int i = 0; i < m->rc_alloc_count; ++i) { + const rc_alloc_record* rc = &m->rc_allocs[i]; + + if (!rc->valid || rc->send_mbit <= EPS) { + continue; + } + + double remaining_after = rc->before.remaining_mbit - rc->send_mbit; + if (remaining_after < 0.0 && remaining_after > -EPS) { + remaining_after = 0.0; + } + + append_flowlet_log(m->interval_id, "allocate_send", + ns->switch_id, m->port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + rc->before.flowlet_id, + rc->before.source_terminal, + rc->before.destination_terminal, + rc->before.creation_interval, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + rc->send_mbit, + remaining_after, + 0.0); + } + + append_switch_log(m->interval_id, "egress", ns->switch_id, m->port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + m->rc_log_sent_total_mbit, + m->rc_log_port_queued_after_mbit, + m->rc_log_shared_queued_before_mbit, + m->rc_log_shared_queued_after_mbit, + ns->shared_buffer_mbit, + 0.0, + m->rc_log_active_after_entries); + + std::vector allocations; + build_allocation_strings_from_rc(m, allocations); + append_switch_training_log(m->interval_id, ns->switch_id, m->port_id, + m->rc_log_target_is_terminal, + m->rc_log_target_index, + m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, + m->rc_log_sent_total_mbit, + m->rc_log_port_queued_after_mbit, + 0.0, + m->rc_log_active_before_entries, + m->rc_log_active_after_entries, + allocations); +} + static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp) { int interval = m->interval_id; @@ -1222,7 +1382,7 @@ static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp m->flowlet_id = out_msg.flowlet_id; m->mbit = mbit; - append_terminal_log(interval, "generate_send", ns->terminal_id, dst, mbit); + log_terminal_generate_event(ns, m); if (cfg.debug_prints) { printf("[fluid terminal] interval=%d terminal=%d dst=%d mbit=%.6f\n", @@ -1238,7 +1398,7 @@ static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp static void handle_terminal_arrival(terminal_state* ns, fluid_msg* m) { ns->received_mbit += m->mbit; ns->received_fragments++; - append_terminal_log(m->interval_id, "receive", ns->terminal_id, m->source_terminal, m->mbit); + log_terminal_receive_event(ns, m); } static void terminal_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { @@ -1289,10 +1449,6 @@ static void terminal_rev_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp } } -static void fluid_commit_logging_begin(void) { fluid_commit_logging_active = true; } - -static void fluid_commit_logging_end(void) { fluid_commit_logging_active = false; } - static void terminal_commit_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw_lp* lp) { (void)b; (void)lp; @@ -1305,15 +1461,11 @@ static void terminal_commit_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw switch (m->event_type) { case WORKLOAD_GENERATE: - if (m->rc_generated) { - append_terminal_log(m->interval_id, "generate_send", ns->terminal_id, - m->destination_terminal, m->mbit); - } + log_terminal_generate_event(ns, m); break; case FLOWLET_ARRIVAL: - append_terminal_log(m->interval_id, "receive", ns->terminal_id, - m->source_terminal, m->mbit); + log_terminal_receive_event(ns, m); break; default: @@ -1323,31 +1475,6 @@ static void terminal_commit_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw fluid_commit_logging_end(); } -static void build_allocation_strings_from_rc(const fluid_msg* m, - std::vector& allocations) { - allocations.clear(); - - for (int i = 0; i < m->rc_alloc_count; ++i) { - const rc_alloc_record* rc = &m->rc_allocs[i]; - - if (!rc->valid || rc->send_mbit <= EPS) { - continue; - } - - double remaining_after = rc->before.remaining_mbit - rc->send_mbit; - if (remaining_after < 0.0 && remaining_after > -EPS) { - remaining_after = 0.0; - } - - std::ostringstream ss; - ss << rc->before.flowlet_id << ':' << rc->before.source_terminal << ':' - << rc->before.destination_terminal << ':' << rc->before.creation_interval << ':' - << rc->before.remaining_mbit << ':' << rc->send_mbit << ':' << remaining_after; - allocations.push_back(ss.str()); - } -} - - static void terminal_finalize(terminal_state* ns, tw_lp* lp) { printf("fluid-terminal gid=%llu terminal=%d switch=%d generated_mbit=%.6f sent_mbit=%.6f " "received_mbit=%.6f generated_flowlets=%d received_fragments=%d\n", @@ -1388,7 +1515,6 @@ static void switch_init(switch_state* ns, tw_lp* lp) { p->is_terminal = 0; p->target_index = sw.links[i].dst_switch; p->capacity_mbit_per_interval = sw.links[i].bandwidth_mbps * cfg.interval_seconds; - p->buffer_mbit = 0.0; /* buffer capacity is switch-wide, not per port */ ns->queues[ns->num_ports - 1] = new std::vector(); } for (int t = 0; t < sw.terminal_count; ++t) { @@ -1400,7 +1526,6 @@ static void switch_init(switch_state* ns, tw_lp* lp) { p->is_terminal = 1; p->target_index = sw.terminal_start + t; p->capacity_mbit_per_interval = sw.terminal_bandwidth_mbps * cfg.interval_seconds; - p->buffer_mbit = 0.0; /* buffer capacity is switch-wide, not per port */ ns->queues[ns->num_ports - 1] = new std::vector(); } @@ -1432,7 +1557,6 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m, tw_lp* lp) { m->rc_port_id = -1; m->rc_accepted_mbit = 0.0; m->rc_dropped_mbit = m->mbit; - m->rc_scheduled_egress = 0; m->rc_prev_scheduled_egress_interval = -1; m->rc_log_target_is_terminal = 0; m->rc_log_target_index = -1; @@ -1446,12 +1570,7 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m, tw_lp* lp) { m->rc_log_active_before_entries = 0; m->rc_log_active_after_entries = 0; ns->dropped_mbit += m->mbit; - append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, - 0.0, 0.0, 0.0, 0.0, shared_before, shared_before, - ns->shared_buffer_mbit, m->mbit, 0); - append_flowlet_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, - m->flowlet_id, m->source_terminal, m->destination_terminal, - m->creation_interval, 0.0, 0.0, 0.0, 0.0, m->mbit); + log_switch_arrival_event(ns, m); return; } @@ -1470,7 +1589,6 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m, tw_lp* lp) { m->rc_coalesced = 0; m->rc_accepted_mbit = 0.0; m->rc_dropped_mbit = 0.0; - m->rc_scheduled_egress = 0; m->rc_prev_scheduled_egress_interval = ns->scheduled_egress_interval[port_id]; m->rc_log_target_is_terminal = p->is_terminal; m->rc_log_target_index = p->target_index; @@ -1504,25 +1622,7 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m, tw_lp* lp) { m->rc_log_flowlet_remaining_after_mbit = flowlet_remaining_after; m->rc_log_active_after_entries = active_after; - if (accepted > EPS) { - append_flowlet_log(m->interval_id, coalesced ? "enqueue_coalesce" : "enqueue", - ns->switch_id, port_id, p->is_terminal, p->target_index, - m->flowlet_id, m->source_terminal, m->destination_terminal, - m->creation_interval, p->capacity_mbit_per_interval, - port_queued_before, 0.0, flowlet_remaining_after, 0.0); - } - if (dropped > EPS) { - append_switch_log(m->interval_id, "drop_shared_buffer_overflow", ns->switch_id, - port_id, p->is_terminal, p->target_index, - p->capacity_mbit_per_interval, port_queued_before, 0.0, - port_queued_after, shared_queued_before, shared_queued_after, - ns->shared_buffer_mbit, dropped, active_after); - append_flowlet_log(m->interval_id, "drop_shared_buffer_overflow", ns->switch_id, - port_id, p->is_terminal, p->target_index, m->flowlet_id, - m->source_terminal, m->destination_terminal, m->creation_interval, - p->capacity_mbit_per_interval, port_queued_before, 0.0, - flowlet_remaining_after, dropped); - } + log_switch_arrival_event(ns, m); if (port_queued_after > EPS) { request_switch_egress(ns, m->interval_id, port_id, lp, m); @@ -1568,7 +1668,6 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { } if (ns->queues[port_id] == NULL) { m->rc_prev_scheduled_egress_interval = -1; - m->rc_scheduled_egress = 0; return; } @@ -1576,7 +1675,6 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { std::vector& qv = *ns->queues[port_id]; m->rc_prev_scheduled_egress_interval = ns->scheduled_egress_interval[port_id]; - m->rc_scheduled_egress = 0; if (ns->scheduled_egress_interval[port_id] == m->interval_id) { ns->scheduled_egress_interval[port_id] = -1; } @@ -1588,7 +1686,6 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { double shared_queued_before = queued_mbit_on_switch(ns); int active_before = active_flowlet_count_on_port(ns, port_id); - std::vector allocations; std::vector send_plan(qv.size(), 0.0); m->rc_alloc_count = 0; m->rc_log_target_is_terminal = p->is_terminal; @@ -1618,21 +1715,6 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { return actual_send; }; - auto record_allocation = [&](const queued_flowlet& before, double send, - double remaining_after) { - append_flowlet_log(m->interval_id, "allocate_send", ns->switch_id, port_id, - p->is_terminal, p->target_index, before.flowlet_id, - before.source_terminal, before.destination_terminal, - before.creation_interval, capacity, queued_before, send, - remaining_after, 0.0); - - std::ostringstream ss; - ss << before.flowlet_id << ':' << before.source_terminal << ':' - << before.destination_terminal << ':' << before.creation_interval << ':' - << before.remaining_mbit << ':' << send << ':' << remaining_after; - allocations.push_back(ss.str()); - }; - if (strcmp(cfg.switch_scheduler, "fifo") == 0) { for (int i = 0; i < (int)qv.size() && remaining_capacity > EPS; ++i) { if (!qv[i].valid || qv[i].remaining_mbit <= EPS) { @@ -1720,7 +1802,6 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { qv[i].remaining_mbit -= send; sent_total += send; - record_allocation(before, send, qv[i].remaining_mbit); } compact_port_queue(ns, port_id); @@ -1734,14 +1815,7 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { m->rc_log_sent_total_mbit = sent_total; m->rc_log_active_after_entries = active_after; - append_switch_log(m->interval_id, "egress", ns->switch_id, port_id, p->is_terminal, - p->target_index, capacity, queued_before, sent_total, queued_after, - shared_queued_before, shared_queued_after, ns->shared_buffer_mbit, - 0.0, active_after); - - append_switch_training_log(m->interval_id, ns->switch_id, port_id, p->is_terminal, - p->target_index, capacity, queued_before, sent_total, queued_after, - 0.0, active_before, active_after, allocations); + log_switch_egress_event(ns, m); if (queued_after > EPS) { /* @@ -1903,119 +1977,12 @@ static void switch_commit_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* switch (m->event_type) { case FLOWLET_ARRIVAL: - if (m->rc_no_route) { - append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, - 0.0, 0.0, 0.0, 0.0, - m->rc_log_shared_queued_before_mbit, - m->rc_log_shared_queued_after_mbit, - ns->shared_buffer_mbit, m->rc_dropped_mbit, 0); - append_flowlet_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, - m->flowlet_id, m->source_terminal, m->destination_terminal, - m->creation_interval, 0.0, 0.0, 0.0, 0.0, - m->rc_dropped_mbit); - } else { - if (m->rc_accepted_mbit > EPS) { - append_flowlet_log(m->interval_id, - m->rc_coalesced ? "enqueue_coalesce" : "enqueue", - ns->switch_id, m->rc_port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->flowlet_id, m->source_terminal, - m->destination_terminal, m->creation_interval, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - 0.0, - m->rc_log_flowlet_remaining_after_mbit, - 0.0); - } - - if (m->rc_dropped_mbit > EPS) { - append_switch_log(m->interval_id, "drop_shared_buffer_overflow", - ns->switch_id, m->rc_port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - 0.0, - m->rc_log_port_queued_after_mbit, - m->rc_log_shared_queued_before_mbit, - m->rc_log_shared_queued_after_mbit, - ns->shared_buffer_mbit, - m->rc_dropped_mbit, - m->rc_log_active_after_entries); - - append_flowlet_log(m->interval_id, "drop_shared_buffer_overflow", - ns->switch_id, m->rc_port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->flowlet_id, m->source_terminal, - m->destination_terminal, m->creation_interval, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - 0.0, - m->rc_log_flowlet_remaining_after_mbit, - m->rc_dropped_mbit); - } - } + log_switch_arrival_event(ns, m); break; - case SWITCH_EGRESS: { - std::vector allocations; - - for (int i = 0; i < m->rc_alloc_count; ++i) { - const rc_alloc_record* rc = &m->rc_allocs[i]; - - if (!rc->valid || rc->send_mbit <= EPS) { - continue; - } - - double remaining_after = rc->before.remaining_mbit - rc->send_mbit; - if (remaining_after < 0.0 && remaining_after > -EPS) { - remaining_after = 0.0; - } - - append_flowlet_log(m->interval_id, "allocate_send", - ns->switch_id, m->port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - rc->before.flowlet_id, - rc->before.source_terminal, - rc->before.destination_terminal, - rc->before.creation_interval, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - rc->send_mbit, - remaining_after, - 0.0); - } - - append_switch_log(m->interval_id, "egress", ns->switch_id, m->port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - m->rc_log_sent_total_mbit, - m->rc_log_port_queued_after_mbit, - m->rc_log_shared_queued_before_mbit, - m->rc_log_shared_queued_after_mbit, - ns->shared_buffer_mbit, - 0.0, - m->rc_log_active_after_entries); - - build_allocation_strings_from_rc(m, allocations); - append_switch_training_log(m->interval_id, ns->switch_id, m->port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - m->rc_log_sent_total_mbit, - m->rc_log_port_queued_after_mbit, - 0.0, - m->rc_log_active_before_entries, - m->rc_log_active_after_entries, - allocations); + case SWITCH_EGRESS: + log_switch_egress_event(ns, m); break; - } default: break; @@ -2024,7 +1991,6 @@ static void switch_commit_event(switch_state* ns, tw_bf* b, fluid_msg* m, tw_lp* fluid_commit_logging_end(); } - static void switch_finalize(switch_state* ns, tw_lp* lp) { double queued = 0.0; for (int p = 0; p < ns->num_ports; ++p) { @@ -2037,9 +2003,14 @@ static void switch_finalize(switch_state* ns, tw_lp* lp) { ns->num_ports, ns->shared_buffer_mbit, ns->received_fragments, ns->sent_fragments, ns->enqueued_mbit, ns->sent_mbit, ns->delivered_local_mbit, ns->dropped_mbit, queued); + + for (int p = 0; p < ns->num_ports; ++p) { + delete ns->queues[p]; + ns->queues[p] = NULL; + } } -const tw_optdef app_opt[] = {TWOPT_GROUP("model-net interval-fluid switch/terminal workload"), +const tw_optdef app_opt[] = {TWOPT_GROUP("interval-fluid switch/terminal workload"), TWOPT_END()}; static const char* find_config_arg(int argc, char** argv) { @@ -2056,38 +2027,112 @@ static const char* find_config_arg(int argc, char** argv) { return NULL; } -static void write_log_headers(int rank) { - if (rank != 0) { +static void write_log_header_file(const char* path, const char* header) { + if (path[0] == '\0') { + return; + } + + std::remove(path); + + std::ofstream out(path, std::ios::out | std::ios::trunc); + if (!out) { + tw_error(TW_LOC, "could not open CSV log file '%s' for header", path); + } + out << header; +} + +static void append_merged_rows_to_file(const char* path, const char* data, int len) { + if (path[0] == '\0' || len <= 0) { + return; + } + + std::ofstream out(path, std::ios::app); + if (!out) { + tw_error(TW_LOC, "could not open CSV log file '%s' for merged append", path); + } + out.write(data, len); +} + +static void merge_log_buffer(const char* path, const std::ostringstream& buffer, MPI_Comm comm) { + if (path[0] == '\0') { return; } - if (cfg.terminal_log_path[0] != '\0') { - std::remove(cfg.terminal_log_path); - std::ofstream out(cfg.terminal_log_path, std::ios::app); - out << "interval,event,terminal,terminal_name,attached_switch,peer_terminal,mbit\n"; + + int rank = 0; + int size = 1; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &size); + + std::string local_rows = buffer.str(); + int local_len = (int)local_rows.size(); + + std::vector recv_counts; + std::vector displs; + if (rank == 0) { + recv_counts.resize(size, 0); + displs.resize(size, 0); } - if (cfg.switch_log_path[0] != '\0') { - std::remove(cfg.switch_log_path); - std::ofstream out(cfg.switch_log_path, std::ios::app); - out << "interval,event,switch,switch_name,port,target_type,target_index,capacity_mbit," - "queued_before_mbit,sent_mbit,queued_after_mbit,shared_queued_before_mbit," - "shared_queued_after_mbit,shared_buffer_mbit,dropped_mbit,active_queue_entries\n"; + + MPI_Gather(&local_len, 1, MPI_INT, + rank == 0 ? recv_counts.data() : NULL, 1, MPI_INT, 0, comm); + + int total_len = 0; + if (rank == 0) { + for (int i = 0; i < size; ++i) { + displs[i] = total_len; + total_len += recv_counts[i]; + } } - if (cfg.flowlet_log_path[0] != '\0') { - std::remove(cfg.flowlet_log_path); - std::ofstream out(cfg.flowlet_log_path, std::ios::app); - out << "interval,event,switch,switch_name,port,target_type,target_index,flowlet_id," - "source_terminal,destination_terminal,creation_interval,age_intervals,capacity_mbit," - "queued_before_mbit,send_mbit,remaining_after_mbit,dropped_mbit\n"; + + std::vector merged; + if (rank == 0 && total_len > 0) { + merged.resize(total_len); } - if (cfg.switch_training_log_path[0] != '\0') { - std::remove(cfg.switch_training_log_path); - std::ofstream out(cfg.switch_training_log_path, std::ios::app); - out << "interval,switch,switch_name,port,target_type,target_index,scheduler,capacity_mbit," - "queued_before_mbit,sent_mbit,queued_after_mbit,dropped_mbit,active_before," - "active_after,allocations\n"; + + MPI_Gatherv(local_len > 0 ? local_rows.data() : NULL, local_len, MPI_CHAR, + rank == 0 && total_len > 0 ? merged.data() : NULL, + rank == 0 ? recv_counts.data() : NULL, + rank == 0 ? displs.data() : NULL, + MPI_CHAR, 0, comm); + + if (rank == 0 && total_len > 0) { + append_merged_rows_to_file(path, merged.data(), total_len); } } +static void merge_all_log_buffers(MPI_Comm comm) { + merge_log_buffer(cfg.terminal_log_path, terminal_log_buffer, comm); + merge_log_buffer(cfg.switch_log_path, switch_log_buffer, comm); + merge_log_buffer(cfg.flowlet_log_path, flowlet_log_buffer, comm); + merge_log_buffer(cfg.switch_training_log_path, switch_training_log_buffer, comm); +} + +static void write_log_headers(int rank) { + if (rank != 0) { + return; + } + + write_log_header_file(cfg.terminal_log_path, + "interval,event,terminal,terminal_name,attached_switch,peer_terminal,mbit\n"); + + write_log_header_file(cfg.switch_log_path, + "interval,event,switch,switch_name,port,target_type,target_index," + "capacity_mbit,queued_before_mbit,sent_mbit,queued_after_mbit," + "shared_queued_before_mbit,shared_queued_after_mbit," + "shared_buffer_mbit,dropped_mbit,active_queue_entries\n"); + + write_log_header_file(cfg.flowlet_log_path, + "interval,event,switch,switch_name,port,target_type,target_index," + "flowlet_id,source_terminal,destination_terminal,creation_interval," + "age_intervals,capacity_mbit,queued_before_mbit,send_mbit," + "remaining_after_mbit,dropped_mbit\n"); + + write_log_header_file(cfg.switch_training_log_path, + "interval,switch,switch_name,port,target_type,target_index,scheduler," + "capacity_mbit,queued_before_mbit,sent_mbit,queued_after_mbit," + "dropped_mbit,active_before,active_after,allocations\n"); +} + int main(int argc, char** argv) { int rank = 0; @@ -2103,8 +2148,8 @@ int main(int argc, char** argv) { return 1; } - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - configuration_load(config_file, MPI_COMM_WORLD, &config); + MPI_Comm_rank(MPI_COMM_CODES, &rank); + configuration_load(config_file, MPI_COMM_CODES, &config); load_config(); validate_ross_message_size_or_abort(rank); @@ -2123,7 +2168,7 @@ int main(int argc, char** argv) { } write_log_headers(rank); - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(MPI_COMM_CODES); if (rank == 0) { printf("fluid-switch config: switches=%zu terminals=%zu interval_seconds=%.6f " @@ -2131,11 +2176,12 @@ int main(int argc, char** argv) { "buffer_mode=shared csv_logs=%s ross_message_size=%d fluid_msg_size=%zu\n", switches.size(), terminals.size(), cfg.interval_seconds, cfg.num_send_intervals, cfg.num_drain_intervals, cfg.switch_scheduler, - fluid_csv_forward_logs_enabled() ? "forward" : "commit", + fluid_csv_forward_logs_enabled() ? "buffered-forward" : "buffered-commit", get_configured_message_size_bytes(), sizeof(fluid_msg)); } tw_run(); + merge_all_log_buffers(MPI_COMM_CODES); tw_end(); return 0; } From bf42b2a7c8cfd65cef7f554398e31d95bca58ca8 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 11:41:47 -0400 Subject: [PATCH 10/20] Add topology generator script for fluid flow simn --- .../generate-fluid-switch-topology.py | 310 ++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100755 src/network-workloads/generate-fluid-switch-topology.py diff --git a/src/network-workloads/generate-fluid-switch-topology.py b/src/network-workloads/generate-fluid-switch-topology.py new file mode 100755 index 00000000..b2acdf43 --- /dev/null +++ b/src/network-workloads/generate-fluid-switch-topology.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +""" +Generate a WAN-like topology YAML file for model-net-fluid-switch. + +The generated graph is intentionally not a symmetric supercomputer-style +topology. It starts with a bidirectional ring to guarantee strong connectivity, +then adds random directed/asymmetric switch-to-switch links. Terminal access +links are generated with higher bandwidth than switch-to-switch links. +""" + +from __future__ import annotations + +import argparse +import random +import sys +from pathlib import Path + + +def positive_int(value: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"expected integer, got {value!r}") from exc + if parsed <= 0: + raise argparse.ArgumentTypeError(f"expected positive integer, got {parsed}") + return parsed + + +def nonnegative_int(value: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"expected integer, got {value!r}") from exc + if parsed < 0: + raise argparse.ArgumentTypeError(f"expected nonnegative integer, got {parsed}") + return parsed + + +def positive_float(value: str) -> float: + try: + parsed = float(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"expected float, got {value!r}") from exc + if parsed <= 0.0: + raise argparse.ArgumentTypeError(f"expected positive float, got {parsed}") + return parsed + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate a random WAN-like topology YAML for model-net-fluid-switch." + ) + + parser.add_argument( + "--switches", + type=positive_int, + default=64, + help="number of switches to generate; default: 64", + ) + parser.add_argument( + "--terminals-per-switch", + type=positive_int, + default=2, + help="number of terminals attached to each switch; default: 2", + ) + parser.add_argument( + "--avg-switch-degree", + type=positive_float, + default=3.0, + help=( + "target average directed switch out-degree, including the ring links; " + "default: 3.0" + ), + ) + parser.add_argument( + "--reverse-link-probability", + type=positive_float, + default=0.35, + help=( + "probability of adding the reverse direction for each random extra link; " + "default: 0.35" + ), + ) + parser.add_argument( + "--switch-link-min-mbps", + type=positive_float, + default=5.0, + help="minimum switch-to-switch bandwidth in Mbps; default: 5", + ) + parser.add_argument( + "--switch-link-max-mbps", + type=positive_float, + default=25.0, + help="maximum switch-to-switch bandwidth in Mbps; default: 25", + ) + parser.add_argument( + "--terminal-link-min-mbps", + type=positive_float, + default=80.0, + help="minimum terminal-to-switch bandwidth in Mbps; default: 80", + ) + parser.add_argument( + "--terminal-link-max-mbps", + type=positive_float, + default=160.0, + help="maximum terminal-to-switch bandwidth in Mbps; default: 160", + ) + parser.add_argument( + "--switch-buffer-min-mb", + type=positive_float, + default=1000.0, + help="minimum shared switch buffer in Mbit; default: 1000", + ) + parser.add_argument( + "--switch-buffer-max-mb", + type=positive_float, + default=4000.0, + help="maximum shared switch buffer in Mbit; default: 4000", + ) + parser.add_argument( + "--seed", + type=nonnegative_int, + default=12345, + help="random seed; default: 12345", + ) + parser.add_argument( + "--output", + type=Path, + default=Path("doc/example/fluid-switch-topology.generated.yaml"), + help=( + "output YAML path, relative to the current working directory unless absolute; " + "default: doc/example/fluid-switch-topology.generated.yaml" + ), + ) + parser.add_argument( + "--name-prefix", + default="S", + help="switch name prefix; default: S", + ) + + args = parser.parse_args() + + if args.switches < 2: + parser.error("--switches must be at least 2 to form a connected switch graph") + + if args.avg_switch_degree < 2.0: + parser.error( + "--avg-switch-degree must be at least 2.0 because the generator " + "uses a bidirectional ring backbone" + ) + + if args.reverse_link_probability < 0.0 or args.reverse_link_probability > 1.0: + parser.error("--reverse-link-probability must be in [0, 1]") + + if args.switch_link_min_mbps > args.switch_link_max_mbps: + parser.error("--switch-link-min-mbps cannot exceed --switch-link-max-mbps") + + if args.terminal_link_min_mbps > args.terminal_link_max_mbps: + parser.error("--terminal-link-min-mbps cannot exceed --terminal-link-max-mbps") + + if args.terminal_link_min_mbps <= args.switch_link_max_mbps: + parser.error( + "terminal access links must be faster than switch-to-switch links: " + "require --terminal-link-min-mbps > --switch-link-max-mbps" + ) + + if args.switch_buffer_min_mb > args.switch_buffer_max_mb: + parser.error("--switch-buffer-min-mb cannot exceed --switch-buffer-max-mb") + + return args + + +def rand_uniform_rounded(rng: random.Random, lo: float, hi: float) -> float: + return round(rng.uniform(lo, hi), 3) + + +def add_link( + links: dict[int, dict[int, float]], + src: int, + dst: int, + bandwidth_mbps: float, +) -> bool: + if src == dst: + return False + if dst in links[src]: + return False + links[src][dst] = bandwidth_mbps + return True + + +def generate_links(args: argparse.Namespace, rng: random.Random) -> dict[int, dict[int, float]]: + n = args.switches + links: dict[int, dict[int, float]] = {i: {} for i in range(n)} + + def new_switch_bw() -> float: + return rand_uniform_rounded( + rng, args.switch_link_min_mbps, args.switch_link_max_mbps + ) + + # Strongly connected but asymmetric backbone. Each direction gets its own + # independently sampled capacity. + for i in range(n): + add_link(links, i, (i + 1) % n, new_switch_bw()) + add_link(links, i, (i - 1) % n, new_switch_bw()) + + target_edges = max(int(round(args.switches * args.avg_switch_degree)), 2 * n) + max_edges = n * (n - 1) + if target_edges > max_edges: + target_edges = max_edges + + current_edges = sum(len(v) for v in links.values()) + attempts = 0 + max_attempts = max(1000, 50 * max_edges) + + while current_edges < target_edges and attempts < max_attempts: + attempts += 1 + src = rng.randrange(n) + dst = rng.randrange(n) + if src == dst: + continue + + if add_link(links, src, dst, new_switch_bw()): + current_edges += 1 + + if current_edges >= target_edges: + break + + if rng.random() < args.reverse_link_probability: + if add_link(links, dst, src, new_switch_bw()): + current_edges += 1 + + if current_edges < target_edges: + raise RuntimeError( + f"could only generate {current_edges} directed switch links; " + f"target was {target_edges}" + ) + + return links + + +def switch_name(index: int, n_switches: int, prefix: str) -> str: + width = max(2, len(str(n_switches - 1))) + return f"{prefix}{index:0{width}d}" + + +def write_topology(args: argparse.Namespace, links: dict[int, dict[int, float]]) -> None: + rng = random.Random(args.seed + 1) + n = args.switches + names = [switch_name(i, n, args.name_prefix) for i in range(n)] + + args.output.parent.mkdir(parents=True, exist_ok=True) + + with args.output.open("w", encoding="utf-8") as f: + f.write("# Generated by src/network-workloads/generate-fluid-switch-topology.py\n") + f.write(f"# switches={args.switches}\n") + f.write(f"# terminals_per_switch={args.terminals_per_switch}\n") + f.write(f"# avg_switch_degree={args.avg_switch_degree}\n") + f.write(f"# seed={args.seed}\n") + f.write("topology:\n") + f.write(" switches:\n") + + for i in range(n): + terminal_bw = rand_uniform_rounded( + rng, args.terminal_link_min_mbps, args.terminal_link_max_mbps + ) + switch_buffer = rand_uniform_rounded( + rng, args.switch_buffer_min_mb, args.switch_buffer_max_mb + ) + + f.write(f" {names[i]}:\n") + f.write(f" terminals: {args.terminals_per_switch}\n") + f.write(f' terminal_bandwidth: "{terminal_bw} Mbps"\n') + f.write(f' switch_buffer: "{switch_buffer} Mb"\n') + f.write(" connections:\n") + + for dst in sorted(links[i]): + f.write(f' {names[dst]}: "{links[i][dst]} Mbps"\n') + + f.write("\n") + + +def summarize(args: argparse.Namespace, links: dict[int, dict[int, float]]) -> None: + edge_count = sum(len(v) for v in links.values()) + degrees = [len(links[i]) for i in range(args.switches)] + bws = [bw for edges in links.values() for bw in edges.values()] + + print(f"wrote {args.output}") + print(f"switches: {args.switches}") + print(f"terminals: {args.switches * args.terminals_per_switch}") + print(f"directed switch links: {edge_count}") + print(f"average directed switch out-degree: {edge_count / args.switches:.3f}") + print(f"min/max directed switch out-degree: {min(degrees)} / {max(degrees)}") + print(f"switch-link Mbps min/max: {min(bws):.3f} / {max(bws):.3f}") + print( + "terminal-link Mbps range: " + f"{args.terminal_link_min_mbps:.3f} / {args.terminal_link_max_mbps:.3f}" + ) + + +def main() -> int: + args = parse_args() + rng = random.Random(args.seed) + links = generate_links(args, rng) + write_topology(args, links) + summarize(args, links) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From b01e38652147ee1da6d9770bbb8a648e166f9b73 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 13:35:14 -0400 Subject: [PATCH 11/20] Replace max_min_fair with round robin for fluid flow simn --- doc/example/fluid-switch.conf.in | 5 +- .../model-net-fluid-switch.cxx | 124 +++++++++++++----- 2 files changed, 95 insertions(+), 34 deletions(-) diff --git a/doc/example/fluid-switch.conf.in b/doc/example/fluid-switch.conf.in index 5339fbd5..216c88c1 100644 --- a/doc/example/fluid-switch.conf.in +++ b/doc/example/fluid-switch.conf.in @@ -33,7 +33,10 @@ FLUID_SWITCH terminal_max_send_fraction_of_link_capacity="1.0"; debug_prints="0"; - switch_scheduler="max_min_fair"; + # This can be either round_robin or fifo. + switch_scheduler="round_robin"; + round_robin_max_entries_per_egress="8"; + round_robin_quantum_mbit="0"; terminal_log_path="../../zmqml_artifacts/fluid-switch/terminal-events.csv"; switch_log_path="../../zmqml_artifacts/fluid-switch/switch-events.csv"; diff --git a/src/network-workloads/model-net-fluid-switch.cxx b/src/network-workloads/model-net-fluid-switch.cxx index faa1cf97..c29a8ea0 100644 --- a/src/network-workloads/model-net-fluid-switch.cxx +++ b/src/network-workloads/model-net-fluid-switch.cxx @@ -90,7 +90,9 @@ struct sim_config { char switch_log_path[1024] = ""; char flowlet_log_path[1024] = ""; char switch_training_log_path[1024] = ""; - char switch_scheduler[64] = "max_min_fair"; + char switch_scheduler[64] = "round_robin"; + int round_robin_max_entries_per_egress = 32; + double round_robin_quantum_mbit = 0.0; }; static sim_config cfg; @@ -143,6 +145,7 @@ struct switch_state { * outstanding for that port. */ int scheduled_egress_interval[MAX_PORTS_PER_SWITCH]; + int rr_next_queue_index[MAX_PORTS_PER_SWITCH]; double enqueued_mbit; double sent_mbit; @@ -192,6 +195,7 @@ struct fluid_msg { int rc_alloc_count; int rc_prev_scheduled_egress_interval; + int rc_prev_rr_next_queue_index; int rc_log_target_is_terminal; int rc_log_target_index; @@ -617,6 +621,10 @@ static void load_config(void) { sizeof(cfg.switch_training_log_path)); read_string_param("FLUID_SWITCH", "switch_scheduler", cfg.switch_scheduler, sizeof(cfg.switch_scheduler)); + read_int_param("FLUID_SWITCH", "round_robin_max_entries_per_egress", + &cfg.round_robin_max_entries_per_egress); + read_double_param("FLUID_SWITCH", "round_robin_quantum_mbit", + &cfg.round_robin_quantum_mbit); read_double_param("FLUID_SWITCH", "interval_seconds", &cfg.interval_seconds); read_int_param("FLUID_SWITCH", "num_send_intervals", &cfg.num_send_intervals); read_int_param("FLUID_SWITCH", "num_drain_intervals", &cfg.num_drain_intervals); @@ -628,13 +636,32 @@ static void load_config(void) { &cfg.terminal_max_send_fraction_of_link_capacity); read_int_param("FLUID_SWITCH", "debug_prints", &cfg.debug_prints); - if (strcmp(cfg.switch_scheduler, "max_min_fair") != 0 && - strcmp(cfg.switch_scheduler, "fifo") != 0) { + if (strcmp(cfg.switch_scheduler, "fifo") != 0 && + strcmp(cfg.switch_scheduler, "round_robin") != 0) { tw_error(TW_LOC, - "FLUID_SWITCH.switch_scheduler must be one of: max_min_fair, fifo; got '%s'", + "FLUID_SWITCH.switch_scheduler must be one of: fifo, round_robin; got '%s'", cfg.switch_scheduler); } + if (cfg.round_robin_max_entries_per_egress <= 0) { + tw_error(TW_LOC, + "FLUID_SWITCH.round_robin_max_entries_per_egress must be positive; got %d", + cfg.round_robin_max_entries_per_egress); + } + + if (cfg.round_robin_max_entries_per_egress > MAX_RC_ALLOCATIONS) { + tw_error(TW_LOC, + "FLUID_SWITCH.round_robin_max_entries_per_egress=%d exceeds " + "MAX_RC_ALLOCATIONS=%d", + cfg.round_robin_max_entries_per_egress, MAX_RC_ALLOCATIONS); + } + + if (cfg.round_robin_quantum_mbit < 0.0) { + tw_error(TW_LOC, + "FLUID_SWITCH.round_robin_quantum_mbit must be nonnegative; got %.12f", + cfg.round_robin_quantum_mbit); + } + if (cfg.topology_yaml_file[0] == '\0') { tw_error(TW_LOC, "FLUID_SWITCH.topology_yaml_file is required"); } @@ -1492,6 +1519,7 @@ static void switch_init(switch_state* ns, tw_lp* lp) { for (int p = 0; p < MAX_PORTS_PER_SWITCH; ++p) { ns->scheduled_egress_interval[p] = -1; + ns->rr_next_queue_index[p] = 0; } const switch_info& sw = switches[ns->switch_id]; @@ -1668,6 +1696,7 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { } if (ns->queues[port_id] == NULL) { m->rc_prev_scheduled_egress_interval = -1; + m->rc_prev_rr_next_queue_index = 0; return; } @@ -1675,6 +1704,7 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { std::vector& qv = *ns->queues[port_id]; m->rc_prev_scheduled_egress_interval = ns->scheduled_egress_interval[port_id]; + m->rc_prev_rr_next_queue_index = ns->rr_next_queue_index[port_id]; if (ns->scheduled_egress_interval[port_id] == m->interval_id) { ns->scheduled_egress_interval[port_id] = -1; } @@ -1725,44 +1755,59 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { double planned_send = add_to_plan(i, requested_send); remaining_capacity -= planned_send; } - } else { - std::vector virtual_remaining(qv.size(), 0.0); - std::vector active; - - for (int i = 0; i < (int)qv.size(); ++i) { - if (qv[i].valid && qv[i].remaining_mbit > EPS) { - virtual_remaining[i] = qv[i].remaining_mbit; - active.push_back(i); + } else if (strcmp(cfg.switch_scheduler, "round_robin") == 0) { + /* + * Bounded round-robin: + * - keep a persistent per-port cursor, + * - visit queued flowlets circularly, + * - serve at most round_robin_max_entries_per_egress flowlets, + * - use a fixed quantum if configured, otherwise use one full + * output-port interval as the quantum. + * + * This avoids touching hundreds of active flowlets in one egress event, + * which keeps reverse metadata, logs, and optimistic rollback bounded. + */ + int qsize = (int)qv.size(); + int entries_to_serve = std::min(active_before, + cfg.round_robin_max_entries_per_egress); + + if (qsize > 0 && entries_to_serve > 0 && remaining_capacity > EPS) { + int idx = ns->rr_next_queue_index[port_id]; + if (idx < 0 || idx >= qsize) { + idx = 0; } - } - while (!active.empty() && remaining_capacity > EPS) { - double share = remaining_capacity / (double)active.size(); - std::vector still_active; - bool made_progress = false; - - for (size_t ai = 0; ai < active.size(); ++ai) { - int idx = active[ai]; + double quantum = cfg.round_robin_quantum_mbit; + if (quantum <= EPS) { + /* + * Default round-robin quantum is one full output-port interval. + * This keeps the scheduler fair over time without splitting one + * interval into many tiny fragments. + */ + quantum = capacity; + } - double planned_send = std::min(virtual_remaining[idx], share); - if (planned_send > EPS) { - send_plan[idx] += planned_send; - virtual_remaining[idx] -= planned_send; + int visited = 0; + int served = 0; + while (remaining_capacity > EPS && visited < qsize && served < entries_to_serve) { + if (qv[idx].valid && qv[idx].remaining_mbit > EPS) { + double requested_send = std::min(qv[idx].remaining_mbit, quantum); + requested_send = std::min(requested_send, remaining_capacity); + double planned_send = add_to_plan(idx, requested_send); remaining_capacity -= planned_send; - made_progress = true; + if (planned_send > EPS) { + served++; + } } - if (virtual_remaining[idx] > EPS) { - still_active.push_back(idx); - } + idx = (idx + 1) % qsize; + visited++; } - active.swap(still_active); - - if (!made_progress) { - break; - } + ns->rr_next_queue_index[port_id] = idx; } + } else { + tw_error(TW_LOC, "unknown switch scheduler '%s'", cfg.switch_scheduler); } for (int i = 0; i < (int)qv.size(); ++i) { @@ -1806,6 +1851,18 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { compact_port_queue(ns, port_id); + /* + * This keeps the round-robin cursor valid after completed flowlets are removed. + * The cursor is restored exactly by rollback through rc_prev_rr_next_queue_index. + */ + if (strcmp(cfg.switch_scheduler, "round_robin") == 0) { + if (qv.empty()) { + ns->rr_next_queue_index[port_id] = 0; + } else { + ns->rr_next_queue_index[port_id] %= (int)qv.size(); + } + } + double queued_after = queued_mbit_on_port(ns, port_id); double shared_queued_after = queued_mbit_on_switch(ns); int active_after = active_flowlet_count_on_port(ns, port_id); @@ -1905,6 +1962,7 @@ static void rollback_switch_egress(switch_state* ns, fluid_msg* m) { } ns->scheduled_egress_interval[port_id] = m->rc_prev_scheduled_egress_interval; + ns->rr_next_queue_index[port_id] = m->rc_prev_rr_next_queue_index; std::vector& qv = *ns->queues[port_id]; const port_desc* p = &ns->ports[port_id]; From de231cf6e216a70178337f0a793ed71131446d75 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 13:50:01 -0400 Subject: [PATCH 12/20] Add fluid flow wan tests for CI --- doc/example/CMakeLists.txt | 6 +- ...logy.yaml => fluid-flow-wan-topology.yaml} | 0 ...-switch.conf.in => fluid-flow-wan.conf.in} | 20 +-- src/CMakeLists.txt | 4 +- ...py => generate-fluid-flow-wan-topology.py} | 10 +- ...witch.cxx => model-net-fluid-flow-wan.cxx} | 62 ++++----- tests/CMakeLists.txt | 31 +++++ tests/fluid-flow-wan-ci.sh | 123 ++++++++++++++++++ 8 files changed, 205 insertions(+), 51 deletions(-) rename doc/example/{fluid-switch-topology.yaml => fluid-flow-wan-topology.yaml} (100%) rename doc/example/{fluid-switch.conf.in => fluid-flow-wan.conf.in} (54%) rename src/network-workloads/{generate-fluid-switch-topology.py => generate-fluid-flow-wan-topology.py} (97%) rename src/network-workloads/{model-net-fluid-switch.cxx => model-net-fluid-flow-wan.cxx} (96%) create mode 100755 tests/fluid-flow-wan-ci.sh diff --git a/doc/example/CMakeLists.txt b/doc/example/CMakeLists.txt index 550c18e2..78d15ab8 100644 --- a/doc/example/CMakeLists.txt +++ b/doc/example/CMakeLists.txt @@ -14,7 +14,7 @@ configure_file(tutorial-ping-pong-surrogate.conf.in tutorial-ping-pong-surrogate configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.template.conf.in @ONLY) configure_file(kb.dfdally-72-event-time-director.conf.in kb.dfdally-72-event-time-director.template.conf.in @ONLY) configure_file(kb.dfdally-72-milc-small.workload.conf.in kb.dfdally-72-milc-small.workload.template.conf.in @ONLY) -configure_file(fluid-switch.conf.in fluid-switch.template.conf.in @ONLY) +configure_file(fluid-flow-wan.conf.in fluid-flow-wan.template.conf.in @ONLY) configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.template.conf.in @ONLY) set(single_quote "'") @@ -43,8 +43,8 @@ configure_file(tutorial-ping-pong-surrogate.conf.in tutorial-ping-pong-surrogate configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.conf) configure_file(kb.dfdally-72-event-time-director.conf.in kb.dfdally-72-event-time-director.conf) configure_file(kb.dfdally-72-milc-small.workload.conf.in kb.dfdally-72-milc-small.workload.conf) -configure_file(fluid-switch.conf.in fluid-switch.conf) +configure_file(fluid-flow-wan.conf.in fluid-flow-wan.conf) configure_file(kb.dfdally-72-milc-small.json kb.dfdally-72-milc-small.json COPYONLY) configure_file(kb.dfdally-72-milc-small.alloc.conf kb.dfdally-72-milc-small.alloc.conf COPYONLY) -configure_file(fluid-switch-topology.yaml fluid-switch-topology.yaml COPYONLY) +configure_file(fluid-flow-wan-topology.yaml fluid-flow-wan-topology.yaml COPYONLY) configure_file(kb.dfdally-72-zeromq-director.conf.in kb.dfdally-72-zeromq-director.conf) diff --git a/doc/example/fluid-switch-topology.yaml b/doc/example/fluid-flow-wan-topology.yaml similarity index 100% rename from doc/example/fluid-switch-topology.yaml rename to doc/example/fluid-flow-wan-topology.yaml diff --git a/doc/example/fluid-switch.conf.in b/doc/example/fluid-flow-wan.conf.in similarity index 54% rename from doc/example/fluid-switch.conf.in rename to doc/example/fluid-flow-wan.conf.in index 216c88c1..22b2b256 100644 --- a/doc/example/fluid-switch.conf.in +++ b/doc/example/fluid-flow-wan.conf.in @@ -1,14 +1,14 @@ # Interval-fluid switch/terminal pure-PDES example. # Run from the build directory, for example: -# mpirun -np 1 ./src/model-net-fluid-switch --synch=1 -- doc/example/fluid-switch.conf +# mpirun -np 1 ./src/model-net-fluid-flow-wan --synch=1 -- doc/example/fluid-flow-wan.conf LPGROUPS { - FLUID_GRP + FLUID_FLOW_WAN_GRP { repetitions="1"; - fluid-switch-lp="3"; - fluid-terminal-lp="6"; + fluid-flow-wan-switch-lp="3"; + fluid-flow-wan-terminal-lp="6"; } } @@ -18,9 +18,9 @@ PARAMS pe_mem_factor="1024"; } -FLUID_SWITCH +FLUID_FLOW_WAN { - topology_yaml_file="fluid-switch-topology.yaml"; + topology_yaml_file="fluid-flow-wan-topology.yaml"; interval_seconds="1"; num_send_intervals="20"; @@ -38,8 +38,8 @@ FLUID_SWITCH round_robin_max_entries_per_egress="8"; round_robin_quantum_mbit="0"; - terminal_log_path="../../zmqml_artifacts/fluid-switch/terminal-events.csv"; - switch_log_path="../../zmqml_artifacts/fluid-switch/switch-events.csv"; - flowlet_log_path="../../zmqml_artifacts/fluid-switch/flowlet-events.csv"; - switch_training_log_path="../../zmqml_artifacts/fluid-switch/switch-training.csv"; + terminal_log_path="../../zmqml_artifacts/fluid-flow-wan/terminal-events.csv"; + switch_log_path="../../zmqml_artifacts/fluid-flow-wan/switch-events.csv"; + flowlet_log_path="../../zmqml_artifacts/fluid-flow-wan/flowlet-events.csv"; + switch_training_log_path="../../zmqml_artifacts/fluid-flow-wan/switch-training.csv"; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 21b21fa3..0f4cba1b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -206,7 +206,7 @@ add_executable(model-net-synthetic network-workloads/model-net-synthetic.c) add_executable(model-net-synthetic-slimfly network-workloads/model-net-synthetic-slimfly.c) add_executable(model-net-synthetic-fattree network-workloads/model-net-synthetic-fattree.c) add_executable(model-net-synthetic-dragonfly-all network-workloads/model-net-synthetic-dragonfly-all.c) -add_executable(model-net-fluid-switch network-workloads/model-net-fluid-switch.cxx) +add_executable(model-net-fluid-flow-wan network-workloads/model-net-fluid-flow-wan.cxx) set(CODES_TARGETS topology-test @@ -215,7 +215,7 @@ set(CODES_TARGETS model-net-synthetic-slimfly model-net-synthetic-fattree model-net-synthetic-dragonfly-all - model-net-fluid-switch + model-net-fluid-flow-wan ) if(USE_DUMPI) diff --git a/src/network-workloads/generate-fluid-switch-topology.py b/src/network-workloads/generate-fluid-flow-wan-topology.py similarity index 97% rename from src/network-workloads/generate-fluid-switch-topology.py rename to src/network-workloads/generate-fluid-flow-wan-topology.py index b2acdf43..280935f5 100755 --- a/src/network-workloads/generate-fluid-switch-topology.py +++ b/src/network-workloads/generate-fluid-flow-wan-topology.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Generate a WAN-like topology YAML file for model-net-fluid-switch. +Generate a WAN-like topology YAML file for model-net-fluid-flow-wan. The generated graph is intentionally not a symmetric supercomputer-style topology. It starts with a bidirectional ring to guarantee strong connectivity, @@ -48,7 +48,7 @@ def positive_float(value: str) -> float: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( - description="Generate a random WAN-like topology YAML for model-net-fluid-switch." + description="Generate a random WAN-like topology YAML for model-net-fluid-flow-wan." ) parser.add_argument( @@ -126,10 +126,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--output", type=Path, - default=Path("doc/example/fluid-switch-topology.generated.yaml"), + default=Path("doc/example/fluid-flow-wan-topology.generated.yaml"), help=( "output YAML path, relative to the current working directory unless absolute; " - "default: doc/example/fluid-switch-topology.generated.yaml" + "default: doc/example/fluid-flow-wan-topology.generated.yaml" ), ) parser.add_argument( @@ -251,7 +251,7 @@ def write_topology(args: argparse.Namespace, links: dict[int, dict[int, float]]) args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("w", encoding="utf-8") as f: - f.write("# Generated by src/network-workloads/generate-fluid-switch-topology.py\n") + f.write("# Generated by src/network-workloads/generate-fluid-flow-wan-topology.py\n") f.write(f"# switches={args.switches}\n") f.write(f"# terminals_per_switch={args.terminals_per_switch}\n") f.write(f"# avg_switch_degree={args.avg_switch_degree}\n") diff --git a/src/network-workloads/model-net-fluid-switch.cxx b/src/network-workloads/model-net-fluid-flow-wan.cxx similarity index 96% rename from src/network-workloads/model-net-fluid-switch.cxx rename to src/network-workloads/model-net-fluid-flow-wan.cxx index c29a8ea0..df349d1a 100644 --- a/src/network-workloads/model-net-fluid-switch.cxx +++ b/src/network-workloads/model-net-fluid-flow-wan.cxx @@ -34,9 +34,9 @@ #include "codes/configuration.h" #include "codes/lp-type-lookup.h" -static const char* GROUP_NAME = "FLUID_GRP"; -static const char* TERMINAL_LP_NAME = "fluid-terminal-lp"; -static const char* SWITCH_LP_NAME = "fluid-switch-lp"; +static const char* GROUP_NAME = "FLUID_FLOW_WAN_GRP"; +static const char* TERMINAL_LP_NAME = "fluid-flow-wan-terminal-lp"; +static const char* SWITCH_LP_NAME = "fluid-flow-wan-switch-lp"; static constexpr int MAX_PORTS_PER_SWITCH = 128; /* @@ -271,7 +271,7 @@ static void validate_ross_message_size_or_abort(int rank) { (size_t)configured_message_size < required_message_size) { if (rank == 0) { fprintf(stderr, - "fluid-switch error: PARAMS.message_size=%d is too small for " + "fluid-flow-wan error: PARAMS.message_size=%d is too small for " "sizeof(fluid_msg)=%zu. Increase PARAMS.message_size in the " "CODES config before calling codes_mapping_setup(). " "fluid_msg is large because MAX_RC_ALLOCATIONS=%d reverse " @@ -609,61 +609,61 @@ static void read_relpath_param(const char* section, const char* key, char* value } static void load_config(void) { - read_relpath_param("FLUID_SWITCH", "topology_yaml_file", cfg.topology_yaml_file, + read_relpath_param("FLUID_FLOW_WAN", "topology_yaml_file", cfg.topology_yaml_file, sizeof(cfg.topology_yaml_file)); - read_relpath_param("FLUID_SWITCH", "terminal_log_path", cfg.terminal_log_path, + read_relpath_param("FLUID_FLOW_WAN", "terminal_log_path", cfg.terminal_log_path, sizeof(cfg.terminal_log_path)); - read_relpath_param("FLUID_SWITCH", "switch_log_path", cfg.switch_log_path, + read_relpath_param("FLUID_FLOW_WAN", "switch_log_path", cfg.switch_log_path, sizeof(cfg.switch_log_path)); - read_relpath_param("FLUID_SWITCH", "flowlet_log_path", cfg.flowlet_log_path, + read_relpath_param("FLUID_FLOW_WAN", "flowlet_log_path", cfg.flowlet_log_path, sizeof(cfg.flowlet_log_path)); - read_relpath_param("FLUID_SWITCH", "switch_training_log_path", cfg.switch_training_log_path, + read_relpath_param("FLUID_FLOW_WAN", "switch_training_log_path", cfg.switch_training_log_path, sizeof(cfg.switch_training_log_path)); - read_string_param("FLUID_SWITCH", "switch_scheduler", cfg.switch_scheduler, + read_string_param("FLUID_FLOW_WAN", "switch_scheduler", cfg.switch_scheduler, sizeof(cfg.switch_scheduler)); - read_int_param("FLUID_SWITCH", "round_robin_max_entries_per_egress", + read_int_param("FLUID_FLOW_WAN", "round_robin_max_entries_per_egress", &cfg.round_robin_max_entries_per_egress); - read_double_param("FLUID_SWITCH", "round_robin_quantum_mbit", + read_double_param("FLUID_FLOW_WAN", "round_robin_quantum_mbit", &cfg.round_robin_quantum_mbit); - read_double_param("FLUID_SWITCH", "interval_seconds", &cfg.interval_seconds); - read_int_param("FLUID_SWITCH", "num_send_intervals", &cfg.num_send_intervals); - read_int_param("FLUID_SWITCH", "num_drain_intervals", &cfg.num_drain_intervals); - read_int_param("FLUID_SWITCH", "rng_seed", &cfg.rng_seed); - read_int_param("FLUID_SWITCH", "terminal_send_every_n_intervals", &cfg.terminal_send_every_n_intervals); - read_double_param("FLUID_SWITCH", "terminal_send_probability", &cfg.terminal_send_probability); - read_double_param("FLUID_SWITCH", "terminal_min_send_mbit", &cfg.terminal_min_send_mbit); - read_double_param("FLUID_SWITCH", "terminal_max_send_fraction_of_link_capacity", + read_double_param("FLUID_FLOW_WAN", "interval_seconds", &cfg.interval_seconds); + read_int_param("FLUID_FLOW_WAN", "num_send_intervals", &cfg.num_send_intervals); + read_int_param("FLUID_FLOW_WAN", "num_drain_intervals", &cfg.num_drain_intervals); + read_int_param("FLUID_FLOW_WAN", "rng_seed", &cfg.rng_seed); + read_int_param("FLUID_FLOW_WAN", "terminal_send_every_n_intervals", &cfg.terminal_send_every_n_intervals); + read_double_param("FLUID_FLOW_WAN", "terminal_send_probability", &cfg.terminal_send_probability); + read_double_param("FLUID_FLOW_WAN", "terminal_min_send_mbit", &cfg.terminal_min_send_mbit); + read_double_param("FLUID_FLOW_WAN", "terminal_max_send_fraction_of_link_capacity", &cfg.terminal_max_send_fraction_of_link_capacity); - read_int_param("FLUID_SWITCH", "debug_prints", &cfg.debug_prints); + read_int_param("FLUID_FLOW_WAN", "debug_prints", &cfg.debug_prints); if (strcmp(cfg.switch_scheduler, "fifo") != 0 && strcmp(cfg.switch_scheduler, "round_robin") != 0) { tw_error(TW_LOC, - "FLUID_SWITCH.switch_scheduler must be one of: fifo, round_robin; got '%s'", + "FLUID_FLOW_WAN.switch_scheduler must be one of: fifo, round_robin; got '%s'", cfg.switch_scheduler); } if (cfg.round_robin_max_entries_per_egress <= 0) { tw_error(TW_LOC, - "FLUID_SWITCH.round_robin_max_entries_per_egress must be positive; got %d", + "FLUID_FLOW_WAN.round_robin_max_entries_per_egress must be positive; got %d", cfg.round_robin_max_entries_per_egress); } if (cfg.round_robin_max_entries_per_egress > MAX_RC_ALLOCATIONS) { tw_error(TW_LOC, - "FLUID_SWITCH.round_robin_max_entries_per_egress=%d exceeds " + "FLUID_FLOW_WAN.round_robin_max_entries_per_egress=%d exceeds " "MAX_RC_ALLOCATIONS=%d", cfg.round_robin_max_entries_per_egress, MAX_RC_ALLOCATIONS); } if (cfg.round_robin_quantum_mbit < 0.0) { tw_error(TW_LOC, - "FLUID_SWITCH.round_robin_quantum_mbit must be nonnegative; got %.12f", + "FLUID_FLOW_WAN.round_robin_quantum_mbit must be nonnegative; got %.12f", cfg.round_robin_quantum_mbit); } if (cfg.topology_yaml_file[0] == '\0') { - tw_error(TW_LOC, "FLUID_SWITCH.topology_yaml_file is required"); + tw_error(TW_LOC, "FLUID_FLOW_WAN.topology_yaml_file is required"); } load_topology_yaml(cfg.topology_yaml_file); compute_routes(); @@ -677,7 +677,7 @@ static tw_lpid get_switch_gid(int switch_id) { tw_lpid gid = 0; /* - * LPGROUPS has one FLUID_GRP repetition containing all switch LPs. + * LPGROUPS has one FLUID_FLOW_WAN_GRP repetition containing all switch LPs. * The switch index is therefore the LP-type offset, not the group repetition. */ codes_mapping_get_lp_id(GROUP_NAME, SWITCH_LP_NAME, NULL, 1, 0, switch_id, &gid); @@ -693,7 +693,7 @@ static tw_lpid get_terminal_gid(int terminal_id) { tw_lpid gid = 0; /* - * LPGROUPS has one FLUID_GRP repetition containing all terminal LPs. + * LPGROUPS has one FLUID_FLOW_WAN_GRP repetition containing all terminal LPs. * The terminal index is therefore the LP-type offset, not the group repetition. */ codes_mapping_get_lp_id(GROUP_NAME, TERMINAL_LP_NAME, NULL, 1, 0, terminal_id, &gid); @@ -1503,7 +1503,7 @@ static void terminal_commit_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw } static void terminal_finalize(terminal_state* ns, tw_lp* lp) { - printf("fluid-terminal gid=%llu terminal=%d switch=%d generated_mbit=%.6f sent_mbit=%.6f " + printf("fluid-flow-wan-terminal gid=%llu terminal=%d switch=%d generated_mbit=%.6f sent_mbit=%.6f " "received_mbit=%.6f generated_flowlets=%d received_fragments=%d\n", (unsigned long long)lp->gid, ns->terminal_id, ns->attached_switch, ns->generated_mbit, ns->sent_to_switch_mbit, ns->received_mbit, @@ -2054,7 +2054,7 @@ static void switch_finalize(switch_state* ns, tw_lp* lp) { for (int p = 0; p < ns->num_ports; ++p) { queued += queued_mbit_on_port(ns, p); } - printf("fluid-switch gid=%llu switch=%d name=%s ports=%d shared_buffer_mbit=%.6f " + printf("fluid-flow-wan gid=%llu switch=%d name=%s ports=%d shared_buffer_mbit=%.6f " "received_fragments=%d sent_fragments=%d enqueued_mbit=%.6f sent_mbit=%.6f " "local_delivery_mbit=%.6f dropped_mbit=%.6f queued_mbit=%.6f\n", (unsigned long long)lp->gid, ns->switch_id, switches[ns->switch_id].name.c_str(), @@ -2229,7 +2229,7 @@ int main(int argc, char** argv) { MPI_Barrier(MPI_COMM_CODES); if (rank == 0) { - printf("fluid-switch config: switches=%zu terminals=%zu interval_seconds=%.6f " + printf("fluid-flow-wan config: switches=%zu terminals=%zu interval_seconds=%.6f " "num_send_intervals=%d num_drain_intervals=%d switch_scheduler=%s " "buffer_mode=shared csv_logs=%s ross_message_size=%d fluid_msg_size=%zu\n", switches.size(), terminals.size(), cfg.interval_seconds, cfg.num_send_intervals, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c972badd..ab31a180 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -298,6 +298,37 @@ if(USE_UNION) set_tests_properties(${union-shell-files} PROPERTIES LABELS "nightly" TIMEOUT 1200) endif() + +# Fluid-flow WAN smoke tests. +# +# These tests start from the CMake-generated doc/example/fluid-flow-wan.conf, +# rewrite only the scheduler and output paths in an isolated test directory, +# and verify that the model runs cleanly and writes CSV logs. +foreach(_ffw_sched fifo round_robin) + foreach(_ffw_mode sequential optimistic) + if(_ffw_mode STREQUAL "sequential") + set(_ffw_synch 1) + set(_ffw_np 1) + else() + set(_ffw_synch 3) + set(_ffw_np 2) + endif() + + set(_ffw_test "fluid-flow-wan-${_ffw_sched}-${_ffw_mode}") + + add_test(NAME ${_ffw_test} + COMMAND "${CMAKE_CURRENT_BINARY_DIR}/run-test.sh" + "${CMAKE_CURRENT_SOURCE_DIR}/fluid-flow-wan-ci.sh" + "${_ffw_sched}" + "${_ffw_synch}" + "${_ffw_np}" + "${_ffw_test}" + WORKING_DIRECTORY "${CODES_BINARY_DIR}") + + set_tests_properties(${_ffw_test} PROPERTIES TIMEOUT 120) + endforeach() +endforeach() + # Equivalence / determinism tests (replacing per-scenario shell scripts). # example-ping-pong-determinism.sh: run the optimistic sim twice, compare # committed-event counts (reproducibility). diff --git a/tests/fluid-flow-wan-ci.sh b/tests/fluid-flow-wan-ci.sh new file mode 100755 index 00000000..83b6d8ef --- /dev/null +++ b/tests/fluid-flow-wan-ci.sh @@ -0,0 +1,123 @@ +#!/bin/bash +set -euo pipefail + +scheduler="${1:?scheduler required: fifo or round_robin}" +synch="${2:?synch mode required: 1 or 3}" +np="${3:?MPI rank count required}" +case_name="${4:-fluid-flow-wan-${scheduler}-synch${synch}}" + +if [[ "$scheduler" != "fifo" && "$scheduler" != "round_robin" ]]; then + echo "unsupported scheduler: $scheduler" + exit 1 +fi + +if [[ -z "${bindir:-}" ]]; then + echo "bindir is not set; this script should be run through tests/run-test.sh" + exit 1 +fi + +if [[ -z "${srcdir:-}" ]]; then + echo "srcdir is not set; this script should be run through tests/run-test.sh" + exit 1 +fi + +binary="$bindir/src/model-net-fluid-flow-wan" +base_conf="$bindir/doc/example/fluid-flow-wan.conf" + +if [[ ! -x "$binary" ]]; then + echo "missing executable: $binary" + exit 1 +fi + +if [[ ! -f "$base_conf" ]]; then + echo "missing generated config: $base_conf" + exit 1 +fi + +topology="$bindir/doc/example/fluid-flow-wan-topology.yaml" +if [[ ! -f "$topology" ]]; then + topology="$srcdir/doc/example/fluid-flow-wan-topology.yaml" +fi + +if [[ ! -f "$topology" ]]; then + echo "missing topology file" + exit 1 +fi + +rm -rf "$case_name" +mkdir -p "$case_name/logs" + +python3 - "$base_conf" "$case_name/fluid-flow-wan.conf" "$scheduler" "$topology" <<'PY' +from pathlib import Path +import re +import sys + +base_conf = Path(sys.argv[1]) +out_conf = Path(sys.argv[2]) +scheduler = sys.argv[3] +topology = Path(sys.argv[4]).resolve() + +text = base_conf.read_text() + +def replace_one(pattern, replacement, label): + global text + text, n = re.subn(pattern, replacement, text, count=1) + if n != 1: + raise SystemExit(f"could not replace {label}") + +replace_one(r'switch_scheduler="[^"]*";', + f'switch_scheduler="{scheduler}";', + "switch_scheduler") + +replace_one(r'topology_yaml_file="[^"]*";', + f'topology_yaml_file="{topology}";', + "topology_yaml_file") + +replace_one(r'terminal_log_path="[^"]*";', + 'terminal_log_path="logs/terminal-events.csv";', + "terminal_log_path") + +replace_one(r'switch_log_path="[^"]*";', + 'switch_log_path="logs/switch-events.csv";', + "switch_log_path") + +replace_one(r'flowlet_log_path="[^"]*";', + 'flowlet_log_path="logs/flowlet-events.csv";', + "flowlet_log_path") + +replace_one(r'switch_training_log_path="[^"]*";', + 'switch_training_log_path="logs/switch-training.csv";', + "switch_training_log_path") + +out_conf.write_text(text) +PY + +( + cd "$case_name" + mpirun -np "$np" "$binary" --synch="$synch" -- fluid-flow-wan.conf \ + > model-output.txt 2> model-output-error.txt +) + +out="$case_name/model-output.txt" + +grep "fluid-flow-wan config:" "$out" +grep "switch_scheduler=$scheduler" "$out" +grep "Net Events Processed" "$out" + +if [[ "$synch" == "1" ]]; then + grep "START SEQUENTIAL SIMULATION" "$out" +else + grep "START PARALLEL OPTIMISTIC SIMULATION" "$out" +fi + +for csv in \ + "$case_name/logs/terminal-events.csv" \ + "$case_name/logs/switch-events.csv" \ + "$case_name/logs/flowlet-events.csv" \ + "$case_name/logs/switch-training.csv" +do + if [[ ! -s "$csv" ]]; then + echo "missing or empty CSV log: $csv" + exit 1 + fi +done From a72907a89b10f576567d52ca8acd19a5939236fb Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 13:57:55 -0400 Subject: [PATCH 13/20] Fix clang format errors --- .../model-net-fluid-flow-wan.cxx | 367 ++++++++---------- 1 file changed, 161 insertions(+), 206 deletions(-) diff --git a/src/network-workloads/model-net-fluid-flow-wan.cxx b/src/network-workloads/model-net-fluid-flow-wan.cxx index df349d1a..7ecf2c71 100644 --- a/src/network-workloads/model-net-fluid-flow-wan.cxx +++ b/src/network-workloads/model-net-fluid-flow-wan.cxx @@ -99,7 +99,7 @@ static sim_config cfg; static std::vector switches; static std::vector terminals; static std::map switch_name_to_id; -static std::vector > next_switch_table; +static std::vector> next_switch_table; static int total_switch_lps = 0; static int total_terminal_lps = 0; @@ -236,18 +236,18 @@ static tw_lptype terminal_lp = { }; static tw_lptype switch_lp = { - (init_f)switch_init, - (pre_run_f)NULL, - (event_f)switch_event, - (revent_f)switch_rev_event, - (commit_f)switch_commit_event, - (final_f)switch_finalize, - (map_f)codes_mapping, - sizeof(switch_state), + (init_f)switch_init, (pre_run_f)NULL, + (event_f)switch_event, (revent_f)switch_rev_event, + (commit_f)switch_commit_event, (final_f)switch_finalize, + (map_f)codes_mapping, sizeof(switch_state), }; -static const tw_lptype* terminal_get_lp_type(void) { return &terminal_lp; } -static const tw_lptype* switch_get_lp_type(void) { return &switch_lp; } +static const tw_lptype* terminal_get_lp_type(void) { + return &terminal_lp; +} +static const tw_lptype* switch_get_lp_type(void) { + return &switch_lp; +} static void add_lp_types(void) { lp_type_register(TERMINAL_LP_NAME, terminal_get_lp_type()); @@ -267,8 +267,7 @@ static void validate_ross_message_size_or_abort(int rank) { int configured_message_size = get_configured_message_size_bytes(); size_t required_message_size = sizeof(fluid_msg); - if (configured_message_size <= 0 || - (size_t)configured_message_size < required_message_size) { + if (configured_message_size <= 0 || (size_t)configured_message_size < required_message_size) { if (rank == 0) { fprintf(stderr, "fluid-flow-wan error: PARAMS.message_size=%d is too small for " @@ -282,7 +281,9 @@ static void validate_ross_message_size_or_abort(int rank) { } } -static double seconds_to_ns(double seconds) { return seconds * 1000.0 * 1000.0 * 1000.0; } +static double seconds_to_ns(double seconds) { + return seconds * 1000.0 * 1000.0 * 1000.0; +} static double event_time_ns(int interval_id, double phase_seconds) { return seconds_to_ns(((double)interval_id * cfg.interval_seconds) + phase_seconds); @@ -319,8 +320,8 @@ static int leading_spaces(const std::string& s) { static std::string strip_quotes(std::string s) { s = trim(s); - if (s.size() >= 2 && ((s.front() == '"' && s.back() == '"') || - (s.front() == '\'' && s.back() == '\''))) { + if (s.size() >= 2 && + ((s.front() == '"' && s.back() == '"') || (s.front() == '\'' && s.back() == '\''))) { return s.substr(1, s.size() - 2); } return s; @@ -476,9 +477,12 @@ static void load_topology_yaml(const char* path) { continue; } if (!in_connections && indent >= 6) { - if (key == "terminals") switches[current_switch].terminal_count = atoi(strip_quotes(value).c_str()); - else if (key == "terminal_bandwidth") switches[current_switch].terminal_bandwidth_mbps = parse_mbps(value); - else if (key == "switch_buffer") switches[current_switch].switch_buffer_mbit = parse_mbit(value); + if (key == "terminals") + switches[current_switch].terminal_count = atoi(strip_quotes(value).c_str()); + else if (key == "terminal_bandwidth") + switches[current_switch].terminal_bandwidth_mbps = parse_mbps(value); + else if (key == "switch_buffer") + switches[current_switch].switch_buffer_mbit = parse_mbit(value); continue; } if (in_connections && indent == 8) { @@ -527,14 +531,22 @@ static void load_topology_yaml(const char* path) { if (terminals.size() < 2) { tw_error(TW_LOC, "topology YAML must define at least two terminals"); } - if (cfg.interval_seconds <= 0.0) tw_error(TW_LOC, "interval_seconds must be positive"); - if (cfg.num_send_intervals <= 0) tw_error(TW_LOC, "num_send_intervals must be positive"); - if (cfg.num_drain_intervals < 0) cfg.num_drain_intervals = 0; - if (cfg.terminal_send_every_n_intervals <= 0) cfg.terminal_send_every_n_intervals = 1; - if (cfg.terminal_send_probability < 0.0) cfg.terminal_send_probability = 0.0; - if (cfg.terminal_send_probability > 1.0) cfg.terminal_send_probability = 1.0; - if (cfg.terminal_max_send_fraction_of_link_capacity < 0.0) cfg.terminal_max_send_fraction_of_link_capacity = 0.0; - if (cfg.terminal_max_send_fraction_of_link_capacity > 1.0) cfg.terminal_max_send_fraction_of_link_capacity = 1.0; + if (cfg.interval_seconds <= 0.0) + tw_error(TW_LOC, "interval_seconds must be positive"); + if (cfg.num_send_intervals <= 0) + tw_error(TW_LOC, "num_send_intervals must be positive"); + if (cfg.num_drain_intervals < 0) + cfg.num_drain_intervals = 0; + if (cfg.terminal_send_every_n_intervals <= 0) + cfg.terminal_send_every_n_intervals = 1; + if (cfg.terminal_send_probability < 0.0) + cfg.terminal_send_probability = 0.0; + if (cfg.terminal_send_probability > 1.0) + cfg.terminal_send_probability = 1.0; + if (cfg.terminal_max_send_fraction_of_link_capacity < 0.0) + cfg.terminal_max_send_fraction_of_link_capacity = 0.0; + if (cfg.terminal_max_send_fraction_of_link_capacity > 1.0) + cfg.terminal_max_send_fraction_of_link_capacity = 1.0; } static void compute_routes(void) { @@ -623,14 +635,15 @@ static void load_config(void) { sizeof(cfg.switch_scheduler)); read_int_param("FLUID_FLOW_WAN", "round_robin_max_entries_per_egress", &cfg.round_robin_max_entries_per_egress); - read_double_param("FLUID_FLOW_WAN", "round_robin_quantum_mbit", - &cfg.round_robin_quantum_mbit); + read_double_param("FLUID_FLOW_WAN", "round_robin_quantum_mbit", &cfg.round_robin_quantum_mbit); read_double_param("FLUID_FLOW_WAN", "interval_seconds", &cfg.interval_seconds); read_int_param("FLUID_FLOW_WAN", "num_send_intervals", &cfg.num_send_intervals); read_int_param("FLUID_FLOW_WAN", "num_drain_intervals", &cfg.num_drain_intervals); read_int_param("FLUID_FLOW_WAN", "rng_seed", &cfg.rng_seed); - read_int_param("FLUID_FLOW_WAN", "terminal_send_every_n_intervals", &cfg.terminal_send_every_n_intervals); - read_double_param("FLUID_FLOW_WAN", "terminal_send_probability", &cfg.terminal_send_probability); + read_int_param("FLUID_FLOW_WAN", "terminal_send_every_n_intervals", + &cfg.terminal_send_every_n_intervals); + read_double_param("FLUID_FLOW_WAN", "terminal_send_probability", + &cfg.terminal_send_probability); read_double_param("FLUID_FLOW_WAN", "terminal_min_send_mbit", &cfg.terminal_min_send_mbit); read_double_param("FLUID_FLOW_WAN", "terminal_max_send_fraction_of_link_capacity", &cfg.terminal_max_send_fraction_of_link_capacity); @@ -657,8 +670,7 @@ static void load_config(void) { } if (cfg.round_robin_quantum_mbit < 0.0) { - tw_error(TW_LOC, - "FLUID_FLOW_WAN.round_robin_quantum_mbit must be nonnegative; got %.12f", + tw_error(TW_LOC, "FLUID_FLOW_WAN.round_robin_quantum_mbit must be nonnegative; got %.12f", cfg.round_robin_quantum_mbit); } @@ -722,12 +734,9 @@ static int find_terminal_port(const switch_state* ns, int dst_terminal) { static double queued_mbit_on_switch(const switch_state* ns); static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, - double* port_queued_before_out, - double* shared_queued_before_out, - double* shared_queued_after_out, - double* dropped_out, - double* flowlet_remaining_after_out, - int* coalesced_out, + double* port_queued_before_out, double* shared_queued_before_out, + double* shared_queued_after_out, double* dropped_out, + double* flowlet_remaining_after_out, int* coalesced_out, int* queue_index_out) { if (port_queued_before_out != NULL) { *port_queued_before_out = 0.0; @@ -802,8 +811,7 @@ static double enqueue_flowlet(switch_state* ns, int port_id, const fluid_msg* m, continue; } - if (qv[i].flowlet_id == m->flowlet_id && - qv[i].source_terminal == m->source_terminal && + if (qv[i].flowlet_id == m->flowlet_id && qv[i].source_terminal == m->source_terminal && qv[i].destination_terminal == m->destination_terminal && qv[i].creation_interval == m->creation_interval) { qv[i].remaining_mbit += accepted; @@ -926,8 +934,7 @@ static int find_queue_index_for_msg(const switch_state* ns, int port_id, const f continue; } - if (qv[i].flowlet_id == m->flowlet_id && - qv[i].source_terminal == m->source_terminal && + if (qv[i].flowlet_id == m->flowlet_id && qv[i].source_terminal == m->source_terminal && qv[i].destination_terminal == m->destination_terminal && qv[i].creation_interval == m->creation_interval) { return i; @@ -942,9 +949,10 @@ static void compact_port_queue(switch_state* ns, int port_id) { return; } std::vector& qv = *ns->queues[port_id]; - qv.erase(std::remove_if(qv.begin(), qv.end(), [](const queued_flowlet& q) { - return !q.valid || q.remaining_mbit <= EPS; - }), + qv.erase(std::remove_if(qv.begin(), qv.end(), + [](const queued_flowlet& q) { + return !q.valid || q.remaining_mbit <= EPS; + }), qv.end()); } @@ -972,9 +980,13 @@ static bool fluid_csv_logs_enabled(void) { return fluid_csv_forward_logs_enabled() || fluid_commit_logging_active; } -static void fluid_commit_logging_begin(void) { fluid_commit_logging_active = true; } +static void fluid_commit_logging_begin(void) { + fluid_commit_logging_active = true; +} -static void fluid_commit_logging_end(void) { fluid_commit_logging_active = false; } +static void fluid_commit_logging_end(void) { + fluid_commit_logging_active = false; +} static void append_log_row(const char* path, std::ostringstream* log_buffer, const std::string& row) { @@ -989,28 +1001,24 @@ static void append_terminal_log(int interval_id, const char* event_name, int ter int peer_id, double mbit) { std::ostringstream row; row << interval_id << ',' << event_name << ',' << terminal_id << ',' - << terminals[terminal_id].name << ',' << terminals[terminal_id].switch_id << ',' - << peer_id << ',' << mbit << '\n'; + << terminals[terminal_id].name << ',' << terminals[terminal_id].switch_id << ',' << peer_id + << ',' << mbit << '\n'; append_log_row(cfg.terminal_log_path, &terminal_log_buffer, row.str()); } static void append_switch_log(int interval_id, const char* event_name, int switch_id, int port_id, int target_is_terminal, int target_index, double capacity_mbit, - double queued_before_mbit, double sent_mbit, - double queued_after_mbit, - double shared_queued_before_mbit, - double shared_queued_after_mbit, - double shared_buffer_mbit, - double dropped_mbit, + double queued_before_mbit, double sent_mbit, double queued_after_mbit, + double shared_queued_before_mbit, double shared_queued_after_mbit, + double shared_buffer_mbit, double dropped_mbit, int active_queue_entries) { std::ostringstream row; - row << interval_id << ',' << event_name << ',' << switch_id << ',' - << switches[switch_id].name << ',' << port_id << ',' - << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' - << capacity_mbit << ',' << queued_before_mbit << ',' << sent_mbit << ',' - << queued_after_mbit << ',' << shared_queued_before_mbit << ',' - << shared_queued_after_mbit << ',' << shared_buffer_mbit << ',' - << dropped_mbit << ',' << active_queue_entries << '\n'; + row << interval_id << ',' << event_name << ',' << switch_id << ',' << switches[switch_id].name + << ',' << port_id << ',' << (target_is_terminal ? "terminal" : "switch") << ',' + << target_index << ',' << capacity_mbit << ',' << queued_before_mbit << ',' << sent_mbit + << ',' << queued_after_mbit << ',' << shared_queued_before_mbit << ',' + << shared_queued_after_mbit << ',' << shared_buffer_mbit << ',' << dropped_mbit << ',' + << active_queue_entries << '\n'; append_log_row(cfg.switch_log_path, &switch_log_buffer, row.str()); } @@ -1018,17 +1026,15 @@ static void append_flowlet_log(int interval_id, const char* event_name, int swit int target_is_terminal, int target_index, unsigned long long flowlet_id, int source_terminal, int destination_terminal, int creation_interval, - double capacity_mbit, double queued_before_mbit, - double send_mbit, double remaining_after_mbit, - double dropped_mbit) { + double capacity_mbit, double queued_before_mbit, double send_mbit, + double remaining_after_mbit, double dropped_mbit) { std::ostringstream row; - row << interval_id << ',' << event_name << ',' << switch_id << ',' - << switches[switch_id].name << ',' << port_id << ',' - << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' - << flowlet_id << ',' << source_terminal << ',' << destination_terminal << ',' - << creation_interval << ',' << (interval_id - creation_interval) << ',' - << capacity_mbit << ',' << queued_before_mbit << ',' << send_mbit << ',' - << remaining_after_mbit << ',' << dropped_mbit << '\n'; + row << interval_id << ',' << event_name << ',' << switch_id << ',' << switches[switch_id].name + << ',' << port_id << ',' << (target_is_terminal ? "terminal" : "switch") << ',' + << target_index << ',' << flowlet_id << ',' << source_terminal << ',' + << destination_terminal << ',' << creation_interval << ',' + << (interval_id - creation_interval) << ',' << capacity_mbit << ',' << queued_before_mbit + << ',' << send_mbit << ',' << remaining_after_mbit << ',' << dropped_mbit << '\n'; append_log_row(cfg.flowlet_log_path, &flowlet_log_buffer, row.str()); } @@ -1036,15 +1042,14 @@ static void append_switch_training_log(int interval_id, int switch_id, int port_ int target_is_terminal, int target_index, double capacity_mbit, double queued_before_mbit, double sent_mbit, double queued_after_mbit, - double dropped_mbit, int active_before, - int active_after, + double dropped_mbit, int active_before, int active_after, const std::vector& allocations) { std::ostringstream row; - row << interval_id << ',' << switch_id << ',' << switches[switch_id].name << ',' - << port_id << ',' << (target_is_terminal ? "terminal" : "switch") << ',' - << target_index << ',' << cfg.switch_scheduler << ',' << capacity_mbit << ',' - << queued_before_mbit << ',' << sent_mbit << ',' << queued_after_mbit << ',' - << dropped_mbit << ',' << active_before << ',' << active_after << ','; + row << interval_id << ',' << switch_id << ',' << switches[switch_id].name << ',' << port_id + << ',' << (target_is_terminal ? "terminal" : "switch") << ',' << target_index << ',' + << cfg.switch_scheduler << ',' << capacity_mbit << ',' << queued_before_mbit << ',' + << sent_mbit << ',' << queued_after_mbit << ',' << dropped_mbit << ',' << active_before + << ',' << active_after << ','; for (size_t i = 0; i < allocations.size(); ++i) { if (i > 0) { row << ';'; @@ -1127,8 +1132,8 @@ static void schedule_switch_egress(int interval_id, int port_id, tw_lp* lp) { static void request_switch_egress(switch_state* ns, int interval_id, int port_id, tw_lp* lp, fluid_msg* cause_msg) { if (port_id < 0 || port_id >= ns->num_ports) { - tw_error(TW_LOC, "invalid request_switch_egress port %d on switch %d", - port_id, ns->switch_id); + tw_error(TW_LOC, "invalid request_switch_egress port %d on switch %d", port_id, + ns->switch_id); } int prev = ns->scheduled_egress_interval[port_id]; @@ -1167,8 +1172,8 @@ static void request_switch_egress(switch_state* ns, int interval_id, int port_id } } -static void schedule_arrival(int interval_id, tw_lpid dst_gid, const fluid_msg* src_msg, double mbit, - int dst_switch, tw_lp* lp) { +static void schedule_arrival(int interval_id, tw_lpid dst_gid, const fluid_msg* src_msg, + double mbit, int dst_switch, tw_lp* lp) { tw_event* e = tw_event_new(dst_gid, delay_until_ns(interval_id, PHASE_ARRIVAL, lp), lp); fluid_msg* m = (fluid_msg*)tw_event_data(e); memset(m, 0, sizeof(*m)); @@ -1246,65 +1251,44 @@ static void log_terminal_generate_event(const terminal_state* ns, const fluid_ms } static void log_terminal_receive_event(const terminal_state* ns, const fluid_msg* m) { - append_terminal_log(m->interval_id, "receive", ns->terminal_id, - m->source_terminal, m->mbit); + append_terminal_log(m->interval_id, "receive", ns->terminal_id, m->source_terminal, m->mbit); } static void log_switch_arrival_event(const switch_state* ns, const fluid_msg* m) { if (m->rc_no_route) { - append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, - 0.0, 0.0, 0.0, 0.0, - m->rc_log_shared_queued_before_mbit, - m->rc_log_shared_queued_after_mbit, - ns->shared_buffer_mbit, m->rc_dropped_mbit, 0); - append_flowlet_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, - m->flowlet_id, m->source_terminal, m->destination_terminal, - m->creation_interval, 0.0, 0.0, 0.0, 0.0, - m->rc_dropped_mbit); + append_switch_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, 0.0, 0.0, 0.0, + 0.0, m->rc_log_shared_queued_before_mbit, + m->rc_log_shared_queued_after_mbit, ns->shared_buffer_mbit, + m->rc_dropped_mbit, 0); + append_flowlet_log(m->interval_id, "drop_no_route", ns->switch_id, -1, 0, -1, m->flowlet_id, + m->source_terminal, m->destination_terminal, m->creation_interval, 0.0, + 0.0, 0.0, 0.0, m->rc_dropped_mbit); return; } if (m->rc_accepted_mbit > EPS) { - append_flowlet_log(m->interval_id, - m->rc_coalesced ? "enqueue_coalesce" : "enqueue", - ns->switch_id, m->rc_port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->flowlet_id, m->source_terminal, - m->destination_terminal, m->creation_interval, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - 0.0, - m->rc_log_flowlet_remaining_after_mbit, - 0.0); + append_flowlet_log(m->interval_id, m->rc_coalesced ? "enqueue_coalesce" : "enqueue", + ns->switch_id, m->rc_port_id, m->rc_log_target_is_terminal, + m->rc_log_target_index, m->flowlet_id, m->source_terminal, + m->destination_terminal, m->creation_interval, m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, 0.0, + m->rc_log_flowlet_remaining_after_mbit, 0.0); } if (m->rc_dropped_mbit > EPS) { - append_switch_log(m->interval_id, "drop_shared_buffer_overflow", - ns->switch_id, m->rc_port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - 0.0, - m->rc_log_port_queued_after_mbit, - m->rc_log_shared_queued_before_mbit, - m->rc_log_shared_queued_after_mbit, - ns->shared_buffer_mbit, - m->rc_dropped_mbit, - m->rc_log_active_after_entries); - - append_flowlet_log(m->interval_id, "drop_shared_buffer_overflow", - ns->switch_id, m->rc_port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->flowlet_id, m->source_terminal, - m->destination_terminal, m->creation_interval, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - 0.0, - m->rc_log_flowlet_remaining_after_mbit, - m->rc_dropped_mbit); + append_switch_log(m->interval_id, "drop_shared_buffer_overflow", ns->switch_id, + m->rc_port_id, m->rc_log_target_is_terminal, m->rc_log_target_index, + m->rc_log_capacity_mbit, m->rc_log_port_queued_before_mbit, 0.0, + m->rc_log_port_queued_after_mbit, m->rc_log_shared_queued_before_mbit, + m->rc_log_shared_queued_after_mbit, ns->shared_buffer_mbit, + m->rc_dropped_mbit, m->rc_log_active_after_entries); + + append_flowlet_log(m->interval_id, "drop_shared_buffer_overflow", ns->switch_id, + m->rc_port_id, m->rc_log_target_is_terminal, m->rc_log_target_index, + m->flowlet_id, m->source_terminal, m->destination_terminal, + m->creation_interval, m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, 0.0, + m->rc_log_flowlet_remaining_after_mbit, m->rc_dropped_mbit); } } @@ -1321,46 +1305,28 @@ static void log_switch_egress_event(const switch_state* ns, const fluid_msg* m) remaining_after = 0.0; } - append_flowlet_log(m->interval_id, "allocate_send", - ns->switch_id, m->port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - rc->before.flowlet_id, - rc->before.source_terminal, - rc->before.destination_terminal, - rc->before.creation_interval, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - rc->send_mbit, - remaining_after, - 0.0); + append_flowlet_log(m->interval_id, "allocate_send", ns->switch_id, m->port_id, + m->rc_log_target_is_terminal, m->rc_log_target_index, + rc->before.flowlet_id, rc->before.source_terminal, + rc->before.destination_terminal, rc->before.creation_interval, + m->rc_log_capacity_mbit, m->rc_log_port_queued_before_mbit, + rc->send_mbit, remaining_after, 0.0); } append_switch_log(m->interval_id, "egress", ns->switch_id, m->port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - m->rc_log_sent_total_mbit, - m->rc_log_port_queued_after_mbit, - m->rc_log_shared_queued_before_mbit, - m->rc_log_shared_queued_after_mbit, - ns->shared_buffer_mbit, - 0.0, + m->rc_log_target_is_terminal, m->rc_log_target_index, m->rc_log_capacity_mbit, + m->rc_log_port_queued_before_mbit, m->rc_log_sent_total_mbit, + m->rc_log_port_queued_after_mbit, m->rc_log_shared_queued_before_mbit, + m->rc_log_shared_queued_after_mbit, ns->shared_buffer_mbit, 0.0, m->rc_log_active_after_entries); std::vector allocations; build_allocation_strings_from_rc(m, allocations); append_switch_training_log(m->interval_id, ns->switch_id, m->port_id, - m->rc_log_target_is_terminal, - m->rc_log_target_index, - m->rc_log_capacity_mbit, - m->rc_log_port_queued_before_mbit, - m->rc_log_sent_total_mbit, - m->rc_log_port_queued_after_mbit, - 0.0, - m->rc_log_active_before_entries, - m->rc_log_active_after_entries, + m->rc_log_target_is_terminal, m->rc_log_target_index, + m->rc_log_capacity_mbit, m->rc_log_port_queued_before_mbit, + m->rc_log_sent_total_mbit, m->rc_log_port_queued_after_mbit, 0.0, + m->rc_log_active_before_entries, m->rc_log_active_after_entries, allocations); } @@ -1412,8 +1378,8 @@ static void handle_workload_generate(terminal_state* ns, fluid_msg* m, tw_lp* lp log_terminal_generate_event(ns, m); if (cfg.debug_prints) { - printf("[fluid terminal] interval=%d terminal=%d dst=%d mbit=%.6f\n", - interval, ns->terminal_id, dst, mbit); + printf("[fluid terminal] interval=%d terminal=%d dst=%d mbit=%.6f\n", interval, + ns->terminal_id, dst, mbit); } } } @@ -1503,11 +1469,11 @@ static void terminal_commit_event(terminal_state* ns, tw_bf* b, fluid_msg* m, tw } static void terminal_finalize(terminal_state* ns, tw_lp* lp) { - printf("fluid-flow-wan-terminal gid=%llu terminal=%d switch=%d generated_mbit=%.6f sent_mbit=%.6f " - "received_mbit=%.6f generated_flowlets=%d received_fragments=%d\n", - (unsigned long long)lp->gid, ns->terminal_id, ns->attached_switch, - ns->generated_mbit, ns->sent_to_switch_mbit, ns->received_mbit, - ns->generated_flowlets, ns->received_fragments); + printf( + "fluid-flow-wan-terminal gid=%llu terminal=%d switch=%d generated_mbit=%.6f sent_mbit=%.6f " + "received_mbit=%.6f generated_flowlets=%d received_fragments=%d\n", + (unsigned long long)lp->gid, ns->terminal_id, ns->attached_switch, ns->generated_mbit, + ns->sent_to_switch_mbit, ns->received_mbit, ns->generated_flowlets, ns->received_fragments); } static void switch_init(switch_state* ns, tw_lp* lp) { @@ -1630,10 +1596,9 @@ static void handle_switch_arrival(switch_state* ns, fluid_msg* m, tw_lp* lp) { m->rc_log_active_before_entries = 0; m->rc_log_active_after_entries = 0; - double accepted = enqueue_flowlet(ns, port_id, m, &port_queued_before, - &shared_queued_before, &shared_queued_after, - &dropped, &flowlet_remaining_after, &coalesced, - &queue_index); + double accepted = enqueue_flowlet(ns, port_id, m, &port_queued_before, &shared_queued_before, + &shared_queued_after, &dropped, &flowlet_remaining_after, + &coalesced, &queue_index); m->rc_queue_index = queue_index; m->rc_coalesced = coalesced; @@ -1768,8 +1733,7 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { * which keeps reverse metadata, logs, and optimistic rollback bounded. */ int qsize = (int)qv.size(); - int entries_to_serve = std::min(active_before, - cfg.round_robin_max_entries_per_egress); + int entries_to_serve = std::min(active_before, cfg.round_robin_max_entries_per_egress); if (qsize > 0 && entries_to_serve > 0 && remaining_capacity > EPS) { int idx = ns->rr_next_queue_index[port_id]; @@ -1821,8 +1785,8 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { tw_error(TW_LOC, "computed send %.12f exceeds remaining %.12f for flowlet %llu " "on switch %d port %d", - send, qv[i].remaining_mbit, - (unsigned long long)qv[i].flowlet_id, ns->switch_id, port_id); + send, qv[i].remaining_mbit, (unsigned long long)qv[i].flowlet_id, + ns->switch_id, port_id); } queued_flowlet before = qv[i]; @@ -1832,8 +1796,7 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { tw_error(TW_LOC, "too many egress allocations for reverse metadata: switch %d port %d " "interval %d count %d max %d", - ns->switch_id, port_id, m->interval_id, m->rc_alloc_count, - MAX_RC_ALLOCATIONS); + ns->switch_id, port_id, m->interval_id, m->rc_alloc_count, MAX_RC_ALLOCATIONS); } rc_alloc_record* rc = &m->rc_allocs[m->rc_alloc_count++]; @@ -1846,7 +1809,6 @@ static void handle_switch_egress(switch_state* ns, fluid_msg* m, tw_lp* lp) { send_flowlet_fragment(ns, port_id, &before, send, m->interval_id, lp); qv[i].remaining_mbit -= send; sent_total += send; - } compact_port_queue(ns, port_id); @@ -1903,8 +1865,7 @@ static void rollback_switch_arrival(switch_state* ns, fluid_msg* m) { ns->received_fragments--; if (!m->rc_no_route && m->rc_port_id >= 0 && m->rc_port_id < ns->num_ports) { - ns->scheduled_egress_interval[m->rc_port_id] = - m->rc_prev_scheduled_egress_interval; + ns->scheduled_egress_interval[m->rc_port_id] = m->rc_prev_scheduled_egress_interval; } if (m->rc_no_route) { @@ -1915,8 +1876,7 @@ static void rollback_switch_arrival(switch_state* ns, fluid_msg* m) { int port_id = m->rc_port_id; if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { - tw_error(TW_LOC, "invalid rollback arrival port %d on switch %d", port_id, - ns->switch_id); + tw_error(TW_LOC, "invalid rollback arrival port %d on switch %d", port_id, ns->switch_id); } ns->dropped_mbit -= m->rc_dropped_mbit; @@ -1933,8 +1893,7 @@ static void rollback_switch_arrival(switch_state* ns, fluid_msg* m) { } if (idx < 0) { - tw_error(TW_LOC, - "could not find flowlet %llu on switch %d port %d during arrival rollback", + tw_error(TW_LOC, "could not find flowlet %llu on switch %d port %d during arrival rollback", (unsigned long long)m->flowlet_id, ns->switch_id, port_id); } @@ -1942,9 +1901,10 @@ static void rollback_switch_arrival(switch_state* ns, fluid_msg* m) { qv[idx].remaining_mbit -= m->rc_accepted_mbit; if (qv[idx].remaining_mbit <= EPS) { - tw_error(TW_LOC, - "coalesced flowlet %llu became empty during arrival rollback on switch %d port %d", - (unsigned long long)m->flowlet_id, ns->switch_id, port_id); + tw_error( + TW_LOC, + "coalesced flowlet %llu became empty during arrival rollback on switch %d port %d", + (unsigned long long)m->flowlet_id, ns->switch_id, port_id); } } else { qv.erase(qv.begin() + idx); @@ -1957,8 +1917,7 @@ static void rollback_switch_egress(switch_state* ns, fluid_msg* m) { int port_id = m->port_id; if (port_id < 0 || port_id >= ns->num_ports || ns->queues[port_id] == NULL) { - tw_error(TW_LOC, "invalid rollback egress port %d on switch %d", port_id, - ns->switch_id); + tw_error(TW_LOC, "invalid rollback egress port %d on switch %d", port_id, ns->switch_id); } ns->scheduled_egress_interval[port_id] = m->rc_prev_scheduled_egress_interval; @@ -1976,8 +1935,7 @@ static void rollback_switch_egress(switch_state* ns, fluid_msg* m) { int idx = rc->queue_index; - if (idx < 0 || idx >= (int)qv.size() || - qv[idx].flowlet_id != rc->before.flowlet_id) { + if (idx < 0 || idx >= (int)qv.size() || qv[idx].flowlet_id != rc->before.flowlet_id) { idx = find_queue_index_for_flowlet(ns, port_id, rc->before); } @@ -2059,8 +2017,7 @@ static void switch_finalize(switch_state* ns, tw_lp* lp) { "local_delivery_mbit=%.6f dropped_mbit=%.6f queued_mbit=%.6f\n", (unsigned long long)lp->gid, ns->switch_id, switches[ns->switch_id].name.c_str(), ns->num_ports, ns->shared_buffer_mbit, ns->received_fragments, ns->sent_fragments, - ns->enqueued_mbit, ns->sent_mbit, ns->delivered_local_mbit, ns->dropped_mbit, - queued); + ns->enqueued_mbit, ns->sent_mbit, ns->delivered_local_mbit, ns->dropped_mbit, queued); for (int p = 0; p < ns->num_ports; ++p) { delete ns->queues[p]; @@ -2068,8 +2025,7 @@ static void switch_finalize(switch_state* ns, tw_lp* lp) { } } -const tw_optdef app_opt[] = {TWOPT_GROUP("interval-fluid switch/terminal workload"), - TWOPT_END()}; +const tw_optdef app_opt[] = {TWOPT_GROUP("interval-fluid switch/terminal workload"), TWOPT_END()}; static const char* find_config_arg(int argc, char** argv) { for (int i = argc - 1; i >= 1; --i) { @@ -2131,8 +2087,7 @@ static void merge_log_buffer(const char* path, const std::ostringstream& buffer, displs.resize(size, 0); } - MPI_Gather(&local_len, 1, MPI_INT, - rank == 0 ? recv_counts.data() : NULL, 1, MPI_INT, 0, comm); + MPI_Gather(&local_len, 1, MPI_INT, rank == 0 ? recv_counts.data() : NULL, 1, MPI_INT, 0, comm); int total_len = 0; if (rank == 0) { @@ -2149,9 +2104,8 @@ static void merge_log_buffer(const char* path, const std::ostringstream& buffer, MPI_Gatherv(local_len > 0 ? local_rows.data() : NULL, local_len, MPI_CHAR, rank == 0 && total_len > 0 ? merged.data() : NULL, - rank == 0 ? recv_counts.data() : NULL, - rank == 0 ? displs.data() : NULL, - MPI_CHAR, 0, comm); + rank == 0 ? recv_counts.data() : NULL, rank == 0 ? displs.data() : NULL, MPI_CHAR, + 0, comm); if (rank == 0 && total_len > 0) { append_merged_rows_to_file(path, merged.data(), total_len); @@ -2170,8 +2124,9 @@ static void write_log_headers(int rank) { return; } - write_log_header_file(cfg.terminal_log_path, - "interval,event,terminal,terminal_name,attached_switch,peer_terminal,mbit\n"); + write_log_header_file( + cfg.terminal_log_path, + "interval,event,terminal,terminal_name,attached_switch,peer_terminal,mbit\n"); write_log_header_file(cfg.switch_log_path, "interval,event,switch,switch_name,port,target_type,target_index," From de76ae31ee4a3730d081be17e1a20719bdc5c4dd Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 14:07:02 -0400 Subject: [PATCH 14/20] Fix MPI EXEC handler for fluid flow wan simn tests --- tests/CMakeLists.txt | 2 ++ tests/fluid-flow-wan-ci.sh | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ab31a180..2fc397ca 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -323,6 +323,8 @@ foreach(_ffw_sched fifo round_robin) "${_ffw_synch}" "${_ffw_np}" "${_ffw_test}" + "${MPIEXEC_EXECUTABLE}" + "${MPIEXEC_NUMPROC_FLAG}" WORKING_DIRECTORY "${CODES_BINARY_DIR}") set_tests_properties(${_ffw_test} PROPERTIES TIMEOUT 120) diff --git a/tests/fluid-flow-wan-ci.sh b/tests/fluid-flow-wan-ci.sh index 83b6d8ef..d1bde13a 100755 --- a/tests/fluid-flow-wan-ci.sh +++ b/tests/fluid-flow-wan-ci.sh @@ -5,6 +5,8 @@ scheduler="${1:?scheduler required: fifo or round_robin}" synch="${2:?synch mode required: 1 or 3}" np="${3:?MPI rank count required}" case_name="${4:-fluid-flow-wan-${scheduler}-synch${synch}}" +mpi_exec="${5:-mpirun}" +mpi_np_flag="${6:--np}" if [[ "$scheduler" != "fifo" && "$scheduler" != "round_robin" ]]; then echo "unsupported scheduler: $scheduler" @@ -92,11 +94,18 @@ replace_one(r'switch_training_log_path="[^"]*";', out_conf.write_text(text) PY -( +if ! ( cd "$case_name" - mpirun -np "$np" "$binary" --synch="$synch" -- fluid-flow-wan.conf \ + "$mpi_exec" "$mpi_np_flag" "$np" "$binary" --synch="$synch" -- fluid-flow-wan.conf \ > model-output.txt 2> model-output-error.txt -) +); then + echo "fluid-flow-wan model run failed" + echo "--- stdout ---" + cat "$case_name/model-output.txt" || true + echo "--- stderr ---" + cat "$case_name/model-output-error.txt" || true + exit 1 +fi out="$case_name/model-output.txt" From fb23b0cb05526d75bdcc90cb30c19fbf2f6fe794 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 14:15:03 -0400 Subject: [PATCH 15/20] Fix yaml loading issue in fluid flow simn test --- tests/fluid-flow-wan-ci.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fluid-flow-wan-ci.sh b/tests/fluid-flow-wan-ci.sh index d1bde13a..5e263f60 100755 --- a/tests/fluid-flow-wan-ci.sh +++ b/tests/fluid-flow-wan-ci.sh @@ -48,8 +48,9 @@ fi rm -rf "$case_name" mkdir -p "$case_name/logs" +cp "$topology" "$case_name/fluid-flow-wan-topology.yaml" -python3 - "$base_conf" "$case_name/fluid-flow-wan.conf" "$scheduler" "$topology" <<'PY' +python3 - "$base_conf" "$case_name/fluid-flow-wan.conf" "$scheduler" <<'PY' from pathlib import Path import re import sys @@ -57,7 +58,6 @@ import sys base_conf = Path(sys.argv[1]) out_conf = Path(sys.argv[2]) scheduler = sys.argv[3] -topology = Path(sys.argv[4]).resolve() text = base_conf.read_text() @@ -72,7 +72,7 @@ replace_one(r'switch_scheduler="[^"]*";', "switch_scheduler") replace_one(r'topology_yaml_file="[^"]*";', - f'topology_yaml_file="{topology}";', + 'topology_yaml_file="fluid-flow-wan-topology.yaml";', "topology_yaml_file") replace_one(r'terminal_log_path="[^"]*";', From 13ca4c7c1ba52523c9002b13fda49fc249311bd7 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 14:28:16 -0400 Subject: [PATCH 16/20] Make fluid flow wan simn tests use sync instead of synch --- tests/fluid-flow-wan-ci.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fluid-flow-wan-ci.sh b/tests/fluid-flow-wan-ci.sh index 5e263f60..57f5d584 100755 --- a/tests/fluid-flow-wan-ci.sh +++ b/tests/fluid-flow-wan-ci.sh @@ -96,7 +96,7 @@ PY if ! ( cd "$case_name" - "$mpi_exec" "$mpi_np_flag" "$np" "$binary" --synch="$synch" -- fluid-flow-wan.conf \ + "$mpi_exec" "$mpi_np_flag" "$np" "$binary" --sync="$synch" -- fluid-flow-wan.conf \ > model-output.txt 2> model-output-error.txt ); then echo "fluid-flow-wan model run failed" @@ -114,9 +114,9 @@ grep "switch_scheduler=$scheduler" "$out" grep "Net Events Processed" "$out" if [[ "$synch" == "1" ]]; then - grep "START SEQUENTIAL SIMULATION" "$out" + grep "csv_logs=buffered-forward" "$out" else - grep "START PARALLEL OPTIMISTIC SIMULATION" "$out" + grep "csv_logs=buffered-commit" "$out" fi for csv in \ From 9297423b732c5a7017d9317f08140ade32abbc1b Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 14:50:25 -0400 Subject: [PATCH 17/20] Fix Ubuntu 24.04 mpich tests --- .github/workflows/build.yml | 54 ++++++++++++++++++++++++++++++++++--- tests/CMakeLists.txt | 35 +++++++++++++++--------- 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec156804..14501b78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -112,6 +112,18 @@ jobs: if [ "${{ matrix.cc }}" = "clang" ]; then sudo apt-get install -y clang fi + if [ "${{ matrix.mpi }}" = "mpich" ]; then + mpi_bin="$RUNNER_TEMP/mpi-bin" + mkdir -p "$mpi_bin" + mpiexec_path="$(command -v mpiexec.mpich || command -v mpiexec)" + mpirun_path="$(command -v mpirun.mpich || command -v mpirun)" + ln -sf "$mpiexec_path" "$mpi_bin/mpiexec" + ln -sf "$mpirun_path" "$mpi_bin/mpirun" + echo "$mpi_bin" >> "$GITHUB_PATH" + echo "CODES_MPIEXEC_EXECUTABLE=$mpi_bin/mpiexec" >> "$GITHUB_ENV" + else + echo "CODES_MPIEXEC_EXECUTABLE=$(command -v mpiexec)" >> "$GITHUB_ENV" + fi - name: Install system dependencies (macOS) if: runner.os == 'macOS' @@ -129,6 +141,7 @@ jobs: # ahead of /usr/bin on PATH for subsequent steps. echo "$(brew --prefix bison)/bin" >> "$GITHUB_PATH" echo "$(brew --prefix flex)/bin" >> "$GITHUB_PATH" + echo "CODES_MPIEXEC_EXECUTABLE=$(command -v mpiexec)" >> "$GITHUB_ENV" - name: Select compiler # Drive the base compiler for both ROSS and CODES via CC/CXX. @@ -151,6 +164,9 @@ jobs: -DCMAKE_BUILD_TYPE=Debug -DROSS_BUILD_MODELS=ON -DCMAKE_INSTALL_PREFIX=$PWD/ross-install + -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" + -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" + -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - name: Build and install ROSS run: cmake --build ross/build --target install -j @@ -161,7 +177,12 @@ jobs: # in CMakePresets.json; ROSS is found via the ROSS_ROOT env var. The two # heavy deps a stock runner might auto-detect are forced off so this leg # stays a deterministic stock build (the `full` job exercises them). - run: cmake --preset debug -DCODES_USE_TORCH=OFF -DCODES_USE_ZEROMQ=OFF + run: > + cmake --preset debug + -DCODES_USE_TORCH=OFF + -DCODES_USE_ZEROMQ=OFF + -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" + -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - name: Build CODES working-directory: codes @@ -214,6 +235,18 @@ jobs: mpich libmpich-dev \ cmake ninja-build pkg-config \ flex bison + mpi_bin="$RUNNER_TEMP/mpi-bin" + mkdir -p "$mpi_bin" + ln -sf "$(command -v mpiexec.mpich || command -v mpiexec)" "$mpi_bin/mpiexec" + ln -sf "$(command -v mpirun.mpich || command -v mpirun)" "$mpi_bin/mpirun" + echo "$mpi_bin" >> "$GITHUB_PATH" + echo "CODES_MPIEXEC_EXECUTABLE=$mpi_bin/mpiexec" >> "$GITHUB_ENV" + mpi_bin="$RUNNER_TEMP/mpi-bin" + mkdir -p "$mpi_bin" + ln -sf "$(command -v mpiexec.mpich || command -v mpiexec)" "$mpi_bin/mpiexec" + ln -sf "$(command -v mpirun.mpich || command -v mpirun)" "$mpi_bin/mpirun" + echo "$mpi_bin" >> "$GITHUB_PATH" + echo "CODES_MPIEXEC_EXECUTABLE=$mpi_bin/mpiexec" >> "$GITHUB_ENV" - name: Configure old ROSS run: > @@ -222,6 +255,7 @@ jobs: -DROSS_BUILD_MODELS=ON -DCMAKE_INSTALL_PREFIX=$PWD/ross-install -DCMAKE_C_COMPILER=mpicc + -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - name: Build and install old ROSS run: cmake --build ross/build --target install -j @@ -334,7 +368,11 @@ jobs: working-directory: codes # The asan/ubsan preset sets CODES_SANITIZER; heavy deps forced off to # match the stock matrix legs. - run: cmake --preset ${{ matrix.sanitizer }} -DCODES_USE_TORCH=OFF -DCODES_USE_ZEROMQ=OFF + run: > + cmake --preset ${{ matrix.sanitizer }} + -DCODES_USE_TORCH=OFF + -DCODES_USE_ZEROMQ=OFF + -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - name: Build CODES working-directory: codes @@ -399,6 +437,12 @@ jobs: mpich libmpich-dev \ cmake ninja-build pkg-config \ flex bison lcov + mpi_bin="$RUNNER_TEMP/mpi-bin" + mkdir -p "$mpi_bin" + ln -sf "$(command -v mpiexec.mpich || command -v mpiexec)" "$mpi_bin/mpiexec" + ln -sf "$(command -v mpirun.mpich || command -v mpirun)" "$mpi_bin/mpirun" + echo "$mpi_bin" >> "$GITHUB_PATH" + echo "CODES_MPIEXEC_EXECUTABLE=$mpi_bin/mpiexec" >> "$GITHUB_ENV" - name: Configure ROSS run: > @@ -414,7 +458,11 @@ jobs: # CODES is checked out at the workspace root here (no `path:`), so the # preset is found in the CWD. The `coverage` preset adds # -DCODES_ENABLE_COVERAGE=ON; heavy deps forced off to match the matrix. - run: cmake --preset coverage -DCODES_USE_TORCH=OFF -DCODES_USE_ZEROMQ=OFF + run: > + cmake --preset coverage + -DCODES_USE_TORCH=OFF + -DCODES_USE_ZEROMQ=OFF + -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - name: Build CODES run: cmake --build --preset coverage diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2fc397ca..6f4c152a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -43,7 +43,7 @@ add_test(NAME codes-unit-smoke COMMAND codes-unit-smoke) # REPEAT for "same command N times", VARIANTS when only a small fragment differs # (e.g. VARIANTS "--sync=1" "--sync=3" for a seq-vs-optimistic check). function(codes_add_equivalence_test) - cmake_parse_arguments(T "" "NAME;BINARY;NP;CONFIG;MARKER;REPEAT;SETUP;REQUIRE" "ARGS;VARIANTS" ${ARGN}) + cmake_parse_arguments(T "" "NAME;BINARY;NP;CONFIG;MARKER;REPEAT;SETUP" "ARGS;VARIANTS;REQUIRE" ${ARGN}) if(NOT T_NAME OR NOT T_BINARY OR NOT T_CONFIG) message(FATAL_ERROR "codes_add_equivalence_test: NAME, BINARY and CONFIG are required") @@ -88,9 +88,11 @@ function(codes_add_equivalence_test) set(_setup --setup "${CMAKE_CURRENT_SOURCE_DIR}/${T_SETUP}") endif() set(_require "") - if(T_REQUIRE) - set(_require --require "${T_REQUIRE}") - endif() + foreach(_req IN LISTS T_REQUIRE) + if(NOT _req STREQUAL "") + list(APPEND _require --require "${_req}") + endif() + endforeach() add_test(NAME ${T_NAME} COMMAND "${CMAKE_CURRENT_BINARY_DIR}/run-test.sh" @@ -122,7 +124,7 @@ endfunction() # [MARKER ] # ) function(codes_add_run_test) - cmake_parse_arguments(T "" "NAME;BINARY;NP;SETUP;MARKER" "ARGS" ${ARGN}) + cmake_parse_arguments(T "" "NAME;BINARY;NP;SETUP;MARKER" "ARGS;REQUIRE" ${ARGN}) if(NOT T_NAME OR NOT T_BINARY) message(FATAL_ERROR "codes_add_run_test: NAME and BINARY are required") endif() @@ -132,12 +134,17 @@ function(codes_add_run_test) set(_run ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${T_NP} ${MPIEXEC_PREFLAGS} "$" ${T_ARGS} ${MPIEXEC_POSTFLAGS}) - if(T_SETUP OR T_MARKER) - # SETUP must be sourced and MARKER checked -> go through the staged runner. + if(T_SETUP OR T_MARKER OR T_REQUIRE) + # SETUP must be sourced and MARKER/REQUIRE checked -> go through the staged runner. set(_opts "") if(T_MARKER) list(APPEND _opts --marker "${T_MARKER}") endif() + foreach(_req IN LISTS T_REQUIRE) + if(NOT _req STREQUAL "") + list(APPEND _opts --require "${_req}") + endif() + endforeach() if(T_SETUP) list(APPEND _opts --setup "${CMAKE_CURRENT_SOURCE_DIR}/${T_SETUP}") endif() @@ -337,9 +344,10 @@ endforeach() codes_add_equivalence_test( NAME example-ping-pong-determinism BINARY tutorial-synthetic-ping-pong - NP 3 - ARGS --sync=3 --num_messages=10 --payload_sz=8192 - CONFIG doc/example/tutorial-ping-pong.conf) + NP 3 + ARGS --sync=3 --num_messages=10 --payload_sz=8192 + CONFIG doc/example/tutorial-ping-pong.conf + REQUIRE "START PARALLEL OPTIMISTIC SIMULATION") # Surrogate-mode determinism: same surrogate config run twice (np 3, sync=3), # committed-event counts must match. The two variants differ only in network @@ -351,7 +359,7 @@ codes_add_equivalence_test( ARGS --sync=3 --num_messages=100 --payload_sz=8192 CONFIG tutorial-ping-pong-surrogate.conf SETUP surrogate-determinism-no-freeze-setup.sh - REQUIRE "Network switch completed") + REQUIRE "Network switch completed" "START PARALLEL OPTIMISTIC SIMULATION") codes_add_equivalence_test( NAME example-ping-pong-surrogate-determinism-freeze BINARY tutorial-synthetic-ping-pong @@ -359,7 +367,7 @@ codes_add_equivalence_test( ARGS --sync=3 --num_messages=100 --payload_sz=8192 CONFIG tutorial-ping-pong-surrogate.conf SETUP surrogate-determinism-freeze-setup.sh - REQUIRE "Network switch completed") + REQUIRE "Network switch completed" "START PARALLEL OPTIMISTIC SIMULATION") # Smoke test: the sim runs cleanly with an empty packet-latency trace path. # Single run (SETUP generates the config; MARKER asserts it produced output). @@ -384,7 +392,8 @@ codes_add_run_test(NAME rc-stack-test BINARY rc-stack-test) codes_add_run_test(NAME modelnet-prio-sched-test-seq BINARY modelnet-prio-sched-test NP 1 ARGS --sync=1 -- "${_tconf}/modelnet-prio-sched-test.conf") codes_add_run_test(NAME modelnet-prio-sched-test-opt BINARY modelnet-prio-sched-test - NP 2 ARGS --sync=3 -- "${_tconf}/modelnet-prio-sched-test.conf") + NP 2 ARGS --sync=3 -- "${_tconf}/modelnet-prio-sched-test.conf" + REQUIRE "START PARALLEL OPTIMISTIC SIMULATION") codes_add_run_test(NAME jobmap-test BINARY jobmap-test ARGS "${_tconf}/jobmap-test-list.conf") codes_add_run_test(NAME map-ctx-test BINARY map-ctx-test ARGS "${_tconf}/map-ctx-test.conf") codes_add_run_test(NAME resource-test BINARY resource-test ARGS --sync=1 "--codes-config=${_tconf}/buffer_test.conf") From eb21844224943b67af8cc6cc5496c18b494217f2 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 15:12:05 -0400 Subject: [PATCH 18/20] Revert MPI launcher workflow override --- .github/workflows/build.yml | 54 +++---------------------------------- 1 file changed, 3 insertions(+), 51 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 14501b78..ec156804 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -112,18 +112,6 @@ jobs: if [ "${{ matrix.cc }}" = "clang" ]; then sudo apt-get install -y clang fi - if [ "${{ matrix.mpi }}" = "mpich" ]; then - mpi_bin="$RUNNER_TEMP/mpi-bin" - mkdir -p "$mpi_bin" - mpiexec_path="$(command -v mpiexec.mpich || command -v mpiexec)" - mpirun_path="$(command -v mpirun.mpich || command -v mpirun)" - ln -sf "$mpiexec_path" "$mpi_bin/mpiexec" - ln -sf "$mpirun_path" "$mpi_bin/mpirun" - echo "$mpi_bin" >> "$GITHUB_PATH" - echo "CODES_MPIEXEC_EXECUTABLE=$mpi_bin/mpiexec" >> "$GITHUB_ENV" - else - echo "CODES_MPIEXEC_EXECUTABLE=$(command -v mpiexec)" >> "$GITHUB_ENV" - fi - name: Install system dependencies (macOS) if: runner.os == 'macOS' @@ -141,7 +129,6 @@ jobs: # ahead of /usr/bin on PATH for subsequent steps. echo "$(brew --prefix bison)/bin" >> "$GITHUB_PATH" echo "$(brew --prefix flex)/bin" >> "$GITHUB_PATH" - echo "CODES_MPIEXEC_EXECUTABLE=$(command -v mpiexec)" >> "$GITHUB_ENV" - name: Select compiler # Drive the base compiler for both ROSS and CODES via CC/CXX. @@ -164,9 +151,6 @@ jobs: -DCMAKE_BUILD_TYPE=Debug -DROSS_BUILD_MODELS=ON -DCMAKE_INSTALL_PREFIX=$PWD/ross-install - -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - name: Build and install ROSS run: cmake --build ross/build --target install -j @@ -177,12 +161,7 @@ jobs: # in CMakePresets.json; ROSS is found via the ROSS_ROOT env var. The two # heavy deps a stock runner might auto-detect are forced off so this leg # stays a deterministic stock build (the `full` job exercises them). - run: > - cmake --preset debug - -DCODES_USE_TORCH=OFF - -DCODES_USE_ZEROMQ=OFF - -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" + run: cmake --preset debug -DCODES_USE_TORCH=OFF -DCODES_USE_ZEROMQ=OFF - name: Build CODES working-directory: codes @@ -235,18 +214,6 @@ jobs: mpich libmpich-dev \ cmake ninja-build pkg-config \ flex bison - mpi_bin="$RUNNER_TEMP/mpi-bin" - mkdir -p "$mpi_bin" - ln -sf "$(command -v mpiexec.mpich || command -v mpiexec)" "$mpi_bin/mpiexec" - ln -sf "$(command -v mpirun.mpich || command -v mpirun)" "$mpi_bin/mpirun" - echo "$mpi_bin" >> "$GITHUB_PATH" - echo "CODES_MPIEXEC_EXECUTABLE=$mpi_bin/mpiexec" >> "$GITHUB_ENV" - mpi_bin="$RUNNER_TEMP/mpi-bin" - mkdir -p "$mpi_bin" - ln -sf "$(command -v mpiexec.mpich || command -v mpiexec)" "$mpi_bin/mpiexec" - ln -sf "$(command -v mpirun.mpich || command -v mpirun)" "$mpi_bin/mpirun" - echo "$mpi_bin" >> "$GITHUB_PATH" - echo "CODES_MPIEXEC_EXECUTABLE=$mpi_bin/mpiexec" >> "$GITHUB_ENV" - name: Configure old ROSS run: > @@ -255,7 +222,6 @@ jobs: -DROSS_BUILD_MODELS=ON -DCMAKE_INSTALL_PREFIX=$PWD/ross-install -DCMAKE_C_COMPILER=mpicc - -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" - name: Build and install old ROSS run: cmake --build ross/build --target install -j @@ -368,11 +334,7 @@ jobs: working-directory: codes # The asan/ubsan preset sets CODES_SANITIZER; heavy deps forced off to # match the stock matrix legs. - run: > - cmake --preset ${{ matrix.sanitizer }} - -DCODES_USE_TORCH=OFF - -DCODES_USE_ZEROMQ=OFF - -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" + run: cmake --preset ${{ matrix.sanitizer }} -DCODES_USE_TORCH=OFF -DCODES_USE_ZEROMQ=OFF - name: Build CODES working-directory: codes @@ -437,12 +399,6 @@ jobs: mpich libmpich-dev \ cmake ninja-build pkg-config \ flex bison lcov - mpi_bin="$RUNNER_TEMP/mpi-bin" - mkdir -p "$mpi_bin" - ln -sf "$(command -v mpiexec.mpich || command -v mpiexec)" "$mpi_bin/mpiexec" - ln -sf "$(command -v mpirun.mpich || command -v mpirun)" "$mpi_bin/mpirun" - echo "$mpi_bin" >> "$GITHUB_PATH" - echo "CODES_MPIEXEC_EXECUTABLE=$mpi_bin/mpiexec" >> "$GITHUB_ENV" - name: Configure ROSS run: > @@ -458,11 +414,7 @@ jobs: # CODES is checked out at the workspace root here (no `path:`), so the # preset is found in the CWD. The `coverage` preset adds # -DCODES_ENABLE_COVERAGE=ON; heavy deps forced off to match the matrix. - run: > - cmake --preset coverage - -DCODES_USE_TORCH=OFF - -DCODES_USE_ZEROMQ=OFF - -DMPIEXEC_EXECUTABLE="$CODES_MPIEXEC_EXECUTABLE" + run: cmake --preset coverage -DCODES_USE_TORCH=OFF -DCODES_USE_ZEROMQ=OFF - name: Build CODES run: cmake --build --preset coverage From 6d4f8b4fb6b0b360316534c3755843e91f425281 Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 15:59:58 -0400 Subject: [PATCH 19/20] Use matching MPI launcher for CTest --- .github/workflows/build.yml | 2 ++ tests/CMakeLists.txt | 38 +++++++++++++++++++++++++++++++++---- tests/equivalence-run.sh | 4 ++++ tests/fluid-flow-wan-ci.sh | 24 +++++++++++++++++------ 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec156804..13d772e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -282,6 +282,7 @@ jobs: env: # ROSS install prefix; find_package(ROSS) honors ROSS_ROOT (CODES presets). ROSS_ROOT: ${{ github.workspace }}/ross-install + UCX_TLS: "self,tcp" # detect_leaks=0: corruption detection first; leaks are the noisiest part # (MPICH internals + CODES's known unfreed globals) and come later with a # suppression file. abort_on_error=1: guarantee a nonzero exit so ctest @@ -379,6 +380,7 @@ jobs: env: # ROSS install prefix; find_package(ROSS) honors ROSS_ROOT (CODES presets). ROSS_ROOT: ${{ github.workspace }}/ross-install + UCX_TLS: "self,tcp" steps: - name: Checkout CODES uses: actions/checkout@v4 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6f4c152a..b158748e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,6 +2,36 @@ enable_testing() configure_file(run-test.sh.in run-test.sh) +# Use an MPI launcher that matches the MPI implementation found by CMake. +# Ubuntu 24.04 runners can expose /usr/bin/mpiexec through alternatives in a +# way that does not match the MPICH headers/libs selected by FindMPI. Prefer +# the implementation-specific MPICH launcher when FindMPI selected MPICH. +set(CODES_TEST_MPIEXEC_EXECUTABLE "${MPIEXEC_EXECUTABLE}") + +set(_codes_mpi_hint + "${MPI_C_COMPILER};${MPI_CXX_COMPILER};${MPI_C_INCLUDE_DIRS};${MPI_CXX_INCLUDE_DIRS};${MPI_C_LIBRARIES};${MPI_CXX_LIBRARIES}") + +if(_codes_mpi_hint MATCHES "mpich") + find_program(CODES_TEST_MPIEXEC_MPICH NAMES mpiexec.mpich mpirun.mpich) + if(CODES_TEST_MPIEXEC_MPICH) + set(CODES_TEST_MPIEXEC_EXECUTABLE "${CODES_TEST_MPIEXEC_MPICH}") + endif() +endif() + +if(NOT CODES_TEST_MPIEXEC_EXECUTABLE) + find_program(CODES_TEST_MPIEXEC_FALLBACK NAMES mpiexec mpirun) + if(CODES_TEST_MPIEXEC_FALLBACK) + set(CODES_TEST_MPIEXEC_EXECUTABLE "${CODES_TEST_MPIEXEC_FALLBACK}") + endif() +endif() + +if(NOT CODES_TEST_MPIEXEC_EXECUTABLE) + message(FATAL_ERROR "Could not find an MPI launcher for tests") +endif() + +message(STATUS "CODES test MPI launcher: ${CODES_TEST_MPIEXEC_EXECUTABLE}") + + option(CODES_ENABLE_ZMQML_HYBRID_TESTS "Register ZMQML hybrid workflow tests that require a running zmqmlserver.py" OFF) @@ -67,7 +97,7 @@ function(codes_add_equivalence_test) else() set(_config "${CODES_BINARY_DIR}/${T_CONFIG}") endif() - set(_launch ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${T_NP} ${MPIEXEC_PREFLAGS}) + set(_launch ${CODES_TEST_MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${T_NP} ${MPIEXEC_PREFLAGS}) set(_cmd "") if(T_VARIANTS) @@ -131,7 +161,7 @@ function(codes_add_run_test) if(NOT T_NP) set(T_NP 1) endif() - set(_run ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${T_NP} ${MPIEXEC_PREFLAGS} + set(_run ${CODES_TEST_MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${T_NP} ${MPIEXEC_PREFLAGS} "$" ${T_ARGS} ${MPIEXEC_POSTFLAGS}) if(T_SETUP OR T_MARKER OR T_REQUIRE) @@ -187,7 +217,7 @@ function(codes_add_lpio_equivalence_test) set(T_NP 1) endif() set(_bin "$") - set(_launch ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${T_NP} ${MPIEXEC_PREFLAGS}) + set(_launch ${CODES_TEST_MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${T_NP} ${MPIEXEC_PREFLAGS}) # Each run writes lp-io into a fresh "lp-io" dir inside its own run-N subdir # (lp_io_prepare mkdir's it and aborts if it already exists, so it must be # relative + per-run-isolated, which the runner's subdirs provide). @@ -330,7 +360,7 @@ foreach(_ffw_sched fifo round_robin) "${_ffw_synch}" "${_ffw_np}" "${_ffw_test}" - "${MPIEXEC_EXECUTABLE}" + "${CODES_TEST_MPIEXEC_EXECUTABLE}" "${MPIEXEC_NUMPROC_FLAG}" WORKING_DIRECTORY "${CODES_BINARY_DIR}") diff --git a/tests/equivalence-run.sh b/tests/equivalence-run.sh index 082e6c35..2c5ce9c2 100755 --- a/tests/equivalence-run.sh +++ b/tests/equivalence-run.sh @@ -89,6 +89,10 @@ run_one() { [[ -z "$req" ]] && continue if ! grep -q "$req" "$out"; then echo "equivalence-run.sh: run ${run_idx} missing required line '$req'" >&2 + echo "--- stdout ---" >&2 + cat "$out" >&2 || true + echo "--- stderr ---" >&2 + cat "$errf" >&2 || true exit 1 fi done diff --git a/tests/fluid-flow-wan-ci.sh b/tests/fluid-flow-wan-ci.sh index 57f5d584..23a87a36 100755 --- a/tests/fluid-flow-wan-ci.sh +++ b/tests/fluid-flow-wan-ci.sh @@ -96,7 +96,7 @@ PY if ! ( cd "$case_name" - "$mpi_exec" "$mpi_np_flag" "$np" "$binary" --sync="$synch" -- fluid-flow-wan.conf \ + "$mpi_exec" "$mpi_np_flag" "$np" "$binary" --synch="$synch" -- fluid-flow-wan.conf \ > model-output.txt 2> model-output-error.txt ); then echo "fluid-flow-wan model run failed" @@ -109,14 +109,26 @@ fi out="$case_name/model-output.txt" -grep "fluid-flow-wan config:" "$out" -grep "switch_scheduler=$scheduler" "$out" -grep "Net Events Processed" "$out" +require_output() { + local pattern="$1" + if ! grep "$pattern" "$out"; then + echo "missing expected output pattern: $pattern" + echo "--- stdout ---" + cat "$out" || true + echo "--- stderr ---" + cat "$case_name/model-output-error.txt" || true + exit 1 + fi +} + +require_output "fluid-flow-wan config:" +require_output "switch_scheduler=$scheduler" +require_output "Net Events Processed" if [[ "$synch" == "1" ]]; then - grep "csv_logs=buffered-forward" "$out" + require_output "csv_logs=buffered-forward" else - grep "csv_logs=buffered-commit" "$out" + require_output "csv_logs=buffered-commit" fi for csv in \ From 9b8b09d99c7ecde248eb1d184caf2f436bcdbded Mon Sep 17 00:00:00 2001 From: Sanjay Chari Date: Thu, 9 Jul 2026 16:14:41 -0400 Subject: [PATCH 20/20] Add diagnostics for MPI used in CI --- .github/workflows/build.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 13d772e8..088b5d31 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,6 +84,7 @@ jobs: # ROSS install prefix; find_package(ROSS) honors ROSS_ROOT, so the CODES # presets need no hardcoded path. Matches the install prefix below. ROSS_ROOT: ${{ github.workspace }}/ross-install + UCX_TLS: "self,tcp" steps: - name: Checkout CODES uses: actions/checkout@v4 @@ -113,6 +114,17 @@ jobs: sudo apt-get install -y clang fi + echo "MPI diagnostics:" + which mpiexec || true + which mpiexec.mpich || true + which mpirun || true + which mpirun.mpich || true + which mpicc || true + mpiexec --version || true + mpiexec.mpich --version || true + mpicc -show || true + dpkg -l | grep -E 'mpich|ucx|openmpi' || true + - name: Install system dependencies (macOS) if: runner.os == 'macOS' run: |