From 8ddce656ac46b3f95d96de5866c35f9bd7e1f29e Mon Sep 17 00:00:00 2001 From: Alexander Van Craen Date: Tue, 11 Aug 2026 11:09:40 +0200 Subject: [PATCH 1/6] add Cray/HPE pm_counters hardware sampler backend Reads whole-node power/energy from Cray/HPE's /sys/cray/pm_counters sysfs interface, to validate hws's software energy measurements against measured hardware ground truth on HLRS's Hunter system. The counter set is discovered and classified at runtime instead of hardcoded, since the exposed files differ across PM counter versions and node generations; confirmed against a real Hunter mi300a node (23 files, PM counters v3). Includes CTest-based regression tests using fixtures from that real dump. --- CMakeLists.txt | 22 ++ include/hws/core.hpp | 5 + .../hws/cray_pm_counters/hardware_sampler.hpp | 149 ++++++++++++ .../cray_pm_counters/pm_counters_samples.hpp | 140 +++++++++++ include/hws/cray_pm_counters/utility.hpp | 102 ++++++++ scripts/hwmon_probe.pbs | 201 ++++++++++++++++ src/hws/cray_pm_counters/CMakeLists.txt | 29 +++ src/hws/cray_pm_counters/hardware_sampler.cpp | 203 ++++++++++++++++ .../cray_pm_counters/pm_counters_samples.cpp | 141 +++++++++++ src/hws/cray_pm_counters/utility.cpp | 150 ++++++++++++ src/hws/system_hardware_sampler.cpp | 90 +++++--- tests/CMakeLists.txt | 9 + tests/cray_pm_counters/CMakeLists.txt | 15 ++ tests/cray_pm_counters/test.cpp | 218 ++++++++++++++++++ 14 files changed, 1437 insertions(+), 37 deletions(-) create mode 100644 include/hws/cray_pm_counters/hardware_sampler.hpp create mode 100644 include/hws/cray_pm_counters/pm_counters_samples.hpp create mode 100644 include/hws/cray_pm_counters/utility.hpp create mode 100644 scripts/hwmon_probe.pbs create mode 100644 src/hws/cray_pm_counters/CMakeLists.txt create mode 100644 src/hws/cray_pm_counters/hardware_sampler.cpp create mode 100644 src/hws/cray_pm_counters/pm_counters_samples.cpp create mode 100644 src/hws/cray_pm_counters/utility.cpp create mode 100644 tests/CMakeLists.txt create mode 100644 tests/cray_pm_counters/CMakeLists.txt create mode 100644 tests/cray_pm_counters/test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ab2b4b..22bfec5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,6 +147,18 @@ else () endif () +#################################################################################################################### +## Cray/HPE node power sampling via pm_counters ## +#################################################################################################################### +set(HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING AUTO CACHE STRING "Enable sampling of node power/energy via Cray's /sys/cray/pm_counters interface.") +set_property(CACHE HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING PROPERTY STRINGS AUTO ON OFF) +if (HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING MATCHES "AUTO" OR HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING) + add_subdirectory(src/hws/cray_pm_counters) +else () + message(STATUS "Hardware sampling via Cray pm_counters disabled!") +endif () + + #################################################################################################################### ## enable MPI support ## #################################################################################################################### @@ -201,6 +213,16 @@ if (HWS_ENABLE_DOCUMENTATION) endif () +######################################################################################################################## +## add tests ## +######################################################################################################################## +option(HWS_ENABLE_TESTING "Build simple regression tests (run via 'ctest')." OFF) +if (HWS_ENABLE_TESTING) + enable_testing() + add_subdirectory(tests) +endif () + + ######################################################################################################################## ## add support for `make install` ## ######################################################################################################################## diff --git a/include/hws/core.hpp b/include/hws/core.hpp index 8c7a474..68dd84b 100644 --- a/include/hws/core.hpp +++ b/include/hws/core.hpp @@ -38,4 +38,9 @@ #include "hws/gpu_intel/level_zero_samples.hpp" #endif +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + #include "hws/cray_pm_counters/hardware_sampler.hpp" + #include "hws/cray_pm_counters/pm_counters_samples.hpp" +#endif + #endif // HWS_CORE_HPP_ diff --git a/include/hws/cray_pm_counters/hardware_sampler.hpp b/include/hws/cray_pm_counters/hardware_sampler.hpp new file mode 100644 index 0000000..75a2fe0 --- /dev/null +++ b/include/hws/cray_pm_counters/hardware_sampler.hpp @@ -0,0 +1,149 @@ +/** + * @file + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + * + * @brief Defines a hardware sampler for whole-node power/energy using Cray/HPE's `/sys/cray/pm_counters` sysfs + * interface (measured, not modeled, node power; includes voltage converter losses). + */ + +#ifndef HWS_CRAY_PM_COUNTERS_HARDWARE_SAMPLER_HPP_ +#define HWS_CRAY_PM_COUNTERS_HARDWARE_SAMPLER_HPP_ +#pragma once + +#include "hws/cray_pm_counters/pm_counters_samples.hpp" // hws::{cray_pm_counters_general_samples, cray_pm_counters_power_samples} +#include "hws/hardware_sampler.hpp" // hws::hardware_sampler +#include "hws/sample_category.hpp" // hws::sample_category + +#include "fmt/ostream.h" // fmt::formatter, fmt::ostream_formatter + +#include // std::chrono::milliseconds, std::chrono_literals namespace +#include // std::filesystem::path +#include // std::ostream forward declaration +#include // std::optional +#include // std::unordered_map + +namespace hws { + +using namespace std::chrono_literals; + +/** + * @brief A hardware sampler for whole-node power/energy using Cray/HPE's `/sys/cray/pm_counters` sysfs interface. + * @details Since `pm_counters` is a per-node (not per-device) sysfs interface, only a single instance of this + * sampler exists per node (mirroring `hws::cpu_hardware_sampler`), even on nodes with an accelerator + * (whose "accel energy"/"accel power" counters `pm_counters` also exposes). In MPI `whole_node` mode + * only the node-local rank 0 creates this sampler; in `per_rank` mode (and the non-MPI, single-process + * case), every rank on a node creates its own instance, so the same node-wide ground truth is sampled and + * reported redundantly once per rank - this mirrors `hws::cpu_hardware_sampler`'s existing behavior. + */ +class cray_pm_counters_hardware_sampler : public hardware_sampler { + public: + /** + * @brief Construct a new pm_counters hardware sampler with the default sampling interval. + * @param[in] category the sample categories that are enabled for hardware sampling (default: all) + */ + explicit cray_pm_counters_hardware_sampler(sample_category category = sample_category::all); + /** + * @brief Construct a new pm_counters hardware sampler with the @p sampling_interval. + * @param[in] sampling_interval the used sampling interval + * @param[in] category the sample categories that are enabled for hardware sampling (default: all) + */ + explicit cray_pm_counters_hardware_sampler(std::chrono::milliseconds sampling_interval, sample_category category = sample_category::all); + + /** + * @brief Delete the copy-constructor (already implicitly deleted due to the base class's std::atomic member). + */ + cray_pm_counters_hardware_sampler(const cray_pm_counters_hardware_sampler &) = delete; + /** + * @brief Delete the move-constructor (already implicitly deleted due to the base class's std::atomic member). + */ + cray_pm_counters_hardware_sampler(cray_pm_counters_hardware_sampler &&) noexcept = delete; + /** + * @brief Delete the copy-assignment operator (already implicitly deleted due to the base class's std::atomic member). + */ + cray_pm_counters_hardware_sampler &operator=(const cray_pm_counters_hardware_sampler &) = delete; + /** + * @brief Delete the move-assignment operator (already implicitly deleted due to the base class's std::atomic member). + */ + cray_pm_counters_hardware_sampler &operator=(cray_pm_counters_hardware_sampler &&) noexcept = delete; + + /** + * @brief Destruct the pm_counters hardware sampler. If the sampler is still running, stops it. + */ + ~cray_pm_counters_hardware_sampler() override; + + /** + * @brief Return the general pm_counters samples of this hardware sampler. + * @return the general pm_counters samples (`[[nodiscard]]`) + */ + [[nodiscard]] const cray_pm_counters_general_samples &general_samples() const noexcept { return general_samples_; } + + /** + * @brief Return the power related pm_counters samples of this hardware sampler. + * @return the power related pm_counters samples (`[[nodiscard]]`) + */ + [[nodiscard]] const cray_pm_counters_power_samples &power_samples() const noexcept { return power_samples_; } + + /** + * @copydoc hws::hardware_sampler::device_identification + */ + [[nodiscard]] std::string device_identification() const final; + + /** + * @copydoc hws::hardware_sampler::samples_only_as_yaml_string() const + */ + [[nodiscard]] std::string samples_only_as_yaml_string() const final; + + private: + /** + * @copydoc hws::hardware_sampler::sampling_loop + */ + void sampling_loop() final; + + /** + * @brief Read every discovered pm_counters file once and append the readings to `general_samples_`/`power_samples_`. + * @details Called both for the very first sample and on every subsequent tick of the sampling loop - all three + * categories (general, energy, power) are re-read every tick, see `cray_pm_counters_general_samples`. + */ + void sample_once(); + + /** + * @brief Derive the hardware's own update period from the already-sampled `raw_scan_hz` metadata. + * @return the hardware tick period, or `std::nullopt` if `raw_scan_hz` wasn't found/couldn't be parsed (`[[nodiscard]]`) + */ + [[nodiscard]] std::optional hardware_tick_period() const; + + /// The general pm_counters samples. + cray_pm_counters_general_samples general_samples_{}; + /// The power related pm_counters samples. + cray_pm_counters_power_samples power_samples_{}; + + /// The full sysfs path of every general metadata file discovered at sampling start, keyed by its `general_samples().get_metadata()` map key. + std::unordered_map general_paths_{}; + /// The full sysfs path of every energy counter discovered at sampling start, keyed by its `power_samples().get_energy_counters()` map key. + std::unordered_map energy_counter_paths_{}; + /// The full sysfs path of every power counter discovered at sampling start, keyed by its `power_samples().get_power_counters()` map key. + std::unordered_map power_counter_paths_{}; +}; + +/** + * @brief Output all pm_counters samples gathered by the @p sampler to the given output-stream @p out. + * @details Sets `std::ios_base::failbit` if the @p sampler is still sampling. + * @param[in,out] out the output-stream to write the pm_counters samples to + * @param[in] sampler the pm_counters hardware sampler + * @return the output-stream + */ +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_hardware_sampler &sampler); + +} // namespace hws + +/// @cond Doxygen_suppress + +template <> +struct fmt::formatter : fmt::ostream_formatter { }; + +/// @endcond + +#endif // HWS_CRAY_PM_COUNTERS_HARDWARE_SAMPLER_HPP_ diff --git a/include/hws/cray_pm_counters/pm_counters_samples.hpp b/include/hws/cray_pm_counters/pm_counters_samples.hpp new file mode 100644 index 0000000..b775246 --- /dev/null +++ b/include/hws/cray_pm_counters/pm_counters_samples.hpp @@ -0,0 +1,140 @@ +/** + * @file + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + * + * @brief Defines the samples gathered from Cray/HPE's `/sys/cray/pm_counters` sysfs interface. + * @details The exact set of files exposed by `/sys/cray/pm_counters` is Cray/HPE-generation specific, so instead of + * hardcoding specific file names, every regular file found below the directory is picked up dynamically + * and classified by its (relative) file name into a measured energy counter, a measured power counter, or + * (everything else, including configured caps and derived/event counters) general metadata - see + * `hws::detail::is_energy_counter_key`/`is_power_counter_key`. All three categories are re-read every + * sampling tick (see `general_samples()` for why even seemingly static metadata like `freshness` needs to + * be). Confirmed against a real HPE Cray EX255a node (`/sys/cray/pm_counters` version 3): 23 files - + * `energy`/`accel[0-3]_energy` (measured energy, J), `power`/`accel[0-3]_power` (measured power, W), and + * general metadata covering both configured limits (`power_cap`/`accel[0-3]_power_cap`), derived/event + * counters (`capped_energy`, `overshoot`, `overshoot_energy`), and protocol metadata (`freshness`, + * `generation`, `raw_scan_hz`, `startup`, `version`). Version 3 telemetry files (but not the `*_cap` + * files) contain `" us"`, e.g. `"2869283299 J 7810213855886 us"` - see + * `pm_counter_reading`. + */ + +#ifndef HWS_CRAY_PM_COUNTERS_PM_COUNTERS_SAMPLES_HPP_ +#define HWS_CRAY_PM_COUNTERS_PM_COUNTERS_SAMPLES_HPP_ +#pragma once + +#include "hws/utility.hpp" // HWS_SAMPLE_STRUCT_FIXED_MEMBER + +#include "fmt/ostream.h" // fmt::formatter, fmt::ostream_formatter + +#include // std::uint64_t +#include // std::ostream forward declaration +#include // std::string +#include // std::unordered_map +#include // std::vector + +namespace hws { + +//*************************************************************************************************************************************// +// general samples // +//*************************************************************************************************************************************// + +/** + * @brief Wrapper class for the non-energy, non-power `/sys/cray/pm_counters` metadata, e.g. `version`, + * `generation`, `raw_scan_hz`, or `freshness`. + * @details Sampled every tick, not just once: while `version`/`startup`/`raw_scan_hz` are static, `freshness` + * increments at `raw_scan_hz` and `generation`/`overshoot` change when a power cap is (re)applied or + * exceeded. HPE's documented consistency check is to read `freshness` before and after a batch of energy + * /power reads; if it's unchanged, that batch is a consistent snapshot - only possible if it's re-sampled. + */ +class cray_pm_counters_general_samples { + // befriend hardware sampler class + friend class cray_pm_counters_hardware_sampler; + /// The map type used for the dynamically discovered metadata entries: file name (relative to `/sys/cray/pm_counters`) -> sampled raw file content. + using metadata_type = std::unordered_map>; + + public: + /** + * @brief Checks whether any general related hardware sample is present. + * @return `true` if any general related hardware sample is, otherwise `false`. + */ + [[nodiscard]] bool has_samples() const; + /** + * @brief Assemble the YAML string containing all available general hardware samples. + * @details Returns an empty string if `has_samples()` returns `false`. + * @return the YAML string (`[[nodiscard]]`) + */ + [[nodiscard]] std::string generate_yaml_string() const; + + HWS_SAMPLE_STRUCT_FIXED_MEMBER(metadata_type, metadata) // dynamically discovered pm_counters metadata entries +}; + +/** + * @brief Output the general @p samples to the given output-stream @p out. + * @param[in,out] out the output-stream to write the general related hardware samples to + * @param[in] samples the pm_counters general related samples + * @return the output-stream + */ +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_general_samples &samples); + +//*************************************************************************************************************************************// +// power samples // +//*************************************************************************************************************************************// + +/** + * @brief Wrapper class for the dynamically discovered energy and power counters below `/sys/cray/pm_counters` + * (e.g. `energy`/`power` for the whole node, `accel[i]_energy`/`accel[i]_power` per APU). + * @details Values are cumulative Joules for energy counters and instantaneous Watts for power counters (confirmed + * against a real HPE Cray EX255a node). On PM counters version 3, telemetry files (but not the `*_cap` + * files) additionally carry a per-sample microsecond timestamp (see `energy_timestamps_us`/ + * `power_timestamps_us`), latched by the HSS - use this instead of the host read time for accurate + * energy-to-power derivatives, since pm_counters only updates at `raw_scan_hz` (10 Hz). + */ +class cray_pm_counters_power_samples { + // befriend hardware sampler class + friend class cray_pm_counters_hardware_sampler; + /// The map type used for the dynamically discovered counters: file name (relative to `/sys/cray/pm_counters`) -> sampled raw values. + using counter_map_type = std::unordered_map>; + + public: + /** + * @brief Checks whether any power related hardware sample is present. + * @return `true` if any power related hardware sample is, otherwise `false`. + */ + [[nodiscard]] bool has_samples() const; + /** + * @brief Assemble the YAML string containing all available power hardware samples. + * @details Returns an empty string if `has_samples()` returns `false`. + * @return the YAML string (`[[nodiscard]]`) + */ + [[nodiscard]] std::string generate_yaml_string() const; + + HWS_SAMPLE_STRUCT_FIXED_MEMBER(counter_map_type, energy_counters) // dynamically discovered *energy* pm_counters entries, in J + HWS_SAMPLE_STRUCT_FIXED_MEMBER(counter_map_type, power_counters) // dynamically discovered *power* pm_counters entries, in W + HWS_SAMPLE_STRUCT_FIXED_MEMBER(counter_map_type, energy_timestamps_us) // per-sample HSS latch timestamp for energy_counters, in us (PM counters v3+) + HWS_SAMPLE_STRUCT_FIXED_MEMBER(counter_map_type, power_timestamps_us) // per-sample HSS latch timestamp for power_counters, in us (PM counters v3+) +}; + +/** + * @brief Output the power related @p samples to the given output-stream @p out. + * @param[in,out] out the output-stream to write the power related hardware samples to + * @param[in] samples the pm_counters power related samples + * @return the output-stream + */ +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_power_samples &samples); + +} // namespace hws + +/// @cond Doxygen_suppress + +template <> +struct fmt::formatter : fmt::ostream_formatter { }; + +template <> +struct fmt::formatter : fmt::ostream_formatter { }; + +/// @endcond + +#endif // HWS_CRAY_PM_COUNTERS_PM_COUNTERS_SAMPLES_HPP_ diff --git a/include/hws/cray_pm_counters/utility.hpp b/include/hws/cray_pm_counters/utility.hpp new file mode 100644 index 0000000..e999a5b --- /dev/null +++ b/include/hws/cray_pm_counters/utility.hpp @@ -0,0 +1,102 @@ +/** + * @file + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + * + * @brief Utility functions to discover and read Cray/HPE's `/sys/cray/pm_counters` sysfs interface. + */ + +#ifndef HWS_CRAY_PM_COUNTERS_UTILITY_HPP_ +#define HWS_CRAY_PM_COUNTERS_UTILITY_HPP_ +#pragma once + +#include // std::uint64_t +#include // std::filesystem::path +#include // std::optional +#include // std::string +#include // std::string_view +#include // std::vector + +namespace hws::detail { + +/// The default root directory of Cray/HPE's pm_counters sysfs interface. +inline constexpr std::string_view default_pm_counters_root = "/sys/cray/pm_counters"; + +/** + * @brief Return the root directory of the pm_counters sysfs interface to use. + * @details Defaults to `hws::detail::default_pm_counters_root`, but can be overridden via the `HWS_PM_COUNTERS_PATH` + * environment variable (e.g. to point at a synthetic directory for local testing off a Cray system). + * @return the pm_counters root directory (`[[nodiscard]]`) + */ +[[nodiscard]] std::filesystem::path pm_counters_root(); + +/** + * @brief Check whether the pm_counters sysfs interface is available on this node. + * @return `true` if `hws::detail::pm_counters_root()` exists and is a directory, otherwise `false` (`[[nodiscard]]`) + */ +[[nodiscard]] bool pm_counters_available(); + +/** + * @brief Recursively list all regular files below `hws::detail::pm_counters_root()`. + * @return the discovered files, or an empty vector if pm_counters isn't available (`[[nodiscard]]`) + */ +[[nodiscard]] std::vector list_pm_counter_files(); + +/** + * @brief Derive the unique map key for @p file: its path relative to `hws::detail::pm_counters_root()`, e.g. + * `energy` or, for a nested file, `accel0/energy` ('/' is a valid, unquoted YAML mapping key character). + * @param[in] file the pm_counters file, must be located below `hws::detail::pm_counters_root()` + * @return the map key (`[[nodiscard]]`) + */ +[[nodiscard]] std::string pm_counter_key(const std::filesystem::path &file); + +/** + * @brief Read the trimmed content of @p file as a raw string. + * @param[in] file the file to read + * @return the trimmed file content, or an empty string if the file couldn't be read (`[[nodiscard]]`) + */ +[[nodiscard]] std::string read_pm_counter_raw(const std::filesystem::path &file); + +/** + * @brief A single pm_counters energy/power reading. + * @details On Hunter (PM counters version 3), telemetry files (e.g. `energy`, `accel0_power`) contain + * `" us"` (e.g. `"2869283299 J 7810213855886 us"`), while power/energy cap + * files and older PM counter versions may only contain `" "` or even just `""`. + */ +struct pm_counter_reading { + /// The counter's raw value (unit depends on the file, e.g. J for energy, W for power). + std::uint64_t value{}; + /// The microsecond timestamp at which the value was latched by the HSS, if the file provides one. + std::optional timestamp_us{}; +}; + +/** + * @brief Read the content of @p file and try to parse it as a `pm_counter_reading`. + * @param[in] file the file to read + * @return the parsed reading, or `std::nullopt` if the file couldn't be read or doesn't start with an unsigned integer (`[[nodiscard]]`) + */ +[[nodiscard]] std::optional read_pm_counter_reading(const std::filesystem::path &file); + +/** + * @brief Check whether the map key @p key (as returned by `pm_counter_key`) looks like a measured energy counter: + * contains "energy" but not "cap" or "overshoot" (e.g. `energy`/`accel0_energy`, but not `capped_energy` + * or `overshoot_energy`, which are derived/event counters, not raw measured node/accelerator energy). + * @param[in] key the map key to check + * @return `true` if @p key looks like a measured energy counter, otherwise `false` (`[[nodiscard]]`) + */ +[[nodiscard]] bool is_energy_counter_key(const std::string &key); + +/** + * @brief Check whether the map key @p key (as returned by `pm_counter_key`) looks like a measured power counter: + * contains "power" but not "cap" or "overshoot" (e.g. `power`/`accel0_power`, but not `power_cap`/ + * `accel0_power_cap`, which are configured limits, not measured power draw). + * @param[in] key the map key to check + * @return `true` if @p key looks like a measured power counter, otherwise `false` (`[[nodiscard]]`) + */ +[[nodiscard]] bool is_power_counter_key(const std::string &key); + +} // namespace hws::detail + +#endif // HWS_CRAY_PM_COUNTERS_UTILITY_HPP_ diff --git a/scripts/hwmon_probe.pbs b/scripts/hwmon_probe.pbs new file mode 100644 index 0000000..76ba996 --- /dev/null +++ b/scripts/hwmon_probe.pbs @@ -0,0 +1,201 @@ +#!/bin/bash +#PBS -N hwmon_probe +#PBS -l select=1:node_type=mi300a:mpiprocs=1 +#PBS -l walltime=00:15:00 +#PBS -j oe +#PBS -q test + +# Site-specific: written for and tested on HLRS's "Hunter" system (HPE Cray EX, PBS Pro scheduler). The +# #PBS directives above (queue name "test", node_type selector "mi300a") are specific to that system and +# will need adapting - or replacing with the equivalent directives for your own scheduler - before use +# elsewhere. Dumps hwmon (amd_energy, amdgpu, cpufreq) and, if present, /sys/cray/pm_counters diagnostics +# for a single compute node. + +set -x + +echo "===================================================================" +echo " Job: $PBS_JOBID" +echo " Host: $(hostname -f)" +echo " Node: $(cat $PBS_NODEFILE 2>/dev/null)" +echo " Date: $(date)" +echo "===================================================================" + +# --------------------------------------------------------------------------- +# Helper: dump every readable *_input / *_label / *_alarm file below a hwmon +# directory, without ever failing the job on a permission error. +# --------------------------------------------------------------------------- +dump_hwmon_dir() { + local hw="$1" + for f in "$hw"/*_input "$hw"/*_label "$hw"/*_average "$hw"/*_max "$hw"/*_crit; do + [ -e "$f" ] || continue + local val + val=$(cat "$f" 2>/dev/null) + if [ -n "$val" ]; then + printf " %-28s = %s\n" "$(basename "$f")" "$val" + else + printf " %-28s = \n" "$(basename "$f")" + fi + done +} + +echo +echo "### --- Alle hwmon-Devices und deren Rohwerte -----------------------" +for hw in /sys/class/hwmon/hwmon*; do + [ -d "$hw" ] || continue + name=$(cat "$hw/name" 2>/dev/null) + echo + echo "-- $hw (name: $name) --" + dump_hwmon_dir "$hw" +done + +echo +echo "### --- /sys/cray/pm_counters: komplettes Layout + Rohwerte ---------" +if [ -d /sys/cray/pm_counters ]; then + echo "-- Verzeichnisstruktur (find) --" + find /sys/cray/pm_counters -mindepth 1 2>/dev/null + + echo + echo "-- Inhalt aller regulaeren Dateien --" + for f in $(find /sys/cray/pm_counters -mindepth 1 -type f 2>/dev/null | sort); do + val=$(cat "$f" 2>/dev/null) + if [ -n "$val" ]; then + printf " %-50s = %s\n" "$f" "$val" + else + printf " %-50s = \n" "$f" + fi + done + + echo + echo "-- zwei Messpunkte fuer alle numerischen (vermutlich Energie-)Zaehler --" + # Falls ein Wert wie "123456 J" oder "110 W" statt reiner Zahl geliefert wird, wird nur die + # fuehrende Zahl fuer den Test/die Arithmetik verwendet (Einheit wird zusaetzlich mitgeloggt). + declare -A pm_t0 + declare -A pm_t0_raw + pm_files=$(find /sys/cray/pm_counters -mindepth 1 -type f 2>/dev/null | sort) + t0=$(date +%s.%N) + for f in $pm_files; do + val_raw=$(cat "$f" 2>/dev/null) + val="${val_raw%% *}" + [[ "$val" =~ ^[0-9]+$ ]] || continue + pm_t0[$f]=$val + pm_t0_raw[$f]=$val_raw + done + + sleep_interval=1 + sleep "$sleep_interval" + + t1=$(date +%s.%N) + dt=$(python3 -c "print($t1-$t0)") + for f in "${!pm_t0[@]}"; do + v1_raw=$(cat "$f" 2>/dev/null) + v1="${v1_raw%% *}" + [[ "$v1" =~ ^[0-9]+$ ]] || continue + delta=$((v1 - ${pm_t0[$f]})) + echo " $f: t0=${pm_t0_raw[$f]} t1=${v1_raw} delta=${delta} dt=${dt}s" + done +else + echo "kein /sys/cray/pm_counters vorhanden" +fi + +echo +echo "### --- amd_energy: zwei Messpunkte fuer Delta/Power-Berechnung -----" +energy_hw="" +for hw in /sys/class/hwmon/hwmon*; do + [ -d "$hw" ] || continue + if [ "$(cat "$hw/name" 2>/dev/null)" = "amd_energy" ]; then + energy_hw="$hw" + break + fi +done + +if [ -n "$energy_hw" ]; then + echo "amd_energy gefunden unter: $energy_hw" + echo + echo "-- verfuegbare Energie-Kanaele --" + for lbl in "$energy_hw"/energy*_label; do + [ -e "$lbl" ] || continue + idx=$(basename "$lbl" | sed -E 's/energy([0-9]+)_label/\1/') + echo " energy${idx}: $(cat "$lbl" 2>/dev/null) (input file: energy${idx}_input, Einheit: uJ, monoton steigend)" + done + + echo + echo "-- Messung 1 (t0) --" + declare -A e_t0 + t0=$(date +%s.%N) + for inp in "$energy_hw"/energy*_input; do + [ -e "$inp" ] || continue + idx=$(basename "$inp" | sed -E 's/energy([0-9]+)_input/\1/') + e_t0[$idx]=$(cat "$inp" 2>/dev/null) + echo " energy${idx}_input = ${e_t0[$idx]} uJ" + done + + sleep_interval=1 + echo + echo "-- warte ${sleep_interval}s --" + sleep "$sleep_interval" + + echo + echo "-- Messung 2 (t1) --" + t1=$(date +%s.%N) + dt=$(echo "$t1 - $t0" | bc -l 2>/dev/null || python3 -c "print($t1-$t0)") + for inp in "$energy_hw"/energy*_input; do + [ -e "$inp" ] || continue + idx=$(basename "$inp" | sed -E 's/energy([0-9]+)_input/\1/') + e_t1=$(cat "$inp" 2>/dev/null) + delta_uj=$((e_t1 - ${e_t0[$idx]})) + # Power[W] = delta_energy[uJ] / 1e6 / delta_t[s] + power_w=$(python3 -c "print(f'{${delta_uj} / 1000000.0 / ${dt}:.3f}')" 2>/dev/null) + echo " energy${idx}_input = ${e_t1} uJ delta = ${delta_uj} uJ dt = ${dt}s -> Power = ${power_w} W" + done +else + echo "kein amd_energy hwmon gefunden" +fi + +echo +echo "### --- amdgpu hwmon: Temperaturen / Power (falls vorhanden) --------" +for hw in /sys/class/hwmon/hwmon*; do + [ -d "$hw" ] || continue + if [ "$(cat "$hw/name" 2>/dev/null)" = "amdgpu" ]; then + echo + echo "-- $hw --" + dump_hwmon_dir "$hw" + fi +done + +echo +echo "### --- cpufreq: aktueller Takt pro Online-CPU -----------------------" +governor_seen="" +for cpu in /sys/devices/system/cpu/cpu[0-9]*; do + [ -d "$cpu/cpufreq" ] || continue + cpuid=$(basename "$cpu") + cur=$(cat "$cpu/cpufreq/scaling_cur_freq" 2>/dev/null) + gov=$(cat "$cpu/cpufreq/scaling_governor" 2>/dev/null) + drv=$(cat "$cpu/cpufreq/scaling_driver" 2>/dev/null) + [ -z "$governor_seen" ] && governor_seen="driver=$drv governor=$gov" + echo " $cpuid: scaling_cur_freq=${cur:-} kHz" +done +echo +echo " (Treiber/Governor, erster gefundener Core: $governor_seen)" + +echo +echo "-- Mittelwert ueber alle Online-CPUs (kHz -> MHz) --" +python3 - <<'EOF' +import glob +vals = [] +for p in glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq/scaling_cur_freq"): + try: + with open(p) as f: + vals.append(int(f.read().strip())) + except Exception: + pass +if vals: + avg_khz = sum(vals) / len(vals) + print(f" n_cores={len(vals)} avg={avg_khz/1000.0:.1f} MHz min={min(vals)/1000.0:.1f} MHz max={max(vals)/1000.0:.1f} MHz") +else: + print(" keine scaling_cur_freq Werte lesbar") +EOF + +echo +echo "===================================================================" +echo " Fertig: $(date)" +echo "===================================================================" diff --git a/src/hws/cray_pm_counters/CMakeLists.txt b/src/hws/cray_pm_counters/CMakeLists.txt new file mode 100644 index 0000000..54125c7 --- /dev/null +++ b/src/hws/cray_pm_counters/CMakeLists.txt @@ -0,0 +1,29 @@ +## Authors: Alexander Van Craen +## Copyright (C): 2024-today All Rights Reserved +## License: This file is released under the MIT license. +## See the LICENSE.md file in the project root for full license information. +######################################################################################################################## + +# pm_counters is a Linux-only (Cray/HPE) sysfs interface, not a linkable library -> nothing to find_package/find_program. +# Availability of /sys/cray/pm_counters is checked at runtime (build host and target host may differ, e.g. cross-compiling +# on a login node for execution on a compute node), so this backend is always compiled in on Linux. +if (NOT UNIX) + if (HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING MATCHES "ON") + message(SEND_ERROR "Cray pm_counters sampling was explicitly requested but is only available on Linux!") + else () + message(STATUS "Cray pm_counters sampling is only available on Linux. Hardware sampling via pm_counters disabled.") + endif () + return() +endif () +message(STATUS "Enable sampling of node power/energy via /sys/cray/pm_counters (availability checked at runtime).") + +# add source file to source file list +target_sources(${HWS_LIBRARY_NAME} PRIVATE + $) + +# add compile definition +target_compile_definitions(${HWS_LIBRARY_NAME} PUBLIC HWS_FOR_CRAY_PM_COUNTERS_ENABLED) diff --git a/src/hws/cray_pm_counters/hardware_sampler.cpp b/src/hws/cray_pm_counters/hardware_sampler.cpp new file mode 100644 index 0000000..433c38b --- /dev/null +++ b/src/hws/cray_pm_counters/hardware_sampler.cpp @@ -0,0 +1,203 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + */ + +#include "hws/cray_pm_counters/hardware_sampler.hpp" + +#include "hws/cray_pm_counters/pm_counters_samples.hpp" // hws::{cray_pm_counters_general_samples, cray_pm_counters_power_samples} +#include "hws/cray_pm_counters/utility.hpp" // hws::detail::{pm_counters_available, list_pm_counter_files, pm_counter_key, read_pm_counter_raw, read_pm_counter_reading, pm_counter_reading, is_energy_counter_key, is_power_counter_key} +#include "hws/hardware_sampler.hpp" // hws::hardware_sampler +#include "hws/sample_category.hpp" // hws::sample_category +#include "hws/utility.hpp" // hws::detail::time_points_to_epoch + +#include "fmt/chrono.h" // direct formatting of std::chrono types +#include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join + +#include // std::max +#include // std::chrono::{steady_clock, milliseconds, duration_cast} +#include // std::exception, std::terminate +#include // std::filesystem::path +#include // std::ios_base +#include // std::cerr, std::endl +#include // std::optional +#include // std::ostream +#include // std::runtime_error +#include // std::string +#include // std::this_thread + +namespace hws { + +cray_pm_counters_hardware_sampler::cray_pm_counters_hardware_sampler(const sample_category category) : + cray_pm_counters_hardware_sampler{ HWS_SAMPLING_INTERVAL, category } { } + +cray_pm_counters_hardware_sampler::cray_pm_counters_hardware_sampler(const std::chrono::milliseconds sampling_interval, const sample_category category) : + hardware_sampler{ sampling_interval, category } { } + +cray_pm_counters_hardware_sampler::~cray_pm_counters_hardware_sampler() { + try { + // if this hardware sampler is still sampling, stop it + if (this->has_sampling_started() && !this->has_sampling_stopped()) { + this->stop_sampling(); + } + } catch (const std::exception &e) { + std::cerr << e.what() << std::endl; + std::terminate(); + } +} + +void cray_pm_counters_hardware_sampler::sample_once() { + if (this->sample_category_enabled(sample_category::general)) { + if (!general_samples_.metadata_.has_value() && !general_paths_.empty()) { + general_samples_.metadata_ = cray_pm_counters_general_samples::metadata_type{}; + } + for (const auto &[key, path] : general_paths_) { + general_samples_.metadata_.value()[key].push_back(detail::read_pm_counter_raw(path)); + } + } + + if (this->sample_category_enabled(sample_category::power)) { + if (!power_samples_.energy_counters_.has_value() && !energy_counter_paths_.empty()) { + power_samples_.energy_counters_ = cray_pm_counters_power_samples::counter_map_type{}; + power_samples_.energy_timestamps_us_ = cray_pm_counters_power_samples::counter_map_type{}; + } + if (!power_samples_.power_counters_.has_value() && !power_counter_paths_.empty()) { + power_samples_.power_counters_ = cray_pm_counters_power_samples::counter_map_type{}; + power_samples_.power_timestamps_us_ = cray_pm_counters_power_samples::counter_map_type{}; + } + for (const auto &[key, path] : energy_counter_paths_) { + const std::optional reading = detail::read_pm_counter_reading(path); + if (reading.has_value()) { + power_samples_.energy_counters_.value()[key].push_back(reading->value); + if (reading->timestamp_us.has_value()) { + power_samples_.energy_timestamps_us_.value()[key].push_back(reading->timestamp_us.value()); + } + } + } + for (const auto &[key, path] : power_counter_paths_) { + const std::optional reading = detail::read_pm_counter_reading(path); + if (reading.has_value()) { + power_samples_.power_counters_.value()[key].push_back(reading->value); + if (reading->timestamp_us.has_value()) { + power_samples_.power_timestamps_us_.value()[key].push_back(reading->timestamp_us.value()); + } + } + } + } +} + +std::optional cray_pm_counters_hardware_sampler::hardware_tick_period() const { + // read directly from the discovered path rather than general_samples_.metadata_: the latter is only populated + // when sample_category::general is enabled, but throttling to the hardware's real rate must apply regardless + // of which sample categories the caller requested (e.g. sample_category::power alone). + const auto it = general_paths_.find("raw_scan_hz"); + if (it == general_paths_.cend()) { + return std::nullopt; + } + + const std::string content = detail::read_pm_counter_raw(it->second); + if (content.empty()) { + return std::nullopt; + } + + try { + const double hz = detail::convert_to(content); + if (hz <= 0.0) { + return std::nullopt; + } + return std::chrono::milliseconds{ static_cast(1000.0 / hz) }; + } catch (const std::exception &) { + return std::nullopt; + } +} + +void cray_pm_counters_hardware_sampler::sampling_loop() { + // + // discover the available pm_counters files once and classify them by name into energy counters, power counters, + // and general metadata; the exact set of files is Cray-generation specific, so every file below + // /sys/cray/pm_counters is picked up dynamically instead of relying on hardcoded names. All three categories + // are re-read every tick (including metadata like "freshness", which HPE's own consistency check relies on + // changing - see cray_pm_counters_general_samples). + // + + this->add_time_point(std::chrono::steady_clock::now()); + + for (const std::filesystem::path &file : detail::list_pm_counter_files()) { + const std::string key = detail::pm_counter_key(file); + + if (detail::is_energy_counter_key(key)) { + energy_counter_paths_.emplace(key, file); + } else if (detail::is_power_counter_key(key)) { + power_counter_paths_.emplace(key, file); + } else { + general_paths_.emplace(key, file); + } + } + + this->sample_once(); + + // pm_counters itself only updates at raw_scan_hz (10 Hz on Hunter); polling faster than that only yields + // duplicate values (and needlessly hammers sysfs), so never poll faster than the hardware's own rate - even if + // the globally configured sampling interval is shorter. Read dynamically rather than hardcoding 10 Hz, since + // raw_scan_hz can differ across Cray/HPE generations. + const std::optional hw_tick_period = this->hardware_tick_period(); + const std::chrono::milliseconds effective_interval = hw_tick_period.has_value() ? std::max(this->sampling_interval(), hw_tick_period.value()) : this->sampling_interval(); + if (hw_tick_period.has_value() && effective_interval > this->sampling_interval()) { + this->add_event(fmt::format("cray_pm_counters: throttled sampling interval from {} to {} to match the hardware's raw_scan_hz", this->sampling_interval(), effective_interval)); + } + + // + // loop until stop_sampling() is called + // + + while (!this->has_sampling_stopped()) { + // only sample values if the sampler currently isn't paused + if (this->is_sampling()) { + // add current time point + this->add_time_point(std::chrono::steady_clock::now()); + + this->sample_once(); + } + + // wait for the (possibly hardware-throttled) sampling interval to pass to retrieve the next sample + std::this_thread::sleep_for(effective_interval); + } +} + +std::string cray_pm_counters_hardware_sampler::device_identification() const { + return "cray_pm_counters_device"; +} + +std::string cray_pm_counters_hardware_sampler::samples_only_as_yaml_string() const { + // check whether it's safe to generate the YAML entry + if (this->is_sampling()) { + throw std::runtime_error{ "Can't create the final YAML entry if the hardware sampler is still running!" }; + } + + return fmt::format("{}{}" + "{}", + general_samples_.generate_yaml_string(), + general_samples_.has_samples() ? "\n" : "", + power_samples_.generate_yaml_string()); +} + +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_hardware_sampler &sampler) { + if (sampler.is_sampling()) { + out.setstate(std::ios_base::failbit); + return out; + } else { + return out << fmt::format("sampling interval: {}\n" + "time points: [{}]\n\n" + "general samples:\n{}\n\n" + "power samples:\n{}", + sampler.sampling_interval(), + fmt::join(detail::time_points_to_epoch(sampler.sampling_time_points()), ", "), + sampler.general_samples(), + sampler.power_samples()); + } +} + +} // namespace hws diff --git a/src/hws/cray_pm_counters/pm_counters_samples.cpp b/src/hws/cray_pm_counters/pm_counters_samples.cpp new file mode 100644 index 0000000..486ce86 --- /dev/null +++ b/src/hws/cray_pm_counters/pm_counters_samples.cpp @@ -0,0 +1,141 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + */ + +#include "hws/cray_pm_counters/pm_counters_samples.hpp" + +#include "hws/cray_pm_counters/utility.hpp" // hws::detail::pm_counters_root +#include "hws/utility.hpp" // hws::detail::map_entry_to_string + +#include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join + +#include // std::sort +#include // std::uint64_t +#include // std::optional +#include // std::ostream +#include // std::string +#include // std::unordered_map +#include // std::vector + +namespace hws { + +namespace { + +// unordered_map has no defined iteration order; sort keys so the generated YAML is reproducible across runs. +template +[[nodiscard]] std::vector sorted_keys(const MapType &map) { + std::vector keys{}; + keys.reserve(map.size()); + for (const auto &[key, value] : map) { + keys.push_back(key); + } + std::sort(keys.begin(), keys.end()); + return keys; +} + +} // namespace + +//*************************************************************************************************************************************// +// general samples // +//*************************************************************************************************************************************// + +bool cray_pm_counters_general_samples::has_samples() const { + return this->metadata_.has_value(); +} + +std::string cray_pm_counters_general_samples::generate_yaml_string() const { + // if no samples are available, return an empty string + if (!this->has_samples()) { + return ""; + } + + std::string str{ "general:\n" }; + const std::string root = detail::pm_counters_root().generic_string(); + + if (this->metadata_.has_value()) { + for (const std::string &key : sorted_keys(this->metadata_.value())) { + str += fmt::format(" {}:\n" + " unit: \"string\"\n" + " source: \"{}/{}\"\n" + " values: [{}]\n", + key, root, key, fmt::join(detail::quote(this->metadata_.value().at(key)), ", ")); + } + } + + return str; +} + +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_general_samples &samples) { + return out << fmt::format("metadata: {}", detail::map_entry_to_string(samples.get_metadata())); +} + +//*************************************************************************************************************************************// +// power samples // +//*************************************************************************************************************************************// + +bool cray_pm_counters_power_samples::has_samples() const { + return this->energy_counters_.has_value() || this->power_counters_.has_value(); +} + +std::string cray_pm_counters_power_samples::generate_yaml_string() const { + // if no samples are available, return an empty string + if (!this->has_samples()) { + return ""; + } + + std::string str{ "power:\n" }; + const std::string root = detail::pm_counters_root().generic_string(); + + // look up the per-sample timestamps (if any) recorded for the counter map key; nullptr if none are available + const auto find_timestamps = [](const std::optional ×tamps, const std::string &key) -> const std::vector * { + if (!timestamps.has_value()) { + return nullptr; + } + const auto it = timestamps.value().find(key); + return it != timestamps.value().cend() ? &it->second : nullptr; + }; + + if (this->energy_counters_.has_value()) { + for (const std::string &key : sorted_keys(this->energy_counters_.value())) { + str += fmt::format(" {}:\n" + " unit: \"J (cumulative)\"\n" + " source: \"{}/{}\"\n" + " values: [{}]\n", + key, root, key, fmt::join(this->energy_counters_.value().at(key), ", ")); + if (const std::vector *timestamps = find_timestamps(this->energy_timestamps_us_, key); timestamps != nullptr) { + str += fmt::format(" timestamps_us: [{}]\n", fmt::join(*timestamps, ", ")); + } + } + } + if (this->power_counters_.has_value()) { + for (const std::string &key : sorted_keys(this->power_counters_.value())) { + str += fmt::format(" {}:\n" + " unit: \"W (instantaneous)\"\n" + " source: \"{}/{}\"\n" + " values: [{}]\n", + key, root, key, fmt::join(this->power_counters_.value().at(key), ", ")); + if (const std::vector *timestamps = find_timestamps(this->power_timestamps_us_, key); timestamps != nullptr) { + str += fmt::format(" timestamps_us: [{}]\n", fmt::join(*timestamps, ", ")); + } + } + } + + return str; +} + +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_power_samples &samples) { + return out << fmt::format("energy_counters: {}\n" + "energy_timestamps_us: {}\n" + "power_counters: {}\n" + "power_timestamps_us: {}", + detail::map_entry_to_string(samples.get_energy_counters()), + detail::map_entry_to_string(samples.get_energy_timestamps_us()), + detail::map_entry_to_string(samples.get_power_counters()), + detail::map_entry_to_string(samples.get_power_timestamps_us())); +} + +} // namespace hws diff --git a/src/hws/cray_pm_counters/utility.cpp b/src/hws/cray_pm_counters/utility.cpp new file mode 100644 index 0000000..69bc2b0 --- /dev/null +++ b/src/hws/cray_pm_counters/utility.cpp @@ -0,0 +1,150 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + */ + +#include "hws/cray_pm_counters/utility.hpp" + +#include "hws/utility.hpp" // hws::detail::{trim, to_lower_case, split, is_integer, convert_to} + +#include // std::uint64_t +#include // std::getenv +#include // std::filesystem::{path, directory_entry, exists, is_directory, recursive_directory_iterator, relative} +#include // std::ifstream +#include // std::optional, std::nullopt +#include // std::ostringstream +#include // std::string +#include // std::string_view +#include // std::error_code +#include // std::vector + +namespace hws::detail { + +std::filesystem::path pm_counters_root() { + // allow overriding the pm_counters root for local testing (e.g. against a synthetic directory off a Cray system) + if (const char *override_path = std::getenv("HWS_PM_COUNTERS_PATH"); override_path != nullptr) { + return std::filesystem::path{ override_path }; + } + return std::filesystem::path{ default_pm_counters_root }; +} + +bool pm_counters_available() { + const std::filesystem::path root = pm_counters_root(); + std::error_code ec{}; + return std::filesystem::exists(root, ec) && std::filesystem::is_directory(root, ec); +} + +std::vector list_pm_counter_files() { + std::vector files{}; + + if (!pm_counters_available()) { + return files; + } + + std::error_code ec{}; + for (const std::filesystem::directory_entry &entry : std::filesystem::recursive_directory_iterator(pm_counters_root(), std::filesystem::directory_options::skip_permission_denied, ec)) { + if (entry.is_regular_file(ec)) { + files.push_back(entry.path()); + } + } + + return files; +} + +std::string pm_counter_key(const std::filesystem::path &file) { + std::error_code ec{}; + std::filesystem::path relative = std::filesystem::relative(file, pm_counters_root(), ec); + if (ec) { + relative = file.filename(); + } + + // '/' has no special meaning in YAML mapping keys, so a nested file's relative path (e.g. "accel0/energy") is + // used as-is; this also means the key always doubles as the real path relative to pm_counters_root() for the + // "source:" field in the generated YAML. All confirmed real HPE pm_counters layouts (2014 CUG slides, 2024 CUG + // slides, a real Hunter node) are flat (no subdirectories), so this rarely matters in practice. + return relative.generic_string(); +} + +std::string read_pm_counter_raw(const std::filesystem::path &file) { + std::ifstream stream{ file }; + if (!stream.is_open()) { + return ""; + } + + std::ostringstream content{}; + content << stream.rdbuf(); + return std::string{ detail::trim(content.str()) }; +} + +std::optional read_pm_counter_reading(const std::filesystem::path &file) { + const std::string content = read_pm_counter_raw(file); + if (content.empty()) { + return std::nullopt; + } + + const std::vector tokens = detail::split(content, ' '); + if (tokens.empty()) { + return std::nullopt; + } + + pm_counter_reading reading{}; + try { + reading.value = detail::convert_to(tokens.front()); + } catch (const std::exception &) { + return std::nullopt; + } + + // PM counters version 3 appends a microsecond timestamp as "... us" to telemetry (but not cap) files + if (tokens.size() >= 2 && tokens.back() == "us" && detail::is_integer(tokens[tokens.size() - 2])) { + try { + reading.timestamp_us = detail::convert_to(tokens[tokens.size() - 2]); + } catch (const std::exception &) { + // ignore an unparsable timestamp, the value itself was already parsed successfully + } + } + + return reading; +} + +namespace { + +// Check whether lower_case_key's last path component (i.e. after the last '_' or '/', or the whole key if it has +// neither) is exactly component - e.g. "accel0_energy" and "energy" both match component "energy", but +// "power_state" or "energy_source" don't match component "power"/"energy" respectively. Anchoring on the last +// component (rather than a plain substring search) avoids misclassifying a future metadata field that merely +// contains "energy"/"power" as a substring without actually being a measured energy/power counter. +[[nodiscard]] bool last_component_is(const std::string &lower_case_key, const std::string &component) { + if (lower_case_key == component) { + return true; + } + const std::size_t suffix_size = component.size() + 1; // +1 for the separator + if (lower_case_key.size() <= suffix_size) { + return false; + } + const char separator = lower_case_key[lower_case_key.size() - suffix_size]; + return (separator == '_' || separator == '/') && lower_case_key.compare(lower_case_key.size() - component.size(), component.size(), component) == 0; +} + +// "power_cap"/"accel[i]_power_cap" are configured limits (not measured draw), and "capped_energy"/"overshoot"/ +// "overshoot_energy" are derived/event counters (confirmed on a real Hunter node) - none of them belong in the +// same map as the actual measured energy/power telemetry this backend exists to compare against hws's own +// software-side measurements, so they're excluded here and fall through to general metadata instead. +[[nodiscard]] bool is_excluded_from_telemetry(const std::string &lower_case_key) { + return lower_case_key.find("cap") != std::string::npos || lower_case_key.find("overshoot") != std::string::npos; +} + +} // namespace + +bool is_energy_counter_key(const std::string &key) { + const std::string lower_case_key = detail::to_lower_case(key); + return last_component_is(lower_case_key, "energy") && !is_excluded_from_telemetry(lower_case_key); +} + +bool is_power_counter_key(const std::string &key) { + const std::string lower_case_key = detail::to_lower_case(key); + return last_component_is(lower_case_key, "power") && !is_excluded_from_telemetry(lower_case_key); +} + +} // namespace hws::detail diff --git a/src/hws/system_hardware_sampler.cpp b/src/hws/system_hardware_sampler.cpp index 4dff468..26aa7f5 100644 --- a/src/hws/system_hardware_sampler.cpp +++ b/src/hws/system_hardware_sampler.cpp @@ -13,6 +13,10 @@ #if defined(HWS_FOR_CPUS_ENABLED) #include "hws/cpu/hardware_sampler.hpp" // hws::cpu_hardware_sampler #endif +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + #include "hws/cray_pm_counters/hardware_sampler.hpp" // hws::cray_pm_counters_hardware_sampler + #include "hws/cray_pm_counters/utility.hpp" // hws::detail::pm_counters_available +#endif #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) #include "hws/gpu_nvidia/hardware_sampler.hpp" // hws::gpu_nvidia_hardware_sampler #include "hws/gpu_nvidia/utility.hpp" // HWS_CUDA_ERROR_CHECK @@ -67,46 +71,53 @@ system_hardware_sampler::system_hardware_sampler(MPI_Comm communicator, const de // create a custom, node-local MPI communicator detail::hostname_comm_info nc{ communicator }; - // CPU: one sampler per node --> node leader only - #if defined(HWS_FOR_CPUS_ENABLED) - if (nc.node_rank == 0) { - samplers_.push_back(std::make_unique(sampling_interval, category)); - } - #endif - - // NVIDIA - #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) - { - const std::vector local = detail::enumerate_local_nvidia_devices(); - const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); - for (int const idx : owned) { - samplers_.push_back(std::make_unique(static_cast(idx), sampling_interval, category)); + // CPU: one sampler per node --> node leader only + #if defined(HWS_FOR_CPUS_ENABLED) + if (nc.node_rank == 0) { + samplers_.push_back(std::make_unique(sampling_interval, category)); } - } - #endif - - // AMD - #if defined(HWS_FOR_AMD_GPUS_ENABLED) - { - const std::vector local = detail::enumerate_local_amd_devices(); - const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); - for (int const idx : owned) { - samplers_.push_back(std::make_unique( - static_cast(idx), sampling_interval, category)); + #endif + + // Cray pm_counters: one sampler per node --> node leader only + #if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + if (nc.node_rank == 0 && detail::pm_counters_available()) { + samplers_.push_back(std::make_unique(sampling_interval, category)); } - } - #endif - - // Intel - #if defined(HWS_FOR_INTEL_GPUS_ENABLED) - { - const std::vector local = detail::enumerate_local_intel_devices(); - const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); - for (int const idx : owned) { - samplers_.push_back(std::make_unique(static_cast(idx), sampling_interval, category)); + #endif + + // NVIDIA + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + { + const std::vector local = detail::enumerate_local_nvidia_devices(); + const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); + for (int const idx : owned) { + samplers_.push_back(std::make_unique(static_cast(idx), sampling_interval, category)); + } } - } - #endif + #endif + + // AMD + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + { + const std::vector local = detail::enumerate_local_amd_devices(); + const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); + for (int const idx : owned) { + samplers_.push_back(std::make_unique( + static_cast(idx), sampling_interval, category)); + } + } + #endif + + // Intel + #if defined(HWS_FOR_INTEL_GPUS_ENABLED) + { + const std::vector local = detail::enumerate_local_intel_devices(); + const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); + for (int const idx : owned) { + samplers_.push_back(std::make_unique(static_cast(idx), sampling_interval, category)); + } + } + #endif } else { throw std::runtime_error{ fmt::format("Unknown MPI sampling mode {}!", static_cast(mode)) }; @@ -285,6 +296,11 @@ void system_hardware_sampler::create_local_samplers(std::chrono::milliseconds sa samplers_.push_back(std::make_unique(sampling_interval, category)); } #endif +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + if (detail::pm_counters_available()) { + samplers_.push_back(std::make_unique(sampling_interval, category)); + } +#endif #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) { int device_count{}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..4ee2c57 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,9 @@ +## Authors: Alexander Van Craen +## Copyright (C): 2024-today All Rights Reserved +## License: This file is released under the MIT license. +## See the LICENSE.md file in the project root for full license information. +######################################################################################################################## + +## one subdirectory per backend, mirroring the include/hws// and src/hws// layout; each +## subdirectory's CMakeLists.txt is responsible for only registering tests if its backend was actually compiled in +add_subdirectory(cray_pm_counters) diff --git a/tests/cray_pm_counters/CMakeLists.txt b/tests/cray_pm_counters/CMakeLists.txt new file mode 100644 index 0000000..af9eccd --- /dev/null +++ b/tests/cray_pm_counters/CMakeLists.txt @@ -0,0 +1,15 @@ +## Authors: Alexander Van Craen +## Copyright (C): 2024-today All Rights Reserved +## License: This file is released under the MIT license. +## See the LICENSE.md file in the project root for full license information. +######################################################################################################################## + +# only build this backend's tests if it was actually compiled into ${HWS_LIBRARY_NAME} +get_target_property(HWS_COMPILE_DEFINITIONS ${HWS_LIBRARY_NAME} COMPILE_DEFINITIONS) + +if ("HWS_FOR_CRAY_PM_COUNTERS_ENABLED" IN_LIST HWS_COMPILE_DEFINITIONS) + add_executable(hws_test_cray_pm_counters test.cpp) + target_link_libraries(hws_test_cray_pm_counters PRIVATE ${HWS_LIBRARY_NAME}) + add_test(NAME cray_pm_counters COMMAND hws_test_cray_pm_counters) + set_tests_properties(cray_pm_counters PROPERTIES TIMEOUT 30) +endif () diff --git a/tests/cray_pm_counters/test.cpp b/tests/cray_pm_counters/test.cpp new file mode 100644 index 0000000..4855792 --- /dev/null +++ b/tests/cray_pm_counters/test.cpp @@ -0,0 +1,218 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + * + * @brief Simple, dependency-free regression checks for the cray_pm_counters backend. + * @details Uses the HWS_PM_COUNTERS_PATH environment variable override to point the backend at synthetic + * directories instead of the real (Cray-only) /sys/cray/pm_counters, so these checks run on any Linux + * machine. The synthetic content mirrors a real dump gathered on an HLRS "Hunter" node (HPE Cray EX255a, + * PM counters version 3, see scripts/hwmon_probe.pbs). Not a general-purpose test framework - just enough to + * catch a regression in the specific bugs found (and fixed) during code review. + */ + +#include "hws/core.hpp" + +#include // std::chrono::milliseconds +#include // std::exit, setenv +#include // std::filesystem +#include // std::ofstream +#include // std::cout, std::cerr +#include // std::string +#include // std::this_thread::sleep_for + +namespace { + +int g_failures = 0; + +void check(bool condition, const std::string &description) { + if (condition) { + std::cout << " ok: " << description << "\n"; + } else { + std::cerr << " FAILED: " << description << "\n"; + ++g_failures; + } +} + +void write_file(const std::filesystem::path &path, const std::string &content) { + std::filesystem::create_directories(path.parent_path()); + std::ofstream file{ path }; + file << content; +} + +void set_pm_counters_path(const std::filesystem::path &path) { + setenv("HWS_PM_COUNTERS_PATH", path.c_str(), 1); +} + +// mirrors the real dump gathered on Hunter (hwmon_probe.o250927): 14 of the 23 real files, enough to exercise +// every classification/parsing rule without duplicating the full dump +std::filesystem::path make_synthetic_pm_counters(const std::filesystem::path &root) { + std::filesystem::remove_all(root); + write_file(root / "energy", "12044825101 J 7810213855886 us\n"); + write_file(root / "power", "724 W 7810213855886 us\n"); + write_file(root / "power_cap", "2894 W\n"); + write_file(root / "accel0_energy", "2869283299 J 7810213855886 us\n"); + write_file(root / "accel0_power", "212 W 7810213855886 us\n"); + write_file(root / "accel0_power_cap", "0 W\n"); + write_file(root / "capped_energy", "12752953 J 7810213855886 us\n"); + write_file(root / "overshoot", "62408\n"); + write_file(root / "overshoot_energy", "3951201 J 7810213855886 us\n"); + write_file(root / "freshness", "60043284\n"); + write_file(root / "generation", "16\n"); + write_file(root / "raw_scan_hz", "10\n"); + write_file(root / "startup", "1778580757634187\n"); + write_file(root / "version", "3\n"); + return root; +} + +void test_absent() { + std::cout << "test_absent\n"; + set_pm_counters_path("/nonexistent/pm_counters_for_hws_test"); + + hws::cray_pm_counters_hardware_sampler sampler{}; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + sampler.stop_sampling(); + + check(!sampler.general_samples().has_samples(), "no general samples when pm_counters is absent"); + check(!sampler.power_samples().has_samples(), "no power samples when pm_counters is absent"); +} + +void test_real_values(const std::filesystem::path &root) { + std::cout << "test_real_values\n"; + set_pm_counters_path(root); + + hws::cray_pm_counters_hardware_sampler sampler{}; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + sampler.stop_sampling(); + + const auto &energy = sampler.power_samples().get_energy_counters(); + const auto &power = sampler.power_samples().get_power_counters(); + const auto &energy_ts = sampler.power_samples().get_energy_timestamps_us(); + const auto &metadata = sampler.general_samples().get_metadata(); + + check(energy.has_value() && energy->count("energy") == 1 && energy->at("energy").front() == 12044825101ull, + "'energy' discovered and parsed as a measured energy counter"); + check(energy.has_value() && energy->count("accel0_energy") == 1, + "'accel0_energy' discovered as a measured energy counter"); + check(power.has_value() && power->count("power") == 1 && power->at("power").front() == 724ull, + "'power' discovered and parsed as a measured power counter"); + check(energy_ts.has_value() && energy_ts->count("energy") == 1 && energy_ts->at("energy").front() == 7810213855886ull, + "'energy's embedded HSS timestamp extracted correctly"); + + // caps and derived/event counters must not pollute the measured telemetry maps (regression: Opus review finding) + check(!energy.has_value() || energy->count("capped_energy") == 0, "'capped_energy' excluded from energy_counters"); + check(!energy.has_value() || energy->count("overshoot_energy") == 0, "'overshoot_energy' excluded from energy_counters"); + check(!power.has_value() || power->count("power_cap") == 0, "'power_cap' excluded from power_counters"); + check(!power.has_value() || power->count("accel0_power_cap") == 0, "'accel0_power_cap' excluded from power_counters"); + + // ... they should show up as general metadata instead + check(metadata.has_value() && metadata->count("power_cap") == 1, "'power_cap' present in general metadata"); + check(metadata.has_value() && metadata->count("capped_energy") == 1, "'capped_energy' present in general metadata"); + check(metadata.has_value() && metadata->count("overshoot") == 1, "'overshoot' present in general metadata"); + // freshness must be re-sampled every tick, not read once (regression check) + check(metadata.has_value() && metadata->count("freshness") == 1 && metadata->at("freshness").size() > 1, + "'freshness' is re-sampled every tick, not just once"); +} + +void test_adversarial_names(const std::filesystem::path &root) { + std::cout << "test_adversarial_names\n"; + write_file(root / "power_state", "ok\n"); + write_file(root / "energy_source", "grid\n"); + set_pm_counters_path(root); + + hws::cray_pm_counters_hardware_sampler sampler{}; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + sampler.stop_sampling(); + + const auto &energy = sampler.power_samples().get_energy_counters(); + const auto &power = sampler.power_samples().get_power_counters(); + const auto &metadata = sampler.general_samples().get_metadata(); + + // regression check (GPT-5.5 review finding): "energy"/"power" must be a path-segment suffix, not any substring + check(!energy.has_value() || energy->count("energy_source") == 0, "'energy_source' not misclassified as an energy counter"); + check(!power.has_value() || power->count("power_state") == 0, "'power_state' not misclassified as a power counter"); + check(metadata.has_value() && metadata->count("power_state") == 1, "'power_state' present in general metadata"); + check(metadata.has_value() && metadata->count("energy_source") == 1, "'energy_source' present in general metadata"); +} + +void test_nested_path(const std::filesystem::path &root) { + std::cout << "test_nested_path\n"; + write_file(root / "nested" / "energy", "111 J 999 us\n"); + set_pm_counters_path(root); + + hws::cray_pm_counters_hardware_sampler sampler{}; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + sampler.stop_sampling(); + + const auto &energy = sampler.power_samples().get_energy_counters(); + // regression check (Opus review finding): nested files use their real relative path as key, not a flattened one + check(energy.has_value() && energy->count("nested/energy") == 1, "nested file uses its un-flattened relative path as map key"); +} + +void test_throttle(const std::filesystem::path &root) { + std::cout << "test_throttle\n"; + set_pm_counters_path(root); + + // request a 10ms interval, far faster than the synthetic hardware's 10 Hz (raw_scan_hz=10); the sampler must + // throttle to the hardware rate instead of hammering the (synthetic) sysfs files + hws::cray_pm_counters_hardware_sampler sampler{ std::chrono::milliseconds(10) }; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(650)); + sampler.stop_sampling(); + + bool throttled = false; + for (const auto &e : sampler.get_events()) { + if (e.name.find("throttled") != std::string::npos) { + throttled = true; + } + } + check(throttled, "a throttling event is recorded for a too-fast configured interval"); + + const auto &energy = sampler.power_samples().get_energy_counters(); + const std::size_t num_samples = (energy.has_value() && energy->count("energy") == 1) ? energy->at("energy").size() : 0; + // ~650ms / 100ms (hardware period) ~= 6-8 samples; a buggy 10ms cadence would produce ~65 + check(num_samples > 0 && num_samples < 20, "sample count matches the hardware-throttled cadence, not the requested 10ms one"); +} + +void test_throttle_power_only_category(const std::filesystem::path &root) { + std::cout << "test_throttle_power_only_category\n"; + set_pm_counters_path(root); + + // regression check (GPT-5.5 review finding): throttling must not depend on sample_category::general being enabled + hws::cray_pm_counters_hardware_sampler sampler{ std::chrono::milliseconds(10), hws::sample_category::power }; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(650)); + sampler.stop_sampling(); + + const auto &energy = sampler.power_samples().get_energy_counters(); + const std::size_t num_samples = (energy.has_value() && energy->count("energy") == 1) ? energy->at("energy").size() : 0; + check(num_samples > 0 && num_samples < 20, "throttling still applies when only sample_category::power is enabled"); +} + +} // namespace + +int main() { + const std::filesystem::path root = std::filesystem::temp_directory_path() / "hws_cray_pm_counters_test"; + + test_absent(); + make_synthetic_pm_counters(root); + test_real_values(root); + test_adversarial_names(root); + test_nested_path(root); + test_throttle(root); + test_throttle_power_only_category(root); + + std::filesystem::remove_all(root); + + if (g_failures > 0) { + std::cerr << "\n" << g_failures << " check(s) failed\n"; + return 1; + } + std::cout << "\nall checks passed\n"; + return 0; +} From c19c7bcf8f5b8334d581b9916541957543f888de Mon Sep 17 00:00:00 2001 From: Alexander Van Craen Date: Tue, 11 Aug 2026 12:58:00 +0200 Subject: [PATCH 2/6] add cross-vendor accel[i] <-> GPU correlation hints to pm_counters Generalizes the pm_counters accel[i] index guess from AMD-only to also cover NVIDIA, since pm_counters exposes the same accel[i] naming convention on Grace-Hopper (EX254n) nodes. Extracts the vendor-specific YAML formatting into a shared, vendor-agnostic hws::detail::accel_correlation_yaml_block() so AMD and NVIDIA share one code path in system_hardware_sampler's device_correlation_hints. Both gpu_amd_hardware_sampler and gpu_nvidia_hardware_sampler gained a pci_bus_id() that queries the same API/handle each sampler already uses for its actual measurements (ROCm SMI's rsmi_dev_pci_id_get(device_id_), NVML's nvmlDeviceGetPciInfo_v3() on the already-resolved handle) instead of a different API family (HIP/CUDA's own PCI bus ID calls). Those differ from ROCm SMI's/NVML's device enumeration under HIP_VISIBLE_DEVICES/CUDA_VISIBLE_DEVICES, which could otherwise silently attribute a correlation guess to the wrong physical device. --- .../hws/cray_pm_counters/hardware_sampler.hpp | 10 +++ include/hws/cray_pm_counters/utility.hpp | 59 ++++++++++++ include/hws/gpu_amd/hardware_sampler.hpp | 22 +++++ include/hws/gpu_amd/utility.hpp | 25 +++++- include/hws/gpu_nvidia/hardware_sampler.hpp | 25 ++++++ include/hws/gpu_nvidia/utility.hpp | 28 +++++- include/hws/system_hardware_sampler.hpp | 15 ++++ include/hws/utility.hpp | 17 ++++ src/hws/cray_pm_counters/hardware_sampler.cpp | 14 ++- src/hws/cray_pm_counters/utility.cpp | 89 +++++++++++++++++++ src/hws/gpu_amd/hardware_sampler.cpp | 15 ++++ src/hws/gpu_amd/utility.cpp | 49 ++++++++-- src/hws/gpu_nvidia/hardware_sampler.cpp | 11 ++- src/hws/gpu_nvidia/utility.cpp | 52 +++++++++-- src/hws/system_hardware_sampler.cpp | 88 +++++++++++++++++- src/hws/utility.cpp | 7 ++ tests/cray_pm_counters/test.cpp | 60 +++++++++++++ 17 files changed, 564 insertions(+), 22 deletions(-) diff --git a/include/hws/cray_pm_counters/hardware_sampler.hpp b/include/hws/cray_pm_counters/hardware_sampler.hpp index 75a2fe0..fdf603f 100644 --- a/include/hws/cray_pm_counters/hardware_sampler.hpp +++ b/include/hws/cray_pm_counters/hardware_sampler.hpp @@ -24,6 +24,7 @@ #include // std::ostream forward declaration #include // std::optional #include // std::unordered_map +#include // std::vector namespace hws { @@ -86,6 +87,15 @@ class cray_pm_counters_hardware_sampler : public hardware_sampler { */ [[nodiscard]] const cray_pm_counters_power_samples &power_samples() const noexcept { return power_samples_; } + /** + * @brief The accelerator indices pm_counters actually exposed on this node (e.g. `{0, 1, 2, 3}` for a 4-APU + * node), derived from the already-sampled `accel_energy` counter keys. + * @details Only meaningful after at least one sample has been taken (i.e. during or after sampling); returns + * an empty vector beforehand. + * @return the sorted accelerator indices (`[[nodiscard]]`) + */ + [[nodiscard]] std::vector discovered_accel_indices() const; + /** * @copydoc hws::hardware_sampler::device_identification */ diff --git a/include/hws/cray_pm_counters/utility.hpp b/include/hws/cray_pm_counters/utility.hpp index e999a5b..1bb6be7 100644 --- a/include/hws/cray_pm_counters/utility.hpp +++ b/include/hws/cray_pm_counters/utility.hpp @@ -12,11 +12,13 @@ #define HWS_CRAY_PM_COUNTERS_UTILITY_HPP_ #pragma once +#include // std::size_t #include // std::uint64_t #include // std::filesystem::path #include // std::optional #include // std::string #include // std::string_view +#include // std::pair #include // std::vector namespace hws::detail { @@ -97,6 +99,63 @@ struct pm_counter_reading { */ [[nodiscard]] bool is_power_counter_key(const std::string &key); +/** + * @brief Extract the accelerator index from an energy or power counter map key, e.g. `"accel2_energy"` -> `2`. + * @param[in] key the map key to check, as returned by `pm_counter_key` + * @return the accelerator index, or `std::nullopt` if @p key isn't of the form `accel_energy`/`accel_power` + * (`[[nodiscard]]`) + */ +[[nodiscard]] std::optional accel_index_from_counter_key(const std::string &key); + +/** + * @brief Extract the sorted, deduplicated set of accelerator indices referenced by @p counter_keys. + * @details Intended to be called with the keys of an already-sampled `cray_pm_counters_power_samples`'s + * `energy_counters`/`power_counters` map, to answer "how many `accel[i]` counters did pm_counters + * actually expose on this node" without re-scanning the filesystem. + * @param[in] counter_keys the energy or power counter map keys to scan, as returned by `pm_counter_key` + * @return the sorted accelerator indices found (`[[nodiscard]]`) + */ +[[nodiscard]] std::vector accel_indices_from_counter_keys(const std::vector &counter_keys); + +/** + * @brief Guess which pm_counters `accel[i]` a given AMD GPU (identified by its PCI bus ID) corresponds to. + * @details UNVERIFIED heuristic: assumes `accel[i]` numbers accelerators in ascending PCI bus address order among + * all physically present AMD GPUs - no HPE documentation defines this correspondence, so this should be + * confirmed empirically (e.g. drive load on a single visible GPU and observe which `accel[i]_power` + * reacts) before being relied on for analysis. See `hws::system_hardware_sampler::device_correlation_hints_as_yaml_string()`, + * the only caller, for how this is surfaced to users. + * @param[in] pci_bus_id the PCI bus ID of the AMD GPU to guess an accel index for + * @param[in] physical_pci_bus_ids_sorted every physically present AMD GPU's PCI bus ID, sorted ascending (as + * returned by `hws::detail::enumerate_all_amd_gpu_pci_bus_ids()`) + * @param[in] accel_indices_sorted the accelerator indices pm_counters exposed, sorted ascending (as returned by + * `cray_pm_counters_hardware_sampler::discovered_accel_indices()`) + * @return the guessed accel index, or `std::nullopt` if @p pci_bus_id isn't found in @p physical_pci_bus_ids_sorted + * or the two lists don't have the same size (a topology count mismatch means the guess isn't safe to make, + * e.g. under a cgroup-isolated partial-node allocation) (`[[nodiscard]]`) + */ +[[nodiscard]] std::optional guess_accel_index(const std::string &pci_bus_id, + const std::vector &physical_pci_bus_ids_sorted, + const std::vector &accel_indices_sorted); + +/** + * @brief Build the YAML sub-block (under a `gpu_vendors:` mapping) describing one GPU vendor's accel[i] + * correlation guesses, e.g. for `"amd"` or `"nvidia"`. + * @details Vendor-agnostic: the caller is responsible for gathering @p visible_devices and + * @p physical_pci_bus_ids_sorted using the right backend (`hws::detail::amd_device_pci_bus_id()`/ + * `enumerate_all_amd_gpu_pci_bus_ids()` or their `nvidia_*` counterparts). Every entry in + * @p physical_pci_bus_ids_sorted and @p accel_indices_sorted is assumed to belong to @p vendor alone - + * don't call this with a mixed-vendor physical topology. + * @param[in] vendor the vendor name to use as the YAML mapping key (e.g. `"amd"`, `"nvidia"`) + * @param[in] visible_devices every visible GPU sampler of this vendor, as (local device index, PCI bus ID) pairs + * @param[in] physical_pci_bus_ids_sorted every physically present GPU of this vendor's PCI bus ID, sorted ascending + * @param[in] accel_indices_sorted the accelerator indices pm_counters exposed, sorted ascending + * @return the YAML sub-block, indented to sit directly under a ` gpu_vendors:\n` key (`[[nodiscard]]`) + */ +[[nodiscard]] std::string accel_correlation_yaml_block(const std::string &vendor, + const std::vector> &visible_devices, + const std::vector &physical_pci_bus_ids_sorted, + const std::vector &accel_indices_sorted); + } // namespace hws::detail #endif // HWS_CRAY_PM_COUNTERS_UTILITY_HPP_ diff --git a/include/hws/gpu_amd/hardware_sampler.hpp b/include/hws/gpu_amd/hardware_sampler.hpp index 668cc9a..7ce1d42 100644 --- a/include/hws/gpu_amd/hardware_sampler.hpp +++ b/include/hws/gpu_amd/hardware_sampler.hpp @@ -116,6 +116,28 @@ class gpu_amd_hardware_sampler : public hardware_sampler { */ [[nodiscard]] const rocm_smi_temperature_samples &temperature_samples() const noexcept { return temperature_samples_; } + /** + * @brief Return the ROCm SMI device index this hardware sampler was constructed with and uses for all of its + * `rsmi_dev_*` calls. + * @details Purely local/informational - since `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` can affect which + * physical device ends up at a given index, don't assume this index is stable across processes or + * reruns; use `pci_bus_id()` to identify the actual physical device. + * @return the ROCm SMI device index (`[[nodiscard]]`) + */ + [[nodiscard]] std::uint32_t device_id() const noexcept { return device_id_; } + + /** + * @brief Return the PCI bus ID (e.g. `"0000:c1:00.0"`) of the physical device this hardware sampler actually + * measures. + * @details Queried via `rsmi_dev_pci_id_get()` using the same `device_id()` index this sampler already uses + * for every other `rsmi_dev_*` call, so - unlike combining `device_id()` with a different API family + * (e.g. HIP's `hipDeviceGetPCIBusId()`) - this is guaranteed to identify the exact physical device + * being sampled, even if `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` causes ROCm SMI's and HIP's + * device enumerations to diverge. Matches the format used by `enumerate_all_amd_gpu_pci_bus_ids()`. + * @return the PCI bus ID (`[[nodiscard]]`) + */ + [[nodiscard]] std::string pci_bus_id() const; + /** * @copydoc hws::hardware_sampler::device_identification */ diff --git a/include/hws/gpu_amd/utility.hpp b/include/hws/gpu_amd/utility.hpp index def0937..c0670a0 100644 --- a/include/hws/gpu_amd/utility.hpp +++ b/include/hws/gpu_amd/utility.hpp @@ -17,11 +17,10 @@ #include // std::runtime_error #include // std::string +#include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) #include "hws/visible_gpu_device.hpp" // hws::detail::visible_gpu_device - - #include // std::vector #endif namespace hws::detail { @@ -74,6 +73,28 @@ namespace hws::detail { */ [[nodiscard]] std::string performance_level_to_string(rsmi_dev_perf_level_t perf_level); +/** + * @brief Return the PCI bus ID (e.g. `"0000:c1:00.0"`) of the AMD GPU device with the given HIP @p local_index. + * @details This is the same, stable identifier used by `enumerate_all_amd_gpu_pci_bus_ids()`, so the two can be + * matched against each other to locate a HIP-visible device within the full physical GPU topology. + * @param[in] local_index the local HIP device index + * @return the PCI bus ID (`[[nodiscard]]`) + */ +[[nodiscard]] std::string amd_device_pci_bus_id(int local_index); + +/** + * @brief Enumerate the PCI bus IDs of every AMD GPU physically present on the node, independent of any + * process-level device visibility filtering (e.g. `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES`). + * @details Reads the kernel's view of devices bound to the `amdgpu` driver directly from + * `/sys/bus/pci/drivers/amdgpu/`, since that sysfs directory - unlike the HIP runtime's device + * enumeration - isn't affected by per-process visible-device environment variables. It *can* still be + * restricted below the true physical device count in a batch job with kernel-level (cgroup) device + * isolation for a partial-node allocation, so callers must treat an unexpectedly low count as "topology + * unknown", not as ground truth. + * @return the sorted PCI bus IDs of all `amdgpu`-bound devices, or an empty vector if the directory doesn't exist + * or isn't readable (`[[nodiscard]]`) + */ +[[nodiscard]] std::vector enumerate_all_amd_gpu_pci_bus_ids(); #if defined(HWS_MPI_SUPPORT_ENABLED) diff --git a/include/hws/gpu_nvidia/hardware_sampler.hpp b/include/hws/gpu_nvidia/hardware_sampler.hpp index 59a5e31..aa14fce 100644 --- a/include/hws/gpu_nvidia/hardware_sampler.hpp +++ b/include/hws/gpu_nvidia/hardware_sampler.hpp @@ -117,6 +117,29 @@ class gpu_nvidia_hardware_sampler : public hardware_sampler { */ [[nodiscard]] const nvml_temperature_samples &temperature_samples() const noexcept { return temperature_samples_; } + /** + * @brief Return the CUDA device index this hardware sampler was constructed with. + * @details Purely local/informational: this index is only used once, at construction, to resolve the NVML + * device handle this sampler actually operates on (see `nvmlDeviceGetHandleByIndex`) - since + * `CUDA_VISIBLE_DEVICES` isn't respected by NVML's own device enumeration, this CUDA-relative index + * is **not** guaranteed to identify the same physical device as `hws::detail::nvidia_device_pci_bus_id(device_id())` + * would. Use `pci_bus_id()` to identify the actual physical device this sampler measures. + * @return the CUDA device index (`[[nodiscard]]`) + */ + [[nodiscard]] std::size_t device_id() const noexcept { return device_id_; } + + /** + * @brief Return the PCI bus ID (e.g. `"0000:c1:00.0"`) of the physical device this hardware sampler actually + * measures. + * @details Queried via `nvmlDeviceGetPciInfo_v3()` on this sampler's already-resolved NVML device handle, so - + * unlike combining `device_id()` with a different API family (e.g. CUDA's `cudaDeviceGetPCIBusId()`) + * - this is guaranteed to identify the exact physical device being sampled, even though + * `CUDA_VISIBLE_DEVICES` and NVML's device enumeration can otherwise diverge. Matches the format + * used by `enumerate_all_nvidia_gpu_pci_bus_ids()`. + * @return the PCI bus ID (`[[nodiscard]]`) + */ + [[nodiscard]] std::string pci_bus_id() const; + /** * @copydoc hws::hardware_sampler::device_identification */ @@ -135,6 +158,8 @@ class gpu_nvidia_hardware_sampler : public hardware_sampler { /// The device handle for the device to sample. detail::nvml_device_handle device_{}; + /// The CUDA device index this hardware sampler was constructed with. + std::size_t device_id_{}; /// The general NVIDIA GPU samples. nvml_general_samples general_samples_{}; diff --git a/include/hws/gpu_nvidia/utility.hpp b/include/hws/gpu_nvidia/utility.hpp index b0b3811..170d98e 100644 --- a/include/hws/gpu_nvidia/utility.hpp +++ b/include/hws/gpu_nvidia/utility.hpp @@ -18,11 +18,10 @@ #include // std::runtime_error #include // std::string +#include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) #include "hws/visible_gpu_device.hpp" // hws::detail::visible_gpu_device - - #include // std::vector #endif namespace hws::detail { @@ -69,6 +68,31 @@ namespace hws::detail { #endif +/** + * @brief Return the PCI bus ID (e.g. `"0000:c1:00.0"`) of the NVIDIA GPU device with the given CUDA @p local_index. + * @details This is the same, stable identifier used by `enumerate_all_nvidia_gpu_pci_bus_ids()`, so the two can be + * matched against each other to locate a CUDA-visible device within the full physical GPU topology. + * @param[in] local_index the local CUDA device index + * @return the PCI bus ID (`[[nodiscard]]`) + */ +[[nodiscard]] std::string nvidia_device_pci_bus_id(int local_index); + +/** + * @brief Enumerate the PCI bus IDs of every NVIDIA GPU physically present on the node, independent of any + * process-level device visibility filtering (e.g. `CUDA_VISIBLE_DEVICES`). + * @details Reads the kernel's view of devices bound to the `nvidia` driver directly from + * `/sys/bus/pci/drivers/nvidia/`, since that sysfs directory - unlike the CUDA runtime's device + * enumeration - isn't affected by per-process visible-device environment variables. It *can* still be + * restricted below the true physical device count in a batch job with kernel-level (cgroup) device + * isolation for a partial-node allocation, so callers must treat an unexpectedly low count as "topology + * unknown", not as ground truth. Untested against real hardware (no NVIDIA GH200/EX254n access) - the + * `amdgpu` counterpart this mirrors (`hws::detail::enumerate_all_amd_gpu_pci_bus_ids()`) is; double-check + * the `nvidia` driver directory name and symlink layout on first use. + * @return the sorted PCI bus IDs of all `nvidia`-bound devices, or an empty vector if the directory doesn't exist + * or isn't readable (`[[nodiscard]]`) + */ +[[nodiscard]] std::vector enumerate_all_nvidia_gpu_pci_bus_ids(); + #if defined(HWS_MPI_SUPPORT_ENABLED) /** diff --git a/include/hws/system_hardware_sampler.hpp b/include/hws/system_hardware_sampler.hpp index 21b3029..875b4eb 100644 --- a/include/hws/system_hardware_sampler.hpp +++ b/include/hws/system_hardware_sampler.hpp @@ -251,6 +251,21 @@ class system_hardware_sampler { * @param category the sample category */ void create_local_samplers(std::chrono::milliseconds sampling_interval, hws::sample_category category); + + /** + * @brief Generate a best-effort, UNVERIFIED YAML hint correlating each visible AMD GPU device with a Cray + * pm_counters `accel[i]` index, if `samplers_` holds both a `cray_pm_counters_hardware_sampler` and at + * least one `gpu_amd_hardware_sampler`. + * @details The generated YAML's own `note`/`verified` fields carry the caveat: this is an ordinal guess + * (`accel[i]` is assumed to number accelerators in ascending PCI bus address order among all + * physically present AMD GPUs), not a vendor-confirmed mapping - no HPE documentation defines this + * correspondence. Degrades gracefully (marks `topology_count_mismatch: true`, omits the guesses but + * still reports the raw PCI bus IDs) if the discovered `accel` count doesn't match the physical AMD + * GPU count, e.g. under a cgroup-isolated partial-node allocation. + * @return the YAML string, or an empty string if the prerequisites aren't met (backend(s) not compiled in, or + * no pm_counters/AMD GPU sampler present) (`[[nodiscard]]`) + */ + [[nodiscard]] std::string device_correlation_hints_as_yaml_string() const; }; } // namespace hws diff --git a/include/hws/utility.hpp b/include/hws/utility.hpp index 2737418..921f29e 100644 --- a/include/hws/utility.hpp +++ b/include/hws/utility.hpp @@ -19,6 +19,7 @@ #include // std::chrono::duration #include // std::trunc #include // std::size_t +#include // std::uint32_t #include // std::optional #include // std::runtime_error #include // std::string, std::stof, std::stod, std::stold @@ -255,6 +256,22 @@ template */ [[nodiscard]] std::string indent_lines(const std::string &text, std::string_view prefix); +/** + * @brief Format a PCI domain/bus/device triplet as the canonical Linux sysfs PCI bus ID string + * `"::.0"` (e.g. `"0000:c1:00.0"`), lowercase hex, 4/2/2 digits. + * @details The function is fixed to `0`: on multi-die/multi-partition accelerators (e.g. AMD MI300-series + * "partitions") the PCI function field is repurposed by the vendor's management library for + * partition/die identification, but the function seen by the OS/sysfs for the *device* itself is + * always `0` - so vendor-provided domain/bus/device values should be combined with a hardcoded `0` + * function here rather than a vendor-reported function value, to stay comparable with sysfs PCI bus IDs + * (see e.g. `hws::detail::enumerate_all_amd_gpu_pci_bus_ids()`/`enumerate_all_nvidia_gpu_pci_bus_ids()`). + * @param[in] domain the PCI domain + * @param[in] bus the PCI bus number + * @param[in] device the PCI device (slot) number + * @return the formatted PCI bus ID string (`[[nodiscard]]`) + */ +[[nodiscard]] std::string format_pci_bus_id(std::uint32_t domain, std::uint32_t bus, std::uint32_t device); + /*****************************************************************************************************/ /** other free functions **/ /*****************************************************************************************************/ diff --git a/src/hws/cray_pm_counters/hardware_sampler.cpp b/src/hws/cray_pm_counters/hardware_sampler.cpp index 433c38b..036fd2e 100644 --- a/src/hws/cray_pm_counters/hardware_sampler.cpp +++ b/src/hws/cray_pm_counters/hardware_sampler.cpp @@ -8,7 +8,7 @@ #include "hws/cray_pm_counters/hardware_sampler.hpp" #include "hws/cray_pm_counters/pm_counters_samples.hpp" // hws::{cray_pm_counters_general_samples, cray_pm_counters_power_samples} -#include "hws/cray_pm_counters/utility.hpp" // hws::detail::{pm_counters_available, list_pm_counter_files, pm_counter_key, read_pm_counter_raw, read_pm_counter_reading, pm_counter_reading, is_energy_counter_key, is_power_counter_key} +#include "hws/cray_pm_counters/utility.hpp" // hws::detail::{pm_counters_available, list_pm_counter_files, pm_counter_key, read_pm_counter_raw, read_pm_counter_reading, pm_counter_reading, is_energy_counter_key, is_power_counter_key, accel_indices_from_counter_keys} #include "hws/hardware_sampler.hpp" // hws::hardware_sampler #include "hws/sample_category.hpp" // hws::sample_category #include "hws/utility.hpp" // hws::detail::time_points_to_epoch @@ -28,6 +28,7 @@ #include // std::runtime_error #include // std::string #include // std::this_thread +#include // std::vector namespace hws { @@ -171,6 +172,17 @@ std::string cray_pm_counters_hardware_sampler::device_identification() const { return "cray_pm_counters_device"; } +std::vector cray_pm_counters_hardware_sampler::discovered_accel_indices() const { + if (!power_samples_.get_energy_counters().has_value()) { + return {}; + } + std::vector keys{}; + for (const auto &entry : power_samples_.get_energy_counters().value()) { + keys.push_back(entry.first); + } + return detail::accel_indices_from_counter_keys(keys); +} + std::string cray_pm_counters_hardware_sampler::samples_only_as_yaml_string() const { // check whether it's safe to generate the YAML entry if (this->is_sampling()) { diff --git a/src/hws/cray_pm_counters/utility.cpp b/src/hws/cray_pm_counters/utility.cpp index 69bc2b0..9d0d732 100644 --- a/src/hws/cray_pm_counters/utility.cpp +++ b/src/hws/cray_pm_counters/utility.cpp @@ -9,6 +9,11 @@ #include "hws/utility.hpp" // hws::detail::{trim, to_lower_case, split, is_integer, convert_to} +#include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join + +#include // std::sort, std::unique +#include // std::size_t #include // std::uint64_t #include // std::getenv #include // std::filesystem::{path, directory_entry, exists, is_directory, recursive_directory_iterator, relative} @@ -18,6 +23,7 @@ #include // std::string #include // std::string_view #include // std::error_code +#include // std::pair #include // std::vector namespace hws::detail { @@ -147,4 +153,87 @@ bool is_power_counter_key(const std::string &key) { return last_component_is(lower_case_key, "power") && !is_excluded_from_telemetry(lower_case_key); } +std::optional accel_index_from_counter_key(const std::string &key) { + const std::string lower_case_key = detail::to_lower_case(key); + constexpr std::string_view prefix = "accel"; + constexpr std::string_view suffixes[] = { "_energy", "_power" }; + + if (lower_case_key.compare(0, prefix.size(), prefix) != 0) { + return std::nullopt; + } + + for (const std::string_view suffix : suffixes) { + if (lower_case_key.size() <= prefix.size() + suffix.size()) { + continue; + } + if (lower_case_key.compare(lower_case_key.size() - suffix.size(), suffix.size(), suffix) != 0) { + continue; + } + const std::string digits = lower_case_key.substr(prefix.size(), lower_case_key.size() - prefix.size() - suffix.size()); + if (!digits.empty() && detail::is_integer(digits)) { + try { + return detail::convert_to(digits); + } catch (const std::exception &) { + return std::nullopt; + } + } + } + return std::nullopt; +} + +std::vector accel_indices_from_counter_keys(const std::vector &counter_keys) { + std::vector indices{}; + for (const std::string &key : counter_keys) { + if (const std::optional idx = accel_index_from_counter_key(key); idx.has_value()) { + indices.push_back(*idx); + } + } + std::sort(indices.begin(), indices.end()); + indices.erase(std::unique(indices.begin(), indices.end()), indices.end()); + return indices; +} + +std::optional guess_accel_index(const std::string &pci_bus_id, + const std::vector &physical_pci_bus_ids_sorted, + const std::vector &accel_indices_sorted) { + if (physical_pci_bus_ids_sorted.empty() || physical_pci_bus_ids_sorted.size() != accel_indices_sorted.size()) { + return std::nullopt; + } + const auto it = std::find(physical_pci_bus_ids_sorted.cbegin(), physical_pci_bus_ids_sorted.cend(), pci_bus_id); + if (it == physical_pci_bus_ids_sorted.cend()) { + return std::nullopt; + } + const auto position = static_cast(std::distance(physical_pci_bus_ids_sorted.cbegin(), it)); + return accel_indices_sorted[position]; +} + +std::string accel_correlation_yaml_block(const std::string &vendor, + const std::vector> &visible_devices, + const std::vector &physical_pci_bus_ids_sorted, + const std::vector &accel_indices_sorted) { + const bool topology_matches = !physical_pci_bus_ids_sorted.empty() && physical_pci_bus_ids_sorted.size() == accel_indices_sorted.size(); + + std::vector visible_entries{}; + for (const auto &[local_index, pci_bus_id] : visible_devices) { + const std::optional guessed = guess_accel_index(pci_bus_id, physical_pci_bus_ids_sorted, accel_indices_sorted); + const std::string guessed_str = guessed.has_value() ? std::to_string(*guessed) : "null"; + visible_entries.push_back(fmt::format(" - local_index: {}\n" + " pci_bus_id: \"{}\"\n" + " guessed_accel_index: {}", + local_index, pci_bus_id, guessed_str)); + } + + std::vector quoted_physical_ids{}; + for (const std::string &id : physical_pci_bus_ids_sorted) { + quoted_physical_ids.push_back(fmt::format("\"{}\"", id)); + } + + return fmt::format(" {}:\n" + " topology_count_mismatch: {}\n" + " physical_pci_bus_ids: [{}]\n" + " visible_gpus:\n" + "{}\n", + vendor, !topology_matches, fmt::join(quoted_physical_ids, ", "), fmt::join(visible_entries, "\n")); +} + } // namespace hws::detail diff --git a/src/hws/gpu_amd/hardware_sampler.cpp b/src/hws/gpu_amd/hardware_sampler.cpp index 7c0a6a2..cbe3b9f 100644 --- a/src/hws/gpu_amd/hardware_sampler.cpp +++ b/src/hws/gpu_amd/hardware_sampler.cpp @@ -681,6 +681,21 @@ std::string gpu_amd_hardware_sampler::device_identification() const { return fmt::format("gpu_amd_device_{}", device_id_); } +std::string gpu_amd_hardware_sampler::pci_bus_id() const { + std::uint64_t bdfid{}; + HWS_ROCM_SMI_ERROR_CHECK(rsmi_dev_pci_id_get(device_id_, &bdfid)) + + // BDFID = (DOMAIN << 32) | (PARTITION << 28) | (BUS << 8) | (DEVICE << 3) | FUNCTION (see ROCm SMI's + // rsmi_dev_pci_id_get documentation). On MI-series partitioned devices the function bits are repurposed for + // the partition ID instead of a real PCI function - but the OS/sysfs-visible PCI address for the device + // itself always has function 0, so the function is intentionally not extracted here; see + // hws::detail::format_pci_bus_id(). + const auto domain = static_cast((bdfid >> 32) & 0xffffffffull); + const auto bus = static_cast((bdfid >> 8) & 0xffull); + const auto device = static_cast((bdfid >> 3) & 0x1full); + return detail::format_pci_bus_id(domain, bus, device); +} + std::string gpu_amd_hardware_sampler::samples_only_as_yaml_string() const { // check whether it's safe to generate the YAML entry if (this->is_sampling()) { diff --git a/src/hws/gpu_amd/utility.cpp b/src/hws/gpu_amd/utility.cpp index 55d6932..a3dedf5 100644 --- a/src/hws/gpu_amd/utility.cpp +++ b/src/hws/gpu_amd/utility.cpp @@ -9,13 +9,16 @@ #include "rocm_smi/rocm_smi.h" // ROCm SMI runtime functions -#include // std::string -#include // std::vector +#include "hip/hip_runtime_api.h" // hipGetDeviceCount, hipDeviceGetPCIBusId + +#include // std::sort +#include // std::filesystem::{directory_iterator, exists, directory_options} +#include // std::string +#include // std::error_code +#include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) && defined(HWS_FOR_AMD_GPUS_ENABLED) #include "hws/visible_gpu_device.hpp" // hws::detail::visible_gpu_device, hws::detail::device_backend_kind - - #include "hip/hip_runtime_api.h" // hipGetDeviceCount, hipDeviceGetPCIBusId #endif namespace hws::detail { @@ -46,6 +49,40 @@ std::string performance_level_to_string(const rsmi_dev_perf_level_t perf_level) } } +std::string amd_device_pci_bus_id(const int local_index) { + char bus_id[64] = {}; + HWS_HIP_ERROR_CHECK(hipDeviceGetPCIBusId(bus_id, sizeof(bus_id), local_index)); + return std::string{ bus_id }; +} + +std::vector enumerate_all_amd_gpu_pci_bus_ids() { + std::vector bus_ids{}; + + const std::filesystem::path amdgpu_driver_dir{ "/sys/bus/pci/drivers/amdgpu" }; + std::error_code ec{}; + if (!std::filesystem::exists(amdgpu_driver_dir, ec) || ec) { + return bus_ids; + } + + for (const std::filesystem::directory_entry &entry : std::filesystem::directory_iterator(amdgpu_driver_dir, std::filesystem::directory_options::skip_permission_denied, ec)) { + if (ec) { + break; + } + // every PCI device bound to the amdgpu driver shows up here as a symlink named after its PCI bus ID, + // e.g. "0000:c1:00.0" -> ../../../devices/.../0000:c1:00.0 + if (!entry.is_symlink(ec)) { + continue; + } + const std::string name = entry.path().filename().string(); + if (name.find(':') != std::string::npos && name.find('.') != std::string::npos) { + bus_ids.push_back(name); + } + } + + std::sort(bus_ids.begin(), bus_ids.end()); + return bus_ids; +} + #if defined(HWS_MPI_SUPPORT_ENABLED) && defined(HWS_FOR_AMD_GPUS_ENABLED) namespace { @@ -58,9 +95,7 @@ namespace { * @return the physical ID of the AMD GPU device */ [[nodiscard]] std::string amd_physical_id(const int local_index) { - char bus_id[64] = {}; - HWS_HIP_ERROR_CHECK(hipDeviceGetPCIBusId(bus_id, sizeof(bus_id), local_index)); - return std::string{ "amd:" } + bus_id; + return std::string{ "amd:" } + amd_device_pci_bus_id(local_index); } } // namespace diff --git a/src/hws/gpu_nvidia/hardware_sampler.cpp b/src/hws/gpu_nvidia/hardware_sampler.cpp index f3c6f53..85be367 100644 --- a/src/hws/gpu_nvidia/hardware_sampler.cpp +++ b/src/hws/gpu_nvidia/hardware_sampler.cpp @@ -45,7 +45,7 @@ gpu_nvidia_hardware_sampler::gpu_nvidia_hardware_sampler(const std::chrono::mill gpu_nvidia_hardware_sampler{ 0, sampling_interval, category } { } gpu_nvidia_hardware_sampler::gpu_nvidia_hardware_sampler(const std::size_t device_id, const std::chrono::milliseconds sampling_interval, const sample_category category) : - hardware_sampler{ sampling_interval, category } { + hardware_sampler{ sampling_interval, category }, device_id_{ device_id } { // make sure that nvmlInit is only called once for all instances if (instances_++ == 0) { HWS_NVML_ERROR_CHECK(nvmlInit()) @@ -568,6 +568,15 @@ std::string gpu_nvidia_hardware_sampler::device_identification() const { return fmt::format("gpu_nvidia_device_{}_{}", pcie_info.device, pcie_info.bus); } +std::string gpu_nvidia_hardware_sampler::pci_bus_id() const { + nvmlPciInfo_st pcie_info{}; + HWS_NVML_ERROR_CHECK(nvmlDeviceGetPciInfo_v3(device_.get_impl().device, &pcie_info)) + // deliberately formatted from the numeric domain/bus/device fields, not pcie_info.busId - NVML's own busId + // string uses an extended 8-digit domain (e.g. "00000000:C1:00.0") that doesn't match the 4-digit lowercase + // sysfs convention used by enumerate_all_nvidia_gpu_pci_bus_ids(); see hws::detail::format_pci_bus_id(). + return detail::format_pci_bus_id(pcie_info.domain, pcie_info.bus, pcie_info.device); +} + std::string gpu_nvidia_hardware_sampler::samples_only_as_yaml_string() const { // check whether it's safe to generate the YAML entry if (this->is_sampling()) { diff --git a/src/hws/gpu_nvidia/utility.cpp b/src/hws/gpu_nvidia/utility.cpp index 97d8c1e..41a2c37 100644 --- a/src/hws/gpu_nvidia/utility.cpp +++ b/src/hws/gpu_nvidia/utility.cpp @@ -7,12 +7,16 @@ #include "hws/gpu_nvidia/utility.hpp" -#include "fmt/format.h" // fmt::format -#include "fmt/ranges.h" // fmt::join -#include "nvml.h" // NVML runtime functions +#include "cuda_runtime_api.h" // cudaGetDeviceCount, cudaDeviceGetPCIBusId +#include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join +#include "nvml.h" // NVML runtime functions -#include // std::string -#include // std::vector +#include // std::sort +#include // std::filesystem::{directory_iterator, exists, directory_options} +#include // std::string +#include // std::error_code +#include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) && defined(HWS_FOR_NVIDIA_GPUS_ENABLED) #include "hws/visible_gpu_device.hpp" // hws::detail::visible_gpu_device, hws::detail::device_backend_kind @@ -60,6 +64,40 @@ std::string throttle_event_reason_to_string(const unsigned long long clocks_even #endif +std::string nvidia_device_pci_bus_id(const int local_index) { + char bus_id[64] = {}; + HWS_CUDA_ERROR_CHECK(cudaDeviceGetPCIBusId(bus_id, sizeof(bus_id), local_index)); + return std::string{ bus_id }; +} + +std::vector enumerate_all_nvidia_gpu_pci_bus_ids() { + std::vector bus_ids{}; + + const std::filesystem::path nvidia_driver_dir{ "/sys/bus/pci/drivers/nvidia" }; + std::error_code ec{}; + if (!std::filesystem::exists(nvidia_driver_dir, ec) || ec) { + return bus_ids; + } + + for (const std::filesystem::directory_entry &entry : std::filesystem::directory_iterator(nvidia_driver_dir, std::filesystem::directory_options::skip_permission_denied, ec)) { + if (ec) { + break; + } + // every PCI device bound to the nvidia driver shows up here as a symlink named after its PCI bus ID, + // e.g. "0000:c1:00.0" -> ../../../devices/.../0000:c1:00.0 + if (!entry.is_symlink(ec)) { + continue; + } + const std::string name = entry.path().filename().string(); + if (name.find(':') != std::string::npos && name.find('.') != std::string::npos) { + bus_ids.push_back(name); + } + } + + std::sort(bus_ids.begin(), bus_ids.end()); + return bus_ids; +} + #if defined(HWS_MPI_SUPPORT_ENABLED) && defined(HWS_FOR_NVIDIA_GPUS_ENABLED) namespace { @@ -72,9 +110,7 @@ namespace { * @return the physical ID of the NVIDIA GPU device */ [[nodiscard]] std::string nvidia_physical_id(const int local_index) { - char bus_id[64] = {}; - HWS_CUDA_ERROR_CHECK(cudaDeviceGetPCIBusId(bus_id, sizeof(bus_id), local_index)); - return std::string{ "nvidia:" } + bus_id; + return std::string{ "nvidia:" } + nvidia_device_pci_bus_id(local_index); } } // namespace diff --git a/src/hws/system_hardware_sampler.cpp b/src/hws/system_hardware_sampler.cpp index 26aa7f5..9273e49 100644 --- a/src/hws/system_hardware_sampler.cpp +++ b/src/hws/system_hardware_sampler.cpp @@ -35,6 +35,7 @@ #endif #include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join #include // std::for_each, std::all_of #include // std::chrono::milliseconds @@ -42,7 +43,10 @@ #include // std::uint32_t #include // std::unique_ptr, std::make_unique #include // std::accumulate +#include // std::optional #include // std::out_of_range +#include // std::string, std::to_string +#include // std::pair #include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) @@ -283,13 +287,95 @@ void system_hardware_sampler::dump_yaml_global(const std::filesystem::path &file #endif std::string system_hardware_sampler::as_yaml_string() const { - return std::accumulate(samplers_.cbegin(), samplers_.cend(), std::string{}, [](const std::string str, const auto &ptr) { return str + ptr->as_yaml_string(); }); + return std::accumulate(samplers_.cbegin(), samplers_.cend(), std::string{}, [](const std::string str, const auto &ptr) { return str + ptr->as_yaml_string(); }) + + this->device_correlation_hints_as_yaml_string(); } std::string system_hardware_sampler::samples_only_as_yaml_string() const { return std::accumulate(samplers_.cbegin(), samplers_.cend(), std::string{}, [](const std::string str, const auto &ptr) { return str + ptr->samples_only_as_yaml_string(); }); } +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) && (defined(HWS_FOR_AMD_GPUS_ENABLED) || defined(HWS_FOR_NVIDIA_GPUS_ENABLED)) +std::string system_hardware_sampler::device_correlation_hints_as_yaml_string() const { + const cray_pm_counters_hardware_sampler *pm_sampler = nullptr; + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + std::vector> amd_devices{}; + #endif + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + std::vector> nvidia_devices{}; + #endif + for (const std::unique_ptr &ptr : samplers_) { + if (const auto *pm = dynamic_cast(ptr.get()); pm != nullptr) { + pm_sampler = pm; + continue; + } + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + if (const auto *amd = dynamic_cast(ptr.get()); amd != nullptr) { + // pci_bus_id() (ROCm SMI, the same API family used for all of this sampler's actual measurements) is + // used here rather than HIP's own hipDeviceGetPCIBusId(), since ROCm SMI's and HIP's device + // enumerations can diverge under HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES - using a different API + // family than the one the sampler measures with could silently attribute the wrong PCI bus ID. + amd_devices.emplace_back(amd->device_id(), amd->pci_bus_id()); + continue; + } + #endif + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + if (const auto *nvidia = dynamic_cast(ptr.get()); nvidia != nullptr) { + // pci_bus_id() (NVML, the same API family used for all of this sampler's actual measurements) is used + // here rather than CUDA's own cudaDeviceGetPCIBusId(), since NVML's device enumeration isn't affected + // by CUDA_VISIBLE_DEVICES while the CUDA runtime's is - using a different API family than the one the + // sampler measures with could silently attribute the wrong PCI bus ID. + nvidia_devices.emplace_back(nvidia->device_id(), nvidia->pci_bus_id()); + continue; + } + #endif + } + + const bool any_visible_gpus = + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + !amd_devices.empty() + #else + false + #endif + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + || !nvidia_devices.empty() + #endif + ; + if (pm_sampler == nullptr || !any_visible_gpus) { + return ""; + } + + // ground truth, no guessing involved: which accel[i] counters pm_counters exposed on this node + const std::vector accel_indices = pm_sampler->discovered_accel_indices(); + + std::string vendor_blocks{}; + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + if (!amd_devices.empty()) { + vendor_blocks += detail::accel_correlation_yaml_block("amd", amd_devices, detail::enumerate_all_amd_gpu_pci_bus_ids(), accel_indices); + } + #endif + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + if (!nvidia_devices.empty()) { + vendor_blocks += detail::accel_correlation_yaml_block("nvidia", nvidia_devices, detail::enumerate_all_nvidia_gpu_pci_bus_ids(), accel_indices); + } + #endif + + return fmt::format("device_correlation_hints:\n" + " note: \"UNVERIFIED heuristic: assumes Cray pm_counters numbers accel[i] in ascending PCI bus address order among all physically present GPUs of a given vendor; this is not confirmed by any HPE documentation. Confirm empirically (e.g. drive load on a single visible GPU and observe which accel[i]_power reacts) before relying on this for analysis.\"\n" + " verified: false\n" + " accel_indices_discovered: [{}]\n" + " gpu_vendors:\n" + "{}" + "\n", + fmt::join(accel_indices, ", "), + vendor_blocks); +} +#else +std::string system_hardware_sampler::device_correlation_hints_as_yaml_string() const { + return ""; +} +#endif + void system_hardware_sampler::create_local_samplers(std::chrono::milliseconds sampling_interval, sample_category category) { #if defined(HWS_FOR_CPUS_ENABLED) { diff --git a/src/hws/utility.cpp b/src/hws/utility.cpp index 406089a..8138ea5 100644 --- a/src/hws/utility.cpp +++ b/src/hws/utility.cpp @@ -7,8 +7,11 @@ #include "hws/utility.hpp" +#include "fmt/format.h" // fmt::format + #include // std::min, std::transform, std::all_of #include // std::tolower, std::isdigit +#include // std::uint32_t #include // std::stringstream #include // std::string #include // std::string_view @@ -77,4 +80,8 @@ std::string indent_lines(const std::string &text, const std::string_view prefix) return out; } +std::string format_pci_bus_id(const std::uint32_t domain, const std::uint32_t bus, const std::uint32_t device) { + return fmt::format("{:04x}:{:02x}:{:02x}.0", domain, bus, device); +} + } // namespace hws::detail diff --git a/tests/cray_pm_counters/test.cpp b/tests/cray_pm_counters/test.cpp index 4855792..987d397 100644 --- a/tests/cray_pm_counters/test.cpp +++ b/tests/cray_pm_counters/test.cpp @@ -13,14 +13,18 @@ */ #include "hws/core.hpp" +#include "hws/cray_pm_counters/utility.hpp" // hws::detail::{accel_index_from_counter_key, accel_indices_from_counter_keys, guess_accel_index, accel_correlation_yaml_block} #include // std::chrono::milliseconds #include // std::exit, setenv #include // std::filesystem #include // std::ofstream #include // std::cout, std::cerr +#include // std::optional #include // std::string #include // std::this_thread::sleep_for +#include // std::pair +#include // std::vector namespace { @@ -194,6 +198,60 @@ void test_throttle_power_only_category(const std::filesystem::path &root) { check(num_samples > 0 && num_samples < 20, "throttling still applies when only sample_category::power is enabled"); } +// pure-logic checks for the accel[i] <-> AMD GPU PCI bus ID correlation guess (see +// hws::system_hardware_sampler::device_correlation_hints_as_yaml_string(), the only caller in the actual library) +// - no ROCm SMI/HIP or real GPU required, since these only deal with plain strings/ints. +void test_accel_correlation_guess() { + std::cout << "test_accel_correlation_guess\n"; + + check(hws::detail::accel_index_from_counter_key("accel0_energy") == std::optional{ 0 }, "'accel0_energy' -> accel index 0"); + check(hws::detail::accel_index_from_counter_key("accel3_power") == std::optional{ 3 }, "'accel3_power' -> accel index 3"); + check(hws::detail::accel_index_from_counter_key("accel12_energy") == std::optional{ 12 }, "multi-digit accel index parsed correctly"); + check(!hws::detail::accel_index_from_counter_key("energy").has_value(), "plain 'energy' (node-wide) has no accel index"); + check(!hws::detail::accel_index_from_counter_key("accel0_power_cap").has_value(), "'accel0_power_cap' has no accel index (not energy/power telemetry)"); + check(!hws::detail::accel_index_from_counter_key("accelerator0_energy").has_value(), "'accelerator0_energy' doesn't match the 'accel_energy' pattern"); + + const std::vector indices = hws::detail::accel_indices_from_counter_keys({ "energy", "accel1_energy", "accel0_energy", "accel1_power", "accel3_energy", "capped_energy" }); + check((indices == std::vector{ 0, 1, 3 }), "accel indices extracted, sorted, and deduplicated across energy+power keys"); + + const std::vector physical{ "0000:0c:00.0", "0000:22:00.0", "0000:c1:00.0", "0000:e2:00.0" }; + const std::vector accel{ 0, 1, 2, 3 }; + + check(hws::detail::guess_accel_index("0000:0c:00.0", physical, accel) == std::optional{ 0 }, "lowest PCI bus id guessed as accel index 0"); + check(hws::detail::guess_accel_index("0000:e2:00.0", physical, accel) == std::optional{ 3 }, "highest PCI bus id guessed as accel index 3"); + check(!hws::detail::guess_accel_index("0000:ff:00.0", physical, accel).has_value(), "unknown PCI bus id yields no guess"); + + // topology count mismatch (e.g. cgroup-isolated partial-node allocation exposing fewer GPUs than pm_counters + // reports accel[i] for) must degrade to "no guess", not a wrong one + const std::vector partial_physical{ "0000:0c:00.0", "0000:22:00.0" }; + check(!hws::detail::guess_accel_index("0000:0c:00.0", partial_physical, accel).has_value(), "topology count mismatch yields no guess, even for a bus id that IS present"); + check(!hws::detail::guess_accel_index("anything", {}, {}).has_value(), "empty physical topology yields no guess"); +} + +// vendor-agnostic YAML formatting for the correlation hints (used for both AMD and NVIDIA, see +// hws::system_hardware_sampler::device_correlation_hints_as_yaml_string()) - also no GPU hardware required. +void test_accel_correlation_yaml_block() { + std::cout << "test_accel_correlation_yaml_block\n"; + + const std::vector accel{ 0, 1 }; + const std::vector physical{ "0000:0c:00.0", "0000:22:00.0" }; + const std::vector> visible{ { 0, "0000:22:00.0" } }; + + const std::string block = hws::detail::accel_correlation_yaml_block("amd", visible, physical, accel); + check(block.rfind(" amd:\n", 0) == 0, "block starts with the vendor key at 4-space indent"); + check(block.find("topology_count_mismatch: false") != std::string::npos, "matching topology reports no mismatch"); + check(block.find("physical_pci_bus_ids: [\"0000:0c:00.0\", \"0000:22:00.0\"]") != std::string::npos, "physical PCI bus ids listed in sorted order"); + check(block.find("local_index: 0") != std::string::npos, "visible device's local index present"); + check(block.find("pci_bus_id: \"0000:22:00.0\"") != std::string::npos, "visible device's PCI bus id present"); + check(block.find("guessed_accel_index: 1") != std::string::npos, "higher (2nd) physical bus id guessed as accel index 1"); + + // topology count mismatch (e.g. no physical GPUs discovered at all) must degrade to "no guess" + const std::string mismatch_block = hws::detail::accel_correlation_yaml_block("nvidia", visible, {}, accel); + check(mismatch_block.rfind(" nvidia:\n", 0) == 0, "vendor key reflects the passed-in vendor name"); + check(mismatch_block.find("topology_count_mismatch: true") != std::string::npos, "empty physical topology reports a mismatch"); + check(mismatch_block.find("guessed_accel_index: null") != std::string::npos, "no guess made under a topology mismatch"); +} + } // namespace int main() { @@ -206,6 +264,8 @@ int main() { test_nested_path(root); test_throttle(root); test_throttle_power_only_category(root); + test_accel_correlation_guess(); + test_accel_correlation_yaml_block(); std::filesystem::remove_all(root); From 92a41047db356b78e5b5c3f069027a4f6d727c9f Mon Sep 17 00:00:00 2001 From: Alexander Van Craen Date: Wed, 12 Aug 2026 12:49:50 +0200 Subject: [PATCH 3/6] fix system_hardware_sampler::dump_yaml to include correlation hints dump_yaml()/dump_yaml_global() wrote each sampler's own YAML document directly instead of going through as_yaml_string(), so device_correlation_hints_as_yaml_string() never made it into a dumped YAML file even though as_yaml_string() included it correctly. Found by running the actual example program on a real Hunter mi300a node - the correlation section was silently missing from track.yaml. dump_yaml(std::string)/dump_yaml(path) now forward to dump_yaml(const char*) instead of duplicating the sampler loop. --- src/hws/system_hardware_sampler.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/hws/system_hardware_sampler.cpp b/src/hws/system_hardware_sampler.cpp index 9273e49..da81006 100644 --- a/src/hws/system_hardware_sampler.cpp +++ b/src/hws/system_hardware_sampler.cpp @@ -41,6 +41,7 @@ #include // std::chrono::milliseconds #include // std::size_t #include // std::uint32_t +#include // std::ofstream #include // std::unique_ptr, std::make_unique #include // std::accumulate #include // std::optional @@ -234,14 +235,24 @@ const std::unique_ptr &system_hardware_sampler::sampler(const void system_hardware_sampler::dump_yaml(const char *filename) const { std::for_each(samplers_.cbegin(), samplers_.cend(), [&filename](const auto &ptr) { ptr->dump_yaml(filename); }); + + // each individual sampler already wrote its own "---"-separated YAML document above; the correlation hints + // aren't tied to any single sampler, so they get one more such document of their own, if there's anything to + // report (see device_correlation_hints_as_yaml_string()). + const std::string hints = this->device_correlation_hints_as_yaml_string(); + if (!hints.empty()) { + std::ofstream file{ filename, std::ios_base::app }; + file << "---\n\n" + << hints; + } } void system_hardware_sampler::dump_yaml(const std::string &filename) const { - std::for_each(samplers_.cbegin(), samplers_.cend(), [&filename](const auto &ptr) { ptr->dump_yaml(filename); }); + this->dump_yaml(filename.c_str()); } void system_hardware_sampler::dump_yaml(const std::filesystem::path &filename) const { - std::for_each(samplers_.cbegin(), samplers_.cend(), [&filename](const auto &ptr) { ptr->dump_yaml(filename); }); + this->dump_yaml(filename.string().c_str()); } #if defined(HWS_MPI_SUPPORT_ENABLED) @@ -269,6 +280,10 @@ void system_hardware_sampler::dump_yaml_global(const char *filename, MPI_Comm co rank_yaml_output += detail::indent_lines(ptr->as_yaml_string(), " "); }); + // not tied to any single sampler, so appended directly rather than under a "sampler_N:" key; empty if there's + // nothing to report (see device_correlation_hints_as_yaml_string()) + rank_yaml_output += this->device_correlation_hints_as_yaml_string(); + const std::string global_yaml_output = detail::gather_yaml_strings_mpi(rank_yaml_output, communicator); if (rank == 0) { From 1884d0317e7e844487feb074fa952032b5d42ff8 Mon Sep 17 00:00:00 2001 From: Alexander Van Craen Date: Wed, 12 Aug 2026 13:47:38 +0200 Subject: [PATCH 4/6] fix turbostat subprocess deadlock and unreliable short sampling intervals The CPU turbostat backend could deadlock: run_subprocess() never combined the child's stdout/stderr, so an unread stderr pipe could fill up and block the child forever, wedging subprocess_join() (reproduced via gdb/wchan on two architectures). Fixed by passing subprocess_option_combined_stdout_stderr. That fix has a side effect: turbostat's own diagnostic messages (e.g. "Disabling Low Power Idle CPU output") now land in the same stream hws parses, which used to assume the header/value pair were always the first two lines. Parsing now takes the *last* two lines instead, since turbostat always prints diagnostics before its one-shot table, never after. Also fixed: turbostat's -i interval was hardcoded to 0.001s (1ms), which empirically (four real machines, full-core load) causes physically implausible readings (multi-kW power, multi-GHz clocks) because turbostat's own internal timing calculation degrades under load at short intervals. This is now the configurable HWS_TURBOSTAT_INTERVAL CMake option; the default stays at 0.001 for backwards compatibility with existing sudoers allowlists and paper-reproducibility setups, but a longer interval (e.g. 1s) is documented as the reliable choice. Related changes: - warn at runtime if sampling_interval is shorter than HWS_TURBOSTAT_INTERVAL - sleep_until() instead of sleep_for() so a slow turbostat call doesn't also get a full extra sampling_interval tacked on afterward - only invoke turbostat if an enabled sample_category actually needs it --- README.md | 20 ++++++++-- include/hws/cpu/hardware_sampler.hpp | 5 +++ src/hws/cpu/CMakeLists.txt | 25 ++++++++++-- src/hws/cpu/hardware_sampler.cpp | 58 +++++++++++++++++++++------- src/hws/cpu/utility.cpp | 7 +++- 5 files changed, 93 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1bd62df..9a50996 100644 --- a/README.md +++ b/README.md @@ -35,14 +35,14 @@ To download the hardware sampling use: ```bash git clone git@github.com:SC-SGS/hardware_sampling.git -cd hardware_sampling +cd hardware_sampling ``` Building the library can be done using the normal CMake approach: ```bash -mkdir build && cd build -cmake -DCMAKE_BUILD_TYPE=Release [optional_options] .. +mkdir build && cd build +cmake -DCMAKE_BUILD_TYPE=Release [optional_options] .. cmake --build . -j ``` @@ -73,6 +73,18 @@ The `[optional_options]` can be one or multiple of: - `HWS_ENABLE_ERROR_CHECKS=ON|OFF` (default: `OFF`): enable sanity checks during hardware sampling, may be problematic with smaller sample intervals - `HWS_SAMPLING_INTERVAL=100ms` (default: `100ms`): set the sampling interval in milliseconds + +- `HWS_TURBOSTAT_INTERVAL=1` (default: `0.001`, kept for backwards compatibility): set the interval in + seconds (`"sec.subsec"`) that `turbostat` itself measures over for a single CPU power/frequency sample + (passed directly to `turbostat`'s own `-i` flag). + **Important:** very short values have been observed to produce + physically implausible readings (multi-kW package power, multi-GHz core clocks) on heavily loaded, + many-core machines, especially with older `turbostat` builds. + **Note:** if this value is larger than `HWS_SAMPLING_INTERVAL`, the `turbostat` backend dominates and + the achieved CPU sampling cadence will be closer to `HWS_TURBOSTAT_INTERVAL` than to + `HWS_SAMPLING_INTERVAL` -- a warning is printed at runtime (once per `cpu_hardware_sampler` instance) + if this is the case. + - `HWS_ENABLE_PYTHON_BINDINGS=ON|OFF` (default: `ON`): enable Python bindings - `HWS_ENABLE_MPI_SUPPORT=ON|OFF|AUTO` (default: `AUTO`): @@ -227,7 +239,7 @@ current clock frequencies, temperatures, or memory consumption. | sample | sample type | CPUs | NVIDIA GPUs | AMD GPUs | Intel GPUs | |:------------------------|:-----------:|:----:|:-----------:|:--------:|:----------:| | num_fans | fixed | - | int | int | int | -| fan_speed_min | fixed | - | % | - | - | +| fan_speed_min | fixed | - | % | - | - | | fan_speed_max | fixed | - | % | RPM | RPM | | temperature_min | fixed | - | - | °C | - | | temperature_max | fixed | - | °C | °C | °C | diff --git a/include/hws/cpu/hardware_sampler.hpp b/include/hws/cpu/hardware_sampler.hpp index d1b4102..18f76bb 100644 --- a/include/hws/cpu/hardware_sampler.hpp +++ b/include/hws/cpu/hardware_sampler.hpp @@ -40,6 +40,11 @@ class cpu_hardware_sampler : public hardware_sampler { * @brief Construct a new CPU hardware sampler with the @p sampling_interval. * @param[in] sampling_interval the used sampling interval * @param[in] category the sample categories that are enabled for hardware sampling (default: all) + * @note If the turbostat backend is used, turbostat itself blocks for `HWS_TURBOSTAT_INTERVAL` seconds + * per invocation (see the CMake option of the same name). If that is larger than + * @p sampling_interval, the turbostat backend dominates and the achieved sampling cadence will be + * closer to `HWS_TURBOSTAT_INTERVAL` than to @p sampling_interval; a warning is printed to + * `std::cerr` in that case. */ explicit cpu_hardware_sampler(std::chrono::milliseconds sampling_interval, sample_category category = sample_category::all); diff --git a/src/hws/cpu/CMakeLists.txt b/src/hws/cpu/CMakeLists.txt index 0896902..c8b241c 100644 --- a/src/hws/cpu/CMakeLists.txt +++ b/src/hws/cpu/CMakeLists.txt @@ -22,18 +22,33 @@ if (HWS_FREE_FOUND) target_compile_definitions(${HWS_LIBRARY_NAME} PUBLIC HWS_VIA_FREE_ENABLED) endif () +## the turbostat "-i" measurement interval, in seconds +## NOTE: very short intervals (e.g. this option's default of 0.001, kept for backwards +## compatibility with the previous hardcoded value and any sudoers/paper-reproducibility setups +## that already depend on it) have been observed to produce physically implausible readings +## (multi-kW package power, multi-GHz core clocks) on heavily loaded, many-core machines, +## especially with older turbostat builds -- turbostat's own internal timing calculation degrades +## under load for short intervals. 1 second was the smallest value that stayed reliable across +## every machine/turbostat-version combination we tested; consider overriding this to "1" unless +## you specifically need to match the old default's exact (sudoers-permitted) command line. +set(HWS_TURBOSTAT_INTERVAL "0.001" CACHE STRING "The interval in seconds ('sec.subsec', passed to turbostat's -i flag) turbostat itself measures over for a single CPU power/frequency sample.") +if (NOT ${HWS_TURBOSTAT_INTERVAL} MATCHES "^[0-9]+(\\.[0-9]+)?$" OR ${HWS_TURBOSTAT_INTERVAL} EQUAL 0) + message(FATAL_ERROR "The HWS_TURBOSTAT_INTERVAL must be a positive number (e.g. \"1\" or \"0.5\"), but is \"${HWS_TURBOSTAT_INTERVAL}\"!") +endif () +message(STATUS "Setting the turbostat measurement interval to ${HWS_TURBOSTAT_INTERVAL}s.") + ## check whether turbostat could be found -> used for the CPU targets as well as for ALL host measurements ## -> checked even if no CPU targets where provided find_program(HWS_TURBOSTAT_FOUND turbostat) if (HWS_TURBOSTAT_FOUND) ## check if the turbostat command works as intended - execute_process(COMMAND sudo -n turbostat -n 1 -i 0.001 -S -q + execute_process(COMMAND sudo -n turbostat -n 1 -i ${HWS_TURBOSTAT_INTERVAL} -S -q RESULT_VARIABLE HWS_TURBOSTAT_WITH_ROOT_N OUTPUT_QUIET ERROR_QUIET) if (HWS_TURBOSTAT_WITH_ROOT_N EQUAL 0) ## can execute with root - execute_process(COMMAND sudo turbostat -n 1 -i 0.001 -S -q + execute_process(COMMAND sudo turbostat -n 1 -i ${HWS_TURBOSTAT_INTERVAL} -S -q RESULT_VARIABLE HWS_TURBOSTAT_WITH_ROOT OUTPUT_QUIET ERROR_QUIET) @@ -50,7 +65,7 @@ if (HWS_TURBOSTAT_FOUND) endif () else () ## check if turbostat can be executed without root -> potential less data - execute_process(COMMAND turbostat -n 1 -i 0.001 -S -q + execute_process(COMMAND turbostat -n 1 -i ${HWS_TURBOSTAT_INTERVAL} -S -q RESULT_VARIABLE HWS_TURBOSTAT_WITHOUT_ROOT OUTPUT_QUIET ERROR_QUIET) @@ -65,6 +80,10 @@ if (HWS_TURBOSTAT_FOUND) message(STATUS "Disabling turbostat support!") endif () endif () + + if (HWS_TURBOSTAT_EXECUTION_TYPE) + target_compile_definitions(${HWS_LIBRARY_NAME} PUBLIC HWS_TURBOSTAT_INTERVAL="${HWS_TURBOSTAT_INTERVAL}") + endif () endif () # check of any CPU related utility could be found diff --git a/src/hws/cpu/hardware_sampler.cpp b/src/hws/cpu/hardware_sampler.cpp index e7efe32..d9dfd73 100644 --- a/src/hws/cpu/hardware_sampler.cpp +++ b/src/hws/cpu/hardware_sampler.cpp @@ -39,7 +39,22 @@ cpu_hardware_sampler::cpu_hardware_sampler(const sample_category category) : cpu_hardware_sampler{ HWS_SAMPLING_INTERVAL, category } { } cpu_hardware_sampler::cpu_hardware_sampler(const std::chrono::milliseconds sampling_interval, const sample_category category) : - hardware_sampler{ sampling_interval, category } { } + hardware_sampler{ sampling_interval, category } { +#if defined(HWS_VIA_TURBOSTAT_ENABLED) + // turbostat itself blocks for HWS_TURBOSTAT_INTERVAL seconds per invocation -> if that's + // longer than the requested sampling interval, the turbostat backend will dominate and the + // achieved cadence will be closer to HWS_TURBOSTAT_INTERVAL than to sampling_interval + const double turbostat_interval_seconds = std::stod(HWS_TURBOSTAT_INTERVAL); + if (std::chrono::duration(sampling_interval).count() < turbostat_interval_seconds) { + std::cerr << fmt::format( + "Warning: the requested CPU sampling interval ({}) is shorter than turbostat's own " + "measurement interval ({}s, set via HWS_TURBOSTAT_INTERVAL) -> the effective sampling " + "cadence while the turbostat backend is active will be closer to {}s per sample.\n", + sampling_interval, turbostat_interval_seconds, turbostat_interval_seconds) + << std::endl; + } +#endif +} cpu_hardware_sampler::~cpu_hardware_sampler() { try { @@ -159,21 +174,28 @@ void cpu_hardware_sampler::sampling_loop() { // get header information #if defined(HWS_VIA_TURBOSTAT_ROOT) // run with sudo - const std::string_view turbostat_command_line = "sudo turbostat -n 1 -i 0.001 -S -q"; + const std::string turbostat_command_line = fmt::format("sudo turbostat -n 1 -i {} -S -q", HWS_TURBOSTAT_INTERVAL); #else // run without sudo - const std::string_view turbostat_command_line = "turbostat -n 1 -i 0.001 -S -q"; + const std::string turbostat_command_line = fmt::format("turbostat -n 1 -i {} -S -q", HWS_TURBOSTAT_INTERVAL); #endif - { + // turbostat feeds all of the following sample categories -> only pay its (potentially multi- + // second, see HWS_TURBOSTAT_INTERVAL) cost per invocation if at least one of them was requested + const bool turbostat_needed = this->sample_category_enabled(sample_category::general | sample_category::clock | sample_category::power | sample_category::temperature | sample_category::gfx | sample_category::idle_state); + + if (turbostat_needed) { // run turbostat const std::string turbostat_output = detail::run_subprocess(turbostat_command_line); - // retrieve the turbostat data + // retrieve the turbostat data; turbostat may prepend diagnostic messages (e.g. "Disabling + // Low Power Idle CPU output") on stderr before its actual header/value lines, and stdout + // and stderr are read through the same combined handle -> the header/value pair is always + // the *last* two lines, never necessarily the first two const std::vector data = detail::split(detail::trim(turbostat_output), '\n'); assert((data.size() >= 2) && "Must read at least two lines!"); - const std::vector header = detail::split(data[0], '\t'); - const std::vector values = detail::split(data[1], '\t'); + const std::vector header = detail::split(data[data.size() - 2], '\t'); + const std::vector values = detail::split(data[data.size() - 1], '\t'); for (std::size_t i = 0; i < header.size(); ++i) { // general samples @@ -387,6 +409,12 @@ void cpu_hardware_sampler::sampling_loop() { // while (!this->has_sampling_stopped()) { + // reference point for this tick's deadline; captured before any of the (potentially slow, + // e.g. turbostat blocking for HWS_TURBOSTAT_INTERVAL seconds) sampling work below so that + // sleep_until() only waits out whatever time is left of the requested interval instead of + // unconditionally adding another full sampling_interval() on top + const auto tick_start = std::chrono::steady_clock::now(); + // only sample values if the sampler currently isn't paused if (this->is_sampling()) { // add current time point @@ -413,15 +441,16 @@ void cpu_hardware_sampler::sampling_loop() { #endif #if defined(HWS_VIA_TURBOSTAT_ENABLED) - { + if (turbostat_needed) { // run turbostat const std::string turbostat_output = detail::run_subprocess(turbostat_command_line); - // retrieve the turbostat data + // retrieve the turbostat data; see the comment at the initial turbostat call above + // for why the header/value pair must be taken from the *last* two lines const std::vector data = detail::split(detail::trim(turbostat_output), '\n'); assert((data.size() >= 2) && "Must read at least two lines!"); - const std::vector header = detail::split(data[0], '\t'); - const std::vector values = detail::split(data[1], '\t'); + const std::vector header = detail::split(data[data.size() - 2], '\t'); + const std::vector values = detail::split(data[data.size() - 1], '\t'); // add values to the respective sample entries for (std::size_t i = 0; i < header.size(); ++i) { @@ -630,8 +659,11 @@ void cpu_hardware_sampler::sampling_loop() { #endif } - // wait for the sampling interval to pass to retrieve the next sample - std::this_thread::sleep_for(this->sampling_interval()); + // wait until this tick's deadline to retrieve the next sample; if the sampling work above + // already took longer than sampling_interval() (e.g. because the turbostat backend blocked + // for HWS_TURBOSTAT_INTERVAL seconds), the deadline is already in the past and this returns + // immediately instead of unconditionally adding another full sampling_interval() of delay + std::this_thread::sleep_until(tick_start + this->sampling_interval()); } } diff --git a/src/hws/cpu/utility.cpp b/src/hws/cpu/utility.cpp index e246282..553771f 100644 --- a/src/hws/cpu/utility.cpp +++ b/src/hws/cpu/utility.cpp @@ -23,8 +23,11 @@ namespace hws::detail { std::string run_subprocess(const std::string_view cmd_line) { - // search PATH for executable - constexpr int options = subprocess_option_e::subprocess_option_search_user_path; + // search PATH for executable; combine stdout and stderr into a single handle since we only + // ever read via subprocess_stdout() below -> without this flag, subprocess_create() opens an + // unread pipe for stderr that can fill up and block the child (and therefore also + // subprocess_join() below) forever if the child ever writes a warning/error to stderr + constexpr int options = subprocess_option_e::subprocess_option_search_user_path | subprocess_option_e::subprocess_option_combined_stdout_stderr; constexpr static std::string::size_type buffer_size = 4096; // extract the separate command line arguments From a029bd0da1731bfb0c85a1986ad6bef9fc1bd432 Mon Sep 17 00:00:00 2001 From: Alexander Van Craen Date: Wed, 12 Aug 2026 13:55:46 +0200 Subject: [PATCH 5/6] fix ROCm SMI device index resolution under device visibility masking gpu_amd_hardware_sampler used the HIP-relative device index directly as the ROCm SMI index for every rsmi_dev_* call. ROCm SMI's own device enumeration isn't filtered by HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES the way HIP's is, so under a non-default visibility mask the sampler measured the wrong physical GPU. Fix: resolve the actual ROCm SMI index once at construction via a PCI-bus-ID match against every ROCm SMI device, throwing if no match is found instead of silently falling back to the HIP-relative index. The original HIP-relative index is kept in a new hip_device_id_ member for the one HIP call that still needs HIP-space, and for system_hardware_sampler's device_correlation_hints output. Also adds the missing find_dependency(MPI) to hwsConfig.cmake.in, alongside the existing ones for CUDA/HIP/ROCm-SMI/Level-Zero. --- cmake/hwsConfig.cmake.in | 9 +++ include/hws/gpu_amd/hardware_sampler.hpp | 28 +++++++-- src/hws/gpu_amd/hardware_sampler.cpp | 80 ++++++++++++++++++++---- src/hws/system_hardware_sampler.cpp | 11 ++-- 4 files changed, 104 insertions(+), 24 deletions(-) diff --git a/cmake/hwsConfig.cmake.in b/cmake/hwsConfig.cmake.in index bde52ae..e14e3c8 100644 --- a/cmake/hwsConfig.cmake.in +++ b/cmake/hwsConfig.cmake.in @@ -112,6 +112,15 @@ if (HWS_HAS_CPU_SUPPORT) endif () endif () +# check whether MPI support is enabled +string_contains("${HWS_COMPILE_DEFINITIONS}" "HWS_MPI_SUPPORT_ENABLED" HWS_HAS_MPI_SUPPORT) +if (HWS_HAS_MPI_SUPPORT) + find_dependency(MPI COMPONENTS CXX) + if (NOT hws_FIND_QUIETLY) + message(STATUS "Enabled MPI support via hws.") + endif () +endif () + # check whether NVIDIA GPUs are supported string_contains("${HWS_COMPILE_DEFINITIONS}" "HWS_FOR_NVIDIA_GPUS_ENABLED" HWS_HAS_GPU_NVIDIA_SUPPORT) if (HWS_HAS_GPU_NVIDIA_SUPPORT) diff --git a/include/hws/gpu_amd/hardware_sampler.hpp b/include/hws/gpu_amd/hardware_sampler.hpp index 7ce1d42..ce0e40f 100644 --- a/include/hws/gpu_amd/hardware_sampler.hpp +++ b/include/hws/gpu_amd/hardware_sampler.hpp @@ -117,15 +117,27 @@ class gpu_amd_hardware_sampler : public hardware_sampler { [[nodiscard]] const rocm_smi_temperature_samples &temperature_samples() const noexcept { return temperature_samples_; } /** - * @brief Return the ROCm SMI device index this hardware sampler was constructed with and uses for all of its - * `rsmi_dev_*` calls. - * @details Purely local/informational - since `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` can affect which - * physical device ends up at a given index, don't assume this index is stable across processes or - * reruns; use `pci_bus_id()` to identify the actual physical device. + * @brief Return the ROCm SMI device index this hardware sampler uses for all of its `rsmi_dev_*` calls. + * @details Resolved at construction time from the HIP-relative index passed to the constructor by matching + * PCI bus IDs, since ROCm SMI's own device enumeration is *not* affected by + * `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` the way HIP's is - the two can otherwise disagree + * about which physical device a given index refers to. Purely local/informational: don't assume + * this index is stable across processes or reruns; use `pci_bus_id()` to identify the actual + * physical device. * @return the ROCm SMI device index (`[[nodiscard]]`) */ [[nodiscard]] std::uint32_t device_id() const noexcept { return device_id_; } + /** + * @brief Return the HIP-relative device index this hardware sampler was constructed with (i.e. the index + * into the process's `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES`-filtered device list). + * @details Unlike `device_id()`, this is exactly the constructor argument, unresolved - the right value to + * report when identifying "the Nth GPU visible to this process/rank" (e.g. for a per-rank device + * list), as opposed to `pci_bus_id()`/`device_id()` which identify the physical device itself. + * @return the HIP-relative device index (`[[nodiscard]]`) + */ + [[nodiscard]] std::uint32_t hip_device_id() const noexcept { return hip_device_id_; } + /** * @brief Return the PCI bus ID (e.g. `"0000:c1:00.0"`) of the physical device this hardware sampler actually * measures. @@ -154,8 +166,12 @@ class gpu_amd_hardware_sampler : public hardware_sampler { */ void sampling_loop() final; - /// The ID of the device to sample. + /// The ROCm SMI device index to sample, resolved from hip_device_id_ via a PCI bus ID match (ROCm SMI's own + /// device enumeration isn't affected by HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES, unlike HIP's). std::uint32_t device_id_{}; + /// The HIP-relative device index this hardware sampler was constructed with; only used for the one HIP call + /// (hipGetDeviceProperties) that needs a HIP-space rather than a ROCm-SMI-space index. + std::uint32_t hip_device_id_{}; /// The general AMD GPU samples. rocm_smi_general_samples general_samples_{}; diff --git a/src/hws/gpu_amd/hardware_sampler.cpp b/src/hws/gpu_amd/hardware_sampler.cpp index cbe3b9f..977b4f4 100644 --- a/src/hws/gpu_amd/hardware_sampler.cpp +++ b/src/hws/gpu_amd/hardware_sampler.cpp @@ -35,6 +35,53 @@ namespace hws { +namespace { + +/** + * @brief Convert a ROCm SMI BDFID (as returned by `rsmi_dev_pci_id_get()`) to a sysfs-style PCI bus ID string. + * @details BDFID = (DOMAIN << 32) | (PARTITION << 28) | (BUS << 8) | (DEVICE << 3) | FUNCTION (see ROCm SMI's + * `rsmi_dev_pci_id_get` documentation). On MI-series partitioned devices the function bits are + * repurposed for the partition ID instead of a real PCI function - but the OS/sysfs-visible PCI address + * for the device itself always has function 0, so the function is intentionally not extracted here; see + * `hws::detail::format_pci_bus_id()`. + */ +[[nodiscard]] std::string bdfid_to_pci_bus_id(const std::uint64_t bdfid) { + const auto domain = static_cast((bdfid >> 32) & 0xffffffffull); + const auto bus = static_cast((bdfid >> 8) & 0xffull); + const auto device = static_cast((bdfid >> 3) & 0x1full); + return detail::format_pci_bus_id(domain, bus, device); +} + +/** + * @brief Resolve the ROCm SMI device index that corresponds to the physical device HIP considers index + * @p hip_device_id, by matching PCI bus IDs. + * @details Necessary because ROCm SMI enumerates every physical AMD GPU on the node unconditionally, while HIP's + * enumeration is filtered/reordered by `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` - the same index + * number in both APIs can refer to different physical devices. Requires `rsmi_init()` to have already + * been called. + * @throws std::runtime_error if ROCm SMI's device count can't be queried, or if none of its devices' PCI bus IDs + * match @p hip_device_id's - silently falling back to @p hip_device_id here would be exactly the + * HIP-index-used-as-RSMI-index bug this function exists to avoid, just triggered by a query failure + * instead of a visibility mask. + */ +[[nodiscard]] std::uint32_t resolve_rsmi_device_id(const std::uint32_t hip_device_id) { + std::uint32_t rsmi_count{}; + if (rsmi_num_monitor_devices(&rsmi_count) != RSMI_STATUS_SUCCESS) { + throw std::runtime_error{ "gpu_amd_hardware_sampler: couldn't query the number of ROCm SMI devices while resolving the physical device for HIP index " + std::to_string(hip_device_id) + "!" }; + } + + const std::string hip_bus_id = detail::amd_device_pci_bus_id(static_cast(hip_device_id)); + for (std::uint32_t rsmi_idx = 0; rsmi_idx < rsmi_count; ++rsmi_idx) { + std::uint64_t bdfid{}; + if (rsmi_dev_pci_id_get(rsmi_idx, &bdfid) == RSMI_STATUS_SUCCESS && bdfid_to_pci_bus_id(bdfid) == hip_bus_id) { + return rsmi_idx; + } + } + throw std::runtime_error{ "gpu_amd_hardware_sampler: couldn't find a ROCm SMI device with PCI bus ID " + hip_bus_id + " (HIP index " + std::to_string(hip_device_id) + ")!" }; +} + +} // namespace + gpu_amd_hardware_sampler::gpu_amd_hardware_sampler(const sample_category category) : gpu_amd_hardware_sampler{ 0, HWS_SAMPLING_INTERVAL, category } { } @@ -46,9 +93,10 @@ gpu_amd_hardware_sampler::gpu_amd_hardware_sampler(const std::chrono::millisecon gpu_amd_hardware_sampler::gpu_amd_hardware_sampler(const std::size_t device_id, const std::chrono::milliseconds sampling_interval, const sample_category category) : hardware_sampler{ sampling_interval, category }, - device_id_{ static_cast(device_id) } { + hip_device_id_{ static_cast(device_id) } { // make sure that rsmi_init is only called once for all instances - if (instances_++ == 0) { + const bool is_first_instance = (instances_++ == 0); + if (is_first_instance) { HWS_ROCM_SMI_ERROR_CHECK(rsmi_init(std::uint64_t{ 0 })) // notify that initialization has been finished init_finished_ = true; @@ -56,6 +104,21 @@ gpu_amd_hardware_sampler::gpu_amd_hardware_sampler(const std::size_t device_id, // wait until init has been finished! while (!init_finished_) { } } + + // resolve device_id_ only after rsmi_init() has definitely run (by this instance or a previous one); if + // resolution throws, this instance never finishes construction and its destructor never runs, so + // instances_/init_finished_ (and, if we were the one that just initialized ROCm SMI, the ROCm SMI runtime + // itself) must be rolled back manually here instead of leaking + try { + device_id_ = resolve_rsmi_device_id(hip_device_id_); + } catch (...) { + --instances_; + if (is_first_instance) { + init_finished_ = false; + rsmi_shut_down(); + } + throw; + } } gpu_amd_hardware_sampler::~gpu_amd_hardware_sampler() { @@ -94,7 +157,7 @@ void gpu_amd_hardware_sampler::sampling_loop() { general_samples_.byte_order_ = "Little Endian"; hipDeviceProp_t prop{}; - if (hipGetDeviceProperties(&prop, static_cast(device_id_)) == hipSuccess) { + if (hipGetDeviceProperties(&prop, static_cast(hip_device_id_)) == hipSuccess) { const std::string architecture{ prop.gcnArchName }; general_samples_.architecture_ = architecture.substr(0, architecture.find_first_of('\0')); } @@ -684,16 +747,7 @@ std::string gpu_amd_hardware_sampler::device_identification() const { std::string gpu_amd_hardware_sampler::pci_bus_id() const { std::uint64_t bdfid{}; HWS_ROCM_SMI_ERROR_CHECK(rsmi_dev_pci_id_get(device_id_, &bdfid)) - - // BDFID = (DOMAIN << 32) | (PARTITION << 28) | (BUS << 8) | (DEVICE << 3) | FUNCTION (see ROCm SMI's - // rsmi_dev_pci_id_get documentation). On MI-series partitioned devices the function bits are repurposed for - // the partition ID instead of a real PCI function - but the OS/sysfs-visible PCI address for the device - // itself always has function 0, so the function is intentionally not extracted here; see - // hws::detail::format_pci_bus_id(). - const auto domain = static_cast((bdfid >> 32) & 0xffffffffull); - const auto bus = static_cast((bdfid >> 8) & 0xffull); - const auto device = static_cast((bdfid >> 3) & 0x1full); - return detail::format_pci_bus_id(domain, bus, device); + return bdfid_to_pci_bus_id(bdfid); } std::string gpu_amd_hardware_sampler::samples_only_as_yaml_string() const { diff --git a/src/hws/system_hardware_sampler.cpp b/src/hws/system_hardware_sampler.cpp index da81006..1230a40 100644 --- a/src/hws/system_hardware_sampler.cpp +++ b/src/hws/system_hardware_sampler.cpp @@ -326,11 +326,12 @@ std::string system_hardware_sampler::device_correlation_hints_as_yaml_string() c } #if defined(HWS_FOR_AMD_GPUS_ENABLED) if (const auto *amd = dynamic_cast(ptr.get()); amd != nullptr) { - // pci_bus_id() (ROCm SMI, the same API family used for all of this sampler's actual measurements) is - // used here rather than HIP's own hipDeviceGetPCIBusId(), since ROCm SMI's and HIP's device - // enumerations can diverge under HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES - using a different API - // family than the one the sampler measures with could silently attribute the wrong PCI bus ID. - amd_devices.emplace_back(amd->device_id(), amd->pci_bus_id()); + // hip_device_id() (the "Nth GPU visible to this process/rank") for local_index, but pci_bus_id() + // (ROCm SMI, the same API family used for all of this sampler's actual measurements) for the PCI bus + // ID - not hip_device_id()'s own HIP-space bus id, since ROCm SMI's and HIP's device enumerations can + // diverge under HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES and using a different API family than the + // one the sampler measures with could silently attribute the wrong PCI bus ID. + amd_devices.emplace_back(amd->hip_device_id(), amd->pci_bus_id()); continue; } #endif From 4b6bac1e43788dbcff6cf9fc1817bf37d768b662 Mon Sep 17 00:00:00 2001 From: Alexander Van Craen Date: Thu, 13 Aug 2026 16:13:41 +0200 Subject: [PATCH 6/6] add Python bindings for the Cray pm_counters backend Mirrors the existing per-backend binding pattern: one .cpp file (bindings/cray_pm_counters_hardware_sampler.cpp) binding CrayPmCountersGeneralSamples, CrayPmCountersPowerSamples and CrayPmCountersHardwareSampler (constructors, general_samples, power_samples, discovered_accel_indices, samples_only_as_yaml_string, __repr__), gated behind HWS_FOR_CRAY_PM_COUNTERS_ENABLED like every other backend's bindings. device_identification() stays unbound, consistent with every other backend. Built and tested locally (non-MPI build) and on a real Hunter mi300a node via a PBS job: SystemHardwareSampler auto-discovery, direct CrayPmCountersHardwareSampler construction, and a dump_yaml roundtrip including the device_correlation_hints section all work correctly. --- bindings/CMakeLists.txt | 3 + .../cray_pm_counters_hardware_sampler.cpp | 55 +++++++++++++++++++ bindings/main.cpp | 7 +++ 3 files changed, 65 insertions(+) create mode 100644 bindings/cray_pm_counters_hardware_sampler.cpp diff --git a/bindings/CMakeLists.txt b/bindings/CMakeLists.txt index 89357b6..c78daf5 100644 --- a/bindings/CMakeLists.txt +++ b/bindings/CMakeLists.txt @@ -54,6 +54,9 @@ endif () if ("HWS_FOR_INTEL_GPUS_ENABLED" IN_LIST HWS_COMPILE_DEFINITIONS) list(APPEND HWS_PYTHON_BINDINGS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/gpu_intel_hardware_sampler.cpp) endif () +if ("HWS_FOR_CRAY_PM_COUNTERS_ENABLED" IN_LIST HWS_COMPILE_DEFINITIONS) + list(APPEND HWS_PYTHON_BINDINGS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/cray_pm_counters_hardware_sampler.cpp) +endif () # create pybind11 module set(HWS_PYTHON_BINDINGS_LIBRARY_NAME HardwareSampling) diff --git a/bindings/cray_pm_counters_hardware_sampler.cpp b/bindings/cray_pm_counters_hardware_sampler.cpp new file mode 100644 index 0000000..f32f600 --- /dev/null +++ b/bindings/cray_pm_counters_hardware_sampler.cpp @@ -0,0 +1,55 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + */ + +#include "hws/cray_pm_counters/hardware_sampler.hpp" // hws::cray_pm_counters_hardware_sampler +#include "hws/cray_pm_counters/pm_counters_samples.hpp" // hws::{cray_pm_counters_general_samples, cray_pm_counters_power_samples} +#include "hws/hardware_sampler.hpp" // hws::hardware_sampler +#include "hws/sample_category.hpp" // hws::sample_category + +#include "fmt/format.h" // fmt::format +#include "pybind11/chrono.h" // automatic bindings for std::chrono::milliseconds +#include "pybind11/pybind11.h" // py::module_ +#include "pybind11/stl.h" // bind STL types + +#include // std::chrono::milliseconds + +namespace py = pybind11; + +void init_cray_pm_counters_hardware_sampler(py::module_ &m) { + // bind the general samples + py::class_(m, "CrayPmCountersGeneralSamples") + .def("has_samples", &hws::cray_pm_counters_general_samples::has_samples, "true if any sample is available, false otherwise") + .def("get_metadata", &hws::cray_pm_counters_general_samples::get_metadata, "the dynamically discovered, non-energy, non-power pm_counters metadata entries (e.g. version, generation, raw_scan_hz, freshness), keyed by file name") + .def("__repr__", [](const hws::cray_pm_counters_general_samples &self) { + return fmt::format("", self); + }); + + // bind the power samples + py::class_(m, "CrayPmCountersPowerSamples") + .def("has_samples", &hws::cray_pm_counters_power_samples::has_samples, "true if any sample is available, false otherwise") + .def("get_energy_counters", &hws::cray_pm_counters_power_samples::get_energy_counters, "the dynamically discovered measured energy counters in J (cumulative), keyed by file name, e.g. 'energy' (whole node) or 'accel0_energy' (per accelerator)") + .def("get_power_counters", &hws::cray_pm_counters_power_samples::get_power_counters, "the dynamically discovered measured power counters in W (instantaneous), keyed by file name, e.g. 'power' (whole node) or 'accel0_power' (per accelerator)") + .def("get_energy_timestamps_us", &hws::cray_pm_counters_power_samples::get_energy_timestamps_us, "per-sample HSS latch timestamp for the energy counters, in us (PM counters v3+ only)") + .def("get_power_timestamps_us", &hws::cray_pm_counters_power_samples::get_power_timestamps_us, "per-sample HSS latch timestamp for the power counters, in us (PM counters v3+ only)") + .def("__repr__", [](const hws::cray_pm_counters_power_samples &self) { + return fmt::format("", self); + }); + + // bind the Cray pm_counters hardware sampler class + py::class_(m, "CrayPmCountersHardwareSampler") + .def(py::init<>(), "construct a new Cray pm_counters hardware sampler with the default sampling interval") + .def(py::init(), "construct a new Cray pm_counters hardware sampler with the default sampling interval sampling only the provided sample_category samples") + .def(py::init(), "construct a new Cray pm_counters hardware sampler with the specified sampling interval") + .def(py::init(), "construct a new Cray pm_counters hardware sampler with the specified sampling interval sampling only the provided sample_category samples") + .def("general_samples", &hws::cray_pm_counters_hardware_sampler::general_samples, "get all general samples") + .def("power_samples", &hws::cray_pm_counters_hardware_sampler::power_samples, "get all power related samples") + .def("discovered_accel_indices", &hws::cray_pm_counters_hardware_sampler::discovered_accel_indices, "the accelerator indices pm_counters actually exposed on this node, derived from the already-sampled accel_energy counter keys") + .def("samples_only_as_yaml_string", &hws::cray_pm_counters_hardware_sampler::samples_only_as_yaml_string, "return all hardware samples as YAML string") + .def("__repr__", [](const hws::cray_pm_counters_hardware_sampler &self) { + return fmt::format("", self); + }); +} diff --git a/bindings/main.cpp b/bindings/main.cpp index 932a897..0fae325 100644 --- a/bindings/main.cpp +++ b/bindings/main.cpp @@ -32,6 +32,7 @@ void init_cpu_hardware_sampler(py::module_ &); void init_gpu_nvidia_hardware_sampler(py::module_ &); void init_gpu_amd_hardware_sampler(py::module_ &); void init_gpu_intel_hardware_sampler(py::module_ &); +void init_cray_pm_counters_hardware_sampler(py::module_ &); void init_version(py::module_ &); PYBIND11_MODULE(HardwareSampling, m) { @@ -77,6 +78,12 @@ PYBIND11_MODULE(HardwareSampling, m) { #endif m.def("has_gpu_intel_hardware_sampler", []() { return HWS_IS_DEFINED(HWS_FOR_INTEL_GPUS_ENABLED); }); + // Cray pm_counters sampling +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + init_cray_pm_counters_hardware_sampler(m); +#endif + m.def("has_cray_pm_counters_hardware_sampler", []() { return HWS_IS_DEFINED(HWS_FOR_CRAY_PM_COUNTERS_ENABLED); }); + init_version(m); }