diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ab2b4b..22bfec5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,6 +147,18 @@ else () endif () +#################################################################################################################### +## Cray/HPE node power sampling via pm_counters ## +#################################################################################################################### +set(HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING AUTO CACHE STRING "Enable sampling of node power/energy via Cray's /sys/cray/pm_counters interface.") +set_property(CACHE HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING PROPERTY STRINGS AUTO ON OFF) +if (HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING MATCHES "AUTO" OR HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING) + add_subdirectory(src/hws/cray_pm_counters) +else () + message(STATUS "Hardware sampling via Cray pm_counters disabled!") +endif () + + #################################################################################################################### ## enable MPI support ## #################################################################################################################### @@ -201,6 +213,16 @@ if (HWS_ENABLE_DOCUMENTATION) endif () +######################################################################################################################## +## add tests ## +######################################################################################################################## +option(HWS_ENABLE_TESTING "Build simple regression tests (run via 'ctest')." OFF) +if (HWS_ENABLE_TESTING) + enable_testing() + add_subdirectory(tests) +endif () + + ######################################################################################################################## ## add support for `make install` ## ######################################################################################################################## diff --git a/README.md b/README.md index 1bd62df..9a50996 100644 --- a/README.md +++ b/README.md @@ -35,14 +35,14 @@ To download the hardware sampling use: ```bash git clone git@github.com:SC-SGS/hardware_sampling.git -cd hardware_sampling +cd hardware_sampling ``` Building the library can be done using the normal CMake approach: ```bash -mkdir build && cd build -cmake -DCMAKE_BUILD_TYPE=Release [optional_options] .. +mkdir build && cd build +cmake -DCMAKE_BUILD_TYPE=Release [optional_options] .. cmake --build . -j ``` @@ -73,6 +73,18 @@ The `[optional_options]` can be one or multiple of: - `HWS_ENABLE_ERROR_CHECKS=ON|OFF` (default: `OFF`): enable sanity checks during hardware sampling, may be problematic with smaller sample intervals - `HWS_SAMPLING_INTERVAL=100ms` (default: `100ms`): set the sampling interval in milliseconds + +- `HWS_TURBOSTAT_INTERVAL=1` (default: `0.001`, kept for backwards compatibility): set the interval in + seconds (`"sec.subsec"`) that `turbostat` itself measures over for a single CPU power/frequency sample + (passed directly to `turbostat`'s own `-i` flag). + **Important:** very short values have been observed to produce + physically implausible readings (multi-kW package power, multi-GHz core clocks) on heavily loaded, + many-core machines, especially with older `turbostat` builds. + **Note:** if this value is larger than `HWS_SAMPLING_INTERVAL`, the `turbostat` backend dominates and + the achieved CPU sampling cadence will be closer to `HWS_TURBOSTAT_INTERVAL` than to + `HWS_SAMPLING_INTERVAL` -- a warning is printed at runtime (once per `cpu_hardware_sampler` instance) + if this is the case. + - `HWS_ENABLE_PYTHON_BINDINGS=ON|OFF` (default: `ON`): enable Python bindings - `HWS_ENABLE_MPI_SUPPORT=ON|OFF|AUTO` (default: `AUTO`): @@ -227,7 +239,7 @@ current clock frequencies, temperatures, or memory consumption. | sample | sample type | CPUs | NVIDIA GPUs | AMD GPUs | Intel GPUs | |:------------------------|:-----------:|:----:|:-----------:|:--------:|:----------:| | num_fans | fixed | - | int | int | int | -| fan_speed_min | fixed | - | % | - | - | +| fan_speed_min | fixed | - | % | - | - | | fan_speed_max | fixed | - | % | RPM | RPM | | temperature_min | fixed | - | - | °C | - | | temperature_max | fixed | - | °C | °C | °C | diff --git a/bindings/CMakeLists.txt b/bindings/CMakeLists.txt index 89357b6..c78daf5 100644 --- a/bindings/CMakeLists.txt +++ b/bindings/CMakeLists.txt @@ -54,6 +54,9 @@ endif () if ("HWS_FOR_INTEL_GPUS_ENABLED" IN_LIST HWS_COMPILE_DEFINITIONS) list(APPEND HWS_PYTHON_BINDINGS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/gpu_intel_hardware_sampler.cpp) endif () +if ("HWS_FOR_CRAY_PM_COUNTERS_ENABLED" IN_LIST HWS_COMPILE_DEFINITIONS) + list(APPEND HWS_PYTHON_BINDINGS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/cray_pm_counters_hardware_sampler.cpp) +endif () # create pybind11 module set(HWS_PYTHON_BINDINGS_LIBRARY_NAME HardwareSampling) diff --git a/bindings/cray_pm_counters_hardware_sampler.cpp b/bindings/cray_pm_counters_hardware_sampler.cpp new file mode 100644 index 0000000..f32f600 --- /dev/null +++ b/bindings/cray_pm_counters_hardware_sampler.cpp @@ -0,0 +1,55 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + */ + +#include "hws/cray_pm_counters/hardware_sampler.hpp" // hws::cray_pm_counters_hardware_sampler +#include "hws/cray_pm_counters/pm_counters_samples.hpp" // hws::{cray_pm_counters_general_samples, cray_pm_counters_power_samples} +#include "hws/hardware_sampler.hpp" // hws::hardware_sampler +#include "hws/sample_category.hpp" // hws::sample_category + +#include "fmt/format.h" // fmt::format +#include "pybind11/chrono.h" // automatic bindings for std::chrono::milliseconds +#include "pybind11/pybind11.h" // py::module_ +#include "pybind11/stl.h" // bind STL types + +#include // std::chrono::milliseconds + +namespace py = pybind11; + +void init_cray_pm_counters_hardware_sampler(py::module_ &m) { + // bind the general samples + py::class_(m, "CrayPmCountersGeneralSamples") + .def("has_samples", &hws::cray_pm_counters_general_samples::has_samples, "true if any sample is available, false otherwise") + .def("get_metadata", &hws::cray_pm_counters_general_samples::get_metadata, "the dynamically discovered, non-energy, non-power pm_counters metadata entries (e.g. version, generation, raw_scan_hz, freshness), keyed by file name") + .def("__repr__", [](const hws::cray_pm_counters_general_samples &self) { + return fmt::format("", self); + }); + + // bind the power samples + py::class_(m, "CrayPmCountersPowerSamples") + .def("has_samples", &hws::cray_pm_counters_power_samples::has_samples, "true if any sample is available, false otherwise") + .def("get_energy_counters", &hws::cray_pm_counters_power_samples::get_energy_counters, "the dynamically discovered measured energy counters in J (cumulative), keyed by file name, e.g. 'energy' (whole node) or 'accel0_energy' (per accelerator)") + .def("get_power_counters", &hws::cray_pm_counters_power_samples::get_power_counters, "the dynamically discovered measured power counters in W (instantaneous), keyed by file name, e.g. 'power' (whole node) or 'accel0_power' (per accelerator)") + .def("get_energy_timestamps_us", &hws::cray_pm_counters_power_samples::get_energy_timestamps_us, "per-sample HSS latch timestamp for the energy counters, in us (PM counters v3+ only)") + .def("get_power_timestamps_us", &hws::cray_pm_counters_power_samples::get_power_timestamps_us, "per-sample HSS latch timestamp for the power counters, in us (PM counters v3+ only)") + .def("__repr__", [](const hws::cray_pm_counters_power_samples &self) { + return fmt::format("", self); + }); + + // bind the Cray pm_counters hardware sampler class + py::class_(m, "CrayPmCountersHardwareSampler") + .def(py::init<>(), "construct a new Cray pm_counters hardware sampler with the default sampling interval") + .def(py::init(), "construct a new Cray pm_counters hardware sampler with the default sampling interval sampling only the provided sample_category samples") + .def(py::init(), "construct a new Cray pm_counters hardware sampler with the specified sampling interval") + .def(py::init(), "construct a new Cray pm_counters hardware sampler with the specified sampling interval sampling only the provided sample_category samples") + .def("general_samples", &hws::cray_pm_counters_hardware_sampler::general_samples, "get all general samples") + .def("power_samples", &hws::cray_pm_counters_hardware_sampler::power_samples, "get all power related samples") + .def("discovered_accel_indices", &hws::cray_pm_counters_hardware_sampler::discovered_accel_indices, "the accelerator indices pm_counters actually exposed on this node, derived from the already-sampled accel_energy counter keys") + .def("samples_only_as_yaml_string", &hws::cray_pm_counters_hardware_sampler::samples_only_as_yaml_string, "return all hardware samples as YAML string") + .def("__repr__", [](const hws::cray_pm_counters_hardware_sampler &self) { + return fmt::format("", self); + }); +} diff --git a/bindings/main.cpp b/bindings/main.cpp index 932a897..0fae325 100644 --- a/bindings/main.cpp +++ b/bindings/main.cpp @@ -32,6 +32,7 @@ void init_cpu_hardware_sampler(py::module_ &); void init_gpu_nvidia_hardware_sampler(py::module_ &); void init_gpu_amd_hardware_sampler(py::module_ &); void init_gpu_intel_hardware_sampler(py::module_ &); +void init_cray_pm_counters_hardware_sampler(py::module_ &); void init_version(py::module_ &); PYBIND11_MODULE(HardwareSampling, m) { @@ -77,6 +78,12 @@ PYBIND11_MODULE(HardwareSampling, m) { #endif m.def("has_gpu_intel_hardware_sampler", []() { return HWS_IS_DEFINED(HWS_FOR_INTEL_GPUS_ENABLED); }); + // Cray pm_counters sampling +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + init_cray_pm_counters_hardware_sampler(m); +#endif + m.def("has_cray_pm_counters_hardware_sampler", []() { return HWS_IS_DEFINED(HWS_FOR_CRAY_PM_COUNTERS_ENABLED); }); + init_version(m); } diff --git a/cmake/hwsConfig.cmake.in b/cmake/hwsConfig.cmake.in index bde52ae..e14e3c8 100644 --- a/cmake/hwsConfig.cmake.in +++ b/cmake/hwsConfig.cmake.in @@ -112,6 +112,15 @@ if (HWS_HAS_CPU_SUPPORT) endif () endif () +# check whether MPI support is enabled +string_contains("${HWS_COMPILE_DEFINITIONS}" "HWS_MPI_SUPPORT_ENABLED" HWS_HAS_MPI_SUPPORT) +if (HWS_HAS_MPI_SUPPORT) + find_dependency(MPI COMPONENTS CXX) + if (NOT hws_FIND_QUIETLY) + message(STATUS "Enabled MPI support via hws.") + endif () +endif () + # check whether NVIDIA GPUs are supported string_contains("${HWS_COMPILE_DEFINITIONS}" "HWS_FOR_NVIDIA_GPUS_ENABLED" HWS_HAS_GPU_NVIDIA_SUPPORT) if (HWS_HAS_GPU_NVIDIA_SUPPORT) diff --git a/include/hws/core.hpp b/include/hws/core.hpp index 8c7a474..68dd84b 100644 --- a/include/hws/core.hpp +++ b/include/hws/core.hpp @@ -38,4 +38,9 @@ #include "hws/gpu_intel/level_zero_samples.hpp" #endif +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + #include "hws/cray_pm_counters/hardware_sampler.hpp" + #include "hws/cray_pm_counters/pm_counters_samples.hpp" +#endif + #endif // HWS_CORE_HPP_ diff --git a/include/hws/cpu/hardware_sampler.hpp b/include/hws/cpu/hardware_sampler.hpp index d1b4102..18f76bb 100644 --- a/include/hws/cpu/hardware_sampler.hpp +++ b/include/hws/cpu/hardware_sampler.hpp @@ -40,6 +40,11 @@ class cpu_hardware_sampler : public hardware_sampler { * @brief Construct a new CPU hardware sampler with the @p sampling_interval. * @param[in] sampling_interval the used sampling interval * @param[in] category the sample categories that are enabled for hardware sampling (default: all) + * @note If the turbostat backend is used, turbostat itself blocks for `HWS_TURBOSTAT_INTERVAL` seconds + * per invocation (see the CMake option of the same name). If that is larger than + * @p sampling_interval, the turbostat backend dominates and the achieved sampling cadence will be + * closer to `HWS_TURBOSTAT_INTERVAL` than to @p sampling_interval; a warning is printed to + * `std::cerr` in that case. */ explicit cpu_hardware_sampler(std::chrono::milliseconds sampling_interval, sample_category category = sample_category::all); diff --git a/include/hws/cray_pm_counters/hardware_sampler.hpp b/include/hws/cray_pm_counters/hardware_sampler.hpp new file mode 100644 index 0000000..fdf603f --- /dev/null +++ b/include/hws/cray_pm_counters/hardware_sampler.hpp @@ -0,0 +1,159 @@ +/** + * @file + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + * + * @brief Defines a hardware sampler for whole-node power/energy using Cray/HPE's `/sys/cray/pm_counters` sysfs + * interface (measured, not modeled, node power; includes voltage converter losses). + */ + +#ifndef HWS_CRAY_PM_COUNTERS_HARDWARE_SAMPLER_HPP_ +#define HWS_CRAY_PM_COUNTERS_HARDWARE_SAMPLER_HPP_ +#pragma once + +#include "hws/cray_pm_counters/pm_counters_samples.hpp" // hws::{cray_pm_counters_general_samples, cray_pm_counters_power_samples} +#include "hws/hardware_sampler.hpp" // hws::hardware_sampler +#include "hws/sample_category.hpp" // hws::sample_category + +#include "fmt/ostream.h" // fmt::formatter, fmt::ostream_formatter + +#include // std::chrono::milliseconds, std::chrono_literals namespace +#include // std::filesystem::path +#include // std::ostream forward declaration +#include // std::optional +#include // std::unordered_map +#include // std::vector + +namespace hws { + +using namespace std::chrono_literals; + +/** + * @brief A hardware sampler for whole-node power/energy using Cray/HPE's `/sys/cray/pm_counters` sysfs interface. + * @details Since `pm_counters` is a per-node (not per-device) sysfs interface, only a single instance of this + * sampler exists per node (mirroring `hws::cpu_hardware_sampler`), even on nodes with an accelerator + * (whose "accel energy"/"accel power" counters `pm_counters` also exposes). In MPI `whole_node` mode + * only the node-local rank 0 creates this sampler; in `per_rank` mode (and the non-MPI, single-process + * case), every rank on a node creates its own instance, so the same node-wide ground truth is sampled and + * reported redundantly once per rank - this mirrors `hws::cpu_hardware_sampler`'s existing behavior. + */ +class cray_pm_counters_hardware_sampler : public hardware_sampler { + public: + /** + * @brief Construct a new pm_counters hardware sampler with the default sampling interval. + * @param[in] category the sample categories that are enabled for hardware sampling (default: all) + */ + explicit cray_pm_counters_hardware_sampler(sample_category category = sample_category::all); + /** + * @brief Construct a new pm_counters hardware sampler with the @p sampling_interval. + * @param[in] sampling_interval the used sampling interval + * @param[in] category the sample categories that are enabled for hardware sampling (default: all) + */ + explicit cray_pm_counters_hardware_sampler(std::chrono::milliseconds sampling_interval, sample_category category = sample_category::all); + + /** + * @brief Delete the copy-constructor (already implicitly deleted due to the base class's std::atomic member). + */ + cray_pm_counters_hardware_sampler(const cray_pm_counters_hardware_sampler &) = delete; + /** + * @brief Delete the move-constructor (already implicitly deleted due to the base class's std::atomic member). + */ + cray_pm_counters_hardware_sampler(cray_pm_counters_hardware_sampler &&) noexcept = delete; + /** + * @brief Delete the copy-assignment operator (already implicitly deleted due to the base class's std::atomic member). + */ + cray_pm_counters_hardware_sampler &operator=(const cray_pm_counters_hardware_sampler &) = delete; + /** + * @brief Delete the move-assignment operator (already implicitly deleted due to the base class's std::atomic member). + */ + cray_pm_counters_hardware_sampler &operator=(cray_pm_counters_hardware_sampler &&) noexcept = delete; + + /** + * @brief Destruct the pm_counters hardware sampler. If the sampler is still running, stops it. + */ + ~cray_pm_counters_hardware_sampler() override; + + /** + * @brief Return the general pm_counters samples of this hardware sampler. + * @return the general pm_counters samples (`[[nodiscard]]`) + */ + [[nodiscard]] const cray_pm_counters_general_samples &general_samples() const noexcept { return general_samples_; } + + /** + * @brief Return the power related pm_counters samples of this hardware sampler. + * @return the power related pm_counters samples (`[[nodiscard]]`) + */ + [[nodiscard]] const cray_pm_counters_power_samples &power_samples() const noexcept { return power_samples_; } + + /** + * @brief The accelerator indices pm_counters actually exposed on this node (e.g. `{0, 1, 2, 3}` for a 4-APU + * node), derived from the already-sampled `accel_energy` counter keys. + * @details Only meaningful after at least one sample has been taken (i.e. during or after sampling); returns + * an empty vector beforehand. + * @return the sorted accelerator indices (`[[nodiscard]]`) + */ + [[nodiscard]] std::vector discovered_accel_indices() const; + + /** + * @copydoc hws::hardware_sampler::device_identification + */ + [[nodiscard]] std::string device_identification() const final; + + /** + * @copydoc hws::hardware_sampler::samples_only_as_yaml_string() const + */ + [[nodiscard]] std::string samples_only_as_yaml_string() const final; + + private: + /** + * @copydoc hws::hardware_sampler::sampling_loop + */ + void sampling_loop() final; + + /** + * @brief Read every discovered pm_counters file once and append the readings to `general_samples_`/`power_samples_`. + * @details Called both for the very first sample and on every subsequent tick of the sampling loop - all three + * categories (general, energy, power) are re-read every tick, see `cray_pm_counters_general_samples`. + */ + void sample_once(); + + /** + * @brief Derive the hardware's own update period from the already-sampled `raw_scan_hz` metadata. + * @return the hardware tick period, or `std::nullopt` if `raw_scan_hz` wasn't found/couldn't be parsed (`[[nodiscard]]`) + */ + [[nodiscard]] std::optional hardware_tick_period() const; + + /// The general pm_counters samples. + cray_pm_counters_general_samples general_samples_{}; + /// The power related pm_counters samples. + cray_pm_counters_power_samples power_samples_{}; + + /// The full sysfs path of every general metadata file discovered at sampling start, keyed by its `general_samples().get_metadata()` map key. + std::unordered_map general_paths_{}; + /// The full sysfs path of every energy counter discovered at sampling start, keyed by its `power_samples().get_energy_counters()` map key. + std::unordered_map energy_counter_paths_{}; + /// The full sysfs path of every power counter discovered at sampling start, keyed by its `power_samples().get_power_counters()` map key. + std::unordered_map power_counter_paths_{}; +}; + +/** + * @brief Output all pm_counters samples gathered by the @p sampler to the given output-stream @p out. + * @details Sets `std::ios_base::failbit` if the @p sampler is still sampling. + * @param[in,out] out the output-stream to write the pm_counters samples to + * @param[in] sampler the pm_counters hardware sampler + * @return the output-stream + */ +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_hardware_sampler &sampler); + +} // namespace hws + +/// @cond Doxygen_suppress + +template <> +struct fmt::formatter : fmt::ostream_formatter { }; + +/// @endcond + +#endif // HWS_CRAY_PM_COUNTERS_HARDWARE_SAMPLER_HPP_ diff --git a/include/hws/cray_pm_counters/pm_counters_samples.hpp b/include/hws/cray_pm_counters/pm_counters_samples.hpp new file mode 100644 index 0000000..b775246 --- /dev/null +++ b/include/hws/cray_pm_counters/pm_counters_samples.hpp @@ -0,0 +1,140 @@ +/** + * @file + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + * + * @brief Defines the samples gathered from Cray/HPE's `/sys/cray/pm_counters` sysfs interface. + * @details The exact set of files exposed by `/sys/cray/pm_counters` is Cray/HPE-generation specific, so instead of + * hardcoding specific file names, every regular file found below the directory is picked up dynamically + * and classified by its (relative) file name into a measured energy counter, a measured power counter, or + * (everything else, including configured caps and derived/event counters) general metadata - see + * `hws::detail::is_energy_counter_key`/`is_power_counter_key`. All three categories are re-read every + * sampling tick (see `general_samples()` for why even seemingly static metadata like `freshness` needs to + * be). Confirmed against a real HPE Cray EX255a node (`/sys/cray/pm_counters` version 3): 23 files - + * `energy`/`accel[0-3]_energy` (measured energy, J), `power`/`accel[0-3]_power` (measured power, W), and + * general metadata covering both configured limits (`power_cap`/`accel[0-3]_power_cap`), derived/event + * counters (`capped_energy`, `overshoot`, `overshoot_energy`), and protocol metadata (`freshness`, + * `generation`, `raw_scan_hz`, `startup`, `version`). Version 3 telemetry files (but not the `*_cap` + * files) contain `" us"`, e.g. `"2869283299 J 7810213855886 us"` - see + * `pm_counter_reading`. + */ + +#ifndef HWS_CRAY_PM_COUNTERS_PM_COUNTERS_SAMPLES_HPP_ +#define HWS_CRAY_PM_COUNTERS_PM_COUNTERS_SAMPLES_HPP_ +#pragma once + +#include "hws/utility.hpp" // HWS_SAMPLE_STRUCT_FIXED_MEMBER + +#include "fmt/ostream.h" // fmt::formatter, fmt::ostream_formatter + +#include // std::uint64_t +#include // std::ostream forward declaration +#include // std::string +#include // std::unordered_map +#include // std::vector + +namespace hws { + +//*************************************************************************************************************************************// +// general samples // +//*************************************************************************************************************************************// + +/** + * @brief Wrapper class for the non-energy, non-power `/sys/cray/pm_counters` metadata, e.g. `version`, + * `generation`, `raw_scan_hz`, or `freshness`. + * @details Sampled every tick, not just once: while `version`/`startup`/`raw_scan_hz` are static, `freshness` + * increments at `raw_scan_hz` and `generation`/`overshoot` change when a power cap is (re)applied or + * exceeded. HPE's documented consistency check is to read `freshness` before and after a batch of energy + * /power reads; if it's unchanged, that batch is a consistent snapshot - only possible if it's re-sampled. + */ +class cray_pm_counters_general_samples { + // befriend hardware sampler class + friend class cray_pm_counters_hardware_sampler; + /// The map type used for the dynamically discovered metadata entries: file name (relative to `/sys/cray/pm_counters`) -> sampled raw file content. + using metadata_type = std::unordered_map>; + + public: + /** + * @brief Checks whether any general related hardware sample is present. + * @return `true` if any general related hardware sample is, otherwise `false`. + */ + [[nodiscard]] bool has_samples() const; + /** + * @brief Assemble the YAML string containing all available general hardware samples. + * @details Returns an empty string if `has_samples()` returns `false`. + * @return the YAML string (`[[nodiscard]]`) + */ + [[nodiscard]] std::string generate_yaml_string() const; + + HWS_SAMPLE_STRUCT_FIXED_MEMBER(metadata_type, metadata) // dynamically discovered pm_counters metadata entries +}; + +/** + * @brief Output the general @p samples to the given output-stream @p out. + * @param[in,out] out the output-stream to write the general related hardware samples to + * @param[in] samples the pm_counters general related samples + * @return the output-stream + */ +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_general_samples &samples); + +//*************************************************************************************************************************************// +// power samples // +//*************************************************************************************************************************************// + +/** + * @brief Wrapper class for the dynamically discovered energy and power counters below `/sys/cray/pm_counters` + * (e.g. `energy`/`power` for the whole node, `accel[i]_energy`/`accel[i]_power` per APU). + * @details Values are cumulative Joules for energy counters and instantaneous Watts for power counters (confirmed + * against a real HPE Cray EX255a node). On PM counters version 3, telemetry files (but not the `*_cap` + * files) additionally carry a per-sample microsecond timestamp (see `energy_timestamps_us`/ + * `power_timestamps_us`), latched by the HSS - use this instead of the host read time for accurate + * energy-to-power derivatives, since pm_counters only updates at `raw_scan_hz` (10 Hz). + */ +class cray_pm_counters_power_samples { + // befriend hardware sampler class + friend class cray_pm_counters_hardware_sampler; + /// The map type used for the dynamically discovered counters: file name (relative to `/sys/cray/pm_counters`) -> sampled raw values. + using counter_map_type = std::unordered_map>; + + public: + /** + * @brief Checks whether any power related hardware sample is present. + * @return `true` if any power related hardware sample is, otherwise `false`. + */ + [[nodiscard]] bool has_samples() const; + /** + * @brief Assemble the YAML string containing all available power hardware samples. + * @details Returns an empty string if `has_samples()` returns `false`. + * @return the YAML string (`[[nodiscard]]`) + */ + [[nodiscard]] std::string generate_yaml_string() const; + + HWS_SAMPLE_STRUCT_FIXED_MEMBER(counter_map_type, energy_counters) // dynamically discovered *energy* pm_counters entries, in J + HWS_SAMPLE_STRUCT_FIXED_MEMBER(counter_map_type, power_counters) // dynamically discovered *power* pm_counters entries, in W + HWS_SAMPLE_STRUCT_FIXED_MEMBER(counter_map_type, energy_timestamps_us) // per-sample HSS latch timestamp for energy_counters, in us (PM counters v3+) + HWS_SAMPLE_STRUCT_FIXED_MEMBER(counter_map_type, power_timestamps_us) // per-sample HSS latch timestamp for power_counters, in us (PM counters v3+) +}; + +/** + * @brief Output the power related @p samples to the given output-stream @p out. + * @param[in,out] out the output-stream to write the power related hardware samples to + * @param[in] samples the pm_counters power related samples + * @return the output-stream + */ +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_power_samples &samples); + +} // namespace hws + +/// @cond Doxygen_suppress + +template <> +struct fmt::formatter : fmt::ostream_formatter { }; + +template <> +struct fmt::formatter : fmt::ostream_formatter { }; + +/// @endcond + +#endif // HWS_CRAY_PM_COUNTERS_PM_COUNTERS_SAMPLES_HPP_ diff --git a/include/hws/cray_pm_counters/utility.hpp b/include/hws/cray_pm_counters/utility.hpp new file mode 100644 index 0000000..1bb6be7 --- /dev/null +++ b/include/hws/cray_pm_counters/utility.hpp @@ -0,0 +1,161 @@ +/** + * @file + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + * + * @brief Utility functions to discover and read Cray/HPE's `/sys/cray/pm_counters` sysfs interface. + */ + +#ifndef HWS_CRAY_PM_COUNTERS_UTILITY_HPP_ +#define HWS_CRAY_PM_COUNTERS_UTILITY_HPP_ +#pragma once + +#include // std::size_t +#include // std::uint64_t +#include // std::filesystem::path +#include // std::optional +#include // std::string +#include // std::string_view +#include // std::pair +#include // std::vector + +namespace hws::detail { + +/// The default root directory of Cray/HPE's pm_counters sysfs interface. +inline constexpr std::string_view default_pm_counters_root = "/sys/cray/pm_counters"; + +/** + * @brief Return the root directory of the pm_counters sysfs interface to use. + * @details Defaults to `hws::detail::default_pm_counters_root`, but can be overridden via the `HWS_PM_COUNTERS_PATH` + * environment variable (e.g. to point at a synthetic directory for local testing off a Cray system). + * @return the pm_counters root directory (`[[nodiscard]]`) + */ +[[nodiscard]] std::filesystem::path pm_counters_root(); + +/** + * @brief Check whether the pm_counters sysfs interface is available on this node. + * @return `true` if `hws::detail::pm_counters_root()` exists and is a directory, otherwise `false` (`[[nodiscard]]`) + */ +[[nodiscard]] bool pm_counters_available(); + +/** + * @brief Recursively list all regular files below `hws::detail::pm_counters_root()`. + * @return the discovered files, or an empty vector if pm_counters isn't available (`[[nodiscard]]`) + */ +[[nodiscard]] std::vector list_pm_counter_files(); + +/** + * @brief Derive the unique map key for @p file: its path relative to `hws::detail::pm_counters_root()`, e.g. + * `energy` or, for a nested file, `accel0/energy` ('/' is a valid, unquoted YAML mapping key character). + * @param[in] file the pm_counters file, must be located below `hws::detail::pm_counters_root()` + * @return the map key (`[[nodiscard]]`) + */ +[[nodiscard]] std::string pm_counter_key(const std::filesystem::path &file); + +/** + * @brief Read the trimmed content of @p file as a raw string. + * @param[in] file the file to read + * @return the trimmed file content, or an empty string if the file couldn't be read (`[[nodiscard]]`) + */ +[[nodiscard]] std::string read_pm_counter_raw(const std::filesystem::path &file); + +/** + * @brief A single pm_counters energy/power reading. + * @details On Hunter (PM counters version 3), telemetry files (e.g. `energy`, `accel0_power`) contain + * `" us"` (e.g. `"2869283299 J 7810213855886 us"`), while power/energy cap + * files and older PM counter versions may only contain `" "` or even just `""`. + */ +struct pm_counter_reading { + /// The counter's raw value (unit depends on the file, e.g. J for energy, W for power). + std::uint64_t value{}; + /// The microsecond timestamp at which the value was latched by the HSS, if the file provides one. + std::optional timestamp_us{}; +}; + +/** + * @brief Read the content of @p file and try to parse it as a `pm_counter_reading`. + * @param[in] file the file to read + * @return the parsed reading, or `std::nullopt` if the file couldn't be read or doesn't start with an unsigned integer (`[[nodiscard]]`) + */ +[[nodiscard]] std::optional read_pm_counter_reading(const std::filesystem::path &file); + +/** + * @brief Check whether the map key @p key (as returned by `pm_counter_key`) looks like a measured energy counter: + * contains "energy" but not "cap" or "overshoot" (e.g. `energy`/`accel0_energy`, but not `capped_energy` + * or `overshoot_energy`, which are derived/event counters, not raw measured node/accelerator energy). + * @param[in] key the map key to check + * @return `true` if @p key looks like a measured energy counter, otherwise `false` (`[[nodiscard]]`) + */ +[[nodiscard]] bool is_energy_counter_key(const std::string &key); + +/** + * @brief Check whether the map key @p key (as returned by `pm_counter_key`) looks like a measured power counter: + * contains "power" but not "cap" or "overshoot" (e.g. `power`/`accel0_power`, but not `power_cap`/ + * `accel0_power_cap`, which are configured limits, not measured power draw). + * @param[in] key the map key to check + * @return `true` if @p key looks like a measured power counter, otherwise `false` (`[[nodiscard]]`) + */ +[[nodiscard]] bool is_power_counter_key(const std::string &key); + +/** + * @brief Extract the accelerator index from an energy or power counter map key, e.g. `"accel2_energy"` -> `2`. + * @param[in] key the map key to check, as returned by `pm_counter_key` + * @return the accelerator index, or `std::nullopt` if @p key isn't of the form `accel_energy`/`accel_power` + * (`[[nodiscard]]`) + */ +[[nodiscard]] std::optional accel_index_from_counter_key(const std::string &key); + +/** + * @brief Extract the sorted, deduplicated set of accelerator indices referenced by @p counter_keys. + * @details Intended to be called with the keys of an already-sampled `cray_pm_counters_power_samples`'s + * `energy_counters`/`power_counters` map, to answer "how many `accel[i]` counters did pm_counters + * actually expose on this node" without re-scanning the filesystem. + * @param[in] counter_keys the energy or power counter map keys to scan, as returned by `pm_counter_key` + * @return the sorted accelerator indices found (`[[nodiscard]]`) + */ +[[nodiscard]] std::vector accel_indices_from_counter_keys(const std::vector &counter_keys); + +/** + * @brief Guess which pm_counters `accel[i]` a given AMD GPU (identified by its PCI bus ID) corresponds to. + * @details UNVERIFIED heuristic: assumes `accel[i]` numbers accelerators in ascending PCI bus address order among + * all physically present AMD GPUs - no HPE documentation defines this correspondence, so this should be + * confirmed empirically (e.g. drive load on a single visible GPU and observe which `accel[i]_power` + * reacts) before being relied on for analysis. See `hws::system_hardware_sampler::device_correlation_hints_as_yaml_string()`, + * the only caller, for how this is surfaced to users. + * @param[in] pci_bus_id the PCI bus ID of the AMD GPU to guess an accel index for + * @param[in] physical_pci_bus_ids_sorted every physically present AMD GPU's PCI bus ID, sorted ascending (as + * returned by `hws::detail::enumerate_all_amd_gpu_pci_bus_ids()`) + * @param[in] accel_indices_sorted the accelerator indices pm_counters exposed, sorted ascending (as returned by + * `cray_pm_counters_hardware_sampler::discovered_accel_indices()`) + * @return the guessed accel index, or `std::nullopt` if @p pci_bus_id isn't found in @p physical_pci_bus_ids_sorted + * or the two lists don't have the same size (a topology count mismatch means the guess isn't safe to make, + * e.g. under a cgroup-isolated partial-node allocation) (`[[nodiscard]]`) + */ +[[nodiscard]] std::optional guess_accel_index(const std::string &pci_bus_id, + const std::vector &physical_pci_bus_ids_sorted, + const std::vector &accel_indices_sorted); + +/** + * @brief Build the YAML sub-block (under a `gpu_vendors:` mapping) describing one GPU vendor's accel[i] + * correlation guesses, e.g. for `"amd"` or `"nvidia"`. + * @details Vendor-agnostic: the caller is responsible for gathering @p visible_devices and + * @p physical_pci_bus_ids_sorted using the right backend (`hws::detail::amd_device_pci_bus_id()`/ + * `enumerate_all_amd_gpu_pci_bus_ids()` or their `nvidia_*` counterparts). Every entry in + * @p physical_pci_bus_ids_sorted and @p accel_indices_sorted is assumed to belong to @p vendor alone - + * don't call this with a mixed-vendor physical topology. + * @param[in] vendor the vendor name to use as the YAML mapping key (e.g. `"amd"`, `"nvidia"`) + * @param[in] visible_devices every visible GPU sampler of this vendor, as (local device index, PCI bus ID) pairs + * @param[in] physical_pci_bus_ids_sorted every physically present GPU of this vendor's PCI bus ID, sorted ascending + * @param[in] accel_indices_sorted the accelerator indices pm_counters exposed, sorted ascending + * @return the YAML sub-block, indented to sit directly under a ` gpu_vendors:\n` key (`[[nodiscard]]`) + */ +[[nodiscard]] std::string accel_correlation_yaml_block(const std::string &vendor, + const std::vector> &visible_devices, + const std::vector &physical_pci_bus_ids_sorted, + const std::vector &accel_indices_sorted); + +} // namespace hws::detail + +#endif // HWS_CRAY_PM_COUNTERS_UTILITY_HPP_ diff --git a/include/hws/gpu_amd/hardware_sampler.hpp b/include/hws/gpu_amd/hardware_sampler.hpp index 668cc9a..ce0e40f 100644 --- a/include/hws/gpu_amd/hardware_sampler.hpp +++ b/include/hws/gpu_amd/hardware_sampler.hpp @@ -116,6 +116,40 @@ class gpu_amd_hardware_sampler : public hardware_sampler { */ [[nodiscard]] const rocm_smi_temperature_samples &temperature_samples() const noexcept { return temperature_samples_; } + /** + * @brief Return the ROCm SMI device index this hardware sampler uses for all of its `rsmi_dev_*` calls. + * @details Resolved at construction time from the HIP-relative index passed to the constructor by matching + * PCI bus IDs, since ROCm SMI's own device enumeration is *not* affected by + * `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` the way HIP's is - the two can otherwise disagree + * about which physical device a given index refers to. Purely local/informational: don't assume + * this index is stable across processes or reruns; use `pci_bus_id()` to identify the actual + * physical device. + * @return the ROCm SMI device index (`[[nodiscard]]`) + */ + [[nodiscard]] std::uint32_t device_id() const noexcept { return device_id_; } + + /** + * @brief Return the HIP-relative device index this hardware sampler was constructed with (i.e. the index + * into the process's `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES`-filtered device list). + * @details Unlike `device_id()`, this is exactly the constructor argument, unresolved - the right value to + * report when identifying "the Nth GPU visible to this process/rank" (e.g. for a per-rank device + * list), as opposed to `pci_bus_id()`/`device_id()` which identify the physical device itself. + * @return the HIP-relative device index (`[[nodiscard]]`) + */ + [[nodiscard]] std::uint32_t hip_device_id() const noexcept { return hip_device_id_; } + + /** + * @brief Return the PCI bus ID (e.g. `"0000:c1:00.0"`) of the physical device this hardware sampler actually + * measures. + * @details Queried via `rsmi_dev_pci_id_get()` using the same `device_id()` index this sampler already uses + * for every other `rsmi_dev_*` call, so - unlike combining `device_id()` with a different API family + * (e.g. HIP's `hipDeviceGetPCIBusId()`) - this is guaranteed to identify the exact physical device + * being sampled, even if `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` causes ROCm SMI's and HIP's + * device enumerations to diverge. Matches the format used by `enumerate_all_amd_gpu_pci_bus_ids()`. + * @return the PCI bus ID (`[[nodiscard]]`) + */ + [[nodiscard]] std::string pci_bus_id() const; + /** * @copydoc hws::hardware_sampler::device_identification */ @@ -132,8 +166,12 @@ class gpu_amd_hardware_sampler : public hardware_sampler { */ void sampling_loop() final; - /// The ID of the device to sample. + /// The ROCm SMI device index to sample, resolved from hip_device_id_ via a PCI bus ID match (ROCm SMI's own + /// device enumeration isn't affected by HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES, unlike HIP's). std::uint32_t device_id_{}; + /// The HIP-relative device index this hardware sampler was constructed with; only used for the one HIP call + /// (hipGetDeviceProperties) that needs a HIP-space rather than a ROCm-SMI-space index. + std::uint32_t hip_device_id_{}; /// The general AMD GPU samples. rocm_smi_general_samples general_samples_{}; diff --git a/include/hws/gpu_amd/utility.hpp b/include/hws/gpu_amd/utility.hpp index def0937..c0670a0 100644 --- a/include/hws/gpu_amd/utility.hpp +++ b/include/hws/gpu_amd/utility.hpp @@ -17,11 +17,10 @@ #include // std::runtime_error #include // std::string +#include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) #include "hws/visible_gpu_device.hpp" // hws::detail::visible_gpu_device - - #include // std::vector #endif namespace hws::detail { @@ -74,6 +73,28 @@ namespace hws::detail { */ [[nodiscard]] std::string performance_level_to_string(rsmi_dev_perf_level_t perf_level); +/** + * @brief Return the PCI bus ID (e.g. `"0000:c1:00.0"`) of the AMD GPU device with the given HIP @p local_index. + * @details This is the same, stable identifier used by `enumerate_all_amd_gpu_pci_bus_ids()`, so the two can be + * matched against each other to locate a HIP-visible device within the full physical GPU topology. + * @param[in] local_index the local HIP device index + * @return the PCI bus ID (`[[nodiscard]]`) + */ +[[nodiscard]] std::string amd_device_pci_bus_id(int local_index); + +/** + * @brief Enumerate the PCI bus IDs of every AMD GPU physically present on the node, independent of any + * process-level device visibility filtering (e.g. `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES`). + * @details Reads the kernel's view of devices bound to the `amdgpu` driver directly from + * `/sys/bus/pci/drivers/amdgpu/`, since that sysfs directory - unlike the HIP runtime's device + * enumeration - isn't affected by per-process visible-device environment variables. It *can* still be + * restricted below the true physical device count in a batch job with kernel-level (cgroup) device + * isolation for a partial-node allocation, so callers must treat an unexpectedly low count as "topology + * unknown", not as ground truth. + * @return the sorted PCI bus IDs of all `amdgpu`-bound devices, or an empty vector if the directory doesn't exist + * or isn't readable (`[[nodiscard]]`) + */ +[[nodiscard]] std::vector enumerate_all_amd_gpu_pci_bus_ids(); #if defined(HWS_MPI_SUPPORT_ENABLED) diff --git a/include/hws/gpu_nvidia/hardware_sampler.hpp b/include/hws/gpu_nvidia/hardware_sampler.hpp index 59a5e31..aa14fce 100644 --- a/include/hws/gpu_nvidia/hardware_sampler.hpp +++ b/include/hws/gpu_nvidia/hardware_sampler.hpp @@ -117,6 +117,29 @@ class gpu_nvidia_hardware_sampler : public hardware_sampler { */ [[nodiscard]] const nvml_temperature_samples &temperature_samples() const noexcept { return temperature_samples_; } + /** + * @brief Return the CUDA device index this hardware sampler was constructed with. + * @details Purely local/informational: this index is only used once, at construction, to resolve the NVML + * device handle this sampler actually operates on (see `nvmlDeviceGetHandleByIndex`) - since + * `CUDA_VISIBLE_DEVICES` isn't respected by NVML's own device enumeration, this CUDA-relative index + * is **not** guaranteed to identify the same physical device as `hws::detail::nvidia_device_pci_bus_id(device_id())` + * would. Use `pci_bus_id()` to identify the actual physical device this sampler measures. + * @return the CUDA device index (`[[nodiscard]]`) + */ + [[nodiscard]] std::size_t device_id() const noexcept { return device_id_; } + + /** + * @brief Return the PCI bus ID (e.g. `"0000:c1:00.0"`) of the physical device this hardware sampler actually + * measures. + * @details Queried via `nvmlDeviceGetPciInfo_v3()` on this sampler's already-resolved NVML device handle, so - + * unlike combining `device_id()` with a different API family (e.g. CUDA's `cudaDeviceGetPCIBusId()`) + * - this is guaranteed to identify the exact physical device being sampled, even though + * `CUDA_VISIBLE_DEVICES` and NVML's device enumeration can otherwise diverge. Matches the format + * used by `enumerate_all_nvidia_gpu_pci_bus_ids()`. + * @return the PCI bus ID (`[[nodiscard]]`) + */ + [[nodiscard]] std::string pci_bus_id() const; + /** * @copydoc hws::hardware_sampler::device_identification */ @@ -135,6 +158,8 @@ class gpu_nvidia_hardware_sampler : public hardware_sampler { /// The device handle for the device to sample. detail::nvml_device_handle device_{}; + /// The CUDA device index this hardware sampler was constructed with. + std::size_t device_id_{}; /// The general NVIDIA GPU samples. nvml_general_samples general_samples_{}; diff --git a/include/hws/gpu_nvidia/utility.hpp b/include/hws/gpu_nvidia/utility.hpp index b0b3811..170d98e 100644 --- a/include/hws/gpu_nvidia/utility.hpp +++ b/include/hws/gpu_nvidia/utility.hpp @@ -18,11 +18,10 @@ #include // std::runtime_error #include // std::string +#include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) #include "hws/visible_gpu_device.hpp" // hws::detail::visible_gpu_device - - #include // std::vector #endif namespace hws::detail { @@ -69,6 +68,31 @@ namespace hws::detail { #endif +/** + * @brief Return the PCI bus ID (e.g. `"0000:c1:00.0"`) of the NVIDIA GPU device with the given CUDA @p local_index. + * @details This is the same, stable identifier used by `enumerate_all_nvidia_gpu_pci_bus_ids()`, so the two can be + * matched against each other to locate a CUDA-visible device within the full physical GPU topology. + * @param[in] local_index the local CUDA device index + * @return the PCI bus ID (`[[nodiscard]]`) + */ +[[nodiscard]] std::string nvidia_device_pci_bus_id(int local_index); + +/** + * @brief Enumerate the PCI bus IDs of every NVIDIA GPU physically present on the node, independent of any + * process-level device visibility filtering (e.g. `CUDA_VISIBLE_DEVICES`). + * @details Reads the kernel's view of devices bound to the `nvidia` driver directly from + * `/sys/bus/pci/drivers/nvidia/`, since that sysfs directory - unlike the CUDA runtime's device + * enumeration - isn't affected by per-process visible-device environment variables. It *can* still be + * restricted below the true physical device count in a batch job with kernel-level (cgroup) device + * isolation for a partial-node allocation, so callers must treat an unexpectedly low count as "topology + * unknown", not as ground truth. Untested against real hardware (no NVIDIA GH200/EX254n access) - the + * `amdgpu` counterpart this mirrors (`hws::detail::enumerate_all_amd_gpu_pci_bus_ids()`) is; double-check + * the `nvidia` driver directory name and symlink layout on first use. + * @return the sorted PCI bus IDs of all `nvidia`-bound devices, or an empty vector if the directory doesn't exist + * or isn't readable (`[[nodiscard]]`) + */ +[[nodiscard]] std::vector enumerate_all_nvidia_gpu_pci_bus_ids(); + #if defined(HWS_MPI_SUPPORT_ENABLED) /** diff --git a/include/hws/system_hardware_sampler.hpp b/include/hws/system_hardware_sampler.hpp index 21b3029..875b4eb 100644 --- a/include/hws/system_hardware_sampler.hpp +++ b/include/hws/system_hardware_sampler.hpp @@ -251,6 +251,21 @@ class system_hardware_sampler { * @param category the sample category */ void create_local_samplers(std::chrono::milliseconds sampling_interval, hws::sample_category category); + + /** + * @brief Generate a best-effort, UNVERIFIED YAML hint correlating each visible AMD GPU device with a Cray + * pm_counters `accel[i]` index, if `samplers_` holds both a `cray_pm_counters_hardware_sampler` and at + * least one `gpu_amd_hardware_sampler`. + * @details The generated YAML's own `note`/`verified` fields carry the caveat: this is an ordinal guess + * (`accel[i]` is assumed to number accelerators in ascending PCI bus address order among all + * physically present AMD GPUs), not a vendor-confirmed mapping - no HPE documentation defines this + * correspondence. Degrades gracefully (marks `topology_count_mismatch: true`, omits the guesses but + * still reports the raw PCI bus IDs) if the discovered `accel` count doesn't match the physical AMD + * GPU count, e.g. under a cgroup-isolated partial-node allocation. + * @return the YAML string, or an empty string if the prerequisites aren't met (backend(s) not compiled in, or + * no pm_counters/AMD GPU sampler present) (`[[nodiscard]]`) + */ + [[nodiscard]] std::string device_correlation_hints_as_yaml_string() const; }; } // namespace hws diff --git a/include/hws/utility.hpp b/include/hws/utility.hpp index 2737418..921f29e 100644 --- a/include/hws/utility.hpp +++ b/include/hws/utility.hpp @@ -19,6 +19,7 @@ #include // std::chrono::duration #include // std::trunc #include // std::size_t +#include // std::uint32_t #include // std::optional #include // std::runtime_error #include // std::string, std::stof, std::stod, std::stold @@ -255,6 +256,22 @@ template */ [[nodiscard]] std::string indent_lines(const std::string &text, std::string_view prefix); +/** + * @brief Format a PCI domain/bus/device triplet as the canonical Linux sysfs PCI bus ID string + * `"::.0"` (e.g. `"0000:c1:00.0"`), lowercase hex, 4/2/2 digits. + * @details The function is fixed to `0`: on multi-die/multi-partition accelerators (e.g. AMD MI300-series + * "partitions") the PCI function field is repurposed by the vendor's management library for + * partition/die identification, but the function seen by the OS/sysfs for the *device* itself is + * always `0` - so vendor-provided domain/bus/device values should be combined with a hardcoded `0` + * function here rather than a vendor-reported function value, to stay comparable with sysfs PCI bus IDs + * (see e.g. `hws::detail::enumerate_all_amd_gpu_pci_bus_ids()`/`enumerate_all_nvidia_gpu_pci_bus_ids()`). + * @param[in] domain the PCI domain + * @param[in] bus the PCI bus number + * @param[in] device the PCI device (slot) number + * @return the formatted PCI bus ID string (`[[nodiscard]]`) + */ +[[nodiscard]] std::string format_pci_bus_id(std::uint32_t domain, std::uint32_t bus, std::uint32_t device); + /*****************************************************************************************************/ /** other free functions **/ /*****************************************************************************************************/ diff --git a/scripts/hwmon_probe.pbs b/scripts/hwmon_probe.pbs new file mode 100644 index 0000000..76ba996 --- /dev/null +++ b/scripts/hwmon_probe.pbs @@ -0,0 +1,201 @@ +#!/bin/bash +#PBS -N hwmon_probe +#PBS -l select=1:node_type=mi300a:mpiprocs=1 +#PBS -l walltime=00:15:00 +#PBS -j oe +#PBS -q test + +# Site-specific: written for and tested on HLRS's "Hunter" system (HPE Cray EX, PBS Pro scheduler). The +# #PBS directives above (queue name "test", node_type selector "mi300a") are specific to that system and +# will need adapting - or replacing with the equivalent directives for your own scheduler - before use +# elsewhere. Dumps hwmon (amd_energy, amdgpu, cpufreq) and, if present, /sys/cray/pm_counters diagnostics +# for a single compute node. + +set -x + +echo "===================================================================" +echo " Job: $PBS_JOBID" +echo " Host: $(hostname -f)" +echo " Node: $(cat $PBS_NODEFILE 2>/dev/null)" +echo " Date: $(date)" +echo "===================================================================" + +# --------------------------------------------------------------------------- +# Helper: dump every readable *_input / *_label / *_alarm file below a hwmon +# directory, without ever failing the job on a permission error. +# --------------------------------------------------------------------------- +dump_hwmon_dir() { + local hw="$1" + for f in "$hw"/*_input "$hw"/*_label "$hw"/*_average "$hw"/*_max "$hw"/*_crit; do + [ -e "$f" ] || continue + local val + val=$(cat "$f" 2>/dev/null) + if [ -n "$val" ]; then + printf " %-28s = %s\n" "$(basename "$f")" "$val" + else + printf " %-28s = \n" "$(basename "$f")" + fi + done +} + +echo +echo "### --- Alle hwmon-Devices und deren Rohwerte -----------------------" +for hw in /sys/class/hwmon/hwmon*; do + [ -d "$hw" ] || continue + name=$(cat "$hw/name" 2>/dev/null) + echo + echo "-- $hw (name: $name) --" + dump_hwmon_dir "$hw" +done + +echo +echo "### --- /sys/cray/pm_counters: komplettes Layout + Rohwerte ---------" +if [ -d /sys/cray/pm_counters ]; then + echo "-- Verzeichnisstruktur (find) --" + find /sys/cray/pm_counters -mindepth 1 2>/dev/null + + echo + echo "-- Inhalt aller regulaeren Dateien --" + for f in $(find /sys/cray/pm_counters -mindepth 1 -type f 2>/dev/null | sort); do + val=$(cat "$f" 2>/dev/null) + if [ -n "$val" ]; then + printf " %-50s = %s\n" "$f" "$val" + else + printf " %-50s = \n" "$f" + fi + done + + echo + echo "-- zwei Messpunkte fuer alle numerischen (vermutlich Energie-)Zaehler --" + # Falls ein Wert wie "123456 J" oder "110 W" statt reiner Zahl geliefert wird, wird nur die + # fuehrende Zahl fuer den Test/die Arithmetik verwendet (Einheit wird zusaetzlich mitgeloggt). + declare -A pm_t0 + declare -A pm_t0_raw + pm_files=$(find /sys/cray/pm_counters -mindepth 1 -type f 2>/dev/null | sort) + t0=$(date +%s.%N) + for f in $pm_files; do + val_raw=$(cat "$f" 2>/dev/null) + val="${val_raw%% *}" + [[ "$val" =~ ^[0-9]+$ ]] || continue + pm_t0[$f]=$val + pm_t0_raw[$f]=$val_raw + done + + sleep_interval=1 + sleep "$sleep_interval" + + t1=$(date +%s.%N) + dt=$(python3 -c "print($t1-$t0)") + for f in "${!pm_t0[@]}"; do + v1_raw=$(cat "$f" 2>/dev/null) + v1="${v1_raw%% *}" + [[ "$v1" =~ ^[0-9]+$ ]] || continue + delta=$((v1 - ${pm_t0[$f]})) + echo " $f: t0=${pm_t0_raw[$f]} t1=${v1_raw} delta=${delta} dt=${dt}s" + done +else + echo "kein /sys/cray/pm_counters vorhanden" +fi + +echo +echo "### --- amd_energy: zwei Messpunkte fuer Delta/Power-Berechnung -----" +energy_hw="" +for hw in /sys/class/hwmon/hwmon*; do + [ -d "$hw" ] || continue + if [ "$(cat "$hw/name" 2>/dev/null)" = "amd_energy" ]; then + energy_hw="$hw" + break + fi +done + +if [ -n "$energy_hw" ]; then + echo "amd_energy gefunden unter: $energy_hw" + echo + echo "-- verfuegbare Energie-Kanaele --" + for lbl in "$energy_hw"/energy*_label; do + [ -e "$lbl" ] || continue + idx=$(basename "$lbl" | sed -E 's/energy([0-9]+)_label/\1/') + echo " energy${idx}: $(cat "$lbl" 2>/dev/null) (input file: energy${idx}_input, Einheit: uJ, monoton steigend)" + done + + echo + echo "-- Messung 1 (t0) --" + declare -A e_t0 + t0=$(date +%s.%N) + for inp in "$energy_hw"/energy*_input; do + [ -e "$inp" ] || continue + idx=$(basename "$inp" | sed -E 's/energy([0-9]+)_input/\1/') + e_t0[$idx]=$(cat "$inp" 2>/dev/null) + echo " energy${idx}_input = ${e_t0[$idx]} uJ" + done + + sleep_interval=1 + echo + echo "-- warte ${sleep_interval}s --" + sleep "$sleep_interval" + + echo + echo "-- Messung 2 (t1) --" + t1=$(date +%s.%N) + dt=$(echo "$t1 - $t0" | bc -l 2>/dev/null || python3 -c "print($t1-$t0)") + for inp in "$energy_hw"/energy*_input; do + [ -e "$inp" ] || continue + idx=$(basename "$inp" | sed -E 's/energy([0-9]+)_input/\1/') + e_t1=$(cat "$inp" 2>/dev/null) + delta_uj=$((e_t1 - ${e_t0[$idx]})) + # Power[W] = delta_energy[uJ] / 1e6 / delta_t[s] + power_w=$(python3 -c "print(f'{${delta_uj} / 1000000.0 / ${dt}:.3f}')" 2>/dev/null) + echo " energy${idx}_input = ${e_t1} uJ delta = ${delta_uj} uJ dt = ${dt}s -> Power = ${power_w} W" + done +else + echo "kein amd_energy hwmon gefunden" +fi + +echo +echo "### --- amdgpu hwmon: Temperaturen / Power (falls vorhanden) --------" +for hw in /sys/class/hwmon/hwmon*; do + [ -d "$hw" ] || continue + if [ "$(cat "$hw/name" 2>/dev/null)" = "amdgpu" ]; then + echo + echo "-- $hw --" + dump_hwmon_dir "$hw" + fi +done + +echo +echo "### --- cpufreq: aktueller Takt pro Online-CPU -----------------------" +governor_seen="" +for cpu in /sys/devices/system/cpu/cpu[0-9]*; do + [ -d "$cpu/cpufreq" ] || continue + cpuid=$(basename "$cpu") + cur=$(cat "$cpu/cpufreq/scaling_cur_freq" 2>/dev/null) + gov=$(cat "$cpu/cpufreq/scaling_governor" 2>/dev/null) + drv=$(cat "$cpu/cpufreq/scaling_driver" 2>/dev/null) + [ -z "$governor_seen" ] && governor_seen="driver=$drv governor=$gov" + echo " $cpuid: scaling_cur_freq=${cur:-} kHz" +done +echo +echo " (Treiber/Governor, erster gefundener Core: $governor_seen)" + +echo +echo "-- Mittelwert ueber alle Online-CPUs (kHz -> MHz) --" +python3 - <<'EOF' +import glob +vals = [] +for p in glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq/scaling_cur_freq"): + try: + with open(p) as f: + vals.append(int(f.read().strip())) + except Exception: + pass +if vals: + avg_khz = sum(vals) / len(vals) + print(f" n_cores={len(vals)} avg={avg_khz/1000.0:.1f} MHz min={min(vals)/1000.0:.1f} MHz max={max(vals)/1000.0:.1f} MHz") +else: + print(" keine scaling_cur_freq Werte lesbar") +EOF + +echo +echo "===================================================================" +echo " Fertig: $(date)" +echo "===================================================================" diff --git a/src/hws/cpu/CMakeLists.txt b/src/hws/cpu/CMakeLists.txt index 0896902..c8b241c 100644 --- a/src/hws/cpu/CMakeLists.txt +++ b/src/hws/cpu/CMakeLists.txt @@ -22,18 +22,33 @@ if (HWS_FREE_FOUND) target_compile_definitions(${HWS_LIBRARY_NAME} PUBLIC HWS_VIA_FREE_ENABLED) endif () +## the turbostat "-i" measurement interval, in seconds +## NOTE: very short intervals (e.g. this option's default of 0.001, kept for backwards +## compatibility with the previous hardcoded value and any sudoers/paper-reproducibility setups +## that already depend on it) have been observed to produce physically implausible readings +## (multi-kW package power, multi-GHz core clocks) on heavily loaded, many-core machines, +## especially with older turbostat builds -- turbostat's own internal timing calculation degrades +## under load for short intervals. 1 second was the smallest value that stayed reliable across +## every machine/turbostat-version combination we tested; consider overriding this to "1" unless +## you specifically need to match the old default's exact (sudoers-permitted) command line. +set(HWS_TURBOSTAT_INTERVAL "0.001" CACHE STRING "The interval in seconds ('sec.subsec', passed to turbostat's -i flag) turbostat itself measures over for a single CPU power/frequency sample.") +if (NOT ${HWS_TURBOSTAT_INTERVAL} MATCHES "^[0-9]+(\\.[0-9]+)?$" OR ${HWS_TURBOSTAT_INTERVAL} EQUAL 0) + message(FATAL_ERROR "The HWS_TURBOSTAT_INTERVAL must be a positive number (e.g. \"1\" or \"0.5\"), but is \"${HWS_TURBOSTAT_INTERVAL}\"!") +endif () +message(STATUS "Setting the turbostat measurement interval to ${HWS_TURBOSTAT_INTERVAL}s.") + ## check whether turbostat could be found -> used for the CPU targets as well as for ALL host measurements ## -> checked even if no CPU targets where provided find_program(HWS_TURBOSTAT_FOUND turbostat) if (HWS_TURBOSTAT_FOUND) ## check if the turbostat command works as intended - execute_process(COMMAND sudo -n turbostat -n 1 -i 0.001 -S -q + execute_process(COMMAND sudo -n turbostat -n 1 -i ${HWS_TURBOSTAT_INTERVAL} -S -q RESULT_VARIABLE HWS_TURBOSTAT_WITH_ROOT_N OUTPUT_QUIET ERROR_QUIET) if (HWS_TURBOSTAT_WITH_ROOT_N EQUAL 0) ## can execute with root - execute_process(COMMAND sudo turbostat -n 1 -i 0.001 -S -q + execute_process(COMMAND sudo turbostat -n 1 -i ${HWS_TURBOSTAT_INTERVAL} -S -q RESULT_VARIABLE HWS_TURBOSTAT_WITH_ROOT OUTPUT_QUIET ERROR_QUIET) @@ -50,7 +65,7 @@ if (HWS_TURBOSTAT_FOUND) endif () else () ## check if turbostat can be executed without root -> potential less data - execute_process(COMMAND turbostat -n 1 -i 0.001 -S -q + execute_process(COMMAND turbostat -n 1 -i ${HWS_TURBOSTAT_INTERVAL} -S -q RESULT_VARIABLE HWS_TURBOSTAT_WITHOUT_ROOT OUTPUT_QUIET ERROR_QUIET) @@ -65,6 +80,10 @@ if (HWS_TURBOSTAT_FOUND) message(STATUS "Disabling turbostat support!") endif () endif () + + if (HWS_TURBOSTAT_EXECUTION_TYPE) + target_compile_definitions(${HWS_LIBRARY_NAME} PUBLIC HWS_TURBOSTAT_INTERVAL="${HWS_TURBOSTAT_INTERVAL}") + endif () endif () # check of any CPU related utility could be found diff --git a/src/hws/cpu/hardware_sampler.cpp b/src/hws/cpu/hardware_sampler.cpp index e7efe32..d9dfd73 100644 --- a/src/hws/cpu/hardware_sampler.cpp +++ b/src/hws/cpu/hardware_sampler.cpp @@ -39,7 +39,22 @@ cpu_hardware_sampler::cpu_hardware_sampler(const sample_category category) : cpu_hardware_sampler{ HWS_SAMPLING_INTERVAL, category } { } cpu_hardware_sampler::cpu_hardware_sampler(const std::chrono::milliseconds sampling_interval, const sample_category category) : - hardware_sampler{ sampling_interval, category } { } + hardware_sampler{ sampling_interval, category } { +#if defined(HWS_VIA_TURBOSTAT_ENABLED) + // turbostat itself blocks for HWS_TURBOSTAT_INTERVAL seconds per invocation -> if that's + // longer than the requested sampling interval, the turbostat backend will dominate and the + // achieved cadence will be closer to HWS_TURBOSTAT_INTERVAL than to sampling_interval + const double turbostat_interval_seconds = std::stod(HWS_TURBOSTAT_INTERVAL); + if (std::chrono::duration(sampling_interval).count() < turbostat_interval_seconds) { + std::cerr << fmt::format( + "Warning: the requested CPU sampling interval ({}) is shorter than turbostat's own " + "measurement interval ({}s, set via HWS_TURBOSTAT_INTERVAL) -> the effective sampling " + "cadence while the turbostat backend is active will be closer to {}s per sample.\n", + sampling_interval, turbostat_interval_seconds, turbostat_interval_seconds) + << std::endl; + } +#endif +} cpu_hardware_sampler::~cpu_hardware_sampler() { try { @@ -159,21 +174,28 @@ void cpu_hardware_sampler::sampling_loop() { // get header information #if defined(HWS_VIA_TURBOSTAT_ROOT) // run with sudo - const std::string_view turbostat_command_line = "sudo turbostat -n 1 -i 0.001 -S -q"; + const std::string turbostat_command_line = fmt::format("sudo turbostat -n 1 -i {} -S -q", HWS_TURBOSTAT_INTERVAL); #else // run without sudo - const std::string_view turbostat_command_line = "turbostat -n 1 -i 0.001 -S -q"; + const std::string turbostat_command_line = fmt::format("turbostat -n 1 -i {} -S -q", HWS_TURBOSTAT_INTERVAL); #endif - { + // turbostat feeds all of the following sample categories -> only pay its (potentially multi- + // second, see HWS_TURBOSTAT_INTERVAL) cost per invocation if at least one of them was requested + const bool turbostat_needed = this->sample_category_enabled(sample_category::general | sample_category::clock | sample_category::power | sample_category::temperature | sample_category::gfx | sample_category::idle_state); + + if (turbostat_needed) { // run turbostat const std::string turbostat_output = detail::run_subprocess(turbostat_command_line); - // retrieve the turbostat data + // retrieve the turbostat data; turbostat may prepend diagnostic messages (e.g. "Disabling + // Low Power Idle CPU output") on stderr before its actual header/value lines, and stdout + // and stderr are read through the same combined handle -> the header/value pair is always + // the *last* two lines, never necessarily the first two const std::vector data = detail::split(detail::trim(turbostat_output), '\n'); assert((data.size() >= 2) && "Must read at least two lines!"); - const std::vector header = detail::split(data[0], '\t'); - const std::vector values = detail::split(data[1], '\t'); + const std::vector header = detail::split(data[data.size() - 2], '\t'); + const std::vector values = detail::split(data[data.size() - 1], '\t'); for (std::size_t i = 0; i < header.size(); ++i) { // general samples @@ -387,6 +409,12 @@ void cpu_hardware_sampler::sampling_loop() { // while (!this->has_sampling_stopped()) { + // reference point for this tick's deadline; captured before any of the (potentially slow, + // e.g. turbostat blocking for HWS_TURBOSTAT_INTERVAL seconds) sampling work below so that + // sleep_until() only waits out whatever time is left of the requested interval instead of + // unconditionally adding another full sampling_interval() on top + const auto tick_start = std::chrono::steady_clock::now(); + // only sample values if the sampler currently isn't paused if (this->is_sampling()) { // add current time point @@ -413,15 +441,16 @@ void cpu_hardware_sampler::sampling_loop() { #endif #if defined(HWS_VIA_TURBOSTAT_ENABLED) - { + if (turbostat_needed) { // run turbostat const std::string turbostat_output = detail::run_subprocess(turbostat_command_line); - // retrieve the turbostat data + // retrieve the turbostat data; see the comment at the initial turbostat call above + // for why the header/value pair must be taken from the *last* two lines const std::vector data = detail::split(detail::trim(turbostat_output), '\n'); assert((data.size() >= 2) && "Must read at least two lines!"); - const std::vector header = detail::split(data[0], '\t'); - const std::vector values = detail::split(data[1], '\t'); + const std::vector header = detail::split(data[data.size() - 2], '\t'); + const std::vector values = detail::split(data[data.size() - 1], '\t'); // add values to the respective sample entries for (std::size_t i = 0; i < header.size(); ++i) { @@ -630,8 +659,11 @@ void cpu_hardware_sampler::sampling_loop() { #endif } - // wait for the sampling interval to pass to retrieve the next sample - std::this_thread::sleep_for(this->sampling_interval()); + // wait until this tick's deadline to retrieve the next sample; if the sampling work above + // already took longer than sampling_interval() (e.g. because the turbostat backend blocked + // for HWS_TURBOSTAT_INTERVAL seconds), the deadline is already in the past and this returns + // immediately instead of unconditionally adding another full sampling_interval() of delay + std::this_thread::sleep_until(tick_start + this->sampling_interval()); } } diff --git a/src/hws/cpu/utility.cpp b/src/hws/cpu/utility.cpp index e246282..553771f 100644 --- a/src/hws/cpu/utility.cpp +++ b/src/hws/cpu/utility.cpp @@ -23,8 +23,11 @@ namespace hws::detail { std::string run_subprocess(const std::string_view cmd_line) { - // search PATH for executable - constexpr int options = subprocess_option_e::subprocess_option_search_user_path; + // search PATH for executable; combine stdout and stderr into a single handle since we only + // ever read via subprocess_stdout() below -> without this flag, subprocess_create() opens an + // unread pipe for stderr that can fill up and block the child (and therefore also + // subprocess_join() below) forever if the child ever writes a warning/error to stderr + constexpr int options = subprocess_option_e::subprocess_option_search_user_path | subprocess_option_e::subprocess_option_combined_stdout_stderr; constexpr static std::string::size_type buffer_size = 4096; // extract the separate command line arguments diff --git a/src/hws/cray_pm_counters/CMakeLists.txt b/src/hws/cray_pm_counters/CMakeLists.txt new file mode 100644 index 0000000..54125c7 --- /dev/null +++ b/src/hws/cray_pm_counters/CMakeLists.txt @@ -0,0 +1,29 @@ +## Authors: Alexander Van Craen +## Copyright (C): 2024-today All Rights Reserved +## License: This file is released under the MIT license. +## See the LICENSE.md file in the project root for full license information. +######################################################################################################################## + +# pm_counters is a Linux-only (Cray/HPE) sysfs interface, not a linkable library -> nothing to find_package/find_program. +# Availability of /sys/cray/pm_counters is checked at runtime (build host and target host may differ, e.g. cross-compiling +# on a login node for execution on a compute node), so this backend is always compiled in on Linux. +if (NOT UNIX) + if (HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING MATCHES "ON") + message(SEND_ERROR "Cray pm_counters sampling was explicitly requested but is only available on Linux!") + else () + message(STATUS "Cray pm_counters sampling is only available on Linux. Hardware sampling via pm_counters disabled.") + endif () + return() +endif () +message(STATUS "Enable sampling of node power/energy via /sys/cray/pm_counters (availability checked at runtime).") + +# add source file to source file list +target_sources(${HWS_LIBRARY_NAME} PRIVATE + $) + +# add compile definition +target_compile_definitions(${HWS_LIBRARY_NAME} PUBLIC HWS_FOR_CRAY_PM_COUNTERS_ENABLED) diff --git a/src/hws/cray_pm_counters/hardware_sampler.cpp b/src/hws/cray_pm_counters/hardware_sampler.cpp new file mode 100644 index 0000000..036fd2e --- /dev/null +++ b/src/hws/cray_pm_counters/hardware_sampler.cpp @@ -0,0 +1,215 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + */ + +#include "hws/cray_pm_counters/hardware_sampler.hpp" + +#include "hws/cray_pm_counters/pm_counters_samples.hpp" // hws::{cray_pm_counters_general_samples, cray_pm_counters_power_samples} +#include "hws/cray_pm_counters/utility.hpp" // hws::detail::{pm_counters_available, list_pm_counter_files, pm_counter_key, read_pm_counter_raw, read_pm_counter_reading, pm_counter_reading, is_energy_counter_key, is_power_counter_key, accel_indices_from_counter_keys} +#include "hws/hardware_sampler.hpp" // hws::hardware_sampler +#include "hws/sample_category.hpp" // hws::sample_category +#include "hws/utility.hpp" // hws::detail::time_points_to_epoch + +#include "fmt/chrono.h" // direct formatting of std::chrono types +#include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join + +#include // std::max +#include // std::chrono::{steady_clock, milliseconds, duration_cast} +#include // std::exception, std::terminate +#include // std::filesystem::path +#include // std::ios_base +#include // std::cerr, std::endl +#include // std::optional +#include // std::ostream +#include // std::runtime_error +#include // std::string +#include // std::this_thread +#include // std::vector + +namespace hws { + +cray_pm_counters_hardware_sampler::cray_pm_counters_hardware_sampler(const sample_category category) : + cray_pm_counters_hardware_sampler{ HWS_SAMPLING_INTERVAL, category } { } + +cray_pm_counters_hardware_sampler::cray_pm_counters_hardware_sampler(const std::chrono::milliseconds sampling_interval, const sample_category category) : + hardware_sampler{ sampling_interval, category } { } + +cray_pm_counters_hardware_sampler::~cray_pm_counters_hardware_sampler() { + try { + // if this hardware sampler is still sampling, stop it + if (this->has_sampling_started() && !this->has_sampling_stopped()) { + this->stop_sampling(); + } + } catch (const std::exception &e) { + std::cerr << e.what() << std::endl; + std::terminate(); + } +} + +void cray_pm_counters_hardware_sampler::sample_once() { + if (this->sample_category_enabled(sample_category::general)) { + if (!general_samples_.metadata_.has_value() && !general_paths_.empty()) { + general_samples_.metadata_ = cray_pm_counters_general_samples::metadata_type{}; + } + for (const auto &[key, path] : general_paths_) { + general_samples_.metadata_.value()[key].push_back(detail::read_pm_counter_raw(path)); + } + } + + if (this->sample_category_enabled(sample_category::power)) { + if (!power_samples_.energy_counters_.has_value() && !energy_counter_paths_.empty()) { + power_samples_.energy_counters_ = cray_pm_counters_power_samples::counter_map_type{}; + power_samples_.energy_timestamps_us_ = cray_pm_counters_power_samples::counter_map_type{}; + } + if (!power_samples_.power_counters_.has_value() && !power_counter_paths_.empty()) { + power_samples_.power_counters_ = cray_pm_counters_power_samples::counter_map_type{}; + power_samples_.power_timestamps_us_ = cray_pm_counters_power_samples::counter_map_type{}; + } + for (const auto &[key, path] : energy_counter_paths_) { + const std::optional reading = detail::read_pm_counter_reading(path); + if (reading.has_value()) { + power_samples_.energy_counters_.value()[key].push_back(reading->value); + if (reading->timestamp_us.has_value()) { + power_samples_.energy_timestamps_us_.value()[key].push_back(reading->timestamp_us.value()); + } + } + } + for (const auto &[key, path] : power_counter_paths_) { + const std::optional reading = detail::read_pm_counter_reading(path); + if (reading.has_value()) { + power_samples_.power_counters_.value()[key].push_back(reading->value); + if (reading->timestamp_us.has_value()) { + power_samples_.power_timestamps_us_.value()[key].push_back(reading->timestamp_us.value()); + } + } + } + } +} + +std::optional cray_pm_counters_hardware_sampler::hardware_tick_period() const { + // read directly from the discovered path rather than general_samples_.metadata_: the latter is only populated + // when sample_category::general is enabled, but throttling to the hardware's real rate must apply regardless + // of which sample categories the caller requested (e.g. sample_category::power alone). + const auto it = general_paths_.find("raw_scan_hz"); + if (it == general_paths_.cend()) { + return std::nullopt; + } + + const std::string content = detail::read_pm_counter_raw(it->second); + if (content.empty()) { + return std::nullopt; + } + + try { + const double hz = detail::convert_to(content); + if (hz <= 0.0) { + return std::nullopt; + } + return std::chrono::milliseconds{ static_cast(1000.0 / hz) }; + } catch (const std::exception &) { + return std::nullopt; + } +} + +void cray_pm_counters_hardware_sampler::sampling_loop() { + // + // discover the available pm_counters files once and classify them by name into energy counters, power counters, + // and general metadata; the exact set of files is Cray-generation specific, so every file below + // /sys/cray/pm_counters is picked up dynamically instead of relying on hardcoded names. All three categories + // are re-read every tick (including metadata like "freshness", which HPE's own consistency check relies on + // changing - see cray_pm_counters_general_samples). + // + + this->add_time_point(std::chrono::steady_clock::now()); + + for (const std::filesystem::path &file : detail::list_pm_counter_files()) { + const std::string key = detail::pm_counter_key(file); + + if (detail::is_energy_counter_key(key)) { + energy_counter_paths_.emplace(key, file); + } else if (detail::is_power_counter_key(key)) { + power_counter_paths_.emplace(key, file); + } else { + general_paths_.emplace(key, file); + } + } + + this->sample_once(); + + // pm_counters itself only updates at raw_scan_hz (10 Hz on Hunter); polling faster than that only yields + // duplicate values (and needlessly hammers sysfs), so never poll faster than the hardware's own rate - even if + // the globally configured sampling interval is shorter. Read dynamically rather than hardcoding 10 Hz, since + // raw_scan_hz can differ across Cray/HPE generations. + const std::optional hw_tick_period = this->hardware_tick_period(); + const std::chrono::milliseconds effective_interval = hw_tick_period.has_value() ? std::max(this->sampling_interval(), hw_tick_period.value()) : this->sampling_interval(); + if (hw_tick_period.has_value() && effective_interval > this->sampling_interval()) { + this->add_event(fmt::format("cray_pm_counters: throttled sampling interval from {} to {} to match the hardware's raw_scan_hz", this->sampling_interval(), effective_interval)); + } + + // + // loop until stop_sampling() is called + // + + while (!this->has_sampling_stopped()) { + // only sample values if the sampler currently isn't paused + if (this->is_sampling()) { + // add current time point + this->add_time_point(std::chrono::steady_clock::now()); + + this->sample_once(); + } + + // wait for the (possibly hardware-throttled) sampling interval to pass to retrieve the next sample + std::this_thread::sleep_for(effective_interval); + } +} + +std::string cray_pm_counters_hardware_sampler::device_identification() const { + return "cray_pm_counters_device"; +} + +std::vector cray_pm_counters_hardware_sampler::discovered_accel_indices() const { + if (!power_samples_.get_energy_counters().has_value()) { + return {}; + } + std::vector keys{}; + for (const auto &entry : power_samples_.get_energy_counters().value()) { + keys.push_back(entry.first); + } + return detail::accel_indices_from_counter_keys(keys); +} + +std::string cray_pm_counters_hardware_sampler::samples_only_as_yaml_string() const { + // check whether it's safe to generate the YAML entry + if (this->is_sampling()) { + throw std::runtime_error{ "Can't create the final YAML entry if the hardware sampler is still running!" }; + } + + return fmt::format("{}{}" + "{}", + general_samples_.generate_yaml_string(), + general_samples_.has_samples() ? "\n" : "", + power_samples_.generate_yaml_string()); +} + +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_hardware_sampler &sampler) { + if (sampler.is_sampling()) { + out.setstate(std::ios_base::failbit); + return out; + } else { + return out << fmt::format("sampling interval: {}\n" + "time points: [{}]\n\n" + "general samples:\n{}\n\n" + "power samples:\n{}", + sampler.sampling_interval(), + fmt::join(detail::time_points_to_epoch(sampler.sampling_time_points()), ", "), + sampler.general_samples(), + sampler.power_samples()); + } +} + +} // namespace hws diff --git a/src/hws/cray_pm_counters/pm_counters_samples.cpp b/src/hws/cray_pm_counters/pm_counters_samples.cpp new file mode 100644 index 0000000..486ce86 --- /dev/null +++ b/src/hws/cray_pm_counters/pm_counters_samples.cpp @@ -0,0 +1,141 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + */ + +#include "hws/cray_pm_counters/pm_counters_samples.hpp" + +#include "hws/cray_pm_counters/utility.hpp" // hws::detail::pm_counters_root +#include "hws/utility.hpp" // hws::detail::map_entry_to_string + +#include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join + +#include // std::sort +#include // std::uint64_t +#include // std::optional +#include // std::ostream +#include // std::string +#include // std::unordered_map +#include // std::vector + +namespace hws { + +namespace { + +// unordered_map has no defined iteration order; sort keys so the generated YAML is reproducible across runs. +template +[[nodiscard]] std::vector sorted_keys(const MapType &map) { + std::vector keys{}; + keys.reserve(map.size()); + for (const auto &[key, value] : map) { + keys.push_back(key); + } + std::sort(keys.begin(), keys.end()); + return keys; +} + +} // namespace + +//*************************************************************************************************************************************// +// general samples // +//*************************************************************************************************************************************// + +bool cray_pm_counters_general_samples::has_samples() const { + return this->metadata_.has_value(); +} + +std::string cray_pm_counters_general_samples::generate_yaml_string() const { + // if no samples are available, return an empty string + if (!this->has_samples()) { + return ""; + } + + std::string str{ "general:\n" }; + const std::string root = detail::pm_counters_root().generic_string(); + + if (this->metadata_.has_value()) { + for (const std::string &key : sorted_keys(this->metadata_.value())) { + str += fmt::format(" {}:\n" + " unit: \"string\"\n" + " source: \"{}/{}\"\n" + " values: [{}]\n", + key, root, key, fmt::join(detail::quote(this->metadata_.value().at(key)), ", ")); + } + } + + return str; +} + +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_general_samples &samples) { + return out << fmt::format("metadata: {}", detail::map_entry_to_string(samples.get_metadata())); +} + +//*************************************************************************************************************************************// +// power samples // +//*************************************************************************************************************************************// + +bool cray_pm_counters_power_samples::has_samples() const { + return this->energy_counters_.has_value() || this->power_counters_.has_value(); +} + +std::string cray_pm_counters_power_samples::generate_yaml_string() const { + // if no samples are available, return an empty string + if (!this->has_samples()) { + return ""; + } + + std::string str{ "power:\n" }; + const std::string root = detail::pm_counters_root().generic_string(); + + // look up the per-sample timestamps (if any) recorded for the counter map key; nullptr if none are available + const auto find_timestamps = [](const std::optional ×tamps, const std::string &key) -> const std::vector * { + if (!timestamps.has_value()) { + return nullptr; + } + const auto it = timestamps.value().find(key); + return it != timestamps.value().cend() ? &it->second : nullptr; + }; + + if (this->energy_counters_.has_value()) { + for (const std::string &key : sorted_keys(this->energy_counters_.value())) { + str += fmt::format(" {}:\n" + " unit: \"J (cumulative)\"\n" + " source: \"{}/{}\"\n" + " values: [{}]\n", + key, root, key, fmt::join(this->energy_counters_.value().at(key), ", ")); + if (const std::vector *timestamps = find_timestamps(this->energy_timestamps_us_, key); timestamps != nullptr) { + str += fmt::format(" timestamps_us: [{}]\n", fmt::join(*timestamps, ", ")); + } + } + } + if (this->power_counters_.has_value()) { + for (const std::string &key : sorted_keys(this->power_counters_.value())) { + str += fmt::format(" {}:\n" + " unit: \"W (instantaneous)\"\n" + " source: \"{}/{}\"\n" + " values: [{}]\n", + key, root, key, fmt::join(this->power_counters_.value().at(key), ", ")); + if (const std::vector *timestamps = find_timestamps(this->power_timestamps_us_, key); timestamps != nullptr) { + str += fmt::format(" timestamps_us: [{}]\n", fmt::join(*timestamps, ", ")); + } + } + } + + return str; +} + +std::ostream &operator<<(std::ostream &out, const cray_pm_counters_power_samples &samples) { + return out << fmt::format("energy_counters: {}\n" + "energy_timestamps_us: {}\n" + "power_counters: {}\n" + "power_timestamps_us: {}", + detail::map_entry_to_string(samples.get_energy_counters()), + detail::map_entry_to_string(samples.get_energy_timestamps_us()), + detail::map_entry_to_string(samples.get_power_counters()), + detail::map_entry_to_string(samples.get_power_timestamps_us())); +} + +} // namespace hws diff --git a/src/hws/cray_pm_counters/utility.cpp b/src/hws/cray_pm_counters/utility.cpp new file mode 100644 index 0000000..9d0d732 --- /dev/null +++ b/src/hws/cray_pm_counters/utility.cpp @@ -0,0 +1,239 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + */ + +#include "hws/cray_pm_counters/utility.hpp" + +#include "hws/utility.hpp" // hws::detail::{trim, to_lower_case, split, is_integer, convert_to} + +#include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join + +#include // std::sort, std::unique +#include // std::size_t +#include // std::uint64_t +#include // std::getenv +#include // std::filesystem::{path, directory_entry, exists, is_directory, recursive_directory_iterator, relative} +#include // std::ifstream +#include // std::optional, std::nullopt +#include // std::ostringstream +#include // std::string +#include // std::string_view +#include // std::error_code +#include // std::pair +#include // std::vector + +namespace hws::detail { + +std::filesystem::path pm_counters_root() { + // allow overriding the pm_counters root for local testing (e.g. against a synthetic directory off a Cray system) + if (const char *override_path = std::getenv("HWS_PM_COUNTERS_PATH"); override_path != nullptr) { + return std::filesystem::path{ override_path }; + } + return std::filesystem::path{ default_pm_counters_root }; +} + +bool pm_counters_available() { + const std::filesystem::path root = pm_counters_root(); + std::error_code ec{}; + return std::filesystem::exists(root, ec) && std::filesystem::is_directory(root, ec); +} + +std::vector list_pm_counter_files() { + std::vector files{}; + + if (!pm_counters_available()) { + return files; + } + + std::error_code ec{}; + for (const std::filesystem::directory_entry &entry : std::filesystem::recursive_directory_iterator(pm_counters_root(), std::filesystem::directory_options::skip_permission_denied, ec)) { + if (entry.is_regular_file(ec)) { + files.push_back(entry.path()); + } + } + + return files; +} + +std::string pm_counter_key(const std::filesystem::path &file) { + std::error_code ec{}; + std::filesystem::path relative = std::filesystem::relative(file, pm_counters_root(), ec); + if (ec) { + relative = file.filename(); + } + + // '/' has no special meaning in YAML mapping keys, so a nested file's relative path (e.g. "accel0/energy") is + // used as-is; this also means the key always doubles as the real path relative to pm_counters_root() for the + // "source:" field in the generated YAML. All confirmed real HPE pm_counters layouts (2014 CUG slides, 2024 CUG + // slides, a real Hunter node) are flat (no subdirectories), so this rarely matters in practice. + return relative.generic_string(); +} + +std::string read_pm_counter_raw(const std::filesystem::path &file) { + std::ifstream stream{ file }; + if (!stream.is_open()) { + return ""; + } + + std::ostringstream content{}; + content << stream.rdbuf(); + return std::string{ detail::trim(content.str()) }; +} + +std::optional read_pm_counter_reading(const std::filesystem::path &file) { + const std::string content = read_pm_counter_raw(file); + if (content.empty()) { + return std::nullopt; + } + + const std::vector tokens = detail::split(content, ' '); + if (tokens.empty()) { + return std::nullopt; + } + + pm_counter_reading reading{}; + try { + reading.value = detail::convert_to(tokens.front()); + } catch (const std::exception &) { + return std::nullopt; + } + + // PM counters version 3 appends a microsecond timestamp as "... us" to telemetry (but not cap) files + if (tokens.size() >= 2 && tokens.back() == "us" && detail::is_integer(tokens[tokens.size() - 2])) { + try { + reading.timestamp_us = detail::convert_to(tokens[tokens.size() - 2]); + } catch (const std::exception &) { + // ignore an unparsable timestamp, the value itself was already parsed successfully + } + } + + return reading; +} + +namespace { + +// Check whether lower_case_key's last path component (i.e. after the last '_' or '/', or the whole key if it has +// neither) is exactly component - e.g. "accel0_energy" and "energy" both match component "energy", but +// "power_state" or "energy_source" don't match component "power"/"energy" respectively. Anchoring on the last +// component (rather than a plain substring search) avoids misclassifying a future metadata field that merely +// contains "energy"/"power" as a substring without actually being a measured energy/power counter. +[[nodiscard]] bool last_component_is(const std::string &lower_case_key, const std::string &component) { + if (lower_case_key == component) { + return true; + } + const std::size_t suffix_size = component.size() + 1; // +1 for the separator + if (lower_case_key.size() <= suffix_size) { + return false; + } + const char separator = lower_case_key[lower_case_key.size() - suffix_size]; + return (separator == '_' || separator == '/') && lower_case_key.compare(lower_case_key.size() - component.size(), component.size(), component) == 0; +} + +// "power_cap"/"accel[i]_power_cap" are configured limits (not measured draw), and "capped_energy"/"overshoot"/ +// "overshoot_energy" are derived/event counters (confirmed on a real Hunter node) - none of them belong in the +// same map as the actual measured energy/power telemetry this backend exists to compare against hws's own +// software-side measurements, so they're excluded here and fall through to general metadata instead. +[[nodiscard]] bool is_excluded_from_telemetry(const std::string &lower_case_key) { + return lower_case_key.find("cap") != std::string::npos || lower_case_key.find("overshoot") != std::string::npos; +} + +} // namespace + +bool is_energy_counter_key(const std::string &key) { + const std::string lower_case_key = detail::to_lower_case(key); + return last_component_is(lower_case_key, "energy") && !is_excluded_from_telemetry(lower_case_key); +} + +bool is_power_counter_key(const std::string &key) { + const std::string lower_case_key = detail::to_lower_case(key); + return last_component_is(lower_case_key, "power") && !is_excluded_from_telemetry(lower_case_key); +} + +std::optional accel_index_from_counter_key(const std::string &key) { + const std::string lower_case_key = detail::to_lower_case(key); + constexpr std::string_view prefix = "accel"; + constexpr std::string_view suffixes[] = { "_energy", "_power" }; + + if (lower_case_key.compare(0, prefix.size(), prefix) != 0) { + return std::nullopt; + } + + for (const std::string_view suffix : suffixes) { + if (lower_case_key.size() <= prefix.size() + suffix.size()) { + continue; + } + if (lower_case_key.compare(lower_case_key.size() - suffix.size(), suffix.size(), suffix) != 0) { + continue; + } + const std::string digits = lower_case_key.substr(prefix.size(), lower_case_key.size() - prefix.size() - suffix.size()); + if (!digits.empty() && detail::is_integer(digits)) { + try { + return detail::convert_to(digits); + } catch (const std::exception &) { + return std::nullopt; + } + } + } + return std::nullopt; +} + +std::vector accel_indices_from_counter_keys(const std::vector &counter_keys) { + std::vector indices{}; + for (const std::string &key : counter_keys) { + if (const std::optional idx = accel_index_from_counter_key(key); idx.has_value()) { + indices.push_back(*idx); + } + } + std::sort(indices.begin(), indices.end()); + indices.erase(std::unique(indices.begin(), indices.end()), indices.end()); + return indices; +} + +std::optional guess_accel_index(const std::string &pci_bus_id, + const std::vector &physical_pci_bus_ids_sorted, + const std::vector &accel_indices_sorted) { + if (physical_pci_bus_ids_sorted.empty() || physical_pci_bus_ids_sorted.size() != accel_indices_sorted.size()) { + return std::nullopt; + } + const auto it = std::find(physical_pci_bus_ids_sorted.cbegin(), physical_pci_bus_ids_sorted.cend(), pci_bus_id); + if (it == physical_pci_bus_ids_sorted.cend()) { + return std::nullopt; + } + const auto position = static_cast(std::distance(physical_pci_bus_ids_sorted.cbegin(), it)); + return accel_indices_sorted[position]; +} + +std::string accel_correlation_yaml_block(const std::string &vendor, + const std::vector> &visible_devices, + const std::vector &physical_pci_bus_ids_sorted, + const std::vector &accel_indices_sorted) { + const bool topology_matches = !physical_pci_bus_ids_sorted.empty() && physical_pci_bus_ids_sorted.size() == accel_indices_sorted.size(); + + std::vector visible_entries{}; + for (const auto &[local_index, pci_bus_id] : visible_devices) { + const std::optional guessed = guess_accel_index(pci_bus_id, physical_pci_bus_ids_sorted, accel_indices_sorted); + const std::string guessed_str = guessed.has_value() ? std::to_string(*guessed) : "null"; + visible_entries.push_back(fmt::format(" - local_index: {}\n" + " pci_bus_id: \"{}\"\n" + " guessed_accel_index: {}", + local_index, pci_bus_id, guessed_str)); + } + + std::vector quoted_physical_ids{}; + for (const std::string &id : physical_pci_bus_ids_sorted) { + quoted_physical_ids.push_back(fmt::format("\"{}\"", id)); + } + + return fmt::format(" {}:\n" + " topology_count_mismatch: {}\n" + " physical_pci_bus_ids: [{}]\n" + " visible_gpus:\n" + "{}\n", + vendor, !topology_matches, fmt::join(quoted_physical_ids, ", "), fmt::join(visible_entries, "\n")); +} + +} // namespace hws::detail diff --git a/src/hws/gpu_amd/hardware_sampler.cpp b/src/hws/gpu_amd/hardware_sampler.cpp index 7c0a6a2..977b4f4 100644 --- a/src/hws/gpu_amd/hardware_sampler.cpp +++ b/src/hws/gpu_amd/hardware_sampler.cpp @@ -35,6 +35,53 @@ namespace hws { +namespace { + +/** + * @brief Convert a ROCm SMI BDFID (as returned by `rsmi_dev_pci_id_get()`) to a sysfs-style PCI bus ID string. + * @details BDFID = (DOMAIN << 32) | (PARTITION << 28) | (BUS << 8) | (DEVICE << 3) | FUNCTION (see ROCm SMI's + * `rsmi_dev_pci_id_get` documentation). On MI-series partitioned devices the function bits are + * repurposed for the partition ID instead of a real PCI function - but the OS/sysfs-visible PCI address + * for the device itself always has function 0, so the function is intentionally not extracted here; see + * `hws::detail::format_pci_bus_id()`. + */ +[[nodiscard]] std::string bdfid_to_pci_bus_id(const std::uint64_t bdfid) { + const auto domain = static_cast((bdfid >> 32) & 0xffffffffull); + const auto bus = static_cast((bdfid >> 8) & 0xffull); + const auto device = static_cast((bdfid >> 3) & 0x1full); + return detail::format_pci_bus_id(domain, bus, device); +} + +/** + * @brief Resolve the ROCm SMI device index that corresponds to the physical device HIP considers index + * @p hip_device_id, by matching PCI bus IDs. + * @details Necessary because ROCm SMI enumerates every physical AMD GPU on the node unconditionally, while HIP's + * enumeration is filtered/reordered by `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` - the same index + * number in both APIs can refer to different physical devices. Requires `rsmi_init()` to have already + * been called. + * @throws std::runtime_error if ROCm SMI's device count can't be queried, or if none of its devices' PCI bus IDs + * match @p hip_device_id's - silently falling back to @p hip_device_id here would be exactly the + * HIP-index-used-as-RSMI-index bug this function exists to avoid, just triggered by a query failure + * instead of a visibility mask. + */ +[[nodiscard]] std::uint32_t resolve_rsmi_device_id(const std::uint32_t hip_device_id) { + std::uint32_t rsmi_count{}; + if (rsmi_num_monitor_devices(&rsmi_count) != RSMI_STATUS_SUCCESS) { + throw std::runtime_error{ "gpu_amd_hardware_sampler: couldn't query the number of ROCm SMI devices while resolving the physical device for HIP index " + std::to_string(hip_device_id) + "!" }; + } + + const std::string hip_bus_id = detail::amd_device_pci_bus_id(static_cast(hip_device_id)); + for (std::uint32_t rsmi_idx = 0; rsmi_idx < rsmi_count; ++rsmi_idx) { + std::uint64_t bdfid{}; + if (rsmi_dev_pci_id_get(rsmi_idx, &bdfid) == RSMI_STATUS_SUCCESS && bdfid_to_pci_bus_id(bdfid) == hip_bus_id) { + return rsmi_idx; + } + } + throw std::runtime_error{ "gpu_amd_hardware_sampler: couldn't find a ROCm SMI device with PCI bus ID " + hip_bus_id + " (HIP index " + std::to_string(hip_device_id) + ")!" }; +} + +} // namespace + gpu_amd_hardware_sampler::gpu_amd_hardware_sampler(const sample_category category) : gpu_amd_hardware_sampler{ 0, HWS_SAMPLING_INTERVAL, category } { } @@ -46,9 +93,10 @@ gpu_amd_hardware_sampler::gpu_amd_hardware_sampler(const std::chrono::millisecon gpu_amd_hardware_sampler::gpu_amd_hardware_sampler(const std::size_t device_id, const std::chrono::milliseconds sampling_interval, const sample_category category) : hardware_sampler{ sampling_interval, category }, - device_id_{ static_cast(device_id) } { + hip_device_id_{ static_cast(device_id) } { // make sure that rsmi_init is only called once for all instances - if (instances_++ == 0) { + const bool is_first_instance = (instances_++ == 0); + if (is_first_instance) { HWS_ROCM_SMI_ERROR_CHECK(rsmi_init(std::uint64_t{ 0 })) // notify that initialization has been finished init_finished_ = true; @@ -56,6 +104,21 @@ gpu_amd_hardware_sampler::gpu_amd_hardware_sampler(const std::size_t device_id, // wait until init has been finished! while (!init_finished_) { } } + + // resolve device_id_ only after rsmi_init() has definitely run (by this instance or a previous one); if + // resolution throws, this instance never finishes construction and its destructor never runs, so + // instances_/init_finished_ (and, if we were the one that just initialized ROCm SMI, the ROCm SMI runtime + // itself) must be rolled back manually here instead of leaking + try { + device_id_ = resolve_rsmi_device_id(hip_device_id_); + } catch (...) { + --instances_; + if (is_first_instance) { + init_finished_ = false; + rsmi_shut_down(); + } + throw; + } } gpu_amd_hardware_sampler::~gpu_amd_hardware_sampler() { @@ -94,7 +157,7 @@ void gpu_amd_hardware_sampler::sampling_loop() { general_samples_.byte_order_ = "Little Endian"; hipDeviceProp_t prop{}; - if (hipGetDeviceProperties(&prop, static_cast(device_id_)) == hipSuccess) { + if (hipGetDeviceProperties(&prop, static_cast(hip_device_id_)) == hipSuccess) { const std::string architecture{ prop.gcnArchName }; general_samples_.architecture_ = architecture.substr(0, architecture.find_first_of('\0')); } @@ -681,6 +744,12 @@ std::string gpu_amd_hardware_sampler::device_identification() const { return fmt::format("gpu_amd_device_{}", device_id_); } +std::string gpu_amd_hardware_sampler::pci_bus_id() const { + std::uint64_t bdfid{}; + HWS_ROCM_SMI_ERROR_CHECK(rsmi_dev_pci_id_get(device_id_, &bdfid)) + return bdfid_to_pci_bus_id(bdfid); +} + std::string gpu_amd_hardware_sampler::samples_only_as_yaml_string() const { // check whether it's safe to generate the YAML entry if (this->is_sampling()) { diff --git a/src/hws/gpu_amd/utility.cpp b/src/hws/gpu_amd/utility.cpp index 55d6932..a3dedf5 100644 --- a/src/hws/gpu_amd/utility.cpp +++ b/src/hws/gpu_amd/utility.cpp @@ -9,13 +9,16 @@ #include "rocm_smi/rocm_smi.h" // ROCm SMI runtime functions -#include // std::string -#include // std::vector +#include "hip/hip_runtime_api.h" // hipGetDeviceCount, hipDeviceGetPCIBusId + +#include // std::sort +#include // std::filesystem::{directory_iterator, exists, directory_options} +#include // std::string +#include // std::error_code +#include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) && defined(HWS_FOR_AMD_GPUS_ENABLED) #include "hws/visible_gpu_device.hpp" // hws::detail::visible_gpu_device, hws::detail::device_backend_kind - - #include "hip/hip_runtime_api.h" // hipGetDeviceCount, hipDeviceGetPCIBusId #endif namespace hws::detail { @@ -46,6 +49,40 @@ std::string performance_level_to_string(const rsmi_dev_perf_level_t perf_level) } } +std::string amd_device_pci_bus_id(const int local_index) { + char bus_id[64] = {}; + HWS_HIP_ERROR_CHECK(hipDeviceGetPCIBusId(bus_id, sizeof(bus_id), local_index)); + return std::string{ bus_id }; +} + +std::vector enumerate_all_amd_gpu_pci_bus_ids() { + std::vector bus_ids{}; + + const std::filesystem::path amdgpu_driver_dir{ "/sys/bus/pci/drivers/amdgpu" }; + std::error_code ec{}; + if (!std::filesystem::exists(amdgpu_driver_dir, ec) || ec) { + return bus_ids; + } + + for (const std::filesystem::directory_entry &entry : std::filesystem::directory_iterator(amdgpu_driver_dir, std::filesystem::directory_options::skip_permission_denied, ec)) { + if (ec) { + break; + } + // every PCI device bound to the amdgpu driver shows up here as a symlink named after its PCI bus ID, + // e.g. "0000:c1:00.0" -> ../../../devices/.../0000:c1:00.0 + if (!entry.is_symlink(ec)) { + continue; + } + const std::string name = entry.path().filename().string(); + if (name.find(':') != std::string::npos && name.find('.') != std::string::npos) { + bus_ids.push_back(name); + } + } + + std::sort(bus_ids.begin(), bus_ids.end()); + return bus_ids; +} + #if defined(HWS_MPI_SUPPORT_ENABLED) && defined(HWS_FOR_AMD_GPUS_ENABLED) namespace { @@ -58,9 +95,7 @@ namespace { * @return the physical ID of the AMD GPU device */ [[nodiscard]] std::string amd_physical_id(const int local_index) { - char bus_id[64] = {}; - HWS_HIP_ERROR_CHECK(hipDeviceGetPCIBusId(bus_id, sizeof(bus_id), local_index)); - return std::string{ "amd:" } + bus_id; + return std::string{ "amd:" } + amd_device_pci_bus_id(local_index); } } // namespace diff --git a/src/hws/gpu_nvidia/hardware_sampler.cpp b/src/hws/gpu_nvidia/hardware_sampler.cpp index f3c6f53..85be367 100644 --- a/src/hws/gpu_nvidia/hardware_sampler.cpp +++ b/src/hws/gpu_nvidia/hardware_sampler.cpp @@ -45,7 +45,7 @@ gpu_nvidia_hardware_sampler::gpu_nvidia_hardware_sampler(const std::chrono::mill gpu_nvidia_hardware_sampler{ 0, sampling_interval, category } { } gpu_nvidia_hardware_sampler::gpu_nvidia_hardware_sampler(const std::size_t device_id, const std::chrono::milliseconds sampling_interval, const sample_category category) : - hardware_sampler{ sampling_interval, category } { + hardware_sampler{ sampling_interval, category }, device_id_{ device_id } { // make sure that nvmlInit is only called once for all instances if (instances_++ == 0) { HWS_NVML_ERROR_CHECK(nvmlInit()) @@ -568,6 +568,15 @@ std::string gpu_nvidia_hardware_sampler::device_identification() const { return fmt::format("gpu_nvidia_device_{}_{}", pcie_info.device, pcie_info.bus); } +std::string gpu_nvidia_hardware_sampler::pci_bus_id() const { + nvmlPciInfo_st pcie_info{}; + HWS_NVML_ERROR_CHECK(nvmlDeviceGetPciInfo_v3(device_.get_impl().device, &pcie_info)) + // deliberately formatted from the numeric domain/bus/device fields, not pcie_info.busId - NVML's own busId + // string uses an extended 8-digit domain (e.g. "00000000:C1:00.0") that doesn't match the 4-digit lowercase + // sysfs convention used by enumerate_all_nvidia_gpu_pci_bus_ids(); see hws::detail::format_pci_bus_id(). + return detail::format_pci_bus_id(pcie_info.domain, pcie_info.bus, pcie_info.device); +} + std::string gpu_nvidia_hardware_sampler::samples_only_as_yaml_string() const { // check whether it's safe to generate the YAML entry if (this->is_sampling()) { diff --git a/src/hws/gpu_nvidia/utility.cpp b/src/hws/gpu_nvidia/utility.cpp index 97d8c1e..41a2c37 100644 --- a/src/hws/gpu_nvidia/utility.cpp +++ b/src/hws/gpu_nvidia/utility.cpp @@ -7,12 +7,16 @@ #include "hws/gpu_nvidia/utility.hpp" -#include "fmt/format.h" // fmt::format -#include "fmt/ranges.h" // fmt::join -#include "nvml.h" // NVML runtime functions +#include "cuda_runtime_api.h" // cudaGetDeviceCount, cudaDeviceGetPCIBusId +#include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join +#include "nvml.h" // NVML runtime functions -#include // std::string -#include // std::vector +#include // std::sort +#include // std::filesystem::{directory_iterator, exists, directory_options} +#include // std::string +#include // std::error_code +#include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) && defined(HWS_FOR_NVIDIA_GPUS_ENABLED) #include "hws/visible_gpu_device.hpp" // hws::detail::visible_gpu_device, hws::detail::device_backend_kind @@ -60,6 +64,40 @@ std::string throttle_event_reason_to_string(const unsigned long long clocks_even #endif +std::string nvidia_device_pci_bus_id(const int local_index) { + char bus_id[64] = {}; + HWS_CUDA_ERROR_CHECK(cudaDeviceGetPCIBusId(bus_id, sizeof(bus_id), local_index)); + return std::string{ bus_id }; +} + +std::vector enumerate_all_nvidia_gpu_pci_bus_ids() { + std::vector bus_ids{}; + + const std::filesystem::path nvidia_driver_dir{ "/sys/bus/pci/drivers/nvidia" }; + std::error_code ec{}; + if (!std::filesystem::exists(nvidia_driver_dir, ec) || ec) { + return bus_ids; + } + + for (const std::filesystem::directory_entry &entry : std::filesystem::directory_iterator(nvidia_driver_dir, std::filesystem::directory_options::skip_permission_denied, ec)) { + if (ec) { + break; + } + // every PCI device bound to the nvidia driver shows up here as a symlink named after its PCI bus ID, + // e.g. "0000:c1:00.0" -> ../../../devices/.../0000:c1:00.0 + if (!entry.is_symlink(ec)) { + continue; + } + const std::string name = entry.path().filename().string(); + if (name.find(':') != std::string::npos && name.find('.') != std::string::npos) { + bus_ids.push_back(name); + } + } + + std::sort(bus_ids.begin(), bus_ids.end()); + return bus_ids; +} + #if defined(HWS_MPI_SUPPORT_ENABLED) && defined(HWS_FOR_NVIDIA_GPUS_ENABLED) namespace { @@ -72,9 +110,7 @@ namespace { * @return the physical ID of the NVIDIA GPU device */ [[nodiscard]] std::string nvidia_physical_id(const int local_index) { - char bus_id[64] = {}; - HWS_CUDA_ERROR_CHECK(cudaDeviceGetPCIBusId(bus_id, sizeof(bus_id), local_index)); - return std::string{ "nvidia:" } + bus_id; + return std::string{ "nvidia:" } + nvidia_device_pci_bus_id(local_index); } } // namespace diff --git a/src/hws/system_hardware_sampler.cpp b/src/hws/system_hardware_sampler.cpp index 4dff468..1230a40 100644 --- a/src/hws/system_hardware_sampler.cpp +++ b/src/hws/system_hardware_sampler.cpp @@ -13,6 +13,10 @@ #if defined(HWS_FOR_CPUS_ENABLED) #include "hws/cpu/hardware_sampler.hpp" // hws::cpu_hardware_sampler #endif +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + #include "hws/cray_pm_counters/hardware_sampler.hpp" // hws::cray_pm_counters_hardware_sampler + #include "hws/cray_pm_counters/utility.hpp" // hws::detail::pm_counters_available +#endif #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) #include "hws/gpu_nvidia/hardware_sampler.hpp" // hws::gpu_nvidia_hardware_sampler #include "hws/gpu_nvidia/utility.hpp" // HWS_CUDA_ERROR_CHECK @@ -31,14 +35,19 @@ #endif #include "fmt/format.h" // fmt::format +#include "fmt/ranges.h" // fmt::join #include // std::for_each, std::all_of #include // std::chrono::milliseconds #include // std::size_t #include // std::uint32_t +#include // std::ofstream #include // std::unique_ptr, std::make_unique #include // std::accumulate +#include // std::optional #include // std::out_of_range +#include // std::string, std::to_string +#include // std::pair #include // std::vector #if defined(HWS_MPI_SUPPORT_ENABLED) @@ -67,46 +76,53 @@ system_hardware_sampler::system_hardware_sampler(MPI_Comm communicator, const de // create a custom, node-local MPI communicator detail::hostname_comm_info nc{ communicator }; - // CPU: one sampler per node --> node leader only - #if defined(HWS_FOR_CPUS_ENABLED) - if (nc.node_rank == 0) { - samplers_.push_back(std::make_unique(sampling_interval, category)); - } - #endif - - // NVIDIA - #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) - { - const std::vector local = detail::enumerate_local_nvidia_devices(); - const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); - for (int const idx : owned) { - samplers_.push_back(std::make_unique(static_cast(idx), sampling_interval, category)); + // CPU: one sampler per node --> node leader only + #if defined(HWS_FOR_CPUS_ENABLED) + if (nc.node_rank == 0) { + samplers_.push_back(std::make_unique(sampling_interval, category)); } - } - #endif + #endif - // AMD - #if defined(HWS_FOR_AMD_GPUS_ENABLED) - { - const std::vector local = detail::enumerate_local_amd_devices(); - const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); - for (int const idx : owned) { - samplers_.push_back(std::make_unique( - static_cast(idx), sampling_interval, category)); + // Cray pm_counters: one sampler per node --> node leader only + #if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + if (nc.node_rank == 0 && detail::pm_counters_available()) { + samplers_.push_back(std::make_unique(sampling_interval, category)); } - } - #endif - - // Intel - #if defined(HWS_FOR_INTEL_GPUS_ENABLED) - { - const std::vector local = detail::enumerate_local_intel_devices(); - const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); - for (int const idx : owned) { - samplers_.push_back(std::make_unique(static_cast(idx), sampling_interval, category)); + #endif + + // NVIDIA + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + { + const std::vector local = detail::enumerate_local_nvidia_devices(); + const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); + for (int const idx : owned) { + samplers_.push_back(std::make_unique(static_cast(idx), sampling_interval, category)); + } } - } - #endif + #endif + + // AMD + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + { + const std::vector local = detail::enumerate_local_amd_devices(); + const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); + for (int const idx : owned) { + samplers_.push_back(std::make_unique( + static_cast(idx), sampling_interval, category)); + } + } + #endif + + // Intel + #if defined(HWS_FOR_INTEL_GPUS_ENABLED) + { + const std::vector local = detail::enumerate_local_intel_devices(); + const std::vector owned = detail::owned_local_indices_for_backend(local, nc.node_comm); + for (int const idx : owned) { + samplers_.push_back(std::make_unique(static_cast(idx), sampling_interval, category)); + } + } + #endif } else { throw std::runtime_error{ fmt::format("Unknown MPI sampling mode {}!", static_cast(mode)) }; @@ -219,14 +235,24 @@ const std::unique_ptr &system_hardware_sampler::sampler(const void system_hardware_sampler::dump_yaml(const char *filename) const { std::for_each(samplers_.cbegin(), samplers_.cend(), [&filename](const auto &ptr) { ptr->dump_yaml(filename); }); + + // each individual sampler already wrote its own "---"-separated YAML document above; the correlation hints + // aren't tied to any single sampler, so they get one more such document of their own, if there's anything to + // report (see device_correlation_hints_as_yaml_string()). + const std::string hints = this->device_correlation_hints_as_yaml_string(); + if (!hints.empty()) { + std::ofstream file{ filename, std::ios_base::app }; + file << "---\n\n" + << hints; + } } void system_hardware_sampler::dump_yaml(const std::string &filename) const { - std::for_each(samplers_.cbegin(), samplers_.cend(), [&filename](const auto &ptr) { ptr->dump_yaml(filename); }); + this->dump_yaml(filename.c_str()); } void system_hardware_sampler::dump_yaml(const std::filesystem::path &filename) const { - std::for_each(samplers_.cbegin(), samplers_.cend(), [&filename](const auto &ptr) { ptr->dump_yaml(filename); }); + this->dump_yaml(filename.string().c_str()); } #if defined(HWS_MPI_SUPPORT_ENABLED) @@ -254,6 +280,10 @@ void system_hardware_sampler::dump_yaml_global(const char *filename, MPI_Comm co rank_yaml_output += detail::indent_lines(ptr->as_yaml_string(), " "); }); + // not tied to any single sampler, so appended directly rather than under a "sampler_N:" key; empty if there's + // nothing to report (see device_correlation_hints_as_yaml_string()) + rank_yaml_output += this->device_correlation_hints_as_yaml_string(); + const std::string global_yaml_output = detail::gather_yaml_strings_mpi(rank_yaml_output, communicator); if (rank == 0) { @@ -272,19 +302,107 @@ void system_hardware_sampler::dump_yaml_global(const std::filesystem::path &file #endif std::string system_hardware_sampler::as_yaml_string() const { - return std::accumulate(samplers_.cbegin(), samplers_.cend(), std::string{}, [](const std::string str, const auto &ptr) { return str + ptr->as_yaml_string(); }); + return std::accumulate(samplers_.cbegin(), samplers_.cend(), std::string{}, [](const std::string str, const auto &ptr) { return str + ptr->as_yaml_string(); }) + + this->device_correlation_hints_as_yaml_string(); } std::string system_hardware_sampler::samples_only_as_yaml_string() const { return std::accumulate(samplers_.cbegin(), samplers_.cend(), std::string{}, [](const std::string str, const auto &ptr) { return str + ptr->samples_only_as_yaml_string(); }); } +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) && (defined(HWS_FOR_AMD_GPUS_ENABLED) || defined(HWS_FOR_NVIDIA_GPUS_ENABLED)) +std::string system_hardware_sampler::device_correlation_hints_as_yaml_string() const { + const cray_pm_counters_hardware_sampler *pm_sampler = nullptr; + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + std::vector> amd_devices{}; + #endif + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + std::vector> nvidia_devices{}; + #endif + for (const std::unique_ptr &ptr : samplers_) { + if (const auto *pm = dynamic_cast(ptr.get()); pm != nullptr) { + pm_sampler = pm; + continue; + } + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + if (const auto *amd = dynamic_cast(ptr.get()); amd != nullptr) { + // hip_device_id() (the "Nth GPU visible to this process/rank") for local_index, but pci_bus_id() + // (ROCm SMI, the same API family used for all of this sampler's actual measurements) for the PCI bus + // ID - not hip_device_id()'s own HIP-space bus id, since ROCm SMI's and HIP's device enumerations can + // diverge under HIP_VISIBLE_DEVICES/ROCR_VISIBLE_DEVICES and using a different API family than the + // one the sampler measures with could silently attribute the wrong PCI bus ID. + amd_devices.emplace_back(amd->hip_device_id(), amd->pci_bus_id()); + continue; + } + #endif + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + if (const auto *nvidia = dynamic_cast(ptr.get()); nvidia != nullptr) { + // pci_bus_id() (NVML, the same API family used for all of this sampler's actual measurements) is used + // here rather than CUDA's own cudaDeviceGetPCIBusId(), since NVML's device enumeration isn't affected + // by CUDA_VISIBLE_DEVICES while the CUDA runtime's is - using a different API family than the one the + // sampler measures with could silently attribute the wrong PCI bus ID. + nvidia_devices.emplace_back(nvidia->device_id(), nvidia->pci_bus_id()); + continue; + } + #endif + } + + const bool any_visible_gpus = + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + !amd_devices.empty() + #else + false + #endif + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + || !nvidia_devices.empty() + #endif + ; + if (pm_sampler == nullptr || !any_visible_gpus) { + return ""; + } + + // ground truth, no guessing involved: which accel[i] counters pm_counters exposed on this node + const std::vector accel_indices = pm_sampler->discovered_accel_indices(); + + std::string vendor_blocks{}; + #if defined(HWS_FOR_AMD_GPUS_ENABLED) + if (!amd_devices.empty()) { + vendor_blocks += detail::accel_correlation_yaml_block("amd", amd_devices, detail::enumerate_all_amd_gpu_pci_bus_ids(), accel_indices); + } + #endif + #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) + if (!nvidia_devices.empty()) { + vendor_blocks += detail::accel_correlation_yaml_block("nvidia", nvidia_devices, detail::enumerate_all_nvidia_gpu_pci_bus_ids(), accel_indices); + } + #endif + + return fmt::format("device_correlation_hints:\n" + " note: \"UNVERIFIED heuristic: assumes Cray pm_counters numbers accel[i] in ascending PCI bus address order among all physically present GPUs of a given vendor; this is not confirmed by any HPE documentation. Confirm empirically (e.g. drive load on a single visible GPU and observe which accel[i]_power reacts) before relying on this for analysis.\"\n" + " verified: false\n" + " accel_indices_discovered: [{}]\n" + " gpu_vendors:\n" + "{}" + "\n", + fmt::join(accel_indices, ", "), + vendor_blocks); +} +#else +std::string system_hardware_sampler::device_correlation_hints_as_yaml_string() const { + return ""; +} +#endif + void system_hardware_sampler::create_local_samplers(std::chrono::milliseconds sampling_interval, sample_category category) { #if defined(HWS_FOR_CPUS_ENABLED) { samplers_.push_back(std::make_unique(sampling_interval, category)); } #endif +#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED) + if (detail::pm_counters_available()) { + samplers_.push_back(std::make_unique(sampling_interval, category)); + } +#endif #if defined(HWS_FOR_NVIDIA_GPUS_ENABLED) { int device_count{}; diff --git a/src/hws/utility.cpp b/src/hws/utility.cpp index 406089a..8138ea5 100644 --- a/src/hws/utility.cpp +++ b/src/hws/utility.cpp @@ -7,8 +7,11 @@ #include "hws/utility.hpp" +#include "fmt/format.h" // fmt::format + #include // std::min, std::transform, std::all_of #include // std::tolower, std::isdigit +#include // std::uint32_t #include // std::stringstream #include // std::string #include // std::string_view @@ -77,4 +80,8 @@ std::string indent_lines(const std::string &text, const std::string_view prefix) return out; } +std::string format_pci_bus_id(const std::uint32_t domain, const std::uint32_t bus, const std::uint32_t device) { + return fmt::format("{:04x}:{:02x}:{:02x}.0", domain, bus, device); +} + } // namespace hws::detail diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..4ee2c57 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,9 @@ +## Authors: Alexander Van Craen +## Copyright (C): 2024-today All Rights Reserved +## License: This file is released under the MIT license. +## See the LICENSE.md file in the project root for full license information. +######################################################################################################################## + +## one subdirectory per backend, mirroring the include/hws// and src/hws// layout; each +## subdirectory's CMakeLists.txt is responsible for only registering tests if its backend was actually compiled in +add_subdirectory(cray_pm_counters) diff --git a/tests/cray_pm_counters/CMakeLists.txt b/tests/cray_pm_counters/CMakeLists.txt new file mode 100644 index 0000000..af9eccd --- /dev/null +++ b/tests/cray_pm_counters/CMakeLists.txt @@ -0,0 +1,15 @@ +## Authors: Alexander Van Craen +## Copyright (C): 2024-today All Rights Reserved +## License: This file is released under the MIT license. +## See the LICENSE.md file in the project root for full license information. +######################################################################################################################## + +# only build this backend's tests if it was actually compiled into ${HWS_LIBRARY_NAME} +get_target_property(HWS_COMPILE_DEFINITIONS ${HWS_LIBRARY_NAME} COMPILE_DEFINITIONS) + +if ("HWS_FOR_CRAY_PM_COUNTERS_ENABLED" IN_LIST HWS_COMPILE_DEFINITIONS) + add_executable(hws_test_cray_pm_counters test.cpp) + target_link_libraries(hws_test_cray_pm_counters PRIVATE ${HWS_LIBRARY_NAME}) + add_test(NAME cray_pm_counters COMMAND hws_test_cray_pm_counters) + set_tests_properties(cray_pm_counters PROPERTIES TIMEOUT 30) +endif () diff --git a/tests/cray_pm_counters/test.cpp b/tests/cray_pm_counters/test.cpp new file mode 100644 index 0000000..987d397 --- /dev/null +++ b/tests/cray_pm_counters/test.cpp @@ -0,0 +1,278 @@ +/** + * @author Alexander Van Craen + * @copyright 2024-today All Rights Reserved + * @license This file is released under the MIT license. + * See the LICENSE.md file in the project root for full license information. + * + * @brief Simple, dependency-free regression checks for the cray_pm_counters backend. + * @details Uses the HWS_PM_COUNTERS_PATH environment variable override to point the backend at synthetic + * directories instead of the real (Cray-only) /sys/cray/pm_counters, so these checks run on any Linux + * machine. The synthetic content mirrors a real dump gathered on an HLRS "Hunter" node (HPE Cray EX255a, + * PM counters version 3, see scripts/hwmon_probe.pbs). Not a general-purpose test framework - just enough to + * catch a regression in the specific bugs found (and fixed) during code review. + */ + +#include "hws/core.hpp" +#include "hws/cray_pm_counters/utility.hpp" // hws::detail::{accel_index_from_counter_key, accel_indices_from_counter_keys, guess_accel_index, accel_correlation_yaml_block} + +#include // std::chrono::milliseconds +#include // std::exit, setenv +#include // std::filesystem +#include // std::ofstream +#include // std::cout, std::cerr +#include // std::optional +#include // std::string +#include // std::this_thread::sleep_for +#include // std::pair +#include // std::vector + +namespace { + +int g_failures = 0; + +void check(bool condition, const std::string &description) { + if (condition) { + std::cout << " ok: " << description << "\n"; + } else { + std::cerr << " FAILED: " << description << "\n"; + ++g_failures; + } +} + +void write_file(const std::filesystem::path &path, const std::string &content) { + std::filesystem::create_directories(path.parent_path()); + std::ofstream file{ path }; + file << content; +} + +void set_pm_counters_path(const std::filesystem::path &path) { + setenv("HWS_PM_COUNTERS_PATH", path.c_str(), 1); +} + +// mirrors the real dump gathered on Hunter (hwmon_probe.o250927): 14 of the 23 real files, enough to exercise +// every classification/parsing rule without duplicating the full dump +std::filesystem::path make_synthetic_pm_counters(const std::filesystem::path &root) { + std::filesystem::remove_all(root); + write_file(root / "energy", "12044825101 J 7810213855886 us\n"); + write_file(root / "power", "724 W 7810213855886 us\n"); + write_file(root / "power_cap", "2894 W\n"); + write_file(root / "accel0_energy", "2869283299 J 7810213855886 us\n"); + write_file(root / "accel0_power", "212 W 7810213855886 us\n"); + write_file(root / "accel0_power_cap", "0 W\n"); + write_file(root / "capped_energy", "12752953 J 7810213855886 us\n"); + write_file(root / "overshoot", "62408\n"); + write_file(root / "overshoot_energy", "3951201 J 7810213855886 us\n"); + write_file(root / "freshness", "60043284\n"); + write_file(root / "generation", "16\n"); + write_file(root / "raw_scan_hz", "10\n"); + write_file(root / "startup", "1778580757634187\n"); + write_file(root / "version", "3\n"); + return root; +} + +void test_absent() { + std::cout << "test_absent\n"; + set_pm_counters_path("/nonexistent/pm_counters_for_hws_test"); + + hws::cray_pm_counters_hardware_sampler sampler{}; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + sampler.stop_sampling(); + + check(!sampler.general_samples().has_samples(), "no general samples when pm_counters is absent"); + check(!sampler.power_samples().has_samples(), "no power samples when pm_counters is absent"); +} + +void test_real_values(const std::filesystem::path &root) { + std::cout << "test_real_values\n"; + set_pm_counters_path(root); + + hws::cray_pm_counters_hardware_sampler sampler{}; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + sampler.stop_sampling(); + + const auto &energy = sampler.power_samples().get_energy_counters(); + const auto &power = sampler.power_samples().get_power_counters(); + const auto &energy_ts = sampler.power_samples().get_energy_timestamps_us(); + const auto &metadata = sampler.general_samples().get_metadata(); + + check(energy.has_value() && energy->count("energy") == 1 && energy->at("energy").front() == 12044825101ull, + "'energy' discovered and parsed as a measured energy counter"); + check(energy.has_value() && energy->count("accel0_energy") == 1, + "'accel0_energy' discovered as a measured energy counter"); + check(power.has_value() && power->count("power") == 1 && power->at("power").front() == 724ull, + "'power' discovered and parsed as a measured power counter"); + check(energy_ts.has_value() && energy_ts->count("energy") == 1 && energy_ts->at("energy").front() == 7810213855886ull, + "'energy's embedded HSS timestamp extracted correctly"); + + // caps and derived/event counters must not pollute the measured telemetry maps (regression: Opus review finding) + check(!energy.has_value() || energy->count("capped_energy") == 0, "'capped_energy' excluded from energy_counters"); + check(!energy.has_value() || energy->count("overshoot_energy") == 0, "'overshoot_energy' excluded from energy_counters"); + check(!power.has_value() || power->count("power_cap") == 0, "'power_cap' excluded from power_counters"); + check(!power.has_value() || power->count("accel0_power_cap") == 0, "'accel0_power_cap' excluded from power_counters"); + + // ... they should show up as general metadata instead + check(metadata.has_value() && metadata->count("power_cap") == 1, "'power_cap' present in general metadata"); + check(metadata.has_value() && metadata->count("capped_energy") == 1, "'capped_energy' present in general metadata"); + check(metadata.has_value() && metadata->count("overshoot") == 1, "'overshoot' present in general metadata"); + // freshness must be re-sampled every tick, not read once (regression check) + check(metadata.has_value() && metadata->count("freshness") == 1 && metadata->at("freshness").size() > 1, + "'freshness' is re-sampled every tick, not just once"); +} + +void test_adversarial_names(const std::filesystem::path &root) { + std::cout << "test_adversarial_names\n"; + write_file(root / "power_state", "ok\n"); + write_file(root / "energy_source", "grid\n"); + set_pm_counters_path(root); + + hws::cray_pm_counters_hardware_sampler sampler{}; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + sampler.stop_sampling(); + + const auto &energy = sampler.power_samples().get_energy_counters(); + const auto &power = sampler.power_samples().get_power_counters(); + const auto &metadata = sampler.general_samples().get_metadata(); + + // regression check (GPT-5.5 review finding): "energy"/"power" must be a path-segment suffix, not any substring + check(!energy.has_value() || energy->count("energy_source") == 0, "'energy_source' not misclassified as an energy counter"); + check(!power.has_value() || power->count("power_state") == 0, "'power_state' not misclassified as a power counter"); + check(metadata.has_value() && metadata->count("power_state") == 1, "'power_state' present in general metadata"); + check(metadata.has_value() && metadata->count("energy_source") == 1, "'energy_source' present in general metadata"); +} + +void test_nested_path(const std::filesystem::path &root) { + std::cout << "test_nested_path\n"; + write_file(root / "nested" / "energy", "111 J 999 us\n"); + set_pm_counters_path(root); + + hws::cray_pm_counters_hardware_sampler sampler{}; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + sampler.stop_sampling(); + + const auto &energy = sampler.power_samples().get_energy_counters(); + // regression check (Opus review finding): nested files use their real relative path as key, not a flattened one + check(energy.has_value() && energy->count("nested/energy") == 1, "nested file uses its un-flattened relative path as map key"); +} + +void test_throttle(const std::filesystem::path &root) { + std::cout << "test_throttle\n"; + set_pm_counters_path(root); + + // request a 10ms interval, far faster than the synthetic hardware's 10 Hz (raw_scan_hz=10); the sampler must + // throttle to the hardware rate instead of hammering the (synthetic) sysfs files + hws::cray_pm_counters_hardware_sampler sampler{ std::chrono::milliseconds(10) }; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(650)); + sampler.stop_sampling(); + + bool throttled = false; + for (const auto &e : sampler.get_events()) { + if (e.name.find("throttled") != std::string::npos) { + throttled = true; + } + } + check(throttled, "a throttling event is recorded for a too-fast configured interval"); + + const auto &energy = sampler.power_samples().get_energy_counters(); + const std::size_t num_samples = (energy.has_value() && energy->count("energy") == 1) ? energy->at("energy").size() : 0; + // ~650ms / 100ms (hardware period) ~= 6-8 samples; a buggy 10ms cadence would produce ~65 + check(num_samples > 0 && num_samples < 20, "sample count matches the hardware-throttled cadence, not the requested 10ms one"); +} + +void test_throttle_power_only_category(const std::filesystem::path &root) { + std::cout << "test_throttle_power_only_category\n"; + set_pm_counters_path(root); + + // regression check (GPT-5.5 review finding): throttling must not depend on sample_category::general being enabled + hws::cray_pm_counters_hardware_sampler sampler{ std::chrono::milliseconds(10), hws::sample_category::power }; + sampler.start_sampling(); + std::this_thread::sleep_for(std::chrono::milliseconds(650)); + sampler.stop_sampling(); + + const auto &energy = sampler.power_samples().get_energy_counters(); + const std::size_t num_samples = (energy.has_value() && energy->count("energy") == 1) ? energy->at("energy").size() : 0; + check(num_samples > 0 && num_samples < 20, "throttling still applies when only sample_category::power is enabled"); +} + +// pure-logic checks for the accel[i] <-> AMD GPU PCI bus ID correlation guess (see +// hws::system_hardware_sampler::device_correlation_hints_as_yaml_string(), the only caller in the actual library) +// - no ROCm SMI/HIP or real GPU required, since these only deal with plain strings/ints. +void test_accel_correlation_guess() { + std::cout << "test_accel_correlation_guess\n"; + + check(hws::detail::accel_index_from_counter_key("accel0_energy") == std::optional{ 0 }, "'accel0_energy' -> accel index 0"); + check(hws::detail::accel_index_from_counter_key("accel3_power") == std::optional{ 3 }, "'accel3_power' -> accel index 3"); + check(hws::detail::accel_index_from_counter_key("accel12_energy") == std::optional{ 12 }, "multi-digit accel index parsed correctly"); + check(!hws::detail::accel_index_from_counter_key("energy").has_value(), "plain 'energy' (node-wide) has no accel index"); + check(!hws::detail::accel_index_from_counter_key("accel0_power_cap").has_value(), "'accel0_power_cap' has no accel index (not energy/power telemetry)"); + check(!hws::detail::accel_index_from_counter_key("accelerator0_energy").has_value(), "'accelerator0_energy' doesn't match the 'accel_energy' pattern"); + + const std::vector indices = hws::detail::accel_indices_from_counter_keys({ "energy", "accel1_energy", "accel0_energy", "accel1_power", "accel3_energy", "capped_energy" }); + check((indices == std::vector{ 0, 1, 3 }), "accel indices extracted, sorted, and deduplicated across energy+power keys"); + + const std::vector physical{ "0000:0c:00.0", "0000:22:00.0", "0000:c1:00.0", "0000:e2:00.0" }; + const std::vector accel{ 0, 1, 2, 3 }; + + check(hws::detail::guess_accel_index("0000:0c:00.0", physical, accel) == std::optional{ 0 }, "lowest PCI bus id guessed as accel index 0"); + check(hws::detail::guess_accel_index("0000:e2:00.0", physical, accel) == std::optional{ 3 }, "highest PCI bus id guessed as accel index 3"); + check(!hws::detail::guess_accel_index("0000:ff:00.0", physical, accel).has_value(), "unknown PCI bus id yields no guess"); + + // topology count mismatch (e.g. cgroup-isolated partial-node allocation exposing fewer GPUs than pm_counters + // reports accel[i] for) must degrade to "no guess", not a wrong one + const std::vector partial_physical{ "0000:0c:00.0", "0000:22:00.0" }; + check(!hws::detail::guess_accel_index("0000:0c:00.0", partial_physical, accel).has_value(), "topology count mismatch yields no guess, even for a bus id that IS present"); + check(!hws::detail::guess_accel_index("anything", {}, {}).has_value(), "empty physical topology yields no guess"); +} + +// vendor-agnostic YAML formatting for the correlation hints (used for both AMD and NVIDIA, see +// hws::system_hardware_sampler::device_correlation_hints_as_yaml_string()) - also no GPU hardware required. +void test_accel_correlation_yaml_block() { + std::cout << "test_accel_correlation_yaml_block\n"; + + const std::vector accel{ 0, 1 }; + const std::vector physical{ "0000:0c:00.0", "0000:22:00.0" }; + const std::vector> visible{ { 0, "0000:22:00.0" } }; + + const std::string block = hws::detail::accel_correlation_yaml_block("amd", visible, physical, accel); + check(block.rfind(" amd:\n", 0) == 0, "block starts with the vendor key at 4-space indent"); + check(block.find("topology_count_mismatch: false") != std::string::npos, "matching topology reports no mismatch"); + check(block.find("physical_pci_bus_ids: [\"0000:0c:00.0\", \"0000:22:00.0\"]") != std::string::npos, "physical PCI bus ids listed in sorted order"); + check(block.find("local_index: 0") != std::string::npos, "visible device's local index present"); + check(block.find("pci_bus_id: \"0000:22:00.0\"") != std::string::npos, "visible device's PCI bus id present"); + check(block.find("guessed_accel_index: 1") != std::string::npos, "higher (2nd) physical bus id guessed as accel index 1"); + + // topology count mismatch (e.g. no physical GPUs discovered at all) must degrade to "no guess" + const std::string mismatch_block = hws::detail::accel_correlation_yaml_block("nvidia", visible, {}, accel); + check(mismatch_block.rfind(" nvidia:\n", 0) == 0, "vendor key reflects the passed-in vendor name"); + check(mismatch_block.find("topology_count_mismatch: true") != std::string::npos, "empty physical topology reports a mismatch"); + check(mismatch_block.find("guessed_accel_index: null") != std::string::npos, "no guess made under a topology mismatch"); +} + +} // namespace + +int main() { + const std::filesystem::path root = std::filesystem::temp_directory_path() / "hws_cray_pm_counters_test"; + + test_absent(); + make_synthetic_pm_counters(root); + test_real_values(root); + test_adversarial_names(root); + test_nested_path(root); + test_throttle(root); + test_throttle_power_only_category(root); + test_accel_correlation_guess(); + test_accel_correlation_yaml_block(); + + std::filesystem::remove_all(root); + + if (g_failures > 0) { + std::cerr << "\n" << g_failures << " check(s) failed\n"; + return 1; + } + std::cout << "\nall checks passed\n"; + return 0; +}