Add Cray/HPE pm_counters backend with cross-vendor GPU correlation hints

CMakeLists.txt

@@ -147,6 +147,18 @@ else ()
 endif ()
 
 
+####################################################################################################################
+##                                  Cray/HPE node power sampling via pm_counters                                  ##
+####################################################################################################################
+set(HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING AUTO CACHE STRING "Enable sampling of node power/energy via Cray's /sys/cray/pm_counters interface.")
+set_property(CACHE HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING PROPERTY STRINGS AUTO ON OFF)
+if (HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING MATCHES "AUTO" OR HWS_ENABLE_CRAY_PM_COUNTERS_SAMPLING)
+    add_subdirectory(src/hws/cray_pm_counters)
+else ()
+    message(STATUS "Hardware sampling via Cray pm_counters disabled!")
+endif ()
+
+
 ####################################################################################################################
 ##                                               enable MPI support                                               ##
 ####################################################################################################################
@@ -201,6 +213,16 @@ if (HWS_ENABLE_DOCUMENTATION)
 endif ()
 
 
+########################################################################################################################
+##                                                    add tests                                                      ##
+########################################################################################################################
+option(HWS_ENABLE_TESTING "Build simple regression tests (run via 'ctest')." OFF)
+if (HWS_ENABLE_TESTING)
+    enable_testing()
+    add_subdirectory(tests)
+endif ()
+
+
 ########################################################################################################################
 ##                                           add support for `make install`                                           ##
 ########################################################################################################################

README.md

@@ -35,14 +35,14 @@ To download the hardware sampling use:
 
 ```bash
 git clone git@github.com:SC-SGS/hardware_sampling.git
-cd hardware_sampling 
+cd hardware_sampling
 ```
 
 Building the library can be done using the normal CMake approach:
 
 ```bash
-mkdir build && cd build 
-cmake -DCMAKE_BUILD_TYPE=Release [optional_options] .. 
+mkdir build && cd build
+cmake -DCMAKE_BUILD_TYPE=Release [optional_options] ..
 cmake --build . -j
 ```
 
@@ -73,6 +73,18 @@ The `[optional_options]` can be one or multiple of:
 - `HWS_ENABLE_ERROR_CHECKS=ON|OFF` (default: `OFF`): enable sanity checks during hardware sampling, may be problematic
   with smaller sample intervals
 - `HWS_SAMPLING_INTERVAL=100ms` (default: `100ms`): set the sampling interval in milliseconds
+
+- `HWS_TURBOSTAT_INTERVAL=1` (default: `0.001`, kept for backwards compatibility): set the interval in
+  seconds (`"sec.subsec"`) that `turbostat` itself measures over for a single CPU power/frequency sample
+  (passed directly to `turbostat`'s own `-i` flag).
+  **Important:** very short values have been observed to produce
+  physically implausible readings (multi-kW package power, multi-GHz core clocks) on heavily loaded,
+  many-core machines, especially with older `turbostat` builds.
+  **Note:** if this value is larger than `HWS_SAMPLING_INTERVAL`, the `turbostat` backend dominates and
+  the achieved CPU sampling cadence will be closer to `HWS_TURBOSTAT_INTERVAL` than to
+  `HWS_SAMPLING_INTERVAL` -- a warning is printed at runtime (once per `cpu_hardware_sampler` instance)
+  if this is the case.
+
 - `HWS_ENABLE_PYTHON_BINDINGS=ON|OFF` (default: `ON`): enable Python bindings
 
 - `HWS_ENABLE_MPI_SUPPORT=ON|OFF|AUTO` (default: `AUTO`):
@@ -227,7 +239,7 @@ current clock frequencies, temperatures, or memory consumption.
 | sample                  | sample type | CPUs | NVIDIA GPUs | AMD GPUs | Intel GPUs |
 |:------------------------|:-----------:|:----:|:-----------:|:--------:|:----------:|
 | num_fans                |    fixed    |  -   |     int     |   int    |    int     |
-| fan_speed_min           |    fixed    |  -   |      %      |    -     |     -      | 
+| fan_speed_min           |    fixed    |  -   |      %      |    -     |     -      |
 | fan_speed_max           |    fixed    |  -   |      %      |   RPM    |    RPM     |
 | temperature_min         |    fixed    |  -   |      -      |    °C    |     -      |
 | temperature_max         |    fixed    |  -   |     °C      |    °C    |     °C     |

bindings/CMakeLists.txt

@@ -54,6 +54,9 @@ endif ()
 if ("HWS_FOR_INTEL_GPUS_ENABLED" IN_LIST HWS_COMPILE_DEFINITIONS)
     list(APPEND HWS_PYTHON_BINDINGS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/gpu_intel_hardware_sampler.cpp)
 endif ()
+if ("HWS_FOR_CRAY_PM_COUNTERS_ENABLED" IN_LIST HWS_COMPILE_DEFINITIONS)
+    list(APPEND HWS_PYTHON_BINDINGS_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/cray_pm_counters_hardware_sampler.cpp)
+endif ()
 
 # create pybind11 module
 set(HWS_PYTHON_BINDINGS_LIBRARY_NAME HardwareSampling)

bindings/cray_pm_counters_hardware_sampler.cpp

@@ -0,0 +1,55 @@
+/**
+ * @author Alexander Van Craen
+ * @copyright 2024-today All Rights Reserved
+ * @license This file is released under the MIT license.
+ *          See the LICENSE.md file in the project root for full license information.
+ */
+
+#include "hws/cray_pm_counters/hardware_sampler.hpp"       // hws::cray_pm_counters_hardware_sampler
+#include "hws/cray_pm_counters/pm_counters_samples.hpp"     // hws::{cray_pm_counters_general_samples, cray_pm_counters_power_samples}
+#include "hws/hardware_sampler.hpp"                         // hws::hardware_sampler
+#include "hws/sample_category.hpp"                          // hws::sample_category
+
+#include "fmt/format.h"         // fmt::format
+#include "pybind11/chrono.h"    // automatic bindings for std::chrono::milliseconds
+#include "pybind11/pybind11.h"  // py::module_
+#include "pybind11/stl.h"       // bind STL types
+
+#include <chrono>  // std::chrono::milliseconds
+
+namespace py = pybind11;
+
+void init_cray_pm_counters_hardware_sampler(py::module_ &m) {
+    // bind the general samples
+    py::class_<hws::cray_pm_counters_general_samples>(m, "CrayPmCountersGeneralSamples")
+        .def("has_samples", &hws::cray_pm_counters_general_samples::has_samples, "true if any sample is available, false otherwise")
+        .def("get_metadata", &hws::cray_pm_counters_general_samples::get_metadata, "the dynamically discovered, non-energy, non-power pm_counters metadata entries (e.g. version, generation, raw_scan_hz, freshness), keyed by file name")
+        .def("__repr__", [](const hws::cray_pm_counters_general_samples &self) {
+            return fmt::format("<HardwareSampling.CrayPmCountersGeneralSamples with\n{}\n>", self);
+        });
+
+    // bind the power samples
+    py::class_<hws::cray_pm_counters_power_samples>(m, "CrayPmCountersPowerSamples")
+        .def("has_samples", &hws::cray_pm_counters_power_samples::has_samples, "true if any sample is available, false otherwise")
+        .def("get_energy_counters", &hws::cray_pm_counters_power_samples::get_energy_counters, "the dynamically discovered measured energy counters in J (cumulative), keyed by file name, e.g. 'energy' (whole node) or 'accel0_energy' (per accelerator)")
+        .def("get_power_counters", &hws::cray_pm_counters_power_samples::get_power_counters, "the dynamically discovered measured power counters in W (instantaneous), keyed by file name, e.g. 'power' (whole node) or 'accel0_power' (per accelerator)")
+        .def("get_energy_timestamps_us", &hws::cray_pm_counters_power_samples::get_energy_timestamps_us, "per-sample HSS latch timestamp for the energy counters, in us (PM counters v3+ only)")
+        .def("get_power_timestamps_us", &hws::cray_pm_counters_power_samples::get_power_timestamps_us, "per-sample HSS latch timestamp for the power counters, in us (PM counters v3+ only)")
+        .def("__repr__", [](const hws::cray_pm_counters_power_samples &self) {
+            return fmt::format("<HardwareSampling.CrayPmCountersPowerSamples with\n{}\n>", self);
+        });
+
+    // bind the Cray pm_counters hardware sampler class
+    py::class_<hws::cray_pm_counters_hardware_sampler, hws::hardware_sampler>(m, "CrayPmCountersHardwareSampler")
+        .def(py::init<>(), "construct a new Cray pm_counters hardware sampler with the default sampling interval")
+        .def(py::init<hws::sample_category>(), "construct a new Cray pm_counters hardware sampler with the default sampling interval sampling only the provided sample_category samples")
+        .def(py::init<std::chrono::milliseconds>(), "construct a new Cray pm_counters hardware sampler with the specified sampling interval")
+        .def(py::init<std::chrono::milliseconds, hws::sample_category>(), "construct a new Cray pm_counters hardware sampler with the specified sampling interval sampling only the provided sample_category samples")
+        .def("general_samples", &hws::cray_pm_counters_hardware_sampler::general_samples, "get all general samples")
+        .def("power_samples", &hws::cray_pm_counters_hardware_sampler::power_samples, "get all power related samples")
+        .def("discovered_accel_indices", &hws::cray_pm_counters_hardware_sampler::discovered_accel_indices, "the accelerator indices pm_counters actually exposed on this node, derived from the already-sampled accel<N>_energy counter keys")
+        .def("samples_only_as_yaml_string", &hws::cray_pm_counters_hardware_sampler::samples_only_as_yaml_string, "return all hardware samples as YAML string")
+        .def("__repr__", [](const hws::cray_pm_counters_hardware_sampler &self) {
+            return fmt::format("<HardwareSampling.CrayPmCountersHardwareSampler with\n{}\n>", self);
+        });
+}

bindings/main.cpp

@@ -32,6 +32,7 @@ void init_cpu_hardware_sampler(py::module_ &);
 void init_gpu_nvidia_hardware_sampler(py::module_ &);
 void init_gpu_amd_hardware_sampler(py::module_ &);
 void init_gpu_intel_hardware_sampler(py::module_ &);
+void init_cray_pm_counters_hardware_sampler(py::module_ &);
 void init_version(py::module_ &);
 
 PYBIND11_MODULE(HardwareSampling, m) {
@@ -77,6 +78,12 @@ PYBIND11_MODULE(HardwareSampling, m) {
 #endif
     m.def("has_gpu_intel_hardware_sampler", []() { return HWS_IS_DEFINED(HWS_FOR_INTEL_GPUS_ENABLED); });
 
+    // Cray pm_counters sampling
+#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED)
+    init_cray_pm_counters_hardware_sampler(m);
+#endif
+    m.def("has_cray_pm_counters_hardware_sampler", []() { return HWS_IS_DEFINED(HWS_FOR_CRAY_PM_COUNTERS_ENABLED); });
+
     init_version(m);
 }
 

cmake/hwsConfig.cmake.in

@@ -112,6 +112,15 @@ if (HWS_HAS_CPU_SUPPORT)
     endif ()
 endif ()
 
+# check whether MPI support is enabled
+string_contains("${HWS_COMPILE_DEFINITIONS}" "HWS_MPI_SUPPORT_ENABLED" HWS_HAS_MPI_SUPPORT)
+if (HWS_HAS_MPI_SUPPORT)
+    find_dependency(MPI COMPONENTS CXX)
+    if (NOT hws_FIND_QUIETLY)
+        message(STATUS "Enabled MPI support via hws.")
+    endif ()
+endif ()
+
 # check whether NVIDIA GPUs are supported
 string_contains("${HWS_COMPILE_DEFINITIONS}" "HWS_FOR_NVIDIA_GPUS_ENABLED" HWS_HAS_GPU_NVIDIA_SUPPORT)
 if (HWS_HAS_GPU_NVIDIA_SUPPORT)

include/hws/core.hpp

@@ -38,4 +38,9 @@
     #include "hws/gpu_intel/level_zero_samples.hpp"
 #endif
 
+#if defined(HWS_FOR_CRAY_PM_COUNTERS_ENABLED)
+    #include "hws/cray_pm_counters/hardware_sampler.hpp"
+    #include "hws/cray_pm_counters/pm_counters_samples.hpp"
+#endif
+
 #endif  // HWS_CORE_HPP_

include/hws/cpu/hardware_sampler.hpp

@@ -40,6 +40,11 @@ class cpu_hardware_sampler : public hardware_sampler {
      * @brief Construct a new CPU hardware sampler with the @p sampling_interval.
      * @param[in] sampling_interval the used sampling interval
      * @param[in] category the sample categories that are enabled for hardware sampling (default: all)
+     * @note If the turbostat backend is used, turbostat itself blocks for `HWS_TURBOSTAT_INTERVAL` seconds
+     *       per invocation (see the CMake option of the same name). If that is larger than
+     *       @p sampling_interval, the turbostat backend dominates and the achieved sampling cadence will be
+     *       closer to `HWS_TURBOSTAT_INTERVAL` than to @p sampling_interval; a warning is printed to
+     *       `std::cerr` in that case.
      */
     explicit cpu_hardware_sampler(std::chrono::milliseconds sampling_interval, sample_category category = sample_category::all);