Preserve libpfm privilege modifiers in performance counters

AUTHORS

@@ -40,6 +40,7 @@ Google Inc.
 Haihan Jiang <haihanj99@gmail.com>
 Henrique Bucher <hbucher@gmail.com>
 International Business Machines Corporation
+Ilya Grashin <igrashin06@gmail.com>
 Ismael Jimenez Martinez <ismael.jimenez.martinez@gmail.com>
 Jern-Kuan Leong <jernkuan@gmail.com>
 JianXiong Zhou <zhoujianxiong2@gmail.com>

CONTRIBUTORS

@@ -60,8 +60,9 @@ Gergő Szitár <szitar.gergo@gmail.com>
 Haihan Jiang <haihanj99@gmail.com>
 Hannes Hauswedell <h2@fsfe.org>
 Henrique Bucher <hbucher@gmail.com>
-Ismael Jimenez Martinez <ismael.jimenez.martinez@gmail.com>
 Iakov Sergeev <yahontu@gmail.com>
+Ilya Grashin <igrashin06@gmail.com>
+Ismael Jimenez Martinez <ismael.jimenez.martinez@gmail.com>
 Jern-Kuan Leong <jernkuan@gmail.com>
 JianXiong Zhou <zhoujianxiong2@gmail.com>
 Joao Paulo Magalhaes <joaoppmagalhaes@gmail.com>

docs/perf_counters.md

@@ -30,6 +30,11 @@ they are platform specific, but some (e.g. `CYCLES` or `INSTRUCTIONS`) are
 mapped by libpfm to platform-specifics - see libpfm
 [documentation](http://perfmon2.sourceforge.net/docs.html) for more details.
 
+By default, counters measure user-mode events only. libpfm privilege modifiers
+can select a different scope: `:u` measures user mode, `:k` measures kernel
+mode, and `:u:k` measures both. Access to kernel-mode counters depends on the
+host's `perf_event_paranoid` setting and the process capabilities.
+
 The counter values are reported back through the [User Counters](../README.md#custom-counters)
 mechanism, meaning, they are available in all the formats (e.g. JSON) supported
 by User Counters.

src/perf_counters.cc

@@ -25,6 +25,7 @@
 #include <linux/perf_event.h>
 #include <sys/stat.h>
 
+#include "perf_counters_libpfm.h"
 #include "perfmon/pfmlib.h"
 #include "perfmon/pfmlib_perf_event.h"
 #endif
@@ -149,8 +150,6 @@ PerfCounters PerfCounters::Create(
   valid_names.reserve(counter_names.size());
   counter_ids.reserve(counter_names.size());
 
-  const int kCounterMode = PFM_PLM3;  // user mode only
-
   // Group leads will be assigned on demand. The idea is that once we cannot
   // create a counter descriptor, the reason is that this group has maxed out
   // so we set the group_id again to -1 and retry - giving the algorithm a
@@ -182,37 +181,16 @@ PerfCounters PerfCounters::Create(
     // Here first means first in group, ie the group leader
     const bool is_first = (group_id < 0);
 
-    // This struct will be populated by libpfm from the counter string
-    // and then fed into the syscall perf_event_open
+    // This struct will be populated by libpfm from the counter string and then
+    // fed into the syscall perf_event_open.
     struct perf_event_attr attr {};
-    attr.size = sizeof(attr);
-
-    // This is the input struct to libpfm.
-    pfm_perf_encode_arg_t arg{};
-    arg.attr = &attr;
-    const int pfm_get = pfm_get_os_event_encoding(name.c_str(), kCounterMode,
-                                                  PFM_OS_PERF_EVENT, &arg);
+    const int pfm_get = ConfigurePerfEventAttr(name.c_str(), is_first, &attr);
     if (pfm_get != PFM_SUCCESS) {
       GetErrorLogInstance()
           << "Unknown performance counter name: " << name << "\n";
       continue;
     }
 
-    // We then proceed to populate the remaining fields in our attribute struct
-    // Note: the man page for perf_event_create suggests inherit = true and
-    // read_format = PERF_FORMAT_GROUP don't work together, but that's not the
-    // case.
-    attr.disabled = is_first;
-    attr.inherit = true;
-    attr.pinned = is_first;
-    attr.exclude_kernel = true;
-    attr.exclude_user = false;
-    attr.exclude_hv = true;
-
-    // Read all counters in a group in one read.
-    attr.read_format = PERF_FORMAT_GROUP;  //| PERF_FORMAT_TOTAL_TIME_ENABLED |
-                                           // PERF_FORMAT_TOTAL_TIME_RUNNING;
-
     uint64_t base_config = attr.config;
     for (uint64_t pmu : GetPMUTypesForEvent(attr)) {
       attr.config = (pmu << PERF_PMU_TYPE_SHIFT) | base_config;

src/perf_counters_libpfm.h

@@ -0,0 +1,52 @@
+// Copyright 2021 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef BENCHMARK_PERF_COUNTERS_LIBPFM_H
+#define BENCHMARK_PERF_COUNTERS_LIBPFM_H
+
+#include <linux/perf_event.h>
+
+#include "perfmon/pfmlib.h"
+#include "perfmon/pfmlib_perf_event.h"
+
+namespace benchmark {
+namespace internal {
+
+inline int ConfigurePerfEventAttr(const char* name, bool is_group_leader,
+                                  perf_event_attr* attr) {
+  *attr = {};
+  attr->size = sizeof(*attr);
+
+  pfm_perf_encode_arg_t arg{};
+  arg.attr = attr;
+  const int kCounterMode = PFM_PLM3;  // user mode unless overridden by name
+  const int status =
+      pfm_get_os_event_encoding(name, kCounterMode, PFM_OS_PERF_EVENT, &arg);
+  if (status != PFM_SUCCESS) {
+    return status;
+  }
+
+  // Preserve the privilege exclusions encoded by libpfm for modifiers such as
+  // ":u" and ":k", and populate only the benchmark-owned group attributes.
+  attr->disabled = is_group_leader;
+  attr->inherit = true;
+  attr->pinned = is_group_leader;
+  attr->read_format = PERF_FORMAT_GROUP;
+  return PFM_SUCCESS;
+}
+
+}  // namespace internal
+}  // namespace benchmark
+
+#endif  // BENCHMARK_PERF_COUNTERS_LIBPFM_H

test/CMakeLists.txt

@@ -256,6 +256,10 @@ if (BENCHMARK_ENABLE_GTEST_TESTS)
   add_gtest(statistics_gtest)
   add_gtest(string_util_gtest)
   add_gtest(perf_counters_gtest)
+  if (PFM_FOUND)
+    target_compile_definitions(perf_counters_gtest PRIVATE HAVE_LIBPFM)
+    target_link_libraries(perf_counters_gtest PFM::libpfm)
+  endif()
   add_gtest(reporter_list_gtest)
   add_gtest(time_unit_gtest)
   add_gtest(min_time_parse_gtest)

test/perf_counters_gtest.cc

@@ -1,3 +1,4 @@
+#include <array>
 #include <mutex>
 #include <random>
 #include <set>
@@ -6,6 +7,12 @@
 #include <vector>
 
 #include "../src/perf_counters.h"
+#if defined HAVE_LIBPFM
+#include <linux/perf_event.h>
+
+#include "../src/perf_counters_libpfm.h"
+#include "perfmon/pfmlib.h"
+#endif
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
@@ -19,6 +26,9 @@ struct MsgHandler {
 using benchmark::internal::PerfCounters;
 using benchmark::internal::PerfCountersMeasurement;
 using benchmark::internal::PerfCounterValues;
+#if defined HAVE_LIBPFM
+using benchmark::internal::ConfigurePerfEventAttr;
+#endif
 using ::testing::AllOf;
 using ::testing::Gt;
 using ::testing::Lt;
@@ -49,6 +59,43 @@ TEST(PerfCountersTest, Init) {
   EXPECT_EQ(PerfCounters::Initialize(), PerfCounters::kSupported);
 }
 
+#if defined HAVE_LIBPFM
+TEST(PerfCountersTest, PreservesLibpfmPrivilegeModifiers) {
+  ASSERT_TRUE(PerfCounters::Initialize());
+
+  struct ExpectedPrivilegeMode {
+    const char* event_name;
+    bool exclude_user;
+    bool exclude_kernel;
+  };
+  const std::array<ExpectedPrivilegeMode, 4> modes = {{
+      {"INSTRUCTIONS", false, true},
+      {"INSTRUCTIONS:u", false, true},
+      {"INSTRUCTIONS:k", true, false},
+      {"INSTRUCTIONS:u:k", false, false},
+  }};
+
+  for (const auto& mode : modes) {
+    perf_event_attr attr{};
+    ASSERT_EQ(ConfigurePerfEventAttr(mode.event_name, true, &attr), PFM_SUCCESS)
+        << mode.event_name;
+    EXPECT_EQ(attr.exclude_user, mode.exclude_user) << mode.event_name;
+    EXPECT_EQ(attr.exclude_kernel, mode.exclude_kernel) << mode.event_name;
+    EXPECT_TRUE(attr.exclude_hv) << mode.event_name;
+    EXPECT_TRUE(attr.disabled) << mode.event_name;
+    EXPECT_TRUE(attr.inherit) << mode.event_name;
+    EXPECT_TRUE(attr.pinned) << mode.event_name;
+    EXPECT_EQ(attr.read_format, PERF_FORMAT_GROUP) << mode.event_name;
+  }
+
+  perf_event_attr follower_attr{};
+  ASSERT_EQ(ConfigurePerfEventAttr("INSTRUCTIONS", false, &follower_attr),
+            PFM_SUCCESS);
+  EXPECT_FALSE(follower_attr.disabled);
+  EXPECT_FALSE(follower_attr.pinned);
+}
+#endif
+
 TEST(PerfCountersTest, OneCounter) {
   if (!HasRequiredPerfCounters({kGenericPerfEvent1})) {
     GTEST_SKIP() << "Requested performance counters are not available.";