diff --git a/AUTHORS b/AUTHORS index ab26bb3ad5..f47ed41209 100644 --- a/AUTHORS +++ b/AUTHORS @@ -40,6 +40,7 @@ Google Inc. Haihan Jiang Henrique Bucher International Business Machines Corporation +Ilya Grashin Ismael Jimenez Martinez Jern-Kuan Leong JianXiong Zhou diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 88c3b1b327..ac40c8eaa9 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -60,8 +60,9 @@ Gergő Szitár Haihan Jiang Hannes Hauswedell Henrique Bucher -Ismael Jimenez Martinez Iakov Sergeev +Ilya Grashin +Ismael Jimenez Martinez Jern-Kuan Leong JianXiong Zhou Joao Paulo Magalhaes diff --git a/docs/perf_counters.md b/docs/perf_counters.md index f342092c99..77b5ca3d3a 100644 --- a/docs/perf_counters.md +++ b/docs/perf_counters.md @@ -30,6 +30,11 @@ they are platform specific, but some (e.g. `CYCLES` or `INSTRUCTIONS`) are mapped by libpfm to platform-specifics - see libpfm [documentation](http://perfmon2.sourceforge.net/docs.html) for more details. +By default, counters measure user-mode events only. libpfm privilege modifiers +can select a different scope: `:u` measures user mode, `:k` measures kernel +mode, and `:u:k` measures both. Access to kernel-mode counters depends on the +host's `perf_event_paranoid` setting and the process capabilities. + The counter values are reported back through the [User Counters](../README.md#custom-counters) mechanism, meaning, they are available in all the formats (e.g. JSON) supported by User Counters. diff --git a/src/perf_counters.cc b/src/perf_counters.cc index e6f220921c..0f3730362d 100644 --- a/src/perf_counters.cc +++ b/src/perf_counters.cc @@ -25,6 +25,7 @@ #include #include +#include "perf_counters_libpfm.h" #include "perfmon/pfmlib.h" #include "perfmon/pfmlib_perf_event.h" #endif @@ -149,8 +150,6 @@ PerfCounters PerfCounters::Create( valid_names.reserve(counter_names.size()); counter_ids.reserve(counter_names.size()); - const int kCounterMode = PFM_PLM3; // user mode only - // Group leads will be assigned on demand. The idea is that once we cannot // create a counter descriptor, the reason is that this group has maxed out // so we set the group_id again to -1 and retry - giving the algorithm a @@ -182,37 +181,16 @@ PerfCounters PerfCounters::Create( // Here first means first in group, ie the group leader const bool is_first = (group_id < 0); - // This struct will be populated by libpfm from the counter string - // and then fed into the syscall perf_event_open + // This struct will be populated by libpfm from the counter string and then + // fed into the syscall perf_event_open. struct perf_event_attr attr {}; - attr.size = sizeof(attr); - - // This is the input struct to libpfm. - pfm_perf_encode_arg_t arg{}; - arg.attr = &attr; - const int pfm_get = pfm_get_os_event_encoding(name.c_str(), kCounterMode, - PFM_OS_PERF_EVENT, &arg); + const int pfm_get = ConfigurePerfEventAttr(name.c_str(), is_first, &attr); if (pfm_get != PFM_SUCCESS) { GetErrorLogInstance() << "Unknown performance counter name: " << name << "\n"; continue; } - // We then proceed to populate the remaining fields in our attribute struct - // Note: the man page for perf_event_create suggests inherit = true and - // read_format = PERF_FORMAT_GROUP don't work together, but that's not the - // case. - attr.disabled = is_first; - attr.inherit = true; - attr.pinned = is_first; - attr.exclude_kernel = true; - attr.exclude_user = false; - attr.exclude_hv = true; - - // Read all counters in a group in one read. - attr.read_format = PERF_FORMAT_GROUP; //| PERF_FORMAT_TOTAL_TIME_ENABLED | - // PERF_FORMAT_TOTAL_TIME_RUNNING; - uint64_t base_config = attr.config; for (uint64_t pmu : GetPMUTypesForEvent(attr)) { attr.config = (pmu << PERF_PMU_TYPE_SHIFT) | base_config; diff --git a/src/perf_counters_libpfm.h b/src/perf_counters_libpfm.h new file mode 100644 index 0000000000..621200d5d6 --- /dev/null +++ b/src/perf_counters_libpfm.h @@ -0,0 +1,52 @@ +// Copyright 2021 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef BENCHMARK_PERF_COUNTERS_LIBPFM_H +#define BENCHMARK_PERF_COUNTERS_LIBPFM_H + +#include + +#include "perfmon/pfmlib.h" +#include "perfmon/pfmlib_perf_event.h" + +namespace benchmark { +namespace internal { + +inline int ConfigurePerfEventAttr(const char* name, bool is_group_leader, + perf_event_attr* attr) { + *attr = {}; + attr->size = sizeof(*attr); + + pfm_perf_encode_arg_t arg{}; + arg.attr = attr; + const int kCounterMode = PFM_PLM3; // user mode unless overridden by name + const int status = + pfm_get_os_event_encoding(name, kCounterMode, PFM_OS_PERF_EVENT, &arg); + if (status != PFM_SUCCESS) { + return status; + } + + // Preserve the privilege exclusions encoded by libpfm for modifiers such as + // ":u" and ":k", and populate only the benchmark-owned group attributes. + attr->disabled = is_group_leader; + attr->inherit = true; + attr->pinned = is_group_leader; + attr->read_format = PERF_FORMAT_GROUP; + return PFM_SUCCESS; +} + +} // namespace internal +} // namespace benchmark + +#endif // BENCHMARK_PERF_COUNTERS_LIBPFM_H diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fe88841dd9..15d7d43d87 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -256,6 +256,10 @@ if (BENCHMARK_ENABLE_GTEST_TESTS) add_gtest(statistics_gtest) add_gtest(string_util_gtest) add_gtest(perf_counters_gtest) + if (PFM_FOUND) + target_compile_definitions(perf_counters_gtest PRIVATE HAVE_LIBPFM) + target_link_libraries(perf_counters_gtest PFM::libpfm) + endif() add_gtest(reporter_list_gtest) add_gtest(time_unit_gtest) add_gtest(min_time_parse_gtest) diff --git a/test/perf_counters_gtest.cc b/test/perf_counters_gtest.cc index f3fd10bc95..382ab573e5 100644 --- a/test/perf_counters_gtest.cc +++ b/test/perf_counters_gtest.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -6,6 +7,12 @@ #include #include "../src/perf_counters.h" +#if defined HAVE_LIBPFM +#include + +#include "../src/perf_counters_libpfm.h" +#include "perfmon/pfmlib.h" +#endif #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -19,6 +26,9 @@ struct MsgHandler { using benchmark::internal::PerfCounters; using benchmark::internal::PerfCountersMeasurement; using benchmark::internal::PerfCounterValues; +#if defined HAVE_LIBPFM +using benchmark::internal::ConfigurePerfEventAttr; +#endif using ::testing::AllOf; using ::testing::Gt; using ::testing::Lt; @@ -49,6 +59,43 @@ TEST(PerfCountersTest, Init) { EXPECT_EQ(PerfCounters::Initialize(), PerfCounters::kSupported); } +#if defined HAVE_LIBPFM +TEST(PerfCountersTest, PreservesLibpfmPrivilegeModifiers) { + ASSERT_TRUE(PerfCounters::Initialize()); + + struct ExpectedPrivilegeMode { + const char* event_name; + bool exclude_user; + bool exclude_kernel; + }; + const std::array modes = {{ + {"INSTRUCTIONS", false, true}, + {"INSTRUCTIONS:u", false, true}, + {"INSTRUCTIONS:k", true, false}, + {"INSTRUCTIONS:u:k", false, false}, + }}; + + for (const auto& mode : modes) { + perf_event_attr attr{}; + ASSERT_EQ(ConfigurePerfEventAttr(mode.event_name, true, &attr), PFM_SUCCESS) + << mode.event_name; + EXPECT_EQ(attr.exclude_user, mode.exclude_user) << mode.event_name; + EXPECT_EQ(attr.exclude_kernel, mode.exclude_kernel) << mode.event_name; + EXPECT_TRUE(attr.exclude_hv) << mode.event_name; + EXPECT_TRUE(attr.disabled) << mode.event_name; + EXPECT_TRUE(attr.inherit) << mode.event_name; + EXPECT_TRUE(attr.pinned) << mode.event_name; + EXPECT_EQ(attr.read_format, PERF_FORMAT_GROUP) << mode.event_name; + } + + perf_event_attr follower_attr{}; + ASSERT_EQ(ConfigurePerfEventAttr("INSTRUCTIONS", false, &follower_attr), + PFM_SUCCESS); + EXPECT_FALSE(follower_attr.disabled); + EXPECT_FALSE(follower_attr.pinned); +} +#endif + TEST(PerfCountersTest, OneCounter) { if (!HasRequiredPerfCounters({kGenericPerfEvent1})) { GTEST_SKIP() << "Requested performance counters are not available.";