diff options
Diffstat (limited to 'tools/power')
52 files changed, 3605 insertions, 1683 deletions
diff --git a/tools/power/acpi/common/cmfsize.c b/tools/power/acpi/common/cmfsize.c index 68b9ea86b86c..af0e558f231c 100644 --- a/tools/power/acpi/common/cmfsize.c +++ b/tools/power/acpi/common/cmfsize.c @@ -3,7 +3,7 @@ * * Module Name: cmfsize - Common get file size function * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ diff --git a/tools/power/acpi/common/getopt.c b/tools/power/acpi/common/getopt.c index 6a0cdba6fdfd..3d63626d80e7 100644 --- a/tools/power/acpi/common/getopt.c +++ b/tools/power/acpi/common/getopt.c @@ -3,7 +3,7 @@ * * Module Name: getopt * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ diff --git a/tools/power/acpi/os_specific/service_layers/oslinuxtbl.c b/tools/power/acpi/os_specific/service_layers/oslinuxtbl.c index 9d70d8c945af..de93067a5da3 100644 --- a/tools/power/acpi/os_specific/service_layers/oslinuxtbl.c +++ b/tools/power/acpi/os_specific/service_layers/oslinuxtbl.c @@ -3,7 +3,7 @@ * * Module Name: oslinuxtbl - Linux OSL for obtaining ACPI tables * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ @@ -19,7 +19,7 @@ ACPI_MODULE_NAME("oslinuxtbl") typedef struct osl_table_info { struct osl_table_info *next; u32 instance; - char signature[ACPI_NAMESEG_SIZE]; + char signature[ACPI_NAMESEG_SIZE] ACPI_NONSTRING; } osl_table_info; @@ -995,7 +995,7 @@ static acpi_status osl_list_customized_tables(char *directory) { void *table_dir; u32 instance; - char temp_name[ACPI_NAMESEG_SIZE]; + char temp_name[ACPI_NAMESEG_SIZE] ACPI_NONSTRING; char *filename; acpi_status status = AE_OK; @@ -1312,7 +1312,7 @@ osl_get_customized_table(char *pathname, { void *table_dir; u32 current_instance = 0; - char temp_name[ACPI_NAMESEG_SIZE]; + char temp_name[ACPI_NAMESEG_SIZE] ACPI_NONSTRING; char table_filename[PATH_MAX]; char *filename; acpi_status status; diff --git a/tools/power/acpi/os_specific/service_layers/osunixdir.c b/tools/power/acpi/os_specific/service_layers/osunixdir.c index 39f3bffd9355..b9bb83116549 100644 --- a/tools/power/acpi/os_specific/service_layers/osunixdir.c +++ b/tools/power/acpi/os_specific/service_layers/osunixdir.c @@ -3,7 +3,7 @@ * * Module Name: osunixdir - Unix directory access interfaces * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ diff --git a/tools/power/acpi/os_specific/service_layers/osunixmap.c b/tools/power/acpi/os_specific/service_layers/osunixmap.c index 2b7d56252684..b93ebc9371a5 100644 --- a/tools/power/acpi/os_specific/service_layers/osunixmap.c +++ b/tools/power/acpi/os_specific/service_layers/osunixmap.c @@ -3,7 +3,7 @@ * * Module Name: osunixmap - Unix OSL for file mappings * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ diff --git a/tools/power/acpi/os_specific/service_layers/osunixxf.c b/tools/power/acpi/os_specific/service_layers/osunixxf.c index 46429417c71a..36f27491713c 100644 --- a/tools/power/acpi/os_specific/service_layers/osunixxf.c +++ b/tools/power/acpi/os_specific/service_layers/osunixxf.c @@ -3,7 +3,7 @@ * * Module Name: osunixxf - UNIX OSL interfaces * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ diff --git a/tools/power/acpi/tools/acpidump/acpidump.h b/tools/power/acpi/tools/acpidump/acpidump.h index 643e3e722340..fe0d23c4f2ed 100644 --- a/tools/power/acpi/tools/acpidump/acpidump.h +++ b/tools/power/acpi/tools/acpidump/acpidump.h @@ -3,7 +3,7 @@ * * Module Name: acpidump.h - Include file for acpi_dump utility * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ diff --git a/tools/power/acpi/tools/acpidump/apdump.c b/tools/power/acpi/tools/acpidump/apdump.c index 0742b00b61a1..7a6223aa703c 100644 --- a/tools/power/acpi/tools/acpidump/apdump.c +++ b/tools/power/acpi/tools/acpidump/apdump.c @@ -3,7 +3,7 @@ * * Module Name: apdump - Dump routines for ACPI tables (acpidump) * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ @@ -86,9 +86,10 @@ u8 ap_is_valid_checksum(struct acpi_table_header *table) if (ACPI_FAILURE(status)) { fprintf(stderr, "%4.4s: Warning: wrong checksum in table\n", table->signature); + return (FALSE); } - return (AE_OK); + return (TRUE); } /****************************************************************************** diff --git a/tools/power/acpi/tools/acpidump/apfiles.c b/tools/power/acpi/tools/acpidump/apfiles.c index 13817f9112c0..d6b8a201480b 100644 --- a/tools/power/acpi/tools/acpidump/apfiles.c +++ b/tools/power/acpi/tools/acpidump/apfiles.c @@ -3,7 +3,7 @@ * * Module Name: apfiles - File-related functions for acpidump utility * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ diff --git a/tools/power/acpi/tools/acpidump/apmain.c b/tools/power/acpi/tools/acpidump/apmain.c index 666a9675e743..9f3850e3af5b 100644 --- a/tools/power/acpi/tools/acpidump/apmain.c +++ b/tools/power/acpi/tools/acpidump/apmain.c @@ -3,7 +3,7 @@ * * Module Name: apmain - Main module for the acpidump utility * - * Copyright (C) 2000 - 2023, Intel Corp. + * Copyright (C) 2000 - 2025, Intel Corp. * *****************************************************************************/ diff --git a/tools/power/acpi/tools/pfrut/pfrut.c b/tools/power/acpi/tools/pfrut/pfrut.c index 44a9ecbd91e8..4d9b0177c312 100644 --- a/tools/power/acpi/tools/pfrut/pfrut.c +++ b/tools/power/acpi/tools/pfrut/pfrut.c @@ -222,6 +222,7 @@ int main(int argc, char *argv[]) fd_update_log = open("/dev/acpi_pfr_telemetry0", O_RDWR); if (fd_update_log < 0) { printf("PFRT device not supported - Quit...\n"); + close(fd_update); return 1; } @@ -265,7 +266,8 @@ int main(int argc, char *argv[]) printf("chunk2_size:%d\n", data_info.chunk2_size); printf("rollover_cnt:%d\n", data_info.rollover_cnt); printf("reset_cnt:%d\n", data_info.reset_cnt); - + close(fd_update); + close(fd_update_log); return 0; } @@ -358,6 +360,7 @@ int main(int argc, char *argv[]) if (ret == -1) { perror("Failed to load capsule file"); + munmap(addr_map_capsule, st.st_size); close(fd_capsule); close(fd_update); close(fd_update_log); @@ -420,7 +423,7 @@ int main(int argc, char *argv[]) if (p_mmap == MAP_FAILED) { perror("mmap error."); close(fd_update_log); - + free(log_buf); return 1; } diff --git a/tools/power/cpupower/.gitignore b/tools/power/cpupower/.gitignore index 5113d5a7aee0..7677329c42a6 100644 --- a/tools/power/cpupower/.gitignore +++ b/tools/power/cpupower/.gitignore @@ -27,6 +27,3 @@ debug/i386/intel_gsic debug/i386/powernow-k8-decode debug/x86_64/centrino-decode debug/x86_64/powernow-k8-decode - -# Clang's compilation database file -compile_commands.json diff --git a/tools/power/cpupower/Makefile b/tools/power/cpupower/Makefile index 51a95239fe06..969716dfe8de 100644 --- a/tools/power/cpupower/Makefile +++ b/tools/power/cpupower/Makefile @@ -2,6 +2,7 @@ # Makefile for cpupower # # Copyright (C) 2005,2006 Dominik Brodowski <linux@dominikbrodowski.net> +# Copyright (C) 2025 Francesco Poli <invernomuto@paranoici.org> # # Based largely on the Makefile for udev by: # @@ -36,9 +37,7 @@ NLS ?= true # cpufreq-bench benchmarking tool CPUFREQ_BENCH ?= true -# Do not build libraries, but build the code in statically -# Libraries are still built, otherwise the Makefile code would -# be rather ugly. +# Build the code, including libraries, statically. export STATIC ?= false # Prefix to the directories we're installing to @@ -52,8 +51,11 @@ DESTDIR ?= # and _should_ modify the PACKAGE_BUGREPORT definition VERSION:= $(shell ./utils/version-gen.sh) -LIB_MAJ= 0.0.1 -LIB_MIN= 1 +LIB_FIX= 1 +LIB_MIN= 0 +LIB_MAJ= 1 +LIB_VER= $(LIB_MAJ).$(LIB_MIN).$(LIB_FIX) + PACKAGE = cpupower PACKAGE_BUGREPORT = linux-pm@vger.kernel.org @@ -68,6 +70,8 @@ bindir ?= /usr/bin sbindir ?= /usr/sbin mandir ?= /usr/man libdir ?= /usr/lib +libexecdir ?= /usr/libexec +unitdir ?= /usr/lib/systemd/system includedir ?= /usr/include localedir ?= /usr/share/locale docdir ?= /usr/share/doc/packages/cpupower @@ -80,6 +84,7 @@ CP = cp -fpR INSTALL = /usr/bin/install -c INSTALL_PROGRAM = ${INSTALL} INSTALL_DATA = ${INSTALL} -m 644 +SETPERM_DATA = chmod 644 #bash completion scripts get sourced and so they should be rw only. INSTALL_SCRIPT = ${INSTALL} -m 644 @@ -200,14 +205,25 @@ $(OUTPUT)lib/%.o: $(LIB_SRC) $(LIB_HEADERS) $(ECHO) " CC " $@ $(QUIET) $(CC) $(CFLAGS) -fPIC -o $@ -c lib/$*.c -$(OUTPUT)libcpupower.so.$(LIB_MAJ): $(LIB_OBJS) +ifeq ($(strip $(STATIC)),true) +LIBCPUPOWER := libcpupower.a +else +LIBCPUPOWER := libcpupower.so.$(LIB_VER) +endif + +$(OUTPUT)$(LIBCPUPOWER): $(LIB_OBJS) +ifeq ($(strip $(STATIC)),true) + $(ECHO) " AR " $@ + $(QUIET) $(AR) rcs $@ $(LIB_OBJS) +else $(ECHO) " LD " $@ $(QUIET) $(CC) -shared $(CFLAGS) $(LDFLAGS) -o $@ \ - -Wl,-soname,libcpupower.so.$(LIB_MIN) $(LIB_OBJS) + -Wl,-soname,libcpupower.so.$(LIB_MAJ) $(LIB_OBJS) @ln -sf $(@F) $(OUTPUT)libcpupower.so - @ln -sf $(@F) $(OUTPUT)libcpupower.so.$(LIB_MIN) + @ln -sf $(@F) $(OUTPUT)libcpupower.so.$(LIB_MAJ) +endif -libcpupower: $(OUTPUT)libcpupower.so.$(LIB_MAJ) +libcpupower: $(OUTPUT)$(LIBCPUPOWER) # Let all .o files depend on its .c file and all headers # Might be worth to put this into utils/Makefile at some point of time @@ -217,7 +233,7 @@ $(OUTPUT)%.o: %.c $(ECHO) " CC " $@ $(QUIET) $(CC) $(CFLAGS) -I./lib -I ./utils -o $@ -c $*.c -$(OUTPUT)cpupower: $(UTIL_OBJS) $(OUTPUT)libcpupower.so.$(LIB_MAJ) +$(OUTPUT)cpupower: $(UTIL_OBJS) $(OUTPUT)$(LIBCPUPOWER) $(ECHO) " CC " $@ ifeq ($(strip $(STATIC)),true) $(QUIET) $(CC) $(CFLAGS) $(LDFLAGS) $(UTIL_OBJS) -lrt -lpci -L$(OUTPUT) -o $@ @@ -262,7 +278,7 @@ update-po: $(OUTPUT)po/$(PACKAGE).pot done; endif -compile-bench: $(OUTPUT)libcpupower.so.$(LIB_MAJ) +compile-bench: $(OUTPUT)$(LIBCPUPOWER) @V=$(V) confdir=$(confdir) $(MAKE) -C bench O=$(OUTPUT) # we compile into subdirectories. if the target directory is not the @@ -280,6 +296,7 @@ clean: -find $(OUTPUT) \( -not -type d \) -and \( -name '*~' -o -name '*.[oas]' \) -type f -print \ | xargs rm -f -rm -f $(OUTPUT)cpupower + -rm -f $(OUTPUT)libcpupower.a -rm -f $(OUTPUT)libcpupower.so* -rm -rf $(OUTPUT)po/*.gmo -rm -rf $(OUTPUT)po/*.pot @@ -288,17 +305,35 @@ clean: install-lib: libcpupower $(INSTALL) -d $(DESTDIR)${libdir} +ifeq ($(strip $(STATIC)),true) + $(CP) $(OUTPUT)libcpupower.a $(DESTDIR)${libdir}/ +else $(CP) $(OUTPUT)libcpupower.so* $(DESTDIR)${libdir}/ +endif $(INSTALL) -d $(DESTDIR)${includedir} $(INSTALL_DATA) lib/cpufreq.h $(DESTDIR)${includedir}/cpufreq.h $(INSTALL_DATA) lib/cpuidle.h $(DESTDIR)${includedir}/cpuidle.h $(INSTALL_DATA) lib/powercap.h $(DESTDIR)${includedir}/powercap.h -install-tools: $(OUTPUT)cpupower +# SYSTEMD=false disables installation of the systemd unit file +SYSTEMD ?= true + +install-systemd: + $(INSTALL) -d $(DESTDIR)${unitdir} + sed 's|___CDIR___|${confdir}|; s|___LDIR___|${libexecdir}|' cpupower.service.in > '$(DESTDIR)${unitdir}/cpupower.service' + $(SETPERM_DATA) '$(DESTDIR)${unitdir}/cpupower.service' + +INSTALL_SYSTEMD := $(if $(filter true,$(strip $(SYSTEMD))),install-systemd) + +install-tools: $(OUTPUT)cpupower $(INSTALL_SYSTEMD) $(INSTALL) -d $(DESTDIR)${bindir} $(INSTALL_PROGRAM) $(OUTPUT)cpupower $(DESTDIR)${bindir} $(INSTALL) -d $(DESTDIR)${bash_completion_dir} $(INSTALL_SCRIPT) cpupower-completion.sh '$(DESTDIR)${bash_completion_dir}/cpupower' + $(INSTALL) -d $(DESTDIR)${confdir} + $(INSTALL_DATA) cpupower-service.conf '$(DESTDIR)${confdir}' + $(INSTALL) -d $(DESTDIR)${libexecdir} + $(INSTALL_PROGRAM) cpupower.sh '$(DESTDIR)${libexecdir}/cpupower' install-man: $(INSTALL_DATA) -D man/cpupower.1 $(DESTDIR)${mandir}/man1/cpupower.1 @@ -322,17 +357,16 @@ install-bench: compile-bench @#DESTDIR must be set from outside to survive @sbindir=$(sbindir) bindir=$(bindir) docdir=$(docdir) confdir=$(confdir) $(MAKE) -C bench O=$(OUTPUT) install -ifeq ($(strip $(STATIC)),true) -install: all install-tools install-man $(INSTALL_NLS) $(INSTALL_BENCH) -else install: all install-lib install-tools install-man $(INSTALL_NLS) $(INSTALL_BENCH) -endif uninstall: - rm -f $(DESTDIR)${libdir}/libcpupower.* - rm -f $(DESTDIR)${includedir}/cpufreq.h - rm -f $(DESTDIR)${includedir}/cpuidle.h - rm -f $(DESTDIR)${bindir}/utils/cpupower + - rm -f $(DESTDIR)${confdir}cpupower-service.conf + - rm -f $(DESTDIR)${libexecdir}/cpupower + - rm -f $(DESTDIR)${unitdir}/cpupower.service - rm -f $(DESTDIR)${mandir}/man1/cpupower.1 - rm -f $(DESTDIR)${mandir}/man1/cpupower-frequency-set.1 - rm -f $(DESTDIR)${mandir}/man1/cpupower-frequency-info.1 @@ -379,4 +413,4 @@ help: @echo ' uninstall - Remove previously installed files from the dir defined by "DESTDIR"' @echo ' cmdline or Makefile config block option (default: "")' -.PHONY: all utils libcpupower update-po create-gmo install-lib install-tools install-man install-gmo install uninstall clean help +.PHONY: all utils libcpupower update-po create-gmo install-lib install-systemd install-tools install-man install-gmo install uninstall clean help diff --git a/tools/power/cpupower/README b/tools/power/cpupower/README index 2678ed81d311..9de449469568 100644 --- a/tools/power/cpupower/README +++ b/tools/power/cpupower/README @@ -59,6 +59,10 @@ $ sudo make install ----------------------------------------------------------------------- | man pages | /usr/man | ----------------------------------------------------------------------- +| systemd service | /usr/lib/systemd/system | +----------------------------------------------------------------------- +| systemd support script | /usr/libexec | +----------------------------------------------------------------------- To put it in other words it makes build results available system-wide, enabling any user to simply start using it without any additional steps @@ -109,6 +113,10 @@ The files will be installed to the following dirs: ----------------------------------------------------------------------- | man pages | ${DESTDIR}/usr/man | ----------------------------------------------------------------------- +| systemd service | ${DESTDIR}/usr/lib/systemd/system | +----------------------------------------------------------------------- +| systemd support script | ${DESTDIR}/usr/libexec | +----------------------------------------------------------------------- If you look at the table for the default 'make' output dirs you will notice that the only difference with the non-default case is the @@ -173,6 +181,26 @@ The issue is that binary cannot find the 'libcpupower' library. So, we shall point to the lib dir: sudo LD_LIBRARY_PATH=lib64/ ./bin/cpupower +systemd service +--------------- + +A systemd service is also provided to run the cpupower utility at boot with +settings read from a configuration file. + +If you want systemd to find the new service after the installation, the service +unit must have been installed in one of the system unit search path directories +(such as '/usr/lib/systemd/system/', which is the default location) and (unless +you are willing to wait for the next reboot) you need to issue the following +command: + +$ sudo systemctl daemon-reload + +If you want to enable this systemd service, edit '/etc/cpupower-service.conf' +(uncommenting at least one of the options, depending on your preferences) +and then issue the following command: + +$ sudo systemctl enable --now cpupower.service + THANKS ------ diff --git a/tools/power/cpupower/bench/parse.c b/tools/power/cpupower/bench/parse.c index 080678d9d74e..bd67c758b33a 100644 --- a/tools/power/cpupower/bench/parse.c +++ b/tools/power/cpupower/bench/parse.c @@ -121,6 +121,10 @@ out_dir: struct config *prepare_default_config() { struct config *config = malloc(sizeof(struct config)); + if (!config) { + perror("malloc"); + return NULL; + } dprintf("loading defaults\n"); diff --git a/tools/power/cpupower/bindings/python/Makefile b/tools/power/cpupower/bindings/python/Makefile index 741f21477432..4527cd732b42 100644 --- a/tools/power/cpupower/bindings/python/Makefile +++ b/tools/power/cpupower/bindings/python/Makefile @@ -1,25 +1,25 @@ # SPDX-License-Identifier: GPL-2.0-only # Makefile for libcpupower's Python bindings # -# This Makefile expects you have already run the makefile for cpupower to build -# the .o files in the lib directory for the bindings to be created. +# This Makefile expects you have already run `make install-lib` in the lib +# directory for the bindings to be created. -CC := gcc +CC ?= gcc +# CFLAGS ?= +LDFLAGS ?= -lcpupower HAVE_SWIG := $(shell if which swig >/dev/null 2>&1; then echo 1; else echo 0; fi) HAVE_PYCONFIG := $(shell if which python-config >/dev/null 2>&1; then echo 1; else echo 0; fi) -LIB_DIR := ../../lib -PY_INCLUDE = $(firstword $(shell python-config --includes)) -OBJECTS_LIB = $(wildcard $(LIB_DIR)/*.o) -INSTALL_DIR = $(shell python3 -c "import site; print(site.getsitepackages()[0])") +PY_INCLUDE ?= $(firstword $(shell python-config --includes)) +INSTALL_DIR ?= $(shell python3 -c "import site; print(site.getsitepackages()[0])") all: _raw_pylibcpupower.so _raw_pylibcpupower.so: raw_pylibcpupower_wrap.o - $(CC) -shared $(OBJECTS_LIB) raw_pylibcpupower_wrap.o -o _raw_pylibcpupower.so + $(CC) -shared $(LDFLAGS) raw_pylibcpupower_wrap.o -o _raw_pylibcpupower.so raw_pylibcpupower_wrap.o: raw_pylibcpupower_wrap.c - $(CC) -fPIC -c raw_pylibcpupower_wrap.c $(PY_INCLUDE) + $(CC) $(CFLAGS) $(PY_INCLUDE) -fPIC -c raw_pylibcpupower_wrap.c raw_pylibcpupower_wrap.c: raw_pylibcpupower.swg ifeq ($(HAVE_SWIG),0) diff --git a/tools/power/cpupower/bindings/python/README b/tools/power/cpupower/bindings/python/README index 952e2e02fd32..2a4896b648b7 100644 --- a/tools/power/cpupower/bindings/python/README +++ b/tools/power/cpupower/bindings/python/README @@ -5,18 +5,21 @@ libcpupower (aside from the libcpupower object files). requirements ------------ -* You need the object files in the libcpupower directory compiled by -cpupower's makefile. +* If you are building completely from upstream; please install libcpupower by +running `make install-lib` within the cpupower directory. This installs the +libcpupower.so file and symlinks needed. Otherwise, please make sure a symlink +to libcpupower.so exists in your library path from your distribution's +packages. * The SWIG program must be installed. -* The Python's development libraries installed. +* The Python's development libraries must be installed. Please check that your version of SWIG is compatible with the version of Python installed on your machine by checking the SWIG changelog on their website. https://swig.org/ Note that while SWIG itself is GPL v3+ licensed; the resulting output, -the bindings code: is permissively licensed + the license of libcpupower's .o -files. For these bindings that means GPL v2. +the bindings code: is permissively licensed + the license of libcpupower's +library files. For these bindings that means GPL v2. Please see https://swig.org/legal.html and the discussion [1] for more details. diff --git a/tools/power/cpupower/cpupower-service.conf b/tools/power/cpupower/cpupower-service.conf new file mode 100644 index 000000000000..abbb46967565 --- /dev/null +++ b/tools/power/cpupower/cpupower-service.conf @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2012, Sébastien Luttringer +# Copyright (C) 2024-2025, Francesco Poli <invernomuto@paranoici.org> + +# Configuration file for cpupower.service systemd service unit +# +# Edit this file (uncommenting at least one of the options, depending on +# your preferences) and then enable cpupower.service, if you want cpupower +# to run at boot with these settings. + +# --- CPU clock frequency --- + +# Define CPU governor +# Valid governors: ondemand, performance, powersave, conservative, userspace +#GOVERNOR='ondemand' + +# Limit frequency range +# Valid suffixes: Hz, kHz (default), MHz, GHz, THz +#MIN_FREQ="2.25GHz" +#MAX_FREQ="3GHz" + +# Set a specific frequency +# Requires userspace governor to be available. +# If this option is set, all the previous frequency options are ignored +#FREQ= + +# --- CPU policy --- + +# Set a register on supported Intel processore which allows software to convey +# its policy for the relative importance of performance versus energy savings to +# the processor. See man CPUPOWER-SET(1) for additional details +#PERF_BIAS= + +# Set the Energy Performance Preference +# Available options can be read from +# /sys/devices/system/cpu/cpufreq/policy0/energy_performance_available_preferences +#EPP= diff --git a/tools/power/cpupower/cpupower.service.in b/tools/power/cpupower/cpupower.service.in new file mode 100644 index 000000000000..fbd5b8c14270 --- /dev/null +++ b/tools/power/cpupower/cpupower.service.in @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2012-2020, Sébastien Luttringer +# Copyright (C) 2024-2025, Francesco Poli <invernomuto@paranoici.org> + +[Unit] +Description=Apply cpupower configuration +ConditionVirtualization=!container + +[Service] +Type=oneshot +EnvironmentFile=-___CDIR___cpupower-service.conf +ExecStart=___LDIR___/cpupower +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/tools/power/cpupower/cpupower.sh b/tools/power/cpupower/cpupower.sh new file mode 100644 index 000000000000..6283e8bf275d --- /dev/null +++ b/tools/power/cpupower/cpupower.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2012, Sébastien Luttringer +# Copyright (C) 2024, Francesco Poli <invernomuto@paranoici.org> + +ESTATUS=0 + +# apply CPU clock frequency options +if test -n "$FREQ" +then + cpupower frequency-set -f "$FREQ" > /dev/null || ESTATUS=1 +elif test -n "${GOVERNOR}${MIN_FREQ}${MAX_FREQ}" +then + cpupower frequency-set \ + ${GOVERNOR:+ -g "$GOVERNOR"} \ + ${MIN_FREQ:+ -d "$MIN_FREQ"} ${MAX_FREQ:+ -u "$MAX_FREQ"} \ + > /dev/null || ESTATUS=1 +fi + +# apply CPU policy options +if test -n "$PERF_BIAS" +then + cpupower set -b "$PERF_BIAS" > /dev/null || ESTATUS=1 +fi + +# apply Energy Performance Preference +if test -n "$EPP" +then + cpupower set -e "$EPP" > /dev/null || ESTATUS=1 +fi + +exit $ESTATUS diff --git a/tools/power/cpupower/lib/cpuidle.c b/tools/power/cpupower/lib/cpuidle.c index 0ecac009273c..2fcb343d8e75 100644 --- a/tools/power/cpupower/lib/cpuidle.c +++ b/tools/power/cpupower/lib/cpuidle.c @@ -150,6 +150,7 @@ unsigned long long cpuidle_state_get_one_value(unsigned int cpu, if (len == 0) return 0; + errno = 0; value = strtoull(linebuf, &endp, 0); if (endp == linebuf || errno == ERANGE) @@ -193,8 +194,7 @@ static char *cpuidle_state_get_one_string(unsigned int cpu, if (result == NULL) return NULL; - if (result[strlen(result) - 1] == '\n') - result[strlen(result) - 1] = '\0'; + result[strcspn(result, "\n")] = '\0'; return result; } @@ -233,6 +233,7 @@ int cpuidle_state_disable(unsigned int cpu, { char value[SYSFS_PATH_MAX]; int bytes_written; + int len; if (cpuidle_state_count(cpu) <= idlestate) return -1; @@ -241,10 +242,10 @@ int cpuidle_state_disable(unsigned int cpu, idlestate_value_files[IDLESTATE_DISABLE])) return -2; - snprintf(value, SYSFS_PATH_MAX, "%u", disable); + len = snprintf(value, SYSFS_PATH_MAX, "%u", disable); bytes_written = cpuidle_state_write_file(cpu, idlestate, "disable", - value, sizeof(disable)); + value, len); if (bytes_written) return 0; return -3; @@ -365,8 +366,7 @@ static char *sysfs_cpuidle_get_one_string(enum cpuidle_string which) if (result == NULL) return NULL; - if (result[strlen(result) - 1] == '\n') - result[strlen(result) - 1] = '\0'; + result[strcspn(result, "\n")] = '\0'; return result; } diff --git a/tools/power/cpupower/lib/cpupower.c b/tools/power/cpupower/lib/cpupower.c index 7a2ef691b20e..d7f7ec6f151c 100644 --- a/tools/power/cpupower/lib/cpupower.c +++ b/tools/power/cpupower/lib/cpupower.c @@ -10,6 +10,7 @@ #include <stdio.h> #include <errno.h> #include <stdlib.h> +#include <string.h> #include "cpupower.h" #include "cpupower_intern.h" @@ -55,7 +56,7 @@ unsigned int cpupower_write_sysfs(const char *path, char *buf, size_t buflen) if (numwritten < 1) { perror(path); close(fd); - return -1; + return 0; } close(fd); @@ -150,15 +151,25 @@ static int __compare(const void *t1, const void *t2) return 0; } +static int __compare_core_cpu_list(const void *t1, const void *t2) +{ + struct cpuid_core_info *top1 = (struct cpuid_core_info *)t1; + struct cpuid_core_info *top2 = (struct cpuid_core_info *)t2; + + return strcmp(top1->core_cpu_list, top2->core_cpu_list); +} + /* * Returns amount of cpus, negative on error, cpu_top must be * passed to cpu_topology_release to free resources * - * Array is sorted after ->pkg, ->core, then ->cpu + * Array is sorted after ->cpu_smt_list ->pkg, ->core */ int get_cpu_topology(struct cpupower_topology *cpu_top) { int cpu, last_pkg, cpus = sysconf(_SC_NPROCESSORS_CONF); + char path[SYSFS_PATH_MAX]; + char *last_cpu_list; cpu_top->core_info = malloc(sizeof(struct cpuid_core_info) * cpus); if (cpu_top->core_info == NULL) @@ -183,6 +194,34 @@ int get_cpu_topology(struct cpupower_topology *cpu_top) cpu_top->core_info[cpu].core = -1; continue; } + if (cpu_top->core_info[cpu].core == -1) { + strncpy(cpu_top->core_info[cpu].core_cpu_list, "-1", CPULIST_BUFFER); + continue; + } + snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/topology/%s", + cpu, "core_cpus_list"); + if (cpupower_read_sysfs( + path, + cpu_top->core_info[cpu].core_cpu_list, + CPULIST_BUFFER) < 1) { + printf("Warning CPU%u has a 0 size core_cpus_list string", cpu); + } + } + + /* Count the number of distinct cpu lists to get the physical core + * count. + */ + qsort(cpu_top->core_info, cpus, sizeof(struct cpuid_core_info), + __compare_core_cpu_list); + + last_cpu_list = cpu_top->core_info[0].core_cpu_list; + cpu_top->cores = 1; + for (cpu = 1; cpu < cpus; cpu++) { + if (strcmp(cpu_top->core_info[cpu].core_cpu_list, last_cpu_list) != 0 && + cpu_top->core_info[cpu].pkg != -1) { + last_cpu_list = cpu_top->core_info[cpu].core_cpu_list; + cpu_top->cores++; + } } qsort(cpu_top->core_info, cpus, sizeof(struct cpuid_core_info), @@ -203,13 +242,6 @@ int get_cpu_topology(struct cpupower_topology *cpu_top) if (!(cpu_top->core_info[0].pkg == -1)) cpu_top->pkgs++; - /* Intel's cores count is not consecutively numbered, there may - * be a core_id of 3, but none of 2. Assume there always is 0 - * Get amount of cores by counting duplicates in a package - for (cpu = 0; cpu_top->core_info[cpu].pkg = 0 && cpu < cpus; cpu++) { - if (cpu_top->core_info[cpu].core == 0) - cpu_top->cores++; - */ return cpus; } diff --git a/tools/power/cpupower/lib/cpupower.h b/tools/power/cpupower/lib/cpupower.h index e4e4292eacec..2e67a080f203 100644 --- a/tools/power/cpupower/lib/cpupower.h +++ b/tools/power/cpupower/lib/cpupower.h @@ -2,6 +2,8 @@ #ifndef __CPUPOWER_CPUPOWER_H__ #define __CPUPOWER_CPUPOWER_H__ +#define CPULIST_BUFFER 5 + struct cpupower_topology { /* Amount of CPU cores, packages and threads per core in the system */ unsigned int cores; @@ -16,6 +18,7 @@ struct cpuid_core_info { int pkg; int core; int cpu; + char core_cpu_list[CPULIST_BUFFER]; /* flags */ unsigned int is_online:1; diff --git a/tools/power/cpupower/man/cpupower-frequency-info.1 b/tools/power/cpupower/man/cpupower-frequency-info.1 index 47fdd7218748..b0d69c9adcbd 100644 --- a/tools/power/cpupower/man/cpupower-frequency-info.1 +++ b/tools/power/cpupower/man/cpupower-frequency-info.1 @@ -32,6 +32,12 @@ Gets the currently used cpufreq policy. \fB\-g\fR \fB\-\-governors\fR Determines available cpufreq governors. .TP +\fB\-b\fR \fB\-\-boost\fR +Gets the current boost state support. +.TP +\fB\-z\fR \fB\-\-epp\fR +Gets the current EPP (energy performance preference). +.TP \fB\-r\fR \fB\-\-related\-cpus\fR Determines which CPUs run at the same hardware frequency. .TP @@ -53,7 +59,7 @@ human\-readable output for the \-f, \-w, \-s and \-y parameters. \fB\-n\fR \fB\-\-no-rounding\fR Output frequencies and latencies without rounding off values. .TP -\fB\-c\fR \fB\-\-perf\fR +\fB\-c\fR \fB\-\-performance\fR Get performances and frequencies capabilities of CPPC, by reading it from hardware (only available on the hardware with CPPC). .TP .SH "REMARKS" diff --git a/tools/power/cpupower/man/cpupower-idle-info.1 b/tools/power/cpupower/man/cpupower-idle-info.1 index 20b6345c53ad..b2f92aba5f5b 100644 --- a/tools/power/cpupower/man/cpupower-idle-info.1 +++ b/tools/power/cpupower/man/cpupower-idle-info.1 @@ -11,10 +11,10 @@ A tool which prints out per cpu idle information helpful to developers and inter .SH "OPTIONS" .LP .TP -\fB\-f\fR \fB\-\-silent\fR +\fB\-s\fR \fB\-\-silent\fR Only print a summary of all available C-states in the system. .TP -\fB\-e\fR \fB\-\-proc\fR +\fB\-o\fR \fB\-\-proc\fR deprecated. Prints out idle information in old /proc/acpi/processor/*/power format. This interface has been removed from the kernel for quite some time, do not let diff --git a/tools/power/cpupower/man/cpupower-info.1 b/tools/power/cpupower/man/cpupower-info.1 index 340bcd0be7de..1f42d8c388a0 100644 --- a/tools/power/cpupower/man/cpupower-info.1 +++ b/tools/power/cpupower/man/cpupower-info.1 @@ -3,7 +3,7 @@ cpupower\-info \- Shows processor power related kernel or hardware configurations .SH SYNOPSIS .ft B -.B cpupower info [ \-b ] +.B cpupower info [\fIoptions\fP] .SH DESCRIPTION \fBcpupower info \fP shows kernel configurations or processor hardware @@ -13,6 +13,13 @@ Some options are platform wide, some affect single cores. By default values of core zero are displayed only. cpupower --cpu all cpuinfo will show the settings of all cores, see cpupower(1) how to choose specific cores. +.SH "OPTIONS" +.LP +.TP +\fB\-b\fR \fB\-\-perf-bias\fR +Gets the current performance bias value. +.TP + .SH "SEE ALSO" Options are described in detail in: diff --git a/tools/power/cpupower/man/cpupower-set.1 b/tools/power/cpupower/man/cpupower-set.1 index 500653ef98c7..8ac82b6f9189 100644 --- a/tools/power/cpupower/man/cpupower-set.1 +++ b/tools/power/cpupower/man/cpupower-set.1 @@ -81,10 +81,11 @@ Refer to the AMD P-State kernel documentation for further information. .RE .PP -\-\-turbo\-boost, \-t +\-\-turbo\-boost, \-\-boost, \-t .RS 4 -This option is used to enable or disable the turbo boost feature on -supported Intel and AMD processors. +This option is used to enable or disable the boost feature on +supported Intel and AMD processors, and other boost supported systems. +(The --boost option is an alias for the --turbo-boost option) This option takes as parameter either \fB1\fP to enable, or \fB0\fP to disable the feature. diff --git a/tools/power/cpupower/utils/cpufreq-info.c b/tools/power/cpupower/utils/cpufreq-info.c index fc750e127404..5a242b491a9d 100644 --- a/tools/power/cpupower/utils/cpufreq-info.c +++ b/tools/power/cpupower/utils/cpufreq-info.c @@ -128,7 +128,7 @@ static int get_boost_mode_x86(unsigned int cpu) /* ToDo: Make this more global */ unsigned long pstates[MAX_HW_PSTATES] = {0,}; - ret = cpufreq_has_boost_support(cpu, &support, &active, &b_states); + ret = cpufreq_has_x86_boost_support(cpu, &support, &active, &b_states); if (ret) { printf(_("Error while evaluating Boost Capabilities" " on CPU %d -- are you root?\n"), cpu); @@ -204,6 +204,18 @@ static int get_boost_mode_x86(unsigned int cpu) return 0; } +static int get_boost_mode_generic(unsigned int cpu) +{ + bool active; + + if (!cpufreq_has_generic_boost_support(&active)) { + printf(_(" boost state support:\n")); + printf(_(" Active: %s\n"), active ? _("yes") : _("no")); + } + + return 0; +} + /* --boost / -b */ static int get_boost_mode(unsigned int cpu) @@ -214,6 +226,8 @@ static int get_boost_mode(unsigned int cpu) cpupower_cpu_info.vendor == X86_VENDOR_HYGON || cpupower_cpu_info.vendor == X86_VENDOR_INTEL) return get_boost_mode_x86(cpu); + else + get_boost_mode_generic(cpu); freqs = cpufreq_get_boost_frequencies(cpu); if (freqs) { @@ -256,7 +270,7 @@ static int get_freq_hardware(unsigned int cpu, unsigned int human) { unsigned long freq; - if (cpupower_cpu_info.caps & CPUPOWER_CAP_APERF) + if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_APERF)) return -EINVAL; freq = cpufreq_get_freq_hardware(cpu); @@ -528,8 +542,6 @@ static struct option info_opts[] = { int cmd_freq_info(int argc, char **argv) { - extern char *optarg; - extern int optind, opterr, optopt; int ret = 0, cont = 1; unsigned int cpu = 0; unsigned int human = 0; diff --git a/tools/power/cpupower/utils/cpufreq-set.c b/tools/power/cpupower/utils/cpufreq-set.c index c5e60a39cfa6..06cd4b280132 100644 --- a/tools/power/cpupower/utils/cpufreq-set.c +++ b/tools/power/cpupower/utils/cpufreq-set.c @@ -195,8 +195,6 @@ static int do_one_cpu(unsigned int cpu, struct cpufreq_policy *new_pol, int cmd_freq_set(int argc, char **argv) { - extern char *optarg; - extern int optind, opterr, optopt; int ret = 0, cont = 1; int double_parm = 0, related = 0, policychange = 0; unsigned long freq = 0; diff --git a/tools/power/cpupower/utils/cpuidle-info.c b/tools/power/cpupower/utils/cpuidle-info.c index e0d17f0de3fe..ccb37125bd37 100644 --- a/tools/power/cpupower/utils/cpuidle-info.c +++ b/tools/power/cpupower/utils/cpuidle-info.c @@ -111,7 +111,7 @@ static void proc_cpuidle_cpu_output(unsigned int cpu) printf(_("max_cstate: C%u\n"), cstates-1); printf(_("maximum allowed latency: %lu usec\n"), max_allowed_cstate); printf(_("states:\t\n")); - for (cstate = 1; cstate < cstates; cstate++) { + for (cstate = 0; cstate < cstates; cstate++) { printf(_(" C%d: " "type[C%d] "), cstate, cstate); printf(_("promotion[--] demotion[--] ")); @@ -139,8 +139,6 @@ static inline void cpuidle_exit(int fail) int cmd_idle_info(int argc, char **argv) { - extern char *optarg; - extern int optind, opterr, optopt; int ret = 0, cont = 1, output_param = 0, verbose = 1; unsigned int cpu = 0; diff --git a/tools/power/cpupower/utils/cpuidle-set.c b/tools/power/cpupower/utils/cpuidle-set.c index a551d1d4ac51..703094f1343c 100644 --- a/tools/power/cpupower/utils/cpuidle-set.c +++ b/tools/power/cpupower/utils/cpuidle-set.c @@ -24,8 +24,6 @@ static struct option info_opts[] = { int cmd_idle_set(int argc, char **argv) { - extern char *optarg; - extern int optind, opterr, optopt; int ret = 0, cont = 1, param = 0, disabled; unsigned long long latency = 0, state_latency; unsigned int cpu = 0, idlestate = 0, idlestates = 0; diff --git a/tools/power/cpupower/utils/cpupower-info.c b/tools/power/cpupower/utils/cpupower-info.c index 18fd7751f509..79154d71e498 100644 --- a/tools/power/cpupower/utils/cpupower-info.c +++ b/tools/power/cpupower/utils/cpupower-info.c @@ -28,8 +28,6 @@ static void print_wrong_arg_exit(void) int cmd_info(int argc, char **argv) { - extern char *optarg; - extern int optind, opterr, optopt; unsigned int cpu; struct utsname uts; diff --git a/tools/power/cpupower/utils/cpupower-set.c b/tools/power/cpupower/utils/cpupower-set.c index 0677b58374ab..c2176b9fa57d 100644 --- a/tools/power/cpupower/utils/cpupower-set.c +++ b/tools/power/cpupower/utils/cpupower-set.c @@ -21,6 +21,7 @@ static struct option set_opts[] = { {"epp", required_argument, NULL, 'e'}, {"amd-pstate-mode", required_argument, NULL, 'm'}, {"turbo-boost", required_argument, NULL, 't'}, + {"boost", required_argument, NULL, 't'}, { }, }; @@ -32,8 +33,6 @@ static void print_wrong_arg_exit(void) int cmd_set(int argc, char **argv) { - extern char *optarg; - extern int optind, opterr, optopt; unsigned int cpu; struct utsname uts; @@ -62,8 +61,8 @@ int cmd_set(int argc, char **argv) params.params = 0; /* parameter parsing */ - while ((ret = getopt_long(argc, argv, "b:e:m:", - set_opts, NULL)) != -1) { + while ((ret = getopt_long(argc, argv, "b:e:m:t:", + set_opts, NULL)) != -1) { switch (ret) { case 'b': if (params.perf_bias) @@ -123,7 +122,11 @@ int cmd_set(int argc, char **argv) } if (params.turbo_boost) { - ret = cpupower_set_turbo_boost(turbo_boost); + if (cpupower_cpu_info.vendor == X86_VENDOR_INTEL) + ret = cpupower_set_intel_turbo_boost(turbo_boost); + else + ret = cpupower_set_generic_turbo_boost(turbo_boost); + if (ret) fprintf(stderr, "Error setting turbo-boost\n"); } diff --git a/tools/power/cpupower/utils/helpers/helpers.h b/tools/power/cpupower/utils/helpers/helpers.h index 95749b8ee475..a3ad80b9c2c2 100644 --- a/tools/power/cpupower/utils/helpers/helpers.h +++ b/tools/power/cpupower/utils/helpers/helpers.h @@ -103,6 +103,9 @@ extern struct cpupower_cpu_info cpupower_cpu_info; /* cpuid and cpuinfo helpers **************************/ +int cpufreq_has_generic_boost_support(bool *active); +int cpupower_set_generic_turbo_boost(int turbo_boost); + /* X86 ONLY ****************************************/ #if defined(__i386__) || defined(__x86_64__) @@ -118,7 +121,6 @@ extern unsigned long long msr_intel_get_turbo_ratio(unsigned int cpu); extern int cpupower_set_epp(unsigned int cpu, char *epp); extern int cpupower_set_amd_pstate_mode(char *mode); -extern int cpupower_set_turbo_boost(int turbo_boost); /* Read/Write msr ****************************/ @@ -139,8 +141,9 @@ extern int decode_pstates(unsigned int cpu, int boost_states, /* AMD HW pstate decoding **************************/ -extern int cpufreq_has_boost_support(unsigned int cpu, int *support, - int *active, int * states); +int cpufreq_has_x86_boost_support(unsigned int cpu, int *support, + int *active, int *states); +int cpupower_set_intel_turbo_boost(int turbo_boost); /* AMD P-State stuff **************************/ bool cpupower_amd_pstate_enabled(void); @@ -181,13 +184,13 @@ static inline int cpupower_set_epp(unsigned int cpu, char *epp) { return -1; }; static inline int cpupower_set_amd_pstate_mode(char *mode) { return -1; }; -static inline int cpupower_set_turbo_boost(int turbo_boost) -{ return -1; }; /* Read/Write msr ****************************/ -static inline int cpufreq_has_boost_support(unsigned int cpu, int *support, - int *active, int * states) +static inline int cpufreq_has_x86_boost_support(unsigned int cpu, int *support, + int *active, int *states) +{ return -1; } +static inline int cpupower_set_intel_turbo_boost(int turbo_boost) { return -1; } static inline bool cpupower_amd_pstate_enabled(void) diff --git a/tools/power/cpupower/utils/helpers/misc.c b/tools/power/cpupower/utils/helpers/misc.c index 76e461ff4f74..eebfc79a4889 100644 --- a/tools/power/cpupower/utils/helpers/misc.c +++ b/tools/power/cpupower/utils/helpers/misc.c @@ -8,18 +8,20 @@ #include "helpers/helpers.h" #include "helpers/sysfs.h" #include "cpufreq.h" +#include "cpupower_intern.h" #if defined(__i386__) || defined(__x86_64__) -#include "cpupower_intern.h" - #define MSR_AMD_HWCR 0xc0010015 -int cpufreq_has_boost_support(unsigned int cpu, int *support, int *active, - int *states) +int cpufreq_has_x86_boost_support(unsigned int cpu, int *support, int *active, + int *states) { int ret; unsigned long long val; + char linebuf[MAX_LINE_LEN]; + char path[SYSFS_PATH_MAX]; + char *endp; *support = *active = *states = 0; @@ -43,8 +45,42 @@ int cpufreq_has_boost_support(unsigned int cpu, int *support, int *active, } } else if (cpupower_cpu_info.caps & CPUPOWER_CAP_AMD_PSTATE) { amd_pstate_boost_init(cpu, support, active); - } else if (cpupower_cpu_info.caps & CPUPOWER_CAP_INTEL_IDA) + } else if (cpupower_cpu_info.caps & CPUPOWER_CAP_INTEL_IDA) { *support = *active = 1; + + snprintf(path, sizeof(path), PATH_TO_CPU "intel_pstate/no_turbo"); + + if (!is_valid_path(path)) + return 0; + + if (cpupower_read_sysfs(path, linebuf, MAX_LINE_LEN) == 0) + return -1; + + val = strtol(linebuf, &endp, 0); + if (endp == linebuf || errno == ERANGE) + return -1; + + *active = !val; + } + return 0; +} + +int cpupower_set_intel_turbo_boost(int turbo_boost) +{ + char path[SYSFS_PATH_MAX]; + char linebuf[2] = {}; + + snprintf(path, sizeof(path), PATH_TO_CPU "intel_pstate/no_turbo"); + + /* Fallback to generic solution when intel_pstate driver not running */ + if (!is_valid_path(path)) + return cpupower_set_generic_turbo_boost(turbo_boost); + + snprintf(linebuf, sizeof(linebuf), "%d", !turbo_boost); + + if (cpupower_write_sysfs(path, linebuf, 2) <= 0) + return -1; + return 0; } @@ -124,24 +160,6 @@ int cpupower_set_amd_pstate_mode(char *mode) return 0; } -int cpupower_set_turbo_boost(int turbo_boost) -{ - char path[SYSFS_PATH_MAX]; - char linebuf[2] = {}; - - snprintf(path, sizeof(path), PATH_TO_CPU "cpufreq/boost"); - - if (!is_valid_path(path)) - return -1; - - snprintf(linebuf, sizeof(linebuf), "%d", turbo_boost); - - if (cpupower_write_sysfs(path, linebuf, 2) <= 0) - return -1; - - return 0; -} - bool cpupower_amd_pstate_enabled(void) { char *driver = cpufreq_get_driver(0); @@ -160,6 +178,39 @@ bool cpupower_amd_pstate_enabled(void) #endif /* #if defined(__i386__) || defined(__x86_64__) */ +int cpufreq_has_generic_boost_support(bool *active) +{ + char path[SYSFS_PATH_MAX]; + char linebuf[2] = {}; + unsigned long val; + char *endp; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpufreq/boost"); + + if (!is_valid_path(path)) + return -EACCES; + + if (cpupower_read_sysfs(path, linebuf, 2) <= 0) + return -EINVAL; + + val = strtoul(linebuf, &endp, 0); + if (endp == linebuf || errno == ERANGE) + return -EINVAL; + + switch (val) { + case 0: + *active = false; + break; + case 1: + *active = true; + break; + default: + return -EINVAL; + } + + return 0; +} + /* get_cpustate * * Gather the information of all online CPUs into bitmask struct @@ -259,3 +310,21 @@ void print_speed(unsigned long speed, int no_rounding) } } } + +int cpupower_set_generic_turbo_boost(int turbo_boost) +{ + char path[SYSFS_PATH_MAX]; + char linebuf[2] = {}; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpufreq/boost"); + + if (!is_valid_path(path)) + return -1; + + snprintf(linebuf, sizeof(linebuf), "%d", turbo_boost); + + if (cpupower_write_sysfs(path, linebuf, 2) <= 0) + return -1; + + return 0; +} diff --git a/tools/power/cpupower/utils/idle_monitor/cpuidle_sysfs.c b/tools/power/cpupower/utils/idle_monitor/cpuidle_sysfs.c index 8b42c2f0a5b0..4225eff9833d 100644 --- a/tools/power/cpupower/utils/idle_monitor/cpuidle_sysfs.c +++ b/tools/power/cpupower/utils/idle_monitor/cpuidle_sysfs.c @@ -70,7 +70,7 @@ static int cpuidle_stop(void) current_count[cpu][state] = cpuidle_state_time(cpu, state); dprint("CPU %d - State: %d - Val: %llu\n", - cpu, state, previous_count[cpu][state]); + cpu, state, current_count[cpu][state]); } } return 0; diff --git a/tools/power/cpupower/utils/idle_monitor/cpupower-monitor.c b/tools/power/cpupower/utils/idle_monitor/cpupower-monitor.c index f746099b5dac..e8b3841d5c0f 100644 --- a/tools/power/cpupower/utils/idle_monitor/cpupower-monitor.c +++ b/tools/power/cpupower/utils/idle_monitor/cpupower-monitor.c @@ -6,6 +6,7 @@ */ +#include <errno.h> #include <stdio.h> #include <unistd.h> #include <stdlib.h> @@ -91,7 +92,11 @@ int fill_string_with_spaces(char *s, int n) return 0; } -#define MAX_COL_WIDTH 6 +#define MAX_COL_WIDTH 6 +#define TOPOLOGY_DEPTH_PKG 3 +#define TOPOLOGY_DEPTH_CORE 2 +#define TOPOLOGY_DEPTH_CPU 1 + void print_header(int topology_depth) { int unsigned mon; @@ -113,12 +118,17 @@ void print_header(int topology_depth) } printf("\n"); - if (topology_depth > 2) + switch (topology_depth) { + case TOPOLOGY_DEPTH_PKG: printf(" PKG|"); - if (topology_depth > 1) + case TOPOLOGY_DEPTH_CORE: printf("CORE|"); - if (topology_depth > 0) + case TOPOLOGY_DEPTH_CPU: printf(" CPU|"); + break; + default: + return; + } for (mon = 0; mon < avail_monitors; mon++) { if (mon != 0) @@ -152,12 +162,17 @@ void print_results(int topology_depth, int cpu) cpu_top.core_info[cpu].pkg == -1) return; - if (topology_depth > 2) + switch (topology_depth) { + case TOPOLOGY_DEPTH_PKG: printf("%4d|", cpu_top.core_info[cpu].pkg); - if (topology_depth > 1) + case TOPOLOGY_DEPTH_CORE: printf("%4d|", cpu_top.core_info[cpu].core); - if (topology_depth > 0) + case TOPOLOGY_DEPTH_CPU: printf("%4d|", cpu_top.core_info[cpu].cpu); + break; + default: + return; + } for (mon = 0; mon < avail_monitors; mon++) { if (mon != 0) @@ -294,7 +309,10 @@ int fork_it(char **argv) if (!child_pid) { /* child */ - execvp(argv[0], argv); + if (execvp(argv[0], argv) == -1) { + printf("Invalid monitor command %s\n", argv[0]); + exit(errno); + } } else { /* parent */ if (child_pid == -1) { @@ -423,11 +441,13 @@ int cmd_monitor(int argc, char **argv) if (avail_monitors == 0) { printf(_("No HW Cstate monitors found\n")); + cpu_topology_release(cpu_top); return 1; } if (mode == list) { list_monitors(); + cpu_topology_release(cpu_top); exit(EXIT_SUCCESS); } @@ -448,15 +468,15 @@ int cmd_monitor(int argc, char **argv) /* ToDo: Topology parsing needs fixing first to do this more generically */ if (cpu_top.pkgs > 1) - print_header(3); + print_header(TOPOLOGY_DEPTH_PKG); else - print_header(1); + print_header(TOPOLOGY_DEPTH_CPU); for (cpu = 0; cpu < cpu_count; cpu++) { if (cpu_top.pkgs > 1) - print_results(3, cpu); + print_results(TOPOLOGY_DEPTH_PKG, cpu); else - print_results(1, cpu); + print_results(TOPOLOGY_DEPTH_CPU, cpu); } for (num = 0; num < avail_monitors; num++) { diff --git a/tools/power/cpupower/utils/idle_monitor/mperf_monitor.c b/tools/power/cpupower/utils/idle_monitor/mperf_monitor.c index 73b6b10cbdd2..5ae02c3d5b64 100644 --- a/tools/power/cpupower/utils/idle_monitor/mperf_monitor.c +++ b/tools/power/cpupower/utils/idle_monitor/mperf_monitor.c @@ -240,9 +240,9 @@ static int mperf_stop(void) int cpu; for (cpu = 0; cpu < cpu_count; cpu++) { - mperf_measure_stats(cpu); - mperf_get_tsc(&tsc_at_measure_end[cpu]); clock_gettime(CLOCK_REALTIME, &time_end[cpu]); + mperf_get_tsc(&tsc_at_measure_end[cpu]); + mperf_measure_stats(cpu); } return 0; diff --git a/tools/power/cpupower/utils/powercap-info.c b/tools/power/cpupower/utils/powercap-info.c index 3ea4486f1a0e..e53033488218 100644 --- a/tools/power/cpupower/utils/powercap-info.c +++ b/tools/power/cpupower/utils/powercap-info.c @@ -38,11 +38,11 @@ static int powercap_print_one_zone(struct powercap_zone *zone) printf(" (%s)\n", mode ? "enabled" : "disabled"); if (zone->has_power_uw) - printf(_("%sPower can be monitored in micro Jules\n"), + printf(_("%sPower can be monitored in micro Watts\n"), pr_prefix); if (zone->has_energy_uj) - printf(_("%sPower can be monitored in micro Watts\n"), + printf(_("%sPower can be monitored in micro Jules\n"), pr_prefix); printf("\n"); diff --git a/tools/power/pm-graph/config/custom-timeline-functions.cfg b/tools/power/pm-graph/config/custom-timeline-functions.cfg index 4f80ad7d7275..0321b59518f3 100644 --- a/tools/power/pm-graph/config/custom-timeline-functions.cfg +++ b/tools/power/pm-graph/config/custom-timeline-functions.cfg @@ -122,13 +122,13 @@ freeze_processes: freeze_kernel_threads: pm_restrict_gfp_mask: acpi_suspend_begin: -suspend_console: +console_suspend_all: acpi_pm_prepare: syscore_suspend: arch_enable_nonboot_cpus_end: syscore_resume: acpi_pm_finish: -resume_console: +console_resume_all: acpi_pm_end: pm_restore_gfp_mask: thaw_processes: diff --git a/tools/power/pm-graph/sleepgraph.py b/tools/power/pm-graph/sleepgraph.py index 918eae58b0b4..1555b51a7d55 100755 --- a/tools/power/pm-graph/sleepgraph.py +++ b/tools/power/pm-graph/sleepgraph.py @@ -210,13 +210,13 @@ class SystemValues: 'hibernate_preallocate_memory': {}, 'create_basic_memory_bitmaps': {}, 'swsusp_write': {}, - 'suspend_console': {}, + 'console_suspend_all': {}, 'acpi_pm_prepare': {}, 'syscore_suspend': {}, 'arch_enable_nonboot_cpus_end': {}, 'syscore_resume': {}, 'acpi_pm_finish': {}, - 'resume_console': {}, + 'console_resume_all': {}, 'acpi_pm_end': {}, 'pm_restore_gfp_mask': {}, 'thaw_processes': {}, @@ -3459,7 +3459,7 @@ def parseTraceLog(live=False): tracewatch = ['irq_wakeup'] if sysvals.usekprobes: tracewatch += ['sync_filesystems', 'freeze_processes', 'syscore_suspend', - 'syscore_resume', 'resume_console', 'thaw_processes', 'CPU_ON', + 'syscore_resume', 'console_resume_all', 'thaw_processes', 'CPU_ON', 'CPU_OFF', 'acpi_suspend'] # extract the callgraph and traceevent data @@ -4017,7 +4017,8 @@ def parseKernelLog(data): 'PM: early restore of devices complete after.*'], 'resume_complete': ['PM: resume of devices complete after.*', 'PM: restore of devices complete after.*'], - 'post_resume': [r'.*Restarting tasks \.\.\..*'], + 'post_resume': [r'.*Restarting tasks \.\.\..*', + 'Done restarting tasks.*'], } # action table (expected events that occur and show up in dmesg) diff --git a/tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py b/tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py index feb9f9421c7b..875b086550d1 100755 --- a/tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py +++ b/tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py @@ -11,7 +11,7 @@ Prerequisites: gnuplot 5.0 or higher gnuplot-py 1.8 or higher (Most of the distributions have these required packages. They may be called - gnuplot-py, phython-gnuplot or phython3-gnuplot, gnuplot-nox, ... ) + gnuplot-py, python-gnuplot or python3-gnuplot, gnuplot-nox, ... ) Kernel config for Linux trace is enabled diff --git a/tools/power/x86/intel-speed-select/Makefile b/tools/power/x86/intel-speed-select/Makefile index 7221f2f55e8b..6b299aae2ded 100644 --- a/tools/power/x86/intel-speed-select/Makefile +++ b/tools/power/x86/intel-speed-select/Makefile @@ -13,7 +13,13 @@ endif # Do not use make's built-in rules # (this improves performance and avoids hard-to-debug behaviour); MAKEFLAGS += -r -override CFLAGS += -O2 -Wall -g -D_GNU_SOURCE -I$(OUTPUT)include -I/usr/include/libnl3 + +NL3_CFLAGS = $(shell pkg-config --cflags libnl-3.0 2>/dev/null) +ifeq ($(NL3_CFLAGS),) +NL3_CFLAGS = -I/usr/include/libnl3 +endif + +override CFLAGS += -O2 -Wall -g -D_GNU_SOURCE -I$(OUTPUT)include $(NL3_CFLAGS) override LDFLAGS += -lnl-genl-3 -lnl-3 ALL_TARGETS := intel-speed-select diff --git a/tools/power/x86/intel-speed-select/isst-config.c b/tools/power/x86/intel-speed-select/isst-config.c index fadfb02b8611..2faff1aead52 100644 --- a/tools/power/x86/intel-speed-select/isst-config.c +++ b/tools/power/x86/intel-speed-select/isst-config.c @@ -16,7 +16,7 @@ struct process_cmd_struct { int arg; }; -static const char *version_str = "v1.21"; +static const char *version_str = "v1.26"; static const int supported_api_ver = 3; static struct isst_if_platform_info isst_platform_info; @@ -26,6 +26,7 @@ static FILE *outf; static int cpu_model; static int cpu_stepping; +static int cpu_family; #define MAX_CPUS_IN_ONE_REQ 512 static short max_target_cpus; @@ -46,8 +47,9 @@ static int force_online_offline; static int auto_mode; static int fact_enable_fail; static int cgroupv2; +static int max_pkg_id; static int max_die_id; -static int max_punit_id; +static int max_die_id_package_0; /* clos related */ static int current_clos = -1; @@ -78,6 +80,23 @@ struct cpu_topology { short die_id; }; +static int read_only; + +static void print_version(void) +{ + fprintf(outf, "Version %s\n", version_str); +} + +static void check_privilege(void) +{ + if (!read_only) + return; + + isst_display_error_info_message(1, "Insufficient privileges", 0, 0); + isst_ctdp_display_information_end(outf); + exit(1); +} + FILE *get_output_file(void) { return outf; @@ -142,15 +161,25 @@ int is_icx_platform(void) return 0; } +static int is_dmr_plus_platform(void) +{ + if (cpu_family == 19) + return 1; + + return 0; +} + static int update_cpu_model(void) { unsigned int ebx, ecx, edx; - unsigned int fms, family; + unsigned int fms; __cpuid(1, fms, ebx, ecx, edx); - family = (fms >> 8) & 0xf; + cpu_family = (fms >> 8) & 0xf; + if (cpu_family == 0xf) + cpu_family += (fms >> 20) & 0xff; cpu_model = (fms >> 4) & 0xf; - if (family == 6 || family == 0xf) + if (cpu_family == 6 || cpu_family == 0xf) cpu_model += ((fms >> 16) & 0xf) << 4; cpu_stepping = fms & 0xf; @@ -557,6 +586,8 @@ void for_each_online_power_domain_in_set(void (*callback)(struct isst_id *, void if (id.pkg < 0 || id.die < 0 || id.punit < 0) continue; + id.die = id.die % (max_die_id_package_0 + 1); + valid_mask[id.pkg][id.die] = 1; if (cpus[id.pkg][id.die][id.punit] == -1) @@ -564,11 +595,11 @@ void for_each_online_power_domain_in_set(void (*callback)(struct isst_id *, void } for (i = 0; i < MAX_PACKAGE_COUNT; i++) { - if (max_die_id == max_punit_id) { + if (max_die_id > max_pkg_id) { for (k = 0; k < MAX_PUNIT_PER_DIE && k < MAX_DIE_PER_PACKAGE; k++) { id.cpu = cpus[i][k][k]; id.pkg = i; - id.die = k; + id.die = get_physical_die_id(id.cpu); id.punit = k; if (isst_is_punit_valid(&id)) callback(&id, arg1, arg2, arg3, arg4); @@ -586,7 +617,10 @@ void for_each_online_power_domain_in_set(void (*callback)(struct isst_id *, void for (k = 0; k < MAX_PUNIT_PER_DIE; k++) { id.cpu = cpus[i][j][k]; id.pkg = i; - id.die = j; + if (id.cpu >= 0) + id.die = get_physical_die_id(id.cpu); + else + id.die = id.pkg; id.punit = k; if (isst_is_punit_valid(&id)) callback(&id, arg1, arg2, arg3, arg4); @@ -788,6 +822,8 @@ static void create_cpu_map(void) cpu_map[i].die_id = die_id; cpu_map[i].core_id = core_id; + if (max_pkg_id < pkg_id) + max_pkg_id = pkg_id; punit_id = 0; @@ -812,8 +848,8 @@ static void create_cpu_map(void) if (max_die_id < die_id) max_die_id = die_id; - if (max_punit_id < cpu_map[i].punit_id) - max_punit_id = cpu_map[i].punit_id; + if (!pkg_id && max_die_id_package_0 < die_id) + max_die_id_package_0 = die_id; debug_printf( "map logical_cpu:%d core: %d die:%d pkg:%d punit:%d punit_cpu:%d punit_core:%d\n", @@ -932,9 +968,11 @@ int isolate_cpus(struct isst_id *id, int mask_size, cpu_set_t *cpu_mask, int lev ret = write(fd, "member", strlen("member")); if (ret == -1) { printf("Can't update to member\n"); + close(fd); return ret; } + close(fd); return 0; } @@ -1105,8 +1143,9 @@ static int isst_fill_platform_info(void) close(fd); if (isst_platform_info.api_version > supported_api_ver) { + print_version(); printf("Incompatible API versions; Upgrade of tool is required\n"); - return -1; + exit(1); } set_platform_ops: @@ -1509,7 +1548,8 @@ display_result: usleep(2000); /* Adjusting uncore freq */ - isst_adjust_uncore_freq(id, tdp_level, &ctdp_level); + if (!is_dmr_plus_platform()) + isst_adjust_uncore_freq(id, tdp_level, &ctdp_level); fprintf(stderr, "Option is set to online/offline\n"); ctdp_level.core_cpumask_size = @@ -1559,6 +1599,8 @@ free_mask: static void set_tdp_level(int arg) { + check_privilege(); + if (cmd_help) { fprintf(stderr, "Set Config TDP level\n"); fprintf(stderr, @@ -1709,6 +1751,9 @@ static int no_turbo(void) return parse_int_file(0, "/sys/devices/system/cpu/intel_pstate/no_turbo"); } +#define U32_MAX ((unsigned int)~0U) +#define S32_MAX ((int)(U32_MAX >> 1)) + static void adjust_scaling_max_from_base_freq(int cpu) { int base_freq, scaling_max_freq; @@ -1716,7 +1761,7 @@ static void adjust_scaling_max_from_base_freq(int cpu) scaling_max_freq = parse_int_file(0, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_max_freq", cpu); base_freq = get_cpufreq_base_freq(cpu); if (scaling_max_freq < base_freq || no_turbo()) - set_cpufreq_scaling_min_max(cpu, 1, base_freq); + set_cpufreq_scaling_min_max(cpu, 1, S32_MAX); } static void adjust_scaling_min_from_base_freq(int cpu) @@ -2027,6 +2072,8 @@ static void set_pbf_enable(int arg) { int enable = arg; + check_privilege(); + if (cmd_help) { if (enable) { fprintf(stderr, @@ -2193,6 +2240,8 @@ static void set_fact_enable(int arg) int i, ret, enable = arg; struct isst_id id; + check_privilege(); + if (cmd_help) { if (enable) { fprintf(stderr, @@ -2342,6 +2391,8 @@ static void set_clos_enable(int arg) { int enable = arg; + check_privilege(); + if (cmd_help) { if (enable) { fprintf(stderr, @@ -2472,6 +2523,8 @@ static void set_clos_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, static void set_clos_config(int arg) { + check_privilege(); + if (cmd_help) { fprintf(stderr, "Set core-power configuration for one of the four clos ids\n"); @@ -2537,6 +2590,8 @@ static void set_clos_assoc_for_cpu(struct isst_id *id, void *arg1, void *arg2, v static void set_clos_assoc(int arg) { + check_privilege(); + if (cmd_help) { fprintf(stderr, "Associate a clos id to a CPU\n"); fprintf(stderr, @@ -2618,6 +2673,8 @@ static void set_turbo_mode(int arg) int i, disable = arg; struct isst_id id; + check_privilege(); + if (cmd_help) { if (disable) fprintf(stderr, "Set turbo mode disable\n"); @@ -2663,6 +2720,7 @@ static void get_set_trl(struct isst_id *id, void *arg1, void *arg2, void *arg3, } if (set) { + check_privilege(); ret = isst_set_trl(id, fact_trl); isst_display_result(id, outf, "turbo-mode", "set-trl", ret); return; @@ -3143,12 +3201,6 @@ static void usage(void) printf("\tTo get full turbo-freq information dump:\n"); printf("\t\tintel-speed-select turbo-freq info -l 0\n"); } - exit(1); -} - -static void print_version(void) -{ - fprintf(outf, "Version %s\n", version_str); exit(0); } @@ -3185,13 +3237,23 @@ static void cmdline(int argc, char **argv) }; if (geteuid() != 0) { - fprintf(stderr, "Must run as root\n"); - exit(0); + int fd; + + fd = open(pathname, O_RDWR); + if (fd < 0) { + fprintf(stderr, "Must run as root\n"); + exit(0); + } + fprintf(stderr, "\nNot running as root, Only read only operations are supported\n"); + close(fd); + read_only = 1; } ret = update_cpu_model(); - if (ret) - err(-1, "Invalid CPU model (%d)\n", cpu_model); + if (ret) { + fprintf(stderr, "Invalid CPU model (%d)\n", cpu_model); + exit(1); + } printf("Intel(R) Speed Select Technology\n"); printf("Executing on CPU model:%d[0x%x]\n", cpu_model, cpu_model); @@ -3255,6 +3317,7 @@ static void cmdline(int argc, char **argv) break; case 'v': print_version(); + exit(0); break; case 'b': oob_mode = 1; diff --git a/tools/power/x86/intel-speed-select/isst-core-tpmi.c b/tools/power/x86/intel-speed-select/isst-core-tpmi.c index da53aaa27fc9..ebaad0dc8ca6 100644 --- a/tools/power/x86/intel-speed-select/isst-core-tpmi.c +++ b/tools/power/x86/intel-speed-select/isst-core-tpmi.c @@ -227,6 +227,7 @@ static int tpmi_get_ctdp_control(struct isst_id *id, int config_index, static int tpmi_get_tdp_info(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level) { + struct isst_perf_level_fabric_info fabric_info; struct isst_perf_level_data_info info; int ret; @@ -253,6 +254,17 @@ static int tpmi_get_tdp_info(struct isst_id *id, int config_index, ctdp_level->uncore_p1 = info.p1_fabric_freq_mhz; ctdp_level->uncore_pm = info.pm_fabric_freq_mhz; + fabric_info.socket_id = id->pkg; + fabric_info.power_domain_id = id->punit; + fabric_info.level = config_index; + + ret = tpmi_process_ioctl(ISST_IF_GET_PERF_LEVEL_FABRIC_INFO, &fabric_info); + if (ret != -1) { + ctdp_level->uncore1_p0 = fabric_info.p0_fabric_freq_mhz[1]; + ctdp_level->uncore1_p1 = fabric_info.p1_fabric_freq_mhz[1]; + ctdp_level->uncore1_pm = fabric_info.pm_fabric_freq_mhz[1]; + } + debug_printf ("cpu:%d ctdp:%d CONFIG_TDP_GET_TDP_INFO tdp_ratio:%d pkg_tdp:%d ctdp_level->t_proc_hot:%d\n", id->cpu, config_index, ctdp_level->tdp_ratio, ctdp_level->pkg_tdp, @@ -440,13 +452,16 @@ static int tpmi_get_pbf_info(struct isst_id *id, int level, return _pbf_get_coremask_info(id, level, pbf_info); } +#define FEATURE_ENABLE_WAIT_US 1000 +#define FEATURE_ENABLE_RETRIES 5 + static int tpmi_set_pbf_fact_status(struct isst_id *id, int pbf, int enable) { struct isst_pkg_ctdp pkg_dev; struct isst_pkg_ctdp_level_info ctdp_level; int current_level; struct isst_perf_feature_control info; - int ret; + int ret, i; ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) @@ -491,6 +506,30 @@ static int tpmi_set_pbf_fact_status(struct isst_id *id, int pbf, int enable) if (ret == -1) return ret; + for (i = 0; i < FEATURE_ENABLE_RETRIES; ++i) { + + usleep(FEATURE_ENABLE_WAIT_US); + + /* Check status */ + ret = isst_get_ctdp_control(id, current_level, &ctdp_level); + if (ret) + return ret; + + debug_printf("pbf_enabled:%d fact_enabled:%d\n", + ctdp_level.pbf_enabled, ctdp_level.fact_enabled); + + if (pbf) { + if (ctdp_level.pbf_enabled == enable) + break; + } else { + if (ctdp_level.fact_enabled == enable) + break; + } + } + + if (i == FEATURE_ENABLE_RETRIES) + return -1; + return 0; } @@ -501,6 +540,7 @@ static int tpmi_get_fact_info(struct isst_id *id, int level, int fact_bucket, int i, j; int ret; + memset(&info, 0, sizeof(info)); info.socket_id = id->pkg; info.power_domain_id = id->punit; info.level = level; @@ -647,7 +687,8 @@ static int tpmi_pm_qos_config(struct isst_id *id, int enable_clos, int priority_type) { struct isst_core_power info; - int i, ret, saved_punit; + int cp_state = 0, cp_cap = 0; + int i, j, ret, saved_punit; info.get_set = 1; info.socket_id = id->pkg; @@ -667,6 +708,19 @@ static int tpmi_pm_qos_config(struct isst_id *id, int enable_clos, id->punit = saved_punit; return ret; } + /* Get status */ + for (j = 0; j < FEATURE_ENABLE_RETRIES; ++j) { + usleep(FEATURE_ENABLE_WAIT_US); + ret = tpmi_read_pm_config(id, &cp_state, &cp_cap); + debug_printf("ret:%d cp_state:%d enable_clos:%d\n", ret, + cp_state, enable_clos); + if (ret || cp_state == enable_clos) + break; + } + if (j == FEATURE_ENABLE_RETRIES) { + id->punit = saved_punit; + return -1; + } } } diff --git a/tools/power/x86/intel-speed-select/isst-display.c b/tools/power/x86/intel-speed-select/isst-display.c index 07ebd08f3202..e4884eb02837 100644 --- a/tools/power/x86/intel-speed-select/isst-display.c +++ b/tools/power/x86/intel-speed-select/isst-display.c @@ -173,7 +173,11 @@ static int print_package_info(struct isst_id *id, FILE *outf) if (out_format_is_json()) { if (api_version() > 1) { - if (id->cpu < 0) + if (id->die < 0 && id->cpu < 0) + snprintf(header, sizeof(header), + "package-%d:die-IO:powerdomain-%d:cpu-None", + id->pkg, id->punit); + else if (id->cpu < 0) snprintf(header, sizeof(header), "package-%d:die-%d:powerdomain-%d:cpu-None", id->pkg, id->die, id->punit); @@ -190,7 +194,10 @@ static int print_package_info(struct isst_id *id, FILE *outf) } snprintf(header, sizeof(header), "package-%d", id->pkg); format_and_print(outf, level++, header, NULL); - snprintf(header, sizeof(header), "die-%d", id->die); + if (id->die < 0) + snprintf(header, sizeof(header), "die-IO"); + else + snprintf(header, sizeof(header), "die-%d", id->die); format_and_print(outf, level++, header, NULL); if (api_version() > 1) { snprintf(header, sizeof(header), "powerdomain-%d", id->punit); @@ -453,6 +460,26 @@ void isst_ctdp_display_information(struct isst_id *id, FILE *outf, int tdp_level format_and_print(outf, level + 2, header, value); } + if (ctdp_level->uncore1_p1) { + snprintf(header, sizeof(header), "uncore-1-frequency-base(MHz)"); + snprintf(value, sizeof(value), "%d", + ctdp_level->uncore1_p1 * isst_get_disp_freq_multiplier()); + format_and_print(outf, level + 2, header, value); + } + if (ctdp_level->uncore1_pm) { + snprintf(header, sizeof(header), "uncore-1-frequency-min(MHz)"); + snprintf(value, sizeof(value), "%d", + ctdp_level->uncore1_pm * isst_get_disp_freq_multiplier()); + format_and_print(outf, level + 2, header, value); + } + + if (ctdp_level->uncore1_p0) { + snprintf(header, sizeof(header), "uncore-1-frequency-max(MHz)"); + snprintf(value, sizeof(value), "%d", + ctdp_level->uncore1_p0 * isst_get_disp_freq_multiplier()); + format_and_print(outf, level + 2, header, value); + } + if (ctdp_level->mem_freq) { snprintf(header, sizeof(header), "max-mem-frequency(MHz)"); snprintf(value, sizeof(value), "%d", diff --git a/tools/power/x86/intel-speed-select/isst.h b/tools/power/x86/intel-speed-select/isst.h index 39ee75677c2c..960f647cfc2d 100644 --- a/tools/power/x86/intel-speed-select/isst.h +++ b/tools/power/x86/intel-speed-select/isst.h @@ -147,6 +147,9 @@ struct isst_pkg_ctdp_level_info { int uncore_p0; int uncore_p1; int uncore_pm; + int uncore1_p0; + int uncore1_p1; + int uncore1_pm; int sse_p1; int avx2_p1; int avx512_p1; diff --git a/tools/power/x86/turbostat/turbostat.8 b/tools/power/x86/turbostat/turbostat.8 index 99bf905ade81..344ede2f8546 100644 --- a/tools/power/x86/turbostat/turbostat.8 +++ b/tools/power/x86/turbostat/turbostat.8 @@ -47,10 +47,11 @@ name as necessary to disambiguate it from others is necessary. Note that option MSRs are read as 64-bits, u32 truncates the displayed value to 32-bits. default: u64 - format: {\fBraw\fP | \fBdelta\fP | \fBpercent\fP} + format: {\fBraw\fP | \fBdelta\fP | \fBpercent\fP | \fBaverage\fP} 'raw' shows the MSR contents in hex. 'delta' shows the difference in values during the measurement interval. 'percent' shows the delta as a percentage of the cycles elapsed. + 'average' similar to raw, but also averaged for node/package summaries (or when using -S). default: delta name: "name_string" @@ -100,7 +101,7 @@ The column name "all" can be used to enable all disabled-by-default built-in cou .PP \fB--show column\fP show only the specified built-in columns. May be invoked multiple times, or with a comma-separated list of column names. .PP -\fB--show CATEGORY --hide CATEGORY\fP Show and hide also accept a single CATEGORY of columns: "all", "topology", "idle", "frequency", "power", "sysfs", "other". +\fB--show CATEGORY --hide CATEGORY\fP Show and hide also accept a comma-separated-list of CATEGORIES of columns: "all", "topology", "idle", "frequency", "power", "cpuidle", "hwidle", "swidle", "cache", "llc", "other". "idle" (enabled by default), includes "hwidle" and "pct_idle". "cpuidle" (default disabled) includes cpuidle software invocation counters. "swidle" includes "cpuidle" plus "pct_idle". "hwidle" includes only hardware based idle residency counters. Older versions of turbostat used the term "sysfs" for what is now "swidle". .PP \fB--Dump\fP displays the raw counter values. .PP @@ -110,10 +111,14 @@ The column name "all" can be used to enable all disabled-by-default built-in cou .PP \fB--no-perf\fP Disable all the uses of the perf API. .PP +\fB--force\fP Force turbostat to run on an unsupported platform (minimal defaults). +.PP \fB--interval seconds\fP overrides the default 5.0 second measurement interval. .PP \fB--num_iterations num\fP number of the measurement iterations. .PP +\fB--header_iterations num\fP print header every num iterations. +.PP \fB--out output_file\fP turbostat output is written to the specified output_file. The file is truncated if it already exists, and it is created if it does not exist. .PP @@ -158,16 +163,30 @@ The system configuration dump (if --quiet is not used) is followed by statistics .PP \fBSMI\fP The number of System Management Interrupts serviced CPU during the measurement interval. While this counter is actually per-CPU, SMI are triggered on all processors, so the number should be the same for all CPUs. .PP -\fBC1, C2, C3...\fP The number times Linux requested the C1, C2, C3 idle state during the measurement interval. The system summary line shows the sum for all CPUs. These are C-state names as exported in /sys/devices/system/cpu/cpu*/cpuidle/state*/name. While their names are generic, their attributes are processor specific. They the system description section of output shows what MWAIT sub-states they are mapped to on each system. +\fBLLCMRPS\fP Last Level Cache Millions of References Per Second. For CPUs with an L3 LLC, this is the number of references that CPU made to the L3 (and the number of misses that CPU made to it's L2). For CPUs with an L2 LLC, this is the number of references to the L2 (and the number of misses to the CPU's L1). The system summary row shows the sum for all CPUs. In both cases, the value displayed is the actual value divided by 1,000,000. If this value is large, then the LLC%hit column is significant. If this value is small, then the LLC%hit column is not significant. +.PP +\fBLLC%hit\fP Last Level Cache Hit Rate %. Hit Rate Percent = 100.0 * Hits/References. The system summary row shows the weighted average for all CPUs (100.0 * Sum_Hits/Sum_References). +.PP +\fBL2MRPS\fP Level-2 Cache Millions of References Per Second. For CPUs with an L2 LLC, this is the same as LLC references. The system summary row shows the sum for all CPUs. In both cases, the value displayed is the actual value divided by 1,000,000. If this value is large, then the L2%hit column is significant. If this value is small, then the L2%hit column is not significant. +.PP +\fBL2%hit\fP Level-2 Cache Hit Rate %. Hit Rate Percent = 100.0 * Hits/References. The system summary row shows the weighted average for all CPUs (100.0 * (Sum_Hits)/Sum_References). +.PP +\fBC1, C2, C3...\fP The number times Linux requested the C1, C2, C3 idle state during the measurement interval. The system summary line shows the sum for all CPUs. These are C-state names as exported in /sys/devices/system/cpu/cpu*/cpuidle/state*/name. While their names are generic, their attributes are processor specific. They the system description section of output shows what MWAIT sub-states they are mapped to on each system. These counters are in the "cpuidle" group, which is disabled, by default. +.PP +\fBC1+, C2+, C3+...\fP The idle governor idle state misprediction statistics. Indicates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a deeper idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/below file. These counters are in the "cpuidle" group, which is disabled, by default. .PP -\fBC1%, C2%, C3%\fP The residency percentage that Linux requested C1, C2, C3.... The system summary is the average of all CPUs in the system. Note that these are software, reflecting what was requested. The hardware counters reflect what was actually achieved. +\fBC1-, C2-, C3-...\fP The idle governor idle state misprediction statistics. Indicates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a shallower idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/above file. These counters are in the "cpuidle" group, which is disabled, by default. .PP -\fBCPU%c1, CPU%c3, CPU%c6, CPU%c7\fP show the percentage residency in hardware core idle states. These numbers are from hardware residency counters. +\fBC1%, C2%, C3%\fP The residency percentage that Linux requested C1, C2, C3.... The system summary is the average of all CPUs in the system. Note that these are software, reflecting what was requested. The hardware counters reflect what was actually achieved. These counters are in the "pct_idle" group, which is enabled by default. +.PP +\fBCPU%c1, CPU%c3, CPU%c6, CPU%c7\fP show the percentage residency in hardware core idle states. These numbers are from hardware residency counters and are in the "hwidle" group, which is enabled, by default. .PP \fBCoreTmp\fP Degrees Celsius reported by the per-core Digital Thermal Sensor. .PP \fBPkgTmp\fP Degrees Celsius reported by the per-package Package Thermal Monitor. .PP +\fBCoreThr\fP Core Thermal Throttling events during the measurement interval. Note that events since boot can be find in /sys/devices/system/cpu/cpu*/thermal_throttle/* +.PP \fBGFX%rc6\fP The percentage of time the GPU is in the "render C6" state, rc6, during the measurement interval. From /sys/class/drm/card0/power/rc6_residency_ms or /sys/class/drm/card0/gt/gt0/rc6_residency_ms or /sys/class/drm/card0/device/tile0/gtN/gtidle/idle_residency_ms depending on the graphics driver being used. .PP \fBGFXMHz\fP Instantaneous snapshot of what sysfs presents at the end of the measurement interval. From /sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz or /sys/class/drm/card0/gt_cur_freq_mhz or /sys/class/drm/card0/gt/gt0/rps_cur_freq_mhz or /sys/class/drm/card0/device/tile0/gtN/freq0/cur_freq depending on the graphics driver being used. @@ -180,6 +199,14 @@ The system configuration dump (if --quiet is not used) is followed by statistics .PP \fBSAMAMHz\fP Instantaneous snapshot of what sysfs presents at the end of the measurement interval. From /sys/class/drm/card0/gt/gt1/rps_act_freq_mhz or /sys/class/drm/card0/device/tile0/gtN/freq0/act_freq depending on the graphics driver being used. .PP +\fBTotl%C0\fP Weighted percentage of time that CPUs are busy. If N CPUs are busy during an interval, the percentage is N * 100%. +.PP +\fBAny%C0\fP Percentage of time that at least one CPU is busy. +.PP +\fBGFX%C0\fP Percentage of time that at least one GFX compute engine is busy. +.PP +\fBCPUGFX%\fP Percentage of time that at least one CPU is busy at the same time as at least one Graphics compute engine is busy. +.PP \fBPkg%pc2, Pkg%pc3, Pkg%pc6, Pkg%pc7\fP percentage residency in hardware package idle states. These numbers are from hardware residency counters. .PP \fBPkgWatt\fP Watts consumed by the whole package. @@ -198,7 +225,9 @@ The system configuration dump (if --quiet is not used) is followed by statistics .PP \fBUncMHz\fP per-package uncore MHz, instantaneous sample. .PP -\fBUMHz1.0\fP per-package uncore MHz for domain=1 and fabric_cluster=0, instantaneous sample. System summary is the average of all packages. +\fBUMHz1.0\fP per-package uncore MHz for pm_domain=1 and fabric_cluster=0, instantaneous sample. System summary is the average of all packages. +Intel Granite Rapids systems use pm_domains 0-2 for CPUs, and 3-4 for IO, with cluster always 0. +For the "--show" and "--hide" options, use "UncMHz" to operate on all UMHz*.* as a group. .SH TOO MUCH INFORMATION EXAMPLE By default, turbostat dumps all possible information -- a system configuration header, followed by columns for all counters. This is ideal for remote debugging, use the "--out" option to save everything to a text file, and get that file to the expert helping you debug. @@ -393,25 +422,24 @@ CPU pCPU%c1 CPU%c1 .fi .SH ADD PERF COUNTER EXAMPLE #2 (using virtual cpu device) -Here we run on hybrid, Raptor Lake platform. -We limit turbostat to show output for just cpu0 (pcore) and cpu12 (ecore). +Here we run on hybrid, Meteor Lake platform. +We limit turbostat to show output for just cpu0 (pcore) and cpu4 (ecore). We add a counter showing number of L3 cache misses, using virtual "cpu" device, labeling it with the column header, "VCMISS". We add a counter showing number of L3 cache misses, using virtual "cpu_core" device, -labeling it with the column header, "PCMISS". This will fail on ecore cpu12. +labeling it with the column header, "PCMISS". This will fail on ecore cpu4. We add a counter showing number of L3 cache misses, using virtual "cpu_atom" device, labeling it with the column header, "ECMISS". This will fail on pcore cpu0. We display it only once, after the conclusion of 0.1 second sleep. .nf -sudo ./turbostat --quiet --cpu 0,12 --show CPU --add perf/cpu/cache-misses,cpu,delta,raw,VCMISS --add perf/cpu_core/cache-misses,cpu,delta,raw,PCMISS --add perf/cpu_atom/cache-misses,cpu,delta,raw,ECMISS sleep .1 +sudo ./turbostat --quiet --cpu 0,4 --show CPU --add perf/cpu/cache-misses,cpu,delta,VCMISS --add perf/cpu_core/cache-misses,cpu,delta,PCMISS --add perf/cpu_atom/cache-misses,cpu,delta,ECMISS sleep 5 turbostat: added_perf_counters_init_: perf/cpu_atom/cache-misses: failed to open counter on cpu0 -turbostat: added_perf_counters_init_: perf/cpu_core/cache-misses: failed to open counter on cpu12 -0.104630 sec -CPU ECMISS PCMISS VCMISS -- 0x0000000000000000 0x0000000000000000 0x0000000000000000 -0 0x0000000000000000 0x0000000000007951 0x0000000000007796 -12 0x000000000001137a 0x0000000000000000 0x0000000000011392 - +turbostat: added_perf_counters_init_: perf/cpu_core/cache-misses: failed to open counter on cpu4 +5.001207 sec +CPU ECMISS PCMISS VCMISS +- 41586506 46291219 87877749 +4 83173012 0 83173040 +0 0 92582439 92582458 .fi .SH ADD PMT COUNTER EXAMPLE @@ -539,6 +567,8 @@ If the upstream version isn't new enough, the development tree can be found here If the development tree doesn't work, please contact the author via chat, or via email with the word "turbostat" on the Subject line. +An old turbostat binary may run on unknown hardware by using "--force", +but results are unsupported. .SH FILES .ta .nf diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 8d5011a0bf60..920694c3c1ec 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -3,7 +3,7 @@ * turbostat -- show CPU frequency and C-state residency * on modern Intel and AMD processors. * - * Copyright (c) 2025 Intel Corporation. + * Copyright (c) 2010 - 2026 Intel Corporation * Len Brown <len.brown@intel.com> */ @@ -67,6 +67,7 @@ #include <stdbool.h> #include <assert.h> #include <linux/kernel.h> +#include <limits.h> #define UNUSED(x) (void)(x) @@ -141,6 +142,7 @@ struct msr_counter { #define FLAGS_SHOW (1 << 1) #define SYSFS_PERCPU (1 << 1) }; +static int use_android_msr_path; struct msr_counter bic[] = { { 0x0, "usec", NULL, 0, 0, 0, NULL, 0 }, @@ -153,7 +155,7 @@ struct msr_counter bic[] = { { 0x0, "TSC_MHz", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "IRQ", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "SMI", NULL, 32, 0, FORMAT_DELTA, NULL, 0 }, - { 0x0, "sysfs", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "cpuidle", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "CPU%c1", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "CPU%c3", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "CPU%c6", NULL, 0, 0, 0, NULL, 0 }, @@ -189,11 +191,13 @@ struct msr_counter bic[] = { { 0x0, "Any%C0", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "GFX%C0", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "CPUGFX%", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "Module", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "Core", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "CPU", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "APIC", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "X2APIC", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "Die", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "L3", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "GFXAMHz", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "IPC", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "CoreThr", NULL, 0, 0, 0, NULL, 0 }, @@ -206,90 +210,257 @@ struct msr_counter bic[] = { { 0x0, "Sys_J", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "NMI", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "CPU%c1e", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "pct_idle", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "LLCMRPS", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "LLC%hit", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "L2MRPS", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "L2%hit", NULL, 0, 0, 0, NULL, 0 }, }; -#define MAX_BIC (sizeof(bic) / sizeof(struct msr_counter)) -#define BIC_USEC (1ULL << 0) -#define BIC_TOD (1ULL << 1) -#define BIC_Package (1ULL << 2) -#define BIC_Node (1ULL << 3) -#define BIC_Avg_MHz (1ULL << 4) -#define BIC_Busy (1ULL << 5) -#define BIC_Bzy_MHz (1ULL << 6) -#define BIC_TSC_MHz (1ULL << 7) -#define BIC_IRQ (1ULL << 8) -#define BIC_SMI (1ULL << 9) -#define BIC_sysfs (1ULL << 10) -#define BIC_CPU_c1 (1ULL << 11) -#define BIC_CPU_c3 (1ULL << 12) -#define BIC_CPU_c6 (1ULL << 13) -#define BIC_CPU_c7 (1ULL << 14) -#define BIC_ThreadC (1ULL << 15) -#define BIC_CoreTmp (1ULL << 16) -#define BIC_CoreCnt (1ULL << 17) -#define BIC_PkgTmp (1ULL << 18) -#define BIC_GFX_rc6 (1ULL << 19) -#define BIC_GFXMHz (1ULL << 20) -#define BIC_Pkgpc2 (1ULL << 21) -#define BIC_Pkgpc3 (1ULL << 22) -#define BIC_Pkgpc6 (1ULL << 23) -#define BIC_Pkgpc7 (1ULL << 24) -#define BIC_Pkgpc8 (1ULL << 25) -#define BIC_Pkgpc9 (1ULL << 26) -#define BIC_Pkgpc10 (1ULL << 27) -#define BIC_CPU_LPI (1ULL << 28) -#define BIC_SYS_LPI (1ULL << 29) -#define BIC_PkgWatt (1ULL << 30) -#define BIC_CorWatt (1ULL << 31) -#define BIC_GFXWatt (1ULL << 32) -#define BIC_PkgCnt (1ULL << 33) -#define BIC_RAMWatt (1ULL << 34) -#define BIC_PKG__ (1ULL << 35) -#define BIC_RAM__ (1ULL << 36) -#define BIC_Pkg_J (1ULL << 37) -#define BIC_Cor_J (1ULL << 38) -#define BIC_GFX_J (1ULL << 39) -#define BIC_RAM_J (1ULL << 40) -#define BIC_Mod_c6 (1ULL << 41) -#define BIC_Totl_c0 (1ULL << 42) -#define BIC_Any_c0 (1ULL << 43) -#define BIC_GFX_c0 (1ULL << 44) -#define BIC_CPUGFX (1ULL << 45) -#define BIC_Core (1ULL << 46) -#define BIC_CPU (1ULL << 47) -#define BIC_APIC (1ULL << 48) -#define BIC_X2APIC (1ULL << 49) -#define BIC_Die (1ULL << 50) -#define BIC_GFXACTMHz (1ULL << 51) -#define BIC_IPC (1ULL << 52) -#define BIC_CORE_THROT_CNT (1ULL << 53) -#define BIC_UNCORE_MHZ (1ULL << 54) -#define BIC_SAM_mc6 (1ULL << 55) -#define BIC_SAMMHz (1ULL << 56) -#define BIC_SAMACTMHz (1ULL << 57) -#define BIC_Diec6 (1ULL << 58) -#define BIC_SysWatt (1ULL << 59) -#define BIC_Sys_J (1ULL << 60) -#define BIC_NMI (1ULL << 61) -#define BIC_CPU_c1e (1ULL << 62) - -#define BIC_TOPOLOGY (BIC_Package | BIC_Node | BIC_CoreCnt | BIC_PkgCnt | BIC_Core | BIC_CPU | BIC_Die) -#define BIC_THERMAL_PWR (BIC_CoreTmp | BIC_PkgTmp | BIC_PkgWatt | BIC_CorWatt | BIC_GFXWatt | BIC_RAMWatt | BIC_PKG__ | BIC_RAM__ | BIC_SysWatt) -#define BIC_FREQUENCY (BIC_Avg_MHz | BIC_Busy | BIC_Bzy_MHz | BIC_TSC_MHz | BIC_GFXMHz | BIC_GFXACTMHz | BIC_SAMMHz | BIC_SAMACTMHz | BIC_UNCORE_MHZ) -#define BIC_IDLE (BIC_Busy | BIC_sysfs | BIC_CPU_c1 | BIC_CPU_c3 | BIC_CPU_c6 | BIC_CPU_c7 | BIC_GFX_rc6 | BIC_Pkgpc2 | BIC_Pkgpc3 | BIC_Pkgpc6 | BIC_Pkgpc7 | BIC_Pkgpc8 | BIC_Pkgpc9 | BIC_Pkgpc10 | BIC_CPU_LPI | BIC_SYS_LPI | BIC_Mod_c6 | BIC_Totl_c0 | BIC_Any_c0 | BIC_GFX_c0 | BIC_CPUGFX | BIC_SAM_mc6 | BIC_Diec6) -#define BIC_OTHER (BIC_IRQ | BIC_NMI | BIC_SMI | BIC_ThreadC | BIC_CoreTmp | BIC_IPC) - -#define BIC_DISABLED_BY_DEFAULT (BIC_USEC | BIC_TOD | BIC_APIC | BIC_X2APIC) - -unsigned long long bic_enabled = (0xFFFFFFFFFFFFFFFFULL & ~BIC_DISABLED_BY_DEFAULT); -unsigned long long bic_present = BIC_USEC | BIC_TOD | BIC_sysfs | BIC_APIC | BIC_X2APIC; - -#define DO_BIC(COUNTER_NAME) (bic_enabled & bic_present & COUNTER_NAME) -#define DO_BIC_READ(COUNTER_NAME) (bic_present & COUNTER_NAME) -#define ENABLE_BIC(COUNTER_NAME) (bic_enabled |= COUNTER_NAME) -#define BIC_PRESENT(COUNTER_BIT) (bic_present |= COUNTER_BIT) -#define BIC_NOT_PRESENT(COUNTER_BIT) (bic_present &= ~COUNTER_BIT) -#define BIC_IS_ENABLED(COUNTER_BIT) (bic_enabled & COUNTER_BIT) +/* n.b. bic_names must match the order in bic[], above */ +enum bic_names { + BIC_USEC, + BIC_TOD, + BIC_Package, + BIC_Node, + BIC_Avg_MHz, + BIC_Busy, + BIC_Bzy_MHz, + BIC_TSC_MHz, + BIC_IRQ, + BIC_SMI, + BIC_cpuidle, + BIC_CPU_c1, + BIC_CPU_c3, + BIC_CPU_c6, + BIC_CPU_c7, + BIC_ThreadC, + BIC_CoreTmp, + BIC_CoreCnt, + BIC_PkgTmp, + BIC_GFX_rc6, + BIC_GFXMHz, + BIC_Pkgpc2, + BIC_Pkgpc3, + BIC_Pkgpc6, + BIC_Pkgpc7, + BIC_Pkgpc8, + BIC_Pkgpc9, + BIC_Pkgpc10, + BIC_CPU_LPI, + BIC_SYS_LPI, + BIC_PkgWatt, + BIC_CorWatt, + BIC_GFXWatt, + BIC_PkgCnt, + BIC_RAMWatt, + BIC_PKG__, + BIC_RAM__, + BIC_Pkg_J, + BIC_Cor_J, + BIC_GFX_J, + BIC_RAM_J, + BIC_Mod_c6, + BIC_Totl_c0, + BIC_Any_c0, + BIC_GFX_c0, + BIC_CPUGFX, + BIC_Module, + BIC_Core, + BIC_CPU, + BIC_APIC, + BIC_X2APIC, + BIC_Die, + BIC_L3, + BIC_GFXACTMHz, + BIC_IPC, + BIC_CORE_THROT_CNT, + BIC_UNCORE_MHZ, + BIC_SAM_mc6, + BIC_SAMMHz, + BIC_SAMACTMHz, + BIC_Diec6, + BIC_SysWatt, + BIC_Sys_J, + BIC_NMI, + BIC_CPU_c1e, + BIC_pct_idle, + BIC_LLC_MRPS, + BIC_LLC_HIT, + BIC_L2_MRPS, + BIC_L2_HIT, + MAX_BIC +}; + +void print_bic_set(char *s, cpu_set_t *set) +{ + int i; + + assert(MAX_BIC < CPU_SETSIZE); + + printf("%s:", s); + + for (i = 0; i < MAX_BIC; ++i) { + + if (CPU_ISSET(i, set)) + printf(" %s", bic[i].name); + } + putchar('\n'); +} + +static cpu_set_t bic_group_topology; +static cpu_set_t bic_group_thermal_pwr; +static cpu_set_t bic_group_frequency; +static cpu_set_t bic_group_hw_idle; +static cpu_set_t bic_group_sw_idle; +static cpu_set_t bic_group_idle; +static cpu_set_t bic_group_cache; +static cpu_set_t bic_group_other; +static cpu_set_t bic_group_disabled_by_default; +static cpu_set_t bic_enabled; +static cpu_set_t bic_present; + +/* modify */ +#define BIC_INIT(set) CPU_ZERO(set) + +#define SET_BIC(COUNTER_NUMBER, set) CPU_SET(COUNTER_NUMBER, set) +#define CLR_BIC(COUNTER_NUMBER, set) CPU_CLR(COUNTER_NUMBER, set) + +#define BIC_PRESENT(COUNTER_NUMBER) SET_BIC(COUNTER_NUMBER, &bic_present) +#define BIC_NOT_PRESENT(COUNTER_NUMBER) CPU_CLR(COUNTER_NUMBER, &bic_present) + +/* test */ +#define BIC_IS_ENABLED(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_enabled) +#define DO_BIC_READ(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_present) +#define DO_BIC(COUNTER_NUMBER) (CPU_ISSET(COUNTER_NUMBER, &bic_enabled) && CPU_ISSET(COUNTER_NUMBER, &bic_present)) + +static void bic_set_all(cpu_set_t *set) +{ + int i; + + assert(MAX_BIC < CPU_SETSIZE); + + for (i = 0; i < MAX_BIC; ++i) + SET_BIC(i, set); +} + +/* + * bic_clear_bits() + * clear all the bits from "clr" in "dst" + */ +static void bic_clear_bits(cpu_set_t *dst, cpu_set_t *clr) +{ + int i; + + assert(MAX_BIC < CPU_SETSIZE); + + for (i = 0; i < MAX_BIC; ++i) + if (CPU_ISSET(i, clr)) + CLR_BIC(i, dst); +} + +static void bic_groups_init(void) +{ + BIC_INIT(&bic_group_topology); + SET_BIC(BIC_Package, &bic_group_topology); + SET_BIC(BIC_Node, &bic_group_topology); + SET_BIC(BIC_CoreCnt, &bic_group_topology); + SET_BIC(BIC_PkgCnt, &bic_group_topology); + SET_BIC(BIC_Module, &bic_group_topology); + SET_BIC(BIC_Core, &bic_group_topology); + SET_BIC(BIC_CPU, &bic_group_topology); + SET_BIC(BIC_Die, &bic_group_topology); + SET_BIC(BIC_L3, &bic_group_topology); + + BIC_INIT(&bic_group_thermal_pwr); + SET_BIC(BIC_CoreTmp, &bic_group_thermal_pwr); + SET_BIC(BIC_PkgTmp, &bic_group_thermal_pwr); + SET_BIC(BIC_PkgWatt, &bic_group_thermal_pwr); + SET_BIC(BIC_CorWatt, &bic_group_thermal_pwr); + SET_BIC(BIC_GFXWatt, &bic_group_thermal_pwr); + SET_BIC(BIC_RAMWatt, &bic_group_thermal_pwr); + SET_BIC(BIC_PKG__, &bic_group_thermal_pwr); + SET_BIC(BIC_RAM__, &bic_group_thermal_pwr); + SET_BIC(BIC_SysWatt, &bic_group_thermal_pwr); + + BIC_INIT(&bic_group_frequency); + SET_BIC(BIC_Avg_MHz, &bic_group_frequency); + SET_BIC(BIC_Busy, &bic_group_frequency); + SET_BIC(BIC_Bzy_MHz, &bic_group_frequency); + SET_BIC(BIC_TSC_MHz, &bic_group_frequency); + SET_BIC(BIC_GFXMHz, &bic_group_frequency); + SET_BIC(BIC_GFXACTMHz, &bic_group_frequency); + SET_BIC(BIC_SAMMHz, &bic_group_frequency); + SET_BIC(BIC_SAMACTMHz, &bic_group_frequency); + SET_BIC(BIC_UNCORE_MHZ, &bic_group_frequency); + + BIC_INIT(&bic_group_hw_idle); + SET_BIC(BIC_Busy, &bic_group_hw_idle); + SET_BIC(BIC_CPU_c1, &bic_group_hw_idle); + SET_BIC(BIC_CPU_c3, &bic_group_hw_idle); + SET_BIC(BIC_CPU_c6, &bic_group_hw_idle); + SET_BIC(BIC_CPU_c7, &bic_group_hw_idle); + SET_BIC(BIC_GFX_rc6, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc2, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc3, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc6, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc7, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc8, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc9, &bic_group_hw_idle); + SET_BIC(BIC_Pkgpc10, &bic_group_hw_idle); + SET_BIC(BIC_CPU_LPI, &bic_group_hw_idle); + SET_BIC(BIC_SYS_LPI, &bic_group_hw_idle); + SET_BIC(BIC_Mod_c6, &bic_group_hw_idle); + SET_BIC(BIC_Totl_c0, &bic_group_hw_idle); + SET_BIC(BIC_Any_c0, &bic_group_hw_idle); + SET_BIC(BIC_GFX_c0, &bic_group_hw_idle); + SET_BIC(BIC_CPUGFX, &bic_group_hw_idle); + SET_BIC(BIC_SAM_mc6, &bic_group_hw_idle); + SET_BIC(BIC_Diec6, &bic_group_hw_idle); + + BIC_INIT(&bic_group_sw_idle); + SET_BIC(BIC_Busy, &bic_group_sw_idle); + SET_BIC(BIC_cpuidle, &bic_group_sw_idle); + SET_BIC(BIC_pct_idle, &bic_group_sw_idle); + + BIC_INIT(&bic_group_idle); + + CPU_OR(&bic_group_idle, &bic_group_idle, &bic_group_hw_idle); + SET_BIC(BIC_pct_idle, &bic_group_idle); + + BIC_INIT(&bic_group_cache); + SET_BIC(BIC_LLC_MRPS, &bic_group_cache); + SET_BIC(BIC_LLC_HIT, &bic_group_cache); + SET_BIC(BIC_L2_MRPS, &bic_group_cache); + SET_BIC(BIC_L2_HIT, &bic_group_cache); + + BIC_INIT(&bic_group_other); + SET_BIC(BIC_IRQ, &bic_group_other); + SET_BIC(BIC_NMI, &bic_group_other); + SET_BIC(BIC_SMI, &bic_group_other); + SET_BIC(BIC_ThreadC, &bic_group_other); + SET_BIC(BIC_CoreTmp, &bic_group_other); + SET_BIC(BIC_IPC, &bic_group_other); + + BIC_INIT(&bic_group_disabled_by_default); + SET_BIC(BIC_USEC, &bic_group_disabled_by_default); + SET_BIC(BIC_TOD, &bic_group_disabled_by_default); + SET_BIC(BIC_cpuidle, &bic_group_disabled_by_default); + SET_BIC(BIC_APIC, &bic_group_disabled_by_default); + SET_BIC(BIC_X2APIC, &bic_group_disabled_by_default); + + BIC_INIT(&bic_enabled); + bic_set_all(&bic_enabled); + bic_clear_bits(&bic_enabled, &bic_group_disabled_by_default); + + BIC_INIT(&bic_present); + SET_BIC(BIC_USEC, &bic_present); + SET_BIC(BIC_TOD, &bic_present); + SET_BIC(BIC_cpuidle, &bic_present); + SET_BIC(BIC_APIC, &bic_present); + SET_BIC(BIC_X2APIC, &bic_present); + SET_BIC(BIC_pct_idle, &bic_present); +} /* * MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit: @@ -313,12 +484,12 @@ unsigned long long bic_present = BIC_USEC | BIC_TOD | BIC_sysfs | BIC_APIC | BIC #define PCL_10 14 /* PC10 */ #define PCLUNL 15 /* Unlimited */ -struct amperf_group_fd; - char *proc_stat = "/proc/stat"; FILE *outf; int *fd_percpu; int *fd_instr_count_percpu; +int *fd_llc_percpu; +int *fd_l2_percpu; struct timeval interval_tv = { 5, 0 }; struct timespec interval_ts = { 5, 0 }; @@ -329,11 +500,13 @@ unsigned int quiet; unsigned int shown; unsigned int sums_need_wide_columns; unsigned int rapl_joules; +unsigned int valid_rapl_msrs; unsigned int summary_only; unsigned int list_header_only; unsigned int dump_only; unsigned int force_load; -unsigned int has_aperf; +unsigned int cpuid_has_aperf_mperf; +unsigned int cpuid_has_hv; unsigned int has_aperf_access; unsigned int has_epb; unsigned int has_turbo; @@ -364,7 +537,7 @@ double rapl_dram_energy_units, rapl_energy_units, rapl_psys_energy_units; double rapl_joule_counter_range; unsigned int crystal_hz; unsigned long long tsc_hz; -int base_cpu; +int master_cpu; unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */ /* IA32_HWP_REQUEST, IA32_HWP_STATUS */ unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */ @@ -399,8 +572,7 @@ static struct gfx_sysfs_info gfx_info[GFX_MAX]; int get_msr(int cpu, off_t offset, unsigned long long *msr); int add_counter(unsigned int msr_num, char *path, char *name, - unsigned int width, enum counter_scope scope, - enum counter_type type, enum counter_format format, int flags, int package_num); + unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int package_num); /* Model specific support Start */ @@ -425,7 +597,7 @@ struct platform_features { bool has_cst_prewake_bit; /* Cstate prewake bit in MSR_IA32_POWER_CTL */ int trl_msrs; /* MSR_TURBO_RATIO_LIMIT/LIMIT1/LIMIT2/SECONDARY, Atom TRL MSRs */ int plr_msrs; /* MSR_CORE/GFX/RING_PERF_LIMIT_REASONS */ - int rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */ + int plat_rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */ bool has_per_core_rapl; /* Indicates cores energy collection is per-core, not per-package. AMD specific for now */ bool has_rapl_divisor; /* Divisor for Energy unit raw value from MSR_RAPL_POWER_UNIT */ bool has_fixed_rapl_unit; /* Fixed Energy Unit used for DRAM RAPL Domain */ @@ -457,7 +629,7 @@ double slm_bclk(void) unsigned int i; double freq; - if (get_msr(base_cpu, MSR_FSB_FREQ, &msr)) + if (get_msr(master_cpu, MSR_FSB_FREQ, &msr)) fprintf(outf, "SLM BCLK: unknown\n"); i = msr & 0xf; @@ -535,7 +707,7 @@ enum rapl_msrs { #define RAPL_PKG_ALL (RAPL_PKG | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO) #define RAPL_DRAM_ALL (RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_DRAM_POWER_INFO) #define RAPL_CORE_ALL (RAPL_CORE | RAPL_CORE_POLICY) -#define RAPL_GFX_ALL (RAPL_GFX | RAPL_GFX_POLIGY) +#define RAPL_GFX_ALL (RAPL_GFX | RAPL_GFX_POLICY) #define RAPL_AMD_F17H (RAPL_AMD_PWR_UNIT | RAPL_AMD_CORE_ENERGY_STAT | RAPL_AMD_PKG_ENERGY_STAT) @@ -580,7 +752,7 @@ static const struct platform_features snb_features = { .cst_limit = CST_LIMIT_SNB, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features snx_features = { @@ -592,7 +764,7 @@ static const struct platform_features snx_features = { .cst_limit = CST_LIMIT_SNB, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, }; static const struct platform_features ivb_features = { @@ -605,7 +777,7 @@ static const struct platform_features ivb_features = { .cst_limit = CST_LIMIT_SNB, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features ivx_features = { @@ -617,7 +789,7 @@ static const struct platform_features ivx_features = { .cst_limit = CST_LIMIT_SNB, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE | TRL_LIMIT1, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, }; static const struct platform_features hsw_features = { @@ -631,7 +803,7 @@ static const struct platform_features hsw_features = { .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features hsx_features = { @@ -645,7 +817,7 @@ static const struct platform_features hsx_features = { .has_irtl_msrs = 1, .trl_msrs = TRL_BASE | TRL_LIMIT1 | TRL_LIMIT2, .plr_msrs = PLR_CORE | PLR_RING, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, .has_fixed_rapl_unit = 1, }; @@ -660,7 +832,7 @@ static const struct platform_features hswl_features = { .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features hswg_features = { @@ -674,7 +846,7 @@ static const struct platform_features hswg_features = { .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features bdw_features = { @@ -687,7 +859,7 @@ static const struct platform_features bdw_features = { .cst_limit = CST_LIMIT_HSW, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features bdwg_features = { @@ -700,7 +872,7 @@ static const struct platform_features bdwg_features = { .cst_limit = CST_LIMIT_HSW, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features bdx_features = { @@ -714,7 +886,7 @@ static const struct platform_features bdx_features = { .has_irtl_msrs = 1, .has_cst_auto_convension = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, .has_fixed_rapl_unit = 1, }; @@ -731,7 +903,7 @@ static const struct platform_features skl_features = { .has_ext_cst_msrs = 1, .trl_msrs = TRL_BASE, .tcc_offset_bits = 6, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, .enable_tsc_tweak = 1, }; @@ -748,7 +920,7 @@ static const struct platform_features cnl_features = { .has_ext_cst_msrs = 1, .trl_msrs = TRL_BASE, .tcc_offset_bits = 6, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, .enable_tsc_tweak = 1, }; @@ -766,7 +938,7 @@ static const struct platform_features adl_features = { .has_ext_cst_msrs = cnl_features.has_ext_cst_msrs, .trl_msrs = cnl_features.trl_msrs, .tcc_offset_bits = cnl_features.tcc_offset_bits, - .rapl_msrs = cnl_features.rapl_msrs, + .plat_rapl_msrs = cnl_features.plat_rapl_msrs, .enable_tsc_tweak = cnl_features.enable_tsc_tweak, }; @@ -784,7 +956,7 @@ static const struct platform_features lnl_features = { .has_ext_cst_msrs = adl_features.has_ext_cst_msrs, .trl_msrs = adl_features.trl_msrs, .tcc_offset_bits = adl_features.tcc_offset_bits, - .rapl_msrs = adl_features.rapl_msrs, + .plat_rapl_msrs = adl_features.plat_rapl_msrs, .enable_tsc_tweak = adl_features.enable_tsc_tweak, }; @@ -799,7 +971,7 @@ static const struct platform_features skx_features = { .has_irtl_msrs = 1, .has_cst_auto_convension = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, .has_fixed_rapl_unit = 1, }; @@ -815,7 +987,7 @@ static const struct platform_features icx_features = { .has_irtl_msrs = 1, .has_cst_prewake_bit = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, .has_fixed_rapl_unit = 1, }; @@ -832,7 +1004,25 @@ static const struct platform_features spr_features = { .has_cst_prewake_bit = 1, .has_fixed_rapl_psys_unit = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, +}; + +static const struct platform_features dmr_features = { + .has_msr_misc_feature_control = spr_features.has_msr_misc_feature_control, + .has_msr_misc_pwr_mgmt = spr_features.has_msr_misc_pwr_mgmt, + .has_nhm_msrs = spr_features.has_nhm_msrs, + .bclk_freq = spr_features.bclk_freq, + .supported_cstates = spr_features.supported_cstates, + .cst_limit = spr_features.cst_limit, + .has_msr_core_c1_res = spr_features.has_msr_core_c1_res, + .has_cst_prewake_bit = spr_features.has_cst_prewake_bit, + .has_fixed_rapl_psys_unit = spr_features.has_fixed_rapl_psys_unit, + .trl_msrs = spr_features.trl_msrs, + .has_msr_module_c6_res_ms = 1, /* DMR has Dual-Core-Module and MC6 MSR */ + .plat_rapl_msrs = 0, /* DMR does not have RAPL MSRs */ + .plr_msrs = 0, /* DMR does not have PLR MSRs */ + .has_irtl_msrs = 0, /* DMR does not have IRTL MSRs */ + .has_config_tdp = 0, /* DMR does not have CTDP MSRs */ }; static const struct platform_features srf_features = { @@ -848,7 +1038,7 @@ static const struct platform_features srf_features = { .has_irtl_msrs = 1, .has_cst_prewake_bit = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, }; static const struct platform_features grr_features = { @@ -864,7 +1054,7 @@ static const struct platform_features grr_features = { .has_irtl_msrs = 1, .has_cst_prewake_bit = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, }; static const struct platform_features slv_features = { @@ -877,7 +1067,7 @@ static const struct platform_features slv_features = { .has_msr_c6_demotion_policy_config = 1, .has_msr_atom_pkg_c6_residency = 1, .trl_msrs = TRL_ATOM, - .rapl_msrs = RAPL_PKG | RAPL_CORE, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE, .has_rapl_divisor = 1, .rapl_quirk_tdp = 30, }; @@ -890,7 +1080,7 @@ static const struct platform_features slvd_features = { .cst_limit = CST_LIMIT_SLV, .has_msr_atom_pkg_c6_residency = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE, .rapl_quirk_tdp = 30, }; @@ -911,7 +1101,7 @@ static const struct platform_features gmt_features = { .cst_limit = CST_LIMIT_GMT, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, }; static const struct platform_features gmtd_features = { @@ -924,7 +1114,7 @@ static const struct platform_features gmtd_features = { .has_irtl_msrs = 1, .has_msr_core_c1_res = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS, }; static const struct platform_features gmtp_features = { @@ -936,7 +1126,7 @@ static const struct platform_features gmtp_features = { .cst_limit = CST_LIMIT_GMT, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, }; static const struct platform_features tmt_features = { @@ -947,7 +1137,7 @@ static const struct platform_features tmt_features = { .cst_limit = CST_LIMIT_GMT, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX, .enable_tsc_tweak = 1, }; @@ -959,7 +1149,7 @@ static const struct platform_features tmtd_features = { .cst_limit = CST_LIMIT_GMT, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL, }; static const struct platform_features knl_features = { @@ -971,7 +1161,7 @@ static const struct platform_features knl_features = { .cst_limit = CST_LIMIT_KNL, .has_msr_knl_core_c6_residency = 1, .trl_msrs = TRL_KNL, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, .has_fixed_rapl_unit = 1, .need_perf_multiplier = 1, }; @@ -980,7 +1170,7 @@ static const struct platform_features default_features = { }; static const struct platform_features amd_features_with_rapl = { - .rapl_msrs = RAPL_AMD_F17H, + .plat_rapl_msrs = RAPL_AMD_F17H, .has_per_core_rapl = 1, .rapl_quirk_tdp = 280, /* This is the max stock TDP of HEDT/Server Fam17h+ chips */ }; @@ -1024,12 +1214,14 @@ static const struct platform_data turbostat_pdata[] = { { INTEL_EMERALDRAPIDS_X, &spr_features }, { INTEL_GRANITERAPIDS_X, &spr_features }, { INTEL_GRANITERAPIDS_D, &spr_features }, + { INTEL_DIAMONDRAPIDS_X, &dmr_features }, { INTEL_LAKEFIELD, &cnl_features }, { INTEL_ALDERLAKE, &adl_features }, { INTEL_ALDERLAKE_L, &adl_features }, { INTEL_RAPTORLAKE, &adl_features }, { INTEL_RAPTORLAKE_P, &adl_features }, { INTEL_RAPTORLAKE_S, &adl_features }, + { INTEL_BARTLETTLAKE, &adl_features }, { INTEL_METEORLAKE, &adl_features }, { INTEL_METEORLAKE_L, &adl_features }, { INTEL_ARROWLAKE_H, &adl_features }, @@ -1037,6 +1229,9 @@ static const struct platform_data turbostat_pdata[] = { { INTEL_ARROWLAKE, &adl_features }, { INTEL_LUNARLAKE_M, &lnl_features }, { INTEL_PANTHERLAKE_L, &lnl_features }, + { INTEL_NOVALAKE, &lnl_features }, + { INTEL_NOVALAKE_L, &lnl_features }, + { INTEL_WILDCATLAKE_L, &lnl_features }, { INTEL_ATOM_SILVERMONT, &slv_features }, { INTEL_ATOM_SILVERMONT_D, &slvd_features }, { INTEL_ATOM_AIRMONT, &amt_features }, @@ -1056,19 +1251,96 @@ static const struct platform_data turbostat_pdata[] = { * Missing support for * INTEL_ICELAKE * INTEL_ATOM_SILVERMONT_MID - * INTEL_ATOM_AIRMONT_MID + * INTEL_ATOM_SILVERMONT_MID2 * INTEL_ATOM_AIRMONT_NP */ { 0, NULL }, }; +struct { + unsigned int uniform; + unsigned int pcore; + unsigned int ecore; + unsigned int lcore; +} perf_pmu_types; + +/* + * Events are enumerated in https://github.com/intel/perfmon + * and tools/perf/pmu-events/arch/x86/.../cache.json + */ +struct perf_l2_events { + unsigned long long refs; /* L2_REQUEST.ALL */ + unsigned long long hits; /* L2_REQUEST.HIT */ +}; + +struct perf_model_support { + unsigned int vfm; + struct perf_l2_events first; + struct perf_l2_events second; + struct perf_l2_events third; +} *perf_model_support; + +/* Perf Cache Events */ +#define PCE(ext_umask, umask) (((unsigned long long) ext_umask) << 40 | umask << 8 | 0x24) + +/* + * Enumerate up to three perf CPU PMU's in a system. + * The first, second, and third columns are populated without skipping, describing + * pcore, ecore, lcore PMUs, in order, if present. (The associated PMU "type" field is + * read from sysfs in all cases.) Eg. + * + * non-hybrid: + * GNR: pcore, {}, {} + * ADL-N: ecore, {}, {} + * hybrid: + * MTL: pcore, ecore, {}% + * ARL-H: pcore, ecore, lcore + * LNL: ecore, ecore%%, {} + * + * % MTL physical lcore share architecture and PMU with ecore, and are thus not enumerated separately. + * %% LNL physical lcore is enumerated by perf as ecore + */ +static struct perf_model_support turbostat_perf_model_support[] = { + { INTEL_SAPPHIRERAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} }, + { INTEL_EMERALDRAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} }, + { INTEL_GRANITERAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} }, + { INTEL_GRANITERAPIDS_D, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} }, + { INTEL_DIAMONDRAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, {}, {} }, + + { INTEL_ATOM_GRACEMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* ADL-N */ + { INTEL_ATOM_CRESTMONT_X, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* SRF */ + { INTEL_ATOM_CRESTMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* GRR */ + { INTEL_ATOM_DARKMONT_X, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {}, {} }, /* CWF */ + + { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_ALDERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_RAPTORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_RAPTORLAKE_P, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_RAPTORLAKE_S, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_METEORLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_METEORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + { INTEL_ARROWLAKE_U, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} }, + + { INTEL_LUNARLAKE_M, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, {} }, + { INTEL_ARROWLAKE_H, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)} }, + { INTEL_ARROWLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, {} }, + + { INTEL_PANTHERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} }, + { INTEL_WILDCATLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} }, + + { INTEL_NOVALAKE, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} }, + { INTEL_NOVALAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} }, + + { 0, {}, {}, {} } +}; + static const struct platform_features *platform; void probe_platform_features(unsigned int family, unsigned int model) { int i; - if (authentic_amd || hygon_genuine) { /* fallback to default features on unsupported models */ force_load++; @@ -1102,11 +1374,25 @@ end: if (platform) return; - fprintf(stderr, "Unsupported platform detected.\n" - "\tSee RUN THE LATEST VERSION on turbostat(8)\n"); + fprintf(stderr, "Unsupported platform detected.\n\tSee RUN THE LATEST VERSION on turbostat(8)\n"); exit(1); } +void init_perf_model_support(unsigned int family, unsigned int model) +{ + int i; + + if (!genuine_intel) + return; + + for (i = 0; turbostat_perf_model_support[i].vfm; i++) { + if (VFM_FAMILY(turbostat_perf_model_support[i].vfm) == family && VFM_MODEL(turbostat_perf_model_support[i].vfm) == model) { + perf_model_support = &turbostat_perf_model_support[i]; + return; + } + } +} + /* Model specific support End */ #define TJMAX_DEFAULT 100 @@ -1121,8 +1407,9 @@ end: int backwards_count; char *progname; -#define CPU_SUBSET_MAXCPUS 1024 /* need to use before probe... */ +#define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */ cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset; +cpu_set_t *perf_pcore_set, *perf_ecore_set, *perf_lcore_set; size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size; #define MAX_ADDED_THREAD_COUNTERS 24 #define MAX_ADDED_CORE_COUNTERS 8 @@ -1182,7 +1469,7 @@ struct rapl_counter_arch_info { int msr_shift; /* Positive mean shift right, negative mean shift left */ double *platform_rapl_msr_scale; /* Scale applied to values read by MSR (platform dependent, filled at runtime) */ unsigned int rci_index; /* Maps data from perf counters to global variables */ - unsigned long long bic; + unsigned int bic_number; double compat_scale; /* Some counters require constant scaling to be in the same range as other, similar ones */ unsigned long long flags; }; @@ -1197,7 +1484,20 @@ static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { .msr_shift = 0, .platform_rapl_msr_scale = &rapl_energy_units, .rci_index = RAPL_RCI_INDEX_ENERGY_PKG, - .bic = BIC_PkgWatt | BIC_Pkg_J, + .bic_number = BIC_PkgWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_PKG, + .perf_subsys = "power", + .perf_name = "energy-pkg", + .msr = MSR_PKG_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_PKG, + .bic_number = BIC_Pkg_J, .compat_scale = 1.0, .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, }, @@ -1210,7 +1510,20 @@ static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { .msr_shift = 0, .platform_rapl_msr_scale = &rapl_energy_units, .rci_index = RAPL_RCI_INDEX_ENERGY_PKG, - .bic = BIC_PkgWatt | BIC_Pkg_J, + .bic_number = BIC_PkgWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_AMD_F17H, + .perf_subsys = "power", + .perf_name = "energy-pkg", + .msr = MSR_PKG_ENERGY_STAT, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_PKG, + .bic_number = BIC_Pkg_J, .compat_scale = 1.0, .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, }, @@ -1223,7 +1536,33 @@ static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { .msr_shift = 0, .platform_rapl_msr_scale = &rapl_energy_units, .rci_index = RAPL_RCI_INDEX_ENERGY_CORES, - .bic = BIC_CorWatt | BIC_Cor_J, + .bic_number = BIC_CorWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_CORE_ENERGY_STATUS, + .perf_subsys = "power", + .perf_name = "energy-cores", + .msr = MSR_PP0_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_CORES, + .bic_number = BIC_Cor_J, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_DRAM, + .perf_subsys = "power", + .perf_name = "energy-ram", + .msr = MSR_DRAM_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_dram_energy_units, + .rci_index = RAPL_RCI_INDEX_DRAM, + .bic_number = BIC_RAMWatt, .compat_scale = 1.0, .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, }, @@ -1236,7 +1575,20 @@ static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { .msr_shift = 0, .platform_rapl_msr_scale = &rapl_dram_energy_units, .rci_index = RAPL_RCI_INDEX_DRAM, - .bic = BIC_RAMWatt | BIC_RAM_J, + .bic_number = BIC_RAM_J, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_GFX, + .perf_subsys = "power", + .perf_name = "energy-gpu", + .msr = MSR_PP1_ENERGY_STATUS, + .msr_mask = 0xFFFFFFFFFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_GFX, + .bic_number = BIC_GFXWatt, .compat_scale = 1.0, .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, }, @@ -1249,7 +1601,7 @@ static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { .msr_shift = 0, .platform_rapl_msr_scale = &rapl_energy_units, .rci_index = RAPL_RCI_INDEX_GFX, - .bic = BIC_GFXWatt | BIC_GFX_J, + .bic_number = BIC_GFX_J, .compat_scale = 1.0, .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, }, @@ -1262,7 +1614,7 @@ static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { .msr_shift = 0, .platform_rapl_msr_scale = &rapl_time_units, .rci_index = RAPL_RCI_INDEX_PKG_PERF_STATUS, - .bic = BIC_PKG__, + .bic_number = BIC_PKG__, .compat_scale = 100.0, .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, }, @@ -1275,7 +1627,7 @@ static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { .msr_shift = 0, .platform_rapl_msr_scale = &rapl_time_units, .rci_index = RAPL_RCI_INDEX_DRAM_PERF_STATUS, - .bic = BIC_RAM__, + .bic_number = BIC_RAM__, .compat_scale = 100.0, .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM, }, @@ -1288,7 +1640,20 @@ static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { .msr_shift = 0, .platform_rapl_msr_scale = &rapl_energy_units, .rci_index = RAPL_RCI_INDEX_CORE_ENERGY, - .bic = BIC_CorWatt | BIC_Cor_J, + .bic_number = BIC_CorWatt, + .compat_scale = 1.0, + .flags = 0, + }, + { + .feature_mask = RAPL_AMD_F17H, + .perf_subsys = NULL, + .perf_name = NULL, + .msr = MSR_CORE_ENERGY_STAT, + .msr_mask = 0xFFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_energy_units, + .rci_index = RAPL_RCI_INDEX_CORE_ENERGY, + .bic_number = BIC_Cor_J, .compat_scale = 1.0, .flags = 0, }, @@ -1301,7 +1666,20 @@ static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = { .msr_shift = 0, .platform_rapl_msr_scale = &rapl_psys_energy_units, .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM, - .bic = BIC_SysWatt | BIC_Sys_J, + .bic_number = BIC_SysWatt, + .compat_scale = 1.0, + .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM, + }, + { + .feature_mask = RAPL_PSYS, + .perf_subsys = "power", + .perf_name = "energy-psys", + .msr = MSR_PLATFORM_ENERGY_STATUS, + .msr_mask = 0x00000000FFFFFFFF, + .msr_shift = 0, + .platform_rapl_msr_scale = &rapl_psys_energy_units, + .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM, + .bic_number = BIC_Sys_J, .compat_scale = 1.0, .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM, }, @@ -1350,7 +1728,7 @@ struct cstate_counter_arch_info { const char *perf_name; unsigned long long msr; unsigned int rci_index; /* Maps data from perf counters to global variables */ - unsigned long long bic; + unsigned int bic_number; unsigned long long flags; int pkg_cstate_limit; }; @@ -1362,7 +1740,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c1-residency", .msr = MSR_CORE_C1_RES, .rci_index = CCSTATE_RCI_INDEX_C1_RESIDENCY, - .bic = BIC_CPU_c1, + .bic_number = BIC_CPU_c1, .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD, .pkg_cstate_limit = 0, }, @@ -1372,7 +1750,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c3-residency", .msr = MSR_CORE_C3_RESIDENCY, .rci_index = CCSTATE_RCI_INDEX_C3_RESIDENCY, - .bic = BIC_CPU_c3, + .bic_number = BIC_CPU_c3, .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, .pkg_cstate_limit = 0, }, @@ -1382,7 +1760,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c6-residency", .msr = MSR_CORE_C6_RESIDENCY, .rci_index = CCSTATE_RCI_INDEX_C6_RESIDENCY, - .bic = BIC_CPU_c6, + .bic_number = BIC_CPU_c6, .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, .pkg_cstate_limit = 0, }, @@ -1392,7 +1770,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c7-residency", .msr = MSR_CORE_C7_RESIDENCY, .rci_index = CCSTATE_RCI_INDEX_C7_RESIDENCY, - .bic = BIC_CPU_c7, + .bic_number = BIC_CPU_c7, .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY, .pkg_cstate_limit = 0, }, @@ -1402,7 +1780,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c2-residency", .msr = MSR_PKG_C2_RESIDENCY, .rci_index = PCSTATE_RCI_INDEX_C2_RESIDENCY, - .bic = BIC_Pkgpc2, + .bic_number = BIC_Pkgpc2, .flags = 0, .pkg_cstate_limit = PCL__2, }, @@ -1412,7 +1790,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c3-residency", .msr = MSR_PKG_C3_RESIDENCY, .rci_index = PCSTATE_RCI_INDEX_C3_RESIDENCY, - .bic = BIC_Pkgpc3, + .bic_number = BIC_Pkgpc3, .flags = 0, .pkg_cstate_limit = PCL__3, }, @@ -1422,7 +1800,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c6-residency", .msr = MSR_PKG_C6_RESIDENCY, .rci_index = PCSTATE_RCI_INDEX_C6_RESIDENCY, - .bic = BIC_Pkgpc6, + .bic_number = BIC_Pkgpc6, .flags = 0, .pkg_cstate_limit = PCL__6, }, @@ -1432,7 +1810,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c7-residency", .msr = MSR_PKG_C7_RESIDENCY, .rci_index = PCSTATE_RCI_INDEX_C7_RESIDENCY, - .bic = BIC_Pkgpc7, + .bic_number = BIC_Pkgpc7, .flags = 0, .pkg_cstate_limit = PCL__7, }, @@ -1442,7 +1820,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c8-residency", .msr = MSR_PKG_C8_RESIDENCY, .rci_index = PCSTATE_RCI_INDEX_C8_RESIDENCY, - .bic = BIC_Pkgpc8, + .bic_number = BIC_Pkgpc8, .flags = 0, .pkg_cstate_limit = PCL__8, }, @@ -1452,7 +1830,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c9-residency", .msr = MSR_PKG_C9_RESIDENCY, .rci_index = PCSTATE_RCI_INDEX_C9_RESIDENCY, - .bic = BIC_Pkgpc9, + .bic_number = BIC_Pkgpc9, .flags = 0, .pkg_cstate_limit = PCL__9, }, @@ -1462,7 +1840,7 @@ static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = { .perf_name = "c10-residency", .msr = MSR_PKG_C10_RESIDENCY, .rci_index = PCSTATE_RCI_INDEX_C10_RESIDENCY, - .bic = BIC_Pkgpc10, + .bic_number = BIC_Pkgpc10, .flags = 0, .pkg_cstate_limit = PCL_10, }, @@ -1627,7 +2005,7 @@ int pmt_telemdir_sort(const struct dirent **a, const struct dirent **b) sscanf((*a)->d_name, "telem%u", &aidx); sscanf((*b)->d_name, "telem%u", &bidx); - return aidx >= bidx; + return (aidx > bidx) ? 1 : (aidx < bidx) ? -1 : 0; } const struct dirent *pmt_diriter_next(struct pmt_diriter_t *iter) @@ -1728,6 +2106,14 @@ void pmt_counter_resize(struct pmt_counter *pcounter, unsigned int new_size) pmt_counter_resize_(pcounter, new_size); } +struct llc_stats { + unsigned long long references; + unsigned long long misses; +}; +struct l2_stats { + unsigned long long references; + unsigned long long hits; +}; struct thread_data { struct timeval tv_begin; struct timeval tv_end; @@ -1740,6 +2126,8 @@ struct thread_data { unsigned long long irq_count; unsigned long long nmi_count; unsigned int smi_count; + struct llc_stats llc; + struct l2_stats l2; unsigned int cpu_id; unsigned int apic_id; unsigned int x2apic_id; @@ -1748,25 +2136,24 @@ struct thread_data { unsigned long long counter[MAX_ADDED_THREAD_COUNTERS]; unsigned long long perf_counter[MAX_ADDED_THREAD_COUNTERS]; unsigned long long pmt_counter[PMT_MAX_ADDED_THREAD_COUNTERS]; -} *thread_even, *thread_odd; +}; struct core_data { - int base_cpu; + int first_cpu; unsigned long long c3; unsigned long long c6; unsigned long long c7; unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */ unsigned int core_temp_c; struct rapl_counter core_energy; /* MSR_CORE_ENERGY_STAT */ - unsigned int core_id; unsigned long long core_throt_cnt; unsigned long long counter[MAX_ADDED_CORE_COUNTERS]; unsigned long long perf_counter[MAX_ADDED_CORE_COUNTERS]; unsigned long long pmt_counter[PMT_MAX_ADDED_CORE_COUNTERS]; -} *core_even, *core_odd; +}; struct pkg_data { - int base_cpu; + int first_cpu; unsigned long long pc2; unsigned long long pc3; unsigned long long pc6; @@ -1786,7 +2173,6 @@ struct pkg_data { long long sam_mc6_ms; unsigned int sam_mhz; unsigned int sam_act_mhz; - unsigned int package_id; struct rapl_counter energy_pkg; /* MSR_PKG_ENERGY_STATUS */ struct rapl_counter energy_dram; /* MSR_DRAM_ENERGY_STATUS */ struct rapl_counter energy_cores; /* MSR_PP0_ENERGY_STATUS */ @@ -1799,26 +2185,10 @@ struct pkg_data { unsigned long long counter[MAX_ADDED_PACKAGE_COUNTERS]; unsigned long long perf_counter[MAX_ADDED_PACKAGE_COUNTERS]; unsigned long long pmt_counter[PMT_MAX_ADDED_PACKAGE_COUNTERS]; -} *package_even, *package_odd; - -#define ODD_COUNTERS thread_odd, core_odd, package_odd -#define EVEN_COUNTERS thread_even, core_even, package_even - -#define GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no) \ - ((thread_base) + \ - ((pkg_no) * \ - topo.nodes_per_pkg * topo.cores_per_node * topo.threads_per_core) + \ - ((node_no) * topo.cores_per_node * topo.threads_per_core) + \ - ((core_no) * topo.threads_per_core) + \ - (thread_no)) - -#define GET_CORE(core_base, core_no, node_no, pkg_no) \ - ((core_base) + \ - ((pkg_no) * topo.nodes_per_pkg * topo.cores_per_node) + \ - ((node_no) * topo.cores_per_node) + \ - (core_no)) +}; -#define GET_PKG(pkg_base, pkg_no) (pkg_base + pkg_no) +#define ODD_COUNTERS odd.threads, odd.cores, odd.packages +#define EVEN_COUNTERS even.threads, even.cores, even.packages /* * The accumulated sum of MSR is defined as a monotonic @@ -1857,7 +2227,7 @@ off_t idx_to_offset(int idx) switch (idx) { case IDX_PKG_ENERGY: - if (platform->rapl_msrs & RAPL_AMD_F17H) + if (platform->plat_rapl_msrs & RAPL_AMD_F17H) offset = MSR_PKG_ENERGY_STAT; else offset = MSR_PKG_ENERGY_STATUS; @@ -1923,19 +2293,19 @@ int idx_valid(int idx) { switch (idx) { case IDX_PKG_ENERGY: - return platform->rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H); + return valid_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H); case IDX_DRAM_ENERGY: - return platform->rapl_msrs & RAPL_DRAM; + return valid_rapl_msrs & RAPL_DRAM; case IDX_PP0_ENERGY: - return platform->rapl_msrs & RAPL_CORE_ENERGY_STATUS; + return valid_rapl_msrs & RAPL_CORE_ENERGY_STATUS; case IDX_PP1_ENERGY: - return platform->rapl_msrs & RAPL_GFX; + return valid_rapl_msrs & RAPL_GFX; case IDX_PKG_PERF: - return platform->rapl_msrs & RAPL_PKG_PERF_STATUS; + return valid_rapl_msrs & RAPL_PKG_PERF_STATUS; case IDX_DRAM_PERF: - return platform->rapl_msrs & RAPL_DRAM_PERF_STATUS; + return valid_rapl_msrs & RAPL_DRAM_PERF_STATUS; case IDX_PSYS_ENERGY: - return platform->rapl_msrs & RAPL_PSYS; + return valid_rapl_msrs & RAPL_PSYS; default: return 0; } @@ -2001,24 +2371,29 @@ static void free_sys_msr_counters(void) sys.added_package_counters -= free_msr_counters_(&sys.pp); } -struct system_summary { - struct thread_data threads; - struct core_data cores; - struct pkg_data packages; -} average; +struct counters { + struct thread_data *threads; + struct core_data *cores; + struct pkg_data *packages; +} average, even, odd; struct platform_counters { struct rapl_counter energy_psys; /* MSR_PLATFORM_ENERGY_STATUS */ } platform_counters_odd, platform_counters_even; +#define MAX_HT_ID 3 /* support SMT-4 */ + struct cpu_topology { - int physical_package_id; + int cpu_id; + int core_id; /* unique within a package */ + int module_id; + int package_id; int die_id; - int logical_cpu_id; + int l3_id; int physical_node_id; int logical_node_id; /* 0-based count within the package */ - int physical_core_id; - int thread_id; + int ht_id; /* unique within a core */ + int ht_sibling_cpu_id[MAX_HT_ID + 1]; int type; cpu_set_t *put_ids; /* Processing Unit/Thread IDs */ } *cpus; @@ -2027,17 +2402,20 @@ struct topo_params { int num_packages; int num_die; int num_cpus; - int num_cores; + int num_cores; /* system wide */ int allowed_packages; int allowed_cpus; int allowed_cores; int max_cpu_num; - int max_core_id; + int max_core_id; /* within a package */ + int min_module_id; /* system wide */ + int max_module_id; /* system wide */ int max_package_id; int max_die_id; + int max_l3_id; int max_node_num; int nodes_per_pkg; - int cores_per_node; + int cores_per_pkg; int threads_per_core; } topo; @@ -2055,66 +2433,106 @@ char *sys_lpi_file_debugfs = "/sys/kernel/debug/pmc_core/slp_s0_residency_usec"; int cpu_is_not_present(int cpu) { + if (cpu < 0) + return 1; + return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set); } int cpu_is_not_allowed(int cpu) { + if (cpu < 0) + return 1; + return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set); } +#define GLOBAL_CORE_ID(core_id, pkg_id) (core_id + pkg_id * (topo.max_core_id + 1)) /* * run func(thread, core, package) in topology order * skip non-present cpus */ +#define PER_THREAD_PARAMS struct thread_data *t, struct core_data *c, struct pkg_data *p + +int has_allowed_lower_ht_sibling(int cpu) +{ + int i; + + for (i = 0; i <= cpus[cpu].ht_id; ++i) { + int sibling_cpu_id = cpus[cpu].ht_sibling_cpu_id[i]; + + if (sibling_cpu_id == cpu) + return 0; + + if (!cpu_is_not_allowed(sibling_cpu_id)) + return 1; + } + return 0; +} + int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *), struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base) { - int retval, pkg_no, core_no, thread_no, node_no; + int cpu, retval; retval = 0; - for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) { - for (node_no = 0; node_no < topo.nodes_per_pkg; node_no++) { - for (core_no = 0; core_no < topo.cores_per_node; ++core_no) { - for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) { - struct thread_data *t; - struct core_data *c; - struct pkg_data *p; - t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no); + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + struct thread_data *t; + struct core_data *c; + struct pkg_data *p; - if (cpu_is_not_allowed(t->cpu_id)) - continue; + int pkg_id = cpus[cpu].package_id; - c = GET_CORE(core_base, core_no, node_no, pkg_no); - p = GET_PKG(pkg_base, pkg_no); + if (cpu_is_not_allowed(cpu)) + continue; - retval |= func(t, c, p); - } - } + if (has_allowed_lower_ht_sibling(cpu)) /* skip HT sibling */ + continue; + + t = &thread_base[cpu]; + c = &core_base[GLOBAL_CORE_ID(cpus[cpu].core_id, pkg_id)]; + p = &pkg_base[pkg_id]; + + retval |= func(t, c, p); + + /* Handle other HT siblings now */ + int i; + + for (i = 0; i <= MAX_HT_ID; ++i) { + int sibling_cpu_id = cpus[cpu].ht_sibling_cpu_id[i]; + + if (sibling_cpu_id < 0) + break; + + if (sibling_cpu_id == cpu) + continue; + + if (cpu_is_not_allowed(sibling_cpu_id)) + continue; + + t = &thread_base[sibling_cpu_id]; + + retval |= func(t, c, p); } } return retval; } -int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c) { - UNUSED(p); - - return ((int)t->cpu_id == c->base_cpu || c->base_cpu < 0); + return ((int)t->cpu_id == c->first_cpu || c->first_cpu < 0); } -int is_cpu_first_core_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int is_cpu_first_core_in_package(struct thread_data *t, struct pkg_data *p) { - UNUSED(c); - - return ((int)t->cpu_id == p->base_cpu || p->base_cpu < 0); + return ((int)t->cpu_id == p->first_cpu || p->first_cpu < 0); } int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p) { - return is_cpu_first_thread_in_core(t, c, p) && is_cpu_first_core_in_package(t, c, p); + return is_cpu_first_thread_in_core(t, c) && is_cpu_first_core_in_package(t, p); } int cpu_migrate(int cpu) @@ -2136,13 +2554,11 @@ int get_msr_fd(int cpu) if (fd) return fd; - - sprintf(pathname, "/dev/cpu/%d/msr", cpu); + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu); fd = open(pathname, O_RDONLY); if (fd < 0) - err(-1, "%s open failed, try chown or chmod +r /dev/cpu/*/msr, " - "or run with --no-msr, or run as root", pathname); - + err(-1, "%s open failed, try chown or chmod +r %s, " + "or run with --no-msr, or run as root", pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr"); fd_percpu[cpu] = fd; return fd; @@ -2150,14 +2566,26 @@ int get_msr_fd(int cpu) static void bic_disable_msr_access(void) { - const unsigned long bic_msrs = BIC_Mod_c6 | BIC_CoreTmp | - BIC_Totl_c0 | BIC_Any_c0 | BIC_GFX_c0 | BIC_CPUGFX | BIC_PkgTmp; - - bic_enabled &= ~bic_msrs; + CLR_BIC(BIC_Mod_c6, &bic_enabled); + CLR_BIC(BIC_CoreTmp, &bic_enabled); + CLR_BIC(BIC_Totl_c0, &bic_enabled); + CLR_BIC(BIC_Any_c0, &bic_enabled); + CLR_BIC(BIC_GFX_c0, &bic_enabled); + CLR_BIC(BIC_CPUGFX, &bic_enabled); + CLR_BIC(BIC_PkgTmp, &bic_enabled); free_sys_msr_counters(); } +static void bic_disable_perf_access(void) +{ + CLR_BIC(BIC_IPC, &bic_enabled); + CLR_BIC(BIC_LLC_MRPS, &bic_enabled); + CLR_BIC(BIC_LLC_HIT, &bic_enabled); + CLR_BIC(BIC_L2_MRPS, &bic_enabled); + CLR_BIC(BIC_L2_HIT, &bic_enabled); +} + static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags) { assert(!no_perf); @@ -2211,23 +2639,52 @@ int get_msr(int cpu, off_t offset, unsigned long long *msr) return 0; } -int probe_msr(int cpu, off_t offset) +int add_msr_counter(int cpu, off_t offset) { ssize_t retval; unsigned long long value; - assert(!no_msr); + if (no_msr) + return -1; + + if (!offset) + return -1; retval = pread(get_msr_fd(cpu), &value, sizeof(value), offset); - /* - * Expect MSRs to accumulate some non-zero value since the system was powered on. - * Treat zero as a read failure. - */ - if (retval != sizeof(value) || value == 0) - return 1; + /* if the read failed, the probe fails */ + if (retval != sizeof(value)) + return -1; - return 0; + if (value == 0) + return 0; + + return 1; +} + +int add_rapl_msr_counter(int cpu, const struct rapl_counter_arch_info *cai) +{ + int ret; + + if (!(valid_rapl_msrs & cai->feature_mask)) + return -1; + + ret = add_msr_counter(cpu, cai->msr); + if (ret < 0) + return -1; + + switch (cai->rci_index) { + case RAPL_RCI_INDEX_ENERGY_PKG: + case RAPL_RCI_INDEX_ENERGY_CORES: + case RAPL_RCI_INDEX_DRAM: + case RAPL_RCI_INDEX_GFX: + case RAPL_RCI_INDEX_ENERGY_PLATFORM: + if (ret == 0) + return 1; + } + + /* PKG,DRAM_PERF_STATUS MSRs, can return any value */ + return 1; } /* Convert CPU ID to domain ID for given added perf counter. */ @@ -2238,10 +2695,10 @@ unsigned int cpu_to_domain(const struct perf_counter_info *pc, int cpu) return cpu; case SCOPE_CORE: - return cpus[cpu].physical_core_id; + return cpus[cpu].core_id; case SCOPE_PACKAGE: - return cpus[cpu].physical_package_id; + return cpus[cpu].package_id; } __builtin_unreachable(); @@ -2252,6 +2709,8 @@ char *deferred_add_names[MAX_DEFERRED]; char *deferred_skip_names[MAX_DEFERRED]; int deferred_add_index; int deferred_skip_index; +unsigned int deferred_add_consumed; +unsigned int deferred_skip_consumed; /* * HIDE_LIST - hide this list of counters, show the rest [default] @@ -2313,9 +2772,7 @@ void help(void) " sets the Thermal Control Circuit temperature in\n" " degrees Celsius\n" " -h, --help\n" - " print this help message\n" - " -v, --version\n" - " print version information\n\nFor more help, run \"man turbostat\"\n"); + " print this help message\n -v, --version\n\t\tprint version information\n\nFor more help, run \"man turbostat\"\n"); } /* @@ -2323,10 +2780,9 @@ void help(void) * for all the strings in comma separate name_list, * set the approprate bit in return value. */ -unsigned long long bic_lookup(char *name_list, enum show_hide_mode mode) +void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode) { unsigned int i; - unsigned long long retval = 0; while (name_list) { char *comma; @@ -2338,36 +2794,49 @@ unsigned long long bic_lookup(char *name_list, enum show_hide_mode mode) for (i = 0; i < MAX_BIC; ++i) { if (!strcmp(name_list, bic[i].name)) { - retval |= (1ULL << i); + SET_BIC(i, ret_set); break; } if (!strcmp(name_list, "all")) { - retval |= ~0; + bic_set_all(ret_set); break; } else if (!strcmp(name_list, "topology")) { - retval |= BIC_TOPOLOGY; + CPU_OR(ret_set, ret_set, &bic_group_topology); break; } else if (!strcmp(name_list, "power")) { - retval |= BIC_THERMAL_PWR; + CPU_OR(ret_set, ret_set, &bic_group_thermal_pwr); break; } else if (!strcmp(name_list, "idle")) { - retval |= BIC_IDLE; + CPU_OR(ret_set, ret_set, &bic_group_idle); + break; + } else if (!strcmp(name_list, "cache")) { + CPU_OR(ret_set, ret_set, &bic_group_cache); + break; + } else if (!strcmp(name_list, "llc")) { + CPU_OR(ret_set, ret_set, &bic_group_cache); + break; + } else if (!strcmp(name_list, "swidle")) { + CPU_OR(ret_set, ret_set, &bic_group_sw_idle); + break; + } else if (!strcmp(name_list, "sysfs")) { /* legacy compatibility */ + CPU_OR(ret_set, ret_set, &bic_group_sw_idle); + break; + } else if (!strcmp(name_list, "hwidle")) { + CPU_OR(ret_set, ret_set, &bic_group_hw_idle); break; } else if (!strcmp(name_list, "frequency")) { - retval |= BIC_FREQUENCY; + CPU_OR(ret_set, ret_set, &bic_group_frequency); break; } else if (!strcmp(name_list, "other")) { - retval |= BIC_OTHER; + CPU_OR(ret_set, ret_set, &bic_group_other); break; } - } if (i == MAX_BIC) { if (mode == SHOW_LIST) { deferred_add_names[deferred_add_index++] = name_list; if (deferred_add_index >= MAX_DEFERRED) { - fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", - MAX_DEFERRED, name_list); + fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", MAX_DEFERRED, name_list); help(); exit(1); } @@ -2376,8 +2845,7 @@ unsigned long long bic_lookup(char *name_list, enum show_hide_mode mode) if (debug) fprintf(stderr, "deferred \"%s\"\n", name_list); if (deferred_skip_index >= MAX_DEFERRED) { - fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", - MAX_DEFERRED, name_list); + fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", MAX_DEFERRED, name_list); help(); exit(1); } @@ -2389,7 +2857,53 @@ unsigned long long bic_lookup(char *name_list, enum show_hide_mode mode) name_list++; } - return retval; +} + +/* + * print_name() + * Print column header name for raw 64-bit counter in 16 columns (at least 8-char plus a tab) + * Otherwise, allow the name + tab to fit within 8-coumn tab-stop. + * In both cases, left justififed, just like other turbostat columns, + * to allow the column values to consume the tab. + * + * Yes, 32-bit counters can overflow 8-columns, and + * 64-bit counters can overflow 16-columns, but that is uncommon. + */ +static inline int print_name(int width, int *printed, char *delim, char *name, enum counter_type type, enum counter_format format) +{ + UNUSED(type); + char *sep = (*printed)++ ? delim : ""; + + if (format == FORMAT_RAW && width >= 64) + return sprintf(outp, "%s%-8s", sep, name); + else + return sprintf(outp, "%s%s", sep, name); +} + +static inline int print_hex_value(int width, int *printed, char *delim, unsigned long long value) +{ + char *sep = (*printed)++ ? delim : ""; + + if (width <= 32) + return sprintf(outp, "%s%08llx", sep, value); + else + return sprintf(outp, "%s%016llx", sep, value); +} + +static inline int print_decimal_value(int width, int *printed, char *delim, unsigned long long value) +{ + char *sep = (*printed)++ ? delim : ""; + + UNUSED(width); + + return sprintf(outp, "%s%lld", sep, value); +} + +static inline int print_float_value(int *printed, char *delim, double value) +{ + char *sep = (*printed)++ ? delim : ""; + + return sprintf(outp, "%s%0.2f", sep, value); } void print_header(char *delim) @@ -2407,8 +2921,12 @@ void print_header(char *delim) outp += sprintf(outp, "%sPackage", (printed++ ? delim : "")); if (DO_BIC(BIC_Die)) outp += sprintf(outp, "%sDie", (printed++ ? delim : "")); + if (DO_BIC(BIC_L3)) + outp += sprintf(outp, "%sL3", (printed++ ? delim : "")); if (DO_BIC(BIC_Node)) outp += sprintf(outp, "%sNode", (printed++ ? delim : "")); + if (DO_BIC(BIC_Module)) + outp += sprintf(outp, "%sModule", (printed++ ? delim : "")); if (DO_BIC(BIC_Core)) outp += sprintf(outp, "%sCore", (printed++ ? delim : "")); if (DO_BIC(BIC_CPU)) @@ -2445,50 +2963,34 @@ void print_header(char *delim) if (DO_BIC(BIC_SMI)) outp += sprintf(outp, "%sSMI", (printed++ ? delim : "")); - for (mp = sys.tp; mp; mp = mp->next) { + if (DO_BIC(BIC_LLC_MRPS)) + outp += sprintf(outp, "%sLLCMRPS", (printed++ ? delim : "")); - if (mp->format == FORMAT_RAW) { - if (mp->width == 64) - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), mp->name); - else - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), mp->name); - } else { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), mp->name); - else - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), mp->name); - } - } + if (DO_BIC(BIC_LLC_HIT)) + outp += sprintf(outp, "%sLLC%%hit", (printed++ ? delim : "")); - for (pp = sys.perf_tp; pp; pp = pp->next) { + if (DO_BIC(BIC_L2_MRPS)) + outp += sprintf(outp, "%sL2MRPS", (printed++ ? delim : "")); - if (pp->format == FORMAT_RAW) { - if (pp->width == 64) - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), pp->name); - } else { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), pp->name); - } - } + if (DO_BIC(BIC_L2_HIT)) + outp += sprintf(outp, "%sL2%%hit", (printed++ ? delim : "")); + + for (mp = sys.tp; mp; mp = mp->next) + outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); + + for (pp = sys.perf_tp; pp; pp = pp->next) + outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format); ppmt = sys.pmt_tp; while (ppmt) { switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), ppmt->name); - else - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), ppmt->name); - + outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; case PMT_TYPE_XTAL_TIME: case PMT_TYPE_TCORE_CLOCK: - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), ppmt->name); + outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; } @@ -2513,63 +3015,36 @@ void print_header(char *delim) if (DO_BIC(BIC_CORE_THROT_CNT)) outp += sprintf(outp, "%sCoreThr", (printed++ ? delim : "")); - if (platform->rapl_msrs && !rapl_joules) { + if (valid_rapl_msrs && !rapl_joules) { if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl) outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : "")); - } else if (platform->rapl_msrs && rapl_joules) { + } else if (valid_rapl_msrs && rapl_joules) { if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl) outp += sprintf(outp, "%sCor_J", (printed++ ? delim : "")); } - for (mp = sys.cp; mp; mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 64) - outp += sprintf(outp, "%s%18.18s", delim, mp->name); - else - outp += sprintf(outp, "%s%10.10s", delim, mp->name); - } else { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", delim, mp->name); - else - outp += sprintf(outp, "%s%s", delim, mp->name); - } - } + for (mp = sys.cp; mp; mp = mp->next) + outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); - for (pp = sys.perf_cp; pp; pp = pp->next) { - - if (pp->format == FORMAT_RAW) { - if (pp->width == 64) - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), pp->name); - } else { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), pp->name); - } - } + for (pp = sys.perf_cp; pp; pp = pp->next) + outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format); ppmt = sys.pmt_cp; while (ppmt) { switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), ppmt->name); - else - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), ppmt->name); + outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; case PMT_TYPE_XTAL_TIME: case PMT_TYPE_TCORE_CLOCK: - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), ppmt->name); + outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; } ppmt = ppmt->next; } - if (DO_BIC(BIC_PkgTmp)) outp += sprintf(outp, "%sPkgTmp", (printed++ ? delim : "")); @@ -2621,7 +3096,7 @@ void print_header(char *delim) if (DO_BIC(BIC_SYS_LPI)) outp += sprintf(outp, "%sSYS%%LPI", (printed++ ? delim : "")); - if (platform->rapl_msrs && !rapl_joules) { + if (!rapl_joules) { if (DO_BIC(BIC_PkgWatt)) outp += sprintf(outp, "%sPkgWatt", (printed++ ? delim : "")); if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl) @@ -2634,7 +3109,7 @@ void print_header(char *delim) outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : "")); if (DO_BIC(BIC_RAM__)) outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : "")); - } else if (platform->rapl_msrs && rapl_joules) { + } else { if (DO_BIC(BIC_Pkg_J)) outp += sprintf(outp, "%sPkg_J", (printed++ ? delim : "")); if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl) @@ -2651,51 +3126,22 @@ void print_header(char *delim) if (DO_BIC(BIC_UNCORE_MHZ)) outp += sprintf(outp, "%sUncMHz", (printed++ ? delim : "")); - for (mp = sys.pp; mp; mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 64) - outp += sprintf(outp, "%s%18.18s", delim, mp->name); - else if (mp->width == 32) - outp += sprintf(outp, "%s%10.10s", delim, mp->name); - else - outp += sprintf(outp, "%s%7.7s", delim, mp->name); - } else { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", delim, mp->name); - else - outp += sprintf(outp, "%s%7.7s", delim, mp->name); - } - } - - for (pp = sys.perf_pp; pp; pp = pp->next) { + for (mp = sys.pp; mp; mp = mp->next) + outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); - if (pp->format == FORMAT_RAW) { - if (pp->width == 64) - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), pp->name); - } else { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), pp->name); - } - } + for (pp = sys.perf_pp; pp; pp = pp->next) + outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format); ppmt = sys.pmt_pp; while (ppmt) { switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), ppmt->name); - else - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), ppmt->name); - + outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; case PMT_TYPE_XTAL_TIME: case PMT_TYPE_TCORE_CLOCK: - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), ppmt->name); + outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; } @@ -2710,11 +3156,38 @@ void print_header(char *delim) outp += sprintf(outp, "\n"); } -int dump_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +/* + * pct(numerator, denominator) + * + * Return sanity checked percentage (100.0 * numerator/denominotor) + * + * n < 0: nan + * d <= 0: nan + * n/d > 1.1: nan + */ +double pct(double numerator, double denominator) +{ + double retval; + + if (numerator < 0) + return nan(""); + + if (denominator <= 0) + return nan(""); + + retval = 100.0 * numerator / denominator; + + if (retval > 110.0) + return nan(""); + + return retval; +} + +int dump_counters(PER_THREAD_PARAMS) { int i; struct msr_counter *mp; - struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even; + struct platform_counters *pplat_cnt = p == odd.packages ? &platform_counters_odd : &platform_counters_even; outp += sprintf(outp, "t %p, c %p, p %p\n", t, c, p); @@ -2735,15 +3208,21 @@ int dump_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p if (DO_BIC(BIC_SMI)) outp += sprintf(outp, "SMI: %d\n", t->smi_count); + outp += sprintf(outp, "LLC refs: %lld", t->llc.references); + outp += sprintf(outp, "LLC miss: %lld", t->llc.misses); + outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses), t->llc.references)); + + outp += sprintf(outp, "L2 refs: %lld", t->l2.references); + outp += sprintf(outp, "L2 hits: %lld", t->l2.hits); + outp += sprintf(outp, "L2 Hit%%: %.2f", pct(t->l2.hits, t->l2.references)); + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - outp += - sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, - t->counter[i], mp->sp->path); + outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path); } } - if (c && is_cpu_first_thread_in_core(t, c, p)) { - outp += sprintf(outp, "core: %d\n", c->core_id); + if (c && is_cpu_first_thread_in_core(t, c)) { + outp += sprintf(outp, "core: 0x%x\n", cpus[t->cpu_id].core_id); outp += sprintf(outp, "c3: %016llX\n", c->c3); outp += sprintf(outp, "c6: %016llX\n", c->c6); outp += sprintf(outp, "c7: %016llX\n", c->c7); @@ -2757,16 +3236,12 @@ int dump_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p outp += sprintf(outp, "Joules: %0llX (scale: %lf)\n", energy_value, energy_scale); for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - outp += - sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, - c->counter[i], mp->sp->path); + outp += sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, c->counter[i], mp->sp->path); } outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us); } - if (p && is_cpu_first_core_in_package(t, c, p)) { - outp += sprintf(outp, "package: %d\n", p->package_id); - + if (p && is_cpu_first_core_in_package(t, p)) { outp += sprintf(outp, "Weighted cores: %016llX\n", p->pkg_wtd_core_c0); outp += sprintf(outp, "Any cores: %016llX\n", p->pkg_any_core_c0); outp += sprintf(outp, "Any GFX: %016llX\n", p->pkg_any_gfxe_c0); @@ -2794,9 +3269,7 @@ int dump_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p outp += sprintf(outp, "PTM: %dC\n", p->pkg_temp_c); for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - outp += - sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, - p->counter[i], mp->sp->path); + outp += sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, p->counter[i], mp->sp->path); } } @@ -2822,16 +3295,57 @@ double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desir return scaled; } +void get_perf_llc_stats(int cpu, struct llc_stats *llc) +{ + struct read_format { + unsigned long long num_read; + struct llc_stats llc; + } r; + const ssize_t expected_read_size = sizeof(r); + ssize_t actual_read_size; + + actual_read_size = read(fd_llc_percpu[cpu], &r, expected_read_size); + + if (actual_read_size == -1) + err(-1, "%s(cpu%d,) %d,,%ld", __func__, cpu, fd_llc_percpu[cpu], expected_read_size); + + llc->references = r.llc.references; + llc->misses = r.llc.misses; + if (actual_read_size != expected_read_size) + warn("%s: failed to read perf_data (req %zu act %zu)", __func__, expected_read_size, actual_read_size); +} + +void get_perf_l2_stats(int cpu, struct l2_stats *l2) +{ + struct read_format { + unsigned long long num_read; + struct l2_stats l2; + } r; + const ssize_t expected_read_size = sizeof(r); + ssize_t actual_read_size; + + actual_read_size = read(fd_l2_percpu[cpu], &r, expected_read_size); + + if (actual_read_size == -1) + err(-1, "%s(cpu%d,) %d,,%ld", __func__, cpu, fd_l2_percpu[cpu], expected_read_size); + + l2->references = r.l2.references; + l2->hits = r.l2.hits; + if (actual_read_size != expected_read_size) + warn("%s: cpu%d: failed to read(%d) perf_data (req %zu act %zu)", __func__, cpu, fd_l2_percpu[cpu], expected_read_size, actual_read_size); +} + /* * column formatting convention & formats */ -int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int format_counters(PER_THREAD_PARAMS) { static int count; struct platform_counters *pplat_cnt = NULL; double interval_float, tsc; - char *fmt8; + char *fmt8 = "%s%.2f"; + int i; struct msr_counter *mp; struct perf_counter_info *pp; @@ -2839,21 +3353,21 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data char *delim = "\t"; int printed = 0; - if (t == &average.threads) { + if (t == average.threads) { pplat_cnt = count & 1 ? &platform_counters_odd : &platform_counters_even; ++count; } /* if showing only 1st thread in core and this isn't one, bail out */ - if (show_core_only && !is_cpu_first_thread_in_core(t, c, p)) + if (show_core_only && !is_cpu_first_thread_in_core(t, c)) return 0; /* if showing only 1st thread in pkg and this isn't one, bail out */ - if (show_pkg_only && !is_cpu_first_core_in_package(t, c, p)) + if (show_pkg_only && !is_cpu_first_core_in_package(t, p)) return 0; /*if not summary line and --cpu is used */ - if ((t != &average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset))) + if ((t != average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset))) return 0; if (DO_BIC(BIC_USEC)) { @@ -2873,13 +3387,17 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data tsc = t->tsc * tsc_tweak; /* topo columns, print blanks on 1st (average) line */ - if (t == &average.threads) { + if (t == average.threads) { if (DO_BIC(BIC_Package)) outp += sprintf(outp, "%s-", (printed++ ? delim : "")); if (DO_BIC(BIC_Die)) outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + if (DO_BIC(BIC_L3)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); if (DO_BIC(BIC_Node)) outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + if (DO_BIC(BIC_Module)) + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); if (DO_BIC(BIC_Core)) outp += sprintf(outp, "%s-", (printed++ ? delim : "")); if (DO_BIC(BIC_CPU)) @@ -2891,7 +3409,7 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data } else { if (DO_BIC(BIC_Package)) { if (p) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->package_id); + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].package_id); else outp += sprintf(outp, "%s-", (printed++ ? delim : "")); } @@ -2901,40 +3419,49 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data else outp += sprintf(outp, "%s-", (printed++ ? delim : "")); } + if (DO_BIC(BIC_L3)) { + if (c) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].l3_id); + else + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } if (DO_BIC(BIC_Node)) { if (t) - outp += sprintf(outp, "%s%d", - (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id); + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id); + else + outp += sprintf(outp, "%s-", (printed++ ? delim : "")); + } + if (DO_BIC(BIC_Module)) { + if (c) + outp += sprintf(outp, "%s0x%x", (printed++ ? delim : ""), cpus[t->cpu_id].module_id); else outp += sprintf(outp, "%s-", (printed++ ? delim : "")); } if (DO_BIC(BIC_Core)) { if (c) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_id); + outp += sprintf(outp, "%s0x%x", (printed++ ? delim : ""), cpus[t->cpu_id].core_id); else outp += sprintf(outp, "%s-", (printed++ ? delim : "")); } if (DO_BIC(BIC_CPU)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->cpu_id); if (DO_BIC(BIC_APIC)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->apic_id); + outp += sprintf(outp, "%s0x%x", (printed++ ? delim : ""), t->apic_id); if (DO_BIC(BIC_X2APIC)) - outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->x2apic_id); + outp += sprintf(outp, "%s0x%x", (printed++ ? delim : ""), t->x2apic_id); } if (DO_BIC(BIC_Avg_MHz)) outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float); if (DO_BIC(BIC_Busy)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->mperf / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf, tsc)); if (DO_BIC(BIC_Bzy_MHz)) { if (has_base_hz) - outp += - sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf); + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf); else - outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), - tsc / units * t->aperf / t->mperf / interval_float); + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), tsc / units * t->aperf / t->mperf / interval_float); } if (DO_BIC(BIC_TSC_MHz)) @@ -2963,96 +3490,86 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data if (DO_BIC(BIC_SMI)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count); - /* Added counters */ + /* LLC Stats */ + if (DO_BIC(BIC_LLC_MRPS)) + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000000); + + if (DO_BIC(BIC_LLC_HIT)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses), t->llc.references)); + + /* L2 Stats */ + if (DO_BIC(BIC_L2_MRPS)) + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->l2.references / interval_float / 1000000); + + if (DO_BIC(BIC_L2_HIT)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct(t->l2.hits, t->l2.references)); + + /* Added Thread Counters */ for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)t->counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->counter[i]); - } else if (mp->format == FORMAT_DELTA) { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->counter[i]); - } else if (mp->format == FORMAT_PERCENT) { + if (mp->format == FORMAT_RAW) + outp += print_hex_value(mp->width, &printed, delim, t->counter[i]); + else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(mp->width, &printed, delim, t->counter[i]); + else if (mp->format == FORMAT_PERCENT) { if (mp->type == COUNTER_USEC) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - t->counter[i] / interval_float / 10000); + outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000); else - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->counter[i] / tsc); + outp += print_float_value(&printed, delim, pct(t->counter[i], tsc)); } } - /* Added perf counters */ + /* Added perf Thread Counters */ for (i = 0, pp = sys.perf_tp; pp; ++i, pp = pp->next) { - if (pp->format == FORMAT_RAW) { - if (pp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)t->perf_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->perf_counter[i]); - } else if (pp->format == FORMAT_DELTA) { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->perf_counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->perf_counter[i]); - } else if (pp->format == FORMAT_PERCENT) { + if (pp->format == FORMAT_RAW) + outp += print_hex_value(pp->width, &printed, delim, t->perf_counter[i]); + else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE) + outp += print_decimal_value(pp->width, &printed, delim, t->perf_counter[i]); + else if (pp->format == FORMAT_PERCENT) { if (pp->type == COUNTER_USEC) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - t->perf_counter[i] / interval_float / 10000); + outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000); else - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->perf_counter[i] / tsc); + outp += print_float_value(&printed, delim, pct(t->perf_counter[i], tsc)); } } + /* Added PMT Thread Counters */ for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { const unsigned long value_raw = t->pmt_counter[i]; double value_converted; switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)t->pmt_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->pmt_counter[i]); - + outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, t->pmt_counter[i]); break; case PMT_TYPE_XTAL_TIME: - value_converted = 100.0 * value_raw / crystal_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / crystal_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = 100.0 * value_raw / tcore_clock_freq_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); } } /* C1 */ if (DO_BIC(BIC_CPU_c1)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->c1 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1, tsc)); /* print per-core data only for 1st thread in core */ - if (!is_cpu_first_thread_in_core(t, c, p)) + if (!is_cpu_first_thread_in_core(t, c)) goto done; if (DO_BIC(BIC_CPU_c3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c3 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3, tsc)); if (DO_BIC(BIC_CPU_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c6 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6, tsc)); if (DO_BIC(BIC_CPU_c7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c7 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7, tsc)); /* Mod%c6 */ if (DO_BIC(BIC_Mod_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->mc6_us / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us, tsc)); if (DO_BIC(BIC_CoreTmp)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c); @@ -3061,77 +3578,53 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data if (DO_BIC(BIC_CORE_THROT_CNT)) outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt); + /* Added Core Counters */ for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)c->counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->counter[i]); - } else if (mp->format == FORMAT_DELTA) { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), c->counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->counter[i]); - } else if (mp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->counter[i] / tsc); - } + if (mp->format == FORMAT_RAW) + outp += print_hex_value(mp->width, &printed, delim, c->counter[i]); + else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]); + else if (mp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(c->counter[i], tsc)); } + /* Added perf Core counters */ for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { - if (pp->format == FORMAT_RAW) { - if (pp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)c->perf_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->perf_counter[i]); - } else if (pp->format == FORMAT_DELTA) { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), c->perf_counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->perf_counter[i]); - } else if (pp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->perf_counter[i] / tsc); - } + if (pp->format == FORMAT_RAW) + outp += print_hex_value(pp->width, &printed, delim, c->perf_counter[i]); + else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE) + outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]); + else if (pp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(c->perf_counter[i], tsc)); } + /* Added PMT Core counters */ for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { const unsigned long value_raw = c->pmt_counter[i]; double value_converted; switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)c->pmt_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->pmt_counter[i]); - + outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, c->pmt_counter[i]); break; case PMT_TYPE_XTAL_TIME: - value_converted = 100.0 * value_raw / crystal_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / crystal_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = 100.0 * value_raw / tcore_clock_freq_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); } } - fmt8 = "%s%.2f"; - if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&c->core_energy, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&c->core_energy, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_JOULES, interval_float)); /* print per-package data only for 1st core in package */ - if (!is_cpu_first_core_in_package(t, c, p)) + if (!is_cpu_first_core_in_package(t, p)) goto done; /* PkgTmp */ @@ -3143,8 +3636,7 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */ outp += sprintf(outp, "%s**.**", (printed++ ? delim : "")); } else { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - p->gfx_rc6_ms / 10.0 / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->gfx_rc6_ms / 10.0 / interval_float); } } @@ -3161,8 +3653,7 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data if (p->sam_mc6_ms == -1) { /* detect GFX counter reset */ outp += sprintf(outp, "%s**.**", (printed++ ? delim : "")); } else { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - p->sam_mc6_ms / 10.0 / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->sam_mc6_ms / 10.0 / interval_float); } } @@ -3176,159 +3667,117 @@ int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data /* Totl%C0, Any%C0 GFX%C0 CPUGFX% */ if (DO_BIC(BIC_Totl_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_wtd_core_c0 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */ if (DO_BIC(BIC_Any_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_core_c0 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0, tsc)); if (DO_BIC(BIC_GFX_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_gfxe_c0 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0, tsc)); if (DO_BIC(BIC_CPUGFX)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_both_core_gfxe_c0 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0, tsc)); if (DO_BIC(BIC_Pkgpc2)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc2 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2, tsc)); if (DO_BIC(BIC_Pkgpc3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc3 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3, tsc)); if (DO_BIC(BIC_Pkgpc6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc6 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6, tsc)); if (DO_BIC(BIC_Pkgpc7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc7 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7, tsc)); if (DO_BIC(BIC_Pkgpc8)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc8 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8, tsc)); if (DO_BIC(BIC_Pkgpc9)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc9 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9, tsc)); if (DO_BIC(BIC_Pkgpc10)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc10 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10, tsc)); if (DO_BIC(BIC_Diec6)) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->die_c6 / crystal_hz / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz, interval_float)); if (DO_BIC(BIC_CPU_LPI)) { if (p->cpu_lpi >= 0) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - 100.0 * p->cpu_lpi / 1000000.0 / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0, interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } if (DO_BIC(BIC_SYS_LPI)) { if (p->sys_lpi >= 0) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - 100.0 * p->sys_lpi / 1000000.0 / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0, interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } if (DO_BIC(BIC_PkgWatt)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_GFXWatt)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_RAMWatt)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_Pkg_J)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_JOULES, interval_float)); if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_JOULES, interval_float)); if (DO_BIC(BIC_GFX_J)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_JOULES, interval_float)); if (DO_BIC(BIC_RAM_J)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float)); if (DO_BIC(BIC_PKG__)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_RAM__)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float)); /* UncMHz */ if (DO_BIC(BIC_UNCORE_MHZ)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz); + /* Added Package Counters */ for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) { - if (mp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)p->counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->counter[i]); - } else if (mp->format == FORMAT_DELTA) { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), p->counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), p->counter[i]); - } else if (mp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->counter[i] / tsc); - } else if (mp->type == COUNTER_K2M) + if (mp->format == FORMAT_RAW) + outp += print_hex_value(mp->width, &printed, delim, p->counter[i]); + else if (mp->type == COUNTER_K2M) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->counter[i] / 1000); + else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]); + else if (mp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(p->counter[i], tsc)); } + /* Added perf Package Counters */ for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { - if (pp->format == FORMAT_RAW) { - if (pp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)p->perf_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->perf_counter[i]); - } else if (pp->format == FORMAT_DELTA) { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), p->perf_counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), p->perf_counter[i]); - } else if (pp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->perf_counter[i] / tsc); - } else if (pp->type == COUNTER_K2M) { - outp += - sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000); - } + if (pp->format == FORMAT_RAW) + outp += print_hex_value(pp->width, &printed, delim, p->perf_counter[i]); + else if (pp->type == COUNTER_K2M) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000); + else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE) + outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]); + else if (pp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(p->perf_counter[i], tsc)); } + /* Added PMT Package Counters */ for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { const unsigned long value_raw = p->pmt_counter[i]; double value_converted; switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)p->pmt_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->pmt_counter[i]); - + outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, p->pmt_counter[i]); break; case PMT_TYPE_XTAL_TIME: - value_converted = 100.0 * value_raw / crystal_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / crystal_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = 100.0 * value_raw / tcore_clock_freq_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float); + outp += print_float_value(&printed, delim, value_converted); } } - if (DO_BIC(BIC_SysWatt) && (t == &average.threads)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_WATTS, interval_float)); - if (DO_BIC(BIC_Sys_J) && (t == &average.threads)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_JOULES, interval_float)); + if (DO_BIC(BIC_SysWatt) && (t == average.threads)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_WATTS, interval_float)); + if (DO_BIC(BIC_Sys_J) && (t == average.threads)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_JOULES, interval_float)); done: if (*(outp - 1) != '\n') @@ -3359,14 +3808,14 @@ void flush_output_stderr(void) outp = output_buffer; } -void format_all_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +void format_all_counters(PER_THREAD_PARAMS) { static int count; if ((!count || (header_iterations && !(count % header_iterations))) || !summary_only) print_header("\t"); - format_counters(&average.threads, &average.cores, &average.packages); + format_counters(average.threads, average.cores, average.packages); count++; @@ -3434,8 +3883,7 @@ int delta_package(struct pkg_data *new, struct pkg_data *old) old->energy_gfx.raw_value = new->energy_gfx.raw_value - old->energy_gfx.raw_value; old->energy_dram.raw_value = new->energy_dram.raw_value - old->energy_dram.raw_value; old->rapl_pkg_perf_status.raw_value = new->rapl_pkg_perf_status.raw_value - old->rapl_pkg_perf_status.raw_value; - old->rapl_dram_perf_status.raw_value = - new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value; + old->rapl_dram_perf_status.raw_value = new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value; for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) @@ -3476,13 +3924,13 @@ void delta_core(struct core_data *new, struct core_data *old) old->c6 = new->c6 - old->c6; old->c7 = new->c7 - old->c7; old->core_temp_c = new->core_temp_c; - old->core_throt_cnt = new->core_throt_cnt; + old->core_throt_cnt = new->core_throt_cnt - old->core_throt_cnt; old->mc6_us = new->mc6_us - old->mc6_us; DELTA_WRAP32(new->core_energy.raw_value, old->core_energy.raw_value); for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) + if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) old->counter[i] = new->counter[i]; else old->counter[i] = new->counter[i] - old->counter[i]; @@ -3542,8 +3990,7 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d /* check for TSC < 1 Mcycles over interval */ if (old->tsc < (1000 * 1000)) errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n" - "You can disable all c-states by booting with \"idle=poll\"\n" - "or just the deep ones with \"processor.max_cstate=1\""); + "You can disable all c-states by booting with \"idle=poll\"\nor just the deep ones with \"processor.max_cstate=1\""); old->c1 = new->c1 - old->c1; @@ -3572,8 +4019,7 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d old->c1 = 0; else { /* normal case, derive c1 */ - old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - - core_delta->c6 - core_delta->c7; + old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - core_delta->c6 - core_delta->c7; } } @@ -3595,8 +4041,20 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d if (DO_BIC(BIC_SMI)) old->smi_count = new->smi_count - old->smi_count; + if (DO_BIC(BIC_LLC_MRPS) || DO_BIC(BIC_LLC_HIT)) + old->llc.references = new->llc.references - old->llc.references; + + if (DO_BIC(BIC_LLC_HIT)) + old->llc.misses = new->llc.misses - old->llc.misses; + + if (DO_BIC(BIC_L2_MRPS) || DO_BIC(BIC_L2_HIT)) + old->l2.references = new->l2.references - old->l2.references; + + if (DO_BIC(BIC_L2_HIT)) + old->l2.hits = new->l2.hits - old->l2.hits; + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW) + if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) old->counter[i] = new->counter[i]; else old->counter[i] = new->counter[i] - old->counter[i]; @@ -3619,20 +4077,19 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d return 0; } -int delta_cpu(struct thread_data *t, struct core_data *c, - struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2) +int delta_cpu(struct thread_data *t, struct core_data *c, struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2) { int retval = 0; /* calculate core delta only for 1st thread in core */ - if (is_cpu_first_thread_in_core(t, c, p)) + if (is_cpu_first_thread_in_core(t, c)) delta_core(c, c2); /* always calculate thread delta */ retval = delta_thread(t, t2, c2); /* c2 is core delta */ /* calculate package delta only for 1st core in package */ - if (is_cpu_first_core_in_package(t, c, p)) + if (is_cpu_first_core_in_package(t, p)) retval |= delta_package(p, p2); return retval; @@ -3650,7 +4107,7 @@ void rapl_counter_clear(struct rapl_counter *c) c->unit = RAPL_UNIT_INVALID; } -void clear_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +void clear_counters(PER_THREAD_PARAMS) { int i; struct msr_counter *mp; @@ -3673,6 +4130,12 @@ void clear_counters(struct thread_data *t, struct core_data *c, struct pkg_data t->nmi_count = 0; t->smi_count = 0; + t->llc.references = 0; + t->llc.misses = 0; + + t->l2.references = 0; + t->l2.hits = 0; + c->c3 = 0; c->c6 = 0; c->c7 = 0; @@ -3747,7 +4210,7 @@ void rapl_counter_accumulate(struct rapl_counter *dst, const struct rapl_counter dst->raw_value += src->raw_value; } -int sum_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int sum_counters(PER_THREAD_PARAMS) { int i; struct msr_counter *mp; @@ -3756,136 +4219,142 @@ int sum_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) /* copy un-changing apic_id's */ if (DO_BIC(BIC_APIC)) - average.threads.apic_id = t->apic_id; + average.threads->apic_id = t->apic_id; if (DO_BIC(BIC_X2APIC)) - average.threads.x2apic_id = t->x2apic_id; + average.threads->x2apic_id = t->x2apic_id; /* remember first tv_begin */ - if (average.threads.tv_begin.tv_sec == 0) - average.threads.tv_begin = procsysfs_tv_begin; + if (average.threads->tv_begin.tv_sec == 0) + average.threads->tv_begin = procsysfs_tv_begin; /* remember last tv_end */ - average.threads.tv_end = t->tv_end; + average.threads->tv_end = t->tv_end; - average.threads.tsc += t->tsc; - average.threads.aperf += t->aperf; - average.threads.mperf += t->mperf; - average.threads.c1 += t->c1; + average.threads->tsc += t->tsc; + average.threads->aperf += t->aperf; + average.threads->mperf += t->mperf; + average.threads->c1 += t->c1; - average.threads.instr_count += t->instr_count; + average.threads->instr_count += t->instr_count; - average.threads.irq_count += t->irq_count; - average.threads.nmi_count += t->nmi_count; - average.threads.smi_count += t->smi_count; + average.threads->irq_count += t->irq_count; + average.threads->nmi_count += t->nmi_count; + average.threads->smi_count += t->smi_count; + + average.threads->llc.references += t->llc.references; + average.threads->llc.misses += t->llc.misses; + + average.threads->l2.references += t->l2.references; + average.threads->l2.hits += t->l2.hits; for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; - average.threads.counter[i] += t->counter[i]; + average.threads->counter[i] += t->counter[i]; } for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; - average.threads.perf_counter[i] += t->perf_counter[i]; + average.threads->perf_counter[i] += t->perf_counter[i]; } for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { - average.threads.pmt_counter[i] += t->pmt_counter[i]; + average.threads->pmt_counter[i] += t->pmt_counter[i]; } /* sum per-core values only for 1st thread in core */ - if (!is_cpu_first_thread_in_core(t, c, p)) + if (!is_cpu_first_thread_in_core(t, c)) return 0; - average.cores.c3 += c->c3; - average.cores.c6 += c->c6; - average.cores.c7 += c->c7; - average.cores.mc6_us += c->mc6_us; + average.cores->c3 += c->c3; + average.cores->c6 += c->c6; + average.cores->c7 += c->c7; + average.cores->mc6_us += c->mc6_us; - average.cores.core_temp_c = MAX(average.cores.core_temp_c, c->core_temp_c); - average.cores.core_throt_cnt = MAX(average.cores.core_throt_cnt, c->core_throt_cnt); + average.cores->core_temp_c = MAX(average.cores->core_temp_c, c->core_temp_c); + average.cores->core_throt_cnt = MAX(average.cores->core_throt_cnt, c->core_throt_cnt); - rapl_counter_accumulate(&average.cores.core_energy, &c->core_energy); + rapl_counter_accumulate(&average.cores->core_energy, &c->core_energy); for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; - average.cores.counter[i] += c->counter[i]; + average.cores->counter[i] += c->counter[i]; } for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; - average.cores.perf_counter[i] += c->perf_counter[i]; + average.cores->perf_counter[i] += c->perf_counter[i]; } for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { - average.cores.pmt_counter[i] += c->pmt_counter[i]; + average.cores->pmt_counter[i] += c->pmt_counter[i]; } /* sum per-pkg values only for 1st core in pkg */ - if (!is_cpu_first_core_in_package(t, c, p)) + if (!is_cpu_first_core_in_package(t, p)) return 0; if (DO_BIC(BIC_Totl_c0)) - average.packages.pkg_wtd_core_c0 += p->pkg_wtd_core_c0; + average.packages->pkg_wtd_core_c0 += p->pkg_wtd_core_c0; if (DO_BIC(BIC_Any_c0)) - average.packages.pkg_any_core_c0 += p->pkg_any_core_c0; + average.packages->pkg_any_core_c0 += p->pkg_any_core_c0; if (DO_BIC(BIC_GFX_c0)) - average.packages.pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0; + average.packages->pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0; if (DO_BIC(BIC_CPUGFX)) - average.packages.pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0; + average.packages->pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0; - average.packages.pc2 += p->pc2; + average.packages->pc2 += p->pc2; if (DO_BIC(BIC_Pkgpc3)) - average.packages.pc3 += p->pc3; + average.packages->pc3 += p->pc3; if (DO_BIC(BIC_Pkgpc6)) - average.packages.pc6 += p->pc6; + average.packages->pc6 += p->pc6; if (DO_BIC(BIC_Pkgpc7)) - average.packages.pc7 += p->pc7; - average.packages.pc8 += p->pc8; - average.packages.pc9 += p->pc9; - average.packages.pc10 += p->pc10; - average.packages.die_c6 += p->die_c6; + average.packages->pc7 += p->pc7; + average.packages->pc8 += p->pc8; + average.packages->pc9 += p->pc9; + average.packages->pc10 += p->pc10; + average.packages->die_c6 += p->die_c6; - average.packages.cpu_lpi = p->cpu_lpi; - average.packages.sys_lpi = p->sys_lpi; + average.packages->cpu_lpi = p->cpu_lpi; + average.packages->sys_lpi = p->sys_lpi; - rapl_counter_accumulate(&average.packages.energy_pkg, &p->energy_pkg); - rapl_counter_accumulate(&average.packages.energy_dram, &p->energy_dram); - rapl_counter_accumulate(&average.packages.energy_cores, &p->energy_cores); - rapl_counter_accumulate(&average.packages.energy_gfx, &p->energy_gfx); + rapl_counter_accumulate(&average.packages->energy_pkg, &p->energy_pkg); + rapl_counter_accumulate(&average.packages->energy_dram, &p->energy_dram); + rapl_counter_accumulate(&average.packages->energy_cores, &p->energy_cores); + rapl_counter_accumulate(&average.packages->energy_gfx, &p->energy_gfx); - average.packages.gfx_rc6_ms = p->gfx_rc6_ms; - average.packages.uncore_mhz = p->uncore_mhz; - average.packages.gfx_mhz = p->gfx_mhz; - average.packages.gfx_act_mhz = p->gfx_act_mhz; - average.packages.sam_mc6_ms = p->sam_mc6_ms; - average.packages.sam_mhz = p->sam_mhz; - average.packages.sam_act_mhz = p->sam_act_mhz; + average.packages->gfx_rc6_ms = p->gfx_rc6_ms; + average.packages->uncore_mhz = p->uncore_mhz; + average.packages->gfx_mhz = p->gfx_mhz; + average.packages->gfx_act_mhz = p->gfx_act_mhz; + average.packages->sam_mc6_ms = p->sam_mc6_ms; + average.packages->sam_mhz = p->sam_mhz; + average.packages->sam_act_mhz = p->sam_act_mhz; - average.packages.pkg_temp_c = MAX(average.packages.pkg_temp_c, p->pkg_temp_c); + average.packages->pkg_temp_c = MAX(average.packages->pkg_temp_c, p->pkg_temp_c); - rapl_counter_accumulate(&average.packages.rapl_pkg_perf_status, &p->rapl_pkg_perf_status); - rapl_counter_accumulate(&average.packages.rapl_dram_perf_status, &p->rapl_dram_perf_status); + rapl_counter_accumulate(&average.packages->rapl_pkg_perf_status, &p->rapl_pkg_perf_status); + rapl_counter_accumulate(&average.packages->rapl_dram_perf_status, &p->rapl_dram_perf_status); for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { if ((mp->format == FORMAT_RAW) && (topo.num_packages == 0)) - average.packages.counter[i] = p->counter[i]; + average.packages->counter[i] = p->counter[i]; else - average.packages.counter[i] += p->counter[i]; + average.packages->counter[i] += p->counter[i]; } for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { if ((pp->format == FORMAT_RAW) && (topo.num_packages == 0)) - average.packages.perf_counter[i] = p->perf_counter[i]; + average.packages->perf_counter[i] = p->perf_counter[i]; else - average.packages.perf_counter[i] += p->perf_counter[i]; + average.packages->perf_counter[i] += p->perf_counter[i]; } for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { - average.packages.pmt_counter[i] += p->pmt_counter[i]; + average.packages->pmt_counter[i] += p->pmt_counter[i]; } return 0; @@ -3895,125 +4364,124 @@ int sum_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) * sum the counters for all cpus in the system * compute the weighted average */ -void compute_average(struct thread_data *t, struct core_data *c, struct pkg_data *p) +void compute_average(PER_THREAD_PARAMS) { int i; struct msr_counter *mp; struct perf_counter_info *pp; struct pmt_counter *ppmt; - clear_counters(&average.threads, &average.cores, &average.packages); + clear_counters(average.threads, average.cores, average.packages); for_all_cpus(sum_counters, t, c, p); /* Use the global time delta for the average. */ - average.threads.tv_delta = tv_delta; + average.threads->tv_delta = tv_delta; - average.threads.tsc /= topo.allowed_cpus; - average.threads.aperf /= topo.allowed_cpus; - average.threads.mperf /= topo.allowed_cpus; - average.threads.instr_count /= topo.allowed_cpus; - average.threads.c1 /= topo.allowed_cpus; + average.threads->tsc /= topo.allowed_cpus; + average.threads->aperf /= topo.allowed_cpus; + average.threads->mperf /= topo.allowed_cpus; + average.threads->instr_count /= topo.allowed_cpus; + average.threads->c1 /= topo.allowed_cpus; - if (average.threads.irq_count > 9999999) + if (average.threads->irq_count > 9999999) sums_need_wide_columns = 1; - if (average.threads.nmi_count > 9999999) + if (average.threads->nmi_count > 9999999) sums_need_wide_columns = 1; - - average.cores.c3 /= topo.allowed_cores; - average.cores.c6 /= topo.allowed_cores; - average.cores.c7 /= topo.allowed_cores; - average.cores.mc6_us /= topo.allowed_cores; + average.cores->c3 /= topo.allowed_cores; + average.cores->c6 /= topo.allowed_cores; + average.cores->c7 /= topo.allowed_cores; + average.cores->mc6_us /= topo.allowed_cores; if (DO_BIC(BIC_Totl_c0)) - average.packages.pkg_wtd_core_c0 /= topo.allowed_packages; + average.packages->pkg_wtd_core_c0 /= topo.allowed_packages; if (DO_BIC(BIC_Any_c0)) - average.packages.pkg_any_core_c0 /= topo.allowed_packages; + average.packages->pkg_any_core_c0 /= topo.allowed_packages; if (DO_BIC(BIC_GFX_c0)) - average.packages.pkg_any_gfxe_c0 /= topo.allowed_packages; + average.packages->pkg_any_gfxe_c0 /= topo.allowed_packages; if (DO_BIC(BIC_CPUGFX)) - average.packages.pkg_both_core_gfxe_c0 /= topo.allowed_packages; + average.packages->pkg_both_core_gfxe_c0 /= topo.allowed_packages; - average.packages.pc2 /= topo.allowed_packages; + average.packages->pc2 /= topo.allowed_packages; if (DO_BIC(BIC_Pkgpc3)) - average.packages.pc3 /= topo.allowed_packages; + average.packages->pc3 /= topo.allowed_packages; if (DO_BIC(BIC_Pkgpc6)) - average.packages.pc6 /= topo.allowed_packages; + average.packages->pc6 /= topo.allowed_packages; if (DO_BIC(BIC_Pkgpc7)) - average.packages.pc7 /= topo.allowed_packages; + average.packages->pc7 /= topo.allowed_packages; - average.packages.pc8 /= topo.allowed_packages; - average.packages.pc9 /= topo.allowed_packages; - average.packages.pc10 /= topo.allowed_packages; - average.packages.die_c6 /= topo.allowed_packages; + average.packages->pc8 /= topo.allowed_packages; + average.packages->pc9 /= topo.allowed_packages; + average.packages->pc10 /= topo.allowed_packages; + average.packages->die_c6 /= topo.allowed_packages; for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; if (mp->type == COUNTER_ITEMS) { - if (average.threads.counter[i] > 9999999) + if (average.threads->counter[i] > 9999999) sums_need_wide_columns = 1; continue; } - average.threads.counter[i] /= topo.allowed_cpus; + average.threads->counter[i] /= topo.allowed_cpus; } for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; if (mp->type == COUNTER_ITEMS) { - if (average.cores.counter[i] > 9999999) + if (average.cores->counter[i] > 9999999) sums_need_wide_columns = 1; } - average.cores.counter[i] /= topo.allowed_cores; + average.cores->counter[i] /= topo.allowed_cores; } for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; if (mp->type == COUNTER_ITEMS) { - if (average.packages.counter[i] > 9999999) + if (average.packages->counter[i] > 9999999) sums_need_wide_columns = 1; } - average.packages.counter[i] /= topo.allowed_packages; + average.packages->counter[i] /= topo.allowed_packages; } for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; if (pp->type == COUNTER_ITEMS) { - if (average.threads.perf_counter[i] > 9999999) + if (average.threads->perf_counter[i] > 9999999) sums_need_wide_columns = 1; continue; } - average.threads.perf_counter[i] /= topo.allowed_cpus; + average.threads->perf_counter[i] /= topo.allowed_cpus; } for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; if (pp->type == COUNTER_ITEMS) { - if (average.cores.perf_counter[i] > 9999999) + if (average.cores->perf_counter[i] > 9999999) sums_need_wide_columns = 1; } - average.cores.perf_counter[i] /= topo.allowed_cores; + average.cores->perf_counter[i] /= topo.allowed_cores; } for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { if (pp->format == FORMAT_RAW) continue; if (pp->type == COUNTER_ITEMS) { - if (average.packages.perf_counter[i] > 9999999) + if (average.packages->perf_counter[i] > 9999999) sums_need_wide_columns = 1; } - average.packages.perf_counter[i] /= topo.allowed_packages; + average.packages->perf_counter[i] /= topo.allowed_packages; } for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { - average.threads.pmt_counter[i] /= topo.allowed_cpus; + average.threads->pmt_counter[i] /= topo.allowed_cpus; } for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { - average.cores.pmt_counter[i] /= topo.allowed_cores; + average.cores->pmt_counter[i] /= topo.allowed_cores; } for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { - average.packages.pmt_counter[i] /= topo.allowed_packages; + average.packages->pmt_counter[i] /= topo.allowed_packages; } } @@ -4092,8 +4560,7 @@ unsigned long long get_legacy_uncore_mhz(int package) */ for (die = 0; die <= topo.max_die_id; ++die) { - sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", - package, die); + sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", package, die); if (access(path, R_OK) == 0) return (snapshot_sysfs_counter(path) / 1000); @@ -4204,11 +4671,6 @@ int get_core_throt_cnt(int cpu, unsigned long long *cnt) return 0; } -struct amperf_group_fd { - int aperf; /* Also the group descriptor */ - int mperf; -}; - static int read_perf_counter_info(const char *const path, const char *const parse_format, void *value_ptr) { int fdmt; @@ -4387,7 +4849,7 @@ void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct pkg_data *p) { - struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even; + struct platform_counters *pplat_cnt = p == odd.packages ? &platform_counters_odd : &platform_counters_even; unsigned long long perf_data[NUM_RAPL_COUNTERS + 1]; struct rapl_counter_info_t *rci; @@ -4408,8 +4870,7 @@ int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct const ssize_t actual_read_size = read(rci->fd_perf, &perf_data[0], sizeof(perf_data)); if (actual_read_size != expected_read_size) - err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, - actual_read_size); + err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size); } for (unsigned int i = 0, pi = 1; i < NUM_RAPL_COUNTERS; ++i) { @@ -4479,7 +4940,7 @@ char *find_sysfs_path_by_id(struct sysfs_path *sp, int id) return NULL; } -int get_cstate_counters(unsigned int cpu, struct thread_data *t, struct core_data *c, struct pkg_data *p) +int get_cstate_counters(unsigned int cpu, PER_THREAD_PARAMS) { /* * Overcommit memory a little bit here, @@ -4647,8 +5108,7 @@ int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t) const ssize_t actual_read_size = read(mci->fd_perf, &perf_data[0], sizeof(perf_data)); if (actual_read_size != expected_read_size) - err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, - actual_read_size); + err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size); } for (unsigned int i = 0, pi = 1; i < NUM_MSR_COUNTERS; ++i) { @@ -4743,12 +5203,29 @@ unsigned long pmt_read_counter(struct pmt_counter *ppmt, unsigned int domain_id) return (value & value_mask) >> value_shift; } +/* Rapl domain enumeration helpers */ +static inline int get_rapl_num_domains(void) +{ + if (!platform->has_per_core_rapl) + return topo.num_packages; + + return GLOBAL_CORE_ID(topo.max_core_id, topo.num_packages) + 1; +} + +static inline int get_rapl_domain_id(int cpu) +{ + if (!platform->has_per_core_rapl) + return cpus[cpu].package_id; + + return GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id); +} + /* * get_counters(...) * migrate to cpu * acquire and record local counters for that cpu */ -int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int get_counters(PER_THREAD_PARAMS) { int cpu = t->cpu_id; unsigned long long msr; @@ -4771,6 +5248,12 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) get_smi_aperf_mperf(cpu, t); + if (DO_BIC(BIC_LLC_MRPS) || DO_BIC(BIC_LLC_HIT)) + get_perf_llc_stats(cpu, &t->llc); + + if (DO_BIC(BIC_L2_MRPS) || DO_BIC(BIC_L2_HIT)) + get_perf_l2_stats(cpu, &t->l2); + if (DO_BIC(BIC_IPC)) if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long)) return -4; @@ -4794,11 +5277,11 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) t->pmt_counter[i] = pmt_read_counter(pp, t->cpu_id); /* collect core counters only for 1st thread in core */ - if (!is_cpu_first_thread_in_core(t, c, p)) + if (!is_cpu_first_thread_in_core(t, c)) goto done; if (platform->has_per_core_rapl) { - status = get_rapl_counters(cpu, c->core_id, c, p); + status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p); if (status != 0) return status; } @@ -4835,10 +5318,10 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) return -10; for (i = 0, pp = sys.pmt_cp; pp; i++, pp = pp->next) - c->pmt_counter[i] = pmt_read_counter(pp, c->core_id); + c->pmt_counter[i] = pmt_read_counter(pp, cpus[t->cpu_id].core_id); /* collect package counters only for 1st core in package */ - if (!is_cpu_first_core_in_package(t, c, p)) + if (!is_cpu_first_core_in_package(t, p)) goto done; if (DO_BIC(BIC_Totl_c0)) { @@ -4864,7 +5347,7 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) p->sys_lpi = cpuidle_cur_sys_lpi_us; if (!platform->has_per_core_rapl) { - status = get_rapl_counters(cpu, p->package_id, c, p); + status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p); if (status != 0) return status; } @@ -4876,7 +5359,7 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) } if (DO_BIC(BIC_UNCORE_MHZ)) - p->uncore_mhz = get_legacy_uncore_mhz(p->package_id); + p->uncore_mhz = get_legacy_uncore_mhz(cpus[t->cpu_id].package_id); if (DO_BIC(BIC_GFX_rc6)) p->gfx_rc6_ms = gfx_info[GFX_rc6].val_ull; @@ -4900,9 +5383,9 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) char *path = NULL; if (mp->msr_num == 0) { - path = find_sysfs_path_by_id(mp->sp, p->package_id); + path = find_sysfs_path_by_id(mp->sp, cpus[t->cpu_id].package_id); if (path == NULL) { - warnx("%s: package_id %d not found", __func__, p->package_id); + warnx("%s: package_id %d not found", __func__, cpus[t->cpu_id].package_id); return -10; } } @@ -4914,7 +5397,7 @@ int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p) return -10; for (i = 0, pp = sys.pmt_pp; pp; i++, pp = pp->next) - p->pmt_counter[i] = pmt_read_counter(pp, p->package_id); + p->pmt_counter[i] = pmt_read_counter(pp, cpus[t->cpu_id].package_id); done: gettimeofday(&t->tv_end, (struct timezone *)NULL); @@ -4927,48 +5410,39 @@ char *pkg_cstate_limit_strings[] = { "unknown", "reserved", "pc0", "pc1", "pc2", "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited" }; -int nhm_pkg_cstate_limits[16] = - { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int nhm_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int snb_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int snb_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int hsw_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int hsw_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int slv_pkg_cstate_limits[16] = - { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int slv_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7 }; -int amt_pkg_cstate_limits[16] = - { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int amt_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int phi_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int phi_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int glm_pkg_cstate_limits[16] = - { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int glm_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int skx_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int skx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int icx_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int icx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; @@ -5012,7 +5486,7 @@ void probe_cst_limit(void) return; } - get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); + get_msr(master_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); pkg_cstate_limit = pkg_cstate_limits[msr & 0xF]; } @@ -5024,9 +5498,9 @@ static void dump_platform_info(void) if (!platform->has_nhm_msrs || no_msr) return; - get_msr(base_cpu, MSR_PLATFORM_INFO, &msr); + get_msr(master_cpu, MSR_PLATFORM_INFO, &msr); - fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", master_cpu, msr); ratio = (msr >> 40) & 0xFF; fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n", ratio, bclk, ratio * bclk); @@ -5042,9 +5516,8 @@ static void dump_power_ctl(void) if (!platform->has_nhm_msrs || no_msr) return; - get_msr(base_cpu, MSR_IA32_POWER_CTL, &msr); - fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", - base_cpu, msr, msr & 0x2 ? "EN" : "DIS"); + get_msr(master_cpu, MSR_IA32_POWER_CTL, &msr); + fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", master_cpu, msr, msr & 0x2 ? "EN" : "DIS"); /* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */ if (platform->has_cst_prewake_bit) @@ -5058,9 +5531,9 @@ static void dump_turbo_ratio_limit2(void) unsigned long long msr; unsigned int ratio; - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT2, &msr); + get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT2, &msr); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", master_cpu, msr); ratio = (msr >> 8) & 0xFF; if (ratio) @@ -5077,9 +5550,9 @@ static void dump_turbo_ratio_limit1(void) unsigned long long msr; unsigned int ratio; - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &msr); + get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT1, &msr); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", master_cpu, msr); ratio = (msr >> 56) & 0xFF; if (ratio) @@ -5120,13 +5593,12 @@ static void dump_turbo_ratio_limits(int trl_msr_offset) unsigned long long msr, core_counts; int shift; - get_msr(base_cpu, trl_msr_offset, &msr); - fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n", - base_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr); + get_msr(master_cpu, trl_msr_offset, &msr); + fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n", master_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr); if (platform->trl_msrs & TRL_CORECOUNT) { - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, core_counts); + get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts); + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", master_cpu, core_counts); } else { core_counts = 0x0807060504030201; } @@ -5137,8 +5609,7 @@ static void dump_turbo_ratio_limits(int trl_msr_offset) ratio = (msr >> shift) & 0xFF; group_size = (core_counts >> shift) & 0xFF; if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", - ratio, bclk, ratio * bclk, group_size); + fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio, bclk, ratio * bclk, group_size); } return; @@ -5149,8 +5620,8 @@ static void dump_atom_turbo_ratio_limits(void) unsigned long long msr; unsigned int ratio; - get_msr(base_cpu, MSR_ATOM_CORE_RATIOS, &msr); - fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF); + get_msr(master_cpu, MSR_ATOM_CORE_RATIOS, &msr); + fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", master_cpu, msr & 0xFFFFFFFF); ratio = (msr >> 0) & 0x3F; if (ratio) @@ -5164,8 +5635,8 @@ static void dump_atom_turbo_ratio_limits(void) if (ratio) fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk); - get_msr(base_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr); - fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF); + get_msr(master_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr); + fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", master_cpu, msr & 0xFFFFFFFF); ratio = (msr >> 24) & 0x3F; if (ratio) @@ -5194,9 +5665,9 @@ static void dump_knl_turbo_ratio_limits(void) unsigned int cores[buckets_no]; unsigned int ratio[buckets_no]; - get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT, &msr); + get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT, &msr); - fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", master_cpu, msr); /* * Turbo encoding in KNL is as follows: @@ -5236,9 +5707,7 @@ static void dump_knl_turbo_ratio_limits(void) for (i = buckets_no - 1; i >= 0; i--) if (i > 0 ? ratio[i] != ratio[i - 1] : 1) - fprintf(outf, - "%d * %.1f = %.1f MHz max turbo %d active cores\n", - ratio[i], bclk, ratio[i] * bclk, cores[i]); + fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio[i], bclk, ratio[i] * bclk, cores[i]); } static void dump_cst_cfg(void) @@ -5248,9 +5717,9 @@ static void dump_cst_cfg(void) if (!platform->has_nhm_msrs || no_msr) return; - get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); + get_msr(master_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr); - fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", base_cpu, msr); + fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", master_cpu, msr); fprintf(outf, " (%s%s%s%s%slocked, pkg-cstate-limit=%d (%s)", (msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "", @@ -5273,12 +5742,12 @@ static void dump_config_tdp(void) { unsigned long long msr; - get_msr(base_cpu, MSR_CONFIG_TDP_NOMINAL, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", base_cpu, msr); + get_msr(master_cpu, MSR_CONFIG_TDP_NOMINAL, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", master_cpu, msr); fprintf(outf, " (base_ratio=%d)\n", (unsigned int)msr & 0xFF); - get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", master_cpu, msr); if (msr) { fprintf(outf, "PKG_MIN_PWR_LVL1=%d ", (unsigned int)(msr >> 48) & 0x7FFF); fprintf(outf, "PKG_MAX_PWR_LVL1=%d ", (unsigned int)(msr >> 32) & 0x7FFF); @@ -5287,8 +5756,8 @@ static void dump_config_tdp(void) } fprintf(outf, ")\n"); - get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", master_cpu, msr); if (msr) { fprintf(outf, "PKG_MIN_PWR_LVL2=%d ", (unsigned int)(msr >> 48) & 0x7FFF); fprintf(outf, "PKG_MAX_PWR_LVL2=%d ", (unsigned int)(msr >> 32) & 0x7FFF); @@ -5297,15 +5766,15 @@ static void dump_config_tdp(void) } fprintf(outf, ")\n"); - get_msr(base_cpu, MSR_CONFIG_TDP_CONTROL, &msr); - fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_CONFIG_TDP_CONTROL, &msr); + fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", master_cpu, msr); if ((msr) & 0x3) fprintf(outf, "TDP_LEVEL=%d ", (unsigned int)(msr) & 0x3); fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1); fprintf(outf, ")\n"); - get_msr(base_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr); - fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", base_cpu, msr); + get_msr(master_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr); + fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", master_cpu, msr); fprintf(outf, "MAX_NON_TURBO_RATIO=%d", (unsigned int)(msr) & 0xFF); fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1); fprintf(outf, ")\n"); @@ -5321,45 +5790,39 @@ void print_irtl(void) return; if (platform->supported_cstates & PC3) { - get_msr(base_cpu, MSR_PKGC3_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + get_msr(master_cpu, MSR_PKGC3_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", master_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC6) { - get_msr(base_cpu, MSR_PKGC6_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + get_msr(master_cpu, MSR_PKGC6_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", master_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC7) { - get_msr(base_cpu, MSR_PKGC7_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + get_msr(master_cpu, MSR_PKGC7_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", master_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC8) { - get_msr(base_cpu, MSR_PKGC8_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + get_msr(master_cpu, MSR_PKGC8_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", master_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC9) { - get_msr(base_cpu, MSR_PKGC9_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + get_msr(master_cpu, MSR_PKGC9_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", master_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC10) { - get_msr(base_cpu, MSR_PKGC10_IRTL, &msr); - fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + get_msr(master_cpu, MSR_PKGC10_IRTL, &msr); + fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", master_cpu, msr); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } } @@ -5393,6 +5856,40 @@ void free_fd_instr_count_percpu(void) fd_instr_count_percpu = NULL; } +void free_fd_llc_percpu(void) +{ + if (!fd_llc_percpu) + return; + + for (int i = 0; i < topo.max_cpu_num + 1; ++i) { + if (fd_llc_percpu[i] != 0) + close(fd_llc_percpu[i]); + } + + free(fd_llc_percpu); + fd_llc_percpu = NULL; + + BIC_NOT_PRESENT(BIC_LLC_MRPS); + BIC_NOT_PRESENT(BIC_LLC_HIT); +} + +void free_fd_l2_percpu(void) +{ + if (!fd_l2_percpu) + return; + + for (int i = 0; i < topo.max_cpu_num + 1; ++i) { + if (fd_l2_percpu[i] != 0) + close(fd_l2_percpu[i]); + } + + free(fd_l2_percpu); + fd_l2_percpu = NULL; + + BIC_NOT_PRESENT(BIC_L2_MRPS); + BIC_NOT_PRESENT(BIC_L2_HIT); +} + void free_fd_cstate(void) { if (!ccstate_counter_info) @@ -5495,21 +5992,36 @@ void free_all_buffers(void) cpu_affinity_set = NULL; cpu_affinity_setsize = 0; - free(thread_even); - free(core_even); - free(package_even); + if (perf_pcore_set) { + CPU_FREE(perf_pcore_set); + perf_pcore_set = NULL; + } + + if (perf_ecore_set) { + CPU_FREE(perf_ecore_set); + perf_ecore_set = NULL; + } + + if (perf_lcore_set) { + CPU_FREE(perf_lcore_set); + perf_lcore_set = NULL; + } + + free(even.threads); + free(even.cores); + free(even.packages); - thread_even = NULL; - core_even = NULL; - package_even = NULL; + even.threads = NULL; + even.cores = NULL; + even.packages = NULL; - free(thread_odd); - free(core_odd); - free(package_odd); + free(odd.threads); + free(odd.cores); + free(odd.packages); - thread_odd = NULL; - core_odd = NULL; - package_odd = NULL; + odd.threads = NULL; + odd.cores = NULL; + odd.packages = NULL; free(output_buffer); output_buffer = NULL; @@ -5517,6 +6029,8 @@ void free_all_buffers(void) free_fd_percpu(); free_fd_instr_count_percpu(); + free_fd_llc_percpu(); + free_fd_l2_percpu(); free_fd_msr(); free_fd_rapl_percpu(); free_fd_cstate(); @@ -5566,7 +6080,7 @@ int cpu_is_first_core_in_package(int cpu) return cpu == parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_siblings_list", cpu); } -int get_physical_package_id(int cpu) +int get_package_id(int cpu) { return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu); } @@ -5576,6 +6090,16 @@ int get_die_id(int cpu) return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/die_id", cpu); } +int get_l3_id(int cpu) +{ + return parse_int_file("/sys/devices/system/cpu/cpu%d/cache/index3/id", cpu); +} + +int get_module_id(int cpu) +{ + return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/cluster_id", cpu); +} + int get_core_id(int cpu) { return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_id", cpu); @@ -5594,7 +6118,7 @@ void set_node_data(void) for (pkg = 0; pkg < topo.num_packages; pkg++) { lnode = 0; for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { - if (cpus[cpu].physical_package_id != pkg) + if (cpus[cpu].package_id != pkg) continue; /* find a cpu with an unset logical_node_id */ if (cpus[cpu].logical_node_id != -1) @@ -5607,7 +6131,7 @@ void set_node_data(void) * the logical_node_id */ for (cpux = cpu; cpux <= topo.max_cpu_num; cpux++) { - if ((cpus[cpux].physical_package_id == pkg) && (cpus[cpux].physical_node_id == node)) { + if ((cpus[cpux].package_id == pkg) && (cpus[cpux].physical_node_id == node)) { cpus[cpux].logical_node_id = lnode; cpu_count++; } @@ -5626,7 +6150,7 @@ int get_physical_node_id(struct cpu_topology *thiscpu) char path[80]; FILE *filep; int i; - int cpu = thiscpu->logical_cpu_id; + int cpu = thiscpu->cpu_id; for (i = 0; i <= topo.max_cpu_num; i++) { sprintf(path, "/sys/devices/system/cpu/cpu%d/node%i/cpulist", cpu, i); @@ -5695,54 +6219,6 @@ static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size) return 0; } -int get_thread_siblings(struct cpu_topology *thiscpu) -{ - char path[80], character; - FILE *filep; - unsigned long map; - int so, shift, sib_core; - int cpu = thiscpu->logical_cpu_id; - int offset = topo.max_cpu_num + 1; - size_t size; - int thread_id = 0; - - thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1)); - if (thiscpu->thread_id < 0) - thiscpu->thread_id = thread_id++; - if (!thiscpu->put_ids) - return -1; - - size = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); - CPU_ZERO_S(size, thiscpu->put_ids); - - sprintf(path, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", cpu); - filep = fopen(path, "r"); - - if (!filep) { - warnx("%s: open failed", path); - return -1; - } - do { - offset -= BITMASK_SIZE; - if (fscanf(filep, "%lx%c", &map, &character) != 2) - err(1, "%s: failed to parse file", path); - for (shift = 0; shift < BITMASK_SIZE; shift++) { - if ((map >> shift) & 0x1) { - so = shift + offset; - sib_core = get_core_id(so); - if (sib_core == thiscpu->physical_core_id) { - CPU_SET_S(so, size, thiscpu->put_ids); - if ((so != cpu) && (cpus[so].thread_id < 0)) - cpus[so].thread_id = thread_id++; - } - } - } - } while (character == ','); - fclose(filep); - - return CPU_COUNT_S(size, thiscpu->put_ids); -} - /* * run func(thread, core, package) in topology order * skip non-present cpus @@ -5754,34 +6230,49 @@ int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *, struct core_data *core_base, struct pkg_data *pkg_base, struct thread_data *thread_base2, struct core_data *core_base2, struct pkg_data *pkg_base2) { - int retval, pkg_no, node_no, core_no, thread_no; + int cpu, retval; retval = 0; - for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) { - for (node_no = 0; node_no < topo.nodes_per_pkg; ++node_no) { - for (core_no = 0; core_no < topo.cores_per_node; ++core_no) { - for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) { - struct thread_data *t, *t2; - struct core_data *c, *c2; - struct pkg_data *p, *p2; + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + struct thread_data *t, *t2; + struct core_data *c, *c2; + struct pkg_data *p, *p2; - t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no); + if (cpu_is_not_allowed(cpu)) + continue; - if (cpu_is_not_allowed(t->cpu_id)) - continue; + if (has_allowed_lower_ht_sibling(cpu)) /* skip HT sibling */ + continue; - t2 = GET_THREAD(thread_base2, thread_no, core_no, node_no, pkg_no); + t = &thread_base[cpu]; + t2 = &thread_base2[cpu]; + c = &core_base[GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id)]; + c2 = &core_base2[GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id)]; + p = &pkg_base[cpus[cpu].package_id]; + p2 = &pkg_base2[cpus[cpu].package_id]; - c = GET_CORE(core_base, core_no, node_no, pkg_no); - c2 = GET_CORE(core_base2, core_no, node_no, pkg_no); + retval |= func(t, c, p, t2, c2, p2); - p = GET_PKG(pkg_base, pkg_no); - p2 = GET_PKG(pkg_base2, pkg_no); + /* Handle HT sibling now */ + int i; - retval |= func(t, c, p, t2, c2, p2); - } - } + for (i = 0; i <= MAX_HT_ID; ++i) { + int sibling_cpu_id = cpus[cpu].ht_sibling_cpu_id[i]; + + if (sibling_cpu_id < 0) + break; + + if (sibling_cpu_id == cpu) + continue; + + if (cpu_is_not_allowed(sibling_cpu_id)) + continue; + + t = &thread_base[sibling_cpu_id]; + t2 = &thread_base2[sibling_cpu_id]; + + retval |= func(t, c, p, t2, c2, p2); } } return retval; @@ -5838,7 +6329,7 @@ static int update_effective_str(bool startup) pos = fgets(buf, 1024, fp); if (!pos) - err(1, "%s: file read failed\n", PATH_EFFECTIVE_CPUS); + err(1, "%s: file read failed", PATH_EFFECTIVE_CPUS); fclose(fp); @@ -5855,13 +6346,15 @@ static void update_effective_set(bool startup) update_effective_str(startup); if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize)) - err(1, "%s: cpu str malformat %s\n", PATH_EFFECTIVE_CPUS, cpu_effective_str); + err(1, "%s: cpu str malformat %s", PATH_EFFECTIVE_CPUS, cpu_effective_str); } void linux_perf_init(void); void msr_perf_init(void); void rapl_perf_init(void); void cstate_perf_init(void); +void perf_llc_init(void); +void perf_l2_init(void); void added_perf_counters_init(void); void pmt_init(void); @@ -5873,23 +6366,24 @@ void re_initialize(void) msr_perf_init(); rapl_perf_init(); cstate_perf_init(); + perf_llc_init(); + perf_l2_init(); added_perf_counters_init(); pmt_init(); - fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, - topo.allowed_cpus); + fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus); } void set_max_cpu_num(void) { FILE *filep; - int base_cpu; + int current_cpu; unsigned long dummy; char pathname[64]; - base_cpu = sched_getcpu(); - if (base_cpu < 0) + current_cpu = sched_getcpu(); + if (current_cpu < 0) err(1, "cannot find calling cpu ID"); - sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", base_cpu); + sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", current_cpu); filep = fopen_or_die(pathname, "r"); topo.max_cpu_num = 0; @@ -5917,9 +6411,13 @@ int mark_cpu_present(int cpu) return 0; } -int init_thread_id(int cpu) +int clear_ht_id(int cpu) { - cpus[cpu].thread_id = -1; + int i; + + cpus[cpu].ht_id = -1; + for (i = 0; i <= MAX_HT_ID; ++i) + cpus[cpu].ht_sibling_cpu_id[i] = -1; return 0; } @@ -6030,6 +6528,7 @@ int snapshot_graphics(int idx) int retval; rewind(gfx_info[idx].fp); + fflush(gfx_info[idx].fp); switch (idx) { case GFX_rc6: @@ -6236,7 +6735,7 @@ int get_msr_sum(int cpu, off_t offset, unsigned long long *msr) timer_t timerid; /* Timer callback, update the sum of MSRs periodically. */ -static int update_msr_sum(struct thread_data *t, struct core_data *c, struct pkg_data *p) +static int update_msr_sum(PER_THREAD_PARAMS) { int i, ret; int cpu = t->cpu_id; @@ -6321,6 +6820,7 @@ release_timer: timer_delete(timerid); release_msr: free(per_cpu_msr_sum); + per_cpu_msr_sum = NULL; } /* @@ -6445,18 +6945,43 @@ restart: } } -void check_dev_msr() +int probe_dev_msr(void) { struct stat sb; char pathname[32]; - if (no_msr) + sprintf(pathname, "/dev/msr%d", master_cpu); + return !stat(pathname, &sb); +} + +int probe_dev_cpu_msr(void) +{ + struct stat sb; + char pathname[32]; + + sprintf(pathname, "/dev/cpu/%d/msr", master_cpu); + return !stat(pathname, &sb); +} + +int probe_msr_driver(void) +{ + if (probe_dev_msr()) { + use_android_msr_path = 1; + return 1; + } + return probe_dev_cpu_msr(); +} + +void check_msr_driver(void) +{ + if (probe_msr_driver()) return; - sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); - if (stat(pathname, &sb)) - if (system("/sbin/modprobe msr > /dev/null 2>&1")) - no_msr = 1; + if (system("/sbin/modprobe msr > /dev/null 2>&1")) + no_msr = 1; + + if (!probe_msr_driver()) + no_msr = 1; } /* @@ -6471,8 +6996,16 @@ int check_for_cap_sys_rawio(void) int ret = 0; caps = cap_get_proc(); - if (caps == NULL) + if (caps == NULL) { + /* + * CONFIG_MULTIUSER=n kernels have no cap_get_proc() + * Allow them to continue and attempt to access MSRs + */ + if (errno == ENOSYS) + return 0; + return 1; + } if (cap_get_flag(caps, CAP_SYS_RAWIO, CAP_EFFECTIVE, &cap_flag_value)) { ret = 1; @@ -6486,7 +7019,7 @@ int check_for_cap_sys_rawio(void) free_and_exit: if (cap_free(caps) == -1) - err(-6, "cap_free\n"); + err(-6, "cap_free"); return ret; } @@ -6503,7 +7036,7 @@ void check_msr_permission(void) failed += check_for_cap_sys_rawio(); /* test file permissions */ - sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", master_cpu); if (euidaccess(pathname, R_OK)) { failed++; } @@ -6532,7 +7065,7 @@ void probe_bclk(void) else return; - get_msr(base_cpu, MSR_PLATFORM_INFO, &msr); + get_msr(master_cpu, MSR_PLATFORM_INFO, &msr); base_ratio = (msr >> 8) & 0xFF; base_hz = base_ratio * bclk * 1000000; @@ -6632,10 +7165,10 @@ static void probe_intel_uncore_frequency_legacy(void) int k, l; char path_base[128]; - sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, - j); + sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, j); - if (access(path_base, R_OK)) + sprintf(path, "%s/current_freq_khz", path_base); + if (access(path, R_OK)) continue; BIC_PRESENT(BIC_UNCORE_MHZ); @@ -6683,16 +7216,16 @@ static void probe_intel_uncore_frequency_cluster(void) } for (i = uncore_max_id; i >= 0; --i) { int k, l; - int package_id, domain_id, cluster_id; + int unc_pkg_id, domain_id, cluster_id; char name_buf[16]; sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i); if (access(path_base, R_OK)) - err(1, "%s: %s\n", __func__, path_base); + err(1, "%s: %s", __func__, path_base); sprintf(path, "%s/package_id", path_base); - package_id = read_sysfs_int(path); + unc_pkg_id = read_sysfs_int(path); sprintf(path, "%s/domain_id", path_base); domain_id = read_sysfs_int(path); @@ -6703,7 +7236,19 @@ static void probe_intel_uncore_frequency_cluster(void) sprintf(path, "%s/current_freq_khz", path_base); sprintf(name_buf, "UMHz%d.%d", domain_id, cluster_id); - add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, package_id); + /* + * Once add_couter() is called, that counter is always read + * and reported -- So it is effectively (enabled & present). + * Only call add_counter() here if legacy BIC_UNCORE_MHZ (UncMHz) + * is (enabled). Since we are in this routine, we + * know we will not probe and set (present) the legacy counter. + * + * This allows "--show/--hide UncMHz" to be effective for + * the clustered MHz counters, as a group. + */ + if BIC_IS_ENABLED + (BIC_UNCORE_MHZ) + add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, unc_pkg_id); if (quiet) continue; @@ -6712,8 +7257,7 @@ static void probe_intel_uncore_frequency_cluster(void) k = read_sysfs_int(path); sprintf(path, "%s/max_freq_khz", path_base); l = read_sysfs_int(path); - fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", package_id, domain_id, - cluster_id, k / 1000, l / 1000); + fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", unc_pkg_id, domain_id, cluster_id, k / 1000, l / 1000); sprintf(path, "%s/initial_min_freq_khz", path_base); k = read_sysfs_int(path); @@ -6868,7 +7412,7 @@ static void dump_sysfs_cstate_config(void) for (state = 0; state < 10; ++state) { - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state); input = fopen(path, "r"); if (input == NULL) continue; @@ -6884,14 +7428,14 @@ static void dump_sysfs_cstate_config(void) remove_underbar(name_buf); - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", base_cpu, state); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", master_cpu, state); input = fopen(path, "r"); if (input == NULL) continue; if (!fgets(desc, sizeof(desc), input)) err(1, "%s: failed to read file", path); - fprintf(outf, "cpu%d: %s: %s", base_cpu, name_buf, desc); + fprintf(outf, "cpu%d: %s: %s", master_cpu, name_buf, desc); fclose(input); } } @@ -6904,7 +7448,7 @@ static void dump_sysfs_pstate_config(void) FILE *input; int turbo; - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", base_cpu); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", master_cpu); input = fopen(path, "r"); if (input == NULL) { fprintf(outf, "NSFOD %s\n", path); @@ -6914,7 +7458,7 @@ static void dump_sysfs_pstate_config(void) err(1, "%s: failed to read file", path); fclose(input); - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", base_cpu); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", master_cpu); input = fopen(path, "r"); if (input == NULL) { fprintf(outf, "NSFOD %s\n", path); @@ -6924,8 +7468,8 @@ static void dump_sysfs_pstate_config(void) err(1, "%s: failed to read file", path); fclose(input); - fprintf(outf, "cpu%d: cpufreq driver: %s", base_cpu, driver_buf); - fprintf(outf, "cpu%d: cpufreq governor: %s", base_cpu, governor_buf); + fprintf(outf, "cpu%d: cpufreq driver: %s", master_cpu, driver_buf); + fprintf(outf, "cpu%d: cpufreq governor: %s", master_cpu, governor_buf); sprintf(path, "/sys/devices/system/cpu/cpufreq/boost"); input = fopen(path, "r"); @@ -6950,7 +7494,7 @@ static void dump_sysfs_pstate_config(void) * print_epb() * Decode the ENERGY_PERF_BIAS MSR */ -int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int print_epb(PER_THREAD_PARAMS) { char *epb_string; int cpu, epb; @@ -6999,7 +7543,7 @@ int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p) * print_hwp() * Decode the MSR_HWP_CAPABILITIES */ -int print_hwp(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int print_hwp(PER_THREAD_PARAMS) { unsigned long long msr; int cpu; @@ -7040,8 +7584,7 @@ int print_hwp(struct thread_data *t, struct core_data *c, struct pkg_data *p) "(high %d guar %d eff %d low %d)\n", cpu, msr, (unsigned int)HWP_HIGHEST_PERF(msr), - (unsigned int)HWP_GUARANTEED_PERF(msr), - (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr)); + (unsigned int)HWP_GUARANTEED_PERF(msr), (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr)); if (get_msr(cpu, MSR_HWP_REQUEST, &msr)) return 0; @@ -7052,8 +7595,7 @@ int print_hwp(struct thread_data *t, struct core_data *c, struct pkg_data *p) (unsigned int)(((msr) >> 0) & 0xff), (unsigned int)(((msr) >> 8) & 0xff), (unsigned int)(((msr) >> 16) & 0xff), - (unsigned int)(((msr) >> 24) & 0xff), - (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1)); + (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1)); if (has_hwp_pkg) { if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr)) @@ -7064,23 +7606,20 @@ int print_hwp(struct thread_data *t, struct core_data *c, struct pkg_data *p) cpu, msr, (unsigned int)(((msr) >> 0) & 0xff), (unsigned int)(((msr) >> 8) & 0xff), - (unsigned int)(((msr) >> 16) & 0xff), - (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3)); + (unsigned int)(((msr) >> 16) & 0xff), (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3)); } if (has_hwp_notify) { if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr)) return 0; fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx " - "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", - cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis"); + "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis"); } if (get_msr(cpu, MSR_HWP_STATUS, &msr)) return 0; fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx " - "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", - cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-"); + "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-"); return 0; } @@ -7088,7 +7627,7 @@ int print_hwp(struct thread_data *t, struct core_data *c, struct pkg_data *p) /* * print_perf_limit() */ -int print_perf_limit(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int print_perf_limit(PER_THREAD_PARAMS) { unsigned long long msr; int cpu; @@ -7125,8 +7664,7 @@ int print_perf_limit(struct thread_data *t, struct core_data *c, struct pkg_data (msr & 1 << 6) ? "VR-Therm, " : "", (msr & 1 << 5) ? "Auto-HWP, " : "", (msr & 1 << 4) ? "Graphics, " : "", - (msr & 1 << 2) ? "bit2, " : "", - (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : ""); + (msr & 1 << 2) ? "bit2, " : "", (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : ""); fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n", (msr & 1 << 31) ? "bit31, " : "", (msr & 1 << 30) ? "bit30, " : "", @@ -7139,8 +7677,7 @@ int print_perf_limit(struct thread_data *t, struct core_data *c, struct pkg_data (msr & 1 << 22) ? "VR-Therm, " : "", (msr & 1 << 21) ? "Auto-HWP, " : "", (msr & 1 << 20) ? "Graphics, " : "", - (msr & 1 << 18) ? "bit18, " : "", - (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : ""); + (msr & 1 << 18) ? "bit18, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : ""); } if (platform->plr_msrs & PLR_GFX) { @@ -7152,16 +7689,14 @@ int print_perf_limit(struct thread_data *t, struct core_data *c, struct pkg_data (msr & 1 << 4) ? "Graphics, " : "", (msr & 1 << 6) ? "VR-Therm, " : "", (msr & 1 << 8) ? "Amps, " : "", - (msr & 1 << 9) ? "GFXPwr, " : "", - (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); + (msr & 1 << 9) ? "GFXPwr, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s)\n", (msr & 1 << 16) ? "PROCHOT, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 20) ? "Graphics, " : "", (msr & 1 << 22) ? "VR-Therm, " : "", (msr & 1 << 24) ? "Amps, " : "", - (msr & 1 << 25) ? "GFXPwr, " : "", - (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); + (msr & 1 << 25) ? "GFXPwr, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); } if (platform->plr_msrs & PLR_RING) { get_msr(cpu, MSR_RING_PERF_LIMIT_REASONS, &msr); @@ -7170,14 +7705,12 @@ int print_perf_limit(struct thread_data *t, struct core_data *c, struct pkg_data (msr & 1 << 0) ? "PROCHOT, " : "", (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 6) ? "VR-Therm, " : "", - (msr & 1 << 8) ? "Amps, " : "", - (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); + (msr & 1 << 8) ? "Amps, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); fprintf(outf, " (Logged: %s%s%s%s%s%s)\n", (msr & 1 << 16) ? "PROCHOT, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 22) ? "VR-Therm, " : "", - (msr & 1 << 24) ? "Amps, " : "", - (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); + (msr & 1 << 24) ? "Amps, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); } return 0; } @@ -7197,8 +7730,8 @@ double get_tdp_intel(void) { unsigned long long msr; - if (platform->rapl_msrs & RAPL_PKG_POWER_INFO) - if (!get_msr(base_cpu, MSR_PKG_POWER_INFO, &msr)) + if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) + if (!get_msr(master_cpu, MSR_PKG_POWER_INFO, &msr)) return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units; return get_quirk_tdp(); } @@ -7213,21 +7746,31 @@ void rapl_probe_intel(void) unsigned long long msr; unsigned int time_unit; double tdp; - const unsigned long long bic_watt_bits = BIC_SysWatt | BIC_PkgWatt | BIC_CorWatt | BIC_RAMWatt | BIC_GFXWatt; - const unsigned long long bic_joules_bits = BIC_Sys_J | BIC_Pkg_J | BIC_Cor_J | BIC_RAM_J | BIC_GFX_J; - if (rapl_joules) - bic_enabled &= ~bic_watt_bits; - else - bic_enabled &= ~bic_joules_bits; + if (rapl_joules) { + CLR_BIC(BIC_SysWatt, &bic_enabled); + CLR_BIC(BIC_PkgWatt, &bic_enabled); + CLR_BIC(BIC_CorWatt, &bic_enabled); + CLR_BIC(BIC_RAMWatt, &bic_enabled); + CLR_BIC(BIC_GFXWatt, &bic_enabled); + } else { + CLR_BIC(BIC_Sys_J, &bic_enabled); + CLR_BIC(BIC_Pkg_J, &bic_enabled); + CLR_BIC(BIC_Cor_J, &bic_enabled); + CLR_BIC(BIC_RAM_J, &bic_enabled); + CLR_BIC(BIC_GFX_J, &bic_enabled); + } + + if (!valid_rapl_msrs || no_msr) + return; - if (!(platform->rapl_msrs & RAPL_PKG_PERF_STATUS)) - bic_enabled &= ~BIC_PKG__; - if (!(platform->rapl_msrs & RAPL_DRAM_PERF_STATUS)) - bic_enabled &= ~BIC_RAM__; + if (!(valid_rapl_msrs & RAPL_PKG_PERF_STATUS)) + CLR_BIC(BIC_PKG__, &bic_enabled); + if (!(valid_rapl_msrs & RAPL_DRAM_PERF_STATUS)) + CLR_BIC(BIC_RAM__, &bic_enabled); /* units on package 0, verify later other packages match */ - if (get_msr(base_cpu, MSR_RAPL_POWER_UNIT, &msr)) + if (get_msr(master_cpu, MSR_RAPL_POWER_UNIT, &msr)) return; rapl_power_units = 1.0 / (1 << (msr & 0xF)); @@ -7263,15 +7806,19 @@ void rapl_probe_amd(void) { unsigned long long msr; double tdp; - const unsigned long long bic_watt_bits = BIC_PkgWatt | BIC_CorWatt; - const unsigned long long bic_joules_bits = BIC_Pkg_J | BIC_Cor_J; - if (rapl_joules) - bic_enabled &= ~bic_watt_bits; - else - bic_enabled &= ~bic_joules_bits; + if (rapl_joules) { + CLR_BIC(BIC_SysWatt, &bic_enabled); + CLR_BIC(BIC_CorWatt, &bic_enabled); + } else { + CLR_BIC(BIC_Pkg_J, &bic_enabled); + CLR_BIC(BIC_Cor_J, &bic_enabled); + } - if (get_msr(base_cpu, MSR_RAPL_PWR_UNIT, &msr)) + if (!valid_rapl_msrs || no_msr) + return; + + if (get_msr(master_cpu, MSR_RAPL_PWR_UNIT, &msr)) return; rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf)); @@ -7291,13 +7838,164 @@ void print_power_limit_msr(int cpu, unsigned long long msr, char *label) cpu, label, ((msr >> 15) & 1) ? "EN" : "DIS", ((msr >> 0) & 0x7FFF) * rapl_power_units, - (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, - (((msr >> 16) & 1) ? "EN" : "DIS")); + (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, (((msr >> 16) & 1) ? "EN" : "DIS")); return; } -int print_rapl(struct thread_data *t, struct core_data *c, struct pkg_data *p) +static int fread_int(char *path, int *val) +{ + FILE *filep; + int ret; + + filep = fopen(path, "r"); + if (!filep) + return -1; + + ret = fscanf(filep, "%d", val); + fclose(filep); + return ret; +} + +static int fread_ull(char *path, unsigned long long *val) +{ + FILE *filep; + int ret; + + filep = fopen(path, "r"); + if (!filep) + return -1; + + ret = fscanf(filep, "%llu", val); + fclose(filep); + return ret; +} + +static int fread_str(char *path, char *buf, int size) +{ + FILE *filep; + int ret; + char *cp; + + filep = fopen(path, "r"); + if (!filep) + return -1; + + ret = fread(buf, 1, size, filep); + fclose(filep); + + /* replace '\n' with '\0' */ + cp = strchr(buf, '\n'); + if (cp != NULL) + *cp = '\0'; + + return ret; +} + +#define PATH_RAPL_SYSFS "/sys/class/powercap" + +static int dump_one_domain(char *domain_path) +{ + char path[PATH_MAX]; + char str[PATH_MAX]; + unsigned long long val; + int constraint; + int enable; + int ret; + + snprintf(path, PATH_MAX, "%s/name", domain_path); + ret = fread_str(path, str, PATH_MAX); + if (ret <= 0) + return -1; + + fprintf(outf, "%s: %s", domain_path + strlen(PATH_RAPL_SYSFS) + 1, str); + + snprintf(path, PATH_MAX, "%s/enabled", domain_path); + ret = fread_int(path, &enable); + if (ret <= 0) + return -1; + + if (!enable) { + fputs(" disabled\n", outf); + return 0; + } + + for (constraint = 0;; constraint++) { + snprintf(path, PATH_MAX, "%s/constraint_%d_time_window_us", domain_path, constraint); + ret = fread_ull(path, &val); + if (ret <= 0) + break; + + if (val > 1000000) + fprintf(outf, " %0.1fs", (double)val / 1000000); + else if (val > 1000) + fprintf(outf, " %0.1fms", (double)val / 1000); + else + fprintf(outf, " %0.1fus", (double)val); + + snprintf(path, PATH_MAX, "%s/constraint_%d_power_limit_uw", domain_path, constraint); + ret = fread_ull(path, &val); + if (ret > 0 && val) + fprintf(outf, ":%lluW", val / 1000000); + + snprintf(path, PATH_MAX, "%s/constraint_%d_max_power_uw", domain_path, constraint); + ret = fread_ull(path, &val); + if (ret > 0 && val) + fprintf(outf, ",max:%lluW", val / 1000000); + } + fputc('\n', outf); + + return 0; +} + +static int print_rapl_sysfs(void) +{ + DIR *dir, *cdir; + struct dirent *entry, *centry; + char path[PATH_MAX]; + char str[PATH_MAX]; + + if ((dir = opendir(PATH_RAPL_SYSFS)) == NULL) { + warn("open %s failed", PATH_RAPL_SYSFS); + return 1; + } + + while ((entry = readdir(dir)) != NULL) { + if (strlen(entry->d_name) > 100) + continue; + + if (strncmp(entry->d_name, "intel-rapl", strlen("intel-rapl"))) + continue; + + snprintf(path, PATH_MAX, "%s/%s/name", PATH_RAPL_SYSFS, entry->d_name); + + /* Parse top level domains first, including package and psys */ + fread_str(path, str, PATH_MAX); + if (strncmp(str, "package", strlen("package")) && strncmp(str, "psys", strlen("psys"))) + continue; + + snprintf(path, PATH_MAX, "%s/%s", PATH_RAPL_SYSFS, entry->d_name); + if ((cdir = opendir(path)) == NULL) { + perror("opendir() error"); + return 1; + } + + dump_one_domain(path); + + while ((centry = readdir(cdir)) != NULL) { + if (strncmp(centry->d_name, "intel-rapl", strlen("intel-rapl"))) + continue; + snprintf(path, PATH_MAX, "%s/%s/%s", PATH_RAPL_SYSFS, entry->d_name, centry->d_name); + dump_one_domain(path); + } + closedir(cdir); + } + + closedir(dir); + return 0; +} + +int print_rapl(PER_THREAD_PARAMS) { unsigned long long msr; const char *msr_name; @@ -7306,7 +8004,7 @@ int print_rapl(struct thread_data *t, struct core_data *c, struct pkg_data *p) UNUSED(c); UNUSED(p); - if (!platform->rapl_msrs) + if (!valid_rapl_msrs) return 0; /* RAPL counters are per package, so print only for 1st thread/package */ @@ -7319,7 +8017,7 @@ int print_rapl(struct thread_data *t, struct core_data *c, struct pkg_data *p) return -1; } - if (platform->rapl_msrs & RAPL_AMD_F17H) { + if (valid_rapl_msrs & RAPL_AMD_F17H) { msr_name = "MSR_RAPL_PWR_UNIT"; if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr)) return -1; @@ -7329,10 +8027,9 @@ int print_rapl(struct thread_data *t, struct core_data *c, struct pkg_data *p) return -1; } - fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, - rapl_power_units, rapl_energy_units, rapl_time_units); + fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, rapl_power_units, rapl_energy_units, rapl_time_units); - if (platform->rapl_msrs & RAPL_PKG_POWER_INFO) { + if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) { if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr)) return -5; @@ -7341,35 +8038,31 @@ int print_rapl(struct thread_data *t, struct core_data *c, struct pkg_data *p) cpu, msr, ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); + ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); } - if (platform->rapl_msrs & RAPL_PKG) { + if (valid_rapl_msrs & RAPL_PKG) { if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr)) return -9; - fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 63) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 63) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "PKG Limit #1"); fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%0.3f Watts, %f* sec, clamp %sabled)\n", cpu, ((msr >> 47) & 1) ? "EN" : "DIS", ((msr >> 32) & 0x7FFF) * rapl_power_units, - (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, - ((msr >> 48) & 1) ? "EN" : "DIS"); + (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, ((msr >> 48) & 1) ? "EN" : "DIS"); if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr)) return -9; fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr); - fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", - cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN"); } - if (platform->rapl_msrs & RAPL_DRAM_POWER_INFO) { + if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) { if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr)) return -6; @@ -7377,31 +8070,28 @@ int print_rapl(struct thread_data *t, struct core_data *c, struct pkg_data *p) cpu, msr, ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); + ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); } - if (platform->rapl_msrs & RAPL_DRAM) { + if (valid_rapl_msrs & RAPL_DRAM) { if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr)) return -9; - fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 31) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "DRAM Limit"); } - if (platform->rapl_msrs & RAPL_CORE_POLICY) { + if (valid_rapl_msrs & RAPL_CORE_POLICY) { if (get_msr(cpu, MSR_PP0_POLICY, &msr)) return -7; fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF); } - if (platform->rapl_msrs & RAPL_CORE_POWER_LIMIT) { + if (valid_rapl_msrs & RAPL_CORE_POWER_LIMIT) { if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr)) return -9; - fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 31) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "Cores Limit"); } - if (platform->rapl_msrs & RAPL_GFX) { + if (valid_rapl_msrs & RAPL_GFX) { if (get_msr(cpu, MSR_PP1_POLICY, &msr)) return -8; @@ -7409,22 +8099,57 @@ int print_rapl(struct thread_data *t, struct core_data *c, struct pkg_data *p) if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr)) return -9; - fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 31) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "GFX Limit"); } return 0; } /* + * probe_rapl_msrs + * + * initialize global valid_rapl_msrs to platform->plat_rapl_msrs + * only if PKG_ENERGY counter is enumerated and reads non-zero + */ +void probe_rapl_msrs(void) +{ + int ret; + off_t offset; + unsigned long long msr_value; + + if (no_msr) + return; + + if ((platform->plat_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H)) == 0) + return; + + offset = idx_to_offset(IDX_PKG_ENERGY); + if (offset < 0) + return; + + ret = get_msr(master_cpu, offset, &msr_value); + if (ret) { + if (debug) + fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset); + return; + } + if (msr_value == 0) { + if (debug) + fprintf(outf, "RAPL_PKG_ENERGY MSR(0x%llx) == ZERO: disabling all RAPL MSRs\n", (unsigned long long)offset); + return; + } + + valid_rapl_msrs = platform->plat_rapl_msrs; /* success */ +} + +/* * probe_rapl() * * sets rapl_power_units, rapl_energy_units, rapl_time_units */ void probe_rapl(void) { - if (!platform->rapl_msrs || no_msr) - return; + probe_rapl_msrs(); if (genuine_intel) rapl_probe_intel(); @@ -7434,6 +8159,11 @@ void probe_rapl(void) if (quiet) return; + print_rapl_sysfs(); + + if (!valid_rapl_msrs || no_msr) + return; + for_all_cpus(print_rapl, ODD_COUNTERS); } @@ -7449,7 +8179,7 @@ void probe_rapl(void) * below this value, including the Digital Thermal Sensor (DTS), * Package Thermal Management Sensor (PTM), and thermal event thresholds. */ -int set_temperature_target(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int set_temperature_target(PER_THREAD_PARAMS) { unsigned long long msr; unsigned int tcc_default, tcc_offset; @@ -7482,7 +8212,7 @@ int set_temperature_target(struct thread_data *t, struct core_data *c, struct pk if (!platform->has_nhm_msrs || no_msr) goto guess; - if (get_msr(base_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr)) + if (get_msr(master_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr)) goto guess; tcc_default = (msr >> 16) & 0xFF; @@ -7491,7 +8221,7 @@ int set_temperature_target(struct thread_data *t, struct core_data *c, struct pk int bits = platform->tcc_offset_bits; unsigned long long enabled = 0; - if (bits && !get_msr(base_cpu, MSR_PLATFORM_INFO, &enabled)) + if (bits && !get_msr(master_cpu, MSR_PLATFORM_INFO, &enabled)) enabled = (enabled >> 30) & 1; if (bits && enabled) { @@ -7517,7 +8247,7 @@ guess: return 0; } -int print_thermal(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int print_thermal(PER_THREAD_PARAMS) { unsigned long long msr; unsigned int dts, dts2; @@ -7535,7 +8265,7 @@ int print_thermal(struct thread_data *t, struct core_data *c, struct pkg_data *p cpu = t->cpu_id; /* DTS is per-core, no need to print for each thread */ - if (!is_cpu_first_thread_in_core(t, c, p)) + if (!is_cpu_first_thread_in_core(t, c)) return 0; if (cpu_migrate(cpu)) { @@ -7543,7 +8273,7 @@ int print_thermal(struct thread_data *t, struct core_data *c, struct pkg_data *p return -1; } - if (do_ptm && is_cpu_first_core_in_package(t, c, p)) { + if (do_ptm && is_cpu_first_core_in_package(t, p)) { if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr)) return 0; @@ -7555,8 +8285,7 @@ int print_thermal(struct thread_data *t, struct core_data *c, struct pkg_data *p dts = (msr >> 16) & 0x7F; dts2 = (msr >> 8) & 0x7F; - fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", - cpu, msr, tj_max - dts, tj_max - dts2); + fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2); } if (do_dts && debug) { @@ -7567,16 +8296,14 @@ int print_thermal(struct thread_data *t, struct core_data *c, struct pkg_data *p dts = (msr >> 16) & 0x7F; resolution = (msr >> 27) & 0xF; - fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", - cpu, msr, tj_max - dts, resolution); + fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", cpu, msr, tj_max - dts, resolution); if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr)) return 0; dts = (msr >> 16) & 0x7F; dts2 = (msr >> 8) & 0x7F; - fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", - cpu, msr, tj_max - dts, tj_max - dts2); + fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2); } return 0; @@ -7597,7 +8324,7 @@ void probe_thermal(void) for_all_cpus(print_thermal, ODD_COUNTERS); } -int get_cpu_type(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int get_cpu_type(PER_THREAD_PARAMS) { unsigned int eax, ebx, ecx, edx; @@ -7629,9 +8356,12 @@ void decode_feature_control_msr(void) if (no_msr) return; - if (!get_msr(base_cpu, MSR_IA32_FEAT_CTL, &msr)) + if (quiet) + return; + + if (!get_msr(master_cpu, MSR_IA32_FEAT_CTL, &msr)) fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n", - base_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : ""); + master_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : ""); } void decode_misc_enable_msr(void) @@ -7644,14 +8374,13 @@ void decode_misc_enable_msr(void) if (!genuine_intel) return; - if (!get_msr(base_cpu, MSR_IA32_MISC_ENABLE, &msr)) + if (!get_msr(master_cpu, MSR_IA32_MISC_ENABLE, &msr)) fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n", - base_cpu, msr, + master_cpu, msr, msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-", msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-", msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-", - msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", - msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : ""); + msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : ""); } void decode_misc_feature_control(void) @@ -7664,11 +8393,10 @@ void decode_misc_feature_control(void) if (!platform->has_msr_misc_feature_control) return; - if (!get_msr(base_cpu, MSR_MISC_FEATURE_CONTROL, &msr)) + if (!get_msr(master_cpu, MSR_MISC_FEATURE_CONTROL, &msr)) fprintf(outf, "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n", - base_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", - msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : ""); + master_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : ""); } /* @@ -7688,10 +8416,9 @@ void decode_misc_pwr_mgmt_msr(void) if (!platform->has_msr_misc_pwr_mgmt) return; - if (!get_msr(base_cpu, MSR_MISC_PWR_MGMT, &msr)) + if (!get_msr(master_cpu, MSR_MISC_PWR_MGMT, &msr)) fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n", - base_cpu, msr, - msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); + master_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); } /* @@ -7710,13 +8437,11 @@ void decode_c6_demotion_policy_msr(void) if (!platform->has_msr_c6_demotion_policy_config) return; - if (!get_msr(base_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr)) - fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", - base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); + if (!get_msr(master_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr)) + fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); - if (!get_msr(base_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr)) - fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", - base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); + if (!get_msr(master_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr)) + fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS"); } void print_dev_latency(void) @@ -7744,91 +8469,191 @@ void print_dev_latency(void) close(fd); } -static int has_instr_count_access(void) +static int has_perf_instr_count_access(void) { int fd; - int has_access; if (no_perf) return 0; - fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0); - has_access = fd != -1; - + fd = open_perf_counter(master_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0); if (fd != -1) close(fd); - if (!has_access) + if (fd == -1) warnx("Failed to access %s. Some of the counters may not be available\n" - "\tRun as root to enable them or use %s to disable the access explicitly", - "instructions retired perf counter", "--no-perf"); + "\tRun as root to enable them or use %s to disable the access explicitly", "perf instructions retired counter", + "'--hide IPC' or '--no-perf'"); - return has_access; + return (fd != -1); } -int add_rapl_perf_counter_(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, - double *scale_, enum rapl_unit *unit_) +int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, double *scale_, enum rapl_unit *unit_) { + int ret = -1; + if (no_perf) return -1; + if (!cai->perf_name) + return -1; + const double scale = read_perf_scale(cai->perf_subsys, cai->perf_name); if (scale == 0.0) - return -1; + goto end; const enum rapl_unit unit = read_perf_rapl_unit(cai->perf_subsys, cai->perf_name); if (unit == RAPL_UNIT_INVALID) - return -1; + goto end; const unsigned int rapl_type = read_perf_type(cai->perf_subsys); const unsigned int rapl_energy_pkg_config = read_perf_config(cai->perf_subsys, cai->perf_name); - const int fd_counter = - open_perf_counter(cpu, rapl_type, rapl_energy_pkg_config, rci->fd_perf, PERF_FORMAT_GROUP); - if (fd_counter == -1) - return -1; + ret = open_perf_counter(cpu, rapl_type, rapl_energy_pkg_config, rci->fd_perf, PERF_FORMAT_GROUP); + if (ret == -1) + goto end; /* If it's the first counter opened, make it a group descriptor */ if (rci->fd_perf == -1) - rci->fd_perf = fd_counter; + rci->fd_perf = ret; *scale_ = scale; *unit_ = unit; - return fd_counter; -} - -int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, - double *scale, enum rapl_unit *unit) -{ - int ret = add_rapl_perf_counter_(cpu, rci, cai, scale, unit); +end: if (debug >= 2) fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu); return ret; } +char cpuset_buf[1024]; +int initialize_cpu_set_from_sysfs(cpu_set_t *cpu_set, char *sysfs_path, char *sysfs_file) +{ + FILE *fp; + char path[128]; + + if (snprintf(path, 128, "%s/%s", sysfs_path, sysfs_file) > 128) + err(-1, "%s %s", sysfs_path, sysfs_file); + + fp = fopen(path, "r"); + if (!fp) { + warn("open %s", path); + return -1; + } + if (fread(cpuset_buf, sizeof(char), 1024, fp) == 0) { + warn("read %s", sysfs_path); + goto err; + } + if (parse_cpu_str(cpuset_buf, cpu_set, cpu_possible_setsize)) { + warnx("%s: cpu str malformat %s\n", sysfs_path, cpu_effective_str); + goto err; + } + return 0; + +err: + fclose(fp); + return -1; +} + +void print_cpu_set(char *s, cpu_set_t *set) +{ + int i; + + assert(MAX_BIC < CPU_SETSIZE); + + printf("%s:", s); + + for (i = 0; i <= topo.max_cpu_num; ++i) + if (CPU_ISSET(i, set)) + printf(" %d", i); + putchar('\n'); +} + +void linux_perf_init_hybrid_cpus(void) +{ + char *perf_cpu_pcore_path = "/sys/devices/cpu_core"; + char *perf_cpu_ecore_path = "/sys/devices/cpu_atom"; + char *perf_cpu_lcore_path = "/sys/devices/cpu_lowpower"; + char path[128]; + + if (!access(perf_cpu_pcore_path, F_OK)) { + perf_pcore_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (perf_pcore_set == NULL) + err(3, "CPU_ALLOC"); + CPU_ZERO_S(cpu_possible_setsize, perf_pcore_set); + initialize_cpu_set_from_sysfs(perf_pcore_set, perf_cpu_pcore_path, "cpus"); + if (debug) + print_cpu_set("perf pcores", perf_pcore_set); + sprintf(path, "%s/%s", perf_cpu_pcore_path, "type"); + perf_pmu_types.pcore = snapshot_sysfs_counter(path); + } + + if (!access(perf_cpu_ecore_path, F_OK)) { + perf_ecore_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (perf_ecore_set == NULL) + err(3, "CPU_ALLOC"); + CPU_ZERO_S(cpu_possible_setsize, perf_ecore_set); + initialize_cpu_set_from_sysfs(perf_ecore_set, perf_cpu_ecore_path, "cpus"); + if (debug) + print_cpu_set("perf ecores", perf_ecore_set); + sprintf(path, "%s/%s", perf_cpu_ecore_path, "type"); + perf_pmu_types.ecore = snapshot_sysfs_counter(path); + } + + if (!access(perf_cpu_lcore_path, F_OK)) { + perf_lcore_set = CPU_ALLOC((topo.max_cpu_num + 1)); + if (perf_lcore_set == NULL) + err(3, "CPU_ALLOC"); + CPU_ZERO_S(cpu_possible_setsize, perf_lcore_set); + initialize_cpu_set_from_sysfs(perf_lcore_set, perf_cpu_lcore_path, "cpus"); + if (debug) + print_cpu_set("perf lcores", perf_lcore_set); + sprintf(path, "%s/%s", perf_cpu_lcore_path, "type"); + perf_pmu_types.lcore = snapshot_sysfs_counter(path); + } +} + /* - * Linux-perf manages the HW instructions-retired counter - * by enabling when requested, and hiding rollover + * Linux-perf related initialization */ void linux_perf_init(void) { + char path[128]; + char *perf_cpu_path = "/sys/devices/cpu"; + if (access("/proc/sys/kernel/perf_event_paranoid", F_OK)) return; - if (BIC_IS_ENABLED(BIC_IPC) && has_aperf) { + if (!access(perf_cpu_path, F_OK)) { + sprintf(path, "%s/%s", perf_cpu_path, "type"); + perf_pmu_types.uniform = snapshot_sysfs_counter(path); + } else { + linux_perf_init_hybrid_cpus(); + } + + if (BIC_IS_ENABLED(BIC_IPC) && cpuid_has_aperf_mperf) { fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); if (fd_instr_count_percpu == NULL) err(-1, "calloc fd_instr_count_percpu"); } + if (BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT)) { + fd_llc_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); + if (fd_llc_percpu == NULL) + err(-1, "calloc fd_llc_percpu"); + } + if (BIC_IS_ENABLED(BIC_L2_MRPS) || BIC_IS_ENABLED(BIC_L2_HIT)) { + fd_l2_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); + if (fd_l2_percpu == NULL) + err(-1, "calloc fd_l2_percpu"); + } } void rapl_perf_init(void) { - const unsigned int num_domains = (platform->has_per_core_rapl ? topo.max_core_id : topo.max_package_id) + 1; + const unsigned int num_domains = get_rapl_num_domains(); bool *domain_visited = calloc(num_domains, sizeof(bool)); rapl_counter_info_perdomain = calloc(num_domains, sizeof(*rapl_counter_info_perdomain)); @@ -7861,6 +8686,9 @@ void rapl_perf_init(void) enum rapl_unit unit; unsigned int next_domain; + if (!BIC_IS_ENABLED(cai->bic_number)) + continue; + memset(domain_visited, 0, num_domains * sizeof(*domain_visited)); for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) { @@ -7869,8 +8697,7 @@ void rapl_perf_init(void) continue; /* Skip already seen and handled RAPL domains */ - next_domain = - platform->has_per_core_rapl ? cpus[cpu].physical_core_id : cpus[cpu].physical_package_id; + next_domain = get_rapl_domain_id(cpu); assert(next_domain < num_domains); @@ -7879,32 +8706,42 @@ void rapl_perf_init(void) domain_visited[next_domain] = 1; - if ((cai->flags & RAPL_COUNTER_FLAG_PLATFORM_COUNTER) && (cpu != base_cpu)) + if ((cai->flags & RAPL_COUNTER_FLAG_PLATFORM_COUNTER) && (cpu != master_cpu)) continue; struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[next_domain]; - /* Check if the counter is enabled and accessible */ - if (BIC_IS_ENABLED(cai->bic) && (platform->rapl_msrs & cai->feature_mask)) { + /* + * rapl_counter_arch_infos[] can have multiple entries describing the same + * counter, due to the difference from different platforms/Vendors. + * E.g. rapl_counter_arch_infos[0] and rapl_counter_arch_infos[1] share the + * same perf_subsys and perf_name, but with different MSR address. + * rapl_counter_arch_infos[0] is for Intel and rapl_counter_arch_infos[1] + * is for AMD. + * In this case, it is possible that multiple rapl_counter_arch_infos[] + * entries are probed just because their perf/msr is duplicate and valid. + * + * Thus need a check to avoid re-probe the same counters. + */ + if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE) + break; - /* Use perf API for this counter */ - if (!no_perf && cai->perf_name - && add_rapl_perf_counter(cpu, rci, cai, &scale, &unit) != -1) { - rci->source[cai->rci_index] = COUNTER_SOURCE_PERF; - rci->scale[cai->rci_index] = scale * cai->compat_scale; - rci->unit[cai->rci_index] = unit; - rci->flags[cai->rci_index] = cai->flags; - - /* Use MSR for this counter */ - } else if (!no_msr && cai->msr && probe_msr(cpu, cai->msr) == 0) { - rci->source[cai->rci_index] = COUNTER_SOURCE_MSR; - rci->msr[cai->rci_index] = cai->msr; - rci->msr_mask[cai->rci_index] = cai->msr_mask; - rci->msr_shift[cai->rci_index] = cai->msr_shift; - rci->unit[cai->rci_index] = RAPL_UNIT_JOULES; - rci->scale[cai->rci_index] = *cai->platform_rapl_msr_scale * cai->compat_scale; - rci->flags[cai->rci_index] = cai->flags; - } + /* Use perf API for this counter */ + if (add_rapl_perf_counter(cpu, rci, cai, &scale, &unit) != -1) { + rci->source[cai->rci_index] = COUNTER_SOURCE_PERF; + rci->scale[cai->rci_index] = scale * cai->compat_scale; + rci->unit[cai->rci_index] = unit; + rci->flags[cai->rci_index] = cai->flags; + + /* Use MSR for this counter */ + } else if (add_rapl_msr_counter(cpu, cai) >= 0) { + rci->source[cai->rci_index] = COUNTER_SOURCE_MSR; + rci->msr[cai->rci_index] = cai->msr; + rci->msr_mask[cai->rci_index] = cai->msr_mask; + rci->msr_shift[cai->rci_index] = cai->msr_shift; + rci->unit[cai->rci_index] = RAPL_UNIT_JOULES; + rci->scale[cai->rci_index] = *cai->platform_rapl_msr_scale * cai->compat_scale; + rci->flags[cai->rci_index] = cai->flags; } if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE) @@ -7913,7 +8750,7 @@ void rapl_perf_init(void) /* If any CPU has access to the counter, make it present */ if (has_counter) - BIC_PRESENT(cai->bic); + BIC_PRESENT(cai->bic_number); } free(domain_visited); @@ -7922,8 +8759,7 @@ void rapl_perf_init(void) /* Assumes msr_counter_info is populated */ static int has_amperf_access(void) { - return msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present && - msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present; + return cpuid_has_aperf_mperf && msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present && msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present; } int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *group_name) @@ -7937,65 +8773,63 @@ int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *gro return NULL; } -int add_cstate_perf_counter_(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai) +int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai) { + int ret = -1; + if (no_perf) return -1; + if (!cai->perf_name) + return -1; + int *pfd_group = get_cstate_perf_group_fd(cci, cai->perf_subsys); if (pfd_group == NULL) - return -1; + goto end; const unsigned int type = read_perf_type(cai->perf_subsys); const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name); - const int fd_counter = open_perf_counter(cpu, type, config, *pfd_group, PERF_FORMAT_GROUP); + ret = open_perf_counter(cpu, type, config, *pfd_group, PERF_FORMAT_GROUP); - if (fd_counter == -1) - return -1; + if (ret == -1) + goto end; /* If it's the first counter opened, make it a group descriptor */ if (*pfd_group == -1) - *pfd_group = fd_counter; - - return fd_counter; -} - -int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai) -{ - int ret = add_cstate_perf_counter_(cpu, cci, cai); + *pfd_group = ret; +end: if (debug >= 2) fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu); return ret; } -int add_msr_perf_counter_(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai) +int add_msr_perf_counter(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai) { + int ret = -1; + if (no_perf) return -1; + if (!cai->perf_name) + return -1; + const unsigned int type = read_perf_type(cai->perf_subsys); const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name); - const int fd_counter = open_perf_counter(cpu, type, config, cci->fd_perf, PERF_FORMAT_GROUP); + ret = open_perf_counter(cpu, type, config, cci->fd_perf, PERF_FORMAT_GROUP); - if (fd_counter == -1) - return -1; + if (ret == -1) + goto end; /* If it's the first counter opened, make it a group descriptor */ if (cci->fd_perf == -1) - cci->fd_perf = fd_counter; - - return fd_counter; -} - -int add_msr_perf_counter(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai) -{ - int ret = add_msr_perf_counter_(cpu, cci, cai); + cci->fd_perf = ret; +end: if (debug) fprintf(stderr, "%s: %s/%s: %d (cpu: %d)\n", __func__, cai->perf_subsys, cai->perf_name, ret, cpu); @@ -8029,12 +8863,12 @@ void msr_perf_init_(void) if (cai->needed) { /* Use perf API for this counter */ - if (!no_perf && cai->perf_name && add_msr_perf_counter(cpu, cci, cai) != -1) { + if (add_msr_perf_counter(cpu, cci, cai) != -1) { cci->source[cai->rci_index] = COUNTER_SOURCE_PERF; cai->present = true; /* User MSR for this counter */ - } else if (!no_msr && cai->msr && probe_msr(cpu, cai->msr) == 0) { + } else if (add_msr_counter(cpu, cai->msr) >= 0) { cci->source[cai->rci_index] = COUNTER_SOURCE_MSR; cci->msr[cai->rci_index] = cai->msr; cci->msr_mask[cai->rci_index] = cai->msr_mask; @@ -8121,8 +8955,8 @@ void cstate_perf_init_(bool soft_c1) if (cpu_is_not_allowed(cpu)) continue; - const int core_id = cpus[cpu].physical_core_id; - const int pkg_id = cpus[cpu].physical_package_id; + const int core_id = cpus[cpu].core_id; + const int pkg_id = cpus[cpu].package_id; assert(core_id < cores_visited_elems); assert(pkg_id < pkg_visited_elems); @@ -8136,19 +8970,17 @@ void cstate_perf_init_(bool soft_c1) if (!per_core && pkg_visited[pkg_id]) continue; - const bool counter_needed = BIC_IS_ENABLED(cai->bic) || - (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY)); + const bool counter_needed = BIC_IS_ENABLED(cai->bic_number) || (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY)); const bool counter_supported = (platform->supported_cstates & cai->feature_mask); if (counter_needed && counter_supported) { /* Use perf API for this counter */ - if (!no_perf && cai->perf_name && add_cstate_perf_counter(cpu, cci, cai) != -1) { + if (add_cstate_perf_counter(cpu, cci, cai) != -1) { cci->source[cai->rci_index] = COUNTER_SOURCE_PERF; /* User MSR for this counter */ - } else if (!no_msr && cai->msr && pkg_cstate_limit >= cai->pkg_cstate_limit - && probe_msr(cpu, cai->msr) == 0) { + } else if (pkg_cstate_limit >= cai->pkg_cstate_limit && add_msr_counter(cpu, cai->msr) >= 0) { cci->source[cai->rci_index] = COUNTER_SOURCE_MSR; cci->msr[cai->rci_index] = cai->msr; } @@ -8163,7 +8995,7 @@ void cstate_perf_init_(bool soft_c1) /* If any CPU has access to the counter, make it present */ if (has_counter) - BIC_PRESENT(cai->bic); + BIC_PRESENT(cai->bic_number); } free(cores_visited); @@ -8247,6 +9079,29 @@ void probe_pstates(void) for_all_cpus(print_perf_limit, ODD_COUNTERS); } +void dump_word_chars(unsigned int word) +{ + int i; + + for (i = 0; i < 4; ++i) + fprintf(outf, "%c", (word >> (i * 8)) & 0xFF); +} + +void dump_cpuid_hypervisor(void) +{ + unsigned int ebx = 0; + unsigned int ecx = 0; + unsigned int edx = 0; + + __cpuid(0x40000000, max_extended_level, ebx, ecx, edx); + + fprintf(outf, "Hypervisor: "); + dump_word_chars(ebx); + dump_word_chars(ecx); + dump_word_chars(edx); + fprintf(outf, "\n"); +} + void process_cpuid() { unsigned int eax, ebx, ecx, edx; @@ -8266,8 +9121,7 @@ void process_cpuid() hygon_genuine = 1; if (!quiet) - fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", - (char *)&ebx, (char *)&edx, (char *)&ecx, max_level); + fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", (char *)&ebx, (char *)&edx, (char *)&ecx, max_level); __cpuid(1, fms, ebx, ecx, edx); family = (fms >> 8) & 0xf; @@ -8279,12 +9133,16 @@ void process_cpuid() model += ((fms >> 16) & 0xf) << 4; ecx_flags = ecx; edx_flags = edx; + cpuid_has_hv = ecx_flags & (1 << 31); if (!no_msr) { - if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) + if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) { warnx("get_msr(UCODE)"); - else + } else { ucode_patch_valid = true; + if (!authentic_amd && !hygon_genuine) + ucode_patch >>= 32; + } } /* @@ -8296,26 +9154,28 @@ void process_cpuid() __cpuid(0x80000000, max_extended_level, ebx, ecx, edx); if (!quiet) { - fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", - family, model, stepping, family, model, stepping); + fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", family, model, stepping, family, model, stepping); if (ucode_patch_valid) - fprintf(outf, " microcode 0x%x", (unsigned int)((ucode_patch >> 32) & 0xFFFFFFFF)); + fprintf(outf, " microcode 0x%x", (unsigned int)ucode_patch); fputc('\n', outf); fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level); - fprintf(outf, "CPUID(1): %s %s %s %s %s %s %s %s %s %s\n", - ecx_flags & (1 << 0) ? "SSE3" : "-", - ecx_flags & (1 << 3) ? "MONITOR" : "-", - ecx_flags & (1 << 6) ? "SMX" : "-", - ecx_flags & (1 << 7) ? "EIST" : "-", - ecx_flags & (1 << 8) ? "TM2" : "-", - edx_flags & (1 << 4) ? "TSC" : "-", - edx_flags & (1 << 5) ? "MSR" : "-", - edx_flags & (1 << 22) ? "ACPI-TM" : "-", - edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-"); - } + fprintf(outf, "CPUID(1): %sSSE3 %sMONITOR %sSMX %sEIST %sTM2 %sHV %sTSC %sMSR %sACPI-TM %sHT %sTM\n", + ecx_flags & (1 << 0) ? "" : "No-", + ecx_flags & (1 << 3) ? "" : "No-", + ecx_flags & (1 << 6) ? "" : "No-", + ecx_flags & (1 << 7) ? "" : "No-", + ecx_flags & (1 << 8) ? "" : "No-", + cpuid_has_hv ? "" : "No-", + edx_flags & (1 << 4) ? "" : "No-", + edx_flags & (1 << 5) ? "" : "No-", + edx_flags & (1 << 22) ? "" : "No-", edx_flags & (1 << 28) ? "" : "No-", edx_flags & (1 << 29) ? "" : "No-"); + } + if (!quiet && cpuid_has_hv) + dump_cpuid_hypervisor(); probe_platform_features(family, model); + init_perf_model_support(family, model); if (!(edx_flags & (1 << 5))) errx(1, "CPUID: no MSR"); @@ -8336,7 +9196,7 @@ void process_cpuid() */ __cpuid(0x6, eax, ebx, ecx, edx); - has_aperf = ecx & (1 << 0); + cpuid_has_aperf_mperf = ecx & (1 << 0); do_dts = eax & (1 << 0); if (do_dts) BIC_PRESENT(BIC_CoreTmp); @@ -8354,19 +9214,18 @@ void process_cpuid() if (!quiet) fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, " "%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n", - has_aperf ? "" : "No-", + cpuid_has_aperf_mperf ? "" : "No-", has_turbo ? "" : "No-", do_dts ? "" : "No-", do_ptm ? "" : "No-", has_hwp ? "" : "No-", has_hwp_notify ? "" : "No-", - has_hwp_activity_window ? "" : "No-", - has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-"); + has_hwp_activity_window ? "" : "No-", has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-"); if (!quiet) decode_misc_enable_msr(); - if (max_level >= 0x7 && !quiet) { + if (max_level >= 0x7) { int has_sgx; ecx = 0; @@ -8375,9 +9234,10 @@ void process_cpuid() has_sgx = ebx & (1 << 2); - is_hybrid = edx & (1 << 15); + is_hybrid = !!(edx & (1 << 15)); - fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-"); + if (!quiet) + fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-"); if (has_sgx) decode_feature_control_msr(); @@ -8395,8 +9255,7 @@ void process_cpuid() if (ebx_tsc != 0) { if (!quiet && (ebx != 0)) - fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", - eax_crystal, ebx_tsc, crystal_hz); + fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", eax_crystal, ebx_tsc, crystal_hz); if (crystal_hz == 0) crystal_hz = platform->crystal_freq; @@ -8404,8 +9263,7 @@ void process_cpuid() if (crystal_hz) { tsc_hz = (unsigned long long)crystal_hz *ebx_tsc / eax_crystal; if (!quiet) - fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n", - tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal); + fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n", tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal); } } } @@ -8428,11 +9286,10 @@ void process_cpuid() tsc_tweak = base_hz / tsc_hz; if (!quiet) - fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", - base_mhz, max_mhz, bus_mhz); + fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", base_mhz, max_mhz, bus_mhz); } - if (has_aperf) + if (cpuid_has_aperf_mperf) aperf_mperf_multiplier = platform->need_perf_multiplier ? 1024 : 1; BIC_PRESENT(BIC_IRQ); @@ -8485,6 +9342,143 @@ void probe_pm_features(void) } /* + * has_perf_llc_access() + * + * return 1 on success, else 0 + */ +int has_perf_llc_access(void) +{ + int fd; + + if (no_perf) + return 0; + + fd = open_perf_counter(master_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP); + if (fd != -1) + close(fd); + + if (fd == -1) + warnx("Failed to access %s. Some of the counters may not be available\n" + "\tRun as root to enable them or use %s to disable the access explicitly", "perf LLC counters", "'--hide LLC' or '--no-perf'"); + + return (fd != -1); +} + +void perf_llc_init(void) +{ + int cpu; + int retval; + + if (no_perf) + return; + if (!(BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT))) + return; + + assert(fd_llc_percpu != 0); + + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + + if (cpu_is_not_allowed(cpu)) + continue; + + fd_llc_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP); + if (fd_llc_percpu[cpu] == -1) { + warnx("%s: perf REFS: failed to open counter on cpu%d", __func__, cpu); + free_fd_llc_percpu(); + return; + } + retval = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, fd_llc_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + warnx("%s: perf MISS: failed to open counter on cpu%d", __func__, cpu); + free_fd_llc_percpu(); + return; + } + } + BIC_PRESENT(BIC_LLC_MRPS); + BIC_PRESENT(BIC_LLC_HIT); +} + +void perf_l2_init(void) +{ + int cpu; + int retval; + + if (no_perf) + return; + if (!(BIC_IS_ENABLED(BIC_L2_MRPS) || BIC_IS_ENABLED(BIC_L2_HIT))) + return; + if (perf_model_support == NULL) + return; + + assert(fd_l2_percpu != 0); + + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + + if (cpu_is_not_allowed(cpu)) + continue; + + if (!is_hybrid) { + fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.uniform, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP); + if (fd_l2_percpu[cpu] == -1) { + warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.uniform, perf_model_support->first.refs); + free_fd_l2_percpu(); + return; + } + retval = open_perf_counter(cpu, perf_pmu_types.uniform, perf_model_support->first.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.uniform, perf_model_support->first.hits); + free_fd_l2_percpu(); + return; + } + continue; + } + if (perf_pcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_pcore_set)) { + fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP); + if (fd_l2_percpu[cpu] == -1) { + warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.refs); + free_fd_l2_percpu(); + return; + } + retval = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.hits); + free_fd_l2_percpu(); + return; + } + } else if (perf_ecore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_ecore_set)) { + fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.refs, -1, PERF_FORMAT_GROUP); + if (fd_l2_percpu[cpu] == -1) { + warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.ecore, perf_model_support->second.refs); + free_fd_l2_percpu(); + return; + } + retval = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.ecore, perf_model_support->second.hits); + free_fd_l2_percpu(); + return; + } + } else if (perf_lcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_lcore_set)) { + fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.refs, -1, PERF_FORMAT_GROUP); + if (fd_l2_percpu[cpu] == -1) { + warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.lcore, perf_model_support->third.refs); + free_fd_l2_percpu(); + return; + } + retval = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.lcore, perf_model_support->third.hits); + free_fd_l2_percpu(); + return; + } + } else + err(-1, "%s: cpu%d: type %d", __func__, cpu, cpus[cpu].type); + } + BIC_PRESENT(BIC_L2_MRPS); + BIC_PRESENT(BIC_L2_HIT); +} + +/* * in /dev/cpu/ return success for names that are numbers * ie. filter out ".", "..", "microcode". */ @@ -8496,31 +9490,35 @@ int dir_filter(const struct dirent *dirp) return 0; } -char *possible_file = "/sys/devices/system/cpu/possible"; -char possible_buf[1024]; - -int initialize_cpu_possible_set(void) +int set_thread_siblings(struct cpu_topology *thiscpu) { - FILE *fp; + char path[80]; + int cpu = thiscpu->cpu_id; + size_t size; + int ht_id = 0; + int i; - fp = fopen(possible_file, "r"); - if (!fp) { - warn("open %s", possible_file); + thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1)); + if (thiscpu->ht_id < 0) + thiscpu->ht_id = 0; /* first CPU in core */ + if (!thiscpu->put_ids) return -1; - } - if (fread(possible_buf, sizeof(char), 1024, fp) == 0) { - warn("read %s", possible_file); - goto err; - } - if (parse_cpu_str(possible_buf, cpu_possible_set, cpu_possible_setsize)) { - warnx("%s: cpu str malformat %s\n", possible_file, cpu_effective_str); - goto err; - } - return 0; -err: - fclose(fp); - return -1; + size = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); + CPU_ZERO_S(size, thiscpu->put_ids); + + sprintf(path, "/sys/devices/system/cpu/cpu%d/topology", cpu); + + initialize_cpu_set_from_sysfs(thiscpu->put_ids, path, "thread_siblings_list"); + + for (i = 0; i <= topo.max_cpu_num; ++i) + if (CPU_ISSET_S(i, size, thiscpu->put_ids)) { + cpus[i].ht_id = ht_id; + cpus[cpu].ht_sibling_cpu_id[ht_id] = i; + ht_id += 1; + } + + return (ht_id - 1); } void topology_probe(bool startup) @@ -8553,6 +9551,8 @@ void topology_probe(bool startup) cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); CPU_ZERO_S(cpu_present_setsize, cpu_present_set); for_all_proc_cpus(mark_cpu_present); + if (debug) + print_cpu_set("present set", cpu_present_set); /* * Allocate and initialize cpu_possible_set @@ -8562,7 +9562,9 @@ void topology_probe(bool startup) err(3, "CPU_ALLOC"); cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set); - initialize_cpu_possible_set(); + initialize_cpu_set_from_sysfs(cpu_possible_set, "/sys/devices/system/cpu", "possible"); + if (debug) + print_cpu_set("possible set", cpu_possible_set); /* * Allocate and initialize cpu_effective_set @@ -8573,6 +9575,8 @@ void topology_probe(bool startup) cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set); update_effective_set(startup); + if (debug) + print_cpu_set("effective set", cpu_effective_set); /* * Allocate and initialize cpu_allowed_set @@ -8616,6 +9620,8 @@ void topology_probe(bool startup) CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set); } + if (debug) + print_cpu_set("allowed set", cpu_allowed_set); if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set)) err(-ENODEV, "No valid cpus found"); @@ -8630,14 +9636,15 @@ void topology_probe(bool startup) cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set); - for_all_proc_cpus(init_thread_id); + for_all_proc_cpus(clear_ht_id); for_all_proc_cpus(set_cpu_hybrid_type); /* * For online cpus - * find max_core_id, max_package_id + * find max_core_id, max_package_id, num_cores (per system) */ + topo.min_module_id = 0x7FFFFFFF; for (i = 0; i <= topo.max_cpu_num; ++i) { int siblings; @@ -8647,50 +9654,70 @@ void topology_probe(bool startup) continue; } - cpus[i].logical_cpu_id = i; + cpus[i].cpu_id = i; /* get package information */ - cpus[i].physical_package_id = get_physical_package_id(i); - if (cpus[i].physical_package_id > max_package_id) - max_package_id = cpus[i].physical_package_id; + cpus[i].package_id = get_package_id(i); + if (cpus[i].package_id > max_package_id) + max_package_id = cpus[i].package_id; /* get die information */ cpus[i].die_id = get_die_id(i); if (cpus[i].die_id > topo.max_die_id) topo.max_die_id = cpus[i].die_id; + /* get l3 information */ + cpus[i].l3_id = get_l3_id(i); + if (cpus[i].l3_id > topo.max_l3_id) + topo.max_l3_id = cpus[i].l3_id; + /* get numa node information */ cpus[i].physical_node_id = get_physical_node_id(&cpus[i]); if (cpus[i].physical_node_id > topo.max_node_num) topo.max_node_num = cpus[i].physical_node_id; + /* get module information */ + cpus[i].module_id = get_module_id(i); + if (cpus[i].module_id > topo.max_module_id) + topo.max_module_id = cpus[i].module_id; + if (cpus[i].module_id < topo.min_module_id) + topo.min_module_id = cpus[i].module_id; + /* get core information */ - cpus[i].physical_core_id = get_core_id(i); - if (cpus[i].physical_core_id > max_core_id) - max_core_id = cpus[i].physical_core_id; + cpus[i].core_id = get_core_id(i); + if (cpus[i].core_id > max_core_id) + max_core_id = cpus[i].core_id; /* get thread information */ - siblings = get_thread_siblings(&cpus[i]); + siblings = set_thread_siblings(&cpus[i]); if (siblings > max_siblings) max_siblings = siblings; - if (cpus[i].thread_id == 0) + if (cpus[i].ht_id == 0) topo.num_cores++; } - topo.max_core_id = max_core_id; + topo.max_core_id = max_core_id; /* within a package */ topo.max_package_id = max_package_id; - topo.cores_per_node = max_core_id + 1; + topo.cores_per_pkg = max_core_id + 1; if (debug > 1) - fprintf(outf, "max_core_id %d, sizing for %d cores per package\n", max_core_id, topo.cores_per_node); + fprintf(outf, "max_core_id %d, sizing for %d cores per package\n", max_core_id, topo.cores_per_pkg); if (!summary_only) BIC_PRESENT(BIC_Core); + if (debug > 1) + fprintf(outf, "min_module_id %d max_module_id %d\n", topo.min_module_id, topo.max_module_id); + if (!summary_only && (topo.min_module_id != topo.max_module_id)) + BIC_PRESENT(BIC_Module); + topo.num_die = topo.max_die_id + 1; if (debug > 1) fprintf(outf, "max_die_id %d, sizing for %d die\n", topo.max_die_id, topo.num_die); if (!summary_only && topo.num_die > 1) BIC_PRESENT(BIC_Die); + if (!summary_only && topo.max_l3_id > 0) + BIC_PRESENT(BIC_L3); + topo.num_packages = max_package_id + 1; if (debug > 1) fprintf(outf, "max_package_id %d, sizing for %d packages\n", max_package_id, topo.num_packages); @@ -8711,46 +9738,66 @@ void topology_probe(bool startup) return; for (i = 0; i <= topo.max_cpu_num; ++i) { + int ht_id; + if (cpu_is_not_present(i)) continue; fprintf(outf, - "cpu %d pkg %d die %d node %d lnode %d core %d thread %d\n", - i, cpus[i].physical_package_id, cpus[i].die_id, - cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].physical_core_id, cpus[i].thread_id); + "cpu %d pkg %d die %d l3 %d node %d lnode %d module 0x%x core %d ht_id %d", + i, cpus[i].package_id, cpus[i].die_id, cpus[i].l3_id, + cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].module_id, cpus[i].core_id, cpus[i].ht_id); + fprintf(outf, " siblings"); + for (ht_id = 0; ht_id <= MAX_HT_ID; ++ht_id) + fprintf(outf, " %d", cpus[i].ht_sibling_cpu_id[ht_id]); + fprintf(outf, "\n"); } } -void allocate_counters(struct thread_data **t, struct core_data **c, struct pkg_data **p) +void allocate_counters_1(struct counters *counters) +{ + counters->threads = calloc(1, sizeof(struct thread_data)); + if (counters->threads == NULL) + goto error; + + counters->cores = calloc(1, sizeof(struct core_data)); + if (counters->cores == NULL) + goto error; + + counters->packages = calloc(1, sizeof(struct pkg_data)); + if (counters->packages == NULL) + goto error; + + return; +error: + err(1, "calloc counters_1"); +} + +void allocate_counters(struct counters *counters) { int i; - int num_cores = topo.cores_per_node * topo.nodes_per_pkg * topo.num_packages; - int num_threads = topo.threads_per_core * num_cores; + int num_cores = topo.cores_per_pkg * topo.num_packages; - *t = calloc(num_threads, sizeof(struct thread_data)); - if (*t == NULL) + counters->threads = calloc(topo.max_cpu_num + 1, sizeof(struct thread_data)); + if (counters->threads == NULL) goto error; - for (i = 0; i < num_threads; i++) - (*t)[i].cpu_id = -1; + for (i = 0; i < topo.max_cpu_num + 1; i++) + (counters->threads)[i].cpu_id = -1; - *c = calloc(num_cores, sizeof(struct core_data)); - if (*c == NULL) + counters->cores = calloc(num_cores, sizeof(struct core_data)); + if (counters->cores == NULL) goto error; - for (i = 0; i < num_cores; i++) { - (*c)[i].core_id = -1; - (*c)[i].base_cpu = -1; - } + for (i = 0; i < num_cores; i++) + (counters->cores)[i].first_cpu = -1; - *p = calloc(topo.num_packages, sizeof(struct pkg_data)); - if (*p == NULL) + counters->packages = calloc(topo.num_packages, sizeof(struct pkg_data)); + if (counters->packages == NULL) goto error; - for (i = 0; i < topo.num_packages; i++) { - (*p)[i].package_id = i; - (*p)[i].base_cpu = -1; - } + for (i = 0; i < topo.num_packages; i++) + (counters->packages)[i].first_cpu = -1; return; error: @@ -8760,17 +9807,15 @@ error: /* * init_counter() * - * set FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE + * set t->cpu_id, FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE */ void init_counter(struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base, int cpu_id) { - int pkg_id = cpus[cpu_id].physical_package_id; + int pkg_id = cpus[cpu_id].package_id; int node_id = cpus[cpu_id].logical_node_id; - int core_id = cpus[cpu_id].physical_core_id; - int thread_id = cpus[cpu_id].thread_id; + int core_id = cpus[cpu_id].core_id; struct thread_data *t; struct core_data *c; - struct pkg_data *p; /* Workaround for systems where physical_node_id==-1 * and logical_node_id==(-1 - topo.num_cpus) @@ -8778,20 +9823,17 @@ void init_counter(struct thread_data *thread_base, struct core_data *core_base, if (node_id < 0) node_id = 0; - t = GET_THREAD(thread_base, thread_id, core_id, node_id, pkg_id); - c = GET_CORE(core_base, core_id, node_id, pkg_id); - p = GET_PKG(pkg_base, pkg_id); + t = &thread_base[cpu_id]; + c = &core_base[GLOBAL_CORE_ID(core_id, pkg_id)]; t->cpu_id = cpu_id; if (!cpu_is_not_allowed(cpu_id)) { - if (c->base_cpu < 0) - c->base_cpu = t->cpu_id; - if (p->base_cpu < 0) - p->base_cpu = t->cpu_id; - } - c->core_id = core_id; - p->package_id = pkg_id; + if (c->first_cpu < 0) + c->first_cpu = t->cpu_id; + if (pkg_base[pkg_id].first_cpu < 0) + pkg_base[pkg_id].first_cpu = t->cpu_id; + } } int initialize_counters(int cpu_id) @@ -8831,12 +9873,12 @@ void allocate_irq_buffers(void) err(-1, "calloc %d NMI", topo.max_cpu_num + 1); } -int update_topo(struct thread_data *t, struct core_data *c, struct pkg_data *p) +int update_topo(PER_THREAD_PARAMS) { topo.allowed_cpus++; - if ((int)t->cpu_id == c->base_cpu) + if ((int)t->cpu_id == c->first_cpu) topo.allowed_cores++; - if ((int)t->cpu_id == p->base_cpu) + if ((int)t->cpu_id == p->first_cpu) topo.allowed_packages++; return 0; @@ -8848,6 +9890,8 @@ void topology_update(void) topo.allowed_cores = 0; topo.allowed_packages = 0; for_all_cpus(update_topo, ODD_COUNTERS); + if (debug) + fprintf(stderr, "allowed_cpus %d allowed_cores %d allowed_packages %d\n", topo.allowed_cpus, topo.allowed_cores, topo.allowed_packages); } void setup_all_buffers(bool startup) @@ -8855,23 +9899,24 @@ void setup_all_buffers(bool startup) topology_probe(startup); allocate_irq_buffers(); allocate_fd_percpu(); - allocate_counters(&thread_even, &core_even, &package_even); - allocate_counters(&thread_odd, &core_odd, &package_odd); + allocate_counters_1(&average); + allocate_counters(&even); + allocate_counters(&odd); allocate_output_buffer(); for_all_proc_cpus(initialize_counters); topology_update(); } -void set_base_cpu(void) +void set_master_cpu(void) { int i; for (i = 0; i < topo.max_cpu_num + 1; ++i) { if (cpu_is_not_allowed(i)) continue; - base_cpu = i; + master_cpu = i; if (debug > 1) - fprintf(outf, "base_cpu = %d\n", base_cpu); + fprintf(outf, "master_cpu = %d\n", master_cpu); return; } err(-ENODEV, "No valid cpus found"); @@ -8889,7 +9934,7 @@ bool has_added_counters(void) void check_msr_access(void) { - check_dev_msr(); + check_msr_driver(); check_msr_permission(); if (no_msr) @@ -8898,8 +9943,16 @@ void check_msr_access(void) void check_perf_access(void) { - if (no_perf || !BIC_IS_ENABLED(BIC_IPC) || !has_instr_count_access()) - bic_enabled &= ~BIC_IPC; + if (BIC_IS_ENABLED(BIC_IPC)) + if (!has_perf_instr_count_access()) + no_perf = 1; + + if (BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT)) + if (!has_perf_llc_access()) + no_perf = 1; + + if (no_perf) + bic_disable_perf_access(); } bool perf_has_hybrid_devices(void) @@ -9009,22 +10062,20 @@ int added_perf_counters_init_(struct perf_counter_info *pinfo) perf_device = "cpu_atom"; break; - default: /* Don't change, we will probably fail and report a problem soon. */ + default: /* Don't change, we will probably fail and report a problem soon. */ break; } } perf_type = read_perf_type(perf_device); if (perf_type == (unsigned int)-1) { - warnx("%s: perf/%s/%s: failed to read %s", - __func__, perf_device, pinfo->event, "type"); + warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "type"); continue; } perf_config = read_perf_config(perf_device, pinfo->event); if (perf_config == (unsigned int)-1) { - warnx("%s: perf/%s/%s: failed to read %s", - __func__, perf_device, pinfo->event, "config"); + warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "config"); continue; } @@ -9035,8 +10086,7 @@ int added_perf_counters_init_(struct perf_counter_info *pinfo) fd_perf = open_perf_counter(cpu, perf_type, perf_config, -1, 0); if (fd_perf == -1) { - warnx("%s: perf/%s/%s: failed to open counter on cpu%d", - __func__, perf_device, pinfo->event, cpu); + warnx("%s: perf/%s/%s: failed to open counter on cpu%d", __func__, perf_device, pinfo->event, cpu); continue; } @@ -9045,8 +10095,7 @@ int added_perf_counters_init_(struct perf_counter_info *pinfo) pinfo->scale = perf_scale; if (debug) - fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n", - perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]); + fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n", perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]); } pinfo = pinfo->next; @@ -9119,7 +10168,7 @@ struct pmt_mmio *pmt_mmio_open(unsigned int target_guid) return NULL; } - for ( ; entry != NULL; entry = pmt_diriter_next(&pmt_iter)) { + for (; entry != NULL; entry = pmt_diriter_next(&pmt_iter)) { if (fstatat(dirfd(pmt_iter.dir), entry->d_name, &st, 0) == -1) break; @@ -9360,8 +10409,7 @@ int pmt_add_counter(unsigned int guid, unsigned int seq, const char *name, enum } if (conflict) { - fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n", - __func__, name); + fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n", __func__, name); exit(1); } @@ -9382,8 +10430,7 @@ void pmt_init(void) if (BIC_IS_ENABLED(BIC_Diec6)) { pmt_add_counter(PMT_MTL_DC6_GUID, PMT_MTL_DC6_SEQ, "Die%c6", PMT_TYPE_XTAL_TIME, - PMT_COUNTER_MTL_DC6_LSB, PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET, - SCOPE_PACKAGE, FORMAT_DELTA, 0, PMT_OPEN_TRY); + PMT_COUNTER_MTL_DC6_LSB, PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET, SCOPE_PACKAGE, FORMAT_DELTA, 0, PMT_OPEN_TRY); } if (BIC_IS_ENABLED(BIC_CPU_c1e)) { @@ -9404,8 +10451,7 @@ void pmt_init(void) * CWF with newer firmware might require a PMT_TYPE_XTAL_TIME intead of PMT_TYPE_TCORE_CLOCK. */ pmt_add_counter(PMT_CWF_MC1E_GUID, seq, "CPU%c1e", PMT_TYPE_TCORE_CLOCK, - PMT_COUNTER_CWF_MC1E_LSB, PMT_COUNTER_CWF_MC1E_MSB, offset, SCOPE_CPU, - FORMAT_DELTA, cpu_num, PMT_OPEN_TRY); + PMT_COUNTER_CWF_MC1E_LSB, PMT_COUNTER_CWF_MC1E_MSB, offset, SCOPE_CPU, FORMAT_DELTA, cpu_num, PMT_OPEN_TRY); /* * Rather complex logic for each time we go to the next loop iteration, @@ -9445,7 +10491,7 @@ next_loop_iter: void turbostat_init() { setup_all_buffers(true); - set_base_cpu(); + set_master_cpu(); check_msr_access(); check_perf_access(); process_cpuid(); @@ -9455,13 +10501,15 @@ void turbostat_init() linux_perf_init(); rapl_perf_init(); cstate_perf_init(); + perf_llc_init(); + perf_l2_init(); added_perf_counters_init(); pmt_init(); for_all_cpus(get_cpu_type, ODD_COUNTERS); for_all_cpus(get_cpu_type, EVEN_COUNTERS); - if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(base_cpu) != -1) + if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(master_cpu) != -1) BIC_PRESENT(BIC_IPC); /* @@ -9469,8 +10517,8 @@ void turbostat_init() * disable more BICs, since it can't be reported accurately. */ if (platform->enable_tsc_tweak && !has_base_hz) { - bic_enabled &= ~BIC_Busy; - bic_enabled &= ~BIC_Bzy_MHz; + CLR_BIC(BIC_Busy, &bic_enabled); + CLR_BIC(BIC_Bzy_MHz, &bic_enabled); } } @@ -9528,6 +10576,7 @@ int fork_it(char **argv) timersub(&tv_odd, &tv_even, &tv_delta); if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) fprintf(outf, "%s: Counter reset detected\n", progname); + delta_platform(&platform_counters_odd, &platform_counters_even); compute_average(EVEN_COUNTERS); format_all_counters(EVEN_COUNTERS); @@ -9559,7 +10608,7 @@ int get_and_dump_counters(void) void print_version() { - fprintf(outf, "turbostat version 2025.02.02 - Len Brown <lenb@kernel.org>\n"); + fprintf(outf, "turbostat version 2026.04.21 - Len Brown <lenb@kernel.org>\n"); } #define COMMAND_LINE_SIZE 2048 @@ -9592,15 +10641,14 @@ struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name) for (mp = head; mp; mp = mp->next) { if (debug) fprintf(stderr, "%s: %s %s\n", __func__, name, mp->name); - if (!strncmp(name, mp->name, strlen(mp->name))) + if (!strcmp(name, mp->name)) return mp; } return NULL; } int add_counter(unsigned int msr_num, char *path, char *name, - unsigned int width, enum counter_scope scope, - enum counter_type type, enum counter_format format, int flags, int id) + unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int id) { struct msr_counter *msrp; @@ -9709,9 +10757,7 @@ int add_counter(unsigned int msr_num, char *path, char *name, struct perf_counter_info *make_perf_counter_info(const char *perf_device, const char *perf_event, const char *name, - unsigned int width, - enum counter_scope scope, - enum counter_type type, enum counter_format format) + unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format) { struct perf_counter_info *pinfo; @@ -9786,8 +10832,7 @@ int add_perf_counter(const char *perf_device, const char *perf_event, const char // FIXME: we might not have debug here yet if (debug) - fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n", - __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope); + fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n", __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope); return 0; } @@ -9856,6 +10901,10 @@ void parse_add_command_msr(char *add_command) format = FORMAT_RAW; goto next; } + if (!strncmp(add_command, "average", strlen("average"))) { + format = FORMAT_AVERAGE; + goto next; + } if (!strncmp(add_command, "delta", strlen("delta"))) { format = FORMAT_DELTA; goto next; @@ -9952,8 +11001,7 @@ int pmt_parse_from_path(const char *target_path, unsigned int *out_guid, unsigne pmt_diriter_init(&pmt_iter); - for (dirname = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH); dirname != NULL; - dirname = pmt_diriter_next(&pmt_iter)) { + for (dirname = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH); dirname != NULL; dirname = pmt_diriter_next(&pmt_iter)) { fd_telem_dir = openat(dirfd(pmt_iter.dir), dirname->d_name, O_RDONLY | O_DIRECTORY); if (fd_telem_dir == -1) @@ -9965,8 +11013,7 @@ int pmt_parse_from_path(const char *target_path, unsigned int *out_guid, unsigne } if (fstat(fd_telem_dir, &stat) == -1) { - fprintf(stderr, "%s: Failed to stat %s directory: %s", __func__, - dirname->d_name, strerror(errno)); + fprintf(stderr, "%s: Failed to stat %s directory: %s", __func__, dirname->d_name, strerror(errno)); continue; } @@ -10015,7 +11062,7 @@ void parse_add_command_pmt(char *add_command) unsigned int lsb; unsigned int msb; unsigned int guid; - unsigned int seq = 0; /* By default, pick first file in a sequence with a given GUID. */ + unsigned int seq = 0; /* By default, pick first file in a sequence with a given GUID. */ unsigned int domain_id; enum counter_scope scope = 0; enum pmt_datatype type = PMT_TYPE_RAW; @@ -10062,8 +11109,7 @@ void parse_add_command_pmt(char *add_command) } if (!has_scope) { - printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n", - __func__); + printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n", __func__); exit(1); } @@ -10128,13 +11174,18 @@ next: has_format = true; } + if (strcmp("average", format_name) == 0) { + format = FORMAT_AVERAGE; + has_format = true; + } + if (strcmp("delta", format_name) == 0) { format = FORMAT_DELTA; has_format = true; } if (!has_format) { - fprintf(stderr, "%s: Invalid format %s. Expected raw or delta\n", __func__, format_name); + fprintf(stderr, "%s: Invalid format %s. Expected raw, average or delta\n", __func__, format_name); exit(1); } } @@ -10179,8 +11230,7 @@ next: } if (direct_path && has_guid) { - printf("%s: path and guid+seq parameters are mutually exclusive\n" - "notice: passed guid=0x%x and path=%s\n", __func__, guid, direct_path); + printf("%s: path and guid+seq parameters are mutually exclusive\nnotice: passed guid=0x%x and path=%s\n", __func__, guid, direct_path); exit(1); } @@ -10224,8 +11274,10 @@ int is_deferred_add(char *name) int i; for (i = 0; i < deferred_add_index; ++i) - if (!strcmp(name, deferred_add_names[i])) + if (!strcmp(name, deferred_add_names[i])) { + deferred_add_consumed |= (1 << i); return 1; + } return 0; } @@ -10234,22 +11286,46 @@ int is_deferred_skip(char *name) int i; for (i = 0; i < deferred_skip_index; ++i) - if (!strcmp(name, deferred_skip_names[i])) + if (!strcmp(name, deferred_skip_names[i])) { + deferred_skip_consumed |= (1 << i); return 1; + } return 0; } -void probe_sysfs(void) +void verify_deferred_consumed(void) +{ + int i; + int fail = 0; + + for (i = 0; i < deferred_add_index; ++i) { + if (!(deferred_add_consumed & (1 << i))) { + warnx("Counter '%s' can not be added.", deferred_add_names[i]); + fail++; + } + } + for (i = 0; i < deferred_skip_index; ++i) { + if (!(deferred_skip_consumed & (1 << i))) { + warnx("Counter '%s' can not be skipped.", deferred_skip_names[i]); + fail++; + } + } + if (fail) + exit(-EINVAL); +} + +void probe_cpuidle_residency(void) { char path[64]; char name_buf[16]; FILE *input; int state; + int min_state = 1024, max_state = 0; char *sp; for (state = 10; state >= 0; --state) { - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state); input = fopen(path, "r"); if (input == NULL) continue; @@ -10269,43 +11345,94 @@ void probe_sysfs(void) sprintf(path, "cpuidle/state%d/time", state); - if (!DO_BIC(BIC_sysfs) && !is_deferred_add(name_buf)) + if (!DO_BIC(BIC_pct_idle) && !is_deferred_add(name_buf)) continue; if (is_deferred_skip(name_buf)) continue; - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0); + add_counter(0, path, name_buf, 32, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0); + + if (state > max_state) + max_state = state; + if (state < min_state) + min_state = state; } +} + +static bool cpuidle_counter_wanted(char *name) +{ + if (is_deferred_skip(name)) + return false; + + return DO_BIC(BIC_cpuidle) || is_deferred_add(name); +} + +void probe_cpuidle_counts(void) +{ + char path[64]; + char name_buf[16]; + FILE *input; + int state; + int min_state = 1024, max_state = 0; + char *sp; + + if (!DO_BIC(BIC_cpuidle) && !deferred_add_index) + return; for (state = 10; state >= 0; --state) { - sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state); + sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state); input = fopen(path, "r"); if (input == NULL) continue; if (!fgets(name_buf, sizeof(name_buf), input)) err(1, "%s: failed to read file", path); - /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */ - sp = strchr(name_buf, '-'); - if (!sp) - sp = strchrnul(name_buf, '\n'); - *sp = '\0'; fclose(input); remove_underbar(name_buf); - sprintf(path, "cpuidle/state%d/usage", state); + /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */ + sp = strchr(name_buf, '-'); + if (!sp) + sp = strchrnul(name_buf, '\n'); - if (!DO_BIC(BIC_sysfs) && !is_deferred_add(name_buf)) - continue; + /* + * The 'below' sysfs file always contains 0 for the deepest state (largest index), + * do not add it. + */ + if (state != max_state) { + /* + * Add 'C1+' for C1, and so on. The 'below' sysfs file always contains 0 for + * the last state, so do not add it. + */ + *sp = '+'; + *(sp + 1) = '\0'; + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/below", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } + } - if (is_deferred_skip(name_buf)) - continue; + *sp = '\0'; + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/usage", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + /* + * The 'above' sysfs file always contains 0 for the shallowest state (smallest + * index), do not add it. + */ + if (state != min_state) { + *sp = '-'; + *(sp + 1) = '\0'; + if (cpuidle_counter_wanted(name_buf)) { + sprintf(path, "cpuidle/state%d/above", state); + add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0); + } + } } - } /* @@ -10383,7 +11510,7 @@ void cmdline(int argc, char **argv) * Parse some options early, because they may make other options invalid, * like adding the MSR counter with --add and at the same time using --no-msr. */ - while ((opt = getopt_long_only(argc, argv, "+MPn:", long_options, &option_index)) != -1) { + while ((opt = getopt_long_only(argc, argv, "+:MP", long_options, &option_index)) != -1) { switch (opt) { case 'M': no_msr = 1; @@ -10397,7 +11524,7 @@ void cmdline(int argc, char **argv) } optind = 0; - while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:o:qMST:v", long_options, &option_index)) != -1) { + while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:N:o:qMPST:v", long_options, &option_index)) != -1) { switch (opt) { case 'a': parse_add_command(optarg); @@ -10415,25 +11542,31 @@ void cmdline(int argc, char **argv) no_perf = 1; break; case 'e': - /* --enable specified counter */ - bic_enabled = bic_enabled | bic_lookup(optarg, SHOW_LIST); + /* --enable specified counter, without clearning existing list */ + bic_lookup(&bic_enabled, optarg, SHOW_LIST); break; case 'f': force_load++; break; case 'd': debug++; - ENABLE_BIC(BIC_DISABLED_BY_DEFAULT); + bic_set_all(&bic_enabled); break; case 'H': /* * --hide: do not show those specified * multiple invocations simply clear more bits in enabled mask */ - bic_enabled &= ~bic_lookup(optarg, HIDE_LIST); + { + cpu_set_t bic_group_hide; + + BIC_INIT(&bic_group_hide); + + bic_lookup(&bic_group_hide, optarg, HIDE_LIST); + bic_clear_bits(&bic_enabled, &bic_group_hide); + } break; case 'h': - default: help(); exit(1); case 'i': @@ -10454,7 +11587,7 @@ void cmdline(int argc, char **argv) rapl_joules++; break; case 'l': - ENABLE_BIC(BIC_DISABLED_BY_DEFAULT); + bic_set_all(&bic_enabled); list_header_only++; quiet++; break; @@ -10469,20 +11602,18 @@ void cmdline(int argc, char **argv) /* Parsed earlier */ break; case 'n': - num_iterations = strtod(optarg, NULL); + num_iterations = strtoul(optarg, NULL, 0); + errno = 0; - if (num_iterations <= 0) { - fprintf(outf, "iterations %d should be positive number\n", num_iterations); - exit(2); - } + if (errno || num_iterations == 0) + errx(-1, "invalid iteration count: %s", optarg); break; case 'N': - header_iterations = strtod(optarg, NULL); + header_iterations = strtoul(optarg, NULL, 0); + errno = 0; - if (header_iterations <= 0) { - fprintf(outf, "iterations %d should be positive number\n", header_iterations); - exit(2); - } + if (errno || header_iterations == 0) + errx(-1, "invalid header iteration count: %s", optarg); break; case 's': /* @@ -10491,9 +11622,8 @@ void cmdline(int argc, char **argv) * subsequent invocations can add to it. */ if (shown == 0) - bic_enabled = bic_lookup(optarg, SHOW_LIST); - else - bic_enabled |= bic_lookup(optarg, SHOW_LIST); + BIC_INIT(&bic_enabled); + bic_lookup(&bic_enabled, optarg, SHOW_LIST); shown = 1; break; case 'S': @@ -10506,6 +11636,9 @@ void cmdline(int argc, char **argv) print_version(); exit(0); break; + default: + help(); + exit(1); } } } @@ -10530,6 +11663,8 @@ int main(int argc, char **argv) { int fd, ret; + bic_groups_init(); + fd = open("/sys/fs/cgroup/cgroup.procs", O_WRONLY); if (fd < 0) goto skip_cgroup_setting; @@ -10549,7 +11684,10 @@ skip_cgroup_setting: print_bootcmd(); } - probe_sysfs(); + probe_cpuidle_residency(); + probe_cpuidle_counts(); + + verify_deferred_consumed(); if (!getuid()) set_rlimit(); diff --git a/tools/power/x86/x86_energy_perf_policy/Makefile b/tools/power/x86/x86_energy_perf_policy/Makefile index 666b325a62a2..d18284667400 100644 --- a/tools/power/x86/x86_energy_perf_policy/Makefile +++ b/tools/power/x86/x86_energy_perf_policy/Makefile @@ -1,8 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 CC = $(CROSS_COMPILE)gcc -BUILD_OUTPUT := $(CURDIR) +BUILD_OUTPUT := $(CURDIR) PREFIX := /usr DESTDIR := +DAY := $(shell date +%Y.%m.%d) +SNAPSHOT = x86_energy_perf_policy-$(DAY) + + ifeq ("$(origin O)", "command line") BUILD_OUTPUT := $(O) @@ -27,3 +31,26 @@ install : x86_energy_perf_policy install -d $(DESTDIR)$(PREFIX)/share/man/man8 install -m 644 x86_energy_perf_policy.8 $(DESTDIR)$(PREFIX)/share/man/man8 +snapshot: x86_energy_perf_policy + @rm -rf $(SNAPSHOT) + @mkdir $(SNAPSHOT) + @cp x86_energy_perf_policy Makefile x86_energy_perf_policy.c x86_energy_perf_policy.8 $(SNAPSHOT) + + @sed -e 's/^#include <linux\/bits.h>/#include "bits.h"/' -e 's/u64/unsigned long long/' ../../../../arch/x86/include/asm/msr-index.h > $(SNAPSHOT)/msr-index.h + @echo '#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))' >> $(SNAPSHOT)/msr-index.h + @echo "#define BIT(x) (1 << (x))" > $(SNAPSHOT)/bits.h + @echo "#define BIT_ULL(nr) (1ULL << (nr))" >> $(SNAPSHOT)/bits.h + @echo "#define GENMASK(h, l) (((~0UL) << (l)) & (~0UL >> (sizeof(long) * 8 - 1 - (h))))" >> $(SNAPSHOT)/bits.h + @echo "#define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (sizeof(long long) * 8 - 1 - (h))))" >> $(SNAPSHOT)/bits.h + + @echo '#define BUILD_BUG_ON(cond) do { enum { compile_time_check ## __COUNTER__ = 1/(!(cond)) }; } while (0)' > $(SNAPSHOT)/build_bug.h + @echo '#define __must_be_array(arr) 0' >> $(SNAPSHOT)/build_bug.h + + @echo PWD=. > $(SNAPSHOT)/Makefile + @echo "CFLAGS += -DMSRHEADER='\"msr-index.h\"'" >> $(SNAPSHOT)/Makefile + @echo "CFLAGS += -DBUILD_BUG_HEADER='\"build_bug.h\"'" >> $(SNAPSHOT)/Makefile + @sed -e's/.*MSRHEADER.*//' Makefile >> $(SNAPSHOT)/Makefile + + @rm -f $(SNAPSHOT).tar.gz + tar cvzf $(SNAPSHOT).tar.gz $(SNAPSHOT) + diff --git a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.8 b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.8 index 78c6361898b1..836553e9a92c 100644 --- a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.8 +++ b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.8 @@ -2,7 +2,7 @@ .\" Distributed under the GPL, Copyleft 1994. .TH X86_ENERGY_PERF_POLICY 8 .SH NAME -x86_energy_perf_policy \- Manage Energy vs. Performance Policy via x86 Model Specific Registers +x86_energy_perf_policy \- Manage Energy vs. Performance Policy .SH SYNOPSIS .B x86_energy_perf_policy .RB "[ options ] [ scope ] [field \ value]" @@ -15,13 +15,20 @@ x86_energy_perf_policy \- Manage Energy vs. Performance Policy via x86 Model Spe .br .RB "other: (\-\-force | \-\-hwp-enable | \-\-turbo-enable) value)" .br +.RB "soc-slider: --soc-slider-balance # | --soc-slider-offset # | --platform-profile <name>" +.br .RB "value: # | default | performance | balance-performance | balance-power | power" .SH DESCRIPTION \fBx86_energy_perf_policy\fP displays and updates energy-performance policy settings specific to -Intel Architecture Processors. Settings are accessed via Model Specific Register (MSR) -updates, no matter if the Linux cpufreq sub-system is enabled or not. +Intel Architecture Processors. It summarizes settings available +in standard Linux interfaces (eg. cpufreq), +and also decodes underlying Model Specific Register (MSRs). +While \fBx86_energy_perf_policy\fP can manage energy-performance policy +using only MSR access, it prefers standard +Linux kernel interfaces, when they are available. +.SH BACKGROUND Policy in MSR_IA32_ENERGY_PERF_BIAS (EPB) may affect a wide range of hardware decisions, such as how aggressively the hardware enters and exits CPU idle states (C-states) @@ -149,6 +156,26 @@ level on this processor, specified in multiples of 100 MHz. in the sliding window that HWP uses to maintain average frequency. This parameter is meaningful only when the "desired" field above is non-zero. Default is 0, allowing the HW to choose. +.SH SOC SLIDER OPTIONS +.PP +Note that the Platform Profile Name must be "SoC Slider", and the +Platform Profile must be "balanced" for the --soc-slider-balance +and --soc-slider-offset options to take effect. +.PP +\fB--soc-slider-balance #\fP write numeric value to the SoC Slider. +Values range from 0 to 6. +Lower values result in higher performance, +and higher values improve energy efficiency. +Actual values are model specific. +.PP +\fB--soc-slider-offset #\fP write the numeric value to the Soc Slider Offset. +The slider offset is the maximum value that software allows the SoC to +autonomously add to the SoC Slider to improve energy efficiency. +The value 0 prohibits the SoC from autonomously changing the slider. +.PP +\fB--platform-profile <name>"\fP set the platform profile to <name>. +Available choices are in platform-profile-0/choices. The Soc Slider +driver currently supports "low-power", "balanced", and "performance". .SH OTHER OPTIONS .PP \fB-f, --force\fP writes the specified values without bounds checking. @@ -200,7 +227,13 @@ runs only as root. .SH FILES .ta .nf -/dev/cpu/*/msr +EPB: /sys/devices/system/cpu/cpu*/power/energy_perf_bias +EPP: /sys/devices/system/cpu/cpu*/cpufreq/energy_performance_preference +MSR: /dev/cpu/*/msr +Platform Profile Name: /sys/class/platform-profile/platform-profile-0/name +Platform Profile: /sys/class/platform-profile/platform-profile-0/profile +SOC Slider Balanced: /sys/module/processor_thermal_soc_slider/parameters/slider_balance +SOC Slider Balanced Offset: /sys/module/processor_thermal_soc_slider/parameters/slider_offset .fi .SH "SEE ALSO" .nf diff --git a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c index ebda9c366b2b..0dc959e30076 100644 --- a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c +++ b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c @@ -4,7 +4,7 @@ * policy preference bias on recent X86 processors. */ /* - * Copyright (c) 2010 - 2017 Intel Corporation. + * Copyright (c) 2010 - 2026 Intel Corporation. * Len Brown <len.brown@intel.com> */ @@ -62,6 +62,7 @@ unsigned char turbo_update_value; unsigned char update_hwp_epp; unsigned char update_hwp_min; unsigned char update_hwp_max; +unsigned char hwp_limits_done_via_sysfs; unsigned char update_hwp_desired; unsigned char update_hwp_window; unsigned char update_hwp_use_pkg; @@ -81,18 +82,35 @@ size_t cpu_setsize; char *proc_stat = "/proc/stat"; -unsigned int has_epb; /* MSR_IA32_ENERGY_PERF_BIAS */ -unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */ +unsigned int has_epb; /* MSR_IA32_ENERGY_PERF_BIAS */ +unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */ /* IA32_HWP_REQUEST, IA32_HWP_STATUS */ -unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */ +unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */ unsigned int has_hwp_activity_window; /* IA32_HWP_REQUEST[bits 41:32] */ unsigned int has_hwp_epp; /* IA32_HWP_REQUEST[bits 31:24] */ unsigned int has_hwp_request_pkg; /* IA32_HWP_REQUEST_PKG */ unsigned int bdx_highest_ratio; +unsigned char update_soc_slider_balance; +unsigned char update_soc_slider_offset; +unsigned char update_platform_profile; +int soc_slider_balance; +int soc_slider_offset; +char platform_profile[64]; + #define PATH_TO_CPU "/sys/devices/system/cpu/" #define SYSFS_PATH_MAX 255 +#define PATH_SOC_SLIDER_BALANCE "/sys/module/processor_thermal_soc_slider/parameters/slider_balance" +#define PATH_SOC_SLIDER_OFFSET "/sys/module/processor_thermal_soc_slider/parameters/slider_offset" +#define PATH_PLATFORM_PROFILE "/sys/class/platform-profile/platform-profile-0/profile" +#define PATH_PLATFORM_PROFILE_NAME "/sys/class/platform-profile/platform-profile-0/name" +#define POWER_SLIDER_NAME "SoC Power Slider" + +static int use_android_msr_path; + +static unsigned int read_sysfs(const char *, char *, size_t); +static int sysfs_read_string(const char *, char *, size_t); /* * maintain compatibility with original implementation, but don't document it: @@ -103,8 +121,8 @@ void usage(void) fprintf(stderr, "scope: --cpu cpu-list [--hwp-use-pkg #] | --pkg pkg-list\n"); fprintf(stderr, "field: --all | --epb | --hwp-epp | --hwp-min | --hwp-max | --hwp-desired\n"); fprintf(stderr, "other: --hwp-enable | --turbo-enable (0 | 1) | --help | --force\n"); - fprintf(stderr, - "value: ( # | \"normal\" | \"performance\" | \"balance-performance\" | \"balance-power\"| \"power\")\n"); + fprintf(stderr, "soc-slider: --soc-slider-balance # | --soc-slider-offset # | --platform-profile <name>\n"); + fprintf(stderr, "value: ( # | \"normal\" | \"performance\" | \"balance-performance\" | \"balance-power\"| \"power\")\n"); fprintf(stderr, "--hwp-window usec\n"); fprintf(stderr, "Specify only Energy Performance BIAS (legacy usage):\n"); @@ -132,6 +150,7 @@ int ratio_2_msr_perf(int ratio) return msr_perf; } + int msr_perf_2_ratio(int msr_perf) { int ratio; @@ -140,8 +159,8 @@ int msr_perf_2_ratio(int msr_perf) if (!bdx_highest_ratio) return msr_perf; - d = (double)msr_perf * (double) bdx_highest_ratio / 255.0; - d = d + 0.5; /* round */ + d = (double)msr_perf * (double)bdx_highest_ratio / 255.0; + d = d + 0.5; /* round */ ratio = (int)d; if (debug) @@ -149,6 +168,7 @@ int msr_perf_2_ratio(int msr_perf) return ratio; } + int parse_cmdline_epb(int i) { if (!has_epb) @@ -195,6 +215,7 @@ int parse_cmdline_hwp_min(int i) } return i; } + /* * "power" changes hwp_max to cap.lowest * All others leave it at cap.highest @@ -214,6 +235,7 @@ int parse_cmdline_hwp_max(int i) } return i; } + /* * for --hwp-des, all strings leave it in autonomous mode * If you want to change it, you need to explicitly pick a value @@ -251,7 +273,7 @@ int parse_cmdline_hwp_window(int i) fprintf(stderr, "--hwp-window: 0 for auto; 1 - 1270000000 usec for window duration\n"); usage(); } - for (exponent = 0; ; ++exponent) { + for (exponent = 0;; ++exponent) { if (debug) printf("%d 10^%d\n", i, exponent); @@ -265,6 +287,7 @@ int parse_cmdline_hwp_window(int i) return (exponent << 7) | i; } + int parse_cmdline_hwp_epp(int i) { update_hwp_epp = 1; @@ -286,6 +309,7 @@ int parse_cmdline_hwp_epp(int i) } return i; } + int parse_cmdline_turbo(int i) { update_turbo = 1; @@ -369,7 +393,7 @@ void validate_cpu_selected_set(void) for (cpu = 0; cpu <= max_cpu_num; ++cpu) { if (CPU_ISSET_S(cpu, cpu_setsize, cpu_selected_set)) if (!CPU_ISSET_S(cpu, cpu_setsize, cpu_present_set)) - errx(1, "Requested cpu% is not present", cpu); + errx(1, "Requested cpu%d is not present", cpu); } } @@ -505,7 +529,7 @@ void parse_cmdline_pkg(char *s) } } -void for_packages(unsigned long long pkg_set, int (func)(int)) +void for_packages(unsigned long long pkg_set, int (func) (int)) { int pkg_num; @@ -515,9 +539,79 @@ void for_packages(unsigned long long pkg_set, int (func)(int)) } } +static int parse_cmdline_int(const char *s, int *out) +{ + char *endp; + long val; + + val = strtol(s, &endp, 0); + if (endp == s || errno == ERANGE) + return -1; + if (*endp != '\0') + return -1; + if (val < INT_MIN || val > INT_MAX) + return -1; + + *out = (int)val; + return 0; +} + void print_version(void) { - printf("x86_energy_perf_policy 17.05.11 (C) Len Brown <len.brown@intel.com>\n"); + printf("x86_energy_perf_policy 2026.04.25 Len Brown <lenb@kernel.org>\n"); +} + +static int platform_profile_access(int mode) +{ + if (access(PATH_PLATFORM_PROFILE, mode)) { + if (debug) + fprintf(stderr, "Can not access %s\n", PATH_PLATFORM_PROFILE); + return 0; + } + + return 1; +} + +static int platform_profile_name_is(char *name) +{ + char buf[64]; + + if (sysfs_read_string(PATH_PLATFORM_PROFILE_NAME, buf, sizeof(buf)) != 0) { + if (debug) + fprintf(stderr, "Can not read %s\n", PATH_PLATFORM_PROFILE_NAME); + return 0; + } + + if (strncmp(buf, name, 16)) { + if (debug) + fprintf(stderr, "%s does not match '%s'\n", PATH_PLATFORM_PROFILE_NAME, name); + return 0; + } + + return 1; +} + +static int soc_slider_access(int mode) +{ + if (!platform_profile_access(R_OK)) + return 0; + + if (!platform_profile_name_is(POWER_SLIDER_NAME)) + return 0; + + if (access(PATH_SOC_SLIDER_BALANCE, mode)) { + if (debug) + fprintf(stderr, "Can not access %s\n", PATH_SOC_SLIDER_BALANCE); + return 0; + } + + if (access(PATH_SOC_SLIDER_OFFSET, mode)) { + if (debug) + fprintf(stderr, "Can not access %s\n", PATH_SOC_SLIDER_OFFSET); + return 0; + } + + return 1; } void cmdline(int argc, char **argv) @@ -526,30 +620,32 @@ void cmdline(int argc, char **argv) int option_index = 0; static struct option long_options[] = { - {"all", required_argument, 0, 'a'}, - {"cpu", required_argument, 0, 'c'}, - {"pkg", required_argument, 0, 'p'}, - {"debug", no_argument, 0, 'd'}, - {"hwp-desired", required_argument, 0, 'D'}, - {"epb", required_argument, 0, 'B'}, - {"force", no_argument, 0, 'f'}, - {"hwp-enable", no_argument, 0, 'e'}, - {"help", no_argument, 0, 'h'}, - {"hwp-epp", required_argument, 0, 'P'}, - {"hwp-min", required_argument, 0, 'm'}, - {"hwp-max", required_argument, 0, 'M'}, - {"read", no_argument, 0, 'r'}, - {"turbo-enable", required_argument, 0, 't'}, - {"hwp-use-pkg", required_argument, 0, 'u'}, - {"version", no_argument, 0, 'v'}, - {"hwp-window", required_argument, 0, 'w'}, - {0, 0, 0, 0 } + { "all", required_argument, 0, 'a' }, + { "cpu", required_argument, 0, 'c' }, + { "pkg", required_argument, 0, 'p' }, + { "debug", no_argument, 0, 'd' }, + { "hwp-desired", required_argument, 0, 'D' }, + { "epb", required_argument, 0, 'B' }, + { "force", no_argument, 0, 'f' }, + { "hwp-enable", no_argument, 0, 'e' }, + { "help", no_argument, 0, 'h' }, + { "hwp-epp", required_argument, 0, 'P' }, + { "hwp-min", required_argument, 0, 'm' }, + { "hwp-max", required_argument, 0, 'M' }, + { "read", no_argument, 0, 'r' }, + { "turbo-enable", required_argument, 0, 't' }, + { "hwp-use-pkg", required_argument, 0, 'u' }, + { "version", no_argument, 0, 'v' }, + { "hwp-window", required_argument, 0, 'w' }, + { "soc-slider-balance", required_argument, 0, 'S' }, + { "soc-slider-offset", required_argument, 0, 'O' }, + { "platform-profile", required_argument, 0, 'F' }, + { 0, 0, 0, 0 } }; progname = argv[0]; - while ((opt = getopt_long_only(argc, argv, "+a:c:dD:E:e:f:m:M:rt:u:vw:", - long_options, &option_index)) != -1) { + while ((opt = getopt_long_only(argc, argv, "+a:c:dD:E:e:f:m:M:rt:u:vw::S:O:F:", long_options, &option_index)) != -1) { switch (opt) { case 'a': parse_cmdline_all(optarg); @@ -576,12 +672,27 @@ void cmdline(int argc, char **argv) case 'D': req_update.hwp_desired = parse_cmdline_hwp_desired(parse_optarg_string(optarg)); break; + case 'F': + if (strlen(optarg) >= sizeof(platform_profile)) + errx(1, "--platform-profile: value too long"); + if (!platform_profile_access(W_OK)) + errx(1, "Can not update platform-profile in '%s'", PATH_PLATFORM_PROFILE); + strcpy(platform_profile, optarg); + update_platform_profile = 1; + break; case 'm': req_update.hwp_min = parse_cmdline_hwp_min(parse_optarg_string(optarg)); break; case 'M': req_update.hwp_max = parse_cmdline_hwp_max(parse_optarg_string(optarg)); break; + case 'O': + if (parse_cmdline_int(optarg, &soc_slider_offset)) + errx(1, "--soc-slider-offset: invalid value"); + if (!soc_slider_access(W_OK)) + errx(1, "Unable to write SOC Slider Offset"); + update_soc_slider_offset = 1; + break; case 'p': parse_cmdline_pkg(optarg); break; @@ -591,6 +702,13 @@ void cmdline(int argc, char **argv) case 'r': /* v1 used -r to specify read-only mode, now the default */ break; + case 'S': + if (parse_cmdline_int(optarg, &soc_slider_balance)) + errx(1, "--soc-slider-balance: invalid value"); + if (!soc_slider_access(W_OK)) + errx(1, "Unable to write SOC Slider-Balance in '%s'", PATH_SOC_SLIDER_BALANCE); + update_soc_slider_balance = 1; + break; case 't': turbo_update_value = parse_cmdline_turbo(parse_optarg_string(optarg)); break; @@ -630,7 +748,7 @@ void cmdline(int argc, char **argv) */ FILE *fopen_or_die(const char *path, const char *mode) { - FILE *filep = fopen(path, "r"); + FILE *filep = fopen(path, mode); if (!filep) err(1, "%s: open failed", path); @@ -644,7 +762,7 @@ void err_on_hypervisor(void) char *buffer; /* On VMs /proc/cpuinfo contains a "flags" entry for hypervisor */ - cpuinfo = fopen_or_die("/proc/cpuinfo", "ro"); + cpuinfo = fopen_or_die("/proc/cpuinfo", "r"); buffer = malloc(4096); if (!buffer) { @@ -659,6 +777,11 @@ void err_on_hypervisor(void) } flags = strstr(buffer, "flags"); + if (!flags) { + fclose(cpuinfo); + free(buffer); + err(1, "Failed to find 'flags' in /proc/cpuinfo"); + } rewind(cpuinfo); fseek(cpuinfo, flags - buffer, SEEK_SET); if (!fgets(buffer, 4096, cpuinfo)) { @@ -673,8 +796,7 @@ void err_on_hypervisor(void) free(buffer); if (hypervisor) - err(-1, - "not supported on this virtual machine"); + err(-1, "not supported on this virtual machine"); } int get_msr(int cpu, int offset, unsigned long long *msr) @@ -683,10 +805,10 @@ int get_msr(int cpu, int offset, unsigned long long *msr) char pathname[32]; int fd; - sprintf(pathname, "/dev/cpu/%d/msr", cpu); + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu); fd = open(pathname, O_RDONLY); if (fd < 0) - err(-1, "%s open failed, try chown or chmod +r /dev/cpu/*/msr, or run as root", pathname); + err(-1, "%s open failed, try chown or chmod +r %s, or run as root", pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr"); retval = pread(fd, msr, sizeof(*msr), offset); if (retval != sizeof(*msr)) { @@ -707,10 +829,10 @@ int put_msr(int cpu, int offset, unsigned long long new_msr) int retval; int fd; - sprintf(pathname, "/dev/cpu/%d/msr", cpu); + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu); fd = open(pathname, O_RDWR); if (fd < 0) - err(-1, "%s open failed, try chown or chmod +r /dev/cpu/*/msr, or run as root", pathname); + err(-1, "%s open failed, try chown or chmod +r %s, or run as root", pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr"); retval = pwrite(fd, &new_msr, sizeof(new_msr), offset); if (retval != sizeof(new_msr)) @@ -742,7 +864,7 @@ static unsigned int read_sysfs(const char *path, char *buf, size_t buflen) buf[numread] = '\0'; close(fd); - return (unsigned int) numread; + return (unsigned int)numread; } static unsigned int write_sysfs(const char *path, char *buf, size_t buflen) @@ -756,14 +878,40 @@ static unsigned int write_sysfs(const char *path, char *buf, size_t buflen) numwritten = write(fd, buf, buflen - 1); if (numwritten < 1) { - perror("write failed\n"); + buf[strcspn(buf, "\n")] = '\0'; + warn("Write '%s' to '%s' failed", buf, path); close(fd); return -1; } close(fd); - return (unsigned int) numwritten; + return (unsigned int)numwritten; +} + +static int sysfs_read_string(const char *path, char *buf, size_t buflen) +{ + unsigned int len; + size_t n; + + len = read_sysfs(path, buf, buflen); + if (!len) + return -1; + + n = strcspn(buf, "\n"); + buf[n] = '\0'; + return 0; +} + +static int sysfs_write_string(const char *path, const char *buf) +{ + char tmp[128]; + int len; + + len = snprintf(tmp, sizeof(tmp), "%s\n", buf); + if (len < 0 || len >= (int)sizeof(tmp)) + return -1; + return write_sysfs(path, tmp, (size_t)len + 1) ? 0 : -1; } void print_hwp_cap(int cpu, struct msr_hwp_cap *cap, char *str) @@ -771,9 +919,9 @@ void print_hwp_cap(int cpu, struct msr_hwp_cap *cap, char *str) if (cpu != -1) printf("cpu%d: ", cpu); - printf("HWP_CAP: low %d eff %d guar %d high %d\n", - cap->lowest, cap->efficient, cap->guaranteed, cap->highest); + printf("HWP_CAP: low %d eff %d guar %d high %d\n", cap->lowest, cap->efficient, cap->guaranteed, cap->highest); } + void read_hwp_cap(int cpu, struct msr_hwp_cap *cap, unsigned int msr_offset) { unsigned long long msr; @@ -795,9 +943,9 @@ void print_hwp_request(int cpu, struct msr_hwp_request *h, char *str) printf("%s", str); printf("HWP_REQ: min %d max %d des %d epp %d window 0x%x (%d*10^%dus) use_pkg %d\n", - h->hwp_min, h->hwp_max, h->hwp_desired, h->hwp_epp, - h->hwp_window, h->hwp_window & 0x7F, (h->hwp_window >> 7) & 0x7, h->hwp_use_pkg); + h->hwp_min, h->hwp_max, h->hwp_desired, h->hwp_epp, h->hwp_window, h->hwp_window & 0x7F, (h->hwp_window >> 7) & 0x7, h->hwp_use_pkg); } + void print_hwp_request_pkg(int pkg, struct msr_hwp_request *h, char *str) { printf("pkg%d: ", pkg); @@ -806,10 +954,10 @@ void print_hwp_request_pkg(int pkg, struct msr_hwp_request *h, char *str) printf("%s", str); printf("HWP_REQ_PKG: min %d max %d des %d epp %d window 0x%x (%d*10^%dus)\n", - h->hwp_min, h->hwp_max, h->hwp_desired, h->hwp_epp, - h->hwp_window, h->hwp_window & 0x7F, (h->hwp_window >> 7) & 0x7); + h->hwp_min, h->hwp_max, h->hwp_desired, h->hwp_epp, h->hwp_window, h->hwp_window & 0x7F, (h->hwp_window >> 7) & 0x7); } -void read_hwp_request(int cpu, struct msr_hwp_request *hwp_req, unsigned int msr_offset) + +void read_hwp_request_msr(int cpu, struct msr_hwp_request *hwp_req, unsigned int msr_offset) { unsigned long long msr; @@ -823,15 +971,13 @@ void read_hwp_request(int cpu, struct msr_hwp_request *hwp_req, unsigned int msr hwp_req->hwp_use_pkg = (((msr) >> 42) & 0x1); } -void write_hwp_request(int cpu, struct msr_hwp_request *hwp_req, unsigned int msr_offset) +void write_hwp_request_msr(int cpu, struct msr_hwp_request *hwp_req, unsigned int msr_offset) { unsigned long long msr = 0; if (debug > 1) printf("cpu%d: requesting min %d max %d des %d epp %d window 0x%0x use_pkg %d\n", - cpu, hwp_req->hwp_min, hwp_req->hwp_max, - hwp_req->hwp_desired, hwp_req->hwp_epp, - hwp_req->hwp_window, hwp_req->hwp_use_pkg); + cpu, hwp_req->hwp_min, hwp_req->hwp_max, hwp_req->hwp_desired, hwp_req->hwp_epp, hwp_req->hwp_window, hwp_req->hwp_use_pkg); msr |= HWP_MIN_PERF(ratio_2_msr_perf(hwp_req->hwp_min)); msr |= HWP_MAX_PERF(ratio_2_msr_perf(hwp_req->hwp_max)); @@ -843,7 +989,7 @@ void write_hwp_request(int cpu, struct msr_hwp_request *hwp_req, unsigned int ms put_msr(cpu, msr_offset, msr); } -static int get_epb(int cpu) +static int get_epb_sysfs(int cpu) { char path[SYSFS_PATH_MAX]; char linebuf[3]; @@ -865,7 +1011,7 @@ static int get_epb(int cpu) return (int)val; } -static int set_epb(int cpu, int val) +static int set_epb_sysfs(int cpu, int val) { char path[SYSFS_PATH_MAX]; char linebuf[3]; @@ -889,20 +1035,72 @@ static int set_epb(int cpu, int val) return (int)val; } +static void print_soc_slider(void) +{ + char buf[64]; + + if (!soc_slider_access(R_OK)) + return; + + if (sysfs_read_string(PATH_SOC_SLIDER_BALANCE, buf, sizeof(buf)) == 0) + printf("soc-slider-balance: %s\n", buf); + + if (sysfs_read_string(PATH_SOC_SLIDER_OFFSET, buf, sizeof(buf)) == 0) + printf("soc-slider-offset: %s\n", buf); +} + +static void print_platform_profile(void) +{ + char buf[64]; + + if (!platform_profile_access(R_OK)) + return; + + if (sysfs_read_string(PATH_PLATFORM_PROFILE_NAME, buf, sizeof(buf)) == 0) + printf("platform-profile-name: %s\n", buf); + + if (sysfs_read_string(PATH_PLATFORM_PROFILE, buf, sizeof(buf)) == 0) + printf("platform-profile: %s\n", buf); +} + +static int update_soc_slider(void) +{ + char tmp[32]; + + if (update_soc_slider_balance) { + snprintf(tmp, sizeof(tmp), "%d", soc_slider_balance); + if (sysfs_write_string(PATH_SOC_SLIDER_BALANCE, tmp)) + err(1, "soc-slider-balance write failed"); + } + + if (update_soc_slider_offset) { + snprintf(tmp, sizeof(tmp), "%d", soc_slider_offset); + if (sysfs_write_string(PATH_SOC_SLIDER_OFFSET, tmp)) + err(1, "soc-slider-offset write failed"); + } + + if (update_platform_profile) { + if (sysfs_write_string(PATH_PLATFORM_PROFILE, platform_profile)) + err(1, "platform-profile write failed"); + } + + return 0; +} + int print_cpu_msrs(int cpu) { struct msr_hwp_request req; struct msr_hwp_cap cap; int epb; - epb = get_epb(cpu); + epb = get_epb_sysfs(cpu); if (epb >= 0) - printf("cpu%d: EPB %u\n", cpu, (unsigned int) epb); + printf("cpu%d: EPB %u\n", cpu, (unsigned int)epb); if (!has_hwp) return 0; - read_hwp_request(cpu, &req, MSR_HWP_REQUEST); + read_hwp_request_msr(cpu, &req, MSR_HWP_REQUEST); print_hwp_request(cpu, &req, ""); read_hwp_cap(cpu, &cap, MSR_HWP_CAPABILITIES); @@ -919,23 +1117,19 @@ int print_pkg_msrs(int pkg) if (!has_hwp) return 0; - read_hwp_request(first_cpu_in_pkg[pkg], &req, MSR_HWP_REQUEST_PKG); + read_hwp_request_msr(first_cpu_in_pkg[pkg], &req, MSR_HWP_REQUEST_PKG); print_hwp_request_pkg(pkg, &req, ""); if (has_hwp_notify) { get_msr(first_cpu_in_pkg[pkg], MSR_HWP_INTERRUPT, &msr); fprintf(stderr, - "pkg%d: MSR_HWP_INTERRUPT: 0x%08llx (Excursion_Min-%sabled, Guaranteed_Perf_Change-%sabled)\n", - pkg, msr, - ((msr) & 0x2) ? "EN" : "Dis", - ((msr) & 0x1) ? "EN" : "Dis"); + "pkg%d: MSR_HWP_INTERRUPT: 0x%08llx (Excursion_Min-%sabled, Guaranteed_Perf_Change-%sabled)\n", + pkg, msr, ((msr) & 0x2) ? "EN" : "Dis", ((msr) & 0x1) ? "EN" : "Dis"); } get_msr(first_cpu_in_pkg[pkg], MSR_HWP_STATUS, &msr); fprintf(stderr, "pkg%d: MSR_HWP_STATUS: 0x%08llx (%sExcursion_Min, %sGuaranteed_Perf_Change)\n", - pkg, msr, - ((msr) & 0x4) ? "" : "No-", - ((msr) & 0x1) ? "" : "No-"); + pkg, msr, ((msr) & 0x4) ? "" : "No-", ((msr) & 0x1) ? "" : "No-"); return 0; } @@ -949,10 +1143,13 @@ int ratio_2_sysfs_khz(int ratio) return ratio * bclk_khz; } + /* * If HWP is enabled and cpufreq sysfs attribtes are present, - * then update sysfs, so that it will not become - * stale when we write to MSRs. + * then update via sysfs. The intel_pstate driver may modify (clip) + * this request, say, when HWP_CAP is outside of PLATFORM_INFO limits, + * and the driver-chosen value takes precidence. + * * (intel_pstate's max_perf_pct and min_perf_pct will follow cpufreq, * so we don't have to touch that.) */ @@ -963,8 +1160,7 @@ void update_cpufreq_scaling_freq(int is_max, int cpu, unsigned int ratio) int retval; int khz; - sprintf(pathname, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_%s_freq", - cpu, is_max ? "max" : "min"); + sprintf(pathname, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_%s_freq", cpu, is_max ? "max" : "min"); fp = fopen(pathname, "w"); if (!fp) { @@ -1007,6 +1203,8 @@ int update_sysfs(int cpu) if (update_hwp_max) update_cpufreq_scaling_freq(1, cpu, req_update.hwp_max); + hwp_limits_done_via_sysfs = 1; + return 0; } @@ -1014,19 +1212,16 @@ int verify_hwp_req_self_consistency(int cpu, struct msr_hwp_request *req) { /* fail if min > max requested */ if (req->hwp_min > req->hwp_max) { - errx(1, "cpu%d: requested hwp-min %d > hwp_max %d", - cpu, req->hwp_min, req->hwp_max); + errx(1, "cpu%d: requested hwp-min %d > hwp_max %d", cpu, req->hwp_min, req->hwp_max); } /* fail if desired > max requestd */ if (req->hwp_desired && (req->hwp_desired > req->hwp_max)) { - errx(1, "cpu%d: requested hwp-desired %d > hwp_max %d", - cpu, req->hwp_desired, req->hwp_max); + errx(1, "cpu%d: requested hwp-desired %d > hwp_max %d", cpu, req->hwp_desired, req->hwp_max); } /* fail if desired < min requestd */ if (req->hwp_desired && (req->hwp_desired < req->hwp_min)) { - errx(1, "cpu%d: requested hwp-desired %d < requested hwp_min %d", - cpu, req->hwp_desired, req->hwp_min); + errx(1, "cpu%d: requested hwp-desired %d < requested hwp_min %d", cpu, req->hwp_desired, req->hwp_min); } return 0; @@ -1036,59 +1231,50 @@ int check_hwp_request_v_hwp_capabilities(int cpu, struct msr_hwp_request *req, s { if (update_hwp_max) { if (req->hwp_max > cap->highest) - errx(1, "cpu%d: requested max %d > capabilities highest %d, use --force?", - cpu, req->hwp_max, cap->highest); + errx(1, "cpu%d: requested max %d > capabilities highest %d, use --force?", cpu, req->hwp_max, cap->highest); if (req->hwp_max < cap->lowest) - errx(1, "cpu%d: requested max %d < capabilities lowest %d, use --force?", - cpu, req->hwp_max, cap->lowest); + errx(1, "cpu%d: requested max %d < capabilities lowest %d, use --force?", cpu, req->hwp_max, cap->lowest); } if (update_hwp_min) { if (req->hwp_min > cap->highest) - errx(1, "cpu%d: requested min %d > capabilities highest %d, use --force?", - cpu, req->hwp_min, cap->highest); + errx(1, "cpu%d: requested min %d > capabilities highest %d, use --force?", cpu, req->hwp_min, cap->highest); if (req->hwp_min < cap->lowest) - errx(1, "cpu%d: requested min %d < capabilities lowest %d, use --force?", - cpu, req->hwp_min, cap->lowest); + errx(1, "cpu%d: requested min %d < capabilities lowest %d, use --force?", cpu, req->hwp_min, cap->lowest); } if (update_hwp_min && update_hwp_max && (req->hwp_min > req->hwp_max)) - errx(1, "cpu%d: requested min %d > requested max %d", - cpu, req->hwp_min, req->hwp_max); + errx(1, "cpu%d: requested min %d > requested max %d", cpu, req->hwp_min, req->hwp_max); if (update_hwp_desired && req->hwp_desired) { if (req->hwp_desired > req->hwp_max) - errx(1, "cpu%d: requested desired %d > requested max %d, use --force?", - cpu, req->hwp_desired, req->hwp_max); + errx(1, "cpu%d: requested desired %d > requested max %d, use --force?", cpu, req->hwp_desired, req->hwp_max); if (req->hwp_desired < req->hwp_min) - errx(1, "cpu%d: requested desired %d < requested min %d, use --force?", - cpu, req->hwp_desired, req->hwp_min); + errx(1, "cpu%d: requested desired %d < requested min %d, use --force?", cpu, req->hwp_desired, req->hwp_min); if (req->hwp_desired < cap->lowest) - errx(1, "cpu%d: requested desired %d < capabilities lowest %d, use --force?", - cpu, req->hwp_desired, cap->lowest); + errx(1, "cpu%d: requested desired %d < capabilities lowest %d, use --force?", cpu, req->hwp_desired, cap->lowest); if (req->hwp_desired > cap->highest) - errx(1, "cpu%d: requested desired %d > capabilities highest %d, use --force?", - cpu, req->hwp_desired, cap->highest); + errx(1, "cpu%d: requested desired %d > capabilities highest %d, use --force?", cpu, req->hwp_desired, cap->highest); } return 0; } -int update_hwp_request(int cpu) +int update_hwp_request_msr(int cpu) { struct msr_hwp_request req; struct msr_hwp_cap cap; int msr_offset = MSR_HWP_REQUEST; - read_hwp_request(cpu, &req, msr_offset); + read_hwp_request_msr(cpu, &req, msr_offset); if (debug) print_hwp_request(cpu, &req, "old: "); - if (update_hwp_min) + if (update_hwp_min && !hwp_limits_done_via_sysfs) req.hwp_min = req_update.hwp_min; - if (update_hwp_max) + if (update_hwp_max && !hwp_limits_done_via_sysfs) req.hwp_max = req_update.hwp_max; if (update_hwp_desired) @@ -1111,15 +1297,16 @@ int update_hwp_request(int cpu) verify_hwp_req_self_consistency(cpu, &req); - write_hwp_request(cpu, &req, msr_offset); + write_hwp_request_msr(cpu, &req, msr_offset); if (debug) { - read_hwp_request(cpu, &req, msr_offset); + read_hwp_request_msr(cpu, &req, msr_offset); print_hwp_request(cpu, &req, "new: "); } return 0; } -int update_hwp_request_pkg(int pkg) + +int update_hwp_request_pkg_msr(int pkg) { struct msr_hwp_request req; struct msr_hwp_cap cap; @@ -1127,7 +1314,7 @@ int update_hwp_request_pkg(int pkg) int msr_offset = MSR_HWP_REQUEST_PKG; - read_hwp_request(cpu, &req, msr_offset); + read_hwp_request_msr(cpu, &req, msr_offset); if (debug) print_hwp_request_pkg(pkg, &req, "old: "); @@ -1155,10 +1342,10 @@ int update_hwp_request_pkg(int pkg) verify_hwp_req_self_consistency(cpu, &req); - write_hwp_request(cpu, &req, msr_offset); + write_hwp_request_msr(cpu, &req, msr_offset); if (debug) { - read_hwp_request(cpu, &req, msr_offset); + read_hwp_request_msr(cpu, &req, msr_offset); print_hwp_request_pkg(pkg, &req, "new: "); } return 0; @@ -1166,30 +1353,38 @@ int update_hwp_request_pkg(int pkg) int enable_hwp_on_cpu(int cpu) { - unsigned long long msr; + unsigned long long old_msr, new_msr; + + get_msr(cpu, MSR_PM_ENABLE, &old_msr); + + if (old_msr & 1) + return 0; /* already enabled */ - get_msr(cpu, MSR_PM_ENABLE, &msr); - put_msr(cpu, MSR_PM_ENABLE, 1); + new_msr = old_msr | 1; + put_msr(cpu, MSR_PM_ENABLE, new_msr); if (verbose) - printf("cpu%d: MSR_PM_ENABLE old: %d new: %d\n", cpu, (unsigned int) msr, 1); + printf("cpu%d: MSR_PM_ENABLE old: %llX new: %llX\n", cpu, old_msr, new_msr); return 0; } -int update_cpu_msrs(int cpu) +int update_cpu_epb_sysfs(int cpu) { - unsigned long long msr; int epb; - if (update_epb) { - epb = get_epb(cpu); - set_epb(cpu, new_epb); + epb = get_epb_sysfs(cpu); + set_epb_sysfs(cpu, new_epb); - if (verbose) - printf("cpu%d: ENERGY_PERF_BIAS old: %d new: %d\n", - cpu, epb, (unsigned int) new_epb); - } + if (verbose) + printf("cpu%d: ENERGY_PERF_BIAS old: %d new: %d\n", cpu, epb, (unsigned int)new_epb); + + return 0; +} + +int update_cpu_msrs(int cpu) +{ + unsigned long long msr; if (update_turbo) { int turbo_is_present_and_disabled; @@ -1198,7 +1393,7 @@ int update_cpu_msrs(int cpu) turbo_is_present_and_disabled = ((msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE) != 0); - if (turbo_update_value == 1) { + if (turbo_update_value == 1) { if (turbo_is_present_and_disabled) { msr &= ~MSR_IA32_MISC_ENABLE_TURBO_DISABLE; put_msr(cpu, MSR_IA32_MISC_ENABLE, msr); @@ -1224,7 +1419,7 @@ int update_cpu_msrs(int cpu) if (!hwp_update_enabled()) return 0; - update_hwp_request(cpu); + update_hwp_request_msr(cpu); return 0; } @@ -1267,6 +1462,7 @@ int set_max_cpu_pkg_num(int cpu) return 0; } + int mark_cpu_present(int cpu) { CPU_SET_S(cpu, cpu_setsize, cpu_present_set); @@ -1277,7 +1473,7 @@ int mark_cpu_present(int cpu) * run func(cpu) on every cpu in /proc/stat * return max_cpu number */ -int for_all_proc_cpus(int (func)(int)) +int for_all_proc_cpus(int (func) (int)) { FILE *fp; int cpu_num; @@ -1304,7 +1500,7 @@ int for_all_proc_cpus(int (func)(int)) return 0; } -void for_all_cpus_in_set(size_t set_size, cpu_set_t *cpu_set, int (func)(int)) +void for_all_cpus_in_set(size_t set_size, cpu_set_t *cpu_set, int (func) (int)) { int cpu_num; @@ -1313,6 +1509,18 @@ void for_all_cpus_in_set(size_t set_size, cpu_set_t *cpu_set, int (func)(int)) func(cpu_num); } +int for_all_cpus_in_set_and(size_t set_size, cpu_set_t *cpu_set, int (func) (int)) +{ + int cpu_num; + int retval = 1; + + for (cpu_num = 0; cpu_num <= max_cpu_num; ++cpu_num) + if (CPU_ISSET_S(cpu_num, set_size, cpu_set)) + retval &= func(cpu_num); + + return retval; +} + void init_data_structures(void) { for_all_proc_cpus(set_max_cpu_pkg_num); @@ -1326,21 +1534,38 @@ void init_data_structures(void) for_all_proc_cpus(mark_cpu_present); } -/* clear has_hwp if it is not enable (or being enabled) */ +int is_hwp_enabled_on_cpu(int cpu_num) +{ + unsigned long long msr; + int retval; + + /* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */ + get_msr(cpu_num, MSR_PM_ENABLE, &msr); + retval = (msr & 1); + if (verbose) + fprintf(stderr, "cpu%d: %sHWP\n", cpu_num, retval ? "" : "No-"); + + return retval; +} + +/* + * verify_hwp_is_enabled() + * + * Set (has_hwp=0) if no HWP feature or any of selected CPU set does not have HWP enabled + */ void verify_hwp_is_enabled(void) { - unsigned long long msr; + int retval; - if (!has_hwp) /* set in early_cpuid() */ + if (!has_hwp) /* set in early_cpuid() */ return; - /* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */ - get_msr(base_cpu, MSR_PM_ENABLE, &msr); - if ((msr & 1) == 0) { + retval = for_all_cpus_in_set_and(cpu_setsize, cpu_selected_set, is_hwp_enabled_on_cpu); + + if (retval == 0) { fprintf(stderr, "HWP can be enabled using '--hwp-enable'\n"); has_hwp = 0; - return; } } @@ -1350,21 +1575,18 @@ int req_update_bounds_check(void) return 0; /* fail if min > max requested */ - if ((update_hwp_max && update_hwp_min) && - (req_update.hwp_min > req_update.hwp_max)) { + if ((update_hwp_max && update_hwp_min) && (req_update.hwp_min > req_update.hwp_max)) { printf("hwp-min %d > hwp_max %d\n", req_update.hwp_min, req_update.hwp_max); return -EINVAL; } /* fail if desired > max requestd */ - if (req_update.hwp_desired && update_hwp_max && - (req_update.hwp_desired > req_update.hwp_max)) { + if (req_update.hwp_desired && update_hwp_max && (req_update.hwp_desired > req_update.hwp_max)) { printf("hwp-desired cannot be greater than hwp_max\n"); return -EINVAL; } /* fail if desired < min requestd */ - if (req_update.hwp_desired && update_hwp_min && - (req_update.hwp_desired < req_update.hwp_min)) { + if (req_update.hwp_desired && update_hwp_min && (req_update.hwp_desired < req_update.hwp_min)) { printf("hwp-desired cannot be less than hwp_min\n"); return -EINVAL; } @@ -1379,21 +1601,35 @@ void set_base_cpu(void) err(-ENODEV, "No valid cpus found"); } +static void probe_android_msr_path(void) +{ + struct stat sb; + char test_path[32]; + + sprintf(test_path, "/dev/msr%d", base_cpu); + if (stat(test_path, &sb) == 0) + use_android_msr_path = 1; +} void probe_dev_msr(void) { struct stat sb; char pathname[32]; - sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); - if (stat(pathname, &sb)) - if (system("/sbin/modprobe msr > /dev/null 2>&1")) - err(-5, "no /dev/cpu/0/msr, Try \"# modprobe msr\" "); + probe_android_msr_path(); + + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", base_cpu); + if (stat(pathname, &sb)) { + if (system("/sbin/modprobe msr > /dev/null 2>&1")) { + if (use_android_msr_path) + err(-5, "no /dev/msr0, Try \"# modprobe msr\" "); + else + err(-5, "no /dev/cpu/0/msr, Try \"# modprobe msr\" "); + } + } } -static void get_cpuid_or_exit(unsigned int leaf, - unsigned int *eax, unsigned int *ebx, - unsigned int *ecx, unsigned int *edx) +static void get_cpuid_or_exit(unsigned int leaf, unsigned int *eax, unsigned int *ebx, unsigned int *ecx, unsigned int *edx) { if (!__get_cpuid(leaf, eax, ebx, ecx, edx)) errx(1, "Processor not supported\n"); @@ -1447,8 +1683,7 @@ void parse_cpuid(void) genuine_intel = 1; if (debug) - fprintf(stderr, "CPUID(0): %.4s%.4s%.4s ", - (char *)&ebx, (char *)&edx, (char *)&ecx); + fprintf(stderr, "CPUID(0): %.4s%.4s%.4s ", (char *)&ebx, (char *)&edx, (char *)&ecx); get_cpuid_or_exit(1, &fms, &ebx, &ecx, &edx); family = (fms >> 8) & 0xf; @@ -1458,23 +1693,18 @@ void parse_cpuid(void) model += ((fms >> 16) & 0xf) << 4; if (debug) { - fprintf(stderr, "%d CPUID levels; family:model:stepping 0x%x:%x:%x (%d:%d:%d)\n", - max_level, family, model, stepping, family, model, stepping); + fprintf(stderr, "%d CPUID levels; family:model:stepping 0x%x:%x:%x (%d:%d:%d)\n", max_level, family, model, stepping, family, model, stepping); fprintf(stderr, "CPUID(1): %s %s %s %s %s %s %s %s\n", ecx & (1 << 0) ? "SSE3" : "-", ecx & (1 << 3) ? "MONITOR" : "-", ecx & (1 << 7) ? "EIST" : "-", ecx & (1 << 8) ? "TM2" : "-", - edx & (1 << 4) ? "TSC" : "-", - edx & (1 << 5) ? "MSR" : "-", - edx & (1 << 22) ? "ACPI-TM" : "-", - edx & (1 << 29) ? "TM" : "-"); + edx & (1 << 4) ? "TSC" : "-", edx & (1 << 5) ? "MSR" : "-", edx & (1 << 22) ? "ACPI-TM" : "-", edx & (1 << 29) ? "TM" : "-"); } if (!(edx & (1 << 5))) errx(1, "CPUID: no MSR"); - get_cpuid_or_exit(0x6, &eax, &ebx, &ecx, &edx); /* turbo_is_enabled already set */ /* has_hwp already set */ @@ -1494,21 +1724,19 @@ void parse_cpuid(void) turbo_is_enabled ? "" : "No-", has_hwp ? "" : "No-", has_hwp_notify ? "" : "No-", - has_hwp_activity_window ? "" : "No-", - has_hwp_epp ? "" : "No-", - has_hwp_request_pkg ? "" : "No-", - has_epb ? "" : "No-"); + has_hwp_activity_window ? "" : "No-", has_hwp_epp ? "" : "No-", has_hwp_request_pkg ? "" : "No-", has_epb ? "" : "No-"); - return; /* success */ + return; /* success */ } int main(int argc, char **argv) { set_base_cpu(); + probe_dev_msr(); init_data_structures(); - early_cpuid(); /* initial cpuid parse before cmdline */ + early_cpuid(); /* initial cpuid parse before cmdline */ cmdline(argc, argv); @@ -1517,7 +1745,7 @@ int main(int argc, char **argv) parse_cpuid(); - /* If CPU-set and PKG-set are not initialized, default to all CPUs */ + /* If CPU-set and PKG-set are not initialized, default to all CPUs */ if ((cpu_selected_set == 0) && (pkg_selected_set == 0)) cpu_selected_set = cpu_present_set; @@ -1535,10 +1763,13 @@ int main(int argc, char **argv) return -EINVAL; /* display information only, no updates to settings */ - if (!update_epb && !update_turbo && !hwp_update_enabled()) { + if (!update_epb && !update_turbo && !hwp_update_enabled() && !update_soc_slider_balance && !update_soc_slider_offset && !update_platform_profile) { if (cpu_selected_set) for_all_cpus_in_set(cpu_setsize, cpu_selected_set, print_cpu_msrs); + print_soc_slider(); + print_platform_profile(); + if (has_hwp_request_pkg) { if (pkg_selected_set == 0) pkg_selected_set = pkg_present_set; @@ -1551,10 +1782,16 @@ int main(int argc, char **argv) /* update CPU set */ if (cpu_selected_set) { + if (update_epb) + for_all_cpus_in_set(cpu_setsize, cpu_selected_set, update_cpu_epb_sysfs); for_all_cpus_in_set(cpu_setsize, cpu_selected_set, update_sysfs); for_all_cpus_in_set(cpu_setsize, cpu_selected_set, update_cpu_msrs); + } else if (pkg_selected_set) - for_packages(pkg_selected_set, update_hwp_request_pkg); + for_packages(pkg_selected_set, update_hwp_request_pkg_msr); + + if (update_soc_slider_balance || update_soc_slider_offset || update_platform_profile) + update_soc_slider(); return 0; } |