diff options
Diffstat (limited to 'tools/perf/util')
156 files changed, 11639 insertions, 2927 deletions
diff --git a/tools/perf/util/Build b/tools/perf/util/Build index 70cc91d00804..5e2265018826 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -6,6 +6,7 @@ perf-util-y += arm64-frame-pointer-unwind-support.o perf-util-y += addr2line.o perf-util-y += addr_location.o perf-util-y += annotate.o +perf-util-y += aslr.o perf-util-y += blake2s.o perf-util-y += block-info.o perf-util-y += block-range.o @@ -75,7 +76,7 @@ perf-util-y += sample.o perf-util-y += sample-raw.o perf-util-y += s390-sample-raw.o perf-util-y += amd-sample-raw.o -perf-util-$(CONFIG_TRACE) += syscalltbl.o + perf-util-y += ordered-events.o perf-util-y += namespaces.o perf-util-y += comm.o @@ -216,6 +217,7 @@ ifndef CONFIG_SETNS perf-util-y += setns.o endif +perf-util-y += unwind.o perf-util-$(CONFIG_LIBDW) += probe-finder.o perf-util-$(CONFIG_LIBDW) += dwarf-aux.o perf-util-$(CONFIG_LIBDW) += dwarf-regs.o @@ -225,13 +227,11 @@ perf-util-$(CONFIG_LIBDW) += annotate-data.o perf-util-$(CONFIG_LIBDW) += libdw.o perf-util-$(CONFIG_LIBDW) += unwind-libdw.o -perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind-local.o perf-util-$(CONFIG_LIBUNWIND) += unwind-libunwind.o -perf-util-$(CONFIG_LIBUNWIND_X86) += libunwind/x86_32.o -perf-util-$(CONFIG_LIBUNWIND_AARCH64) += libunwind/arm64.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arch/ ifeq ($(CONFIG_LIBTRACEEVENT),y) - perf-util-$(CONFIG_LIBBABELTRACE) += data-convert-bt.o + perf-util-$(CONFIG_BABELTRACE2_CTF_WRITER) += data-convert-bt.o endif perf-util-y += data-convert-json.o @@ -441,3 +441,18 @@ $(OUTPUT)%.pylint_log: % $(Q)$(call echo-cmd,test)pylint "$<" > $@ || (cat $@ && rm $@ && false) perf-util-y += $(PYLINT_TEST_LOGS) + +ifeq ($(CONFIG_PERF_BPF_SKEL),y) +include $(srctree)/tools/perf/bpf_skel.mak + +$(OUTPUT)util/bpf_ftrace.o: $(SKEL_OUT)/func_latency.skel.h +$(OUTPUT)util/bpf-filter.o: $(SKEL_OUT)/sample_filter.skel.h +$(OUTPUT)util/bpf_kwork_top.o: $(SKEL_OUT)/kwork_top.skel.h +$(OUTPUT)util/bpf_off_cpu.o: $(SKEL_OUT)/off_cpu.skel.h +$(OUTPUT)util/bpf-trace-summary.o: $(SKEL_OUT)/syscall_summary.skel.h +$(OUTPUT)util/bpf_counter_cgroup.o: $(SKEL_OUT)/bperf_cgroup.skel.h +$(OUTPUT)util/bpf_trace_augment.o: $(SKEL_OUT)/augmented_raw_syscalls.skel.h +$(OUTPUT)util/bpf_counter.o: $(SKEL_OUT)/bpf_prog_profiler.skel.h $(SKEL_OUT)/bperf_leader.skel.h $(SKEL_OUT)/bperf_follower.skel.h +$(OUTPUT)util/bpf_lock_contention.o: $(SKEL_OUT)/lock_contention.skel.h +$(OUTPUT)util/bpf_kwork.o: $(SKEL_OUT)/kwork_trace.skel.h +endif diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c index b084dee76b1a..394c061fbeb3 100644 --- a/tools/perf/util/amd-sample-raw.c +++ b/tools/perf/util/amd-sample-raw.c @@ -21,6 +21,31 @@ static u32 cpu_family, cpu_model, ibs_fetch_type, ibs_op_type; static bool zen4_ibs_extensions; static bool ldlat_cap; static bool dtlb_pgsize_cap; +static bool rmtsocket_cap; +static bool strmst_cap; + +/* + * Status fields of IBS_FETCH_CTL and IBS_FETCH_CTL_EXT are valid only if + * IBS_FETCH_CTL[PhyAddrValid] is set. + */ +static int fetch_ctl_depends_on_phy_addr_valid(void) +{ + static int depends = -1; /* -1: Don't know, 1: Yes, 0: No */ + + if (depends != -1) + return depends; + + depends = 0; + if (cpu_family > 0x1a || + (cpu_family == 0x1a && ( + (cpu_model >= 0x50 && cpu_model <= 0x5f) || + (cpu_model >= 0x80 && cpu_model <= 0xaf) || + (cpu_model >= 0xc0 && cpu_model <= 0xcf)))) { + depends = 1; + } + + return depends; +} static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) { @@ -43,6 +68,18 @@ static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) const char *ic_miss_str = NULL; const char *l1tlb_pgsz_str = NULL; char l3_miss_str[sizeof(" L3MissOnly _ FetchOcMiss _ FetchL3Miss _")] = ""; + char l3_miss_only_str[sizeof(" L3MissOnly _")] = ""; + + if (fetch_ctl_depends_on_phy_addr_valid() && !reg.phy_addr_valid) { + snprintf(l3_miss_only_str, sizeof(l3_miss_only_str), + " L3MissOnly %d", reg.l3_miss_only); + + printf("ibs_fetch_ctl:\t%016llx MaxCnt %7d Cnt %7d En %d Val %d Comp %d " + "PhyAddrValid 0 RandEn %d%s\n", reg.val, reg.fetch_maxcnt << 4, + reg.fetch_cnt << 4, reg.fetch_en, reg.fetch_val, reg.fetch_comp, + reg.rand_en, l3_miss_only_str); + return; + } if (cpu_family == 0x19 && cpu_model < 0x10) { /* @@ -72,8 +109,11 @@ static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) l3_miss_str); } -static void pr_ic_ibs_extd_ctl(union ic_ibs_extd_ctl reg) +static void pr_ic_ibs_extd_ctl(union ibs_fetch_ctl fetch_ctl, union ic_ibs_extd_ctl reg) { + if (fetch_ctl_depends_on_phy_addr_valid() && !fetch_ctl.phy_addr_valid) + return; + printf("ic_ibs_ext_ctl:\t%016llx IbsItlbRefillLat %3d\n", reg.val, reg.itlb_refill_lat); } @@ -126,8 +166,16 @@ static void pr_ibs_op_data2_extended(union ibs_op_data2 reg) /* 13 to 31 are reserved. Avoid printing them. */ }; int data_src = (reg.data_src_hi << 3) | reg.data_src_lo; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; + + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); - printf("ibs_op_data2:\t%016llx %sRmtNode %d%s\n", reg.val, + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, (data_src == 1 || data_src == 2 || data_src == 5) ? (reg.cache_hit_st ? "CacheHitSt 1=O-State " : "CacheHitSt 0=M-state ") : "", reg.rmt_node, @@ -146,8 +194,16 @@ static void pr_ibs_op_data2_default(union ibs_op_data2 reg) " DataSrc 6=(reserved)", " DataSrc 7=Other" }; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; - printf("ibs_op_data2:\t%016llx %sRmtNode %d%s\n", reg.val, + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); + + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, reg.data_src_lo == 2 ? (reg.cache_hit_st ? "CacheHitSt 1=O-State " : "CacheHitSt 0=M-state ") : "", reg.rmt_node, data_src_str[reg.data_src_lo]); @@ -172,6 +228,7 @@ static void pr_ibs_op_data3(union ibs_op_data3 reg) char dc_l1_l2tlb_miss_str[sizeof(" DcL1TlbMiss _ DcL2TlbMiss _")] = ""; char dc_l1tlb_hit_str[sizeof(" DcL1TlbHit2M _ DcL1TlbHit1G _")] = ""; char op_mem_width_str[sizeof(" OpMemWidth _____ bytes")] = ""; + char tlb_refill_lat_str[sizeof(" TlbRefillLat _____")] = ""; char dc_l2tlb_hit_2m_str[sizeof(" DcL2TlbHit2M _")] = ""; char dc_l2tlb_hit_1g_str[sizeof(" DcL2TlbHit1G _")] = ""; char dc_page_size_str[sizeof(" DcPageSize ____")] = ""; @@ -214,17 +271,23 @@ static void pr_ibs_op_data3(union ibs_op_data3 reg) " DcL2TlbHit1G %d", reg.dc_l2_tlb_hit_1g); } + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (!zen4_ibs_extensions || reg.dc_phy_addr_valid) { + snprintf(tlb_refill_lat_str, sizeof(tlb_refill_lat_str), + " TlbRefillLat %5d", reg.tlb_refill_lat); + } + printf("ibs_op_data3:\t%016llx LdOp %d StOp %d%s%s%s DcMiss %d DcMisAcc %d " "DcWcMemAcc %d DcUcMemAcc %d DcLockedOp %d DcMissNoMabAlloc %d " "DcLinAddrValid %d DcPhyAddrValid %d%s%s SwPf %d%s%s " - "DcMissLat %5d TlbRefillLat %5d\n", + "DcMissLat %5d%s\n", reg.val, reg.ld_op, reg.st_op, dc_l1_l2tlb_miss_str, dtlb_pgsize_cap ? dc_page_size_str : dc_l1tlb_hit_str, dc_l2tlb_hit_2m_str, reg.dc_miss, reg.dc_mis_acc, reg.dc_wc_mem_acc, reg.dc_uc_mem_acc, reg.dc_locked_op, reg.dc_miss_no_mab_alloc, reg.dc_lin_addr_valid, reg.dc_phy_addr_valid, dc_l2tlb_hit_1g_str, l2_miss_str, reg.sw_pf, op_mem_width_str, op_dc_miss_open_mem_reqs_str, - reg.dc_miss_lat, reg.tlb_refill_lat); + reg.dc_miss_lat, tlb_refill_lat_str); } /* @@ -253,8 +316,12 @@ static void amd_dump_ibs_op(struct perf_sample *sample) pr_ibs_op_data3(*op_data3); if (op_data3->dc_lin_addr_valid) printf("IbsDCLinAd:\t%016llx\n", *(rip + 4)); - if (op_data3->dc_phy_addr_valid) + + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (op_data3->dc_phy_addr_valid && *(rip + 5) && + (!zen4_ibs_extensions || op_data3->dc_lin_addr_valid)) { printf("IbsDCPhysAd:\t%016llx\n", *(rip + 5)); + } if (op_data->op_brn_ret && *(rip + 6)) printf("IbsBrTarget:\t%016llx\n", *(rip + 6)); } @@ -274,7 +341,7 @@ static void amd_dump_ibs_fetch(struct perf_sample *sample) printf("IbsFetchLinAd:\t%016llx\n", *addr++); if (fetch_ctl->phy_addr_valid) printf("IbsFetchPhysAd:\t%016llx\n", *addr); - pr_ic_ibs_extd_ctl(*extd_ctl); + pr_ic_ibs_extd_ctl(*fetch_ctl, *extd_ctl); } /* @@ -380,6 +447,12 @@ bool evlist__has_amd_ibs(struct evlist *evlist) if (perf_env__find_pmu_cap(env, "ibs_op", "dtlb_pgsize")) dtlb_pgsize_cap = 1; + if (perf_env__find_pmu_cap(env, "ibs_op", "rmtsocket")) + rmtsocket_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "strmst")) + strmst_cap = 1; + if (ibs_fetch_type || ibs_op_type) { if (!cpu_family) parse_cpuid(env); diff --git a/tools/perf/util/annotate-data.c b/tools/perf/util/annotate-data.c index 1eff0a27237d..63e3c54fab42 100644 --- a/tools/perf/util/annotate-data.c +++ b/tools/perf/util/annotate-data.c @@ -74,7 +74,8 @@ void pr_debug_type_name(Dwarf_Die *die, enum type_state_kind kind) break; } - dwarf_aggregate_size(die, &size); + if (dwarf_aggregate_size(die, &size) != 0) + size = 0; strbuf_init(&sb, 32); die_get_typename_from_type(die, &sb); @@ -146,9 +147,9 @@ static void pr_debug_scope(Dwarf_Die *scope_die) tag = dwarf_tag(scope_die); if (tag == DW_TAG_subprogram) - pr_info("[function] %s\n", dwarf_diename(scope_die)); + pr_info("[function] %s\n", die_name(scope_die)); else if (tag == DW_TAG_inlined_subroutine) - pr_info("[inlined] %s\n", dwarf_diename(scope_die)); + pr_info("[inlined] %s\n", die_name(scope_die)); else if (tag == DW_TAG_lexical_block) pr_info("[block]\n"); else @@ -250,9 +251,12 @@ static int __add_member_cb(Dwarf_Die *die, void *arg) if (dwarf_aggregate_size(&die_mem, &size) < 0) size = 0; - if (dwarf_attr_integrate(die, DW_AT_data_member_location, &attr)) - dwarf_formudata(&attr, &loc); - else { + if (dwarf_attr_integrate(die, DW_AT_data_member_location, &attr)) { + if (dwarf_formudata(&attr, &loc) != 0) { + if (die_get_data_member_location(die, &loc) != 0) + loc = 0; + } + } else { /* bitfield member */ if (dwarf_attr_integrate(die, DW_AT_data_bit_offset, &attr) && dwarf_formudata(&attr, &loc) == 0) @@ -273,7 +277,9 @@ static int __add_member_cb(Dwarf_Die *die, void *arg) dwarf_diename(die), (long)bit_size) < 0) member->var_name = NULL; } else { - member->var_name = strdup(dwarf_diename(die)); + const char *name = dwarf_diename(die); + + member->var_name = name ? strdup(name) : NULL; } if (member->var_name == NULL) { @@ -370,7 +376,8 @@ static struct annotated_data_type *dso__findnew_data_type(struct dso *dso, if (dwarf_tag(type_die) == DW_TAG_typedef) die_get_real_type(type_die, type_die); - dwarf_aggregate_size(type_die, &size); + if (dwarf_aggregate_size(type_die, &size) != 0) + size = 0; /* Check existing nodes in dso->data_types tree */ key.self.type_name = type_name; @@ -1569,7 +1576,7 @@ static int find_data_type_die(struct data_loc_info *dloc, Dwarf_Die *type_die) offset = loc->offset; pr_debug_dtp("CU for %s (die:%#lx)\n", - dwarf_diename(&cu_die), (long)dwarf_dieoffset(&cu_die)); + die_name(&cu_die), (long)dwarf_dieoffset(&cu_die)); if (reg == DWARF_REG_PC) { if (get_global_var_type(&cu_die, dloc, dloc->ip, dloc->var_addr, @@ -1636,7 +1643,7 @@ retry: } pr_debug_dtp("found \"%s\" (die: %#lx) in scope=%d/%d (die: %#lx) ", - dwarf_diename(&var_die), (long)dwarf_dieoffset(&var_die), + die_name(&var_die), (long)dwarf_dieoffset(&var_die), i+1, nr_scopes, (long)dwarf_dieoffset(&scopes[i])); if (reg == DWARF_REG_PC) { diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index e745f3034a0e..02505222d8c2 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -213,9 +213,10 @@ static int __symbol__account_cycles(struct cyc_hist *ch, } static int __symbol__inc_addr_samples(struct map_symbol *ms, - struct annotated_source *src, struct evsel *evsel, u64 addr, + struct annotated_source *src, u64 addr, struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; struct symbol *sym = ms->sym; long hash_key; u64 offset; @@ -235,7 +236,8 @@ static int __symbol__inc_addr_samples(struct map_symbol *ms, h = annotated_source__histogram(src, evsel); if (h == NULL) { pr_debug("%s(%d): ENOMEM! sym->name=%s, start=%#" PRIx64 ", addr=%#" PRIx64 ", end=%#" PRIx64 ", func: %d\n", - __func__, __LINE__, sym->name, sym->start, addr, sym->end, sym->type == STT_FUNC); + __func__, __LINE__, sym->name, sym->start, addr, sym->end, + symbol__type(sym) == STT_FUNC); return -ENOMEM; } @@ -318,7 +320,7 @@ alloc_histograms: } static int symbol__inc_addr_samples(struct map_symbol *ms, - struct evsel *evsel, u64 addr, + u64 addr, struct perf_sample *sample) { struct symbol *sym = ms->sym; @@ -326,8 +328,8 @@ static int symbol__inc_addr_samples(struct map_symbol *ms, if (sym == NULL) return 0; - src = symbol__hists(sym, evsel->evlist->core.nr_entries); - return src ? __symbol__inc_addr_samples(ms, src, evsel, addr, sample) : 0; + src = symbol__hists(sym, sample->evsel->evlist->core.nr_entries); + return src ? __symbol__inc_addr_samples(ms, src, addr, sample) : 0; } static int symbol__account_br_cntr(struct annotated_branch *branch, @@ -581,16 +583,14 @@ static int annotation__compute_ipc(struct annotation *notes, size_t size, return 0; } -int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample, - struct evsel *evsel) +int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample) { - return symbol__inc_addr_samples(&ams->ms, evsel, ams->al_addr, sample); + return symbol__inc_addr_samples(&ams->ms, ams->al_addr, sample); } -int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, - struct evsel *evsel, u64 ip) +int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, u64 ip) { - return symbol__inc_addr_samples(&he->ms, evsel, ip, sample); + return symbol__inc_addr_samples(&he->ms, ip, sample); } @@ -2224,7 +2224,7 @@ int symbol__annotate2(struct map_symbol *ms, struct evsel *evsel, annotation__init_column_widths(notes, sym); annotation__update_column_widths(notes); - sym->annotate2 = 1; + symbol__set_annotate2(sym, true); return 0; } diff --git a/tools/perf/util/annotate.h b/tools/perf/util/annotate.h index 696e36dbf013..1aa6df7d1618 100644 --- a/tools/perf/util/annotate.h +++ b/tools/perf/util/annotate.h @@ -422,8 +422,7 @@ static inline struct annotation *symbol__annotation(struct symbol *sym) return (void *)sym - symbol_conf.priv_size; } -int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample, - struct evsel *evsel); +int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample); struct annotated_branch *annotation__get_branch(struct annotation *notes); @@ -433,8 +432,7 @@ int addr_map_symbol__account_cycles(struct addr_map_symbol *ams, struct evsel *evsel, u64 br_cntr); -int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, - struct evsel *evsel, u64 addr); +int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, u64 addr); struct annotated_source *symbol__hists(struct symbol *sym, int nr_hists); void symbol__annotate_zero_histograms(struct symbol *sym); diff --git a/tools/perf/util/arm-spe-decoder/Build b/tools/perf/util/arm-spe-decoder/Build index ab500e0efe24..97a298d1e279 100644 --- a/tools/perf/util/arm-spe-decoder/Build +++ b/tools/perf/util/arm-spe-decoder/Build @@ -1 +1,3 @@ perf-util-y += arm-spe-pkt-decoder.o arm-spe-decoder.o + +CFLAGS_arm-spe-pkt-decoder.o += -I$(srctree)/tools/arch/arm64/include/ -I$(OUTPUT)arch/arm64/include/generated/ diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c index 9e02b2bdd117..7a3a4815fd37 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c @@ -120,7 +120,8 @@ static int arm_spe_get_data(struct arm_spe_decoder *decoder) return decoder->len; } -static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder) +static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder, + struct arm_spe_pkt *packet) { int ret; @@ -134,7 +135,8 @@ static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder) } ret = arm_spe_get_packet(decoder->buf, decoder->len, - &decoder->packet); + packet, decoder->midr); + if (ret <= 0) { /* Move forward for 1 byte */ decoder->buf += 1; @@ -144,7 +146,7 @@ static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder) decoder->buf += ret; decoder->len -= ret; - } while (decoder->packet.type == ARM_SPE_PAD); + } while (packet->type == ARM_SPE_PAD); return 1; } @@ -154,19 +156,20 @@ static int arm_spe_read_record(struct arm_spe_decoder *decoder) int err; int idx; u64 payload, ip; + struct arm_spe_pkt packet; memset(&decoder->record, 0x0, sizeof(decoder->record)); decoder->record.context_id = (u64)-1; while (1) { - err = arm_spe_get_next_packet(decoder); + err = arm_spe_get_next_packet(decoder, &packet); if (err <= 0) return err; - idx = decoder->packet.index; - payload = decoder->packet.payload; + idx = packet.index; + payload = packet.payload; - switch (decoder->packet.type) { + switch (packet.type) { case ARM_SPE_TIMESTAMP: decoder->record.timestamp = payload; return 1; diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h index 3310e05122f0..0cbcb501edc9 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h @@ -147,8 +147,7 @@ struct arm_spe_decoder { const unsigned char *buf; size_t len; - - struct arm_spe_pkt packet; + u64 midr; }; struct arm_spe_decoder *arm_spe_decoder_new(struct arm_spe_params *params); diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c index 5769ba2f4140..600677318f84 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c @@ -8,6 +8,7 @@ #include <string.h> #include <endian.h> #include <byteswap.h> +#include <linux/bitmap.h> #include <linux/bitops.h> #include <stdarg.h> #include <linux/kernel.h> @@ -15,6 +16,8 @@ #include "arm-spe-pkt-decoder.h" +#include "../../arm64/include/asm/cputype.h" + static const char * const arm_spe_packet_name[] = { [ARM_SPE_PAD] = "PAD", [ARM_SPE_END] = "END", @@ -222,11 +225,12 @@ static int arm_spe_do_get_packet(const unsigned char *buf, size_t len, } int arm_spe_get_packet(const unsigned char *buf, size_t len, - struct arm_spe_pkt *packet) + struct arm_spe_pkt *packet, u64 midr) { int ret; ret = arm_spe_do_get_packet(buf, len, packet); + packet->midr = midr; /* put multiple consecutive PADs on the same line, up to * the fixed-width output format of 16 bytes per line. */ @@ -276,6 +280,73 @@ static int arm_spe_pkt_out_string(int *err, char **buf_p, size_t *blen, return ret; } +struct ev_string { + u8 event; + const char *desc; +}; + +static const struct ev_string common_ev_strings[] = { + { .event = EV_EXCEPTION_GEN, .desc = "EXCEPTION-GEN" }, + { .event = EV_RETIRED, .desc = "RETIRED" }, + { .event = EV_L1D_ACCESS, .desc = "L1D-ACCESS" }, + { .event = EV_L1D_REFILL, .desc = "L1D-REFILL" }, + { .event = EV_TLB_ACCESS, .desc = "TLB-ACCESS" }, + { .event = EV_TLB_WALK, .desc = "TLB-REFILL" }, + { .event = EV_NOT_TAKEN, .desc = "NOT-TAKEN" }, + { .event = EV_MISPRED, .desc = "MISPRED" }, + { .event = EV_LLC_ACCESS, .desc = "LLC-ACCESS" }, + { .event = EV_LLC_MISS, .desc = "LLC-REFILL" }, + { .event = EV_REMOTE_ACCESS, .desc = "REMOTE-ACCESS" }, + { .event = EV_ALIGNMENT, .desc = "ALIGNMENT" }, + { .event = EV_TRANSACTIONAL, .desc = "TXN" }, + { .event = EV_PARTIAL_PREDICATE, .desc = "SVE-PARTIAL-PRED" }, + { .event = EV_EMPTY_PREDICATE, .desc = "SVE-EMPTY-PRED" }, + { .event = EV_L2D_ACCESS, .desc = "L2D-ACCESS" }, + { .event = EV_L2D_MISS, .desc = "L2D-MISS" }, + { .event = EV_CACHE_DATA_MODIFIED, .desc = "HITM" }, + { .event = EV_RECENTLY_FETCHED, .desc = "LFB" }, + { .event = EV_DATA_SNOOPED, .desc = "SNOOPED" }, + { .event = EV_STREAMING_SVE_MODE, .desc = "STREAMING-SVE" }, + { .event = EV_SMCU, .desc = "SMCU" }, + { .event = 0, .desc = NULL }, +}; + +static const struct ev_string n1_event_strings[] = { + { .event = 12, .desc = "LATE-PREFETCH" }, + { .event = 0, .desc = NULL }, +}; + +static u64 print_event_list(int *err, char **buf, size_t *buf_len, + const struct ev_string *ev_strings, u64 payload) +{ + for (const struct ev_string *ev = ev_strings; ev->desc != NULL; ev++) { + if (payload & BIT_ULL(ev->event)) + arm_spe_pkt_out_string(err, buf, buf_len, " %s", ev->desc); + payload &= ~BIT_ULL(ev->event); + } + return payload; +} + +struct event_print_handle { + const struct midr_range *midr_ranges; + const struct ev_string *ev_strings; +}; + +#define EV_PRINT(range, strings) \ + { \ + .midr_ranges = range, \ + .ev_strings = strings, \ + } + +static const struct midr_range n1_event_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + {}, +}; + +static const struct event_print_handle event_print_handles[] = { + EV_PRINT(n1_event_encoding_cpus, n1_event_strings), +}; + static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, char *buf, size_t buf_len) { @@ -283,51 +354,34 @@ static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, int err = 0; arm_spe_pkt_out_string(&err, &buf, &buf_len, "EV"); + payload = print_event_list(&err, &buf, &buf_len, common_ev_strings, + payload); + + /* Try to decode IMPDEF bits for known CPUs */ + for (unsigned int i = 0; i < ARRAY_SIZE(event_print_handles); i++) { + if (is_midr_in_range_list(packet->midr, + event_print_handles[i].midr_ranges)) + payload = print_event_list(&err, &buf, &buf_len, + event_print_handles[i].ev_strings, + payload); + } - if (payload & BIT(EV_EXCEPTION_GEN)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCEPTION-GEN"); - if (payload & BIT(EV_RETIRED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " RETIRED"); - if (payload & BIT(EV_L1D_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-ACCESS"); - if (payload & BIT(EV_L1D_REFILL)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " L1D-REFILL"); - if (payload & BIT(EV_TLB_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-ACCESS"); - if (payload & BIT(EV_TLB_WALK)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " TLB-REFILL"); - if (payload & BIT(EV_NOT_TAKEN)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " NOT-TAKEN"); - if (payload & BIT(EV_MISPRED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " MISPRED"); - if (payload & BIT(EV_LLC_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-ACCESS"); - if (payload & BIT(EV_LLC_MISS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " LLC-REFILL"); - if (payload & BIT(EV_REMOTE_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " REMOTE-ACCESS"); - if (payload & BIT(EV_ALIGNMENT)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " ALIGNMENT"); - if (payload & BIT(EV_TRANSACTIONAL)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " TXN"); - if (payload & BIT(EV_PARTIAL_PREDICATE)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-PARTIAL-PRED"); - if (payload & BIT(EV_EMPTY_PREDICATE)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-EMPTY-PRED"); - if (payload & BIT(EV_L2D_ACCESS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " L2D-ACCESS"); - if (payload & BIT(EV_L2D_MISS)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " L2D-MISS"); - if (payload & BIT(EV_CACHE_DATA_MODIFIED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " HITM"); - if (payload & BIT(EV_RECENTLY_FETCHED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " LFB"); - if (payload & BIT(EV_DATA_SNOOPED)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " SNOOPED"); - if (payload & BIT(EV_STREAMING_SVE_MODE)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " STREAMING-SVE"); - if (payload & BIT(EV_SMCU)) - arm_spe_pkt_out_string(&err, &buf, &buf_len, " SMCU"); + /* + * Print remaining IMPDEF bits that weren't printed above as raw + * "IMPDEF:1,2,3,4" etc. + */ + if (payload) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " IMPDEF:"); + for (int i = 0; i < 64; i++) { + const char *sep = payload & (payload - 1) ? "," : ""; + + if (payload & BIT_ULL(i)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "%d%s", i, + sep); + payload &= ~BIT_ULL(i); + } + } + } return err; } diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h index adf4cde320aa..a3300bec4990 100644 --- a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h @@ -11,7 +11,7 @@ #include <stddef.h> #include <stdint.h> -#define ARM_SPE_PKT_DESC_MAX 256 +#define ARM_SPE_PKT_DESC_MAX 512 #define ARM_SPE_NEED_MORE_BYTES -1 #define ARM_SPE_BAD_PACKET -2 @@ -35,6 +35,7 @@ struct arm_spe_pkt { enum arm_spe_pkt_type type; unsigned char index; uint64_t payload; + uint64_t midr; }; /* Short header (HEADER0) and extended header (HEADER1) */ @@ -184,7 +185,7 @@ enum arm_spe_events { const char *arm_spe_pkt_name(enum arm_spe_pkt_type); int arm_spe_get_packet(const unsigned char *buf, size_t len, - struct arm_spe_pkt *packet); + struct arm_spe_pkt *packet, u64 midr); int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, size_t len); #endif diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index e5835042acdf..552f063f126e 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -134,8 +134,10 @@ struct data_source_handle { .ds_synth = arm_spe__synth_##func, \ } +static int arm_spe__get_midr(struct arm_spe *spe, int cpu, u64 *midr); + static void arm_spe_dump(struct arm_spe *spe __maybe_unused, - unsigned char *buf, size_t len) + unsigned char *buf, size_t len, u64 midr) { struct arm_spe_pkt packet; size_t pos = 0; @@ -148,7 +150,8 @@ static void arm_spe_dump(struct arm_spe *spe __maybe_unused, len); while (len) { - ret = arm_spe_get_packet(buf, len, &packet); + ret = arm_spe_get_packet(buf, len, &packet, midr); + if (ret > 0) pkt_len = ret; else @@ -174,10 +177,10 @@ static void arm_spe_dump(struct arm_spe *spe __maybe_unused, } static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf, - size_t len) + size_t len, u64 midr) { printf(".\n"); - arm_spe_dump(spe, buf, len); + arm_spe_dump(spe, buf, len, midr); } static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data) @@ -302,8 +305,10 @@ static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe, if (speq->thread) { speq->pid = thread__pid(speq->thread); - if (queue->cpu == -1) + if (queue->cpu == -1) { speq->cpu = thread__cpu(speq->thread); + arm_spe__get_midr(spe, speq->cpu, &speq->decoder->midr); + } } } @@ -482,10 +487,30 @@ static void arm_spe__prep_branch_stack(struct arm_spe_queue *speq) bstack->hw_idx = -1ULL; } -static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type) +static int arm_spe__inject_event(struct arm_spe *spe, union perf_event *event, + struct perf_sample *sample, u64 type) { - event->header.size = perf_event__sample_event_size(sample, type, 0); - return perf_event__synthesize_sample(event, type, 0, sample); + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && spe->session && spe->session->evlist) + evsel = evlist__id2evsel(spe->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + event->header.type = PERF_RECORD_SAMPLE; + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); } static inline int @@ -497,7 +522,7 @@ arm_spe_deliver_synth_event(struct arm_spe *spe, int ret; if (spe->synth_opts.inject) { - ret = arm_spe__inject_event(event, sample, spe->sample_type); + ret = arm_spe__inject_event(spe, event, sample, spe->sample_type); if (ret) return ret; } @@ -972,14 +997,9 @@ static void arm_spe__synth_memory_level(struct arm_spe_queue *speq, } } -static void arm_spe__synth_ds(struct arm_spe_queue *speq, - const struct arm_spe_record *record, - union perf_mem_data_src *data_src) +static int arm_spe__get_midr(struct arm_spe *spe, int cpu, u64 *midr) { - struct arm_spe *spe = speq->spe; - u64 *metadata = NULL; - u64 midr; - unsigned int i; + u64 *metadata; /* Metadata version 1 assumes all CPUs are the same (old behavior) */ if (spe->metadata_ver == 1) { @@ -987,15 +1007,35 @@ static void arm_spe__synth_ds(struct arm_spe_queue *speq, pr_warning_once("Old SPE metadata, re-record to improve decode accuracy\n"); cpuid = perf_env__cpuid(perf_session__env(spe->session)); - midr = strtol(cpuid, NULL, 16); - } else { - metadata = arm_spe__get_metadata_by_cpu(spe, speq->cpu); - if (!metadata) - return; + if (!cpuid) + goto err; - midr = metadata[ARM_SPE_CPU_MIDR]; + *midr = strtol(cpuid, NULL, 16); + return 0; } + metadata = arm_spe__get_metadata_by_cpu(spe, cpu); + if (!metadata) + goto err; + + *midr = metadata[ARM_SPE_CPU_MIDR]; + return 0; + +err: + pr_warning_once("Failed to get MIDR for CPU %d\n", cpu); + return -EINVAL; +} + +static void arm_spe__synth_ds(struct arm_spe_queue *speq, + const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + u64 midr; + unsigned int i; + + if (arm_spe__get_midr(speq->spe, speq->cpu, &midr)) + return; + for (i = 0; i < ARRAY_SIZE(data_source_handles); i++) { if (is_midr_in_range_list(midr, data_source_handles[i].midr_ranges)) { return data_source_handles[i].ds_synth(record, data_src); @@ -1233,6 +1273,7 @@ static int arm_spe__setup_queue(struct arm_spe *spe, if (queue->cpu != -1) speq->cpu = queue->cpu; + arm_spe__get_midr(spe, queue->cpu, &speq->decoder->midr); if (!speq->on_heap) { int ret; @@ -1475,8 +1516,11 @@ static int arm_spe_process_auxtrace_event(struct perf_session *session, /* Dump here now we have copied a piped trace out of the pipe */ if (dump_trace) { if (auxtrace_buffer__get_data(buffer, fd)) { + u64 midr = 0; + + arm_spe__get_midr(spe, buffer->cpu.cpu, &midr); arm_spe_dump_event(spe, buffer->data, - buffer->size); + buffer->size, midr); auxtrace_buffer__put_data(buffer); } } @@ -1958,7 +2002,7 @@ int arm_spe_process_auxtrace_info(union perf_event *event, spe->tc.time_mult = tc->time_mult; spe->tc.time_zero = tc->time_zero; - if (event_contains(*tc, time_cycles)) { + if (event_contains(*tc, cap_user_time_short)) { spe->tc.time_cycles = tc->time_cycles; spe->tc.time_mask = tc->time_mask; spe->tc.cap_user_time_zero = tc->cap_user_time_zero; diff --git a/tools/perf/util/arm64-frame-pointer-unwind-support.c b/tools/perf/util/arm64-frame-pointer-unwind-support.c index 858ce2b01812..3af8c7a466e0 100644 --- a/tools/perf/util/arm64-frame-pointer-unwind-support.c +++ b/tools/perf/util/arm64-frame-pointer-unwind-support.c @@ -2,6 +2,7 @@ #include "arm64-frame-pointer-unwind-support.h" #include "callchain.h" #include "event.h" +#include "record.h" #include "unwind.h" #include <string.h> @@ -16,6 +17,11 @@ struct entries { #define SMPL_REG_MASK(b) (1ULL << (b)) +void add_leaf_frame_caller_opts_aarch64(struct record_opts *opts) +{ + opts->sample_user_regs |= SMPL_REG_MASK(PERF_REG_ARM64_LR); +} + static bool get_leaf_frame_caller_enabled(struct perf_sample *sample) { struct regs_dump *regs; diff --git a/tools/perf/util/arm64-frame-pointer-unwind-support.h b/tools/perf/util/arm64-frame-pointer-unwind-support.h index 42d3a45490f5..ba35b295bfcd 100644 --- a/tools/perf/util/arm64-frame-pointer-unwind-support.h +++ b/tools/perf/util/arm64-frame-pointer-unwind-support.h @@ -5,8 +5,10 @@ #include <linux/types.h> struct perf_sample; +struct record_opts; struct thread; +void add_leaf_frame_caller_opts_aarch64(struct record_opts *opts); u64 get_leaf_frame_caller_aarch64(struct perf_sample *sample, struct thread *thread, int user_idx); #endif /* __PERF_ARM_FRAME_POINTER_UNWIND_SUPPORT_H */ diff --git a/tools/perf/util/aslr.c b/tools/perf/util/aslr.c new file mode 100644 index 000000000000..6a7542e7db82 --- /dev/null +++ b/tools/perf/util/aslr.c @@ -0,0 +1,1409 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "aslr.h" + +#include "addr_location.h" +#include "debug.h" +#include "event.h" +#include "evsel.h" +#include "evlist.h" +#include "machine.h" +#include "map.h" +#include "thread.h" +#include "tool.h" +#include "session.h" +#include "data.h" +#include "dso.h" +#include "pmus.h" + +#include <internal/lib.h> /* page_size */ +#include <linux/compiler.h> +#include <linux/zalloc.h> +#include <errno.h> +#include <inttypes.h> +#include <unistd.h> +#include <byteswap.h> + +/** + * struct remap_addresses_key - Key for mapping original addresses to remapped ones. + * @dso: Pointer to the DSO (Dynamic Shared Object) associated with the mapping. + * @invariant: Unique offset invariant within the VMA (Virtual Memory Area). + * Calculated as `start - pgoff`. This value remains constant when + * perf's internal `maps__fixup_overlap_and_insert` splits a map into + * fragmented VMA pieces due to overlapping events, allowing us to + * resolve split maps consistently back to the original VMA. + * @pid: Process ID associated with the mapping. + */ +struct remap_addresses_key { + struct machine *machine; + struct dso *dso; + u64 invariant; + pid_t pid; +}; + +struct aslr_mapping { + struct list_head node; + u64 orig_start; + u64 len; + u64 remap_start; +}; + +struct aslr_evsel_priv { + u64 orig_sample_type; + u64 orig_sample_regs_user; + u64 orig_sample_regs_intr; + int orig_sample_size; +}; + +static size_t evsel_hash(long key, void *ctx __maybe_unused) +{ + return (size_t)key; +} + +static bool evsel_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +struct process_top_address { + u64 remapped_max; +}; +struct aslr_tool { + /** @tool: The tool implemented here and a pointer to a delegate to process the data. */ + struct delegate_tool tool; + /** @machines: The machines with the input, not remapped, virtual address layout. */ + struct machines machines; + /** @event_copy: Buffer used to create an event to pass to the delegate. */ + char event_copy[PERF_SAMPLE_MAX_SIZE] __aligned(8); + /** @remap_addresses: mapping from remap_addresses_key to remapped address. */ + struct hashmap remap_addresses; + /** @top_addresses: mapping from process to max remapped address. */ + struct hashmap top_addresses; + /** + * @evsel_orig_attrs: mapping from evsel pointer to its original + * unstripped sample_type and registers bitmasks. + */ + struct hashmap evsel_orig_attrs; +}; + +static const pid_t kernel_pid = -1; + +/* Start remapping user processes from a small non-zero offset. */ +static const u64 user_space_start = 0x200000; +static const u64 kernel_space_start_64 = 0xffff800010000000ULL; +static const u64 kernel_space_start_32 = 0x80000000ULL; + +static size_t remap_addresses__hash(long _key, void *ctx __maybe_unused) +{ + struct remap_addresses_key *key = (struct remap_addresses_key *)_key; + void *dso_ptr = key->dso ? RC_CHK_ACCESS(key->dso) : NULL; + + return (size_t)key->machine ^ (size_t)dso_ptr ^ key->invariant ^ key->pid; +} + +static bool remap_addresses__equal(long _key1, long _key2, void *ctx __maybe_unused) +{ + struct remap_addresses_key *key1 = (struct remap_addresses_key *)_key1; + struct remap_addresses_key *key2 = (struct remap_addresses_key *)_key2; + + return key1->machine == key2->machine && + RC_CHK_EQUAL(key1->dso, key2->dso) && + key1->invariant == key2->invariant && + key1->pid == key2->pid; +} + +struct top_addresses_key { + struct machine *machine; + pid_t pid; +}; + +static size_t top_addresses__hash(long _key, void *ctx __maybe_unused) +{ + struct top_addresses_key *key = (struct top_addresses_key *)_key; + + return (size_t)key->machine ^ key->pid; +} + +static bool top_addresses__equal(long _key1, long _key2, void *ctx __maybe_unused) +{ + struct top_addresses_key *key1 = (struct top_addresses_key *)_key1; + struct top_addresses_key *key2 = (struct top_addresses_key *)_key2; + + return key1->machine == key2->machine && key1->pid == key2->pid; +} + +static u64 round_up_to_page_size(u64 addr) +{ + return (addr + page_size - 1) & ~((u64)page_size - 1); +} + +static u64 aslr_tool__remap_address(struct aslr_tool *aslr, + struct thread *aslr_thread, + u8 cpumode, + u64 addr) +{ + struct addr_location al; + struct remap_addresses_key key; + u64 *remapped_invariant_ptr = NULL; + u64 remap_addr = 0; + u8 effective_cpumode = cpumode; + struct dso *dso; + const char *dso_name; + + if (!aslr_thread) + return 0; /* No thread. */ + + addr_location__init(&al); + if (!thread__find_map(aslr_thread, cpumode, addr, &al)) { + /* + * If lookup fails with specified cpumode, try fallback to the other space + * to be robust against bad cpumode in samples. + */ + if (cpumode == PERF_RECORD_MISC_KERNEL) + effective_cpumode = PERF_RECORD_MISC_USER; + else if (cpumode == PERF_RECORD_MISC_USER) + effective_cpumode = PERF_RECORD_MISC_KERNEL; + else if (cpumode == PERF_RECORD_MISC_GUEST_KERNEL) + effective_cpumode = PERF_RECORD_MISC_GUEST_USER; + else if (cpumode == PERF_RECORD_MISC_GUEST_USER) + effective_cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + + if (!thread__find_map(aslr_thread, effective_cpumode, addr, &al)) { + addr_location__exit(&al); + return 0; /* No mmap. */ + } + } + + dso = map__dso(al.map); + dso_name = dso ? dso__long_name(dso) : NULL; + + key.machine = maps__machine(thread__maps(aslr_thread)); + key.dso = dso; + if (dso && !is_anon_memory(dso_name) && !is_no_dso_memory(dso_name)) + key.invariant = map__start(al.map) - map__pgoff(al.map); + else + key.invariant = map__start(al.map); + key.pid = (effective_cpumode == PERF_RECORD_MISC_KERNEL || + effective_cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_pid : thread__pid(aslr_thread); + + if (hashmap__find(&aslr->remap_addresses, &key, &remapped_invariant_ptr)) { + remap_addr = *remapped_invariant_ptr + map__pgoff(al.map) + + (addr - map__start(al.map)); + } else { + pr_debug("Cannot find a remapped entry for address %" PRIx64 " in mapping %" PRIx64 "(%zu) for pid=%d\n", + addr, map__start(al.map), map__size(al.map), key.pid); + } + + addr_location__exit(&al); + return remap_addr; +} + +struct aslr_machine_priv { + bool kernel_maps_loaded; +}; + +static int aslr_tool__preload_kernel_maps(struct machine *machine) +{ + struct aslr_machine_priv *mpriv = machine->priv; + + if (!mpriv) { + mpriv = zalloc(sizeof(*mpriv)); + if (!mpriv) + return -ENOMEM; + machine->priv = mpriv; + } + + if (!mpriv->kernel_maps_loaded) { + struct maps *kmaps = machine__kernel_maps(machine); + + if (kmaps) { + int err = maps__load_maps(kmaps); + + if (err < 0) { + pr_err("ASLR: Failed to preload kernel maps for machine pid %d\n", + machine->pid); + return err; + } + } + mpriv->kernel_maps_loaded = true; + } + return 0; +} + +static void aslr_tool__free_machine_priv(struct machine *machine) +{ + free(machine->priv); + machine->priv = NULL; +} + +static void aslr_tool__destroy_machines_priv(struct machines *machines) +{ + struct rb_node *nd; + + aslr_tool__free_machine_priv(&machines->host); + for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) { + struct machine *machine = rb_entry(nd, struct machine, rb_node); + + aslr_tool__free_machine_priv(machine); + } +} + +static u64 aslr_tool__findnew_mapping(struct aslr_tool *aslr, + struct machine *session_machine, + struct thread *aslr_thread, + u8 cpumode, u64 start, + u64 len, u64 pgoff) +{ + /* Address location for dso lookup. */ + struct addr_location al; + /* Original ASLR address based key for the remap table. */ + struct remap_addresses_key remap_key; + /* The address in the ASLR sanitized address space less pg_off. */ + u64 *remapped_invariant_ptr; + /* Key for the maximum address in a process. */ + struct top_addresses_key top_addr_key; + /* Value in top address table. */ + struct process_top_address *top = NULL; + /* Address in ASLR sanitized address space. */ + u64 remap_addr; + /* Potentially allocated remap table key. */ + struct remap_addresses_key *new_remap_key = NULL; + /* + * Potentially allocated remap table key. + * TODO: Avoid allocation necessary for perf 32-bit binary support. + */ + u64 *new_remap_val = NULL; + int err; + + if (!aslr_thread) + return 0; + + /* The key to look up an incoming address to the outgoing value. */ + addr_location__init(&al); + remap_key.machine = maps__machine(thread__maps(aslr_thread)); + remap_key.pid = (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_pid : thread__pid(aslr_thread); + if (thread__find_map(aslr_thread, cpumode, start, &al)) { + struct dso *dso = map__dso(al.map); + const char *dso_name = dso ? dso__long_name(dso) : NULL; + + remap_key.dso = dso; + if (dso && !is_anon_memory(dso_name) && !is_no_dso_memory(dso_name)) + remap_key.invariant = map__start(al.map) - map__pgoff(al.map); + else + remap_key.invariant = map__start(al.map); + } else { + remap_key.dso = NULL; + remap_key.invariant = start; + } + + /* The key to look up top allocated address. */ + top_addr_key.machine = remap_key.machine; + top_addr_key.pid = remap_key.pid; + + if (hashmap__find(&aslr->remap_addresses, &remap_key, &remapped_invariant_ptr)) { + /* Mmap already exists. */ + u64 calculated_max; + + if (al.map) { + /* + * The cached value is the base of the invariant. We add the + * offset into the VMA (start - map__start), plus the map's + * pgoff, to get the precise virtual address within this chunk. + */ + remap_addr = *remapped_invariant_ptr + map__pgoff(al.map) + + (start - map__start(al.map)); + } else { + /* + * For unmapped memory (e.g. kernel anonymous), the cached value + * was stored offset by pgoff. Adding pgoff yields the true remap_addr. + */ + remap_addr = *remapped_invariant_ptr + pgoff; + } + + calculated_max = remap_addr + len; + + /* See if top mapping was expanded. */ + if (hashmap__find(&aslr->top_addresses, &top_addr_key, &top)) { + if (calculated_max > top->remapped_max) + top->remapped_max = calculated_max; + } + addr_location__exit(&al); + return remap_addr; + } + /* No mmap, create an entry from the top address. */ + if (hashmap__find(&aslr->top_addresses, &top_addr_key, &top)) { + struct addr_location prev_al; + bool is_contiguous = false; + + /* Current max allocated mmap address within the process. */ + remap_addr = top->remapped_max; + + addr_location__init(&prev_al); + if (thread__find_map(aslr_thread, cpumode, start - 1, &prev_al)) { + if (map__end(prev_al.map) == start) + is_contiguous = true; + } + addr_location__exit(&prev_al); + + if (is_contiguous) { + /* Contiguous mapping, do not add 1 page gap! */ + remap_addr = round_up_to_page_size(remap_addr); + } else { + /* Give 1 page gap from current max page. */ + remap_addr = round_up_to_page_size(remap_addr); + remap_addr += page_size; + } + if (remap_addr + len > top->remapped_max) + top->remapped_max = remap_addr + len; + } else { + /* First address of the process, allocate key and first top address. */ + struct top_addresses_key *tk; + struct process_top_address *top_val; + struct perf_env *env = session_machine ? session_machine->env : NULL; + bool is_64 = env ? perf_env__kernel_is_64_bit(env) : (sizeof(void *) == 8); + u64 kernel_start_addr = is_64 ? kernel_space_start_64 : kernel_space_start_32; + + remap_addr = (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_start_addr : user_space_start; + remap_addr = round_up_to_page_size(remap_addr); + + tk = malloc(sizeof(*tk)); + top_val = malloc(sizeof(*top_val)); + if (!tk || !top_val) { + err = -ENOMEM; + } else { + *tk = top_addr_key; + top_val->remapped_max = remap_addr + len; + err = hashmap__insert(&aslr->top_addresses, tk, top_val, + HASHMAP_ADD, NULL, NULL); + } + if (err) { + errno = -err; + pr_err("Failure to add ASLR process top address %m\n"); + free(tk); + free(top_val); + addr_location__exit(&al); + return 0; + } + } + /* Create rmeapping entry. */ + new_remap_key = malloc(sizeof(*new_remap_key)); + new_remap_val = malloc(sizeof(u64)); + if (!new_remap_key || !new_remap_val) { + err = -ENOMEM; + } else { + *new_remap_key = remap_key; + new_remap_key->dso = dso__get(remap_key.dso); + if (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) { + if (al.map) { + *new_remap_val = remap_addr - + (start - map__start(al.map)) - + map__pgoff(al.map); + } else { + /* + * Subtract pgoff from the base virtual address so that + * when the lookup path adds pgoff back, it perfectly + * cancels out and returns remap_addr. + */ + *new_remap_val = remap_addr - pgoff; + } + } else { + *new_remap_val = remap_addr - (al.map ? (start - map__start(al.map)) + + map__pgoff(al.map) : pgoff); + } + err = hashmap__add(&aslr->remap_addresses, new_remap_key, new_remap_val); + if (err) + dso__put(new_remap_key->dso); + } + if (err) { + errno = -err; + pr_err("Failure to add ASLR remapping %m\n"); + free(new_remap_key); + free(new_remap_val); + addr_location__exit(&al); + return 0; + } + addr_location__exit(&al); + return remap_addr; +} + +static int aslr_tool__process_mmap(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + u8 cpumode; + struct thread *thread; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_mmap(tool, event, sample, aslr_machine); + if (err) + return err; + + thread = machine__findnew_thread(aslr_machine, event->mmap.pid, event->mmap.tid); + if (!thread) + return -ENOMEM; + memcpy(&new_event->mmap, &event->mmap, event->mmap.header.size); + /* Remaps the mmap.start. */ + new_event->mmap.start = aslr_tool__findnew_mapping(aslr, machine, thread, cpumode, + event->mmap.start, + event->mmap.len, + event->mmap.pgoff); + /* + * For anonymous memory (and kernel maps), the kernel populates the + * event's pgoff field with the original un-obfuscated virtual address + * in bytes (i.e. (addr >> PAGE_SHIFT) << PAGE_SHIFT). + * We must overwrite pgoff with the new remapped byte address to prevent + * leaking the original ASLR layout. + */ + if (is_anon_memory(event->mmap.filename) || is_no_dso_memory(event->mmap.filename) || + ((cpumode == PERF_RECORD_MISC_KERNEL || cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !is_kernel_module(event->mmap.filename, cpumode))) + new_event->mmap.pgoff = new_event->mmap.start; + err = delegate->mmap(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_mmap2(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + u8 cpumode; + struct thread *thread; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_mmap2(tool, event, sample, aslr_machine); + if (err) + return err; + + thread = machine__findnew_thread(aslr_machine, event->mmap2.pid, event->mmap2.tid); + if (!thread) + return -ENOMEM; + memcpy(&new_event->mmap2, &event->mmap2, event->mmap2.header.size); + /* Remaps the mmap.start. */ + new_event->mmap2.start = aslr_tool__findnew_mapping(aslr, machine, thread, cpumode, + event->mmap2.start, + event->mmap2.len, + event->mmap2.pgoff); + /* + * For anonymous memory (and kernel maps), the kernel populates the + * event's pgoff field with the original un-obfuscated virtual address + * in bytes (i.e. (addr >> PAGE_SHIFT) << PAGE_SHIFT). + * We must overwrite pgoff with the new remapped byte address to prevent + * leaking the original ASLR layout. + */ + if (is_anon_memory(event->mmap2.filename) || is_no_dso_memory(event->mmap2.filename) || + ((cpumode == PERF_RECORD_MISC_KERNEL || cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !is_kernel_module(event->mmap2.filename, cpumode))) + new_event->mmap2.pgoff = new_event->mmap2.start; + err = delegate->mmap2(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_comm(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_comm(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->comm(delegate, event, sample, machine); +} + +static int aslr_tool__process_fork(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_fork(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->fork(delegate, event, sample, machine); +} + +static int aslr_tool__process_exit(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_exit(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->exit(delegate, event, sample, machine); +} + +static int aslr_tool__process_text_poke(const struct perf_tool *tool __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + /* Drop in case the instruction encodes an ASLR revealing address. */ + return 0; +} + +static int aslr_tool__process_ksymbol(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + struct thread *thread; + struct machine *aslr_machine; + bool is_unregister; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + thread = machine__findnew_thread(aslr_machine, kernel_pid, 0); + if (!thread) + return -ENOMEM; + + is_unregister = (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER); + + memcpy(&new_event->ksymbol, &event->ksymbol, event->ksymbol.header.size); + + if (is_unregister) { + new_event->ksymbol.addr = aslr_tool__findnew_mapping(aslr, machine, thread, + PERF_RECORD_MISC_KERNEL, + event->ksymbol.addr, + event->ksymbol.len, + /*pgoff=*/0); + err = perf_event__process_ksymbol(tool, event, sample, aslr_machine); + } else { + err = perf_event__process_ksymbol(tool, event, sample, aslr_machine); + new_event->ksymbol.addr = aslr_tool__findnew_mapping(aslr, machine, thread, + PERF_RECORD_MISC_KERNEL, + event->ksymbol.addr, + event->ksymbol.len, + /*pgoff=*/0); + } + if (err) { + thread__put(thread); + return err; + } + + err = delegate->ksymbol(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_sample(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct evsel *evsel = sample->evsel; + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + int ret; + int orig_sample_size; + u64 sample_type; + struct thread *thread; + struct machine *aslr_machine; + __u64 max_i; + __u64 max_j; + union perf_event *new_event; + struct perf_sample new_sample; + __u64 *in_array, *out_array; + u8 cpumode; + u64 addr; + size_t i; + size_t j; + struct aslr_evsel_priv *priv = NULL; + u64 orig_sample_type; + u64 orig_regs_user; + u64 orig_regs_intr; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + if (evsel__is_dummy_event(evsel)) + return delegate->sample(delegate, event, sample, machine); + + ret = -EFAULT; + + if (hashmap__find(&aslr->evsel_orig_attrs, evsel, &priv)) { + orig_sample_type = priv->orig_sample_type; + orig_regs_user = priv->orig_sample_regs_user; + orig_regs_intr = priv->orig_sample_regs_intr; + } else { + orig_sample_type = evsel->core.attr.sample_type; + orig_regs_user = evsel->core.attr.sample_regs_user; + orig_regs_intr = evsel->core.attr.sample_regs_intr; + } + + orig_sample_size = evsel->sample_size; + + sample_type = orig_sample_type; + sample_type &= ~PERF_SAMPLE_REGS_USER; + sample_type &= ~PERF_SAMPLE_REGS_INTR; + sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + + max_i = (event->header.size - sizeof(struct perf_event_header)) / sizeof(__u64); + max_j = (PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_event_header)) / sizeof(__u64); + new_event = (union perf_event *)aslr->event_copy; + cpumode = sample->cpumode; + i = 0; + j = 0; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + thread = machine__findnew_thread(aslr_machine, sample->pid, sample->tid); + + if (!thread) + return -ENOMEM; + + if (max_i > PERF_SAMPLE_MAX_SIZE / sizeof(u64)) + goto out_put; + + new_event->sample.header = event->sample.header; + + in_array = &event->sample.array[0]; + out_array = &new_event->sample.array[0]; + +#define CHECK_BOUNDS(required_i, required_j) \ + (i + (required_i) > max_i || j + (required_j) > max_j) + +#define COPY_U64() \ + do { \ + if (CHECK_BOUNDS(1, 1)) { \ + ret = -EFAULT; \ + goto out_put; \ + } \ + out_array[j++] = in_array[i++]; \ + } while (0) + +#define REMAP_U64(addr_field) \ + do { \ + u64 remapped; \ + if (CHECK_BOUNDS(1, 1)) { \ + ret = -EFAULT; \ + goto out_put; \ + } \ + remapped = aslr_tool__remap_address(aslr, thread, cpumode, addr_field); \ + out_array[j++] = remapped; \ + i++; \ + } while (0) + + if (orig_sample_type & PERF_SAMPLE_IDENTIFIER) + COPY_U64(); /* id */ + if (orig_sample_type & PERF_SAMPLE_IP) + REMAP_U64(sample->ip); + if (orig_sample_type & PERF_SAMPLE_TID) { + union { + u64 val64; + u32 val32[2]; + } u; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + u.val32[0] = sample->pid; + u.val32[1] = sample->tid; + out_array[j++] = u.val64; + i++; + } + if (orig_sample_type & PERF_SAMPLE_TIME) + COPY_U64(); /* time */ + if (orig_sample_type & PERF_SAMPLE_ADDR) + REMAP_U64(sample->addr); + if (orig_sample_type & PERF_SAMPLE_ID) + COPY_U64(); /* id */ + if (orig_sample_type & PERF_SAMPLE_STREAM_ID) + COPY_U64(); /* stream_id */ + if (orig_sample_type & PERF_SAMPLE_CPU) + COPY_U64(); /* cpu, res */ + if (orig_sample_type & PERF_SAMPLE_PERIOD) + COPY_U64(); /* period */ + if (orig_sample_type & PERF_SAMPLE_READ) { + if ((evsel->core.attr.read_format & PERF_FORMAT_GROUP) == 0) { + COPY_U64(); /* value */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + COPY_U64(); /* time_enabled */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + COPY_U64(); /* time_running */ + if (evsel->core.attr.read_format & PERF_FORMAT_ID) + COPY_U64(); /* id */ + if (evsel->core.attr.read_format & PERF_FORMAT_LOST) + COPY_U64(); /* lost */ + } else { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + COPY_U64(); /* time_enabled */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + COPY_U64(); /* time_running */ + for (u64 cntr = 0; cntr < nr; cntr++) { + COPY_U64(); /* value */ + if (evsel->core.attr.read_format & PERF_FORMAT_ID) + COPY_U64(); /* id */ + if (evsel->core.attr.read_format & PERF_FORMAT_LOST) + COPY_U64(); /* lost */ + } + } + } + if (orig_sample_type & PERF_SAMPLE_CALLCHAIN) { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + + for (u64 cntr = 0; cntr < nr; cntr++) { + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + addr = in_array[i++]; + if (addr >= PERF_CONTEXT_MAX) { + out_array[j++] = addr; + switch (addr) { + case PERF_CONTEXT_HV: + cpumode = PERF_RECORD_MISC_HYPERVISOR; + break; + case PERF_CONTEXT_KERNEL: + cpumode = PERF_RECORD_MISC_KERNEL; + break; + case PERF_CONTEXT_USER: + cpumode = PERF_RECORD_MISC_USER; + break; + case PERF_CONTEXT_GUEST: + cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + break; + case PERF_CONTEXT_GUEST_KERNEL: + cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + break; + case PERF_CONTEXT_GUEST_USER: + cpumode = PERF_RECORD_MISC_GUEST_USER; + break; + case PERF_CONTEXT_USER_DEFERRED: + if (cntr + 1 >= nr) { + pr_debug("Truncated callchain deferred cookie context\n"); + ret = 0; + goto out_put; + } + /* + * Immediately followed by a 64-bit + * stitching cookie. Skip/Copy it! + */ + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + out_array[j++] = in_array[i++]; + cntr++; + cpumode = PERF_RECORD_MISC_USER; + break; + default: + pr_debug("invalid callchain context: %"PRIx64"\n", addr); + ret = 0; + goto out_put; + } + continue; + } + addr = aslr_tool__remap_address(aslr, thread, cpumode, addr); + out_array[j++] = addr; + } + } + if (orig_sample_type & PERF_SAMPLE_RAW) { + size_t bytes = sizeof(u32) + sample->raw_size; + size_t u64_words = (bytes + 7) / 8; + + if (i + u64_words > max_i || j + u64_words > max_j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], bytes); + i += u64_words; + j += u64_words; + /* + * TODO: certain raw samples can be remapped, such as + * tracepoints by examining their fields. + */ + pr_debug("Dropping raw samples as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_BRANCH_STACK) { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) + COPY_U64(); /* hw_idx */ + + if (nr > (ULLONG_MAX / 3)) { + ret = -EFAULT; + goto out_put; + } + if (nr * 3 > max_i - i || nr * 3 > max_j - j) { + ret = -EFAULT; + goto out_put; + } + for (u64 cntr = 0; cntr < nr; cntr++) { + u64 from = in_array[i++]; + u64 to = in_array[i++]; + + from = aslr_tool__remap_address(aslr, thread, sample->cpumode, from); + to = aslr_tool__remap_address(aslr, thread, sample->cpumode, to); + + out_array[j++] = from; + out_array[j++] = to; + out_array[j++] = in_array[i++]; /* flags */ + } + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) { + if (nr > max_i - i || nr > max_j - j) { + ret = -EFAULT; + goto out_put; + } + for (u64 cntr = 0; cntr < nr; cntr++) + COPY_U64(); + } + } + if (orig_sample_type & PERF_SAMPLE_REGS_USER) { + u64 abi; + + if (CHECK_BOUNDS(1, 0)) { + ret = -EFAULT; + goto out_put; + } + abi = in_array[i++]; + if (abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 nr = hweight64(orig_regs_user); + + if (nr > max_i - i) { + ret = -EFAULT; + goto out_put; + } + i += nr; + } + } + if (orig_sample_type & PERF_SAMPLE_STACK_USER) { + u64 size; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + size = in_array[i]; + COPY_U64(); + if (size > 0) { + size_t u64_words = size / 8 + (size % 8 ? 1 : 0); + + if (u64_words > max_i - i || u64_words > max_j - j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], size); + if (size % 8) { + size_t pad = 8 - (size % 8); + + memset(((char *)&out_array[j]) + size, 0, pad); + } + i += u64_words; + j += u64_words; + } + /* TODO: can this be less conservative? */ + pr_debug("Dropping stack user sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_WEIGHT_TYPE) + COPY_U64(); /* perf_sample_weight */ + if (orig_sample_type & PERF_SAMPLE_DATA_SRC) + COPY_U64(); /* data_src */ + if (orig_sample_type & PERF_SAMPLE_TRANSACTION) + COPY_U64(); /* transaction */ + if (orig_sample_type & PERF_SAMPLE_REGS_INTR) { + u64 abi; + + if (CHECK_BOUNDS(1, 0)) { + ret = -EFAULT; + goto out_put; + } + abi = in_array[i++]; + if (abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 nr = hweight64(orig_regs_intr); + + if (nr > max_i - i) { + ret = -EFAULT; + goto out_put; + } + i += nr; + } + } + if (orig_sample_type & PERF_SAMPLE_PHYS_ADDR) { + COPY_U64(); /* phys_addr */ + /* TODO: can this be less conservative? */ + pr_debug("Dropping physical address sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_CGROUP) + COPY_U64(); /* cgroup */ + if (orig_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) + COPY_U64(); /* data_page_size */ + if (orig_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) + COPY_U64(); /* code_page_size */ + + if (orig_sample_type & PERF_SAMPLE_AUX) { + u64 size; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + out_array[j] = in_array[i]; + size = out_array[j++]; + i++; + if (size > 0) { + size_t u64_words = size / 8 + (size % 8 ? 1 : 0); + + if (u64_words > max_i - i || u64_words > max_j - j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], size); + if (size % 8) { + size_t pad = 8 - (size % 8); + + memset(((char *)&out_array[j]) + size, 0, pad); + } + i += u64_words; + j += u64_words; + } + /* TODO: can this be less conservative? */ + pr_debug("Dropping aux sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + + if (evsel__is_offcpu_event(evsel)) { + /* TODO: can this be less conservative? */ + pr_debug("Dropping off-CPU sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + + new_event->sample.header.size = sizeof(struct perf_event_header) + j * sizeof(u64); + /* Temporarily override evsel attributes to match the stripped new_event format! */ + evsel->sample_size = __evsel__sample_size(sample_type); + evsel->core.attr.sample_type = sample_type; + evsel->core.attr.sample_regs_user = 0; + evsel->core.attr.sample_regs_intr = 0; + perf_sample__init(&new_sample, /*all=*/ true); + ret = __evsel__parse_sample(evsel, new_event, &new_sample, /*needs_swap=*/false); + + if (ret) { + /* Restore original attributes immediately if parsing fails */ + evsel->sample_size = orig_sample_size; + evsel->core.attr.sample_type = orig_sample_type; + evsel->core.attr.sample_regs_user = orig_regs_user; + evsel->core.attr.sample_regs_intr = orig_regs_intr; + perf_sample__exit(&new_sample); + goto out_put; + } + + new_sample.evsel = evsel; + ret = delegate->sample(delegate, new_event, &new_sample, machine); + perf_sample__exit(&new_sample); + + /* Restore original attributes so trace ingestion never desynchronizes! */ + evsel->sample_size = orig_sample_size; + evsel->core.attr.sample_type = orig_sample_type; + evsel->core.attr.sample_regs_user = orig_regs_user; + evsel->core.attr.sample_regs_intr = orig_regs_intr; + +out_put: + thread__put(thread); + return ret; +} + +#undef CHECK_BOUNDS +#undef COPY_U64 +#undef REMAP_U64 + +static int skipn(int fd, off_t n) +{ + char buf[4096]; + ssize_t ret; + + while (n > 0) { + ret = read(fd, buf, min_t(off_t, n, (off_t)sizeof(buf))); + if (ret <= 0) + return ret; + n -= ret; + } + + return 0; +} + +static s64 aslr_tool__process_auxtrace(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + pr_warning_once("ASLR: Dropping auxtrace data as it cannot be obfuscated.\n"); + if (perf_data__is_pipe(session->data)) { + /* Copy behavior of the stub by reading all pipe data. */ + int err = skipn(perf_data__fd(session->data), event->auxtrace.size); + + if (err < 0) + return err; + } + return event->auxtrace.size; +} + +static int aslr_tool__process_auxtrace_info(const struct perf_tool *tool __maybe_unused, + struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +static int aslr_tool__process_auxtrace_error(const struct perf_tool *tool __maybe_unused, + struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +void aslr_tool__strip_attr_event(union perf_event *event, struct evlist *evlist) +{ + u32 attr_size; + + if (!evlist) + return; + + attr_size = event->attr.attr.size ?: PERF_ATTR_SIZE_VER0; + + if (attr_size >= (offsetof(struct perf_event_attr, sample_type) + sizeof(u64))) { + event->attr.attr.sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + + if (attr_size >= (offsetof(struct perf_event_attr, sample_regs_user) + sizeof(u64))) + event->attr.attr.sample_regs_user = 0; + if (attr_size >= (offsetof(struct perf_event_attr, sample_regs_intr) + sizeof(u64))) + event->attr.attr.sample_regs_intr = 0; + } + + if (attr_size >= (offsetof(struct perf_event_attr, type) + sizeof(u32))) { + u32 type = event->attr.attr.type; + + if (type == PERF_TYPE_BREAKPOINT && + attr_size >= (offsetof(struct perf_event_attr, bp_addr) + sizeof(u64))) { + event->attr.attr.bp_addr = 0; + } else if (type >= PERF_TYPE_MAX) { + struct perf_pmu *pmu; + + pmu = perf_pmus__find_by_type(type); + if (pmu && (!strcmp(pmu->name, "kprobe") || + !strcmp(pmu->name, "uprobe"))) { + if (attr_size >= (offsetof(struct perf_event_attr, config1) + sizeof(u64))) + event->attr.attr.config1 = 0; + if (attr_size >= (offsetof(struct perf_event_attr, config2) + sizeof(u64))) + event->attr.attr.config2 = 0; + } + } + } +} + +static int aslr_tool__init(struct aslr_tool *aslr, struct perf_tool *delegate) +{ + delegate_tool__init(&aslr->tool, delegate); + aslr->tool.tool.ordered_events = true; + + if (machines__init(&aslr->machines)) + return -ENOMEM; + + hashmap__init(&aslr->remap_addresses, + remap_addresses__hash, remap_addresses__equal, + /*ctx=*/NULL); + hashmap__init(&aslr->top_addresses, + top_addresses__hash, top_addresses__equal, + /*ctx=*/NULL); + hashmap__init(&aslr->evsel_orig_attrs, + evsel_hash, evsel_equal, + /*ctx=*/NULL); + + aslr->tool.tool.sample = aslr_tool__process_sample; + /* read - reads a counter, okay to delegate. */ + aslr->tool.tool.mmap = aslr_tool__process_mmap; + aslr->tool.tool.mmap2 = aslr_tool__process_mmap2; + aslr->tool.tool.comm = aslr_tool__process_comm; + aslr->tool.tool.fork = aslr_tool__process_fork; + aslr->tool.tool.exit = aslr_tool__process_exit; + /* namesspaces, cgroup, lost, lost_sample, aux, */ + /* itrace_start, aux_output_hw_id, context_switch, throttle, unthrottle */ + /* - no virtual addresses. */ + aslr->tool.tool.ksymbol = aslr_tool__process_ksymbol; + /* bpf - no virtual address. */ + aslr->tool.tool.text_poke = aslr_tool__process_text_poke; + /* + * event_update, tracing_data, finished_round, build_id, id_index, + * auxtrace_info, auxtrace_error, time_conv, thread_map, cpu_map, + * stat_config, stat, feature, finished_init, bpf_metadata, compressed, + * auxtrace - no virtual addresses. + */ + aslr->tool.tool.auxtrace = aslr_tool__process_auxtrace; + aslr->tool.tool.auxtrace_info = aslr_tool__process_auxtrace_info; + aslr->tool.tool.auxtrace_error = aslr_tool__process_auxtrace_error; + + return 0; +} + +struct perf_tool *aslr_tool__new(struct perf_tool *delegate) +{ + struct aslr_tool *aslr = zalloc(sizeof(*aslr)); + + if (!aslr) + return NULL; + + if (aslr_tool__init(aslr, delegate)) { + free(aslr); + return NULL; + } + return &aslr->tool.tool; +} + +void aslr_tool__delete(struct perf_tool *tool) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct hashmap_entry *cur; + size_t bkt; + struct rb_node *nd; + + if (!tool) + return; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + + hashmap__for_each_entry(&aslr->remap_addresses, cur, bkt) { + struct remap_addresses_key *key = (struct remap_addresses_key *)cur->pkey; + + if (key) + dso__put(key->dso); + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__for_each_entry(&aslr->top_addresses, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__for_each_entry(&aslr->evsel_orig_attrs, cur, bkt) { + zfree(&cur->pvalue); + } + + hashmap__clear(&aslr->remap_addresses); + hashmap__clear(&aslr->top_addresses); + hashmap__clear(&aslr->evsel_orig_attrs); + aslr_tool__destroy_machines_priv(&aslr->machines); + machines__destroy_kernel_maps(&aslr->machines); + + while ((nd = rb_first_cached(&aslr->machines.guests)) != NULL) { + struct machine *machine = rb_entry(nd, struct machine, rb_node); + + rb_erase_cached(nd, &aslr->machines.guests); + machine__delete(machine); + } + + machines__exit(&aslr->machines); + free(aslr); +} + +int aslr_tool__cache_orig_attrs(struct perf_tool *tool, struct evsel *evsel) +{ + struct delegate_tool *del_tool = container_of(tool, struct delegate_tool, tool); + struct aslr_tool *aslr = container_of(del_tool, struct aslr_tool, tool); + struct aslr_evsel_priv *priv = zalloc(sizeof(*priv)); + int err; + + if (!priv) + return -ENOMEM; + + priv->orig_sample_type = evsel->core.attr.sample_type; + priv->orig_sample_regs_user = evsel->core.attr.sample_regs_user; + priv->orig_sample_regs_intr = evsel->core.attr.sample_regs_intr; + priv->orig_sample_size = evsel->sample_size; + + err = hashmap__add(&aslr->evsel_orig_attrs, evsel, priv); + if (err) { + free(priv); + return err; + } + return 0; +} + +void aslr_tool__strip_evlist(const struct perf_tool *tool __maybe_unused, struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + evsel->core.attr.sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + evsel->core.attr.sample_regs_user = 0; + evsel->core.attr.sample_regs_intr = 0; + evsel->sample_size = __evsel__sample_size(evsel->core.attr.sample_type); + evsel__calc_id_pos(evsel); + + if (evsel->core.attr.type == PERF_TYPE_BREAKPOINT) { + evsel->core.attr.bp_addr = 0; + } else if (evsel->core.attr.type >= PERF_TYPE_MAX) { + struct perf_pmu *pmu = perf_pmus__find_by_type(evsel->core.attr.type); + + if (pmu && (!strcmp(pmu->name, "kprobe") || + !strcmp(pmu->name, "uprobe"))) { + evsel->core.attr.config1 = 0; + evsel->core.attr.config2 = 0; + } + } + } +} + +void aslr_tool__restore_evlist(const struct perf_tool *tool, struct evlist *evlist) +{ + const struct delegate_tool *del_tool = container_of(tool, const struct delegate_tool, tool); + const struct aslr_tool *aslr = container_of(del_tool, const struct aslr_tool, tool); + struct evsel *evsel; + struct aslr_evsel_priv *priv; + + evlist__for_each_entry(evlist, evsel) { + if (hashmap__find(&aslr->evsel_orig_attrs, evsel, &priv)) { + evsel->core.attr.sample_type = priv->orig_sample_type; + evsel->core.attr.sample_regs_user = priv->orig_sample_regs_user; + evsel->core.attr.sample_regs_intr = priv->orig_sample_regs_intr; + evsel->sample_size = priv->orig_sample_size; + evsel__calc_id_pos(evsel); + } + } +} diff --git a/tools/perf/util/aslr.h b/tools/perf/util/aslr.h new file mode 100644 index 000000000000..522e31c8e2c0 --- /dev/null +++ b/tools/perf/util/aslr.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ASLR_H +#define __PERF_ASLR_H + +#include <linux/perf_event.h> + +#define ASLR_SUPPORTED_SAMPLE_TYPE ( \ + PERF_SAMPLE_IDENTIFIER | \ + PERF_SAMPLE_IP | \ + PERF_SAMPLE_TID | \ + PERF_SAMPLE_TIME | \ + PERF_SAMPLE_ADDR | \ + PERF_SAMPLE_ID | \ + PERF_SAMPLE_STREAM_ID | \ + PERF_SAMPLE_CPU | \ + PERF_SAMPLE_PERIOD | \ + PERF_SAMPLE_READ | \ + PERF_SAMPLE_CALLCHAIN | \ + PERF_SAMPLE_RAW | \ + PERF_SAMPLE_BRANCH_STACK | \ + PERF_SAMPLE_STACK_USER | \ + PERF_SAMPLE_WEIGHT_TYPE | \ + PERF_SAMPLE_DATA_SRC | \ + PERF_SAMPLE_TRANSACTION | \ + PERF_SAMPLE_PHYS_ADDR | \ + PERF_SAMPLE_CGROUP | \ + PERF_SAMPLE_DATA_PAGE_SIZE | \ + PERF_SAMPLE_CODE_PAGE_SIZE | \ + PERF_SAMPLE_AUX) + +struct perf_tool; +struct evsel; +struct evlist; +union perf_event; + +struct perf_tool *aslr_tool__new(struct perf_tool *delegate); +void aslr_tool__delete(struct perf_tool *tool); + +void aslr_tool__strip_attr_event(union perf_event *event, struct evlist *evlist); +int aslr_tool__cache_orig_attrs(struct perf_tool *tool, struct evsel *evsel); +void aslr_tool__strip_evlist(const struct perf_tool *tool, struct evlist *evlist); +void aslr_tool__restore_evlist(const struct perf_tool *tool, struct evlist *evlist); + +#endif /* __PERF_ASLR_H */ diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index a224687ffbc1..4cd2caf54015 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -372,7 +372,8 @@ static bool filter_cpu(struct perf_session *session, struct perf_cpu cpu) { unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap; - return cpu_bitmap && cpu.cpu != -1 && !test_bit(cpu.cpu, cpu_bitmap); + return cpu_bitmap && cpu.cpu >= 0 && cpu.cpu < MAX_NR_CPUS && + !test_bit(cpu.cpu, cpu_bitmap); } static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues, @@ -896,6 +897,21 @@ regroup: return 0; } +/** + * auxtrace_record__init - Initialize an AUX area tracing record. + * @evlist: The list of events to check for AUX area tracing event. + * @err: Pointer to an integer to store return code. + * + * This function looks through the @evlist to determine which AUX area + * tracing hardware is being used and initializes the auxtrace_record + * structure. + * + * Return: + * a) A pointer to the struct auxtrace_record with @err = 0 on success. + * b) NULL with @err = 0 if no AUX area tracing event is found/supported + * (not considered an error). + * c) NULL with non-zero @err on actual auxtrace_record__init failure. + */ struct auxtrace_record *__weak auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err) { @@ -1759,7 +1775,7 @@ static const char * const auxtrace_error_type_name[] = { [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace", }; -static const char *auxtrace_error_name(int type) +static const char *auxtrace_error_name(unsigned int type) { const char *error_type_name = NULL; @@ -1775,6 +1791,7 @@ size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp) struct perf_record_auxtrace_error *e = &event->auxtrace_error; unsigned long long nsecs = e->time; const char *msg = e->msg; + int msg_max; int ret; ret = fprintf(fp, " %s error type %u", @@ -1792,11 +1809,26 @@ size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp) if (!e->fmt) msg = (const char *)&e->time; - if (e->fmt >= 2 && e->machine_pid) + /* Bound msg to the bytes actually within the event, capped at the array size */ + msg_max = (int)((void *)event + event->header.size - (void *)msg); + if (msg_max < 0) + msg_max = 0; + if (msg_max > (int)sizeof(e->msg)) + msg_max = sizeof(e->msg); + + /* + * Unlike the swap path which downgrades fmt in place, + * native-endian events are mmap'd read-only — check size + * instead to avoid accessing machine_pid/vcpu OOB. + */ + if (e->fmt >= 2 && + event->header.size >= offsetof(typeof(event->auxtrace_error), vcpu) + + sizeof(event->auxtrace_error.vcpu) && + e->machine_pid) ret += fprintf(fp, " machine_pid %d vcpu %d", e->machine_pid, e->vcpu); - ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %s\n", - e->cpu, e->pid, e->tid, e->ip, e->code, msg); + ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %.*s\n", + e->cpu, e->pid, e->tid, e->ip, e->code, msg_max, msg); return ret; } @@ -2663,7 +2695,7 @@ static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt, { /* Same name, and global or the n'th found or any */ return !arch__compare_symbol_names(name, sym->name) && - ((!idx && sym->binding == STB_GLOBAL) || + ((!idx && symbol__binding(sym) == STB_GLOBAL) || (idx > 0 && ++*cnt == idx) || idx < 0); } @@ -2681,8 +2713,8 @@ static void print_duplicate_syms(struct dso *dso, const char *sym_name) if (dso_sym_match(sym, sym_name, &cnt, -1)) { pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n", ++cnt, sym->start, - sym->binding == STB_GLOBAL ? 'g' : - sym->binding == STB_LOCAL ? 'l' : 'w', + symbol__binding(sym) == STB_GLOBAL ? 'g' : + symbol__binding(sym) == STB_LOCAL ? 'l' : 'w', sym->name); near = true; } else if (near) { diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c index a27945c279ef..fa3ebc8ea7f0 100644 --- a/tools/perf/util/bpf-event.c +++ b/tools/perf/util/bpf-event.c @@ -36,7 +36,7 @@ static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) size_t i; for (i = 0; i < len; i++) - ret += snprintf(buf + ret, size - ret, "%02x", data[i]); + ret += scnprintf(buf + ret, size - ret, "%02x", data[i]); return ret; } @@ -59,6 +59,10 @@ static int machine__process_bpf_event_load(struct machine *machine, return 0; info_linear = info_node->info_linear; + /* jited_ksyms is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_KSYMS))) + return 0; + for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); u64 addr = addrs[i]; @@ -140,22 +144,26 @@ static int synthesize_bpf_prog_name(char *buf, int size, const struct btf_type *t; int name_len; - name_len = snprintf(buf, size, "bpf_prog_"); + name_len = scnprintf(buf, size, "bpf_prog_"); name_len += snprintf_hex(buf + name_len, size - name_len, prog_tags[sub_id], BPF_TAG_SIZE); - if (btf) { + if (btf && + info->func_info_rec_size >= sizeof(*finfo) && + sub_id < info->nr_func_info) { finfo = func_infos + sub_id * info->func_info_rec_size; t = btf__type_by_id(btf, finfo->type_id); - short_name = btf__name_by_offset(btf, t->name_off); + if (t) + short_name = btf__name_by_offset(btf, t->name_off); } else if (sub_id == 0 && sub_prog_cnt == 1) { /* no subprog */ if (info->name[0]) short_name = info->name; } else short_name = "F"; - if (short_name) - name_len += snprintf(buf + name_len, size - name_len, - "_%s", short_name); + if (short_name) { + name_len += scnprintf(buf + name_len, size - name_len, + "_%s", short_name); + } return name_len; } @@ -365,6 +373,15 @@ static struct bpf_metadata *bpf_metadata_alloc(__u32 nr_prog_tags, event_size = sizeof(metadata->event->bpf_metadata) + nr_variables * sizeof(metadata->event->bpf_metadata.entries[0]); + /* + * header.size is __u16. synthesize_perf_record_bpf_metadata() + * adds machine->id_hdr_size (up to ~64 bytes) after this, so + * leave headroom to prevent the final size from wrapping. + */ + if (event_size > UINT16_MAX - 256) { + bpf_metadata_free(metadata); + return NULL; + } metadata->event = zalloc(event_size); if (!metadata->event) { bpf_metadata_free(metadata); @@ -393,8 +410,10 @@ static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info) continue; metadata = bpf_metadata_alloc(info->nr_prog_tags, map.num_vars); - if (!metadata) + if (!metadata) { + bpf_metadata_free_map_data(&map); continue; + } bpf_metadata_fill_event(&map, &metadata->event->bpf_metadata); @@ -869,6 +888,7 @@ static int perf_env__add_bpf_info(struct perf_env *env, u32 id) if (!perf_env__insert_bpf_prog_info(env, info_node)) { pr_debug("%s: duplicate add bpf info request for id %u\n", __func__, btf_id); + bpf_metadata_free(info_node->metadata); free(info_linear); free(info_node); goto out; @@ -943,12 +963,15 @@ int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); } -void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, struct perf_env *env, FILE *fp) { - __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); - __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + struct bpf_prog_info *info = &info_linear->info; + __u64 required_arrays = (1UL << PERF_BPIL_JITED_KSYMS) | + (1UL << PERF_BPIL_JITED_FUNC_LENS); + __u32 *prog_lens; + __u64 *prog_addrs; char name[KSYM_NAME_LEN]; struct btf *btf = NULL; u32 sub_prog_cnt, i; @@ -958,6 +981,13 @@ void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, sub_prog_cnt != info->nr_jited_func_lens) return; + /* Ensure the arrays were present and converted by bpil_offs_to_addr() */ + if ((info_linear->arrays & required_arrays) != required_arrays) + return; + + prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); + prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + if (info->btf_id) { struct btf_node *node; diff --git a/tools/perf/util/bpf-event.h b/tools/perf/util/bpf-event.h index 60d2c6637af5..da4eeb4a1a73 100644 --- a/tools/perf/util/bpf-event.h +++ b/tools/perf/util/bpf-event.h @@ -40,7 +40,7 @@ struct btf_node { int machine__process_bpf(struct machine *machine, union perf_event *event, struct perf_sample *sample); int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env); -void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info, +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, struct perf_env *env, FILE *fp); void bpf_metadata_free(struct bpf_metadata *metadata); @@ -58,7 +58,7 @@ static inline int evlist__add_bpf_sb_event(struct evlist *evlist __maybe_unused, return 0; } -static inline void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info __maybe_unused, +static inline void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear __maybe_unused, struct perf_env *env __maybe_unused, FILE *fp __maybe_unused) { diff --git a/tools/perf/util/bpf-trace-summary.c b/tools/perf/util/bpf-trace-summary.c index cf6e1e4402d5..9a31dbf06cbb 100644 --- a/tools/perf/util/bpf-trace-summary.c +++ b/tools/perf/util/bpf-trace-summary.c @@ -6,7 +6,7 @@ #include <stdlib.h> #include "dwarf-regs.h" /* for EM_HOST */ -#include "syscalltbl.h" +#include "trace/beauty/syscalltbl.h" #include "util/cgroup.h" #include "util/hashmap.h" #include "util/trace.h" diff --git a/tools/perf/util/bpf-utils.c b/tools/perf/util/bpf-utils.c index d6d2c9c190f7..69148197b1ef 100644 --- a/tools/perf/util/bpf-utils.c +++ b/tools/perf/util/bpf-utils.c @@ -264,12 +264,28 @@ void bpil_offs_to_addr(struct perf_bpil *info_linear) for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { const struct bpil_array_desc *desc = &bpil_array_desc[i]; __u64 addr, offs; + __u32 count, size; if ((info_linear->arrays & (1UL << i)) == 0) continue; offs = bpf_prog_info_read_offset_u64(&info_linear->info, desc->array_offset); + count = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->count_offset); + size = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->size_offset); + /* offset and extent from perf.data are untrusted — keep within data[] */ + if (offs >= info_linear->data_len || + (u64)count * size > info_linear->data_len - offs) { + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, 0); + bpf_prog_info_set_offset_u32(&info_linear->info, + desc->count_offset, 0); + /* clear the bit so bpil_addr_to_offs() won't reverse a zeroed address */ + info_linear->arrays &= ~(1UL << i); + continue; + } addr = offs + ptr_to_u64(info_linear->data); bpf_prog_info_set_offset_u64(&info_linear->info, desc->array_offset, addr); diff --git a/tools/perf/util/bpf_counter_cgroup.c b/tools/perf/util/bpf_counter_cgroup.c index 519fee3dc3d0..e1ce5aa3b957 100644 --- a/tools/perf/util/bpf_counter_cgroup.c +++ b/tools/perf/util/bpf_counter_cgroup.c @@ -104,6 +104,11 @@ static int bperf_load_program(struct evlist *evlist) set_max_rlimit(); + if (nr_cgroups == 0 || evlist->core.nr_entries % nr_cgroups != 0) { + pr_err("Invalid cgroup or event count\n"); + return -EINVAL; + } + test_max_events_program_load(); skel = bperf_cgroup_bpf__open(); @@ -187,6 +192,21 @@ static int bperf_load_program(struct evlist *evlist) } /* + * Propagate supported flag from leaders to followers. Follower events + * are not opened, so their supported flag remains false. + */ + { + struct evsel *leader; + int num_events = evlist->core.nr_entries / nr_cgroups; + + evlist__for_each_entry(evlist, evsel) { + leader = evlist__find_evsel(evlist, evsel->core.idx % num_events); + if (leader) + evsel->supported = leader->supported; + } + } + + /* * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check * whether the kernel support it */ diff --git a/tools/perf/util/bpf_kwork.c b/tools/perf/util/bpf_kwork.c index d3a2e548f2b6..2248f462a847 100644 --- a/tools/perf/util/bpf_kwork.c +++ b/tools/perf/util/bpf_kwork.c @@ -273,6 +273,7 @@ static int add_work(struct perf_kwork *kwork, .cpu = key->cpu, }; enum kwork_class_type type = key->type; + int ret = 0; if (!valid_kwork_class_type(type)) { pr_debug("Invalid class type %d to add work\n", type); @@ -287,8 +288,10 @@ static int add_work(struct perf_kwork *kwork, return -1; work = kwork->add_work(kwork, tmp.class, &tmp); - if (work == NULL) - return -1; + if (work == NULL) { + ret = -1; + goto out; + } if (kwork->report == KWORK_REPORT_RUNTIME) { work->nr_atoms = data->nr; @@ -304,13 +307,16 @@ static int add_work(struct perf_kwork *kwork, work->max_latency_end = data->max_time_end; } else { pr_debug("Invalid bpf report type %d\n", kwork->report); - return -1; + ret = -1; + goto out; } kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec; kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec; - return 0; +out: + work_exit(&tmp); + return ret; } int perf_kwork__report_read_bpf(struct perf_kwork *kwork) diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index cbd7435579fe..b1cfa63a488f 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -186,6 +186,7 @@ int lock_contention_prepare(struct lock_contention *con) int ncpus = 1, ntasks = 1, ntypes = 1, naddrs = 1, ncgrps = 1, nslabs = 1; struct evlist *evlist = con->evlist; struct target *target = con->target; + bool has_mmap_lock = false; /* make sure it loads the kernel map before lookup */ map__load(machine__kernel_map(con->machine)); @@ -244,6 +245,11 @@ int lock_contention_prepare(struct lock_contention *con) unsigned long *addrs; for (i = 0; i < con->filters->nr_syms; i++) { + if (!strcmp(con->filters->syms[i], "mmap_lock")) { + has_mmap_lock = true; + continue; + } + sym = machine__find_kernel_symbol_by_name(con->machine, con->filters->syms[i], &kmap); @@ -263,7 +269,7 @@ int lock_contention_prepare(struct lock_contention *con) addrs[con->filters->nr_addrs++] = map__unmap_ip(kmap, sym->start); con->filters->addrs = addrs; } - naddrs = con->filters->nr_addrs; + naddrs = con->filters->nr_addrs ?: has_mmap_lock; skel->rodata->has_addr = 1; } @@ -298,6 +304,7 @@ int lock_contention_prepare(struct lock_contention *con) skel->rodata->aggr_mode = con->aggr_mode; skel->rodata->needs_callstack = con->save_callstack; skel->rodata->lock_owner = con->owner; + skel->rodata->has_mmap_lock = has_mmap_lock; if (con->aggr_mode == LOCK_AGGR_CGROUP || con->filters->nr_cgrps) { if (cgroup_is_v2("perf_event")) @@ -463,8 +470,8 @@ static void update_lock_stat(int map_fd, int pid, u64 end_ts, stat_key.lock_addr_or_cgroup = ts_data->lock; break; case LOCK_AGGR_CGROUP: - /* TODO */ - return; + stat_key.lock_addr_or_cgroup = ts_data->cgroup_id; + break; default: return; } diff --git a/tools/perf/util/bpf_skel/bench_uprobe.bpf.c b/tools/perf/util/bpf_skel/bench_uprobe.bpf.c deleted file mode 100644 index a01c7f791fcd..000000000000 --- a/tools/perf/util/bpf_skel/bench_uprobe.bpf.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) -// Copyright (c) 2023 Red Hat -#include "vmlinux.h" -#include <bpf/bpf_tracing.h> - -unsigned int nr_uprobes; -unsigned int nr_uretprobes; - -SEC("uprobe") -int BPF_UPROBE(empty) -{ - return 0; -} - -SEC("uprobe") -int BPF_UPROBE(trace_printk) -{ - char fmt[] = "perf bench uprobe %u"; - - bpf_trace_printk(fmt, sizeof(fmt), ++nr_uprobes); - return 0; -} - -SEC("uretprobe") -int BPF_URETPROBE(empty_ret) -{ - return 0; -} - -SEC("uretprobe") -int BPF_URETPROBE(trace_printk_ret) -{ - char fmt[] = "perf bench uretprobe %u"; - - bpf_trace_printk(fmt, sizeof(fmt), ++nr_uretprobes); - return 0; -} - -char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c index 96e7d853b9ed..0d9c6f55050e 100644 --- a/tools/perf/util/bpf_skel/lock_contention.bpf.c +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -175,6 +175,13 @@ struct mm_struct___new { struct rw_semaphore mmap_lock; } __attribute__((preserve_access_index)); +struct cas_ctx { + struct contention_data *data; + u64 duration; + int max_done; + int min_done; +}; + extern struct kmem_cache *bpf_get_kmem_cache(u64 addr) __ksym __weak; /* control flags */ @@ -184,6 +191,7 @@ const volatile int has_type; const volatile int has_addr; const volatile int has_cgroup; const volatile int has_slab; +const volatile int has_mmap_lock; const volatile int needs_callstack; const volatile int stack_skip; const volatile int lock_owner; @@ -214,6 +222,8 @@ int data_map_full; struct task_struct *bpf_task_from_pid(s32 pid) __ksym __weak; void bpf_task_release(struct task_struct *p) __ksym __weak; +static inline __u32 check_lock_type(__u64 lock, __u32 flags); + static inline __u64 get_current_cgroup_id(void) { struct task_struct *task; @@ -239,6 +249,8 @@ static inline __u64 get_current_cgroup_id(void) static inline int can_record(u64 *ctx) { + bool is_addr_ok = false; + if (has_cpu) { __u32 cpu = bpf_get_smp_processor_id(); __u8 *ok; @@ -271,8 +283,10 @@ static inline int can_record(u64 *ctx) __u64 addr = ctx[0]; ok = bpf_map_lookup_elem(&addr_filter, &addr); - if (!ok && !has_slab) + if (!ok && !has_slab && !has_mmap_lock) return 0; + + is_addr_ok = !!ok; } if (has_cgroup) { @@ -284,6 +298,10 @@ static inline int can_record(u64 *ctx) return 0; } + if (is_addr_ok) + return 1; + + /* slab and mmap_lock are part of the addr_filter */ if (has_slab && bpf_get_kmem_cache) { __u8 *ok; __u64 addr = ctx[0]; @@ -291,7 +309,17 @@ static inline int can_record(u64 *ctx) kmem_cache_addr = (long)bpf_get_kmem_cache(addr); ok = bpf_map_lookup_elem(&slab_filter, &kmem_cache_addr); - if (!ok) + if (ok) + return 1; + else if (!has_mmap_lock) + return 0; + } + + if (has_mmap_lock) { + __u64 lock = ctx[0]; + __u32 flag = ctx[1]; + + if (check_lock_type(lock, flag) != LCD_F_MMAP_LOCK) return 0; } @@ -486,16 +514,49 @@ static inline s32 get_owner_stack_id(u64 *stacktrace) return -1; } +static long cas_min_max_cb(u64 idx, void *arg) +{ + struct cas_ctx *ctx = arg; + + if (!ctx->max_done) { + u64 old_max = ctx->data->max_time; + if (old_max >= ctx->duration) { + ctx->max_done = 1; + } else { + u64 r = __sync_val_compare_and_swap( + &ctx->data->max_time, old_max, ctx->duration); + if (r == old_max) + ctx->max_done = 1; + } + } + + if (!ctx->min_done) { + u64 old_min = ctx->data->min_time; + if (old_min <= ctx->duration) { + ctx->min_done = 1; + } else { + u64 r = __sync_val_compare_and_swap( + &ctx->data->min_time, old_min, ctx->duration); + if (r == old_min) + ctx->min_done = 1; + } + } + + return (ctx->max_done && ctx->min_done) ? 1 : 0; +} + static inline void update_contention_data(struct contention_data *data, u64 duration, u32 count) { __sync_fetch_and_add(&data->total_time, duration); __sync_fetch_and_add(&data->count, count); - /* FIXME: need atomic operations */ - if (data->max_time < duration) - data->max_time = duration; - if (data->min_time > duration) - data->min_time = duration; + struct cas_ctx ctx = { + .data = data, + .duration = duration, + .max_done = 0, + .min_done = 0, + }; + bpf_loop(64, cas_min_max_cb, &ctx, 0); } static inline void update_owner_stat(u32 id, u64 duration, u32 flags) @@ -536,6 +597,8 @@ int contention_begin(u64 *ctx) pelem->timestamp = bpf_ktime_get_ns(); pelem->lock = (__u64)ctx[0]; pelem->flags = (__u32)ctx[1]; + if (aggr_mode == LOCK_AGGR_CGROUP) + pelem->cgroup_id = get_current_cgroup_id(); if (needs_callstack) { u32 i = 0; @@ -771,7 +834,7 @@ skip_owner: key.stack_id = pelem->stack_id; break; case LOCK_AGGR_CGROUP: - key.lock_addr_or_cgroup = get_current_cgroup_id(); + key.lock_addr_or_cgroup = pelem->cgroup_id; break; default: /* should not happen */ diff --git a/tools/perf/util/bpf_skel/lock_data.h b/tools/perf/util/bpf_skel/lock_data.h index 28c5e5aced7f..652e114e6b87 100644 --- a/tools/perf/util/bpf_skel/lock_data.h +++ b/tools/perf/util/bpf_skel/lock_data.h @@ -13,6 +13,7 @@ struct owner_tracing_data { struct tstamp_data { u64 timestamp; u64 lock; + u64 cgroup_id; u32 flags; s32 stack_id; }; diff --git a/tools/perf/util/btf.c b/tools/perf/util/btf.c index bb163fe87767..50d98f3e83bf 100644 --- a/tools/perf/util/btf.c +++ b/tools/perf/util/btf.c @@ -14,7 +14,7 @@ const struct btf_member *__btf_type__find_member_by_name(struct btf *btf, { const struct btf_type *t = btf__type_by_id(btf, type_id); const struct btf_member *m; - int i; + u32 i; for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { const char *current_member_name = btf__name_by_offset(btf, m->name_off); diff --git a/tools/perf/util/build-id.c b/tools/perf/util/build-id.c index fdb35133fde4..eb95ab90f974 100644 --- a/tools/perf/util/build-id.c +++ b/tools/perf/util/build-id.c @@ -10,6 +10,7 @@ #include "util.h" // lsdir(), mkdir_p(), rm_rf() #include <dirent.h> #include <errno.h> +#include <inttypes.h> #include <stdio.h> #include <sys/stat.h> #include <sys/types.h> @@ -55,7 +56,6 @@ static int mark_dso_hit_callback(struct callchain_cursor_node *node, void *data int build_id__mark_dso_hit(const struct perf_tool *tool __maybe_unused, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) { struct addr_location al; @@ -63,8 +63,8 @@ int build_id__mark_dso_hit(const struct perf_tool *tool __maybe_unused, sample->tid); if (thread == NULL) { - pr_err("problem processing %d event, skipping it.\n", - event->header.type); + pr_err("problem processing %s event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), sample->file_offset); return -1; } @@ -74,7 +74,7 @@ int build_id__mark_dso_hit(const struct perf_tool *tool __maybe_unused, addr_location__exit(&al); - sample__for_each_callchain_node(thread, evsel, sample, PERF_MAX_STACK_DEPTH, + sample__for_each_callchain_node(thread, sample, PERF_MAX_STACK_DEPTH, /*symbols=*/false, mark_dso_hit_callback, /*data=*/NULL); @@ -93,8 +93,11 @@ int build_id__snprintf(const struct build_id *build_id, char *bf, size_t bf_size return 0; } - for (size_t i = 0; i < build_id->size && offs < bf_size; ++i) - offs += snprintf(bf + offs, bf_size - offs, "%02x", build_id->data[i]); + if (bf_size > 0) + bf[0] = '\0'; + + for (size_t i = 0; i < build_id->size && offs + 1 < bf_size; ++i) + offs += scnprintf(bf + offs, bf_size - offs, "%02x", build_id->data[i]); return offs; } diff --git a/tools/perf/util/build-id.h b/tools/perf/util/build-id.h index 47e621cebe1b..73bad90b06f9 100644 --- a/tools/perf/util/build-id.h +++ b/tools/perf/util/build-id.h @@ -34,12 +34,7 @@ char *__dso__build_id_filename(const struct dso *dso, char *bf, size_t size, bool is_debug, bool is_kallsyms); int build_id__mark_dso_hit(const struct perf_tool *tool, union perf_event *event, - struct perf_sample *sample, struct evsel *evsel, - struct machine *machine); - -int perf_event__inject_buildid(const struct perf_tool *tool, union perf_event *event, - struct perf_sample *sample, struct evsel *evsel, - struct machine *machine); + struct perf_sample *sample, struct machine *machine); bool perf_session__read_build_ids(struct perf_session *session, bool with_hits); int perf_session__write_buildid_table(struct perf_session *session, diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index f031cbbeeba8..31c675cbab63 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -801,7 +801,7 @@ static enum match_result match_chain(struct callchain_cursor_node *node, * symbol start. Otherwise do a faster comparison based * on the symbol start address. */ - if (cnode->ms.sym->inlined || node->ms.sym->inlined) { + if (symbol__inlined(cnode->ms.sym) || symbol__inlined(node->ms.sym)) { match = match_chain_strings(cnode->ms.sym->name, node->ms.sym->name); if (match != MATCH_ERROR) @@ -1170,7 +1170,7 @@ int callchain_cursor_append(struct callchain_cursor *cursor, int sample__resolve_callchain(struct perf_sample *sample, struct callchain_cursor *cursor, struct symbol **parent, - struct evsel *evsel, struct addr_location *al, + struct addr_location *al, int max_stack) { if (sample->callchain == NULL && !symbol_conf.show_branchflag_count) @@ -1178,7 +1178,7 @@ int sample__resolve_callchain(struct perf_sample *sample, if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain || perf_hpp_list.parent || symbol_conf.show_branchflag_count) { - return thread__resolve_callchain(al->thread, cursor, evsel, sample, + return thread__resolve_callchain(al->thread, cursor, sample, parent, al, max_stack); } return 0; @@ -1245,7 +1245,7 @@ char *callchain_list__sym_name(struct callchain_list *cl, int printed; if (cl->ms.sym) { - const char *inlined = cl->ms.sym->inlined ? " (inlined)" : ""; + const char *inlined = symbol__inlined(cl->ms.sym) ? " (inlined)" : ""; if (show_srcline && cl->srcline) printed = scnprintf(bf, bfsize, "%s %s%s", @@ -1578,6 +1578,21 @@ void free_callchain(struct callchain_root *root) free_callchain_node(&root->node); } +void callchain_cursor_cleanup(struct callchain_cursor *cursor) +{ + struct callchain_cursor_node *node, *next; + + callchain_cursor_reset(cursor); + + for (node = cursor->first; node; node = next) { + next = node->next; + free(node); + } + cursor->first = NULL; + cursor->last = &cursor->first; + cursor->curr = NULL; +} + static u64 decay_callchain_node(struct callchain_node *node) { struct callchain_node *child; @@ -1853,7 +1868,7 @@ s64 callchain_avg_cycles(struct callchain_node *cnode) return cycles; } -int sample__for_each_callchain_node(struct thread *thread, struct evsel *evsel, +int sample__for_each_callchain_node(struct thread *thread, struct perf_sample *sample, int max_stack, bool symbols, callchain_iter_fn cb, void *data) { @@ -1864,7 +1879,7 @@ int sample__for_each_callchain_node(struct thread *thread, struct evsel *evsel, return -ENOMEM; /* Fill in the callchain. */ - ret = __thread__resolve_callchain(thread, cursor, evsel, sample, + ret = __thread__resolve_callchain(thread, cursor, sample, /*parent=*/NULL, /*root_al=*/NULL, max_stack, symbols); if (ret) diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h index 06d463ccc7a0..8b1e405d1cdf 100644 --- a/tools/perf/util/callchain.h +++ b/tools/perf/util/callchain.h @@ -8,11 +8,9 @@ #include "branch.h" struct addr_location; -struct evsel; struct hist_entry; struct hists; struct ip_callchain; -struct map; struct perf_sample; struct record_opts; struct thread; @@ -245,7 +243,7 @@ int record_opts__parse_callchain(struct record_opts *record, int sample__resolve_callchain(struct perf_sample *sample, struct callchain_cursor *cursor, struct symbol **parent, - struct evsel *evsel, struct addr_location *al, + struct addr_location *al, int max_stack); int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample); int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node, @@ -277,8 +275,6 @@ static inline int arch_skip_callchain_idx(struct thread *thread __maybe_unused, } #endif -void arch__add_leaf_frame_record_opts(struct record_opts *opts); - char *callchain_list__sym_name(struct callchain_list *cl, char *bf, size_t bfsize, bool show_dso); char *callchain_node__scnprintf_value(struct callchain_node *node, @@ -290,6 +286,7 @@ int callchain_list_counts__printf_value(struct callchain_list *clist, FILE *fp, char *bf, int bfsize); void free_callchain(struct callchain_root *root); +void callchain_cursor_cleanup(struct callchain_cursor *cursor); void decay_callchain(struct callchain_root *root); int callchain_node__make_parent_list(struct callchain_node *node); @@ -308,7 +305,7 @@ s64 callchain_avg_cycles(struct callchain_node *cnode); typedef int (*callchain_iter_fn)(struct callchain_cursor_node *node, void *data); -int sample__for_each_callchain_node(struct thread *thread, struct evsel *evsel, +int sample__for_each_callchain_node(struct thread *thread, struct perf_sample *sample, int max_stack, bool symbols, callchain_iter_fn cb, void *data); diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c index 25cf6e15ec27..5ad537fea436 100644 --- a/tools/perf/util/capstone.c +++ b/tools/perf/util/capstone.c @@ -1,7 +1,19 @@ // SPDX-License-Identifier: GPL-2.0 #include "capstone.h" -#include "annotate.h" + +#include <errno.h> +#include <inttypes.h> +#include <string.h> + +#include <dlfcn.h> +#include <elf.h> +#include <fcntl.h> +#include <linux/ctype.h> + +#include <capstone/capstone.h> + #include "addr_location.h" +#include "annotate.h" #include "debug.h" #include "disasm.h" #include "dso.h" @@ -11,13 +23,6 @@ #include "print_insn.h" #include "symbol.h" #include "thread.h" -#include <dlfcn.h> -#include <errno.h> -#include <fcntl.h> -#include <inttypes.h> -#include <string.h> - -#include <capstone/capstone.h> #ifdef LIBCAPSTONE_DLOPEN static void *perf_cs_dll_handle(void) @@ -137,37 +142,70 @@ static enum cs_err perf_cs_close(csh *handle) #endif } -static int capstone_init(struct machine *machine, csh *cs_handle, bool is64, +static bool e_machine_to_capstone(uint16_t e_machine, bool is64, bool is_big_endian, + enum cs_arch *arch, enum cs_mode *mode) +{ + *mode = is_big_endian ? CS_MODE_BIG_ENDIAN : CS_MODE_LITTLE_ENDIAN; + + switch (e_machine) { + case EM_X86_64: + case EM_386: + *arch = CS_ARCH_X86; + *mode |= is64 ? CS_MODE_64 : CS_MODE_32; + return true; + case EM_AARCH64: + *arch = CS_ARCH_ARM64; + *mode |= CS_MODE_ARM; + return true; + case EM_ARM: + *arch = CS_ARCH_ARM; + *mode |= CS_MODE_ARM | CS_MODE_V8; + return true; + case EM_S390: + *arch = CS_ARCH_SYSZ; + return true; + case EM_MIPS: + *arch = CS_ARCH_MIPS; + *mode |= is64 ? CS_MODE_MIPS64 : CS_MODE_MIPS32; + return true; + case EM_PPC: + *arch = CS_ARCH_PPC; + return true; + case EM_PPC64: + *arch = CS_ARCH_PPC; + *mode |= CS_MODE_64; + return true; + case EM_SPARC: + *arch = CS_ARCH_SPARC; + return true; + case EM_SPARCV9: + *arch = CS_ARCH_SPARC; + *mode |= CS_MODE_V9; + return true; + case EM_RISCV: + *arch = CS_ARCH_RISCV; + *mode |= (is64 ? CS_MODE_RISCV64 : CS_MODE_RISCV32) | CS_MODE_RISCVC; + return true; + default: + return false; + } +} + +static int capstone_init(uint16_t e_machine, csh *cs_handle, bool is64, bool is_big_endian, bool disassembler_style) { enum cs_arch arch; enum cs_mode mode; - if (machine__is(machine, "x86_64") && is64) { - arch = CS_ARCH_X86; - mode = CS_MODE_64; - } else if (machine__normalized_is(machine, "x86")) { - arch = CS_ARCH_X86; - mode = CS_MODE_32; - } else if (machine__normalized_is(machine, "arm64")) { - arch = CS_ARCH_ARM64; - mode = CS_MODE_ARM; - } else if (machine__normalized_is(machine, "arm")) { - arch = CS_ARCH_ARM; - mode = CS_MODE_ARM + CS_MODE_V8; - } else if (machine__normalized_is(machine, "s390")) { - arch = CS_ARCH_SYSZ; - mode = CS_MODE_BIG_ENDIAN; - } else { + if (!e_machine_to_capstone(e_machine, is64, is_big_endian, &arch, &mode)) return -1; - } if (perf_cs_open(arch, mode, cs_handle) != CS_ERR_OK) { pr_warning_once("cs_open failed\n"); return -1; } - if (machine__normalized_is(machine, "x86")) { + if (arch == CS_ARCH_X86) { /* * In case of using capstone_init while symbol__disassemble * setting CS_OPT_SYNTAX_ATT depends if disassembler_style opts @@ -211,29 +249,28 @@ static size_t print_insn_x86(struct thread *thread, u8 cpumode, struct cs_insn * return printed; } - -ssize_t capstone__fprintf_insn_asm(struct machine *machine __maybe_unused, - struct thread *thread __maybe_unused, - u8 cpumode __maybe_unused, bool is64bit __maybe_unused, - const uint8_t *code __maybe_unused, - size_t code_size __maybe_unused, - uint64_t ip __maybe_unused, int *lenp __maybe_unused, - int print_opts __maybe_unused, FILE *fp __maybe_unused) +ssize_t capstone__fprintf_insn_asm(struct machine *machine, struct thread *thread, u8 cpumode, + bool is64bit, const uint8_t *code, size_t code_size, uint64_t ip, + int *lenp, int print_opts, FILE *fp) { size_t printed; struct cs_insn *insn; csh cs_handle; size_t count; + bool is_big_endian = false; + uint16_t e_machine = thread__e_machine_endian(thread, machine, + /*e_flags=*/NULL, &is_big_endian); int ret; /* TODO: Try to initiate capstone only once but need a proper place. */ - ret = capstone_init(machine, &cs_handle, is64bit, true); + ret = capstone_init(e_machine, &cs_handle, is64bit, is_big_endian, + /*disassembler_style=*/true); if (ret < 0) return ret; count = perf_cs_disasm(cs_handle, code, code_size, ip, 1, &insn); if (count > 0) { - if (machine__normalized_is(machine, "x86")) + if (e_machine == EM_X86_64 || e_machine == EM_386) printed = print_insn_x86(thread, cpumode, &insn[0], print_opts, fp); else printed = fprintf(fp, "%s %s", insn[0].mnemonic, insn[0].op_str); @@ -322,9 +359,8 @@ static int find_file_offset(u64 start, u64 len, u64 pgoff, void *arg) return 0; } -int symbol__disassemble_capstone(const char *filename __maybe_unused, - struct symbol *sym __maybe_unused, - struct annotate_args *args __maybe_unused) +int symbol__disassemble_capstone(const char *filename, struct symbol *sym, + struct annotate_args *args) { struct annotation *notes = symbol__annotation(sym); struct map *map = args->ms->map; @@ -344,6 +380,8 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, char disasm_buf[512]; struct disasm_line *dl; bool disassembler_style = false; + uint16_t e_machine; + bool is_big_endian = false; if (args->options->objdump_path) return -1; @@ -373,8 +411,10 @@ int symbol__disassemble_capstone(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(thread__maps(args->ms->thread)), &handle, is_64bit, - disassembler_style) < 0) + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) goto err; needs_cs_close = true; @@ -466,6 +506,8 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, struct disasm_line *dl; u32 *line; bool disassembler_style = false; + uint16_t e_machine; + bool is_big_endian = false; if (args->options->objdump_path) return -1; @@ -484,8 +526,10 @@ int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, !strcmp(args->options->disassembler_style, "att")) disassembler_style = true; - if (capstone_init(maps__machine(thread__maps(args->ms->thread)), &handle, is_64bit, - disassembler_style) < 0) + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) goto err; needs_cs_close = true; diff --git a/tools/perf/util/config.c b/tools/perf/util/config.c index 087002fb1b9b..7988149dc7ed 100644 --- a/tools/perf/util/config.c +++ b/tools/perf/util/config.c @@ -23,6 +23,7 @@ #include "build-id.h" #include "debug.h" #include "config.h" +#include "unwind.h" #include <sys/types.h> #include <sys/stat.h> #include <stdlib.h> @@ -525,6 +526,9 @@ int perf_default_config(const char *var, const char *value, if (strstarts(var, "addr2line.")) return addr2line_configure(var, value, dummy); + if (strstarts(var, "unwind.")) + return unwind__configure(var, value, dummy); + /* Add other config variables here. */ return 0; } diff --git a/tools/perf/util/cpumap.c b/tools/perf/util/cpumap.c index 11922e1ded84..23ebe9b97f8e 100644 --- a/tools/perf/util/cpumap.c +++ b/tools/perf/util/cpumap.c @@ -10,6 +10,7 @@ #include <linux/bitmap.h> #include "asm/bug.h" +#include <linux/compiler.h> #include <linux/ctype.h> #include <linux/zalloc.h> #include <internal/cpumap.h> @@ -40,15 +41,16 @@ bool perf_record_cpu_map_data__test_bit(int i, /* Read ith mask value from data into the given 64-bit sized bitmap */ static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu_map_data *data, - int i, unsigned long *bitmap) + int i, unsigned long *bitmap, + u16 long_size) { #if __SIZEOF_LONG__ == 8 - if (data->mask32_data.long_size == 4) + if (long_size == 4) bitmap[0] = data->mask32_data.mask[i]; else bitmap[0] = data->mask64_data.mask[i]; #else - if (data->mask32_data.long_size == 4) { + if (long_size == 4) { bitmap[0] = data->mask32_data.mask[i]; bitmap[1] = 0; } else { @@ -64,24 +66,27 @@ static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu } static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_map_data *data) { + /* Snapshot nr — data is mmap'd and could change between reads */ + u16 nr = READ_ONCE(data->cpus_data.nr); struct perf_cpu_map *map; - map = perf_cpu_map__empty_new(data->cpus_data.nr); + map = perf_cpu_map__empty_new(nr); if (!map) return NULL; - for (unsigned int i = 0; i < data->cpus_data.nr; i++) { + for (unsigned int i = 0; i < nr; i++) { + u16 cpu = READ_ONCE(data->cpus_data.cpu[i]); /* * Special treatment for -1, which is not real cpu number, * and we need to use (int) -1 to initialize map[i], * otherwise it would become 65535. */ - if (data->cpus_data.cpu[i] == (u16) -1) { + if (cpu == (u16) -1) { RC_CHK_ACCESS(map)->map[i].cpu = -1; - } else if (data->cpus_data.cpu[i] < INT16_MAX) { - RC_CHK_ACCESS(map)->map[i].cpu = (int16_t) data->cpus_data.cpu[i]; + } else if (cpu < INT16_MAX) { + RC_CHK_ACCESS(map)->map[i].cpu = (int16_t) cpu; } else { - pr_err("Invalid cpumap entry %u\n", data->cpus_data.cpu[i]); + pr_err("Invalid cpumap entry %u\n", cpu); perf_cpu_map__put(map); return NULL; } @@ -93,11 +98,21 @@ static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_m static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_data *data) { DECLARE_BITMAP(local_copy, 64); - int weight = 0, mask_nr = data->mask32_data.nr; + int weight = 0, mask_nr; + /* Snapshot before validation — data is mmap'd and could change */ + u16 long_size = READ_ONCE(data->mask32_data.long_size); struct perf_cpu_map *map; + /* long_size must be 4 or 8; other values overflow cpus_per_i below */ + if (long_size != 4 && long_size != 8) { + pr_warning("WARNING: cpu_map mask: unsupported long_size %u\n", long_size); + return NULL; + } + + mask_nr = READ_ONCE(data->mask32_data.nr); + for (int i = 0; i < mask_nr; i++) { - perf_record_cpu_map_data__read_one_mask(data, i, local_copy); + perf_record_cpu_map_data__read_one_mask(data, i, local_copy, long_size); weight += bitmap_weight(local_copy, 64); } @@ -106,11 +121,14 @@ static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_ return NULL; for (int i = 0, j = 0; i < mask_nr; i++) { - int cpus_per_i = (i * data->mask32_data.long_size * BITS_PER_BYTE); + int cpus_per_i = (i * long_size * BITS_PER_BYTE); int cpu; - perf_record_cpu_map_data__read_one_mask(data, i, local_copy); + perf_record_cpu_map_data__read_one_mask(data, i, local_copy, long_size); for_each_set_bit(cpu, local_copy, 64) { + /* Guard against more set bits than the first pass counted */ + if (j >= weight) + break; if (cpu + cpus_per_i < INT16_MAX) { RC_CHK_ACCESS(map)->map[j++].cpu = cpu + cpus_per_i; } else { @@ -126,18 +144,28 @@ static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_ static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data) { + /* Snapshot fields — data is mmap'd and could change between reads */ + u16 start_cpu = READ_ONCE(data->range_cpu_data.start_cpu); + u16 end_cpu = READ_ONCE(data->range_cpu_data.end_cpu); + u16 any_cpu = READ_ONCE(data->range_cpu_data.any_cpu); struct perf_cpu_map *map; unsigned int i = 0; - map = perf_cpu_map__empty_new(data->range_cpu_data.end_cpu - - data->range_cpu_data.start_cpu + 1 + data->range_cpu_data.any_cpu); + if (end_cpu < start_cpu) { + pr_warning("WARNING: cpu_map range: end_cpu %u < start_cpu %u\n", + end_cpu, start_cpu); + return NULL; + } + + /* any_cpu is boolean (0 or 1), not a count — clamp to avoid inflated nr */ + map = perf_cpu_map__empty_new(end_cpu - start_cpu + 1 + !!any_cpu); if (!map) return NULL; - if (data->range_cpu_data.any_cpu) + if (any_cpu) RC_CHK_ACCESS(map)->map[i++].cpu = -1; - for (int cpu = data->range_cpu_data.start_cpu; cpu <= data->range_cpu_data.end_cpu; + for (int cpu = start_cpu; cpu <= end_cpu; i++, cpu++) { if (cpu < INT16_MAX) { RC_CHK_ACCESS(map)->map[i].cpu = cpu; @@ -420,6 +448,12 @@ static int get_max_num(char *path, int *max) buf[num] = '\0'; + /* empty file — nothing to parse */ + if (num == 0) { + err = -1; + goto out; + } + /* start on the right, to find highest node num */ while (--num) { if ((buf[num] == ',') || (buf[num] == '-')) { @@ -466,6 +500,16 @@ static void set_max_cpu_num(void) if (ret) goto out; + /* + * struct perf_cpu.cpu is int16_t (libperf ABI) — clamp to avoid + * truncation to negative. See tools/lib/perf/TODO for the ABI + * widening plan. + */ + if (max > INT16_MAX) { + pr_warning("WARNING: max possible cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; + } max_cpu_num.cpu = max; /* get the highest present cpu number for a sparse allocation */ @@ -478,11 +522,12 @@ static void set_max_cpu_num(void) ret = get_max_num(path, &max); if (!ret && max > INT16_MAX) { - pr_err("Read out of bounds max cpus of %d\n", max); - ret = -1; + pr_warning("WARNING: max present cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; } if (!ret) - max_present_cpu_num.cpu = (int16_t)max; + max_present_cpu_num.cpu = max; out: if (ret) pr_err("Failed to read max cpus, using default of %d\n", max_cpu_num.cpu); @@ -548,6 +593,10 @@ int cpu__get_node(struct perf_cpu cpu) return -1; } + /* cpunode_map allocated for max_cpu_num entries; input may be untrusted */ + if (cpu.cpu < 0 || cpu.cpu >= max_cpu_num.cpu) + return -1; + return cpunode_map[cpu.cpu]; } @@ -615,7 +664,9 @@ int cpu__setup_cpunode_map(void) while ((dent2 = readdir(dir2)) != NULL) { if (dent2->d_type != DT_LNK || sscanf(dent2->d_name, "cpu%u", &cpu) < 1) continue; - cpunode_map[cpu] = mem; + /* cpunode_map allocated for max_cpu_num entries */ + if (cpu < (unsigned int)max_cpu_num.cpu) + cpunode_map[cpu] = mem; } closedir(dir2); } @@ -641,21 +692,21 @@ size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size) if (start == -1) { start = i; if (last) { - ret += snprintf(buf + ret, size - ret, - "%s%d", COMMA, - perf_cpu_map__cpu(map, i).cpu); + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, i).cpu); } } else if (((i - start) != (cpu.cpu - perf_cpu_map__cpu(map, start).cpu)) || last) { int end = i - 1; if (start == end) { - ret += snprintf(buf + ret, size - ret, - "%s%d", COMMA, - perf_cpu_map__cpu(map, start).cpu); + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, start).cpu); } else { - ret += snprintf(buf + ret, size - ret, - "%s%d-%d", COMMA, - perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); + ret += scnprintf(buf + ret, size - ret, + "%s%d-%d", COMMA, + perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); } first = false; start = i; diff --git a/tools/perf/util/cs-etm-base.c b/tools/perf/util/cs-etm-base.c index 4abe416e3feb..aebef71d3a0a 100644 --- a/tools/perf/util/cs-etm-base.c +++ b/tools/perf/util/cs-etm-base.c @@ -170,7 +170,9 @@ int cs_etm__process_auxtrace_info(union perf_event *event, u64 *ptr = NULL; u64 hdr_version; - if (auxtrace_info->header.size < (event_header_size + INFO_HEADER_SIZE)) + /* Ensure priv[] is large enough for the global header entries */ + if (auxtrace_info->header.size < (event_header_size + INFO_HEADER_SIZE + + CS_ETM_HEADER_SIZE)) return -EINVAL; /* First the global part */ diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c index dee3020ceaa9..26940f1f1b0b 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -402,6 +402,8 @@ cs_etm_decoder__buffer_packet(struct cs_etm_queue *etmq, packet_queue->packet_buffer[et].flags = 0; packet_queue->packet_buffer[et].exception_number = UINT32_MAX; packet_queue->packet_buffer[et].trace_chan_id = trace_chan_id; + packet_queue->packet_buffer[et].el = ocsd_EL_unknown; + packet_queue->packet_buffer[et].tid = -1; if (packet_queue->packet_count == CS_ETM_PACKET_MAX_BUFFER - 1) return OCSD_RESP_WAIT; @@ -449,6 +451,7 @@ cs_etm_decoder__buffer_range(struct cs_etm_queue *etmq, packet->last_instr_type = elem->last_i_type; packet->last_instr_subtype = elem->last_i_subtype; packet->last_instr_cond = elem->last_instr_cond; + packet->el = elem->context.exception_level; if (elem->last_i_type == OCSD_INSTR_BR || elem->last_i_type == OCSD_INSTR_BR_INDIRECT) packet->last_instr_taken_branch = elem->last_instr_exec; @@ -525,7 +528,9 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, const ocsd_generic_trace_elem *elem, const uint8_t trace_chan_id) { + struct cs_etm_packet *packet; pid_t tid = -1; + int ret; /* * Process the PE_CONTEXT packets if we have a valid contextID or VMID. @@ -546,12 +551,18 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, break; } - if (cs_etm__etmq_set_tid_el(etmq, tid, trace_chan_id, - elem->context.exception_level)) + if (cs_etm__etmq_update_decode_context(etmq, trace_chan_id, + elem->context.exception_level, tid)) return OCSD_RESP_FATAL_SYS_ERR; - if (tid == -1) - return OCSD_RESP_CONT; + ret = cs_etm_decoder__buffer_packet(etmq, packet_queue, trace_chan_id, + CS_ETM_CONTEXT); + if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT) + return ret; + + packet = &packet_queue->packet_buffer[packet_queue->tail]; + packet->tid = tid; + packet->el = elem->context.exception_level; /* * A timestamp is generated after a PE_CONTEXT element so make sure @@ -559,7 +570,7 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, */ cs_etm_decoder__reset_timestamp(packet_queue); - return OCSD_RESP_CONT; + return ret; } static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer( diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 8a639d2e51a4..5d0664ff73b7 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -6,6 +6,7 @@ * Author: Mathieu Poirier <mathieu.poirier@linaro.org> */ +#include <limits.h> #include <linux/bitfield.h> #include <linux/bitops.h> #include <linux/coresight-pmu.h> @@ -85,15 +86,22 @@ struct cs_etm_traceid_queue { u64 period_instructions; size_t last_branch_pos; union perf_event *event_buf; - struct thread *thread; - struct thread *prev_packet_thread; - ocsd_ex_level prev_packet_el; - ocsd_ex_level el; struct branch_stack *last_branch; struct branch_stack *last_branch_rb; struct cs_etm_packet *prev_packet; struct cs_etm_packet *packet; struct cs_etm_packet_queue packet_queue; + + struct thread *decode_thread; + ocsd_ex_level decode_el; + + /* + * The frontend accesses the EL from '[prev_]packet' because it needs + * previous EL for branch and current EL for instruction samples. It's + * not possible to change thread in a single branch sample so no need to + * store or access the thread through the packet. + */ + struct thread *frontend_thread; }; enum cs_etm_format { @@ -284,8 +292,11 @@ static struct cs_etm_queue *cs_etm__get_queue(struct cs_etm_auxtrace *etm, int c { if (etm->per_thread_decoding) return etm->queues.queue_array[0].priv; - else - return etm->queues.queue_array[cpu].priv; + + if (cpu < 0 || cpu >= (int)etm->queues.nr_queues) + return NULL; + + return etm->queues.queue_array[cpu].priv; } static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id, @@ -298,6 +309,9 @@ static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id * queue associated with that CPU so only one decoder is made. */ etmq = cs_etm__get_queue(etm, cpu_metadata[CS_ETM_CPU]); + if (!etmq) + return -EINVAL; + if (etmq->format == UNFORMATTED) return cs_etm__insert_trace_id_node(etmq, trace_chan_id, cpu_metadata); @@ -310,6 +324,9 @@ static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id int ret; etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + ret = cs_etm__insert_trace_id_node(etmq, trace_chan_id, cpu_metadata); if (ret) @@ -350,6 +367,9 @@ static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, u32 sink_id = FIELD_GET(CS_AUX_HW_ID_SINK_ID_MASK, hw_id); u8 trace_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id); + if (!etmq) + return -EINVAL; + /* * Check sink id hasn't changed in per-cpu mode. In per-thread mode, * let it pass for now until an actual overlapping trace ID is hit. In @@ -367,6 +387,9 @@ static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) { struct cs_etm_queue *other_etmq = etm->queues.queue_array[i].priv; + if (!other_etmq) + continue; + /* Different sinks, skip */ if (other_etmq->sink_id != etmq->sink_id) continue; @@ -388,6 +411,9 @@ static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, } cpu_data = get_cpu_data(etm, cpu); + if (!cpu_data) + return -EINVAL; + ret = cs_etm__insert_trace_id_node(etmq, trace_id, cpu_data); if (ret) return ret; @@ -614,10 +640,11 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; tidq->trace_chan_id = trace_chan_id; - tidq->el = tidq->prev_packet_el = ocsd_EL_unknown; - tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1, + tidq->decode_el = ocsd_EL_unknown; + tidq->frontend_thread = machine__findnew_thread(&etm->session->machines.host, -1, + queue->tid); + tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1, queue->tid); - tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host); tidq->packet = zalloc(sizeof(struct cs_etm_packet)); if (!tidq->packet) @@ -750,21 +777,10 @@ static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm, /* * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for * the next incoming packet. - * - * Threads and exception levels are also tracked for both the - * previous and current packets. This is because the previous - * packet is used for the 'from' IP for branch samples, so the - * thread at that time must also be assigned to that sample. - * Across discontinuity packets the thread can change, so by - * tracking the thread for the previous packet the branch sample - * will have the correct info. */ tmp = tidq->packet; tidq->packet = tidq->prev_packet; tidq->prev_packet = tmp; - tidq->prev_packet_el = tidq->el; - thread__put(tidq->prev_packet_thread); - tidq->prev_packet_thread = thread__get(tidq->thread); } } @@ -937,8 +953,8 @@ static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq) /* Free this traceid_queue from the array */ tidq = etmq->traceid_queues[idx]; - thread__zput(tidq->thread); - thread__zput(tidq->prev_packet_thread); + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); zfree(&tidq->event_buf); zfree(&tidq->last_branch); zfree(&tidq->last_branch_rb); @@ -1083,47 +1099,43 @@ static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address, } } -static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, - u64 address, size_t size, u8 *buffer, - const ocsd_mem_space_acc_t mem_space) +static u32 __cs_etm__mem_access(struct cs_etm_queue *etmq, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space, + ocsd_ex_level el, struct thread *thread) { u8 cpumode; u64 offset; int len; struct addr_location al; struct dso *dso; - struct cs_etm_traceid_queue *tidq; int ret = 0; if (!etmq) return 0; addr_location__init(&al); - tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); - if (!tidq) - goto out; /* - * We've already tracked EL along side the PID in cs_etm__set_thread() - * so double check that it matches what OpenCSD thinks as well. It - * doesn't distinguish between EL0 and EL1 for this mem access callback - * so we had to do the extra tracking. Skip validation if it's any of - * the 'any' values. + * We track EL for the frontend and the backend when receiving context + * and range packets. OpenCSD doesn't distinguish between EL0 and EL1 + * for this mem access callback so we had to do the extra tracking. Skip + * validation if it's any of the 'any' values. */ if (!(mem_space == OCSD_MEM_SPACE_ANY || mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) { if (mem_space & OCSD_MEM_SPACE_EL1N) { /* Includes both non secure EL1 and EL0 */ - assert(tidq->el == ocsd_EL1 || tidq->el == ocsd_EL0); + assert(el == ocsd_EL1 || el == ocsd_EL0); } else if (mem_space & OCSD_MEM_SPACE_EL2) - assert(tidq->el == ocsd_EL2); + assert(el == ocsd_EL2); else if (mem_space & OCSD_MEM_SPACE_EL3) - assert(tidq->el == ocsd_EL3); + assert(el == ocsd_EL3); } - cpumode = cs_etm__cpu_mode(etmq, address, tidq->el); + cpumode = cs_etm__cpu_mode(etmq, address, el); - if (!thread__find_map(tidq->thread, cpumode, address, &al)) + if (!thread__find_map(thread, cpumode, address, &al)) goto out; dso = map__dso(al.map); @@ -1138,7 +1150,7 @@ static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, map__load(al.map); - len = dso__data_read_offset(dso, maps__machine(thread__maps(tidq->thread)), + len = dso__data_read_offset(dso, maps__machine(thread__maps(thread)), offset, buffer, size); if (len <= 0) { @@ -1158,6 +1170,30 @@ out: return ret; } +static u32 cs_etm__frontend_mem_access(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 address, size_t size, u8 *buffer) +{ + return __cs_etm__mem_access(etmq, address, size, buffer, 0, packet->el, + tidq->frontend_thread); +} + +static u32 cs_etm__decoder_mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space) +{ + struct cs_etm_traceid_queue *tidq; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (!tidq) + return 0; + + return __cs_etm__mem_access(etmq, address, size, buffer, + mem_space, tidq->decode_el, + tidq->decode_thread); +} + static struct cs_etm_queue *cs_etm__alloc_queue(void) { struct cs_etm_queue *etmq = zalloc(sizeof(*etmq)); @@ -1333,12 +1369,13 @@ void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq) } static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq, - u8 trace_chan_id, u64 addr) + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 addr) { u8 instrBytes[2]; - cs_etm__mem_access(etmq, trace_chan_id, addr, ARRAY_SIZE(instrBytes), - instrBytes, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + ARRAY_SIZE(instrBytes), instrBytes); /* * T32 instruction size is indicated by bits[15:11] of the first * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111 @@ -1371,16 +1408,16 @@ u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet) } static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq, - u64 trace_chan_id, - const struct cs_etm_packet *packet, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 offset) { if (packet->isa == CS_ETM_ISA_T32) { u64 addr = packet->start_addr; while (offset) { - addr += cs_etm__t32_instr_size(etmq, - trace_chan_id, addr); + addr += cs_etm__t32_instr_size(etmq, tidq, packet, + addr); offset--; } return addr; @@ -1422,11 +1459,29 @@ static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq, bs->nr += 1; } -static int cs_etm__inject_event(union perf_event *event, +static int cs_etm__inject_event(struct cs_etm_auxtrace *etm, union perf_event *event, struct perf_sample *sample, u64 type) { - event->header.size = perf_event__sample_event_size(sample, type, 0); - return perf_event__synthesize_sample(event, type, 0, sample); + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && etm->session && etm->session->evlist) + evsel = evlist__id2evsel(etm->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); } @@ -1472,34 +1527,51 @@ cs_etm__get_trace(struct cs_etm_queue *etmq) return etmq->buf_len; } -static void cs_etm__set_thread(struct cs_etm_queue *etmq, - struct cs_etm_traceid_queue *tidq, pid_t tid, - ocsd_ex_level el) +/* + * Convert a raw thread number to a thread struct and assign it to **thread. + */ +static int cs_etm__etmq_update_thread(struct cs_etm_queue *etmq, + ocsd_ex_level el, pid_t tid, + struct thread **thread) { struct machine *machine = cs_etm__get_machine(etmq, el); + if (!machine || !*thread) + return -EINVAL; + if (tid != -1) { - thread__zput(tidq->thread); - tidq->thread = machine__find_thread(machine, -1, tid); + thread__zput(*thread); + *thread = machine__find_thread(machine, -1, tid); } /* Couldn't find a known thread */ - if (!tidq->thread) - tidq->thread = machine__idle_thread(machine); + if (!*thread) + *thread = machine__idle_thread(machine); - tidq->el = el; + return 0; } -int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid, - u8 trace_chan_id, ocsd_ex_level el) +/* + * Set the thread and EL of the decode context which is ahead in time of the + * frontend context. + */ +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, + ocsd_ex_level el, pid_t tid) { struct cs_etm_traceid_queue *tidq; + int ret; tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); if (!tidq) return -EINVAL; - cs_etm__set_thread(etmq, tidq, tid, el); + ret = cs_etm__etmq_update_thread(etmq, el, tid, + &tidq->decode_thread); + if (ret) + return ret; + + tidq->decode_el = el; return 0; } @@ -1509,8 +1581,8 @@ bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq) } static void cs_etm__copy_insn(struct cs_etm_queue *etmq, - u64 trace_chan_id, - const struct cs_etm_packet *packet, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, struct perf_sample *sample) { /* @@ -1527,14 +1599,14 @@ static void cs_etm__copy_insn(struct cs_etm_queue *etmq, * cs_etm__t32_instr_size(). */ if (packet->isa == CS_ETM_ISA_T32) - sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id, + sample->insn_len = cs_etm__t32_instr_size(etmq, tidq, packet, sample->ip); /* Otherwise, A64 and A32 instruction size are always 32-bit. */ else sample->insn_len = 4; - cs_etm__mem_access(etmq, trace_chan_id, sample->ip, sample->insn_len, - (void *)sample->insn, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, sample->ip, + sample->insn_len, (void *)sample->insn); } u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp) @@ -1561,6 +1633,7 @@ static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq, static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 addr, u64 period) { int ret = 0; @@ -1570,29 +1643,29 @@ static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, perf_sample__init(&sample, /*all=*/true); event->sample.header.type = PERF_RECORD_SAMPLE; - event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, tidq->el); + event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, packet->el); event->sample.header.size = sizeof(struct perf_event_header); /* Set time field based on etm auxtrace config. */ sample.time = cs_etm__resolve_sample_time(etmq, tidq); sample.ip = addr; - sample.pid = thread__pid(tidq->thread); - sample.tid = thread__tid(tidq->thread); + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); sample.id = etmq->etm->instructions_id; sample.stream_id = etmq->etm->instructions_id; sample.period = period; - sample.cpu = tidq->packet->cpu; + sample.cpu = packet->cpu; sample.flags = tidq->prev_packet->flags; sample.cpumode = event->sample.header.misc; - cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample); + cs_etm__copy_insn(etmq, tidq, packet, &sample); if (etm->synth_opts.last_branch) sample.branch_stack = tidq->last_branch; if (etm->synth_opts.inject) { - ret = cs_etm__inject_event(event, &sample, + ret = cs_etm__inject_event(etm, event, &sample, etm->instructions_sample_type); if (ret) return ret; @@ -1631,15 +1704,15 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, event->sample.header.type = PERF_RECORD_SAMPLE; event->sample.header.misc = cs_etm__cpu_mode(etmq, ip, - tidq->prev_packet_el); + tidq->prev_packet->el); event->sample.header.size = sizeof(struct perf_event_header); /* Set time field based on etm auxtrace config. */ sample.time = cs_etm__resolve_sample_time(etmq, tidq); sample.ip = ip; - sample.pid = thread__pid(tidq->prev_packet_thread); - sample.tid = thread__tid(tidq->prev_packet_thread); + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); sample.addr = cs_etm__first_executed_instr(tidq->packet); sample.id = etmq->etm->branches_id; sample.stream_id = etmq->etm->branches_id; @@ -1648,8 +1721,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, sample.flags = tidq->prev_packet->flags; sample.cpumode = event->sample.header.misc; - cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet, - &sample); + cs_etm__copy_insn(etmq, tidq, tidq->prev_packet, &sample); /* * perf report cannot handle events without a branch stack @@ -1667,7 +1739,7 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, } if (etm->synth_opts.inject) { - ret = cs_etm__inject_event(event, &sample, + ret = cs_etm__inject_event(etm, event, &sample, etm->branches_sample_type); if (ret) return ret; @@ -1770,7 +1842,6 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, { struct cs_etm_auxtrace *etm = etmq->etm; int ret; - u8 trace_chan_id = tidq->trace_chan_id; u64 instrs_prev; /* Get instructions remainder from previous packet */ @@ -1856,10 +1927,10 @@ static int cs_etm__sample(struct cs_etm_queue *etmq, * been executed, but PC has not advanced to next * instruction) */ - addr = cs_etm__instr_addr(etmq, trace_chan_id, - tidq->packet, offset - 1); + addr = cs_etm__instr_addr(etmq, tidq, tidq->packet, + offset - 1); ret = cs_etm__synth_instruction_sample( - etmq, tidq, addr, + etmq, tidq, tidq->packet, addr, etm->instructions_sample_period); if (ret) return ret; @@ -1941,7 +2012,7 @@ static int cs_etm__flush(struct cs_etm_queue *etmq, addr = cs_etm__last_executed_instr(tidq->prev_packet); err = cs_etm__synth_instruction_sample( - etmq, tidq, addr, + etmq, tidq, tidq->prev_packet, addr, tidq->period_instructions); if (err) return err; @@ -1996,7 +2067,7 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq, addr = cs_etm__last_executed_instr(tidq->prev_packet); err = cs_etm__synth_instruction_sample( - etmq, tidq, addr, + etmq, tidq, tidq->prev_packet, addr, tidq->period_instructions); if (err) return err; @@ -2033,9 +2104,9 @@ static int cs_etm__get_data_block(struct cs_etm_queue *etmq) return etmq->buf_len; } -static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, - struct cs_etm_packet *packet, - u64 end_addr) +static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 end_addr) { /* Initialise to keep compiler happy */ u16 instr16 = 0; @@ -2057,8 +2128,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, * so below only read 2 bytes as instruction size for T32. */ addr = end_addr - 2; - cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr16), - (u8 *)&instr16, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr16), (u8 *)&instr16); if ((instr16 & 0xFF00) == 0xDF00) return true; @@ -2073,8 +2144,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, * +---------+---------+-------------------------+ */ addr = end_addr - 4; - cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32), - (u8 *)&instr32, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); if ((instr32 & 0x0F000000) == 0x0F000000 && (instr32 & 0xF0000000) != 0xF0000000) return true; @@ -2090,8 +2161,8 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, * +-----------------------+---------+-----------+ */ addr = end_addr - 4; - cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32), - (u8 *)&instr32, 0); + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); if ((instr32 & 0xFFE0001F) == 0xd4000001) return true; @@ -2107,7 +2178,6 @@ static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id, static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, struct cs_etm_traceid_queue *tidq, u64 magic) { - u8 trace_chan_id = tidq->trace_chan_id; struct cs_etm_packet *packet = tidq->packet; struct cs_etm_packet *prev_packet = tidq->prev_packet; @@ -2122,7 +2192,7 @@ static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, */ if (magic == __perf_cs_etmv4_magic) { if (packet->exception_number == CS_ETMV4_EXC_CALL && - cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, + cs_etm__is_svc_instr(etmq, tidq, prev_packet, prev_packet->end_addr)) return true; } @@ -2160,7 +2230,6 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, struct cs_etm_traceid_queue *tidq, u64 magic) { - u8 trace_chan_id = tidq->trace_chan_id; struct cs_etm_packet *packet = tidq->packet; struct cs_etm_packet *prev_packet = tidq->prev_packet; @@ -2186,7 +2255,7 @@ static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, * (SMC, HVC) are taken as sync exceptions. */ if (packet->exception_number == CS_ETMV4_EXC_CALL && - !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet, + !cs_etm__is_svc_instr(etmq, tidq, prev_packet, prev_packet->end_addr)) return true; @@ -2210,7 +2279,6 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, { struct cs_etm_packet *packet = tidq->packet; struct cs_etm_packet *prev_packet = tidq->prev_packet; - u8 trace_chan_id = tidq->trace_chan_id; u64 magic; int ret; @@ -2291,11 +2359,11 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, if (prev_packet->flags == (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT) && - cs_etm__is_svc_instr(etmq, trace_chan_id, - packet, packet->start_addr)) + cs_etm__is_svc_instr(etmq, tidq, packet, packet->start_addr)) { prev_packet->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_SYSCALLRET; + } break; case CS_ETM_DISCONTINUITY: /* @@ -2376,6 +2444,7 @@ static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT; break; + case CS_ETM_CONTEXT: case CS_ETM_EMPTY: default: break; @@ -2451,6 +2520,19 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, */ cs_etm__sample(etmq, tidq); break; + case CS_ETM_CONTEXT: + /* + * Update context but don't swap packet. Keep the + * previous one for branch source address info, if + * tracing the kernel the context packet will be emitted + * between two ranges. + */ + ret = cs_etm__etmq_update_thread(etmq, tidq->packet->el, + tidq->packet->tid, + &tidq->frontend_thread); + if (ret) + goto out; + break; case CS_ETM_EXCEPTION: case CS_ETM_EXCEPTION_RET: /* @@ -2479,6 +2561,7 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, } } +out: return ret; } @@ -2602,7 +2685,7 @@ static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, if (!tidq) continue; - if (tid == -1 || thread__tid(tidq->thread) == tid) + if (tid == -1 || thread__tid(tidq->frontend_thread) == tid) cs_etm__run_per_thread_timeless_decoder(etmq); } else cs_etm__run_per_cpu_timeless_decoder(etmq); @@ -3079,6 +3162,9 @@ static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_o aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { struct cs_etm_queue *etmq = cs_etm__get_queue(etm, auxtrace_event->cpu); + if (!etmq) + return -EINVAL; + /* * If this AUX event was inside this buffer somewhere, create a new auxtrace event * based on the sizes of the aux event, and queue that fragment. @@ -3310,7 +3396,7 @@ static int cs_etm__create_queue_decoders(struct cs_etm_queue *etmq) */ if (cs_etm_decoder__add_mem_access_cb(etmq->decoder, 0x0L, ((u64) -1L), - cs_etm__mem_access)) + cs_etm__decoder_mem_access)) goto out_free_decoder; zfree(&t_params); @@ -3366,6 +3452,18 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, /* First the global part */ ptr = (u64 *) auxtrace_info->priv; num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff; + + /* + * Bound num_cpu by the event size: the global header consumes + * CS_ETM_HEADER_SIZE bytes, and each CPU needs at least one u64 + * metadata entry after that. + */ + priv_size = total_size - event_header_size - INFO_HEADER_SIZE - + CS_ETM_HEADER_SIZE; + if (num_cpu <= 0 || priv_size <= 0 || + num_cpu > priv_size / (int)sizeof(u64)) + return -EINVAL; + metadata = zalloc(sizeof(*metadata) * num_cpu); if (!metadata) return -ENOMEM; @@ -3404,7 +3502,13 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, goto err_free_metadata; } - if ((int) metadata[j][CS_ETM_CPU] > max_cpu) + /* CPU id comes from perf.data and must fit max_cpu + 1 without overflow */ + if (metadata[j][CS_ETM_CPU] >= INT_MAX) { + err = -EINVAL; + goto err_free_metadata; + } + + if ((int)metadata[j][CS_ETM_CPU] > max_cpu) max_cpu = metadata[j][CS_ETM_CPU]; } @@ -3496,7 +3600,7 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, etm->tc.time_shift = tc->time_shift; etm->tc.time_mult = tc->time_mult; etm->tc.time_zero = tc->time_zero; - if (event_contains(*tc, time_cycles)) { + if (event_contains(*tc, cap_user_time_short)) { etm->tc.time_cycles = tc->time_cycles; etm->tc.time_mask = tc->time_mask; etm->tc.cap_user_time_zero = tc->cap_user_time_zero; diff --git a/tools/perf/util/cs-etm.h b/tools/perf/util/cs-etm.h index aa9bb4a32eca..b81099c2b301 100644 --- a/tools/perf/util/cs-etm.h +++ b/tools/perf/util/cs-etm.h @@ -158,6 +158,7 @@ enum cs_etm_sample_type { CS_ETM_DISCONTINUITY, CS_ETM_EXCEPTION, CS_ETM_EXCEPTION_RET, + CS_ETM_CONTEXT, }; enum cs_etm_isa { @@ -184,6 +185,8 @@ struct cs_etm_packet { u8 last_instr_size; u8 trace_chan_id; int cpu; + int el; + pid_t tid; }; #define CS_ETM_PACKET_MAX_BUFFER 1024 @@ -259,8 +262,9 @@ enum cs_etm_pid_fmt { #include <opencsd/ocsd_if_types.h> int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu); enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq); -int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid, - u8 trace_chan_id, ocsd_ex_level el); +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, ocsd_ex_level el, + pid_t tid); bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq); void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, u8 trace_chan_id); diff --git a/tools/perf/util/data-convert-bt.c b/tools/perf/util/data-convert-bt.c index 3b8f2df823a9..5ff46bfcd0e1 100644 --- a/tools/perf/util/data-convert-bt.c +++ b/tools/perf/util/data-convert-bt.c @@ -11,14 +11,13 @@ #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/zalloc.h> -#include <babeltrace/ctf-writer/writer.h> -#include <babeltrace/ctf-writer/clock.h> -#include <babeltrace/ctf-writer/stream.h> -#include <babeltrace/ctf-writer/event.h> -#include <babeltrace/ctf-writer/event-types.h> -#include <babeltrace/ctf-writer/event-fields.h> -#include <babeltrace/ctf-ir/utils.h> -#include <babeltrace/ctf/events.h> +#include <babeltrace2-ctf-writer/writer.h> +#include <babeltrace2-ctf-writer/clock.h> +#include <babeltrace2-ctf-writer/stream.h> +#include <babeltrace2-ctf-writer/event.h> +#include <babeltrace2-ctf-writer/event-types.h> +#include <babeltrace2-ctf-writer/event-fields.h> +#include <babeltrace2-ctf-writer/utils.h> #include "asm/bug.h" #include "data-convert.h" #include "session.h" @@ -121,13 +120,13 @@ static int value_set(struct bt_ctf_field_type *type, } if (sign) { - ret = bt_ctf_field_signed_integer_set_value(field, val); + ret = bt_ctf_field_integer_signed_set_value(field, val); if (ret) { pr_err("failed to set field value %s\n", name); goto err; } } else { - ret = bt_ctf_field_unsigned_integer_set_value(field, val); + ret = bt_ctf_field_integer_unsigned_set_value(field, val); if (ret) { pr_err("failed to set field value %s\n", name); goto err; @@ -374,10 +373,10 @@ static int add_tracepoint_field_value(struct ctf_writer *cw, data + offset + i * len, len); if (!(flags & TEP_FIELD_IS_SIGNED)) - ret = bt_ctf_field_unsigned_integer_set_value( + ret = bt_ctf_field_integer_unsigned_set_value( field, value_int); else - ret = bt_ctf_field_signed_integer_set_value( + ret = bt_ctf_field_integer_signed_set_value( field, adjust_signedness(value_int, len)); } @@ -471,7 +470,7 @@ add_bpf_output_values(struct bt_ctf_event_class *event_class, goto put_len_type; } - ret = bt_ctf_field_unsigned_integer_set_value(len_field, nr_elements); + ret = bt_ctf_field_integer_unsigned_set_value(len_field, nr_elements); if (ret) { pr_err("failed to set field value for raw_len\n"); goto put_len_field; @@ -500,7 +499,7 @@ add_bpf_output_values(struct bt_ctf_event_class *event_class, struct bt_ctf_field *elem_field = bt_ctf_field_sequence_get_field(seq_field, i); - ret = bt_ctf_field_unsigned_integer_set_value(elem_field, + ret = bt_ctf_field_integer_unsigned_set_value(elem_field, ((u32 *)(sample->raw_data))[i]); bt_ctf_field_put(elem_field); @@ -545,7 +544,7 @@ add_callchain_output_values(struct bt_ctf_event_class *event_class, goto put_len_type; } - ret = bt_ctf_field_unsigned_integer_set_value(len_field, nr_elements); + ret = bt_ctf_field_integer_unsigned_set_value(len_field, nr_elements); if (ret) { pr_err("failed to set field value for perf_callchain_size\n"); goto put_len_field; @@ -575,7 +574,7 @@ add_callchain_output_values(struct bt_ctf_event_class *event_class, struct bt_ctf_field *elem_field = bt_ctf_field_sequence_get_field(seq_field, i); - ret = bt_ctf_field_unsigned_integer_set_value(elem_field, + ret = bt_ctf_field_integer_unsigned_set_value(elem_field, ((u64 *)(callchain->ips))[i]); bt_ctf_field_put(elem_field); @@ -728,7 +727,7 @@ static struct ctf_stream *ctf_stream__create(struct ctf_writer *cw, int cpu) goto out; } - ret = bt_ctf_field_unsigned_integer_set_value(cpu_field, (u32) cpu); + ret = bt_ctf_field_integer_unsigned_set_value(cpu_field, (u32) cpu); if (ret) { pr_err("Failed to update CPU number\n"); goto out; @@ -803,10 +802,10 @@ static bool is_flush_needed(struct ctf_stream *cs) static int process_sample_event(const struct perf_tool *tool, union perf_event *_event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine __maybe_unused) { struct convert *c = container_of(tool, struct convert, tool); + struct evsel *evsel = sample->evsel; struct evsel_priv *priv = evsel->priv; struct ctf_writer *cw = &c->writer; struct ctf_stream *cs; @@ -1414,7 +1413,7 @@ do { \ ADD("host", env->hostname); ADD("sysname", "Linux"); - ADD("release", env->os_release); + ADD("release", perf_env__os_release(env)); ADD("version", env->version); ADD("machine", env->arch); ADD("domain", "kernel"); diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c index d526c91312ed..40412c3dbdb2 100644 --- a/tools/perf/util/data-convert-json.c +++ b/tools/perf/util/data-convert-json.c @@ -16,6 +16,7 @@ #include "linux/err.h" #include "util/auxtrace.h" #include "util/debug.h" +#include "util/env.h" #include "util/dso.h" #include "util/event.h" #include "util/evsel.h" @@ -159,13 +160,12 @@ static void output_sample_callchain_entry(const struct perf_tool *tool, static int process_sample_event(const struct perf_tool *tool, union perf_event *event __maybe_unused, struct perf_sample *sample, - struct evsel *evsel __maybe_unused, struct machine *machine) { struct convert_json *c = container_of(tool, struct convert_json, tool); FILE *out = c->out; struct addr_location al; - u64 sample_type = __evlist__combined_sample_type(evsel->evlist); + u64 sample_type = __evlist__combined_sample_type(sample->evsel->evlist); u8 cpumode = PERF_RECORD_MISC_USER; addr_location__init(&al); @@ -177,6 +177,7 @@ static int process_sample_event(const struct perf_tool *tool, if (perf_time__ranges_skip_sample(c->ptime_range, c->range_num, sample->time)) { ++c->skipped; + addr_location__exit(&al); return 0; } @@ -245,7 +246,7 @@ static int process_sample_event(const struct perf_tool *tool, #ifdef HAVE_LIBTRACEEVENT if (sample->raw_data) { - struct tep_event *tp_format = evsel__tp_format(evsel); + struct tep_event *tp_format = evsel__tp_format(sample->evsel); struct tep_format_field **fields = tp_format ? tep_event_fields(tp_format) : NULL; if (fields) { @@ -273,7 +274,7 @@ static void output_headers(struct perf_session *session, struct convert_json *c) { struct stat st; const struct perf_header *header = &session->header; - const struct perf_env *env = perf_session__env(session); + struct perf_env *env = perf_session__env(session); int ret; int fd = perf_data__fd(session->data); int i; @@ -297,7 +298,8 @@ static void output_headers(struct perf_session *session, struct convert_json *c) output_json_key_format(out, true, 2, "feat-offset", "%" PRIu64, header->feat_offset); output_json_key_string(out, true, 2, "hostname", env->hostname); - output_json_key_string(out, true, 2, "os-release", env->os_release); + output_json_key_string(out, true, 2, "os-release", + perf_env__os_release(env)); output_json_key_string(out, true, 2, "arch", env->arch); if (env->cpu_desc) diff --git a/tools/perf/util/data-convert.h b/tools/perf/util/data-convert.h index ee651fa680a1..a96240f15671 100644 --- a/tools/perf/util/data-convert.h +++ b/tools/perf/util/data-convert.h @@ -11,10 +11,10 @@ struct perf_data_convert_opts { const char *time_str; }; -#ifdef HAVE_LIBBABELTRACE_SUPPORT +#ifdef HAVE_BABELTRACE2_CTF_WRITER_SUPPORT int bt_convert__perf2ctf(const char *input_name, const char *to_ctf, struct perf_data_convert_opts *opts); -#endif /* HAVE_LIBBABELTRACE_SUPPORT */ +#endif /* HAVE_BABELTRACE2_CTF_WRITER_SUPPORT */ int bt_convert__perf2json(const char *input_name, const char *to_ctf, struct perf_data_convert_opts *opts); diff --git a/tools/perf/util/db-export.c b/tools/perf/util/db-export.c index cc2bb1af4243..ba54b1119ee6 100644 --- a/tools/perf/util/db-export.c +++ b/tools/perf/util/db-export.c @@ -208,8 +208,7 @@ static int db_ids_from_al(struct db_export *dbe, struct addr_location *al, static struct call_path *call_path_from_sample(struct db_export *dbe, struct machine *machine, struct thread *thread, - struct perf_sample *sample, - struct evsel *evsel) + struct perf_sample *sample) { u64 kernel_start = machine__kernel_start(machine); struct call_path *current = &dbe->cpr->call_path; @@ -227,7 +226,7 @@ static struct call_path *call_path_from_sample(struct db_export *dbe, */ callchain_param.order = ORDER_CALLER; cursor = get_tls_callchain_cursor(); - err = thread__resolve_callchain(thread, cursor, evsel, + err = thread__resolve_callchain(thread, cursor, sample, NULL, NULL, PERF_MAX_STACK_DEPTH); if (err) { callchain_param.order = saved_order; @@ -345,14 +344,13 @@ static int db_export__threads(struct db_export *dbe, struct thread *thread, } int db_export__sample(struct db_export *dbe, union perf_event *event, - struct perf_sample *sample, struct evsel *evsel, + struct perf_sample *sample, struct addr_location *al, struct addr_location *addr_al) { struct thread *thread = al->thread; struct export_sample es = { .event = event, .sample = sample, - .evsel = evsel, .al = al, }; struct thread *main_thread; @@ -365,7 +363,7 @@ int db_export__sample(struct db_export *dbe, union perf_event *event, if (!machine) return -1; - err = db_export__evsel(dbe, evsel); + err = db_export__evsel(dbe, sample->evsel); if (err) return err; @@ -390,8 +388,7 @@ int db_export__sample(struct db_export *dbe, union perf_event *event, if (dbe->cpr) { struct call_path *cp = call_path_from_sample(dbe, machine, - thread, sample, - evsel); + thread, sample); if (cp) { db_export__call_path(dbe, cp); es.call_path_id = cp->db_id; diff --git a/tools/perf/util/db-export.h b/tools/perf/util/db-export.h index 23983cb35706..1abbfd398e3a 100644 --- a/tools/perf/util/db-export.h +++ b/tools/perf/util/db-export.h @@ -25,7 +25,6 @@ struct call_return; struct export_sample { union perf_event *event; struct perf_sample *sample; - struct evsel *evsel; struct addr_location *al; u64 db_id; u64 comm_db_id; @@ -96,7 +95,7 @@ int db_export__symbol(struct db_export *dbe, struct symbol *sym, int db_export__branch_type(struct db_export *dbe, u32 branch_type, const char *name); int db_export__sample(struct db_export *dbe, union perf_event *event, - struct perf_sample *sample, struct evsel *evsel, + struct perf_sample *sample, struct addr_location *al, struct addr_location *addr_al); int db_export__branch_types(struct db_export *dbe); diff --git a/tools/perf/util/debuginfo.c b/tools/perf/util/debuginfo.c index 0e35c13abd04..84a78b30ceac 100644 --- a/tools/perf/util/debuginfo.c +++ b/tools/perf/util/debuginfo.c @@ -42,6 +42,7 @@ static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, { GElf_Addr dummy; int fd; + bool fd_consumed = false; fd = open(path, O_RDONLY); if (fd < 0) @@ -55,6 +56,7 @@ static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, dbg->mod = dwfl_report_offline(dbg->dwfl, "", "", fd); if (!dbg->mod) goto error; + fd_consumed = true; dbg->dbg = dwfl_module_getdwarf(dbg->mod, &dbg->bias); if (!dbg->dbg) @@ -62,13 +64,14 @@ static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, dwfl_module_build_id(dbg->mod, &dbg->build_id, &dummy); - dwfl_report_end(dbg->dwfl, NULL, NULL); + if (dwfl_report_end(dbg->dwfl, NULL, NULL) != 0) + goto error; return 0; error: if (dbg->dwfl) dwfl_end(dbg->dwfl); - else + if (!fd_consumed) close(fd); memset(dbg, 0, sizeof(*dbg)); @@ -167,7 +170,7 @@ int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, /* Search the relocation related .text section */ for (i = 0; i < n; i++) { p = dwfl_module_relocation_info(dbg->mod, i, &shndx); - if (strcmp(p, ".text") == 0) { + if (p && strcmp(p, ".text") == 0) { /* OK, get the section header */ scn = elf_getscn(elf, shndx); if (!scn) diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c index 59ba88e1f744..0a1a7e9cf3ef 100644 --- a/tools/perf/util/disasm.c +++ b/tools/perf/util/disasm.c @@ -1577,8 +1577,11 @@ int symbol__disassemble(struct symbol *sym, struct annotate_args *args) if (dso__decompress_kmodule_path(dso, symfs_filename, tmp, sizeof(tmp)) < 0) return -1; - decomp = true; - strcpy(symfs_filename, tmp); + /* empty pathname means file wasn't actually compressed */ + if (tmp[0] != '\0') { + decomp = true; + strcpy(symfs_filename, tmp); + } } /* diff --git a/tools/perf/util/dlfilter.c b/tools/perf/util/dlfilter.c index dc31b5e7149e..e11e144af62b 100644 --- a/tools/perf/util/dlfilter.c +++ b/tools/perf/util/dlfilter.c @@ -56,7 +56,7 @@ static void al_to_d_al(struct addr_location *al, struct perf_dlfilter_al *d_al) d_al->symoff = al->addr - map__start(al->map) - sym->start; else d_al->symoff = 0; - d_al->sym_binding = sym->binding; + d_al->sym_binding = symbol__binding(sym); } else { d_al->sym = NULL; d_al->sym_start = 0; diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index b791e1b6b2cf..2309196d8df3 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -343,10 +343,16 @@ int filename__decompress(const char *name, char *pathname, * To keep this transparent, we detect this and return the file * descriptor to the uncompressed file. */ - if (!compressions[comp].is_compressed(name)) - return open(name, O_RDONLY); + if (!compressions[comp].is_compressed(name)) { + fd = open(name, O_RDONLY | O_CLOEXEC); + if (fd < 0) + *err = errno; + if (pathname && len > 0) + pathname[0] = '\0'; + return fd; + } - fd = mkstemp(tmpbuf); + fd = mkostemp(tmpbuf, O_CLOEXEC); if (fd < 0) { *err = errno; return -1; @@ -594,12 +600,28 @@ static char *dso__get_filename(struct dso *dso, const char *root_dir, size_t len = sizeof(newpath); if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) { - errno = *dso__load_errno(dso); + /* + * Use a standard errno value, not the negative custom + * DSO_LOAD_ERRNO stored in dso__load_errno(dso): + * __open_dso() computes fd = -errno, so a negative + * errno produces a positive fd that looks valid. + */ + errno = EIO; goto out; } - *decomp = true; - strcpy(name, newpath); + /* empty pathname means file wasn't actually compressed */ + if (newpath[0] != '\0') { + char *tmp = strdup(newpath); + + if (!tmp) { + unlink(newpath); + goto out; + } + free(name); + name = tmp; + *decomp = true; + } } return name; @@ -864,6 +886,12 @@ static ssize_t bpf_read(struct dso *dso, u64 offset, char *data) return -1; } + /* jited_prog_insns is only valid if bpil_offs_to_addr() converted it */ + if (!(node->info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + return -1; + } + len = node->info_linear->info.jited_prog_len; buf = (u8 *)(uintptr_t)node->info_linear->info.jited_prog_insns; @@ -1220,7 +1248,8 @@ static enum dso_swap_type dso_swap_type__from_elf_data(unsigned char eidata) } /* Reads e_machine from fd, optionally caching data in dso. */ -uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags) +uint16_t dso__read_e_machine_endian(struct dso *optional_dso, int fd, uint32_t *e_flags, + bool *is_big_endian) { uint16_t e_machine = EM_NONE; unsigned char e_ident[EI_NIDENT]; @@ -1250,6 +1279,9 @@ uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags if (swap_type == DSO_SWAP__UNSET) return EM_NONE; // Bad ELF data encoding. + if (is_big_endian) + *is_big_endian = (e_ident[EI_DATA] == ELFDATA2MSB); + /* Cache the need for swapping. */ if (optional_dso) { assert(dso__needs_swap(optional_dso) == DSO_SWAP__UNSET || @@ -1288,7 +1320,8 @@ uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags return e_machine; } -uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_flags) +uint16_t dso__e_machine_endian(struct dso *dso, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian) { uint16_t e_machine = EM_NONE; int fd; @@ -1308,9 +1341,11 @@ uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_fl case DSO_BINARY_TYPE__BPF_IMAGE: case DSO_BINARY_TYPE__OOL: case DSO_BINARY_TYPE__JAVA_JIT: - if (e_flags) - *e_flags = EF_HOST; - return EM_HOST; + if (is_big_endian) { + *is_big_endian = perf_arch_is_big_endian( + machine && machine->env ? perf_env__arch(machine->env) : NULL); + } + return perf_env__e_machine(machine ? machine->env : NULL, e_flags); case DSO_BINARY_TYPE__DEBUGLINK: case DSO_BINARY_TYPE__BUILD_ID_CACHE: case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: @@ -1338,7 +1373,7 @@ uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_fl try_to_open_dso(dso, machine); fd = dso__data(dso)->fd; if (fd >= 0) - e_machine = dso__read_e_machine(dso, fd, e_flags); + e_machine = dso__read_e_machine_endian(dso, fd, e_flags, is_big_endian); else if (e_flags) *e_flags = 0; @@ -1766,7 +1801,7 @@ void dso__read_running_kernel_build_id(struct dso *dso, struct machine *machine) if (machine__is_default_guest(machine)) return; - sprintf(path, "%s/sys/kernel/notes", machine->root_dir); + snprintf(path, sizeof(path), "%s/sys/kernel/notes", machine->root_dir); sysfs__read_build_id(path, &bid); dso__set_build_id(dso, &bid); } @@ -1904,7 +1939,7 @@ static const u8 *__dso__read_symbol(struct dso *dso, const char *symfs_filename, int saved_errno; nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); - fd = open(symfs_filename, O_RDONLY); + fd = open(symfs_filename, O_RDONLY | O_CLOEXEC); saved_errno = errno; nsinfo__mountns_exit(&nsc); if (fd < 0) { @@ -1972,6 +2007,10 @@ const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, return NULL; } info_linear = info_node->info_linear; + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } assert(len <= info_linear->info.jited_prog_len); *out_buf_len = len; return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns); diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index ede691e9a249..2916b954a804 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -866,8 +866,18 @@ int dso__data_file_size(struct dso *dso, struct machine *machine); off_t dso__data_size(struct dso *dso, struct machine *machine); ssize_t dso__data_read_offset(struct dso *dso, struct machine *machine, u64 offset, u8 *data, ssize_t size); -uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags); -uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_flags); +uint16_t dso__read_e_machine_endian(struct dso *optional_dso, int fd, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags) +{ + return dso__read_e_machine_endian(optional_dso, fd, e_flags, NULL); +} +uint16_t dso__e_machine_endian(struct dso *dso, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_flags) +{ + return dso__e_machine_endian(dso, machine, e_flags, NULL); +} ssize_t dso__data_read_addr(struct dso *dso, struct map *map, struct machine *machine, u64 addr, u8 *data, ssize_t size); diff --git a/tools/perf/util/dwarf-aux.c b/tools/perf/util/dwarf-aux.c index 92db2fccc788..d7160f87ac7d 100644 --- a/tools/perf/util/dwarf-aux.c +++ b/tools/perf/util/dwarf-aux.c @@ -125,7 +125,8 @@ int cu_find_lineinfo(Dwarf_Die *cu_die, Dwarf_Addr addr, && die_entrypc(&die_mem, &faddr) == 0 && faddr == addr) { *fname = die_get_decl_file(&die_mem); - dwarf_decl_line(&die_mem, lineno); + if (dwarf_decl_line(&die_mem, lineno) != 0) + return -ENOENT; goto out; } @@ -171,7 +172,6 @@ int cu_walk_functions_at(Dwarf_Die *cu_die, Dwarf_Addr addr, } return ret; - } /** @@ -460,7 +460,7 @@ int die_get_data_member_location(Dwarf_Die *mb_die, Dwarf_Word *offs) size_t nexpr; int ret; - if (dwarf_attr(mb_die, DW_AT_data_member_location, &attr) == NULL) + if (dwarf_attr_integrate(mb_die, DW_AT_data_member_location, &attr) == NULL) return -ENOENT; if (dwarf_formudata(&attr, offs) != 0) { @@ -620,7 +620,7 @@ Dwarf_Die *die_find_tailfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, ad.addr = addr; ad.die_mem = die_mem; /* dwarf_getscopes can't find subprogram. */ - if (!dwarf_getfuncs(cu_die, __die_search_func_tail_cb, &ad, 0)) + if (dwarf_getfuncs(cu_die, __die_search_func_tail_cb, &ad, 0) <= 0) return NULL; else return die_mem; @@ -659,7 +659,7 @@ Dwarf_Die *die_find_realfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, ad.addr = addr; ad.die_mem = die_mem; /* dwarf_getscopes can't find subprogram. */ - if (!dwarf_getfuncs(cu_die, __die_search_func_cb, &ad, 0)) + if (dwarf_getfuncs(cu_die, __die_search_func_cb, &ad, 0) <= 0) return NULL; else return die_mem; @@ -796,8 +796,7 @@ static int __die_walk_instances_cb(Dwarf_Die *inst, void *data) /* Ignore redundant instances */ if (dwarf_tag(inst) == DW_TAG_inlined_subroutine) { - dwarf_decl_line(origin, &tmp); - if (die_get_call_lineno(inst) == tmp) { + if (dwarf_decl_line(origin, &tmp) == 0 && die_get_call_lineno(inst) == tmp) { tmp = die_get_decl_fileno(origin); if (die_get_call_fileno(inst) == tmp) return DIE_FIND_CB_CONTINUE; @@ -951,11 +950,6 @@ int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) cu_die = dwarf_diecu(rt_die, &die_mem, NULL, NULL); dwarf_decl_line(rt_die, &decl); decf = die_get_decl_file(rt_die); - if (!decf) { - pr_debug2("Failed to get the declared file name of %s\n", - dwarf_diename(rt_die)); - return -EINVAL; - } } else cu_die = rt_die; if (!cu_die) { @@ -999,11 +993,12 @@ int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) if (die_find_inlinefunc(rt_die, addr, &die_mem)) { /* Call-site check */ inf = die_get_call_file(&die_mem); - if ((inf && !strcmp(inf, decf)) && + if ((inf == decf || (inf && decf && !strcmp(inf, decf))) && die_get_call_lineno(&die_mem) == lineno) goto found; - dwarf_decl_line(&die_mem, &inl); + if (dwarf_decl_line(&die_mem, &inl) != 0) + inl = 0; if (inl != decl || decf != die_get_decl_file(&die_mem)) continue; @@ -1035,8 +1030,10 @@ found: .data = data, .retval = 0, }; - dwarf_getfuncs(cu_die, __die_walk_culines_cb, ¶m, 0); - ret = param.retval; + if (dwarf_getfuncs(cu_die, __die_walk_culines_cb, ¶m, 0) < 0) + ret = -EINVAL; + else + ret = param.retval; } return ret; @@ -1940,10 +1937,12 @@ static bool die_get_postprologue_addr(unsigned long entrypc_idx, break; } - dwarf_lineaddr(line, postprologue_addr); - if (*postprologue_addr >= highpc) - dwarf_lineaddr(dwarf_onesrcline(lines, i - 1), - postprologue_addr); + if (dwarf_lineaddr(line, postprologue_addr) != 0) + return false; + if (*postprologue_addr >= highpc) { + if (dwarf_lineaddr(dwarf_onesrcline(lines, i - 1), postprologue_addr) != 0) + return false; + } return true; } diff --git a/tools/perf/util/dwarf-aux.h b/tools/perf/util/dwarf-aux.h index a79968a2e573..161f0bf980b6 100644 --- a/tools/perf/util/dwarf-aux.h +++ b/tools/perf/util/dwarf-aux.h @@ -10,6 +10,11 @@ #include <elfutils/libdwfl.h> #include <elfutils/version.h> +static inline const char *die_name(Dwarf_Die *die) +{ + return dwarf_diename(die) ?: "<unknown>"; +} + struct strbuf; /* Find the realpath of the target file */ diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c index 1e54e2c86360..c0e2b9d5f0b2 100644 --- a/tools/perf/util/env.c +++ b/tools/perf/util/env.c @@ -1,10 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 #include "cpumap.h" +#include "dwarf-regs.h" #include "debug.h" #include "env.h" #include "util/header.h" #include "util/rwsem.h" #include <linux/compiler.h> +#include <linux/kernel.h> #include <linux/ctype.h> #include <linux/rbtree.h> #include <linux/string.h> @@ -248,6 +250,8 @@ void perf_env__exit(struct perf_env *env) { int i, j; + mutex_destroy(&env->lock); + perf_env__purge_bpf(env); perf_env__purge_cgroups(env); zfree(&env->hostname); @@ -305,19 +309,32 @@ void perf_env__init(struct perf_env *env) init_rwsem(&env->bpf_progs.lock); #endif env->kernel_is_64_bit = -1; + mutex_init(&env->lock); } static void perf_env__init_kernel_mode(struct perf_env *env) { - const char *arch = perf_env__raw_arch(env); + const char *arch = env->arch; + + if (!arch) { + static struct utsname uts = { .machine[0] = '\0', }; - if (!strncmp(arch, "x86_64", 6) || !strncmp(arch, "aarch64", 7) || - !strncmp(arch, "arm64", 5) || !strncmp(arch, "mips64", 6) || - !strncmp(arch, "parisc64", 8) || !strncmp(arch, "riscv64", 7) || - !strncmp(arch, "s390x", 5) || !strncmp(arch, "sparc64", 7)) - env->kernel_is_64_bit = 1; - else - env->kernel_is_64_bit = 0; + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') + arch = uts.machine; + } + + if (arch) { + if (strstr(arch, "64") || strstr(arch, "s390x")) + env->kernel_is_64_bit = 1; + else + env->kernel_is_64_bit = 0; + return; + } + + /* Fallback if completely unresolvable (assume host-bitness) */ + env->kernel_is_64_bit = (sizeof(void *) == 8) ? 1 : 0; } int perf_env__kernel_is_64_bit(struct perf_env *env) @@ -328,6 +345,60 @@ int perf_env__kernel_is_64_bit(struct perf_env *env) return env->kernel_is_64_bit; } +bool perf_arch_is_big_endian(const char *arch) +{ + if (!arch) + return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__; + + if (str_ends_with(arch, "_be") || !strcmp(arch, "sparc") || !strcmp(arch, "sparc64") || + !strcmp(arch, "s390") || !strcmp(arch, "s390x") || !strcmp(arch, "powerpc") || + !strcmp(arch, "ppc") || !strcmp(arch, "ppc64") || + !strcmp(arch, "mips") || !strcmp(arch, "mips64") || !strcmp(arch, "parisc") || + !strcmp(arch, "parisc64") || !strcmp(arch, "m68k") || + !strcmp(arch, "armeb") || !strcmp(arch, "mipseb") || !strcmp(arch, "mips64eb")) + return true; + + return false; +} + +const char *perf_env__os_release(struct perf_env *env) +{ + struct utsname uts; + int ret; + const char *release; + + if (!env) + return perf_version_string; + + mutex_lock(&env->lock); + if (env->os_release) { + release = env->os_release; + goto out; + } + + /* + * If env->arch is set, this is an offline target environment. + * If the os_release is not populated in the file, we do not want + * to poison it with the host's release which would break guest checks. + */ + if (env->arch) { + release = NULL; + goto out; + } + + /* + * The os_release is being accessed but wasn't initialized from a data + * file, assume this is 'live' mode and use the release from uname. If + * uname or strdup fails then use the current perf tool version. + */ + ret = uname(&uts); + env->os_release = strdup(ret < 0 ? perf_version_string : uts.release); + release = env->os_release ?: perf_version_string; +out: + mutex_unlock(&env->lock); + return release; +} + int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]) { int i; @@ -440,19 +511,6 @@ int perf_env__read_cpuid(struct perf_env *env) return 0; } -static int perf_env__read_arch(struct perf_env *env) -{ - struct utsname uts; - - if (env->arch) - return 0; - - if (!uname(&uts)) - env->arch = strdup(uts.machine); - - return env->arch ? 0 : -ENOMEM; -} - static int perf_env__read_nr_cpus_avail(struct perf_env *env) { if (env->nr_cpus_avail == 0) @@ -571,11 +629,6 @@ error: return ret; } -const char *perf_env__raw_arch(struct perf_env *env) -{ - return env && !perf_env__read_arch(env) ? env->arch : "unknown"; -} - int perf_env__nr_cpus_avail(struct perf_env *env) { return env && !perf_env__read_nr_cpus_avail(env) ? env->nr_cpus_avail : 0; @@ -588,67 +641,244 @@ void cpu_cache_level__free(struct cpu_cache_level *cache) zfree(&cache->size); } +struct arch_to_e_machine { + const char *prefix; + uint16_t e_machine; +}; + /* - * Return architecture name in a normalized form. - * The conversion logic comes from the Makefile. + * A mapping from an arch prefix string to an ELF machine that can be used in a + * bsearch. Some arch prefixes are shared an need additional processing as + * marked next to the architecture. The prefixes handle both perf's architecture + * naming and those from uname. */ -static const char *normalize_arch(char *arch) -{ - if (!strcmp(arch, "x86_64")) - return "x86"; - if (arch[0] == 'i' && arch[2] == '8' && arch[3] == '6') - return "x86"; - if (!strcmp(arch, "sun4u") || !strncmp(arch, "sparc", 5)) - return "sparc"; - if (!strncmp(arch, "aarch64", 7) || !strncmp(arch, "arm64", 5)) - return "arm64"; - if (!strncmp(arch, "arm", 3) || !strcmp(arch, "sa110")) - return "arm"; - if (!strncmp(arch, "s390", 4)) - return "s390"; - if (!strncmp(arch, "parisc", 6)) - return "parisc"; - if (!strncmp(arch, "powerpc", 7) || !strncmp(arch, "ppc", 3)) - return "powerpc"; - if (!strncmp(arch, "mips", 4)) - return "mips"; - if (!strncmp(arch, "sh", 2) && isdigit(arch[2])) - return "sh"; - if (!strncmp(arch, "loongarch", 9)) - return "loongarch"; +static const struct arch_to_e_machine prefix_to_e_machine[] = { + {"aarch64", EM_AARCH64}, + {"alpha", EM_ALPHA}, + {"arc", EM_ARC}, + {"arm", EM_ARM}, /* Check also for EM_AARCH64. */ + {"avr", EM_AVR}, /* Check also for EM_AVR32. */ + {"bfin", EM_BLACKFIN}, + {"blackfin", EM_BLACKFIN}, + {"cris", EM_CRIS}, + {"csky", EM_CSKY}, + {"hppa", EM_PARISC}, + {"i386", EM_386}, + {"i486", EM_386}, + {"i586", EM_386}, + {"i686", EM_386}, + {"loongarch", EM_LOONGARCH}, + {"m32r", EM_M32R}, + {"m68k", EM_68K}, + {"microblaze", EM_MICROBLAZE}, + {"mips", EM_MIPS}, + {"msp430", EM_MSP430}, + {"parisc", EM_PARISC}, + {"powerpc", EM_PPC}, /* Check also for EM_PPC64. */ + {"ppc", EM_PPC}, /* Check also for EM_PPC64. */ + {"riscv", EM_RISCV}, + {"s390", EM_S390}, + {"sa110", EM_ARM}, + {"sh", EM_SH}, + {"sparc", EM_SPARC}, /* Check also for EM_SPARCV9. */ + {"sun4u", EM_SPARC}, + {"x86", EM_X86_64}, /* Check also for EM_386. */ + {"xtensa", EM_XTENSA}, +}; + +static int compare_prefix(const void *key, const void *element) +{ + const char *search_key = key; + const struct arch_to_e_machine *map_element = element; + size_t prefix_len = strlen(map_element->prefix); - return arch; + return strncmp(search_key, map_element->prefix, prefix_len); } -const char *perf_env__arch(struct perf_env *env) +static uint16_t perf_arch_to_e_machine(const char *perf_arch, int is_64_bit) +{ + /* Binary search for a matching prefix. */ + const struct arch_to_e_machine *result; + + if (!perf_arch) + return EM_HOST; + + result = bsearch(perf_arch, + prefix_to_e_machine, ARRAY_SIZE(prefix_to_e_machine), + sizeof(prefix_to_e_machine[0]), + compare_prefix); + + if (!result) { + pr_debug("Unknown perf arch for ELF machine mapping: %s\n", perf_arch); + return EM_NONE; + } + + /* + * Handle conflicting prefixes. If the is_64_bit is unknown (-1) then + * assume 64-bit. We can't use perf_env__kernel_is_64_bit as that + * depends on the arch string. + */ + switch (result->e_machine) { + case EM_ARM: + return !strcmp(perf_arch, "arm64") || !strcmp(perf_arch, "aarch64") + ? EM_AARCH64 : EM_ARM; + case EM_AVR: + return !strcmp(perf_arch, "avr32") ? EM_AVR32 : EM_AVR; + case EM_PPC: + if (is_64_bit == 1) + return EM_PPC64; + if (is_64_bit == 0) + return EM_PPC; + return strstarts(perf_arch, "ppc64") ? EM_PPC64 : EM_PPC; + case EM_SPARC: + if (is_64_bit == 1) + return EM_SPARCV9; + if (is_64_bit == 0) + return EM_SPARC; + return !strcmp(perf_arch, "sparc64") || !strcmp(perf_arch, "sun4u") + ? EM_SPARCV9 : EM_SPARC; + case EM_X86_64: + if (is_64_bit == 1) + return EM_X86_64; + if (is_64_bit == 0) + return EM_386; + return !strcmp(perf_arch, "x86_64") || !strcmp(perf_arch, "x86") + ? EM_X86_64 : EM_386; + default: + return result->e_machine; + } +} + +static const char *e_machine_to_perf_arch(uint16_t e_machine) +{ + /* + * Table for if either the perf arch string differs from uname or there + * are >1 ELF machine with the prefix. + */ + static const struct arch_to_e_machine extras[] = { + {"arm64", EM_AARCH64}, + {"avr32", EM_AVR32}, + {"powerpc", EM_PPC}, + {"powerpc", EM_PPC64}, + {"sparc", EM_SPARCV9}, + {"x86", EM_386}, + {"x86", EM_X86_64}, + {"none", EM_NONE}, + }; + + for (size_t i = 0; i < ARRAY_SIZE(extras); i++) { + if (extras[i].e_machine == e_machine) + return extras[i].prefix; + } + + for (size_t i = 0; i < ARRAY_SIZE(prefix_to_e_machine); i++) { + if (prefix_to_e_machine[i].e_machine == e_machine) + return prefix_to_e_machine[i].prefix; + + } + return "unknown"; +} + +uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags) { - char *arch_name; + uint16_t e_machine = EM_NONE; + const char *arch = NULL; + int is_64_bit = -1; + + if (e_flags) + *e_flags = 0; + + if (env) { + arch = env->arch; + is_64_bit = env->kernel_is_64_bit; + } - if (!env || !env->arch) { /* Assume local operation */ + if (!arch) { static struct utsname uts = { .machine[0] = '\0', }; - if (uts.machine[0] == '\0' && uname(&uts) < 0) - return NULL; - arch_name = uts.machine; - } else - arch_name = env->arch; - return normalize_arch(arch_name); + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') + arch = uts.machine; + } + + e_machine = perf_arch_to_e_machine(arch, is_64_bit); + + if (e_flags) + *e_flags = (e_machine == EM_HOST) ? EF_HOST : 0; + + return e_machine; } -#if defined(HAVE_LIBTRACEEVENT) -#include "trace/beauty/arch_errno_names.c" -#endif +uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags) +{ + uint16_t e_machine; + uint32_t local_e_flags = 0; + + if (env && env->e_machine != EM_NONE) { + if (e_flags) + *e_flags = env->e_flags; + + return env->e_machine; + } + e_machine = perf_env__e_machine_nocache(env, &local_e_flags); + /* + * Only cache the e_machine in perf_env if env->arch is not NULL. + * If env->arch is NULL, the e_machine is just a fallback to EM_HOST. + * Caching it permanently would prevent dynamic, more accurate + * thread-based session e_machine scanning later in + * perf_session__e_machine(). + */ + if (env && env->arch) { + env->e_machine = e_machine; + env->e_flags = local_e_flags; + } + if (e_flags) + *e_flags = local_e_flags; -const char *perf_env__arch_strerrno(struct perf_env *env __maybe_unused, int err __maybe_unused) + return e_machine; +} + +const char *perf_env__arch(struct perf_env *env) { -#if defined(HAVE_LIBTRACEEVENT) - if (env->arch_strerrno == NULL) - env->arch_strerrno = arch_syscalls__strerrno_function(perf_env__arch(env)); + uint16_t e_machine; + const char *arch; - return env->arch_strerrno ? env->arch_strerrno(err) : "no arch specific strerrno function"; -#else - return "!HAVE_LIBTRACEEVENT"; -#endif + if (!env) { + static struct utsname uts = { .machine[0] = '\0', }; + uint16_t host_e_machine; + + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') { + host_e_machine = perf_arch_to_e_machine(uts.machine, -1); + return e_machine_to_perf_arch(host_e_machine); + } + return e_machine_to_perf_arch(EM_HOST); + } + + /* + * Lazily compute/allocate arch. The e_machine may have been + * read from a data file and so may not be EM_HOST. + */ + e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); + arch = e_machine_to_perf_arch(e_machine); + + if (e_machine == EM_RISCV && perf_env__kernel_is_64_bit(env) == 1) + arch = "riscv64"; + else if (e_machine == EM_MIPS && perf_env__kernel_is_64_bit(env) == 1) + arch = "mips64"; + else if (e_machine == EM_PARISC && perf_env__kernel_is_64_bit(env) == 1) + arch = "parisc64"; + + return arch; +} + +const char *arch_syscalls__strerrno(uint16_t e_machine, int err); + +const char *perf_env__arch_strerrno(uint16_t e_machine, int err) +{ + return arch_syscalls__strerrno(e_machine, err); } const char *perf_env__cpuid(struct perf_env *env) @@ -825,6 +1055,7 @@ bool x86__is_amd_cpu(void) struct perf_env env = { .total_mem = 0, }; bool is_amd; + perf_env__init(&env); perf_env__cpuid(&env); is_amd = perf_env__is_x86_amd_cpu(&env); perf_env__exit(&env); @@ -847,6 +1078,7 @@ bool x86__is_intel_cpu(void) struct perf_env env = { .total_mem = 0, }; bool is_intel; + perf_env__init(&env); perf_env__cpuid(&env); is_intel = perf_env__is_x86_intel_cpu(&env); perf_env__exit(&env); diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h index c7052ac1f856..7acca39b42ff 100644 --- a/tools/perf/util/env.h +++ b/tools/perf/util/env.h @@ -6,6 +6,7 @@ #include <linux/rbtree.h> #include "cpumap.h" #include "rwsem.h" +#include "mutex.h" struct perf_cpu_map; @@ -67,8 +68,6 @@ struct cpu_domain_map { struct domain_info **domains; }; -typedef const char *(arch_syscalls__strerrno_t)(int err); - struct perf_env { char *hostname; char *os_release; @@ -158,7 +157,8 @@ struct perf_env { */ bool enabled; } clock; - arch_syscalls__strerrno_t *arch_strerrno; + /* Protects lazy environment initialization (e.g. os_release, e_machine). */ + struct mutex lock; }; enum perf_compress_type { @@ -175,6 +175,8 @@ void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, void perf_env__exit(struct perf_env *env); int perf_env__kernel_is_64_bit(struct perf_env *env); +bool perf_arch_is_big_endian(const char *arch); +const char *perf_env__os_release(struct perf_env *env); int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]); @@ -185,12 +187,27 @@ const char *perf_env__pmu_mappings(struct perf_env *env); int perf_env__read_cpu_topology_map(struct perf_env *env); +/* + * Safe accessor for env->cpu[] topology array. env->cpu can be NULL when + * reading old-format perf.data that predates topology information — + * process_cpu_topology() in header.c frees it while nr_cpus_avail remains + * set, so callers must not index env->cpu[] without this check. + */ +static inline struct cpu_topology_map * +perf_env__get_cpu_topology(struct perf_env *env, struct perf_cpu cpu) +{ + if (env->cpu && cpu.cpu >= 0 && cpu.cpu < env->nr_cpus_avail) + return &env->cpu[cpu.cpu]; + return NULL; +} + void cpu_cache_level__free(struct cpu_cache_level *cache); +uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags); +uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags); const char *perf_env__arch(struct perf_env *env); -const char *perf_env__arch_strerrno(struct perf_env *env, int err); +const char *perf_env__arch_strerrno(uint16_t e_machine, int err); const char *perf_env__cpuid(struct perf_env *env); -const char *perf_env__raw_arch(struct perf_env *env); int perf_env__nr_cpus_avail(struct perf_env *env); void perf_env__init(struct perf_env *env); diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c index 66f4843bb235..ea75816d126a 100644 --- a/tools/perf/util/event.c +++ b/tools/perf/util/event.c @@ -14,6 +14,7 @@ #include <linux/perf_event.h> #include "cpumap.h" #include "dso.h" +#include "env.h" #include "event.h" #include "debug.h" #include "hist.h" @@ -836,8 +837,18 @@ int machine__resolve(struct machine *machine, struct addr_location *al, if (al->cpu >= 0) { struct perf_env *env = machine->env; - if (env && env->cpu) - al->socket = env->cpu[al->cpu].socket_id; + /* + * Bounds-check al->cpu (s32) before casting to struct perf_cpu + * (int16_t): without this, e.g. 65536 truncates to 0 and silently + * returns CPU 0's topology. Can go once perf_cpu.cpu is widened. + */ + if (env && al->cpu < env->nr_cpus_avail) { + struct cpu_topology_map *topo; + + topo = perf_env__get_cpu_topology(env, (struct perf_cpu){ al->cpu }); + if (topo) + al->socket = topo->socket_id; + } } /* Account for possible out-of-order switch events. */ diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index ee971d15b3c6..1a238b245b3a 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -529,7 +529,7 @@ static int evlist__is_enabled(struct evlist *evlist) static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl_dummy) { - struct evsel *pos; + struct evsel *pos, *member; struct evlist_cpu_iterator evlist_cpu_itr; bool has_imm = false; @@ -561,6 +561,9 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl if (excl_dummy && evsel__is_dummy_event(pos)) continue; pos->disabled = true; + + for_each_group_member(member, pos) + member->disabled = true; } /* @@ -590,7 +593,7 @@ void evlist__disable_evsel(struct evlist *evlist, char *evsel_name) static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_dummy) { - struct evsel *pos; + struct evsel *pos, *member; struct evlist_cpu_iterator evlist_cpu_itr; evlist__for_each_cpu(evlist_cpu_itr, evlist) { @@ -611,6 +614,9 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_ if (excl_dummy && evsel__is_dummy_event(pos)) continue; pos->disabled = false; + + for_each_group_member(member, pos) + member->disabled = false; } /* diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 2ee87fd84d3e..ea9fa04429f0 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -11,68 +11,71 @@ */ #define __SANE_USERSPACE_TYPES__ -#include <byteswap.h> +#include "evsel.h" + #include <errno.h> #include <inttypes.h> +#include <stdlib.h> + +#include <dirent.h> #include <linux/bitops.h> -#include <api/fs/fs.h> -#include <api/fs/tracing_path.h> -#include <linux/hw_breakpoint.h> -#include <linux/perf_event.h> #include <linux/compiler.h> +#include <linux/ctype.h> #include <linux/err.h> +#include <linux/hw_breakpoint.h> +#include <linux/perf_event.h> #include <linux/zalloc.h> #include <sys/ioctl.h> #include <sys/resource.h> #include <sys/syscall.h> #include <sys/types.h> -#include <dirent.h> -#include <stdlib.h> + +#include <api/fs/fs.h> +#include <api/fs/tracing_path.h> +#include <byteswap.h> +#include <internal/lib.h> +#include <internal/threadmap.h> +#include <internal/xyarray.h> +#include <perf/cpumap.h> #include <perf/evsel.h> + +#include "../perf-sys.h" #include "asm/bug.h" +#include "bpf-filter.h" #include "bpf_counter.h" #include "callchain.h" #include "cgroup.h" #include "counts.h" +#include "debug.h" +#include "drm_pmu.h" #include "dwarf-regs.h" +#include "env.h" #include "event.h" -#include "evsel.h" -#include "time-utils.h" -#include "util/env.h" -#include "util/evsel_config.h" -#include "util/evsel_fprintf.h" #include "evlist.h" -#include <perf/cpumap.h> -#include "thread_map.h" -#include "target.h" +#include "evsel_config.h" +#include "evsel_fprintf.h" +#include "hashmap.h" +#include "hist.h" +#include "hwmon_pmu.h" +#include "intel-tpebs.h" +#include "memswap.h" +#include "off_cpu.h" +#include "parse-branch-options.h" #include "perf_regs.h" +#include "pmu.h" +#include "pmus.h" #include "record.h" -#include "debug.h" -#include "trace-event.h" +#include "rlimit.h" #include "session.h" #include "stat.h" #include "string2.h" -#include "memswap.h" -#include "util.h" -#include "util/hashmap.h" -#include "off_cpu.h" -#include "pmu.h" -#include "pmus.h" -#include "drm_pmu.h" -#include "hwmon_pmu.h" +#include "target.h" +#include "thread_map.h" +#include "time-utils.h" #include "tool_pmu.h" #include "tp_pmu.h" -#include "rlimit.h" -#include "../perf-sys.h" -#include "util/parse-branch-options.h" -#include "util/bpf-filter.h" -#include "util/hist.h" -#include <internal/xyarray.h> -#include <internal/lib.h> -#include <internal/threadmap.h> -#include "util/intel-tpebs.h" - -#include <linux/ctype.h> +#include "trace-event.h" +#include "util.h" #ifdef HAVE_LIBTRACEEVENT #include <event-parse.h> @@ -251,6 +254,58 @@ const char *evsel__pmu_name(const struct evsel *evsel) return event_type(evsel->core.attr.type); } +enum evsel_probe_type { + PROBE__UNKNOWN = 0, + PROBE__NOPE = 1, + PROBE__KPROBE = 2, + PROBE__UPROBE = 3, + /* + * Ftrace-based dynamic probes (kprobes/uprobes/fprobes created via + * tracefs) report PMU "tracepoint", not "kprobe"/"uprobe". Detect + * them by the __probe_ip field that the kernel adds to all dynamic + * probe formats. + */ + PROBE__FTRACE = 4, +}; + +static void evsel__resolve_probe_type(struct evsel *evsel) +{ + const char *name = evsel__pmu_name(evsel); + + if (!strcmp(name, "kprobe")) + evsel->probe_type = PROBE__KPROBE; + else if (!strcmp(name, "uprobe")) + evsel->probe_type = PROBE__UPROBE; + else if (!strcmp(name, "tracepoint") && evsel__field(evsel, "__probe_ip")) + evsel->probe_type = PROBE__FTRACE; + else + evsel->probe_type = PROBE__NOPE; +} + +bool evsel__is_probe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type > PROBE__NOPE; +} + +bool evsel__is_kprobe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type == PROBE__KPROBE; +} + +bool evsel__is_uprobe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type == PROBE__UPROBE; +} + #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) int __evsel__sample_size(u64 sample_type) @@ -410,6 +465,7 @@ void evsel__init(struct evsel *evsel, evsel->supported = true; evsel->alternate_hw_config = PERF_COUNT_HW_MAX; evsel->script_output_type = -1; // FIXME: OUTPUT_TYPE_UNSET, see builtin-script.c + evsel->probe_type = PROBE__UNKNOWN; } struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx) @@ -1396,6 +1452,12 @@ void evsel__set_config_if_unset(struct evsel *evsel, const char *config_name, perf_pmu__format_pack(format->bits, val, vp, /*zero=*/true); } +bool evsel__config_exists(const struct evsel *evsel, const char *config_name) +{ + struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, config_name); + + return format && !bitmap_empty(format->bits, PERF_PMU_FORMAT_BITS); +} int evsel__get_config_val(const struct evsel *evsel, const char *config_name, u64 *val) @@ -1795,27 +1857,114 @@ int evsel__append_addr_filter(struct evsel *evsel, const char *filter) /* Caller has to clear disabled after going through all CPUs. */ int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx) { - return perf_evsel__enable_cpu(&evsel->core, cpu_map_idx); + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_enable_cpu(evsel, cpu_map_idx); + else + err = perf_evsel__enable_cpu(&evsel->core, cpu_map_idx); + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel (opened with group_fd = -1) + * and are skipped by the kernel when enabling the + * group leader. We must manually enable them in + * userspace. + */ + int mem_err = evsel__enable_cpu(member, cpu_map_idx); + + if (mem_err) + return mem_err; + } + } + } + return err; } int evsel__enable(struct evsel *evsel) { - int err = perf_evsel__enable(&evsel->core); + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_enable(evsel); + else + err = perf_evsel__enable(&evsel->core); if (!err) evsel->disabled = false; + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel (opened with group_fd = -1) + * and are skipped by the kernel when enabling the + * group leader. We must manually enable them in + * userspace. + */ + int mem_err = evsel__enable(member); + + if (mem_err) + return mem_err; + } + member->disabled = false; + } + } + return err; } /* Caller has to set disabled after going through all CPUs. */ int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx) { - return perf_evsel__disable_cpu(&evsel->core, cpu_map_idx); + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_disable_cpu(evsel, cpu_map_idx); + else + err = perf_evsel__disable_cpu(&evsel->core, cpu_map_idx); + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel and are skipped by the + * kernel when disabling the group leader. We must + * manually disable them in userspace. + */ + int mem_err = evsel__disable_cpu(member, cpu_map_idx); + + if (mem_err) + return mem_err; + } + } + } + return err; } int evsel__disable(struct evsel *evsel) { - int err = perf_evsel__disable(&evsel->core); + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_disable(evsel); + else + err = perf_evsel__disable(&evsel->core); + /* * We mark it disabled here so that tools that disable a event can * ignore events after they disable it. I.e. the ring buffer may have @@ -1825,6 +1974,27 @@ int evsel__disable(struct evsel *evsel) if (!err) evsel->disabled = true; + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel and are skipped by the + * kernel when disabling the group leader. We must + * manually disable them in userspace. + */ + int mem_err = evsel__disable(member); + + if (mem_err) + return mem_err; + } + member->disabled = true; + } + } + return err; } @@ -1885,8 +2055,10 @@ void evsel__exit(struct evsel *evsel) evsel__priv_destructor(evsel->priv); perf_evsel__object.fini(evsel); if (evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME || - evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) - xyarray__delete(evsel->start_times); + evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) { + xyarray__delete(evsel->process_time.start_times); + xyarray__delete(evsel->process_time.accumulated_times); + } } void evsel__delete(struct evsel *evsel) @@ -3003,52 +3175,62 @@ int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads) return ret; } -static int perf_evsel__parse_id_sample(const struct evsel *evsel, - const union perf_event *event, +static int perf_evsel__parse_id_sample(const union perf_event *event, struct perf_sample *sample) { + const struct evsel *evsel = sample->evsel; u64 type = evsel->core.attr.sample_type; const __u64 *array = event->sample.array; bool swapped = evsel->needs_swap; union u64_swap u; - - array += ((event->header.size - - sizeof(event->header)) / sizeof(u64)) - 1; + int i = ((event->header.size - sizeof(event->header)) / sizeof(u64)) - 1; if (type & PERF_SAMPLE_IDENTIFIER) { - sample->id = *array; - array--; + if (i < 0) + return -EFAULT; + + sample->id = array[i--]; } if (type & PERF_SAMPLE_CPU) { - u.val64 = *array; + if (i < 0) + return -EFAULT; + + u.val64 = array[i--]; if (swapped) { /* undo swap of u64, then swap on individual u32s */ u.val64 = bswap_64(u.val64); u.val32[0] = bswap_32(u.val32[0]); } - sample->cpu = u.val32[0]; - array--; } if (type & PERF_SAMPLE_STREAM_ID) { - sample->stream_id = *array; - array--; + if (i < 0) + return -EFAULT; + + sample->stream_id = array[i--]; } if (type & PERF_SAMPLE_ID) { - sample->id = *array; - array--; + if (i < 0) + return -EFAULT; + + sample->id = array[i--]; } if (type & PERF_SAMPLE_TIME) { - sample->time = *array; - array--; + if (i < 0) + return -EFAULT; + + sample->time = array[i--]; } if (type & PERF_SAMPLE_TID) { - u.val64 = *array; + if (i < 0) + return -EFAULT; + + u.val64 = array[i--]; if (swapped) { /* undo swap of u64, then swap on individual u32s */ u.val64 = bswap_64(u.val64); @@ -3058,7 +3240,6 @@ static int perf_evsel__parse_id_sample(const struct evsel *evsel, sample->pid = u.val32[0]; sample->tid = u.val32[1]; - array--; } return 0; @@ -3209,11 +3390,11 @@ out_efault: return -EFAULT; } -int evsel__parse_sample(struct evsel *evsel, union perf_event *event, - struct perf_sample *data) +int __evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data, bool needs_swap) { u64 type = evsel->core.attr.sample_type; - bool swapped = evsel->needs_swap; + bool swapped = needs_swap; const __u64 *array; u16 max_size = event->header.size; const void *endp = (void *)event + max_size; @@ -3244,15 +3425,18 @@ int evsel__parse_sample(struct evsel *evsel, union perf_event *event, data->deferred_cookie = event->callchain_deferred.cookie; - if (evsel->core.attr.sample_id_all) - perf_evsel__parse_id_sample(evsel, event, data); - + if (evsel->core.attr.sample_id_all) { + if (perf_evsel__parse_id_sample(event, data)) + goto out_efault; + } return 0; } if (event->header.type != PERF_RECORD_SAMPLE) { - if (evsel->core.attr.sample_id_all) - perf_evsel__parse_id_sample(evsel, event, data); + if (evsel->core.attr.sample_id_all) { + if (perf_evsel__parse_id_sample(event, data)) + goto out_efault; + } return 0; } @@ -3614,12 +3798,13 @@ int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event, if (event->header.type != PERF_RECORD_SAMPLE) { struct perf_sample data = { + .evsel = evsel, .time = -1ULL, }; if (!evsel->core.attr.sample_id_all) return -1; - if (perf_evsel__parse_id_sample(evsel, event, &data)) + if (perf_evsel__parse_id_sample(event, &data)) return -1; *timestamp = data.time; @@ -3687,22 +3872,63 @@ struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *na return tp_format ? tep_find_common_field(tp_format, name) : NULL; } -void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name) +static bool out_of_bounds(const struct tep_format_field *field, int offset, int size, u32 raw_size) { - struct tep_format_field *field = evsel__field(evsel, name); - int offset; + if (offset < 0) { + pr_warning("Negative trace point field offset %d in %s\n", + offset, field->name); + return true; + } + if (size < 0) { + pr_warning("Negative trace point field size %d in %s\n", + size, field->name); + return true; + } + if ((u32)offset + (u32)size > raw_size) { + pr_warning("Out of bound tracepoint field (%s) offset %d size %d in %u\n", + field->name, offset, size, raw_size); + return true; + } + return false; +} + +void *perf_sample__rawptr(struct perf_sample *sample, const char *name) +{ + struct tep_format_field *field = evsel__field(sample->evsel, name); + int offset, size; if (!field) return NULL; offset = field->offset; - + size = field->size; if (field->flags & TEP_FIELD_IS_DYNAMIC) { - offset = *(int *)(sample->raw_data + field->offset); - offset &= 0xffff; - if (tep_field_is_relative(field->flags)) + int dynamic_data; + + if (out_of_bounds(field, offset, 4, sample->raw_size)) + return NULL; + + dynamic_data = *(int *)(sample->raw_data + field->offset); + + if (sample->evsel->needs_swap) + dynamic_data = bswap_32(dynamic_data); + + offset = dynamic_data & 0xffff; + size = (dynamic_data >> 16) & 0xffff; + + if (tep_field_is_relative(field->flags)) { + /* + * Newer kernel feature: Relative offsets (__rel_loc). + * If the relative flag is set, the parsed offset is not + * absolute from the start of the record. Instead, it is + * relative to the *end* of the dynamic field descriptor + * itself. + */ offset += field->offset + field->size; + } } + if (out_of_bounds(field, offset, size, sample->raw_size)) + return NULL; return sample->raw_data + offset; } @@ -3713,6 +3939,9 @@ u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sam u64 value; void *ptr = sample->raw_data + field->offset; + if (out_of_bounds(field, field->offset, field->size, sample->raw_size)) + return 0; + switch (field->size) { case 1: return *(u8 *)ptr; @@ -3746,21 +3975,21 @@ u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sam return 0; } -u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name) +u64 perf_sample__intval(struct perf_sample *sample, const char *name) { - struct tep_format_field *field = evsel__field(evsel, name); + struct tep_format_field *field = evsel__field(sample->evsel, name); - return field ? format_field__intval(field, sample, evsel->needs_swap) : 0; + return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; } -u64 evsel__intval_common(struct evsel *evsel, struct perf_sample *sample, const char *name) +u64 perf_sample__intval_common(struct perf_sample *sample, const char *name) { - struct tep_format_field *field = evsel__common_field(evsel, name); + struct tep_format_field *field = evsel__common_field(sample->evsel, name); - return field ? format_field__intval(field, sample, evsel->needs_swap) : 0; + return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; } -char evsel__taskstate(struct evsel *evsel, struct perf_sample *sample, const char *name) +char perf_sample__taskstate(struct perf_sample *sample, const char *name) { static struct tep_format_field *prev_state_field; static const char *states; @@ -3769,7 +3998,7 @@ char evsel__taskstate(struct evsel *evsel, struct perf_sample *sample, const cha unsigned int bit; char state = '?'; /* '?' denotes unknown task state */ - field = evsel__field(evsel, name); + field = evsel__field(sample->evsel, name); if (!field) return state; @@ -3788,7 +4017,7 @@ char evsel__taskstate(struct evsel *evsel, struct perf_sample *sample, const cha * * We can change this if we have a good reason in the future. */ - val = evsel__intval(evsel, sample, name); + val = perf_sample__intval(sample, name); bit = val ? ffs(val) : 0; state = (!bit || bit > strlen(states)) ? 'R' : states[bit-1]; return state; diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 339b5c08a33d..163fc2b6a7ea 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -126,6 +126,7 @@ struct evsel { bool needs_uniquify; bool fallenback_eacces; bool fallenback_eopnotsupp; + u8 probe_type:3; struct hashmap *per_pkg_mask; int err; int script_output_type; @@ -190,12 +191,18 @@ struct evsel { double max; } retirement_latency; /* duration_time is a single global time. */ - __u64 start_time; + struct { + __u64 start_time; + __u64 accumulated_time; + } duration_time; /* * user_time and system_time read an initial value potentially * per-CPU or per-pid. */ - struct xyarray *start_times; + struct { + struct xyarray *start_times; + struct xyarray *accumulated_times; + } process_time; }; /* Is the tool's fd for /proc/pid/stat or /proc/stat. */ bool pid_stat; @@ -253,6 +260,10 @@ struct perf_pmu *evsel__find_pmu(const struct evsel *evsel); const char *evsel__pmu_name(const struct evsel *evsel); bool evsel__is_aux_event(const struct evsel *evsel); +bool evsel__is_probe(struct evsel *evsel); +bool evsel__is_kprobe(struct evsel *evsel); +bool evsel__is_uprobe(struct evsel *evsel); + struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx); static inline struct evsel *evsel__new(struct perf_event_attr *attr) @@ -350,6 +361,11 @@ void arch_evsel__apply_ratio_to_prev(struct evsel *evsel, struct perf_event_attr int evsel__set_filter(struct evsel *evsel, const char *filter); int evsel__append_tp_filter(struct evsel *evsel, const char *filter); int evsel__append_addr_filter(struct evsel *evsel, const char *filter); +static inline bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel) +{ + return evsel->pmu && evsel->pmu->type > PERF_PMU_TYPE_PE_END; +} + int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx); int evsel__enable(struct evsel *evsel); int evsel__disable(struct evsel *evsel); @@ -371,14 +387,14 @@ bool evsel__precise_ip_fallback(struct evsel *evsel); struct perf_sample; #ifdef HAVE_LIBTRACEEVENT -void *evsel__rawptr(struct evsel *evsel, struct perf_sample *sample, const char *name); -u64 evsel__intval(struct evsel *evsel, struct perf_sample *sample, const char *name); -u64 evsel__intval_common(struct evsel *evsel, struct perf_sample *sample, const char *name); -char evsel__taskstate(struct evsel *evsel, struct perf_sample *sample, const char *name); +void *perf_sample__rawptr(struct perf_sample *sample, const char *name); +u64 perf_sample__intval(struct perf_sample *sample, const char *name); +u64 perf_sample__intval_common(struct perf_sample *sample, const char *name); +char perf_sample__taskstate(struct perf_sample *sample, const char *name); -static inline char *evsel__strval(struct evsel *evsel, struct perf_sample *sample, const char *name) +static inline char *perf_sample__strval(struct perf_sample *sample, const char *name) { - return evsel__rawptr(evsel, sample, name); + return perf_sample__rawptr(sample, name); } #endif @@ -386,8 +402,22 @@ struct tep_format_field; u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, bool needs_swap); +#ifdef HAVE_LIBTRACEEVENT struct tep_format_field *evsel__field(struct evsel *evsel, const char *name); struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name); +#else +static inline struct tep_format_field * +evsel__field(struct evsel *evsel __maybe_unused, const char *name __maybe_unused) +{ + return NULL; +} + +static inline struct tep_format_field * +evsel__common_field(struct evsel *evsel __maybe_unused, const char *name __maybe_unused) +{ + return NULL; +} +#endif bool __evsel__match(const struct evsel *evsel, u32 type, u64 config); @@ -421,8 +451,14 @@ static inline int evsel__read_on_cpu_scaled(struct evsel *evsel, int cpu_map_idx return __evsel__read_on_cpu(evsel, cpu_map_idx, thread, true); } -int evsel__parse_sample(struct evsel *evsel, union perf_event *event, - struct perf_sample *sample); +int __evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data, bool needs_swap); + +static inline int evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data) +{ + return __evsel__parse_sample(evsel, event, data, evsel->needs_swap); +} int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event, u64 *timestamp); @@ -578,6 +614,7 @@ void evsel__uniquify_counter(struct evsel *counter); ((((src) >> (pos)) & ((1ull << (size)) - 1)) << (63 - ((pos) + (size) - 1))) u64 evsel__bitfield_swap_branch_flags(u64 value); +bool evsel__config_exists(const struct evsel *evsel, const char *config_name); int evsel__get_config_val(const struct evsel *evsel, const char *config_name, u64 *val); void evsel__set_config_if_unset(struct evsel *evsel, const char *config_name, diff --git a/tools/perf/util/evsel_fprintf.c b/tools/perf/util/evsel_fprintf.c index 5521d00bff2c..0f7a25500a44 100644 --- a/tools/perf/util/evsel_fprintf.c +++ b/tools/perf/util/evsel_fprintf.c @@ -146,7 +146,7 @@ int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, sym = node->ms.sym; map = node->ms.map; - if (sym && sym->ignore && print_skip_ignored) + if (sym && symbol__ignore(sym) && print_skip_ignored) goto next; printed += fprintf(fp, "%-*.*s", left_alignment, left_alignment, " "); @@ -182,7 +182,7 @@ int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, addr_location__exit(&node_al); } - if (print_dso && (!sym || !sym->inlined)) + if (print_dso && (!sym || !symbol__inlined(sym))) printed += map__fprintf_dsoname_dsoff(map, print_dsoff, addr, fp); if (print_srcline) { @@ -192,7 +192,7 @@ int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, printed += map__fprintf_srcline(map, addr, "\n ", fp); } - if (sym && sym->inlined) + if (sym && symbol__inlined(sym)) printed += fprintf(fp, " (inlined)"); if (!print_oneline) diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c index 644769e92708..232998fef72b 100644 --- a/tools/perf/util/expr.c +++ b/tools/perf/util/expr.c @@ -27,6 +27,7 @@ struct expr_id_data { struct { double val; int source_count; + int aggr_nr; } val; struct { double val; @@ -151,8 +152,8 @@ int expr__add_id_val(struct expr_parse_ctx *ctx, const char *id, double val) } /* Caller must make sure id is allocated */ -int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, - double val, int source_count) +int expr__add_id_val_source_count_aggr_nr(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count, int aggr_nr) { struct expr_id_data *data_ptr = NULL, *old_data = NULL; char *old_key = NULL; @@ -163,6 +164,7 @@ int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, return -ENOMEM; data_ptr->val.val = val; data_ptr->val.source_count = source_count; + data_ptr->val.aggr_nr = aggr_nr; data_ptr->kind = EXPR_ID_DATA__VALUE; ret = hashmap__set(ctx->ids, id, data_ptr, &old_key, &old_data); @@ -171,12 +173,20 @@ int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, } else if (old_data) { data_ptr->val.val += old_data->val.val; data_ptr->val.source_count += old_data->val.source_count; + data_ptr->val.aggr_nr += old_data->val.aggr_nr; } free(old_key); free(old_data); return ret; } +/* Caller must make sure id is allocated */ +int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count) +{ + return expr__add_id_val_source_count_aggr_nr(ctx, id, val, source_count, 1); +} + int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref) { struct expr_id_data *data_ptr = NULL, *old_data = NULL; @@ -390,8 +400,16 @@ double expr_id_data__value(const struct expr_id_data *data) double expr_id_data__source_count(const struct expr_id_data *data) { - assert(data->kind == EXPR_ID_DATA__VALUE); - return data->val.source_count; + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.source_count; + return 1.0; +} + +double expr_id_data__aggr_nr(const struct expr_id_data *data) +{ + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.aggr_nr; + return 1.0; } double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx) diff --git a/tools/perf/util/expr.h b/tools/perf/util/expr.h index c0cec29ddc29..ed12e4007d2d 100644 --- a/tools/perf/util/expr.h +++ b/tools/perf/util/expr.h @@ -36,7 +36,9 @@ void expr__del_id(struct expr_parse_ctx *ctx, const char *id); int expr__add_id(struct expr_parse_ctx *ctx, const char *id); int expr__add_id_val(struct expr_parse_ctx *ctx, const char *id, double val); int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, - double val, int source_count); + double val, int source_count); +int expr__add_id_val_source_count_aggr_nr(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count, int aggr_nr); int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref); int expr__get_id(struct expr_parse_ctx *ctx, const char *id, struct expr_id_data **data); @@ -53,6 +55,8 @@ int expr__find_ids(const char *expr, const char *one, double expr_id_data__value(const struct expr_id_data *data); double expr_id_data__source_count(const struct expr_id_data *data); +double expr_id_data__aggr_nr(const struct expr_id_data *data); + double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx); double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id); double expr__strcmp_cpuid_str(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id); diff --git a/tools/perf/util/expr.l b/tools/perf/util/expr.l index a2fc43159ee9..f16ccff278d0 100644 --- a/tools/perf/util/expr.l +++ b/tools/perf/util/expr.l @@ -121,6 +121,7 @@ min { return MIN; } if { return IF; } else { return ELSE; } source_count { return SOURCE_COUNT; } +aggr_nr { return AGGR_NR; } has_event { return HAS_EVENT; } strcmp_cpuid_str { return STRCMP_CPUID_STR; } NaN { return nan_value(yyscanner); } diff --git a/tools/perf/util/expr.y b/tools/perf/util/expr.y index e364790babb5..e20f649354cf 100644 --- a/tools/perf/util/expr.y +++ b/tools/perf/util/expr.y @@ -41,7 +41,7 @@ int expr_lex(YYSTYPE * yylval_param , void *yyscanner); } ids; } -%token ID NUMBER MIN MAX IF ELSE LITERAL D_RATIO SOURCE_COUNT HAS_EVENT STRCMP_CPUID_STR EXPR_ERROR +%token ID NUMBER MIN MAX IF ELSE LITERAL D_RATIO SOURCE_COUNT AGGR_NR HAS_EVENT STRCMP_CPUID_STR EXPR_ERROR %left MIN MAX IF %left '|' %left '^' @@ -87,8 +87,14 @@ static struct ids union_expr(struct ids ids1, struct ids ids2) return result; } +enum expr_id_kind { + EXPR_ID_KIND__VALUE, + EXPR_ID_KIND__SOURCE_COUNT, + EXPR_ID_KIND__AGGR_NR, +}; + static struct ids handle_id(struct expr_parse_ctx *ctx, char *id, - bool compute_ids, bool source_count) + bool compute_ids, enum expr_id_kind kind) { struct ids result; @@ -101,9 +107,12 @@ static struct ids handle_id(struct expr_parse_ctx *ctx, char *id, result.val = NAN; if (expr__resolve_id(ctx, id, &data) == 0) { - result.val = source_count - ? expr_id_data__source_count(data) - : expr_id_data__value(data); + if (kind == EXPR_ID_KIND__SOURCE_COUNT) + result.val = expr_id_data__source_count(data); + else if (kind == EXPR_ID_KIND__AGGR_NR) + result.val = expr_id_data__aggr_nr(data); + else + result.val = expr_id_data__value(data); } result.ids = NULL; free(id); @@ -201,8 +210,9 @@ expr: NUMBER $$.val = $1; $$.ids = NULL; } -| ID { $$ = handle_id(ctx, $1, compute_ids, /*source_count=*/false); } -| SOURCE_COUNT '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, /*source_count=*/true); } +| ID { $$ = handle_id(ctx, $1, compute_ids, EXPR_ID_KIND__VALUE); } +| SOURCE_COUNT '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, EXPR_ID_KIND__SOURCE_COUNT); } +| AGGR_NR '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, EXPR_ID_KIND__AGGR_NR); } | HAS_EVENT '(' ID ')' { $$.val = expr__has_event(ctx, compute_ids, $3); diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index f30e48eb3fc3..091d8f7f6bd2 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <errno.h> #include <inttypes.h> +#include <limits.h> #include "string2.h" #include <sys/param.h> #include <sys/types.h> @@ -47,6 +48,7 @@ #include <api/io_dir.h> #include "asm/bug.h" #include "tool.h" +#include "../perf.h" #include "time-utils.h" #include "units.h" #include "util/util.h" // perf_exe() @@ -63,6 +65,25 @@ #include <event-parse.h> #endif +/* + * nr_ids * sizeof(struct perf_sample_id) must not overflow + * size_t on 32-bit; the struct is ~104 bytes (32-bit) or + * ~184 bytes (64-bit), so 1<<24 (16M) keeps the product + * under 2 GB on 32-bit. + * + * This is a per-attribute cap only — the total across all + * attributes is not capped because legitimate high-core-count + * workloads (e.g. 5000 tracepoints × 4096 CPUs) can exceed + * a single-attribute limit. + */ +#define MAX_IDS_PER_ATTR (1 << 24) +/* + * Cap nr_attrs to prevent resource exhaustion from crafted + * files. 65536 is well beyond any real workload (perf stat + * typically uses < 100 events) but prevents u64-to-int + * truncation on the attr count. + */ +#define MAX_NR_ATTRS (1 << 16) #define MAX_BPF_DATA_LEN (256 * 1024 * 1024) #define MAX_BPF_PROGS 131072 #define MAX_CACHE_ENTRIES 32768 @@ -158,15 +179,25 @@ int do_write(struct feat_fd *ff, const void *buf, size_t size) /* Return: 0 if succeeded, -ERR if failed. */ static int do_write_bitmap(struct feat_fd *ff, unsigned long *set, u64 size) { - u64 *p = (u64 *) set; + size_t byte_size = BITS_TO_LONGS(size) * sizeof(unsigned long); int i, ret; ret = do_write(ff, &size, sizeof(size)); if (ret < 0) return ret; + /* + * The on-disk format uses u64 elements, but the in-memory bitmap + * uses unsigned long, which is only 4 bytes on 32-bit architectures. + * Copy with bounded size so the last element doesn't read past the + * bitmap allocation when BITS_TO_LONGS(size) is odd. + */ for (i = 0; (u64) i < BITS_TO_U64(size); i++) { - ret = do_write(ff, p + i, sizeof(*p)); + u64 val = 0; + size_t off = i * sizeof(val); + + memcpy(&val, (char *)set + off, min(sizeof(val), byte_size - off)); + ret = do_write(ff, &val, sizeof(val)); if (ret < 0) return ret; } @@ -213,23 +244,32 @@ static int __do_read_fd(struct feat_fd *ff, void *addr, ssize_t size) if (ret != size) return ret < 0 ? (int)ret : -1; + ff->offset += size; return 0; } static int __do_read_buf(struct feat_fd *ff, void *addr, ssize_t size) { - if (size > (ssize_t)ff->size - ff->offset) - return -1; - memcpy(addr, ff->buf + ff->offset, size); ff->offset += size; return 0; - } static int __do_read(struct feat_fd *ff, void *addr, ssize_t size) { + /* + * Reject negative sizes, which on 32-bit can occur when a + * u32 >= 0x80000000 is passed as ssize_t. The cast to + * ssize_t is safe because perf_header__process_sections() + * validates that each section fits within the file size + * before any feature callback reaches here, and only + * feature sections (metadata like build IDs, topology, etc.) + * use this path — these cannot legitimately approach 2GB. + */ + if (size < 0 || size > (ssize_t)ff->size - ff->offset) + return -1; + if (!ff->buf) return __do_read_fd(ff, addr, size); return __do_read_buf(ff, addr, size); @@ -269,16 +309,25 @@ static char *do_read_string(struct feat_fd *ff) if (do_read_u32(ff, &len)) return NULL; + /* At least the null terminator. */ + if (len < 1 || len > ff->size - ff->offset) { + pr_debug("do_read_string: invalid length %u (remaining %zu)\n", + len, (size_t)(ff->size - ff->offset)); + return NULL; + } + buf = malloc(len); if (!buf) return NULL; if (!__do_read(ff, buf, len)) { /* - * strings are padded by zeroes - * thus the actual strlen of buf - * may be less than len + * do_write_string() writes len including the null + * terminator, padded to NAME_ALIGN. Ensure the + * string is always null-terminated even if the file + * data has been tampered with. */ + buf[len - 1] = '\0'; return buf; } @@ -297,7 +346,20 @@ static int do_read_bitmap(struct feat_fd *ff, unsigned long **pset, u64 *psize) if (ret) return ret; - set = bitmap_zalloc(size); + /* Bitmap APIs use int for nbits; reject u64 values that truncate. */ + if (size > INT_MAX || + BITS_TO_U64(size) > (ff->size - ff->offset) / sizeof(u64)) { + pr_debug("do_read_bitmap: size %" PRIu64 " exceeds section bounds\n", size); + return -1; + } + + /* + * bitmap_zalloc() allocates in unsigned long units, which are only + * 4 bytes on 32-bit architectures. The read loop below casts the + * buffer to u64 * and writes 8-byte elements, so allocate in u64 + * units to ensure the buffer is large enough. + */ + set = calloc(BITS_TO_U64(size), sizeof(u64)); if (!set) return -ENOMEM; @@ -370,30 +432,40 @@ static int write_osrelease(struct feat_fd *ff, struct evlist *evlist __maybe_unused) { struct utsname uts; - int ret; + const char *release = NULL; - ret = uname(&uts); - if (ret < 0) - return -1; + if (evlist->session) + release = perf_env__os_release(perf_session__env(evlist->session)); + + if (!release) { + int ret = uname(&uts); - return do_write_string(ff, uts.release); + if (ret < 0) + return -1; + release = uts.release; + } + return do_write_string(ff, release); } -static int write_arch(struct feat_fd *ff, - struct evlist *evlist __maybe_unused) +static int write_arch(struct feat_fd *ff, struct evlist *evlist) { struct utsname uts; - int ret; + const char *arch = NULL; - ret = uname(&uts); - if (ret < 0) - return -1; + if (evlist->session) + arch = perf_env__arch(perf_session__env(evlist->session)); + + if (!arch) { + int ret = uname(&uts); - return do_write_string(ff, uts.machine); + if (ret < 0) + return -1; + arch = uts.machine; + } + return do_write_string(ff, arch); } -static int write_e_machine(struct feat_fd *ff, - struct evlist *evlist __maybe_unused) +static int write_e_machine(struct feat_fd *ff, struct evlist *evlist) { /* e_machine expanded from 16 to 32-bits for alignment. */ uint32_t e_flags; @@ -2035,8 +2107,7 @@ static void print_bpf_prog_info(struct feat_fd *ff __maybe_unused, FILE *fp) node = rb_entry(next, struct bpf_prog_info_node, rb_node); next = rb_next(&node->rb_node); - __bpf_event__print_bpf_prog_info(&node->info_linear->info, - env, fp); + __bpf_event__print_bpf_prog_info(node->info_linear, env, fp); } up_read(&env->bpf_progs.lock); @@ -2135,9 +2206,28 @@ static struct evsel *read_event_desc(struct feat_fd *ff) if (do_read_u32(ff, &nre)) goto error; + /* Size of each of the nre attributes. */ if (do_read_u32(ff, &sz)) goto error; + /* + * Require at least one event with an attr no smaller than the + * first published struct, and reject sz values where + * sz + sizeof(u32) would overflow size_t (possible on 32-bit) + * or nre == UINT32_MAX where nre + 1 wraps to 0 in the calloc. + * + * The minimum section footprint per event is sz bytes for the + * attr plus a u32 for the id count, check that nre events fit. + */ + if (!nre || sz < PERF_ATTR_SIZE_VER0 || + sz > ff->size || (size_t)sz > SIZE_MAX - sizeof(u32) || + nre == UINT32_MAX || + nre > (ff->size - ff->offset) / (sz + sizeof(u32))) { + pr_err("Invalid HEADER_EVENT_DESC: nre=%u sz=%u (min %d)\n", + nre, sz, PERF_ATTR_SIZE_VER0); + goto error; + } + /* buffer to hold on file attr struct */ buf = malloc(sz); if (!buf) @@ -2153,6 +2243,9 @@ static struct evsel *read_event_desc(struct feat_fd *ff) msz = sz; for (i = 0, evsel = events; i < nre; evsel++, i++) { + struct perf_event_attr *attr = buf; + u32 attr_size; + evsel->core.idx = i; /* @@ -2162,6 +2255,32 @@ static struct evsel *read_event_desc(struct feat_fd *ff) if (__do_read(ff, buf, sz)) goto error; + /* Reject before attr_swap to prevent OOB via bswap_safe() */ + attr_size = ff->ph->needs_swap ? bswap_32(attr->size) : attr->size; + /* ABI0: size == 0 means the producer didn't set it */ + if (!attr_size) { + attr_size = PERF_ATTR_SIZE_VER0; + /* + * Write back so free_event_desc() doesn't + * treat this event as the end-of-array sentinel + * (it iterates while attr.size != 0). + * + * Only for native — the swap path must NOT + * write native-endian VER0 here because + * perf_event__attr_swap() would re-swap it + * to 0x40000000, defeating bswap_safe() bounds. + * perf_event__attr_swap() has its own ABI0 + * fallback that sets VER0 after swapping. + */ + if (!ff->ph->needs_swap) + attr->size = attr_size; + } + if (attr_size < PERF_ATTR_SIZE_VER0 || attr_size > sz) { + pr_err("Event %d attr.size (%u) invalid (min: %d, max: %u)\n", + i, attr_size, PERF_ATTR_SIZE_VER0, sz); + goto error; + } + if (ff->ph->needs_swap) perf_event__attr_swap(buf); @@ -2183,6 +2302,12 @@ static struct evsel *read_event_desc(struct feat_fd *ff) if (!nr) continue; + /* Prevent oversized allocation from crafted nr */ + if (nr > (ff->size - ff->offset) / sizeof(*id)) { + pr_err("Event %d: id count %u exceeds remaining section\n", i, nr); + goto error; + } + id = calloc(nr, sizeof(*id)); if (!id) goto error; @@ -2341,15 +2466,16 @@ static void print_cpu_pmu_caps(struct feat_fd *ff, FILE *fp) static void print_pmu_caps(struct feat_fd *ff, FILE *fp) { struct perf_env *env = &ff->ph->env; - struct pmu_caps *pmu_caps; + uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); for (int i = 0; i < env->nr_pmus_with_caps; i++) { - pmu_caps = &env->pmu_caps[i]; + struct pmu_caps *pmu_caps = &env->pmu_caps[i]; + __print_pmu_caps(fp, pmu_caps->nr_caps, pmu_caps->caps, pmu_caps->pmu_name); } - if (strcmp(perf_env__arch(env), "x86") == 0 && + if ((e_machine == EM_X86_64 || e_machine == EM_386) && perf_env__has_pmu_mapping(env, "ibs_op")) { char *max_precise = perf_env__find_pmu_cap(env, "cpu", "max_precise"); @@ -2578,7 +2704,13 @@ static int perf_header__read_build_ids_abi_quirk(struct perf_header *header, } old_bev; struct perf_record_header_build_id bev; char filename[PATH_MAX]; - u64 limit = offset + size; + u64 limit; + + /* Prevent offset + size from wrapping past ULLONG_MAX */ + if (size > ULLONG_MAX - offset) + return -1; + + limit = offset + size; while (offset < limit) { ssize_t len; @@ -2589,6 +2721,10 @@ static int perf_header__read_build_ids_abi_quirk(struct perf_header *header, if (header->needs_swap) perf_event_header__bswap(&old_bev.header); + /* size == 0 loops forever; size > remaining reads past section */ + if (old_bev.header.size == 0 || old_bev.header.size > limit - offset) + return -1; + len = old_bev.header.size - sizeof(old_bev); if (len < 0 || len >= PATH_MAX) { pr_warning("invalid build_id filename length %zd\n", len); @@ -2597,6 +2733,13 @@ static int perf_header__read_build_ids_abi_quirk(struct perf_header *header, if (readn(input, filename, len) != len) return -1; + /* + * The file data may lack a null terminator, which could + * indicate a corrupt or crafted perf.data file. Ensure + * filename is always a valid C string before passing it + * to functions like machine__findnew_dso(). + */ + filename[len] = '\0'; bev.header = old_bev.header; @@ -2624,17 +2767,32 @@ static int perf_header__read_build_ids(struct perf_header *header, struct perf_session *session = container_of(header, struct perf_session, header); struct perf_record_header_build_id bev; char filename[PATH_MAX]; - u64 limit = offset + size, orig_offset = offset; + u64 limit, orig_offset = offset; int err = -1; + /* Prevent offset + size from wrapping past ULLONG_MAX */ + if (size > ULLONG_MAX - offset) + return -1; + + limit = offset + size; + while (offset < limit) { ssize_t len; if (readn(input, &bev, sizeof(bev)) != sizeof(bev)) goto out; - if (header->needs_swap) + if (header->needs_swap) { perf_event_header__bswap(&bev.header); + bev.pid = bswap_32(bev.pid); + } + + /* + * size == 0 would loop forever (offset never advances); + * size > remaining would read past the section boundary. + */ + if (bev.header.size == 0 || bev.header.size > limit - offset) + goto out; len = bev.header.size - sizeof(bev); if (len < 0 || len >= PATH_MAX) { @@ -2645,6 +2803,13 @@ static int perf_header__read_build_ids(struct perf_header *header, if (readn(input, filename, len) != len) goto out; /* + * The file data may lack a null terminator, which could + * indicate a corrupt or crafted perf.data file. Ensure + * filename is always a valid C string before passing it + * to functions like machine__findnew_dso(). + */ + filename[len] = '\0'; + /* * The a1645ce1 changeset: * * "perf: 'perf kvm' tool for monitoring guest performance from host" @@ -2657,7 +2822,9 @@ static int perf_header__read_build_ids(struct perf_header *header, * '[kernel.kallsyms]' string for the kernel build-id has the * first 4 characters chopped off (where the pid_t sits). */ - if (memcmp(filename, "nel.kallsyms]", 13) == 0) { + /* Guard short filenames against memcmp reading past the buffer */ + if (len >= (ssize_t)sizeof("nel.kallsyms]") - 1 && + memcmp(filename, "nel.kallsyms]", sizeof("nel.kallsyms]") - 1) == 0) { if (lseek(input, orig_offset, SEEK_SET) == (off_t)-1) return -1; return perf_header__read_build_ids_abi_quirk(header, input, offset, size); @@ -2684,10 +2851,18 @@ static int process_##__feat(struct feat_fd *ff, void *data __maybe_unused) \ FEAT_PROCESS_STR_FUN(hostname, hostname); FEAT_PROCESS_STR_FUN(osrelease, os_release); FEAT_PROCESS_STR_FUN(version, version); -FEAT_PROCESS_STR_FUN(arch, arch); FEAT_PROCESS_STR_FUN(cpudesc, cpu_desc); FEAT_PROCESS_STR_FUN(cpuid, cpuid); +static int process_arch(struct feat_fd *ff, void *data __maybe_unused) +{ + free(ff->ph->env.arch); + ff->ph->env.arch = do_read_string(ff); + if (!ff->ph->env.arch) + return -ENOMEM; + return 0; +} + static int process_e_machine(struct feat_fd *ff, void *data __maybe_unused) { int ret; @@ -2706,14 +2881,21 @@ static int process_tracing_data(struct feat_fd *ff __maybe_unused, void *data __ return ret < 0 ? -1 : 0; #else - pr_err("ERROR: Trying to read tracing data without libtraceevent support.\n"); - return -1; + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("Tracing data present but libtraceevent not available, skipping.\n"); + return 0; #endif } static int process_build_id(struct feat_fd *ff, void *data __maybe_unused) { - if (perf_header__read_build_ids(ff->ph, ff->fd, ff->offset, ff->size)) + /* lseek fails in pipe mode — fall back to ff->offset */ + off_t offset = lseek(ff->fd, 0, SEEK_CUR); + + if (offset == (off_t)-1) + offset = ff->offset; + + if (perf_header__read_build_ids(ff->ph, ff->fd, offset, ff->size)) pr_debug("Failed to read buildids, continuing...\n"); return 0; } @@ -2732,6 +2914,17 @@ static int process_nrcpus(struct feat_fd *ff, void *data __maybe_unused) if (ret) return ret; + /* + * Cap at 1M CPUs — generous for any real system but prevents + * stack overflow from VLA allocations sized by nr_cpus_avail + * (e.g. DECLARE_BITMAP in builtin-c2c.c node_entry()). + */ + if (nr_cpus_avail > (1U << 20)) { + pr_err("Invalid HEADER_NRCPUS: nr_cpus_avail (%u) exceeds maximum (%u)\n", + nr_cpus_avail, 1U << 20); + return -1; + } + if (nr_cpus_online > nr_cpus_avail) { pr_err("Invalid HEADER_NRCPUS: nr_cpus_online (%u) > nr_cpus_avail (%u)\n", nr_cpus_online, nr_cpus_avail); @@ -3357,7 +3550,8 @@ static int process_mem_topology(struct feat_fd *ff, return -1; } - if (ff->size < 3 * sizeof(u64) + nr * 2 * sizeof(u64)) { + /* Per node: node_id(u64) + mem_size(u64) + bitmap_nr_bits(u64) */ + if (ff->size < 3 * sizeof(u64) + nr * 3 * sizeof(u64)) { pr_err("Invalid HEADER_MEM_TOPOLOGY: section too small (%zu) for %llu nodes\n", ff->size, (unsigned long long)nr); return -1; @@ -3392,7 +3586,7 @@ static int process_mem_topology(struct feat_fd *ff, out: if (ret) - free(nodes); + memory_node__delete_nodes(nodes, nr); return ret; } @@ -3601,8 +3795,9 @@ out: up_write(&env->bpf_progs.lock); return err; #else - pr_err("ERROR: Trying to read bpf_prog_info without libbpf support.\n"); - return -1; + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("BPF prog info present but libbpf not available, skipping.\n"); + return 0; #endif // HAVE_LIBBPF_SUPPORT } @@ -3670,8 +3865,9 @@ out: free(node); return err; #else - pr_err("ERROR: Trying to read btf data without libbpf support.\n"); - return -1; + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("BTF data present but libbpf not available, skipping.\n"); + return 0; #endif // HAVE_LIBBPF_SUPPORT } @@ -3695,6 +3891,23 @@ static int process_compressed(struct feat_fd *ff, if (do_read_u32(ff, &(env->comp_mmap_len))) return -1; + /* + * FIXME: perf.data should record the recording system's page + * size — it affects mmap buffer alignment, sample addresses, + * and data_page_size/code_page_size interpretation. Without + * it we assume 4K (the smallest Linux page size) as a safe + * minimum alignment for comp_mmap_len validation. + * + * No upper-bound cap: perf_session__process_compressed_event() + * checks decomp_len + sizeof(struct decomp) against SIZE_MAX + * before allocating, which handles 32-bit safety. + */ + if (env->comp_mmap_len < 4096 || env->comp_mmap_len % 4096) { + pr_err("Invalid HEADER_COMPRESSED: comp_mmap_len (%u) must be a 4K-aligned value >= 4096\n", + env->comp_mmap_len); + return -1; + } + return 0; } @@ -4088,6 +4301,7 @@ static int perf_file_section__fprintf_info(struct perf_file_section *section, ff = (struct feat_fd) { .fd = fd, .ph = ph, + .size = section->size, }; if (!feat_ops[feat].full_only || hd->full) @@ -4423,8 +4637,13 @@ int perf_session__inject_header(struct perf_session *session, static int perf_header__getbuffer64(struct perf_header *header, int fd, void *buf, size_t size) { - if (readn(fd, buf, size) <= 0) + ssize_t n = readn(fd, buf, size); + + if (n <= 0) { + if (n == 0) + errno = EIO; return -1; + } if (header->needs_swap) mem_bswap_64(buf, size); @@ -4443,6 +4662,7 @@ int perf_header__process_sections(struct perf_header *header, int fd, int sec_size; int feat; int err; + struct stat st; nr_sections = bitmap_weight(header->adds_features, HEADER_FEAT_BITS); if (!nr_sections) @@ -4460,7 +4680,29 @@ int perf_header__process_sections(struct perf_header *header, int fd, if (err < 0) goto out_free; + if (fstat(fd, &st) < 0) { + pr_err("Failed to stat the perf data file\n"); + err = -1; + goto out_free; + } + for_each_set_bit(feat, header->adds_features, header->last_feat) { + /* + * FIXME: block devices have st_size == 0, so we skip + * bounds checking entirely. Historically perf never + * prevented using a block device as input, but it + * probably should — there's no valid use case for it + * and it bypasses all file-size validation. + */ + if (S_ISREG(st.st_mode) && + (sec->offset > (u64)st.st_size || + sec->size > (u64)st.st_size - sec->offset)) { + pr_err("Feature %s (%d) section extends past EOF (offset=%" PRIu64 ", size=%" PRIu64 ", file=%" PRIu64 ")\n", + header_feat__name(feat), feat, + sec->offset, sec->size, (u64)st.st_size); + err = -1; + goto out_free; + } err = process(sec++, header, feat, fd, data); if (err < 0) goto out_free; @@ -4687,7 +4929,7 @@ static int perf_file_section__process(struct perf_file_section *section, .fd = fd, .ph = ph, .size = section->size, - .offset = section->offset, + .offset = 0, }; if (lseek(fd, section->offset, SEEK_SET) == (off_t)-1) { @@ -4758,6 +5000,8 @@ static int read_attr(int fd, struct perf_header *ph, if (ret <= 0) { pr_debug("cannot read %d bytes of header attr\n", PERF_ATTR_SIZE_VER0); + if (ret == 0) + errno = EIO; return -1; } @@ -4770,9 +5014,15 @@ static int read_attr(int fd, struct perf_header *ph, if (sz == 0) { /* assume ABI0 */ sz = PERF_ATTR_SIZE_VER0; + } else if (sz < PERF_ATTR_SIZE_VER0) { + pr_debug("bad attr size %zu, expected at least %d\n", + sz, PERF_ATTR_SIZE_VER0); + errno = EINVAL; + return -1; } else if (sz > our_sz) { pr_debug("file uses a more recent and unsupported ABI" " (%zu bytes extra)\n", sz - our_sz); + errno = EINVAL; return -1; } /* what we have not yet read and that we know about */ @@ -4782,11 +5032,21 @@ static int read_attr(int fd, struct perf_header *ph, ptr += PERF_ATTR_SIZE_VER0; ret = readn(fd, ptr, left); + if (ret <= 0) { + if (ret == 0) + errno = EIO; + return -1; + } } /* read perf_file_section, ids are read in caller */ ret = readn(fd, &f_attr->ids, sizeof(f_attr->ids)); + if (ret <= 0) { + if (ret == 0) + errno = EIO; + return -1; + } - return ret <= 0 ? -1 : 0; + return 0; } #ifdef HAVE_LIBTRACEEVENT @@ -4842,7 +5102,8 @@ int perf_session__read_header(struct perf_session *session) struct perf_file_header f_header; struct perf_file_attr f_attr; u64 f_id; - int nr_attrs, nr_ids, i, j, err; + struct stat input_stat; + int nr_attrs, nr_ids, i, j, err = -ENOMEM; int fd = perf_data__fd(data); session->evlist = evlist__new(); @@ -4862,6 +5123,7 @@ int perf_session__read_header(struct perf_session *session) return err; } + err = -ENOMEM; if (perf_file_header__read(&f_header, header, fd) < 0) return -EINVAL; @@ -4889,6 +5151,15 @@ int perf_session__read_header(struct perf_session *session) return -EINVAL; } + if (fstat(fd, &input_stat) < 0) + return -errno; + + /* Check before assigning to int to avoid u64-to-int truncation */ + if (f_header.attrs.size / f_header.attr_size > MAX_NR_ATTRS) { + pr_err("Too many attributes: %" PRIu64 " (max %d)\n", + f_header.attrs.size / f_header.attr_size, MAX_NR_ATTRS); + return -EINVAL; + } nr_attrs = f_header.attrs.size / f_header.attr_size; lseek(fd, f_header.attrs.offset, SEEK_SET); @@ -4905,6 +5176,45 @@ int perf_session__read_header(struct perf_session *session) perf_event__attr_swap(&f_attr.attr); } + /* + * Validate ids section: must be aligned to u64, and + * the count must fit in an int to avoid truncation in + * nr_ids and size_t overflow in perf_evsel__alloc_id() + * on 32-bit architectures. + */ + if (f_attr.ids.size % sizeof(u64)) { + pr_err("Invalid ids section size %" PRIu64 " for attr %d, not aligned to u64\n", + f_attr.ids.size, i); + err = -EINVAL; + goto out_delete_evlist; + } + + /* + * Cap the ID count to avoid int truncation of nr_ids + * on 64-bit and size_t overflow in the allocation + * paths (nr_ids * sizeof(u64), nr_ids * + * sizeof(struct perf_sample_id)) on 32-bit. + */ + if (f_attr.ids.size / sizeof(u64) > MAX_IDS_PER_ATTR) { + pr_err("Invalid ids section size %" PRIu64 " for attr %d, too many IDs\n", + f_attr.ids.size, i); + err = -EINVAL; + goto out_delete_evlist; + } + + /* + * FIXME: see perf_header__process_sections() — block + * devices bypass this check because st_size is 0. + */ + if (S_ISREG(input_stat.st_mode) && + (f_attr.ids.offset > (u64)input_stat.st_size || + f_attr.ids.size > (u64)input_stat.st_size - f_attr.ids.offset)) { + pr_err("Invalid ids section for attr %d: offset=%" PRIu64 " size=%" PRIu64 " exceeds file size %" PRIu64 "\n", + i, f_attr.ids.offset, f_attr.ids.size, (u64)input_stat.st_size); + err = -EINVAL; + goto out_delete_evlist; + } + tmp = lseek(fd, 0, SEEK_CUR); evsel = evsel__new(&f_attr.attr); @@ -4939,15 +5249,54 @@ int perf_session__read_header(struct perf_session *session) lseek(fd, tmp, SEEK_SET); } + /* + * Skip feature section processing for truncated files + * (data.size == 0 means recording was interrupted). The + * section table is unreliable in that case, and the event + * data can still be processed without the feature headers. + * Clear the bitmap so has_feat() returns false and tools + * use their "feature not present" fallbacks instead of + * accessing uninitialized env fields. + */ + if (f_header.data.size == 0) { + bitmap_zero(header->adds_features, HEADER_FEAT_BITS); + } else { #ifdef HAVE_LIBTRACEEVENT - perf_header__process_sections(header, fd, &session->tevent, - perf_file_section__process); + err = perf_header__process_sections(header, fd, &session->tevent, + perf_file_section__process); + if (err < 0) + goto out_delete_evlist; - if (evlist__prepare_tracepoint_events(session->evlist, session->tevent.pevent)) - goto out_delete_evlist; + if (evlist__prepare_tracepoint_events(session->evlist, + session->tevent.pevent)) { + err = -ENOMEM; + goto out_delete_evlist; + } #else - perf_header__process_sections(header, fd, NULL, perf_file_section__process); + err = perf_header__process_sections(header, fd, NULL, + perf_file_section__process); + if (err < 0) + goto out_delete_evlist; #endif + } + + /* + * Without nr_cpus_avail the sample CPU bounds check in + * perf_session__deliver_event() is bypassed, allowing crafted + * CPU IDs to reach downstream consumers that index fixed-size + * arrays (timechart, kwork, sched — all sized MAX_NR_CPUS). + * + * This can happen with truncated files (interrupted recording + * loses all feature sections), very old files that predate + * HEADER_NRCPUS, or crafted files that omit it. Fall back to + * MAX_NR_CPUS so the bounds check is still effective — any + * CPU ID below that limit is safe for all downstream arrays. + */ + if (header->env.nr_cpus_avail == 0) { + header->env.nr_cpus_avail = MAX_NR_CPUS; + pr_warning("WARNING: perf.data is missing HEADER_NRCPUS, using MAX_NR_CPUS (%d) as CPU bound\n", + MAX_NR_CPUS); + } return 0; out_errno: @@ -4956,7 +5305,7 @@ out_errno: out_delete_evlist: evlist__delete(session->evlist); session->evlist = NULL; - return -ENOMEM; + return err; } int perf_event__process_feature(const struct perf_tool *tool __maybe_unused, @@ -5059,15 +5408,76 @@ size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp) switch (ev->type) { case PERF_EVENT_UPDATE__SCALE: + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) { + ret += fprintf(fp, "... scale: (truncated)\n"); + break; + } ret += fprintf(fp, "... scale: %f\n", ev->scale.scale); break; case PERF_EVENT_UPDATE__UNIT: - ret += fprintf(fp, "... unit: %s\n", ev->unit); - break; - case PERF_EVENT_UPDATE__NAME: - ret += fprintf(fp, "... name: %s\n", ev->name); + case PERF_EVENT_UPDATE__NAME: { + size_t str_off = offsetof(struct perf_record_event_update, unit); + size_t max_len = event->header.size > str_off ? + event->header.size - str_off : 0; + + if (max_len == 0 || strnlen(ev->unit, max_len) == max_len) { + ret += fprintf(fp, "... %s: (unterminated)\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name"); + break; + } + ret += fprintf(fp, "... %s: %s\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name", + ev->unit); break; - case PERF_EVENT_UPDATE__CPUS: + } + case PERF_EVENT_UPDATE__CPUS: { + size_t cpus_off = offsetof(struct perf_record_event_update, cpus); + u32 cpus_payload; + + if (event->header.size < cpus_off + sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map)) { + ret += fprintf(fp, "... cpus: (truncated)\n"); + break; + } + + /* + * Validate nr against payload — this function may be + * called from the stub handler (dump_trace path) which + * bypasses perf_event__process_event_update() validation. + */ + cpus_payload = event->header.size - cpus_off; + if (ev->cpus.cpus.type == PERF_CPU_MAP__CPUS) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, cpus_data.cpu) || + ev->cpus.cpus.cpus_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, cpus_data.cpu)) / + sizeof(ev->cpus.cpus.cpus_data.cpu[0])) { + ret += fprintf(fp, "... cpus: nr %u exceeds payload\n", + ev->cpus.cpus.cpus_data.nr); + break; + } + } else if (ev->cpus.cpus.type == PERF_CPU_MAP__MASK) { + if (ev->cpus.cpus.mask32_data.long_size == 4) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask32_data.mask) || + ev->cpus.cpus.mask32_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask32_data.mask)) / + sizeof(ev->cpus.cpus.mask32_data.mask[0])) { + ret += fprintf(fp, "... cpus: mask nr %u exceeds payload\n", + ev->cpus.cpus.mask32_data.nr); + break; + } + } else if (ev->cpus.cpus.mask64_data.long_size == 8) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask) || + ev->cpus.cpus.mask64_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) / + sizeof(ev->cpus.cpus.mask64_data.mask[0])) { + ret += fprintf(fp, "... cpus: mask nr %u exceeds payload\n", + ev->cpus.cpus.mask64_data.nr); + break; + } + } + } + ret += fprintf(fp, "... "); map = cpu_map__new_data(&ev->cpus.cpus); @@ -5077,6 +5487,7 @@ size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp) } else ret += fprintf(fp, "failed to get cpus\n"); break; + } default: ret += fprintf(fp, "... unknown type\n"); break; @@ -5094,11 +5505,42 @@ int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, union perf_event *event, struct evlist **pevlist) { - u32 i, n_ids; + struct perf_event_attr attr; + u32 i, n_ids, raw_attr_size; u64 *ids; + size_t attr_size, copy_size; struct evsel *evsel; struct evlist *evlist = *pevlist; + /* + * HEADER_ATTR event layout (pipe/inject mode): + * + * [header (8 bytes)] [attr (attr_size bytes)] [id0 id1 ... idN] + * |<------------------ header.size --------------------------->| + * + * attr_size varies across perf versions: VER0 = 64 bytes, + * current sizeof(struct perf_event_attr) = larger. A newer + * producer may emit a larger attr than we understand. + * + * attr.size == 0 (ABI0) means the producer didn't set it + * (e.g., bench/inject-buildid, older perf). Treat as VER0. + * + * Require 8-byte alignment so the u64 ID array is aligned + * and attr.size fits cleanly within the payload. + * + * Read attr.size once — the event may be on a shared mmap + * and re-reading could yield a different value. + */ + raw_attr_size = event->attr.attr.size; + if (event->header.size < sizeof(event->header) + PERF_ATTR_SIZE_VER0 || + (raw_attr_size && (raw_attr_size < PERF_ATTR_SIZE_VER0 || + raw_attr_size % sizeof(u64) || + raw_attr_size > event->header.size - sizeof(event->header)))) { + pr_err("PERF_RECORD_HEADER_ATTR: invalid attr.size %u (event size %u, min %d)\n", + raw_attr_size, event->header.size, PERF_ATTR_SIZE_VER0); + return -EINVAL; + } + if (dump_trace) perf_event__fprintf_attr(event, stdout); @@ -5108,13 +5550,46 @@ int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, return -ENOMEM; } - evsel = evsel__new(&event->attr.attr); + /* + * attr_size = footprint of the attr in the event — determines + * where the ID array starts. For ABI0, assume VER0 (64 bytes). + * + * copy_size = how much we copy into our local struct, capped at + * sizeof(attr) so a newer producer's larger attr doesn't + * overflow. Fields beyond copy_size are zeroed. + * + * Do NOT write attr_size back to the event — native-endian + * files use MAP_SHARED (read-only), writing would SIGSEGV. + * The swap path handles ABI0 in perf_event__attr_swap() + * which writes to the writable MAP_PRIVATE copy instead. + */ + attr_size = raw_attr_size ?: PERF_ATTR_SIZE_VER0; + copy_size = min(attr_size, sizeof(attr)); + memcpy(&attr, &event->attr.attr, copy_size); + if (copy_size < sizeof(attr)) + memset((void *)&attr + copy_size, 0, sizeof(attr) - copy_size); + + /* + * Normalize ABI0: the swap path sets attr.size = VER0 on the + * event, but the native path leaves it as 0. Set it on the + * local copy so perf inject re-synthesizes with consistent + * layout regardless of endianness. + */ + attr.size = attr_size; + + evsel = evsel__new(&attr); if (evsel == NULL) return -ENOMEM; evlist__add(evlist, evsel); - n_ids = event->header.size - sizeof(event->header) - event->attr.attr.size; + /* + * IDs occupy the remainder after header + attr. Use attr_size + * (not copy_size) — even if the producer's attr is larger than + * our struct, the IDs start after attr_size bytes in the event. + * Validation above guarantees attr_size <= payload size. + */ + n_ids = event->header.size - sizeof(event->header) - attr_size; n_ids = n_ids / sizeof(u64); /* * We don't have the cpu and thread maps on the header, so @@ -5124,7 +5599,13 @@ int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, if (perf_evsel__alloc_id(&evsel->core, 1, n_ids)) return -ENOMEM; - ids = perf_record_header_attr_id(event); + /* + * Locate IDs at attr_size bytes past the attr start in the + * event. Cannot use perf_record_header_attr_id() — that + * macro reads event->attr.attr.size, which is 0 for ABI0 + * on the native-endian path (no swap handler to fix it up). + */ + ids = (void *)&event->attr.attr + attr_size; for (i = 0; i < n_ids; i++) { perf_evlist__id_add(&evlist->core, &evsel->core, 0, i, ids[i]); } @@ -5141,6 +5622,83 @@ int perf_event__process_event_update(const struct perf_tool *tool __maybe_unused struct evsel *evsel; struct perf_cpu_map *map; + /* + * Validate payload before dump_trace or processing — both + * paths access variant-specific fields without further checks. + */ + if (ev->type == PERF_EVENT_UPDATE__UNIT || + ev->type == PERF_EVENT_UPDATE__NAME) { + size_t str_off = offsetof(struct perf_record_event_update, unit); + size_t max_len = event->header.size > str_off ? + event->header.size - str_off : 0; + + if (max_len == 0 || strnlen(ev->unit, max_len) == max_len) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: %s not null-terminated, skipping\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name"); + return 0; + } + } else if (ev->type == PERF_EVENT_UPDATE__SCALE) { + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: SCALE payload too small, skipping\n"); + return 0; + } + } else if (ev->type == PERF_EVENT_UPDATE__CPUS) { + size_t cpus_off = offsetof(struct perf_record_event_update, cpus); + size_t min_cpus = sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map); + u32 cpus_payload; + + if (event->header.size < cpus_off + min_cpus) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: CPUS payload too small, skipping\n"); + return 0; + } + + /* + * Validate per-variant nr against the remaining + * payload on the native path — the swap path clamps + * nr in perf_event__event_update_swap(), but native + * events are read-only and cannot be clamped in place. + * cpu_map__new_data() trusts nr for allocation and + * iteration, so unchecked values cause OOB reads. + */ + cpus_payload = event->header.size - cpus_off; + switch (ev->cpus.cpus.type) { + case PERF_CPU_MAP__CPUS: + if (ev->cpus.cpus.cpus_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, cpus_data.cpu)) / + sizeof(ev->cpus.cpus.cpus_data.cpu[0])) { + pr_warning("WARNING: EVENT_UPDATE CPUS: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.cpus_data.nr); + return 0; + } + break; + case PERF_CPU_MAP__MASK: + if (ev->cpus.cpus.mask32_data.long_size == 4) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask32_data.mask) || + ev->cpus.cpus.mask32_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask32_data.mask)) / + sizeof(ev->cpus.cpus.mask32_data.mask[0])) { + pr_warning("WARNING: EVENT_UPDATE MASK: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.mask32_data.nr); + return 0; + } + } else if (ev->cpus.cpus.mask64_data.long_size == 8) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask) || + ev->cpus.cpus.mask64_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) / + sizeof(ev->cpus.cpus.mask64_data.mask[0])) { + pr_warning("WARNING: EVENT_UPDATE MASK: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.mask64_data.nr); + return 0; + } + } + break; + default: + break; + } + } + if (dump_trace) perf_event__fprintf_event_update(event, stdout); @@ -5172,6 +5730,7 @@ int perf_event__process_event_update(const struct perf_tool *tool __maybe_unused evsel->core.pmu_cpus = map; } else pr_err("failed to get event_update cpus\n"); + break; default: break; } diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index 747fdc455c80..c93915625ee7 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -935,8 +935,8 @@ iter_add_single_mem_entry(struct hist_entry_iter *iter, struct addr_location *al { u64 cost; struct mem_info *mi = iter->mi; - struct hists *hists = evsel__hists(iter->evsel); struct perf_sample *sample = iter->sample; + struct hists *hists = evsel__hists(sample->evsel); struct hist_entry *he; if (mi == NULL) @@ -968,7 +968,7 @@ static int iter_finish_mem_entry(struct hist_entry_iter *iter, struct addr_location *al __maybe_unused) { - struct evsel *evsel = iter->evsel; + struct evsel *evsel = iter->sample->evsel; struct hists *hists = evsel__hists(evsel); struct hist_entry *he = iter->he; int err = -EINVAL; @@ -1036,9 +1036,9 @@ static int iter_add_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) { struct branch_info *bi; - struct evsel *evsel = iter->evsel; - struct hists *hists = evsel__hists(evsel); struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hists *hists = evsel__hists(evsel); struct hist_entry *he = NULL; int i = iter->curr; int err = 0; @@ -1078,7 +1078,7 @@ static int iter_finish_branch_entry(struct hist_entry_iter *iter, struct addr_location *al __maybe_unused) { - struct evsel *evsel = iter->evsel; + struct evsel *evsel = iter->sample->evsel; struct hists *hists = evsel__hists(evsel); for (int i = 0; i < iter->total; i++) @@ -1103,8 +1103,8 @@ iter_prepare_normal_entry(struct hist_entry_iter *iter __maybe_unused, static int iter_add_single_normal_entry(struct hist_entry_iter *iter, struct addr_location *al) { - struct evsel *evsel = iter->evsel; struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; struct hist_entry *he; he = hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL, @@ -1121,8 +1121,8 @@ iter_finish_normal_entry(struct hist_entry_iter *iter, struct addr_location *al __maybe_unused) { struct hist_entry *he = iter->he; - struct evsel *evsel = iter->evsel; struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; if (he == NULL) return 0; @@ -1165,9 +1165,9 @@ static int iter_add_single_cumulative_entry(struct hist_entry_iter *iter, struct addr_location *al) { - struct evsel *evsel = iter->evsel; - struct hists *hists = evsel__hists(evsel); struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hists *hists = evsel__hists(evsel); struct hist_entry **he_cache = iter->he_cache; struct hist_entry *he; int err = 0; @@ -1224,8 +1224,8 @@ static int iter_add_next_cumulative_entry(struct hist_entry_iter *iter, struct addr_location *al) { - struct evsel *evsel = iter->evsel; struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; struct hist_entry **he_cache = iter->he_cache; struct hist_entry *he; struct hist_entry he_tmp = { @@ -1342,7 +1342,7 @@ int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al, alm = map__get(al->map); err = sample__resolve_callchain(iter->sample, get_tls_callchain_cursor(), &iter->parent, - iter->evsel, al, max_stack_depth); + al, max_stack_depth); if (err) { map__put(alm); return err; @@ -2826,7 +2826,7 @@ int hists__unlink(struct hists *hists) void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, struct perf_sample *sample, bool nonany_branch_mode, - u64 *total_cycles, struct evsel *evsel) + u64 *total_cycles) { struct branch_info *bi; struct branch_entry *entries = perf_sample__branch_entries(sample); @@ -2850,7 +2850,7 @@ void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, for (int i = bs->nr - 1; i >= 0; i--) { addr_map_symbol__account_cycles(&bi[i].from, nonany_branch_mode ? NULL : prev, - bi[i].flags.cycles, evsel, + bi[i].flags.cycles, sample->evsel, bi[i].branch_stack_cntr); prev = &bi[i].to; @@ -2963,9 +2963,10 @@ int __hists__scnprintf_title(struct hists *hists, char *bf, size_t size, bool sh ev_name, sample_freq_str, enable_ref ? ref : " ", nr_events); - if (hists->uid_filter_str) - printed += snprintf(bf + printed, size - printed, - ", UID: %s", hists->uid_filter_str); + if (hists->uid_filter_str) { + printed += scnprintf(bf + printed, size - printed, + ", UID: %s", hists->uid_filter_str); + } if (thread) { if (hists__has(hists, thread)) { printed += scnprintf(bf + printed, size - printed, @@ -3040,7 +3041,7 @@ static void hists__delete_remaining_entries(struct rb_root_cached *root) } } -static void hists__delete_all_entries(struct hists *hists) +void hists__delete_all_entries(struct hists *hists) { hists__delete_entries(hists); hists__delete_remaining_entries(&hists->entries_in_array[0]); diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h index d97a4efb9250..b830cbe7f95b 100644 --- a/tools/perf/util/hist.h +++ b/tools/perf/util/hist.h @@ -156,7 +156,6 @@ struct hist_entry_iter { int total; int curr; - struct evsel *evsel; struct perf_sample *sample; struct hist_entry *he; struct symbol *parent; @@ -392,6 +391,7 @@ int hists__collapse_resort(struct hists *hists, struct ui_progress *prog); void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel); void hists__delete_entries(struct hists *hists); +void hists__delete_all_entries(struct hists *hists); void hists__output_recalc_col_len(struct hists *hists, int max_rows); struct hist_entry *hists__get_entry(struct hists *hists, int idx); @@ -799,7 +799,7 @@ unsigned int hists__overhead_width(struct hists *hists); void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, struct perf_sample *sample, bool nonany_branch_mode, - u64 *total_cycles, struct evsel *evsel); + u64 *total_cycles); struct option; int parse_filter_percentage(const struct option *opt, const char *arg, int unset); diff --git a/tools/perf/util/hwmon_pmu.c b/tools/perf/util/hwmon_pmu.c index fb3ffa8d32ad..ed544dca70c3 100644 --- a/tools/perf/util/hwmon_pmu.c +++ b/tools/perf/util/hwmon_pmu.c @@ -202,7 +202,8 @@ bool parse_hwmon_filename(const char *filename, fn_item_len = strlen(fn_item); if (fn_item_len > 6 && !strcmp(&fn_item[fn_item_len - 6], "_alarm")) { assert(strlen(LONGEST_HWMON_ITEM_STR) < sizeof(fn_type)); - strlcpy(fn_type, fn_item, fn_item_len - 5); + /* fn_item_len - 5 strips "_alarm"; clamp to buffer size */ + strlcpy(fn_type, fn_item, min_t(size_t, fn_item_len - 5, sizeof(fn_type))); fn_item = fn_type; *alarm = true; } @@ -289,13 +290,16 @@ static int hwmon_pmu__read_events(struct hwmon_pmu *pmu) if (fd < 0) continue; - read_len = read(fd, buf, sizeof(buf)); + read_len = read(fd, buf, sizeof(buf) - 1); while (read_len > 0 && buf[read_len - 1] == '\n') read_len--; - if (read_len > 0) - buf[read_len] = '\0'; + if (read_len <= 0) { + close(fd); + continue; + } + buf[read_len] = '\0'; if (buf[0] == '\0') { pr_debug("hwmon_pmu: empty label file %s %s\n", @@ -431,8 +435,8 @@ static size_t hwmon_pmu__describe_items(struct hwmon_pmu *hwm, char *out_buf, si hwmon_item_strs[bit], is_alarm ? "_alarm" : ""); fd = openat(dir, buf, O_RDONLY); - if (fd > 0) { - ssize_t read_len = read(fd, buf, sizeof(buf)); + if (fd >= 0) { + ssize_t read_len = read(fd, buf, sizeof(buf) - 1); while (read_len > 0 && buf[read_len - 1] == '\n') read_len--; @@ -442,12 +446,12 @@ static size_t hwmon_pmu__describe_items(struct hwmon_pmu *hwm, char *out_buf, si buf[read_len] = '\0'; val = strtoll(buf, /*endptr=*/NULL, 10); - len += snprintf(out_buf + len, out_buf_len - len, "%s%s%s=%g%s", - len == 0 ? " " : ", ", - hwmon_item_strs[bit], - is_alarm ? "_alarm" : "", - (double)val / 1000.0, - hwmon_units[key.type]); + len += scnprintf(out_buf + len, out_buf_len - len, "%s%s%s=%g%s", + len == 0 ? " " : ", ", + hwmon_item_strs[bit], + is_alarm ? "_alarm" : "", + (double)val / 1000.0, + hwmon_units[key.type]); } close(fd); } @@ -514,14 +518,14 @@ int hwmon_pmu__for_each_event(struct perf_pmu *pmu, void *state, pmu_event_callb int ret; size_t len; - len = snprintf(alias_buf, sizeof(alias_buf), "%s%d", - hwmon_type_strs[key.type], key.num); + scnprintf(alias_buf, sizeof(alias_buf), "%s%d", + hwmon_type_strs[key.type], key.num); if (!info.name) { info.name = info.alias; info.alias = NULL; } - len = snprintf(desc_buf, sizeof(desc_buf), "%s in unit %s named %s.", + len = scnprintf(desc_buf, sizeof(desc_buf), "%s in unit %s named %s.", hwmon_desc[key.type], pmu->name + 6, value->label ?: info.name); @@ -816,7 +820,7 @@ int evsel__hwmon_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) count = perf_counts(evsel->counts, cpu_map_idx, thread); fd = FD(evsel, cpu_map_idx, thread); - len = pread(fd, buf, sizeof(buf), 0); + len = pread(fd, buf, sizeof(buf) - 1, 0); if (len <= 0) { count->lost++; return -EINVAL; diff --git a/tools/perf/util/intel-bts.c b/tools/perf/util/intel-bts.c index 382255393fb3..0b18ebd13f7c 100644 --- a/tools/perf/util/intel-bts.c +++ b/tools/perf/util/intel-bts.c @@ -303,7 +303,8 @@ static int intel_bts_synth_branch_sample(struct intel_bts_queue *btsq, event.sample.header.size = bts->branches_event_size; ret = perf_event__synthesize_sample(&event, bts->branches_sample_type, - 0, &sample); + /*read_format=*/0, /*branch_sample_type=*/0, + &sample); if (ret) return ret; } diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c index 72c7a4e15d61..f90fcbc4302d 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c +++ b/tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c @@ -220,7 +220,6 @@ const char *dump_insn(struct perf_insn *x, uint64_t ip __maybe_unused, { struct insn insn; int n, i, ret; - int left; ret = insn_decode(&insn, inbuf, inlen, x->is64bit ? INSN_MODE_64 : INSN_MODE_32); @@ -229,13 +228,9 @@ const char *dump_insn(struct perf_insn *x, uint64_t ip __maybe_unused, return "<bad>"; if (lenp) *lenp = insn.length; - left = sizeof(x->out); - n = snprintf(x->out, left, "insn: "); - left -= n; - for (i = 0; i < insn.length; i++) { - n += snprintf(x->out + n, left, "%02x ", inbuf[i]); - left -= n; - } + n = scnprintf(x->out, sizeof(x->out), "insn: "); + for (i = 0; i < insn.length; i++) + n += scnprintf(x->out + n, sizeof(x->out) - n, "%02x ", inbuf[i]); return x->out; } diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index fc9eec8b54b8..56a9e439f5f8 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -1307,7 +1307,8 @@ static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt, goto out_free; } - if (pt->synth_opts.last_branch || pt->synth_opts.other_events) { + if (pt->synth_opts.last_branch || pt->synth_opts.add_last_branch || + pt->synth_opts.other_events) { unsigned int entry_cnt = max(LBRS_MAX, pt->br_stack_sz); ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt); @@ -1728,11 +1729,30 @@ static void intel_pt_prep_b_sample(struct intel_pt *pt, event->sample.header.misc = sample->cpumode; } -static int intel_pt_inject_event(union perf_event *event, +static int intel_pt_inject_event(struct intel_pt *pt, union perf_event *event, struct perf_sample *sample, u64 type) { - event->header.size = perf_event__sample_event_size(sample, type, 0); - return perf_event__synthesize_sample(event, type, 0, sample); + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && pt->session && pt->session->evlist) + evsel = evlist__id2evsel(pt->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + event->header.type = PERF_RECORD_SAMPLE; + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); } static inline int intel_pt_opt_inject(struct intel_pt *pt, @@ -1742,7 +1762,7 @@ static inline int intel_pt_opt_inject(struct intel_pt *pt, if (!pt->synth_opts.inject) return 0; - return intel_pt_inject_event(event, sample, type); + return intel_pt_inject_event(pt, event, sample, type); } static int intel_pt_deliver_synth_event(struct intel_pt *pt, @@ -2486,7 +2506,7 @@ static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evse intel_pt_add_xmm(intr_regs, pos, items, regs_mask); } - if (sample_type & PERF_SAMPLE_BRANCH_STACK) { + if ((sample_type | evsel->synth_sample_type) & PERF_SAMPLE_BRANCH_STACK) { if (items->mask[INTEL_PT_LBR_0_POS] || items->mask[INTEL_PT_LBR_1_POS] || items->mask[INTEL_PT_LBR_2_POS]) { @@ -2557,7 +2577,8 @@ static int intel_pt_do_synth_pebs_sample(struct intel_pt_queue *ptq, struct evse sample.transaction = txn; } - ret = intel_pt_deliver_synth_event(pt, event, &sample, sample_type); + ret = intel_pt_deliver_synth_event(pt, event, &sample, + sample_type | evsel->synth_sample_type); perf_sample__exit(&sample); return ret; } @@ -2979,7 +3000,7 @@ static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip) start = dso__first_symbol(map__dso(map)); for (sym = start; sym; sym = dso__next_symbol(sym)) { - if (sym->binding == STB_GLOBAL && + if (symbol__binding(sym) == STB_GLOBAL && !strcmp(sym->name, "__switch_to")) { ip = map__unmap_ip(map, sym->start); if (ip >= map__start(map) && ip < map__end(map)) { @@ -3426,7 +3447,7 @@ static int intel_pt_process_switch(struct intel_pt *pt, if (evsel != pt->switch_evsel) return 0; - tid = evsel__intval(evsel, sample, "next_pid"); + tid = perf_sample__intval(sample, "next_pid"); cpu = sample->cpu; intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n", diff --git a/tools/perf/util/intel-tpebs.c b/tools/perf/util/intel-tpebs.c index 8b615dc94e9e..bc3b79bfa01a 100644 --- a/tools/perf/util/intel-tpebs.c +++ b/tools/perf/util/intel-tpebs.c @@ -37,6 +37,7 @@ static pthread_t tpebs_reader_thread; static struct child_process tpebs_cmd; static int control_fd[2], ack_fd[2]; static struct mutex tpebs_mtx; +static bool tpebs_stopping; struct tpebs_retire_lat { struct list_head nd; @@ -52,16 +53,18 @@ struct tpebs_retire_lat { bool started; }; -static void tpebs_mtx_init(void) +static void tpebs_init(void) { mutex_init(&tpebs_mtx); + control_fd[0] = control_fd[1] = -1; + ack_fd[0] = ack_fd[1] = -1; } static struct mutex *tpebs_mtx_get(void) { - static pthread_once_t tpebs_mtx_once = PTHREAD_ONCE_INIT; + static pthread_once_t tpebs_once = PTHREAD_ONCE_INIT; - pthread_once(&tpebs_mtx_once, tpebs_mtx_init); + pthread_once(&tpebs_once, tpebs_init); return &tpebs_mtx; } @@ -111,6 +114,7 @@ static int evsel__tpebs_start_perf_record(struct evsel *evsel) /* Note, no workload given so system wide is implied. */ assert(tpebs_cmd.pid == 0); + memset(&tpebs_cmd, 0, sizeof(tpebs_cmd)); tpebs_cmd.argv = record_argv; tpebs_cmd.out = -1; ret = start_command(&tpebs_cmd); @@ -185,7 +189,6 @@ static bool should_ignore_sample(const struct perf_sample *sample, const struct static int process_sample_event(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine __maybe_unused) { struct tpebs_retire_lat *t; @@ -196,7 +199,7 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused, mutex_unlock(tpebs_mtx_get()); return 0; } - t = tpebs_retire_lat__find(evsel); + t = tpebs_retire_lat__find(sample->evsel); if (!t) { mutex_unlock(tpebs_mtx_get()); return -EINVAL; @@ -321,20 +324,43 @@ static int tpebs_stop(void) EXCLUSIVE_LOCKS_REQUIRED(tpebs_mtx_get()) { int ret = 0; + if (tpebs_stopping) + return 0; + /* Like tpebs_start, we should only run tpebs_end once. */ if (tpebs_cmd.pid != 0) { + pid_t actual_pid = tpebs_cmd.pid; + + tpebs_stopping = true; tpebs_send_record_cmd(EVLIST_CTL_CMD_STOP_TAG); tpebs_cmd.pid = 0; mutex_unlock(tpebs_mtx_get()); pthread_join(tpebs_reader_thread, NULL); mutex_lock(tpebs_mtx_get()); - close(control_fd[0]); - close(control_fd[1]); - close(ack_fd[0]); - close(ack_fd[1]); - close(tpebs_cmd.out); + if (control_fd[0] >= 0) { + close(control_fd[0]); + control_fd[0] = -1; + } + if (control_fd[1] >= 0) { + close(control_fd[1]); + control_fd[1] = -1; + } + if (ack_fd[0] >= 0) { + close(ack_fd[0]); + ack_fd[0] = -1; + } + if (ack_fd[1] >= 0) { + close(ack_fd[1]); + ack_fd[1] = -1; + } + if (tpebs_cmd.out >= 0) { + close(tpebs_cmd.out); + tpebs_cmd.out = -1; + } + tpebs_cmd.pid = actual_pid; ret = finish_command(&tpebs_cmd); tpebs_cmd.pid = 0; + tpebs_stopping = false; if (ret == -ERR_RUN_COMMAND_WAITPID_SIGNAL) ret = 0; } @@ -487,30 +513,42 @@ int evsel__tpebs_open(struct evsel *evsel) { int ret; bool tpebs_empty; + bool started_process = false; /* We should only run tpebs_start when tpebs_recording is enabled. */ if (!tpebs_recording) return 0; + + mutex_lock(tpebs_mtx_get()); + if (tpebs_stopping) { + mutex_unlock(tpebs_mtx_get()); + return -EBUSY; + } /* Only start the events once. */ if (tpebs_cmd.pid != 0) { struct tpebs_retire_lat *t; bool valid; - mutex_lock(tpebs_mtx_get()); t = tpebs_retire_lat__find(evsel); valid = t && t->started; mutex_unlock(tpebs_mtx_get()); /* May fail as the event wasn't started. */ return valid ? 0 : -EBUSY; } + mutex_unlock(tpebs_mtx_get()); ret = evsel__tpebs_prepare(evsel); if (ret) return ret; mutex_lock(tpebs_mtx_get()); + if (tpebs_stopping || tpebs_cmd.pid != 0) { + ret = -EBUSY; + goto out; + } tpebs_empty = list_empty(&tpebs_results); if (!tpebs_empty) { + started_process = true; /*Create control and ack fd for --control*/ if (pipe(control_fd) < 0) { pr_err("tpebs: Failed to create control fifo"); @@ -530,7 +568,6 @@ int evsel__tpebs_open(struct evsel *evsel) if (pthread_create(&tpebs_reader_thread, /*attr=*/NULL, __sample_reader, /*arg=*/NULL)) { kill(tpebs_cmd.pid, SIGTERM); - close(tpebs_cmd.out); pr_err("Could not create thread to process sample data.\n"); ret = -1; goto out; @@ -541,8 +578,38 @@ out: if (ret) { struct tpebs_retire_lat *t = tpebs_retire_lat__find(evsel); - list_del_init(&t->nd); - tpebs_retire_lat__delete(t); + if (t) { + list_del_init(&t->nd); + tpebs_retire_lat__delete(t); + } + + if (started_process) { + if (tpebs_cmd.pid > 0) { + kill(tpebs_cmd.pid, SIGTERM); + finish_command(&tpebs_cmd); + tpebs_cmd.pid = 0; + } + if (tpebs_cmd.out >= 0) { + close(tpebs_cmd.out); + tpebs_cmd.out = -1; + } + if (control_fd[0] >= 0) { + close(control_fd[0]); + control_fd[0] = -1; + } + if (control_fd[1] >= 0) { + close(control_fd[1]); + control_fd[1] = -1; + } + if (ack_fd[0] >= 0) { + close(ack_fd[0]); + ack_fd[0] = -1; + } + if (ack_fd[1] >= 0) { + close(ack_fd[1]); + ack_fd[1] = -1; + } + } } mutex_unlock(tpebs_mtx_get()); return ret; diff --git a/tools/perf/util/jitdump.c b/tools/perf/util/jitdump.c index e0ce8b904729..83005b30b9bf 100644 --- a/tools/perf/util/jitdump.c +++ b/tools/perf/util/jitdump.c @@ -234,7 +234,7 @@ jit_open(struct jit_buf_desc *jd, const char *name) /* * keep dirname for generating files and mmap records */ - strncpy(jd->dir, name, PATH_MAX); + strncpy(jd->dir, name, PATH_MAX - 1); jd->dir[PATH_MAX - 1] = '\0'; dirname(jd->dir); free(buf); @@ -409,7 +409,7 @@ static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp) * checks the event size and assigns these extended fields if these * fields are contained in the event. */ - if (event_contains(*time_conv, time_cycles)) { + if (event_contains(*time_conv, cap_user_time_short)) { tc.time_cycles = time_conv->time_cycles; tc.time_mask = time_conv->time_mask; tc.cap_user_time_zero = time_conv->cap_user_time_zero; @@ -642,7 +642,7 @@ static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr) ret = jit_inject_event(jd, event); if (!ret) - build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine); + build_id__mark_dso_hit(tool, event, &sample, jd->machine); out: perf_sample__exit(&sample); return ret; diff --git a/tools/perf/util/jitdump.h b/tools/perf/util/jitdump.h index ab2842def83d..f57bfebb20ff 100644 --- a/tools/perf/util/jitdump.h +++ b/tools/perf/util/jitdump.h @@ -11,9 +11,8 @@ #ifndef JITDUMP_H #define JITDUMP_H -#include <sys/time.h> -#include <time.h> #include <stdint.h> +#include <string.h> /* JiTD */ #define JITHEADER_MAGIC 0x4A695444 diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c b/tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c index c640dcd8af7c..018b0db0e6e7 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c @@ -17,12 +17,11 @@ static const char * const __kvm_events_tp[] = { NULL, }; -static void event_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_get_key(struct perf_sample *sample, struct event_key *key) { key->info = 0; - key->key = evsel__intval(evsel, sample, kvm_exit_reason(EM_AARCH64)); + key->key = perf_sample__intval(sample, kvm_exit_reason(EM_AARCH64)); key->exit_reasons = arm64_exit_reasons; /* @@ -31,24 +30,22 @@ static void event_get_key(struct evsel *evsel, * properly decode event's est_ec. */ if (key->key == ARM_EXCEPTION_TRAP) { - key->key = evsel__intval(evsel, sample, kvm_trap_exit_reason); + key->key = perf_sample__intval(sample, kvm_trap_exit_reason); key->exit_reasons = arm64_trap_exit_reasons; } } -static bool event_begin(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +static bool event_begin(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return evsel__name_is(evsel, kvm_entry_trace(EM_AARCH64)); + return evsel__name_is(sample->evsel, kvm_entry_trace(EM_AARCH64)); } -static bool event_end(struct evsel *evsel, - struct perf_sample *sample, +static bool event_end(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, kvm_exit_trace(EM_AARCH64))) { - event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, kvm_exit_trace(EM_AARCH64))) { + event_get_key(sample, key); return true; } return false; diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c b/tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c index b802e516b138..a04cd09e3361 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c @@ -53,14 +53,12 @@ static const char * const __kvm_events_tp[] = { NULL, }; -static bool event_begin(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +static bool event_begin(struct perf_sample *sample, struct event_key *key) { - return exit_event_begin(evsel, sample, key); + return exit_event_begin(sample, key); } -static bool event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +static bool event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { /* @@ -71,17 +69,16 @@ static bool event_end(struct evsel *evsel, * kvm:kvm_enter means returning to vmm and then to guest * kvm:kvm_reenter means returning to guest immediately */ - return evsel__name_is(evsel, kvm_entry_trace(EM_LOONGARCH)) || - evsel__name_is(evsel, kvm_reenter_trace); + return evsel__name_is(sample->evsel, kvm_entry_trace(EM_LOONGARCH)) || + evsel__name_is(sample->evsel, kvm_reenter_trace); } -static void event_gspr_get_key(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +static void event_gspr_get_key(struct perf_sample *sample, struct event_key *key) { unsigned int insn; key->key = LOONGARCH_EXCEPTION_OTHERS; - insn = evsel__intval(evsel, sample, "inst_word"); + insn = perf_sample__intval(sample, "inst_word"); switch (insn >> 24) { case 0: diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c b/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c index 42182d70beb6..96d9c4ae0209 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c @@ -28,12 +28,11 @@ static const char * const ppc_book3s_hv_kvm_tp[] = { /* 1 extra placeholder for NULL */ static const char *__kvm_events_tp[NR_TPS + 1]; -static void hcall_event_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void hcall_event_get_key(struct perf_sample *sample, struct event_key *key) { key->info = 0; - key->key = evsel__intval(evsel, sample, "req"); + key->key = perf_sample__intval(sample, "req"); } static const char *get_hcall_exit_reason(u64 exit_code) @@ -51,18 +50,16 @@ static const char *get_hcall_exit_reason(u64 exit_code) return "UNKNOWN"; } -static bool hcall_event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +static bool hcall_event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return evsel__name_is(evsel, __kvm_events_tp[3]); + return evsel__name_is(sample->evsel, __kvm_events_tp[3]); } -static bool hcall_event_begin(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +static bool hcall_event_begin(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, __kvm_events_tp[2])) { - hcall_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, __kvm_events_tp[2])) { + hcall_event_get_key(sample, key); return true; } diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c b/tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c index 8d4d5d6ce720..967bba261a47 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c @@ -20,30 +20,27 @@ static const char * const __kvm_events_tp[] = { NULL, }; -static void event_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_get_key(struct perf_sample *sample, struct event_key *key) { int xlen = 64; // TODO: 32-bit support. key->info = 0; - key->key = evsel__intval(evsel, sample, kvm_exit_reason(EM_RISCV)) & ~CAUSE_IRQ_FLAG(xlen); + key->key = perf_sample__intval(sample, kvm_exit_reason(EM_RISCV)) & ~CAUSE_IRQ_FLAG(xlen); key->exit_reasons = riscv_exit_reasons; } -static bool event_begin(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +static bool event_begin(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return evsel__name_is(evsel, kvm_entry_trace(EM_RISCV)); + return evsel__name_is(sample->evsel, kvm_entry_trace(EM_RISCV)); } -static bool event_end(struct evsel *evsel, - struct perf_sample *sample, +static bool event_end(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, kvm_exit_trace(EM_RISCV))) { - event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, kvm_exit_trace(EM_RISCV))) { + event_get_key(sample, key); return true; } return false; diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-s390.c b/tools/perf/util/kvm-stat-arch/kvm-stat-s390.c index 7e29169f5bb0..4771fc69fa39 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-s390.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-s390.c @@ -18,38 +18,34 @@ define_exit_reasons_table(sie_sigp_order_codes, sigp_order_codes); define_exit_reasons_table(sie_diagnose_codes, diagnose_codes); define_exit_reasons_table(sie_icpt_prog_codes, icpt_prog_codes); -static void event_icpt_insn_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_icpt_insn_get_key(struct perf_sample *sample, struct event_key *key) { u64 insn; - insn = evsel__intval(evsel, sample, "instruction"); + insn = perf_sample__intval(sample, "instruction"); key->key = icpt_insn_decoder(insn); key->exit_reasons = sie_icpt_insn_codes; } -static void event_sigp_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_sigp_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "order_code"); + key->key = perf_sample__intval(sample, "order_code"); key->exit_reasons = sie_sigp_order_codes; } -static void event_diag_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_diag_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "code"); + key->key = perf_sample__intval(sample, "code"); key->exit_reasons = sie_diagnose_codes; } -static void event_icpt_prog_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void event_icpt_prog_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "code"); + key->key = perf_sample__intval(sample, "code"); key->exit_reasons = sie_icpt_prog_codes; } diff --git a/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c b/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c index 0f626db3a439..788d216f0852 100644 --- a/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c +++ b/tools/perf/util/kvm-stat-arch/kvm-stat-x86.c @@ -24,45 +24,43 @@ static const struct kvm_events_ops exit_events = { * the time of MMIO write: kvm_mmio(KVM_TRACE_MMIO_WRITE...) -> kvm_entry * the time of MMIO read: kvm_exit -> kvm_mmio(KVM_TRACE_MMIO_READ...). */ -static void mmio_event_get_key(struct evsel *evsel, struct perf_sample *sample, +static void mmio_event_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "gpa"); - key->info = evsel__intval(evsel, sample, "type"); + key->key = perf_sample__intval(sample, "gpa"); + key->info = perf_sample__intval(sample, "type"); } #define KVM_TRACE_MMIO_READ_UNSATISFIED 0 #define KVM_TRACE_MMIO_READ 1 #define KVM_TRACE_MMIO_WRITE 2 -static bool mmio_event_begin(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +static bool mmio_event_begin(struct perf_sample *sample, struct event_key *key) { /* MMIO read begin event in kernel. */ - if (kvm_exit_event(evsel)) + if (kvm_exit_event(sample->evsel)) return true; /* MMIO write begin event in kernel. */ - if (evsel__name_is(evsel, "kvm:kvm_mmio") && - evsel__intval(evsel, sample, "type") == KVM_TRACE_MMIO_WRITE) { - mmio_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, "kvm:kvm_mmio") && + perf_sample__intval(sample, "type") == KVM_TRACE_MMIO_WRITE) { + mmio_event_get_key(sample, key); return true; } return false; } -static bool mmio_event_end(struct evsel *evsel, struct perf_sample *sample, - struct event_key *key) +static bool mmio_event_end(struct perf_sample *sample, struct event_key *key) { /* MMIO write end event in kernel. */ - if (kvm_entry_event(evsel)) + if (kvm_entry_event(sample->evsel)) return true; /* MMIO read end event in kernel.*/ - if (evsel__name_is(evsel, "kvm:kvm_mmio") && - evsel__intval(evsel, sample, "type") == KVM_TRACE_MMIO_READ) { - mmio_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, "kvm:kvm_mmio") && + perf_sample__intval(sample, "type") == KVM_TRACE_MMIO_READ) { + mmio_event_get_key(sample, key); return true; } @@ -86,31 +84,27 @@ static const struct kvm_events_ops mmio_events = { }; /* The time of emulation pio access is from kvm_pio to kvm_entry. */ -static void ioport_event_get_key(struct evsel *evsel, - struct perf_sample *sample, +static void ioport_event_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "port"); - key->info = evsel__intval(evsel, sample, "rw"); + key->key = perf_sample__intval(sample, "port"); + key->info = perf_sample__intval(sample, "rw"); } -static bool ioport_event_begin(struct evsel *evsel, - struct perf_sample *sample, +static bool ioport_event_begin(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, "kvm:kvm_pio")) { - ioport_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, "kvm:kvm_pio")) { + ioport_event_get_key(sample, key); return true; } return false; } -static bool ioport_event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, - struct event_key *key __maybe_unused) +static bool ioport_event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return kvm_entry_event(evsel); + return kvm_entry_event(sample->evsel); } static void ioport_event_decode_key(struct perf_kvm_stat *kvm __maybe_unused, @@ -130,31 +124,25 @@ static const struct kvm_events_ops ioport_events = { }; /* The time of emulation msr is from kvm_msr to kvm_entry. */ -static void msr_event_get_key(struct evsel *evsel, - struct perf_sample *sample, - struct event_key *key) +static void msr_event_get_key(struct perf_sample *sample, struct event_key *key) { - key->key = evsel__intval(evsel, sample, "ecx"); - key->info = evsel__intval(evsel, sample, "write"); + key->key = perf_sample__intval(sample, "ecx"); + key->info = perf_sample__intval(sample, "write"); } -static bool msr_event_begin(struct evsel *evsel, - struct perf_sample *sample, - struct event_key *key) +static bool msr_event_begin(struct perf_sample *sample, struct event_key *key) { - if (evsel__name_is(evsel, "kvm:kvm_msr")) { - msr_event_get_key(evsel, sample, key); + if (evsel__name_is(sample->evsel, "kvm:kvm_msr")) { + msr_event_get_key(sample, key); return true; } return false; } -static bool msr_event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, - struct event_key *key __maybe_unused) +static bool msr_event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return kvm_entry_event(evsel); + return kvm_entry_event(sample->evsel); } static void msr_event_decode_key(struct perf_kvm_stat *kvm __maybe_unused, diff --git a/tools/perf/util/kvm-stat.c b/tools/perf/util/kvm-stat.c index 27f16810498c..755ab659a05c 100644 --- a/tools/perf/util/kvm-stat.c +++ b/tools/perf/util/kvm-stat.c @@ -11,22 +11,20 @@ bool kvm_exit_event(struct evsel *evsel) return evsel__name_is(evsel, kvm_exit_trace(e_machine)); } -void exit_event_get_key(struct evsel *evsel, - struct perf_sample *sample, +void exit_event_get_key(struct perf_sample *sample, struct event_key *key) { - uint16_t e_machine = evsel__e_machine(evsel, /*e_flags=*/NULL); + uint16_t e_machine = evsel__e_machine(sample->evsel, /*e_flags=*/NULL); key->info = 0; - key->key = evsel__intval(evsel, sample, kvm_exit_reason(e_machine)); + key->key = perf_sample__intval(sample, kvm_exit_reason(e_machine)); } -bool exit_event_begin(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key) +bool exit_event_begin(struct perf_sample *sample, struct event_key *key) { - if (kvm_exit_event(evsel)) { - exit_event_get_key(evsel, sample, key); + if (kvm_exit_event(sample->evsel)) { + exit_event_get_key(sample, key); return true; } @@ -40,11 +38,10 @@ bool kvm_entry_event(struct evsel *evsel) return evsel__name_is(evsel, kvm_entry_trace(e_machine)); } -bool exit_event_end(struct evsel *evsel, - struct perf_sample *sample __maybe_unused, +bool exit_event_end(struct perf_sample *sample, struct event_key *key __maybe_unused) { - return kvm_entry_event(evsel); + return kvm_entry_event(sample->evsel); } static const char *get_exit_reason(struct perf_kvm_stat *kvm, diff --git a/tools/perf/util/kvm-stat.h b/tools/perf/util/kvm-stat.h index 4a998aaece5d..cdbd921a555f 100644 --- a/tools/perf/util/kvm-stat.h +++ b/tools/perf/util/kvm-stat.h @@ -53,18 +53,15 @@ struct kvm_event { }; struct child_event_ops { - void (*get_key)(struct evsel *evsel, - struct perf_sample *sample, + void (*get_key)(struct perf_sample *sample, struct event_key *key); const char *name; }; struct kvm_events_ops { - bool (*is_begin_event)(struct evsel *evsel, - struct perf_sample *sample, + bool (*is_begin_event)(struct perf_sample *sample, struct event_key *key); - bool (*is_end_event)(struct evsel *evsel, - struct perf_sample *sample, struct event_key *key); + bool (*is_end_event)(struct perf_sample *sample, struct event_key *key); const struct child_event_ops *child_ops; void (*decode_key)(struct perf_kvm_stat *kvm, struct event_key *key, char *decode); @@ -116,14 +113,11 @@ struct kvm_reg_events_ops { #ifdef HAVE_LIBTRACEEVENT -void exit_event_get_key(struct evsel *evsel, - struct perf_sample *sample, +void exit_event_get_key(struct perf_sample *sample, struct event_key *key); -bool exit_event_begin(struct evsel *evsel, - struct perf_sample *sample, +bool exit_event_begin(struct perf_sample *sample, struct event_key *key); -bool exit_event_end(struct evsel *evsel, - struct perf_sample *sample, +bool exit_event_end(struct perf_sample *sample, struct event_key *key); void exit_event_decode_key(struct perf_kvm_stat *kvm, struct event_key *key, diff --git a/tools/perf/util/kwork.h b/tools/perf/util/kwork.h index db00269b73f2..6ec70dcc4157 100644 --- a/tools/perf/util/kwork.h +++ b/tools/perf/util/kwork.h @@ -5,6 +5,7 @@ #include "util/tool.h" #include "util/time-utils.h" +#include <stdlib.h> #include <linux/bitmap.h> #include <linux/list.h> #include <linux/rbtree.h> @@ -157,7 +158,6 @@ struct kwork_class { struct kwork_class *class, struct kwork_work *work, enum kwork_trace_type src_type, - struct evsel *evsel, struct perf_sample *sample, struct machine *machine); @@ -165,21 +165,29 @@ struct kwork_class { char *buf, int len); }; +static inline void work_exit(struct kwork_work *work) +{ + if (work) { + free(work->name); + work->name = NULL; + } +} + struct trace_kwork_handler { int (*raise_event)(struct perf_kwork *kwork, - struct kwork_class *class, struct evsel *evsel, + struct kwork_class *class, struct perf_sample *sample, struct machine *machine); int (*entry_event)(struct perf_kwork *kwork, - struct kwork_class *class, struct evsel *evsel, + struct kwork_class *class, struct perf_sample *sample, struct machine *machine); int (*exit_event)(struct perf_kwork *kwork, - struct kwork_class *class, struct evsel *evsel, + struct kwork_class *class, struct perf_sample *sample, struct machine *machine); int (*sched_switch_event)(struct perf_kwork *kwork, - struct kwork_class *class, struct evsel *evsel, + struct kwork_class *class, struct perf_sample *sample, struct machine *machine); }; @@ -194,6 +202,7 @@ struct __top_cpus_runtime { struct kwork_top_stat { DECLARE_BITMAP(all_cpus_bitmap, MAX_NR_CPUS); struct __top_cpus_runtime *cpus_runtime; + unsigned int nr_skipped_cpu; }; struct perf_kwork { diff --git a/tools/perf/util/libdw.c b/tools/perf/util/libdw.c index 216977884103..d5d2958902c0 100644 --- a/tools/perf/util/libdw.c +++ b/tools/perf/util/libdw.c @@ -4,6 +4,7 @@ #include "srcline.h" #include "symbol.h" #include "dwarf-aux.h" +#include "callchain.h" #include <fcntl.h> #include <unistd.h> #include <elfutils/libdwfl.h> @@ -60,7 +61,10 @@ struct Dwfl *dso__libdw_dwfl(struct dso *dso) return NULL; } - dwfl_report_end(dwfl, /*removed=*/NULL, /*arg=*/NULL); + if (dwfl_report_end(dwfl, /*removed=*/NULL, /*arg=*/NULL) != 0) { + dwfl_end(dwfl); + return NULL; + } dso__set_libdw(dso, dwfl); return dwfl; @@ -72,43 +76,65 @@ struct libdw_a2l_cb_args { struct inline_node *node; char *leaf_srcline; bool leaf_srcline_used; + int err; }; static int libdw_a2l_cb(Dwarf_Die *die, void *_args) { struct libdw_a2l_cb_args *args = _args; - struct symbol *inline_sym = new_inline_sym(args->dso, args->sym, dwarf_diename(die)); + struct symbol *inline_sym = new_inline_sym(args->dso, args->sym, die_name(die)); const char *call_fname = die_get_call_file(die); + int call_lineno = die_get_call_lineno(die); char *call_srcline = srcline__unknown; - struct inline_list *ilist; if (!inline_sym) - return -ENOMEM; + goto abort_enomem; /* Assign caller information to the parent. */ if (call_fname) - call_srcline = srcline_from_fileline(call_fname, die_get_call_lineno(die)); + call_srcline = srcline_from_fileline(call_fname, call_lineno >= 0 ? call_lineno : 0); + + if (!list_empty(&args->node->val)) { + struct inline_list *parent; - list_for_each_entry(ilist, &args->node->val, list) { - if (args->leaf_srcline == ilist->srcline) + if (callchain_param.order == ORDER_CALLEE) + parent = list_first_entry(&args->node->val, struct inline_list, list); + else + parent = list_last_entry(&args->node->val, struct inline_list, list); + + if (args->leaf_srcline == parent->srcline) args->leaf_srcline_used = false; - else if (ilist->srcline != srcline__unknown) - free(ilist->srcline); - ilist->srcline = call_srcline; + else if (parent->srcline != srcline__unknown) + free(parent->srcline); + parent->srcline = call_srcline; call_srcline = NULL; - break; } if (call_srcline && call_srcline != srcline__unknown) free(call_srcline); /* Add this symbol to the chain as the leaf. */ if (!args->leaf_srcline_used) { - inline_list__append_tail(inline_sym, args->leaf_srcline, args->node); + if (inline_list__append_tail(inline_sym, args->leaf_srcline, args->node) != 0) + goto abort_delete_sym; args->leaf_srcline_used = true; } else { - inline_list__append_tail(inline_sym, strdup(args->leaf_srcline), args->node); + char *srcline = strdup(args->leaf_srcline); + + if (!srcline) + goto abort_delete_sym; + if (inline_list__append_tail(inline_sym, srcline, args->node) != 0) { + free(srcline); + goto abort_delete_sym; + } } return 0; + +abort_delete_sym: + if (symbol__inlined(inline_sym)) + symbol__delete(inline_sym); +abort_enomem: + args->err = -ENOMEM; + return DWARF_CB_ABORT; } int libdw__addr2line(u64 addr, char **file, unsigned int *line_nr, @@ -162,11 +188,29 @@ int libdw__addr2line(u64 addr, char **file, unsigned int *line_nr, .leaf_srcline = srcline_from_fileline(src ?: "<unknown>", lineno), }; + if (!args.leaf_srcline) { + if (file && *file) { + free(*file); + *file = NULL; + } + return 0; + } + /* Walk from the parent down to the leaf. */ - cu_walk_functions_at(cudie, addr, libdw_a2l_cb, &args); + if (cudie) + cu_walk_functions_at(cudie, addr, libdw_a2l_cb, &args); if (!args.leaf_srcline_used) free(args.leaf_srcline); + + if (args.err) { + if (file && *file) { + free(*file); + *file = NULL; + } + inline_node__clear_frames(node); + return 0; + } } return 1; } diff --git a/tools/perf/util/libunwind-arch/Build b/tools/perf/util/libunwind-arch/Build new file mode 100644 index 000000000000..80d3571918b1 --- /dev/null +++ b/tools/perf/util/libunwind-arch/Build @@ -0,0 +1,11 @@ +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arch.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arm64.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arm.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-loongarch.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-mips.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-ppc32.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-ppc64.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-riscv.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-s390.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-i386.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-x86_64.o diff --git a/tools/perf/util/libunwind-arch/libunwind-arch.c b/tools/perf/util/libunwind-arch/libunwind-arch.c new file mode 100644 index 000000000000..9a74cf3c8729 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-arch.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include <elf.h> +#include <errno.h> + +int get_perf_regnum_for_unw_regnum(unsigned int e_machine, int unw_regnum) +{ + switch (e_machine) { + case EM_ARM: + return __get_perf_regnum_for_unw_regnum_arm(unw_regnum); + case EM_AARCH64: + return __get_perf_regnum_for_unw_regnum_arm64(unw_regnum); + case EM_LOONGARCH: + return __get_perf_regnum_for_unw_regnum_loongarch(unw_regnum); + case EM_MIPS: + return __get_perf_regnum_for_unw_regnum_mips(unw_regnum); + case EM_PPC: + return __get_perf_regnum_for_unw_regnum_ppc32(unw_regnum); + case EM_PPC64: + return __get_perf_regnum_for_unw_regnum_ppc64(unw_regnum); + case EM_RISCV: + return __get_perf_regnum_for_unw_regnum_riscv(unw_regnum); + case EM_S390: + return __get_perf_regnum_for_unw_regnum_s390(unw_regnum); + case EM_386: + return __get_perf_regnum_for_unw_regnum_i386(unw_regnum); + case EM_X86_64: + return __get_perf_regnum_for_unw_regnum_x86_64(unw_regnum); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return -EINVAL; + } +} + + +void libunwind_arch__flush_access(struct maps *maps) +{ + unsigned int e_machine = maps__e_machine(maps); + + switch (e_machine) { + case EM_NONE: + break; // No libunwind info on the maps. + case EM_ARM: + __libunwind_arch__flush_access_arm(maps); + break; + case EM_AARCH64: + __libunwind_arch__flush_access_arm64(maps); + break; + case EM_LOONGARCH: + __libunwind_arch__flush_access_loongarch(maps); + break; + case EM_MIPS: + __libunwind_arch__flush_access_mips(maps); + break; + case EM_PPC: + __libunwind_arch__flush_access_ppc32(maps); + break; + case EM_PPC64: + __libunwind_arch__flush_access_ppc64(maps); + break; + case EM_RISCV: + __libunwind_arch__flush_access_riscv(maps); + break; + case EM_S390: + __libunwind_arch__flush_access_s390(maps); + break; + case EM_386: + __libunwind_arch__flush_access_i386(maps); + break; + case EM_X86_64: + __libunwind_arch__flush_access_x86_64(maps); + break; + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + break; + } +} + +void libunwind_arch__finish_access(struct maps *maps) +{ + unsigned int e_machine = maps__e_machine(maps); + + switch (e_machine) { + case EM_NONE: + break; // No libunwind info on the maps. + case EM_ARM: + __libunwind_arch__finish_access_arm(maps); + break; + case EM_AARCH64: + __libunwind_arch__finish_access_arm64(maps); + break; + case EM_LOONGARCH: + __libunwind_arch__finish_access_loongarch(maps); + break; + case EM_MIPS: + __libunwind_arch__finish_access_mips(maps); + break; + case EM_PPC: + __libunwind_arch__finish_access_ppc32(maps); + break; + case EM_PPC64: + __libunwind_arch__finish_access_ppc64(maps); + break; + case EM_RISCV: + __libunwind_arch__finish_access_riscv(maps); + break; + case EM_S390: + __libunwind_arch__finish_access_s390(maps); + break; + case EM_386: + __libunwind_arch__finish_access_i386(maps); + break; + case EM_X86_64: + __libunwind_arch__finish_access_x86_64(maps); + break; + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + break; + } +} + +void *libunwind_arch__create_addr_space(unsigned int e_machine) +{ + switch (e_machine) { + case EM_ARM: + return __libunwind_arch__create_addr_space_arm(); + case EM_AARCH64: + return __libunwind_arch__create_addr_space_arm64(); + case EM_LOONGARCH: + return __libunwind_arch__create_addr_space_loongarch(); + case EM_MIPS: + return __libunwind_arch__create_addr_space_mips(); + case EM_PPC: + return __libunwind_arch__create_addr_space_ppc32(); + case EM_PPC64: + return __libunwind_arch__create_addr_space_ppc64(); + case EM_RISCV: + return __libunwind_arch__create_addr_space_riscv(); + case EM_S390: + return __libunwind_arch__create_addr_space_s390(); + case EM_386: + return __libunwind_arch__create_addr_space_i386(); + case EM_X86_64: + return __libunwind_arch__create_addr_space_x86_64(); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return NULL; + } +} + +int libunwind_arch__dwarf_search_unwind_table(unsigned int e_machine, + void *as, + uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg) +{ + switch (e_machine) { + case EM_ARM: + return __libunwind_arch__dwarf_search_unwind_table_arm(as, ip, di, pi, + need_unwind_info, arg); + case EM_AARCH64: + return __libunwind_arch__dwarf_search_unwind_table_arm64(as, ip, di, pi, + need_unwind_info, arg); + case EM_LOONGARCH: + return __libunwind_arch__dwarf_search_unwind_table_loongarch(as, ip, di, pi, + need_unwind_info, arg); + case EM_MIPS: + return __libunwind_arch__dwarf_search_unwind_table_mips(as, ip, di, pi, + need_unwind_info, arg); + case EM_PPC: + return __libunwind_arch__dwarf_search_unwind_table_ppc32(as, ip, di, pi, + need_unwind_info, arg); + case EM_PPC64: + return __libunwind_arch__dwarf_search_unwind_table_ppc64(as, ip, di, pi, + need_unwind_info, arg); + case EM_RISCV: + return __libunwind_arch__dwarf_search_unwind_table_riscv(as, ip, di, pi, + need_unwind_info, arg); + case EM_S390: + return __libunwind_arch__dwarf_search_unwind_table_s390(as, ip, di, pi, + need_unwind_info, arg); + case EM_386: + return __libunwind_arch__dwarf_search_unwind_table_i386(as, ip, di, pi, + need_unwind_info, arg); + case EM_X86_64: + return __libunwind_arch__dwarf_search_unwind_table_x86_64(as, ip, di, pi, + need_unwind_info, arg); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return -EINVAL; + } +} + +int libunwind_arch__dwarf_find_debug_frame(unsigned int e_machine, + int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end) +{ + switch (e_machine) { + case EM_ARM: + return __libunwind_arch__dwarf_find_debug_frame_arm(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_AARCH64: + return __libunwind_arch__dwarf_find_debug_frame_arm64(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_LOONGARCH: + return __libunwind_arch__dwarf_find_debug_frame_loongarch(found, di_debug, ip, + segbase, obj_name, + start, end); + case EM_MIPS: + return __libunwind_arch__dwarf_find_debug_frame_mips(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_PPC: + return __libunwind_arch__dwarf_find_debug_frame_ppc32(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_PPC64: + return __libunwind_arch__dwarf_find_debug_frame_ppc64(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_RISCV: + return __libunwind_arch__dwarf_find_debug_frame_riscv(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_S390: + return __libunwind_arch__dwarf_find_debug_frame_s390(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_386: + return __libunwind_arch__dwarf_find_debug_frame_i386(found, di_debug, ip, segbase, + obj_name, start, end); + case EM_X86_64: + return __libunwind_arch__dwarf_find_debug_frame_x86_64(found, di_debug, ip, segbase, + obj_name, start, end); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return -EINVAL; + } +} + +struct unwind_info *libunwind_arch_unwind_info__new(struct thread *thread, + struct perf_sample *sample, int max_stack, + bool best_effort, uint16_t e_machine, + uint64_t first_ip) +{ + switch (e_machine) { + case EM_ARM: + return __libunwind_arch_unwind_info__new_arm(thread, sample, max_stack, + best_effort, first_ip); + case EM_AARCH64: + return __libunwind_arch_unwind_info__new_arm64(thread, sample, max_stack, + best_effort, first_ip); + case EM_LOONGARCH: + return __libunwind_arch_unwind_info__new_loongarch(thread, sample, max_stack, + best_effort, first_ip); + case EM_MIPS: + return __libunwind_arch_unwind_info__new_mips(thread, sample, max_stack, + best_effort, first_ip); + case EM_PPC: + return __libunwind_arch_unwind_info__new_ppc32(thread, sample, max_stack, + best_effort, first_ip); + case EM_PPC64: + return __libunwind_arch_unwind_info__new_ppc64(thread, sample, max_stack, + best_effort, first_ip); + case EM_RISCV: + return __libunwind_arch_unwind_info__new_riscv(thread, sample, max_stack, + best_effort, first_ip); + case EM_S390: + return __libunwind_arch_unwind_info__new_s390(thread, sample, max_stack, + best_effort, first_ip); + case EM_386: + return __libunwind_arch_unwind_info__new_i386(thread, sample, max_stack, + best_effort, first_ip); + case EM_X86_64: + return __libunwind_arch_unwind_info__new_x86_64(thread, sample, max_stack, + best_effort, first_ip); + default: + pr_err("ELF MACHINE %x is not supported.\n", e_machine); + return NULL; + } +} + +void libunwind_arch_unwind_info__delete(struct unwind_info *ui) +{ + free(ui); +} + +int libunwind_arch__unwind_step(struct unwind_info *ui) +{ + switch (ui->e_machine) { + case EM_ARM: + return __libunwind_arch__unwind_step_arm(ui); + case EM_AARCH64: + return __libunwind_arch__unwind_step_arm64(ui); + case EM_LOONGARCH: + return __libunwind_arch__unwind_step_loongarch(ui); + case EM_MIPS: + return __libunwind_arch__unwind_step_mips(ui); + case EM_PPC: + return __libunwind_arch__unwind_step_ppc32(ui); + case EM_PPC64: + return __libunwind_arch__unwind_step_ppc64(ui); + case EM_RISCV: + return __libunwind_arch__unwind_step_riscv(ui); + case EM_S390: + return __libunwind_arch__unwind_step_s390(ui); + case EM_386: + return __libunwind_arch__unwind_step_i386(ui); + case EM_X86_64: + return __libunwind_arch__unwind_step_x86_64(ui); + default: + pr_err("ELF MACHINE %x is not supported.\n", ui->e_machine); + return -EINVAL; + } +} diff --git a/tools/perf/util/libunwind-arch/libunwind-arch.h b/tools/perf/util/libunwind-arch/libunwind-arch.h new file mode 100644 index 000000000000..74a09cd58f38 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-arch.h @@ -0,0 +1,296 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LIBUNWIND_ARCH_H +#define __LIBUNWIND_ARCH_H + +#include <stdbool.h> +#include <stdint.h> +#include <stdlib.h> + +struct machine; +struct maps; +struct perf_sample; +struct thread; + +struct unwind_info { + struct machine *machine; + struct thread *thread; + struct perf_sample *sample; + void *cursor; + uint64_t *ips; + int cur_ip; + int max_ips; + unsigned int unw_word_t_size; + uint16_t e_machine; + bool best_effort; +}; + +struct libarch_unwind__dyn_info { + uint64_t start_ip; + uint64_t end_ip; + uint64_t segbase; + uint64_t table_data; + uint64_t table_len; +}; +struct libarch_unwind__proc_info; + +int __get_perf_regnum_for_unw_regnum_arm(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_arm64(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_loongarch(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_mips(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_ppc32(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_ppc64(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_riscv(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_s390(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_i386(int unw_regnum); +int __get_perf_regnum_for_unw_regnum_x86_64(int unw_regnum); +int get_perf_regnum_for_unw_regnum(unsigned int e_machine, int unw_regnum); + +void __libunwind_arch__flush_access_arm(struct maps *maps); +void __libunwind_arch__flush_access_arm64(struct maps *maps); +void __libunwind_arch__flush_access_loongarch(struct maps *maps); +void __libunwind_arch__flush_access_mips(struct maps *maps); +void __libunwind_arch__flush_access_ppc32(struct maps *maps); +void __libunwind_arch__flush_access_ppc64(struct maps *maps); +void __libunwind_arch__flush_access_riscv(struct maps *maps); +void __libunwind_arch__flush_access_s390(struct maps *maps); +void __libunwind_arch__flush_access_i386(struct maps *maps); +void __libunwind_arch__flush_access_x86_64(struct maps *maps); +void libunwind_arch__flush_access(struct maps *maps); + +void __libunwind_arch__finish_access_arm(struct maps *maps); +void __libunwind_arch__finish_access_arm64(struct maps *maps); +void __libunwind_arch__finish_access_loongarch(struct maps *maps); +void __libunwind_arch__finish_access_mips(struct maps *maps); +void __libunwind_arch__finish_access_ppc32(struct maps *maps); +void __libunwind_arch__finish_access_ppc64(struct maps *maps); +void __libunwind_arch__finish_access_riscv(struct maps *maps); +void __libunwind_arch__finish_access_s390(struct maps *maps); +void __libunwind_arch__finish_access_i386(struct maps *maps); +void __libunwind_arch__finish_access_x86_64(struct maps *maps); +void libunwind_arch__finish_access(struct maps *maps); + +void *__libunwind_arch__create_addr_space_arm(void); +void *__libunwind_arch__create_addr_space_arm64(void); +void *__libunwind_arch__create_addr_space_loongarch(void); +void *__libunwind_arch__create_addr_space_mips(void); +void *__libunwind_arch__create_addr_space_ppc32(void); +void *__libunwind_arch__create_addr_space_ppc64(void); +void *__libunwind_arch__create_addr_space_riscv(void); +void *__libunwind_arch__create_addr_space_s390(void); +void *__libunwind_arch__create_addr_space_i386(void); +void *__libunwind_arch__create_addr_space_x86_64(void); +void *libunwind_arch__create_addr_space(unsigned int e_machine); + +int __libunwind__find_proc_info(void *as, uint64_t ip, void *pi, int need_unwind_info, void *arg); +int __libunwind__access_mem(void *as, uint64_t addr, void *valp_word, int __write, void *arg); +int __libunwind__access_reg(void *as, int regnum, void *valp_word, int __write, void *arg); + +int __libunwind_arch__dwarf_search_unwind_table_arm(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_arm64(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_loongarch(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_mips(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_ppc32(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_ppc64(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_riscv(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_s390(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_i386(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int __libunwind_arch__dwarf_search_unwind_table_x86_64(void *as, uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); +int libunwind_arch__dwarf_search_unwind_table(unsigned int e_machine, + void *as, + uint64_t ip, + struct libarch_unwind__dyn_info *di, + void *pi, + int need_unwind_info, + void *arg); + +int __libunwind_arch__dwarf_find_debug_frame_arm(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_arm64(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_loongarch(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_mips(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_ppc32(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_ppc64(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_riscv(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_s390(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_i386(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int __libunwind_arch__dwarf_find_debug_frame_x86_64(int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); +int libunwind_arch__dwarf_find_debug_frame(unsigned int e_machine, + int found, + struct libarch_unwind__dyn_info *di_debug, + uint64_t ip, + uint64_t segbase, + const char *obj_name, + uint64_t start, + uint64_t end); + +struct unwind_info *__libunwind_arch_unwind_info__new_arm(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_arm64(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_loongarch(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_mips(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_ppc32(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_ppc64(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_riscv(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_s390(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_i386(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *__libunwind_arch_unwind_info__new_x86_64(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint64_t first_ip); +struct unwind_info *libunwind_arch_unwind_info__new(struct thread *thread, + struct perf_sample *sample, + int max_stack, + bool best_effort, + uint16_t e_machine, + uint64_t first_ip); + +void libunwind_arch_unwind_info__delete(struct unwind_info *ui); + +int __libunwind_arch__unwind_step_arm(struct unwind_info *ui); +int __libunwind_arch__unwind_step_arm64(struct unwind_info *ui); +int __libunwind_arch__unwind_step_loongarch(struct unwind_info *ui); +int __libunwind_arch__unwind_step_mips(struct unwind_info *ui); +int __libunwind_arch__unwind_step_ppc32(struct unwind_info *ui); +int __libunwind_arch__unwind_step_ppc64(struct unwind_info *ui); +int __libunwind_arch__unwind_step_riscv(struct unwind_info *ui); +int __libunwind_arch__unwind_step_s390(struct unwind_info *ui); +int __libunwind_arch__unwind_step_i386(struct unwind_info *ui); +int __libunwind_arch__unwind_step_x86_64(struct unwind_info *ui); +int libunwind_arch__unwind_step(struct unwind_info *ui); + +#endif /* __LIBUNWIND_ARCH_H */ diff --git a/tools/perf/util/libunwind-arch/libunwind-arm.c b/tools/perf/util/libunwind-arch/libunwind-arm.c new file mode 100644 index 000000000000..61b1c5a70b7f --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-arm.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/arm/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT +#include <libunwind-arm.h> +#endif + +int __get_perf_regnum_for_unw_regnum_arm(int unw_regnum) +{ + if (unw_regnum < 0 || unw_regnum >= PERF_REG_ARM_MAX) { + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } + return unw_regnum; +} + +void __libunwind_arch__flush_access_arm(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_arm(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + + +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_arm(void) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_arm(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_ARM_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_ARM) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_arm(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_ARM_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_ARM) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_arm(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_ARM; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_arm(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-arm64.c b/tools/perf/util/libunwind-arch/libunwind-arm64.c new file mode 100644 index 000000000000..4d467dd440a1 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-arm64.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/arm64/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT +#include <libunwind-aarch64.h> +#endif + +int __get_perf_regnum_for_unw_regnum_arm64(int unw_regnum) +{ + if (unw_regnum < 0 || unw_regnum >= PERF_REG_ARM64_EXTENDED_MAX) { + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } + return unw_regnum; +} + +void __libunwind_arch__flush_access_arm64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_arm64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_arm64(void) +{ +#ifdef HAVE_LIBUNWIND_AARCH64_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_ARM64_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_arm64(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM64_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_ARM64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_ARM64) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_arm64(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_ARM64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_ARM64) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_arm64(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM64_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_AARCH64; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_arm64(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_ARM64_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-i386.c b/tools/perf/util/libunwind-arch/libunwind-i386.c new file mode 100644 index 000000000000..3423be45096a --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-i386.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/x86/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_X86_SUPPORT +#include <libunwind-x86.h> +#endif + +int __get_perf_regnum_for_unw_regnum_i386(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_X86_SUPPORT + return -EINVAL; +#else + static const int perf_i386_regnums[] = { +#define REGNUM(reg) [UNW_X86_E ## reg] = PERF_REG_X86_ ## reg + REGNUM(AX), + REGNUM(DX), + REGNUM(CX), + REGNUM(BX), + REGNUM(SI), + REGNUM(DI), + REGNUM(BP), + REGNUM(SP), + REGNUM(IP), +#undef REGNUM + }; + + if (unw_regnum == UNW_X86_EAX) + return PERF_REG_X86_AX; + + if (unw_regnum < 0 || unw_regnum >= (int)ARRAY_SIZE(perf_i386_regnums) || + perf_i386_regnums[unw_regnum] == 0) { + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } + + return perf_i386_regnums[unw_regnum]; +#endif // HAVE_LIBUNWIND_X86_SUPPORT +} + +void __libunwind_arch__flush_access_i386(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_i386(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_X86_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_i386(void) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_X86_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_i386(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_X86_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_X86) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_i386(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_X86_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_X86) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_i386(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_I386; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_i386(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-loongarch.c b/tools/perf/util/libunwind-arch/libunwind-loongarch.c new file mode 100644 index 000000000000..ff225a540b43 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-loongarch.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/loongarch/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT +#include <libunwind-loongarch64.h> +#endif + +int __get_perf_regnum_for_unw_regnum_loongarch(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_LOONGARCH64_R1 ... UNW_LOONGARCH64_R31: + return unw_regnum - UNW_LOONGARCH64_R1 + PERF_REG_LOONGARCH_R1; + case UNW_LOONGARCH64_PC: + return PERF_REG_LOONGARCH_PC; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_LOONGARCH64_SUPPORT +} + +void __libunwind_arch__flush_access_loongarch(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_loongarch(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_loongarch(void) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH64_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_LOONGARCH_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_loongarch(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_LOONGARCH_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_LOONGARCH) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_loongarch(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_LOONGARCH_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_LOONGARCH) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_loongarch(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_LOONGARCH; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_loongarch(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_LOONGARCH_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-mips.c b/tools/perf/util/libunwind-arch/libunwind-mips.c new file mode 100644 index 000000000000..ed14a8f94fa7 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-mips.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/mips/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT +#include <libunwind-mips.h> +#endif + +int __get_perf_regnum_for_unw_regnum_mips(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_MIPS_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_MIPS_R1 ... UNW_MIPS_R25: + return unw_regnum - UNW_MIPS_R1 + PERF_REG_MIPS_R1; + case UNW_MIPS_R28 ... UNW_MIPS_R31: + return unw_regnum - UNW_MIPS_R28 + PERF_REG_MIPS_R28; + case UNW_MIPS_PC: + return PERF_REG_MIPS_PC; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_MIPS_SUPPORT +} + +void __libunwind_arch__flush_access_mips(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_mips(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_mips(void) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_mips(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_MIPS_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_MIPS) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_mips(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_MIPS_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_MIPS) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_mips(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_MIPS; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_mips(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_MIPS_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-ppc32.c b/tools/perf/util/libunwind-arch/libunwind-ppc32.c new file mode 100644 index 000000000000..289a3b8cde75 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-ppc32.c @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/powerpc/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT +#include <libunwind-ppc32.h> +#endif + +int __get_perf_regnum_for_unw_regnum_ppc32(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_PPC32_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_PPC32_R0 ... UNW_PPC32_R31: + return unw_regnum - UNW_PPC32_R0 + PERF_REG_POWERPC_R0; + case UNW_PPC32_LR: + return PERF_REG_POWERPC_LINK; + case UNW_PPC32_CTR: + return PERF_REG_POWERPC_CTR; + case UNW_PPC32_XER: + return PERF_REG_POWERPC_XER; + case UNW_PPC32_NIP: + return PERF_REG_POWERPC_NIP; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_PPC32_SUPPORT +} + +void __libunwind_arch__flush_access_ppc32(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_ppc32(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_ppc32(void) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_ppc32(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_PPC32_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_PPC32) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_ppc32(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_PPC32_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_PPC32) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_ppc32(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_PPC; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_ppc32(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC32_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-ppc64.c b/tools/perf/util/libunwind-arch/libunwind-ppc64.c new file mode 100644 index 000000000000..a4c9b6cc3c00 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-ppc64.c @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/powerpc/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT +#include <libunwind-ppc64.h> +#endif + +int __get_perf_regnum_for_unw_regnum_ppc64(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_PPC64_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_PPC64_R0 ... UNW_PPC64_R31: + return unw_regnum - UNW_PPC64_R0 + PERF_REG_POWERPC_R0; + case UNW_PPC64_LR: + return PERF_REG_POWERPC_LINK; + case UNW_PPC64_CTR: + return PERF_REG_POWERPC_CTR; + case UNW_PPC64_XER: + return PERF_REG_POWERPC_XER; + case UNW_PPC64_NIP: + return PERF_REG_POWERPC_NIP; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_PPC64_SUPPORT +} + +void __libunwind_arch__flush_access_ppc64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_ppc64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_ppc64(void) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_ppc64(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_PPC64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_PPC64) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_ppc64(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_PPC64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_PPC64) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_ppc64(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_PPC64; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_ppc64(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_PPC64_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-riscv.c b/tools/perf/util/libunwind-arch/libunwind-riscv.c new file mode 100644 index 000000000000..3220690cd7d1 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-riscv.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/riscv/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT +#include <libunwind-riscv.h> +#endif + +int __get_perf_regnum_for_unw_regnum_riscv(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_RISCV_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_RISCV_X1 ... UNW_RISCV_X31: + return unw_regnum - UNW_RISCV_X1 + PERF_REG_RISCV_RA; + case UNW_RISCV_PC: + return PERF_REG_RISCV_PC; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_RISCV_SUPPORT +} + +void __libunwind_arch__flush_access_riscv(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_riscv(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_riscv(void) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_riscv(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_RISCV_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_RISCV) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_riscv(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_RISCV_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_RISCV) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_riscv(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_RISCV; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_riscv(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_RISCV_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-s390.c b/tools/perf/util/libunwind-arch/libunwind-s390.c new file mode 100644 index 000000000000..3e57cfc451c6 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-s390.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/s390/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT +#include <libunwind-s390x.h> +#endif + +int __get_perf_regnum_for_unw_regnum_s390(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_S390X_SUPPORT + return -EINVAL; +#else + switch (unw_regnum) { + case UNW_S390X_R0 ... UNW_S390X_R15: + return unw_regnum - UNW_S390X_R0 + PERF_REG_S390_R0; + case UNW_S390X_F0 ... UNW_S390X_F15: + return unw_regnum - UNW_S390X_F0 + PERF_REG_S390_FP0; + case UNW_S390X_IP: + return PERF_REG_S390_PC; + default: + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } +#endif // HAVE_LIBUNWIND_S390X_SUPPORT +} + +void __libunwind_arch__flush_access_s390(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_s390(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_s390(void) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_s390(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_S390X_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_S390X) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_s390(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_S390X_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_S390X) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_s390(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_S390; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_s390(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_S390X_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind-arch/libunwind-x86_64.c b/tools/perf/util/libunwind-arch/libunwind-x86_64.c new file mode 100644 index 000000000000..b13e9b1cc7a3 --- /dev/null +++ b/tools/perf/util/libunwind-arch/libunwind-x86_64.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "libunwind-arch.h" +#include "../debug.h" +#include "../maps.h" +#include "../thread.h" +#include "../../../arch/x86/include/uapi/asm/perf_regs.h" +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> +#include <elf.h> +#include <errno.h> + +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT +#include <libunwind-x86_64.h> +#endif + +int __get_perf_regnum_for_unw_regnum_x86_64(int unw_regnum __maybe_unused) +{ +#ifndef HAVE_LIBUNWIND_X86_64_SUPPORT + return -EINVAL; +#else + static const int perf_x86_64_regnums[] = { +#define REGNUM(reg) [UNW_X86_64_R ## reg] = PERF_REG_X86_ ## reg + REGNUM(AX), + REGNUM(DX), + REGNUM(CX), + REGNUM(BX), + REGNUM(SI), + REGNUM(DI), + REGNUM(BP), + REGNUM(SP), + REGNUM(IP), +#undef REGNUM +#define REGNUM(reg) [UNW_X86_64_ ## reg] = PERF_REG_X86_ ## reg + REGNUM(R8), + REGNUM(R9), + REGNUM(R10), + REGNUM(R11), + REGNUM(R12), + REGNUM(R13), + REGNUM(R14), + REGNUM(R15), +#undef REGNUM + }; + + if (unw_regnum == UNW_X86_64_RAX) + return PERF_REG_X86_AX; + + if (unw_regnum < 0 || unw_regnum >= (int)ARRAY_SIZE(perf_x86_64_regnums) || + perf_x86_64_regnums[unw_regnum] == 0) { + pr_err("unwind: invalid reg id %d\n", unw_regnum); + return -EINVAL; + } + return perf_x86_64_regnums[unw_regnum]; +#endif // HAVE_LIBUNWIND_X86_64_SUPPORT +} + +void __libunwind_arch__flush_access_x86_64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + unw_flush_cache(maps__addr_space(maps), 0, 0); +#endif +} + +void __libunwind_arch__finish_access_x86_64(struct maps *maps __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + unw_destroy_addr_space(maps__addr_space(maps)); +#endif +} + +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT +static int find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, + int need_unwind_info, void *arg) +{ + return __libunwind__find_proc_info(as, ip, pi, need_unwind_info, arg); +} + +static void put_unwind_info(unw_addr_space_t __maybe_unused as, + unw_proc_info_t *pi __maybe_unused, + void *arg __maybe_unused) +{ + pr_debug("unwind: put_unwind_info called\n"); +} + +static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused *dil_addr, + void __maybe_unused *arg) +{ + return -UNW_ENOINFO; +} + +static int access_mem(unw_addr_space_t as, unw_word_t addr, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_mem(as, addr, valp, __write, arg); +} + +static int access_reg(unw_addr_space_t as, unw_regnum_t regnum, unw_word_t *valp, + int __write, void *arg) +{ + return __libunwind__access_reg(as, regnum, valp, __write, arg); +} + +static int access_fpreg(unw_addr_space_t __maybe_unused as, + unw_regnum_t __maybe_unused num, + unw_fpreg_t __maybe_unused *val, + int __maybe_unused __write, + void __maybe_unused *arg) +{ + pr_err("unwind: access_fpreg unsupported\n"); + return -UNW_EINVAL; +} + +static int resume(unw_addr_space_t __maybe_unused as, + unw_cursor_t __maybe_unused *cu, + void __maybe_unused *arg) +{ + pr_err("unwind: resume unsupported\n"); + return -UNW_EINVAL; +} + +static int get_proc_name(unw_addr_space_t __maybe_unused as, + unw_word_t __maybe_unused addr, + char __maybe_unused *bufp, size_t __maybe_unused buf_len, + unw_word_t __maybe_unused *offp, void __maybe_unused *arg) +{ + pr_err("unwind: get_proc_name unsupported\n"); + return -UNW_EINVAL; +} +#endif + +void *__libunwind_arch__create_addr_space_x86_64(void) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + static unw_accessors_t accessors = { + .find_proc_info = find_proc_info, + .put_unwind_info = put_unwind_info, + .get_dyn_info_list_addr = get_dyn_info_list_addr, + .access_mem = access_mem, + .access_reg = access_reg, + .access_fpreg = access_fpreg, + .resume = resume, + .get_proc_name = get_proc_name, + }; + unw_addr_space_t addr_space; + + addr_space = unw_create_addr_space(&accessors, /*byte_order=*/0); + unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); + return addr_space; +#else + return NULL; +#endif +} + +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT +extern int UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, + unw_word_t ip, + unw_dyn_info_t *di, + unw_proc_info_t *pi, + int need_unwind_info, void *arg); +#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) +#endif + +int __libunwind_arch__dwarf_search_unwind_table_x86_64(void *as __maybe_unused, + uint64_t ip __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + void *pi __maybe_unused, + int need_unwind_info __maybe_unused, + void *arg __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_search_unwind_table(as, ip, &di, pi, need_unwind_info, arg); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.rti.segbase; + _di->table_data = di.u.rti.table_data; + _di->table_len = di.u.rti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +#if defined(HAVE_LIBUNWIND_X86_64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_X86_64) +extern int UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, + unw_word_t ip, + unw_word_t segbase, + const char *obj_name, unw_word_t start, + unw_word_t end); +#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) +#endif + +int __libunwind_arch__dwarf_find_debug_frame_x86_64(int found __maybe_unused, + struct libarch_unwind__dyn_info *_di __maybe_unused, + uint64_t ip __maybe_unused, + uint64_t segbase __maybe_unused, + const char *obj_name __maybe_unused, + uint64_t start __maybe_unused, + uint64_t end __maybe_unused) +{ +#if defined(HAVE_LIBUNWIND_X86_64_SUPPORT) && !defined(NO_LIBUNWIND_DEBUG_FRAME_X86_64) + unw_dyn_info_t di = { + .format = UNW_INFO_FORMAT_REMOTE_TABLE, + .start_ip = _di->start_ip, + .end_ip = _di->end_ip, + .u = { + .rti = { + .segbase = _di->segbase, + .table_data = _di->table_data, + .table_len = _di->table_len, + }, + }, + }; + int ret = dwarf_find_debug_frame(found, &di, ip, segbase, obj_name, start, end); + + _di->start_ip = di.start_ip; + _di->end_ip = di.end_ip; + _di->segbase = di.u.ti.segbase; + _di->table_data = di.u.ti.table_data; + _di->table_len = di.u.ti.table_len; + return ret; +#else + return -EINVAL; +#endif +} + +struct unwind_info *__libunwind_arch_unwind_info__new_x86_64(struct thread *thread __maybe_unused, + struct perf_sample *sample __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused, + uint64_t first_ip __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + struct arch_unwind_info { + struct unwind_info ui; + unw_cursor_t _cursor; + uint64_t _ips[]; + }; + + struct maps *maps = thread__maps(thread); + void *addr_space = maps__addr_space(maps); + struct arch_unwind_info *ui; + int ret; + + if (addr_space == NULL) + return NULL; + + ui = zalloc(sizeof(*ui) + sizeof(ui->_ips[0]) * max_stack); + if (!ui) + return NULL; + + ui->ui.machine = maps__machine(maps); + ui->ui.thread = thread; + ui->ui.sample = sample; + ui->ui.cursor = &ui->_cursor; + ui->ui.ips = &ui->_ips[0]; + ui->ui.ips[0] = first_ip; + ui->ui.cur_ip = 1; + ui->ui.max_ips = max_stack; + ui->ui.unw_word_t_size = sizeof(unw_word_t); + ui->ui.e_machine = EM_X86_64; + ui->ui.best_effort = best_effort; + + ret = unw_init_remote(&ui->_cursor, addr_space, &ui->ui); + if (ret) { + if (!best_effort) + pr_err("libunwind: %s\n", unw_strerror(ret)); + free(ui); + return NULL; + } + return &ui->ui; +#else + return NULL; +#endif +} + +int __libunwind_arch__unwind_step_x86_64(struct unwind_info *ui __maybe_unused) +{ +#ifdef HAVE_LIBUNWIND_X86_64_SUPPORT + int ret; + + if (ui->cur_ip >= ui->max_ips) + return 0; + + ret = unw_step(ui->cursor); + if (ret > 0) { + uint64_t ip; + + unw_get_reg(ui->cursor, UNW_REG_IP, &ip); + + if (unw_is_signal_frame(ui->cursor) <= 0) { + /* + * Decrement the IP for any non-activation frames. This + * is required to properly find the srcline for caller + * frames. See also the documentation for + * dwfl_frame_pc(), which this code tries to replicate. + */ + --ip; + } + ui->ips[ui->cur_ip++] = ip; + } + return ret; +#else + return -EINVAL; +#endif +} diff --git a/tools/perf/util/libunwind/arm64.c b/tools/perf/util/libunwind/arm64.c deleted file mode 100644 index 37ecef0c53b9..000000000000 --- a/tools/perf/util/libunwind/arm64.c +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * This file setups defines to compile arch specific binary from the - * generic one. - * - * The function 'LIBUNWIND__ARCH_REG_ID' name is set according to arch - * name and the definition of this function is included directly from - * 'arch/arm64/util/unwind-libunwind.c', to make sure that this function - * is defined no matter what arch the host is. - * - * Finally, the arch specific unwind methods are exported which will - * be assigned to each arm64 thread. - */ - -#define REMOTE_UNWIND_LIBUNWIND - -/* Define arch specific functions & regs for libunwind, should be - * defined before including "unwind.h" - */ -#define LIBUNWIND__ARCH_REG_ID(regnum) libunwind__arm64_reg_id(regnum) - -#include "unwind.h" -#include "libunwind-aarch64.h" -#define perf_event_arm_regs perf_event_arm64_regs -#include <../../../arch/arm64/include/uapi/asm/perf_regs.h> -#undef perf_event_arm_regs -#include "../../arch/arm64/util/unwind-libunwind.c" - -/* NO_LIBUNWIND_DEBUG_FRAME is a feature flag for local libunwind, - * assign NO_LIBUNWIND_DEBUG_FRAME_AARCH64 to it for compiling arm64 - * unwind methods. - */ -#undef NO_LIBUNWIND_DEBUG_FRAME -#ifdef NO_LIBUNWIND_DEBUG_FRAME_AARCH64 -#define NO_LIBUNWIND_DEBUG_FRAME -#endif -#include "util/unwind-libunwind-local.c" - -struct unwind_libunwind_ops * -arm64_unwind_libunwind_ops = &_unwind_libunwind_ops; diff --git a/tools/perf/util/libunwind/x86_32.c b/tools/perf/util/libunwind/x86_32.c deleted file mode 100644 index 1697dece1b74..000000000000 --- a/tools/perf/util/libunwind/x86_32.c +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * This file setups defines to compile arch specific binary from the - * generic one. - * - * The function 'LIBUNWIND__ARCH_REG_ID' name is set according to arch - * name and the definition of this function is included directly from - * 'arch/x86/util/unwind-libunwind.c', to make sure that this function - * is defined no matter what arch the host is. - * - * Finally, the arch specific unwind methods are exported which will - * be assigned to each x86 thread. - */ - -#define REMOTE_UNWIND_LIBUNWIND - -/* Define arch specific functions & regs for libunwind, should be - * defined before including "unwind.h" - */ -#define LIBUNWIND__ARCH_REG_ID(regnum) libunwind__x86_reg_id(regnum) - -#include "unwind.h" -#include "libunwind-x86.h" -#include <../../../../arch/x86/include/uapi/asm/perf_regs.h> - -/* HAVE_ARCH_X86_64_SUPPORT is used in'arch/x86/util/unwind-libunwind.c' - * for x86_32, we undef it to compile code for x86_32 only. - */ -#undef HAVE_ARCH_X86_64_SUPPORT -#include "../../arch/x86/util/unwind-libunwind.c" - -/* Explicitly define NO_LIBUNWIND_DEBUG_FRAME, because non-ARM has no - * dwarf_find_debug_frame() function. - */ -#ifndef NO_LIBUNWIND_DEBUG_FRAME -#define NO_LIBUNWIND_DEBUG_FRAME -#endif -#include "util/unwind-libunwind-local.c" - -struct unwind_libunwind_ops * -x86_32_unwind_libunwind_ops = &_unwind_libunwind_ops; diff --git a/tools/perf/util/lock-contention.c b/tools/perf/util/lock-contention.c index 92e7b7b572a2..119a7206f3cd 100644 --- a/tools/perf/util/lock-contention.c +++ b/tools/perf/util/lock-contention.c @@ -104,7 +104,8 @@ bool match_callstack_filter(struct machine *machine, u64 *callstack, int max_sta struct map *kmap; struct symbol *sym; u64 ip; - const char *arch = perf_env__arch(machine->env); + uint16_t e_machine = perf_env__e_machine(machine->env, /*e_flags=*/NULL); + bool is_powerpc = e_machine == EM_PPC64 || e_machine == EM_PPC; if (list_empty(&callstack_filters)) return true; @@ -125,8 +126,7 @@ bool match_callstack_filter(struct machine *machine, u64 *callstack, int max_sta * incase first or second callstack index entry has 0 * address for powerpc. */ - if (!callstack || (!callstack[i] && (strcmp(arch, "powerpc") || - (i != 1 && i != 2)))) + if (!callstack || (!callstack[i] && (!is_powerpc || (i != 1 && i != 2)))) break; ip = callstack[i]; diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index e76f8c86e62a..0d2ebf6a84bc 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -4,10 +4,12 @@ #include <inttypes.h> #include <regex.h> #include <stdlib.h> +#include <string.h> #include "callchain.h" #include "debug.h" #include "dso.h" #include "env.h" +#include "dwarf-regs.h" #include "event.h" #include "evsel.h" #include "hist.h" @@ -77,15 +79,14 @@ int machine__init(struct machine *machine, const char *root_dir, pid_t pid) int err = -ENOMEM; memset(machine, 0, sizeof(*machine)); - machine->kmaps = maps__new(machine); - if (machine->kmaps == NULL) - return -ENOMEM; - RB_CLEAR_NODE(&machine->rb_node); dsos__init(&machine->dsos); - threads__init(&machine->threads); + machine->kmaps = maps__new(machine); + if (machine->kmaps == NULL) + goto out; + machine->vdso_info = NULL; machine->env = NULL; @@ -122,11 +123,11 @@ int machine__init(struct machine *machine, const char *root_dir, pid_t pid) out: if (err) { - zfree(&machine->kmaps); + maps__zput(machine->kmaps); zfree(&machine->root_dir); zfree(&machine->mmap_name); } - return 0; + return err; } static struct machine *__machine__new_host(struct perf_env *host_env, bool kernel_maps) @@ -136,7 +137,10 @@ static struct machine *__machine__new_host(struct perf_env *host_env, bool kerne if (!machine) return NULL; - machine__init(machine, "", HOST_KERNEL_ID); + if (machine__init(machine, "", HOST_KERNEL_ID) != 0) { + free(machine); + return NULL; + } if (kernel_maps && machine__create_kernel_maps(machine) < 0) { free(machine); @@ -229,10 +233,12 @@ void machine__delete(struct machine *machine) } } -void machines__init(struct machines *machines) +int machines__init(struct machines *machines) { - machine__init(&machines->host, "", HOST_KERNEL_ID); + int err = machine__init(&machines->host, "", HOST_KERNEL_ID); + machines->guests = RB_ROOT_CACHED; + return err; } void machines__exit(struct machines *machines) @@ -327,7 +333,7 @@ struct machine *machines__findnew(struct machines *machines, pid_t pid) if ((pid != HOST_KERNEL_ID) && (pid != DEFAULT_GUEST_KERNEL_ID) && (symbol_conf.guestmount)) { - sprintf(path, "%s/%d", symbol_conf.guestmount, pid); + snprintf(path, sizeof(path), "%s/%d", symbol_conf.guestmount, pid); if (access(path, R_OK)) { static struct strlist *seen; @@ -729,9 +735,15 @@ static int machine__process_ksymbol_register(struct machine *machine, { struct symbol *sym; struct dso *dso = NULL; - struct map *map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); + struct map *map; int err = 0; + /* Ignore mapping symbols in ksymbol events - check early before any state mutation */ + if (is_ignored_kernel_symbol(event->ksymbol.name)) + return 0; + + map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); + if (!map) { dso = dso__new(event->ksymbol.name); @@ -790,6 +802,10 @@ static int machine__process_ksymbol_unregister(struct machine *machine, struct symbol *sym; struct map *map; + /* Ignore mapping symbols in ksymbol events */ + if (is_ignored_kernel_symbol(event->ksymbol.name)) + return 0; + map = maps__find(machine__kernel_maps(machine), event->ksymbol.addr); if (!map) return 0; @@ -814,6 +830,11 @@ int machine__process_ksymbol(struct machine *machine __maybe_unused, if (dump_trace) perf_event__fprintf_ksymbol(event, stdout); + if (event->header.size < offsetof(struct perf_record_ksymbol, name) + 2 || + !memchr(event->ksymbol.name, '\0', + event->header.size - offsetof(struct perf_record_ksymbol, name))) + return -EINVAL; + /* no need to process non-JIT BPF as it cannot get samples */ if (event->ksymbol.len == 0) return 0; @@ -1078,7 +1099,7 @@ static u64 find_entry_trampoline(struct dso *dso) unsigned int i; for (; sym; sym = dso__next_symbol(sym)) { - if (sym->binding != STB_GLOBAL) + if (symbol__binding(sym) != STB_GLOBAL) continue; for (i = 0; i < ARRAY_SIZE(syms); i++) { if (!strcmp(sym->name, syms[i])) @@ -1239,9 +1260,9 @@ int machines__create_guest_kernel_maps(struct machines *machines) namelist[i]->d_name); continue; } - sprintf(path, "%s/%s/proc/kallsyms", - symbol_conf.guestmount, - namelist[i]->d_name); + snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", + symbol_conf.guestmount, + namelist[i]->d_name); ret = access(path, R_OK); if (ret) { pr_debug("Can't access file %s\n", path); @@ -1319,7 +1340,7 @@ static char *get_kernel_version(const char *root_dir) char *name, *tmp; const char *prefix = "Linux version "; - sprintf(version, "%s/proc/version", root_dir); + snprintf(version, sizeof(version), "%s/proc/version", root_dir); file = fopen(version, "r"); if (!file) return NULL; @@ -1522,22 +1543,30 @@ static void machine__set_kernel_mmap(struct machine *machine, map__set_end(machine->vmlinux_map, ~0ULL); } -static int machine__update_kernel_mmap(struct machine *machine, - u64 start, u64 end) +struct kernel_mmap_mutation_ctx { + u64 start; + u64 end; +}; + +static int kernel_mmap_mutate_cb(struct map *map, void *data) { - struct map *orig, *updated; - int err; + struct kernel_mmap_mutation_ctx *ctx = data; - orig = machine->vmlinux_map; - updated = map__get(orig); + map__set_start(map, ctx->start); + map__set_end(map, ctx->end); + if (ctx->start == 0 && ctx->end == 0) + map__set_end(map, ~0ULL); + return 0; +} - machine->vmlinux_map = updated; - maps__remove(machine__kernel_maps(machine), orig); - machine__set_kernel_mmap(machine, start, end); - err = maps__insert(machine__kernel_maps(machine), updated); - map__put(orig); +static int machine__update_kernel_mmap(struct machine *machine, + u64 start, u64 end) +{ + struct kernel_mmap_mutation_ctx ctx = { .start = start, .end = end }; - return err; + return maps__mutate_mapping(machine__kernel_maps(machine), + machine->vmlinux_map, + kernel_mmap_mutate_cb, &ctx); } int machine__create_kernel_maps(struct machine *machine) @@ -1611,10 +1640,24 @@ static bool machine__uses_kcore(struct machine *machine) return dsos__for_each_dso(&machine->dsos, machine__uses_kcore_cb, NULL) != 0 ? true : false; } +static bool machine__is(struct machine *machine, uint16_t e_machine) +{ + if (!machine) + return false; + + if (!machine->env) { + if (machine__is_host(machine)) + return e_machine == EM_HOST; + return false; + } + + return perf_env__e_machine(machine->env, NULL) == e_machine; +} + static bool perf_event__is_extra_kernel_mmap(struct machine *machine, struct extra_kernel_map *xm) { - return machine__is(machine, "x86_64") && + return machine__is(machine, EM_X86_64) && is_entry_trampoline(xm->name); } @@ -2770,7 +2813,7 @@ static int find_prev_cpumode(struct ip_callchain *chain, struct thread *thread, static u64 get_leaf_frame_caller(struct perf_sample *sample, struct thread *thread, int usr_idx) { - if (machine__normalized_is(maps__machine(thread__maps(thread)), "arm64")) + if (thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL) == EM_AARCH64) return get_leaf_frame_caller_aarch64(sample, thread, usr_idx); else return 0; @@ -2778,13 +2821,13 @@ static u64 get_leaf_frame_caller(struct perf_sample *sample, static int thread__resolve_callchain_sample(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, struct symbol **parent, struct addr_location *root_al, int max_stack, bool symbols) { + struct evsel *evsel = sample->evsel; struct branch_stack *branch = sample->branch_stack; struct branch_entry *entries = perf_sample__branch_entries(sample); struct ip_callchain *chain = sample->callchain; @@ -2986,10 +3029,11 @@ static int unwind_entry(struct unwind_entry *entry, void *arg) static int thread__resolve_callchain_unwind(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, int max_stack, bool symbols) { + struct evsel *evsel = sample->evsel; + /* Can we do dwarf post unwind? */ if (!((evsel->core.attr.sample_type & PERF_SAMPLE_REGS_USER) && (evsel->core.attr.sample_type & PERF_SAMPLE_STACK_USER))) @@ -3009,7 +3053,6 @@ static int thread__resolve_callchain_unwind(struct thread *thread, int __thread__resolve_callchain(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, struct symbol **parent, struct addr_location *root_al, @@ -3025,22 +3068,22 @@ int __thread__resolve_callchain(struct thread *thread, if (callchain_param.order == ORDER_CALLEE) { ret = thread__resolve_callchain_sample(thread, cursor, - evsel, sample, + sample, parent, root_al, max_stack, symbols); if (ret) return ret; ret = thread__resolve_callchain_unwind(thread, cursor, - evsel, sample, + sample, max_stack, symbols); } else { ret = thread__resolve_callchain_unwind(thread, cursor, - evsel, sample, + sample, max_stack, symbols); if (ret) return ret; ret = thread__resolve_callchain_sample(thread, cursor, - evsel, sample, + sample, parent, root_al, max_stack, symbols); } @@ -3141,20 +3184,6 @@ int machine__set_current_tid(struct machine *machine, int cpu, pid_t pid, return 0; } -/* - * Compares the raw arch string. N.B. see instead perf_env__arch() or - * machine__normalized_is() if a normalized arch is needed. - */ -bool machine__is(struct machine *machine, const char *arch) -{ - return machine && !strcmp(perf_env__raw_arch(machine->env), arch); -} - -bool machine__normalized_is(struct machine *machine, const char *arch) -{ - return machine && !strcmp(perf_env__arch(machine->env), arch); -} - int machine__nr_cpus_avail(struct machine *machine) { return machine ? perf_env__nr_cpus_avail(machine->env) : 0; @@ -3181,7 +3210,7 @@ int machine__get_kernel_start(struct machine *machine) * start of kernel text, but still above 2^63. So leave * kernel_start = 1ULL << 63 for x86_64. */ - if (!err && !machine__is(machine, "x86_64")) + if (!err && !machine__is(machine, EM_X86_64)) machine->kernel_start = map__start(map); } return err; diff --git a/tools/perf/util/machine.h b/tools/perf/util/machine.h index 22a42c5825fa..26f9827062f5 100644 --- a/tools/perf/util/machine.h +++ b/tools/perf/util/machine.h @@ -152,7 +152,7 @@ struct machines { struct rb_root_cached guests; }; -void machines__init(struct machines *machines); +int machines__init(struct machines *machines); void machines__exit(struct machines *machines); void machines__process_guests(struct machines *machines, @@ -187,7 +187,6 @@ struct callchain_cursor; int __thread__resolve_callchain(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, struct symbol **parent, struct addr_location *root_al, @@ -196,7 +195,6 @@ int __thread__resolve_callchain(struct thread *thread, static inline int thread__resolve_callchain(struct thread *thread, struct callchain_cursor *cursor, - struct evsel *evsel, struct perf_sample *sample, struct symbol **parent, struct addr_location *root_al, @@ -204,7 +202,6 @@ static inline int thread__resolve_callchain(struct thread *thread, { return __thread__resolve_callchain(thread, cursor, - evsel, sample, parent, root_al, @@ -227,8 +224,6 @@ static inline bool machine__is_host(struct machine *machine) } bool machine__is_lock_function(struct machine *machine, u64 addr); -bool machine__is(struct machine *machine, const char *arch); -bool machine__normalized_is(struct machine *machine, const char *arch); int machine__nr_cpus_avail(struct machine *machine); struct thread *machine__findnew_thread(struct machine *machine, pid_t pid, pid_t tid); diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index 81a97ac34077..f808df2fe77b 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -40,6 +40,7 @@ DECLARE_RC_STRUCT(maps) { #ifdef HAVE_LIBUNWIND_SUPPORT void *addr_space; const struct unwind_libunwind_ops *unwind_libunwind_ops; + uint16_t e_machine; #endif #ifdef HAVE_LIBDW_SUPPORT void *libdw_addr_space_dwfl; @@ -197,14 +198,14 @@ void maps__set_addr_space(struct maps *maps, void *addr_space) RC_CHK_ACCESS(maps)->addr_space = addr_space; } -const struct unwind_libunwind_ops *maps__unwind_libunwind_ops(const struct maps *maps) +uint16_t maps__e_machine(const struct maps *maps) { - return RC_CHK_ACCESS(maps)->unwind_libunwind_ops; + return RC_CHK_ACCESS(maps)->e_machine; } -void maps__set_unwind_libunwind_ops(struct maps *maps, const struct unwind_libunwind_ops *ops) +void maps__set_e_machine(struct maps *maps, uint16_t e_machine) { - RC_CHK_ACCESS(maps)->unwind_libunwind_ops = ops; + RC_CHK_ACCESS(maps)->e_machine = e_machine; } #endif #ifdef HAVE_LIBDW_SUPPORT @@ -575,6 +576,48 @@ void maps__remove(struct maps *maps, struct map *map) #endif } +/** + * maps__mutate_mapping - Apply write-protected mutations to a map. + * @maps: The maps collection containing the map. + * @map: The map to mutate. + * @mutate_cb: Callback function that performs the actual mutations. + * @data: Private data passed to the callback. + * + * This acquires the write lock on the maps semaphore to safely protect + * concurrent readers from seeing partially mutated or unsorted map boundaries. + * + * WARNING: Acquiring down_write() here can trigger a recursive self-deadlock if + * the caller already holds the read lock (e.g., during maps__for_each_map() or + * maps__find() iteration paths that trigger lazy symbol loading). To completely + * avoid this deadlock, all kernel/module maps must be pre-loaded up-front (via + * maps__load_maps()) under a clean, single-threaded context before entering + * multi-threaded event processing loops. + */ +int maps__mutate_mapping(struct maps *maps, struct map *map, + int (*mutate_cb)(struct map *map, void *data), void *data) +{ + int err = 0; + + if (maps) { + down_write(maps__lock(maps)); + + err = mutate_cb(map, data); + + RC_CHK_ACCESS(maps)->maps_by_address_sorted = false; + RC_CHK_ACCESS(maps)->maps_by_name_sorted = false; + + up_write(maps__lock(maps)); + +#ifdef HAVE_LIBDW_SUPPORT + libdw__invalidate_dwfl(maps, maps__libdw_addr_space_dwfl(maps)); +#endif + } else { + err = mutate_cb(map, data); + } + + return err; +} + bool maps__empty(struct maps *maps) { bool res; @@ -625,6 +668,41 @@ int maps__for_each_map(struct maps *maps, int (*cb)(struct map *map, void *data) return ret; } +int maps__load_maps(struct maps *maps) +{ + struct map **maps_copy; + unsigned int nr_maps; + int err = 0; + + if (!maps) + return 0; + + down_read(maps__lock(maps)); + nr_maps = maps__nr_maps(maps); + if (nr_maps == 0) { + up_read(maps__lock(maps)); + return 0; + } + maps_copy = calloc(nr_maps, sizeof(*maps_copy)); + if (!maps_copy) { + up_read(maps__lock(maps)); + return -ENOMEM; + } + for (unsigned int i = 0; i < nr_maps; i++) + maps_copy[i] = map__get(maps__maps_by_address(maps)[i]); + up_read(maps__lock(maps)); + + for (unsigned int i = 0; i < nr_maps; i++) { + if (map__load(maps_copy[i]) < 0) { + pr_warning("Failed to load map %s\n", dso__name(map__dso(maps_copy[i]))); + err = -1; + } + map__put(maps_copy[i]); + } + free(maps_copy); + return err; +} + void maps__remove_maps(struct maps *maps, bool (*cb)(struct map *map, void *data), void *data) { struct map **maps_by_address; @@ -667,40 +745,57 @@ struct symbol *maps__find_symbol(struct maps *maps, u64 addr, struct map **mapp) return result; } -struct maps__find_symbol_by_name_args { - struct map **mapp; - const char *name; - struct symbol *sym; -}; - -static int maps__find_symbol_by_name_cb(struct map *map, void *data) +struct symbol *maps__find_symbol_by_name(struct maps *maps, const char *name, struct map **mapp) { - struct maps__find_symbol_by_name_args *args = data; + struct map **maps_copy; + unsigned int nr_maps; + struct symbol *sym = NULL; - args->sym = map__find_symbol_by_name(map, args->name); - if (!args->sym) - return 0; + if (!maps) + return NULL; - if (!map__contains_symbol(map, args->sym)) { - args->sym = NULL; - return 0; + /* + * First, ensure all maps are loaded. We pre-load them outside of any + * read-to-write locks to avoid deadlocks. Even if some fail, we proceed. + */ + maps__load_maps(maps); + + /* + * Create a local snapshot of the maps while holding the read lock. + * This prevents deadlocking if iteration triggers further map insertions. + */ + down_read(maps__lock(maps)); + nr_maps = maps__nr_maps(maps); + maps_copy = calloc(nr_maps, sizeof(*maps_copy)); + if (maps_copy) { + for (unsigned int i = 0; i < nr_maps; i++) { + struct map *map = maps__maps_by_address(maps)[i]; + + maps_copy[i] = map__get(map); + } } + up_read(maps__lock(maps)); - if (args->mapp != NULL) - *args->mapp = map__get(map); - return 1; -} + if (!maps_copy) + return NULL; -struct symbol *maps__find_symbol_by_name(struct maps *maps, const char *name, struct map **mapp) -{ - struct maps__find_symbol_by_name_args args = { - .mapp = mapp, - .name = name, - .sym = NULL, - }; + for (unsigned int i = 0; i < nr_maps; i++) { + struct map *map = maps_copy[i]; + + sym = map__find_symbol_by_name(map, name); + if (sym && map__contains_symbol(map, sym)) { + if (mapp) + *mapp = map__get(map); + break; + } + sym = NULL; + } + + for (unsigned int i = 0; i < nr_maps; i++) + map__put(maps_copy[i]); - maps__for_each_map(maps, maps__find_symbol_by_name_cb, &args); - return args.sym; + free(maps_copy); + return sym; } int maps__find_ams(struct maps *maps, struct addr_map_symbol *ams) @@ -1039,6 +1134,14 @@ int maps__copy_from(struct maps *dest, struct maps *parent) down_write(maps__lock(dest)); down_read(maps__lock(parent)); +#ifdef HAVE_LIBUNWIND_SUPPORT + err = unwind__prepare_access(dest, maps__e_machine(parent)); + if (err) { + up_read(maps__lock(parent)); + up_write(maps__lock(dest)); + return err; + } +#endif parent_maps_by_address = maps__maps_by_address(parent); n = maps__nr_maps(parent); if (maps__nr_maps(dest) == 0) { @@ -1068,14 +1171,11 @@ int maps__copy_from(struct maps *dest, struct maps *parent) if (!new) err = -ENOMEM; else { - err = unwind__prepare_access(dest, new, NULL); - if (!err) { - dest_maps_by_address[i] = new; - map__set_kmap_maps(new, dest); - if (dest_maps_by_name) - dest_maps_by_name[i] = map__get(new); - RC_CHK_ACCESS(dest)->nr_maps = i + 1; - } + dest_maps_by_address[i] = new; + map__set_kmap_maps(new, dest); + if (dest_maps_by_name) + dest_maps_by_name[i] = map__get(new); + RC_CHK_ACCESS(dest)->nr_maps = i + 1; } if (err) map__put(new); @@ -1093,9 +1193,7 @@ int maps__copy_from(struct maps *dest, struct maps *parent) if (!new) err = -ENOMEM; else { - err = unwind__prepare_access(dest, new, NULL); - if (!err) - err = __maps__insert(dest, new); + err = __maps__insert(dest, new); } map__put(new); } diff --git a/tools/perf/util/maps.h b/tools/perf/util/maps.h index 20c52084ba9e..4ec9b7453a3b 100644 --- a/tools/perf/util/maps.h +++ b/tools/perf/util/maps.h @@ -49,8 +49,8 @@ refcount_t *maps__refcnt(struct maps *maps); /* Test only. */ #ifdef HAVE_LIBUNWIND_SUPPORT void *maps__addr_space(const struct maps *maps); void maps__set_addr_space(struct maps *maps, void *addr_space); -const struct unwind_libunwind_ops *maps__unwind_libunwind_ops(const struct maps *maps); -void maps__set_unwind_libunwind_ops(struct maps *maps, const struct unwind_libunwind_ops *ops); +uint16_t maps__e_machine(const struct maps *maps); +void maps__set_e_machine(struct maps *maps, uint16_t e_machine); #endif #ifdef HAVE_LIBDW_SUPPORT void *maps__libdw_addr_space_dwfl(const struct maps *maps); @@ -59,8 +59,11 @@ void maps__set_libdw_addr_space_dwfl(struct maps *maps, void *dwfl); size_t maps__fprintf(struct maps *maps, FILE *fp); +int maps__load_maps(struct maps *maps); int maps__insert(struct maps *maps, struct map *map); void maps__remove(struct maps *maps, struct map *map); +int maps__mutate_mapping(struct maps *maps, struct map *map, + int (*mutate_cb)(struct map *map, void *data), void *data); struct map *maps__find(struct maps *maps, u64 addr); struct symbol *maps__find_symbol(struct maps *maps, u64 addr, struct map **mapp); diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index 4db9578efd81..c2ce3e53aaee 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -295,7 +295,8 @@ static int setup_metric_events(const char *pmu, struct hashmap *ids, const char *metric_id; struct evsel *ev; size_t ids_size, matched_events, i; - bool all_pmus = !strcmp(pmu, "all") || perf_pmus__num_core_pmus() == 1 || !is_pmu_core(pmu); + bool all_pmus = !strcmp(pmu, "all") || !strcmp(pmu, "default_core") || + perf_pmus__num_core_pmus() == 1 || !is_pmu_core(pmu); *out_metric_events = NULL; ids_size = hashmap__size(ids); @@ -415,14 +416,9 @@ static int metricgroup__sys_event_iter(const struct pmu_metric *pm, if (!pm->metric_expr || !pm->compat) return 0; - while ((pmu = perf_pmus__scan(pmu))) { - - if (!pmu->id || !pmu_uncore_identifier_match(pm->compat, pmu->id)) - continue; - - return d->fn(pm, table, d->data); - } - return 0; + /* Only process with the iterator if there is a a PMU that matches the ID. */ + pmu = perf_pmus__scan_for_uncore_id(pmu, pm->compat); + return pmu ? d->fn(pm, table, d->data) : 0; } int metricgroup__for_each_metric(const struct pmu_metrics_table *table, pmu_metric_iter_fn fn, diff --git a/tools/perf/util/mmap.c b/tools/perf/util/mmap.c index b69f926d314b..358e70c4f3ed 100644 --- a/tools/perf/util/mmap.c +++ b/tools/perf/util/mmap.c @@ -89,10 +89,10 @@ static int perf_mmap__aio_alloc(struct mmap *map, int idx) static void perf_mmap__aio_free(struct mmap *map, int idx) { - if (map->aio.data[idx]) { - munmap(map->aio.data[idx], mmap__mmap_len(map)); - map->aio.data[idx] = NULL; - } + if (!map->aio.data || !map->aio.data[idx]) + return; + munmap(map->aio.data[idx], mmap__mmap_len(map)); + map->aio.data[idx] = NULL; } static int perf_mmap__aio_bind(struct mmap *map, int idx, struct perf_cpu cpu, int affinity) @@ -104,9 +104,15 @@ static int perf_mmap__aio_bind(struct mmap *map, int idx, struct perf_cpu cpu, i int err = 0; if (affinity != PERF_AFFINITY_SYS && cpu__max_node() > 1) { + int node; + data = map->aio.data[idx]; mmap_len = mmap__mmap_len(map); - node_index = cpu__get_node(cpu); + node = cpu__get_node(cpu); + /* -1 sign-extends to ULONG_MAX, wrapping bitmap_zalloc(0) and OOB __set_bit */ + if (node < 0) + return 0; + node_index = node; node_mask = bitmap_zalloc(node_index + 1); if (!node_mask) { pr_err("Failed to allocate node mask for mbind: error %m\n"); @@ -135,6 +141,8 @@ static int perf_mmap__aio_alloc(struct mmap *map, int idx) static void perf_mmap__aio_free(struct mmap *map, int idx) { + if (!map->aio.data) + return; zfree(&(map->aio.data[idx])); } @@ -230,6 +238,8 @@ static void perf_mmap__aio_munmap(struct mmap *map __maybe_unused) void mmap__munmap(struct mmap *map) { bitmap_free(map->affinity_mask.bits); + map->affinity_mask.bits = NULL; + map->affinity_mask.nbits = 0; zstd_fini(&map->zstd_data); diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 1497e1f2a08c..943569e82b82 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -2251,6 +2251,33 @@ static int parse_events__sort_events_and_fix_groups(struct list_head *list) } last_event_was_forced_leader = (force_grouped_leader == pos); } + + /* + * Make sure the first wildcard match is the earliest entry in the list. + * Since list_sort might have reordered the aliases, the original leader + * might not be at the head of the list anymore. We find the first + * alias in the sorted list and make it the new leader, and redirect + * all other aliases to it. + */ + list_for_each_entry(pos, list, core.node) { + struct evsel *orig_leader = pos->first_wildcard_match; + + if (!orig_leader) + continue; + + if (orig_leader->first_wildcard_match) { + /* Original leader was redirected to a new leader */ + pos->first_wildcard_match = orig_leader->first_wildcard_match; + } else if (pos->core.idx < orig_leader->core.idx) { + /* + * We are earlier than the original leader in sorted order, + * and no earlier alias has claimed leadership yet. + */ + orig_leader->first_wildcard_match = pos; + pos->first_wildcard_match = NULL; + } + } + list_for_each_entry(pos, list, core.node) { struct evsel *pos_leader = evsel__leader(pos); diff --git a/tools/perf/util/perf-regs-arch/perf_regs_riscv.c b/tools/perf/util/perf-regs-arch/perf_regs_riscv.c index 5b5f21fcba8c..bf769304c97c 100644 --- a/tools/perf/util/perf-regs-arch/perf_regs_riscv.c +++ b/tools/perf/util/perf-regs-arch/perf_regs_riscv.c @@ -1,8 +1,136 @@ // SPDX-License-Identifier: GPL-2.0 +#include <errno.h> +#include <regex.h> +#include <string.h> +#include <linux/kernel.h> +#include <linux/zalloc.h> + +#include "../debug.h" #include "../perf_regs.h" #include "../../arch/riscv/include/perf_regs.h" +/* + * RISC-V SDT argument formats (GCC 'nor' constraint): + * + * Register: REG e.g. a0, t1, s0, sp + * Memory: NUM(REG) e.g. 8(a0), -20(s0) + * Constant: NUM e.g. 99 (not supported by uprobe, skip) + * + * Note: 'zero' (x0) is hardwired to 0 and not in pt_regs; skip it. + * + * Uprobe target format: + * Register: %REG e.g. %a0 + * Memory: +NUM(%REG) or -NUM(%REG) + */ + +/* RISC-V register ABI names: ra, sp, gp, tp, t0-t6, s0-s11, a0-a7 */ +#define SDT_OP_REGEX1 "^(ra|sp|gp|tp|t[0-6]|s[0-9]|s1[01]|a[0-7])$" + +/* RISC-V memory operand: [-]NUM(REG) */ +#define SDT_OP_REGEX2 "^(\\-)?([0-9]+)\\((ra|sp|gp|tp|t[0-6]|s[0-9]|s1[01]|a[0-7])\\)$" + +static regex_t sdt_op_regex1, sdt_op_regex2; + +static int sdt_init_op_regex(void) +{ + static int initialized; + int ret = 0; + + if (initialized) + return 0; + + ret = regcomp(&sdt_op_regex1, SDT_OP_REGEX1, REG_EXTENDED); + if (ret) + goto error; + + ret = regcomp(&sdt_op_regex2, SDT_OP_REGEX2, REG_EXTENDED); + if (ret) + goto free_regex1; + + initialized = 1; + return 0; + +free_regex1: + regfree(&sdt_op_regex1); +error: + pr_debug4("Regex compilation error.\n"); + return -ret; +} + +/* + * Parse OP and convert it into uprobe format. + * Possible variants of OP (RISC-V, GCC 'nor' constraint): + * + * Format Example Uprobe + * ---------------------------------------- + * REG a0 %a0 + * NUM(REG) 8(a0) +8(%a0) + * -NUM(REG) -20(s0) -20(%s0) + * NUM 99 (skip, constant not supported) + */ +int __perf_sdt_arg_parse_op_riscv(char *old_op, char **new_op) +{ + int ret, new_len; + regmatch_t rm[4]; + char prefix; + + /* + * Constant argument: pure integer with no trailing '(' (e.g. "99", "-1"). + * uprobe does not support immediate values, so skip them. + * Memory operands like "8(a0)" or "-20(s0)" contain '(' so are NOT + * treated as constants here; they will be matched by REGEX2 below. + */ + if (strchr(old_op, '(') == NULL && + ((*old_op >= '0' && *old_op <= '9') || + (*old_op == '-' && old_op[1] >= '0' && old_op[1] <= '9'))) { + pr_debug4("Skipping unsupported SDT argument: %s\n", old_op); + return SDT_ARG_SKIP; + } + + ret = sdt_init_op_regex(); + if (ret < 0) + return ret; + + if (!regexec(&sdt_op_regex1, old_op, 2, rm, 0)) { + /* REG --> %REG */ + new_len = 2; /* % NULL */ + new_len += (int)(rm[1].rm_eo - rm[1].rm_so); + + *new_op = zalloc(new_len); + if (!*new_op) + return -ENOMEM; + + scnprintf(*new_op, new_len, "%%%.*s", + (int)(rm[1].rm_eo - rm[1].rm_so), old_op + rm[1].rm_so); + } else if (!regexec(&sdt_op_regex2, old_op, 4, rm, 0)) { + /* + * NUM(REG) or -NUM(REG) --> +NUM(%REG) or -NUM(%REG) + * rm[1]: optional '-' + * rm[2]: decimal offset + * rm[3]: register name + */ + prefix = (rm[1].rm_so == -1) ? '+' : '-'; + + new_len = 5; /* sign ( % ) NULL */ + new_len += (int)(rm[2].rm_eo - rm[2].rm_so); + new_len += (int)(rm[3].rm_eo - rm[3].rm_so); + + *new_op = zalloc(new_len); + if (!*new_op) + return -ENOMEM; + + scnprintf(*new_op, new_len, "%c%.*s(%%%.*s)", prefix, + (int)(rm[2].rm_eo - rm[2].rm_so), old_op + rm[2].rm_so, + (int)(rm[3].rm_eo - rm[3].rm_so), old_op + rm[3].rm_so); + } else { + pr_debug4("Skipping unsupported SDT argument: %s\n", old_op); + return SDT_ARG_SKIP; + } + + return SDT_ARG_VALID; +} + uint64_t __perf_reg_mask_riscv(bool intr __maybe_unused) { return PERF_REGS_MASK; diff --git a/tools/perf/util/perf_event_attr_fprintf.c b/tools/perf/util/perf_event_attr_fprintf.c index 741c3d657a8b..3933639d76c5 100644 --- a/tools/perf/util/perf_event_attr_fprintf.c +++ b/tools/perf/util/perf_event_attr_fprintf.c @@ -275,24 +275,56 @@ static void __p_config_id(struct perf_pmu *pmu, char *buf, size_t size, u32 type #define p_type_id(val) __p_type_id(buf, BUF_SIZE, pmu, val) #define p_config_id(val) __p_config_id(pmu, buf, BUF_SIZE, attr->type, val) -#define PRINT_ATTRn(_n, _f, _p, _a) \ -do { \ - if (_a || attr->_f) { \ - _p(attr->_f); \ - ret += attr__fprintf(fp, _n, buf, priv);\ - } \ +#define PRINT_ATTRn(_n, _f, _p, _a) \ +do { \ + if (attr_size >= offsetof(struct perf_event_attr, _f) + \ + sizeof(attr->_f) && \ + (_a || attr->_f)) { \ + _p(attr->_f); \ + ret += attr__fprintf(fp, _n, buf, priv); \ + } \ +} while (0) + +/* bitfield members share an offset; most are within PERF_ATTR_SIZE_VER0 */ +#define PRINT_ATTRn_bf(_n, _f, _p, _a) \ +do { \ + if (_a || attr->_f) { \ + _p(attr->_f); \ + ret += attr__fprintf(fp, _n, buf, priv); \ + } \ } while (0) #define PRINT_ATTRf(_f, _p) PRINT_ATTRn(#_f, _f, _p, false) +#define PRINT_ATTRf_bf(_f, _p) PRINT_ATTRn_bf(#_f, _f, _p, false) int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr, attr__fprintf_f attr__fprintf, void *priv) { struct perf_pmu *pmu = perf_pmus__find_by_type(attr->type); + /* + * size == 0 means ABI0 — the producer didn't set attr.size. + * perf_event__fprintf_attr() may pass the raw mmap'd event + * before the local copy, so default to PERF_ATTR_SIZE_VER0 + * (the ABI0 footprint) to avoid reading past the attr into + * the ID array that follows it in HEADER_ATTR events. + */ + u32 attr_size = attr->size ?: PERF_ATTR_SIZE_VER0; char buf[BUF_SIZE]; int ret = 0; - if (!pmu && (attr->type == PERF_TYPE_HARDWARE || attr->type == PERF_TYPE_HW_CACHE)) { + /* + * Cap to what we understand: all callers store the attr in a + * buffer of sizeof(*attr) bytes (perf.data read path copies + * min(attr.size, sizeof), BPF augmented path copies into a + * fixed-size value[] array). A spoofed attr->size larger + * than sizeof would cause PRINT_ATTRn to read past the + * actual buffer. + */ + if (attr_size > sizeof(*attr)) + attr_size = sizeof(*attr); + + if (!pmu && attr_size >= offsetof(struct perf_event_attr, config) + sizeof(attr->config) && + (attr->type == PERF_TYPE_HARDWARE || attr->type == PERF_TYPE_HW_CACHE)) { u32 extended_type = attr->config >> PERF_PMU_TYPE_SHIFT; if (extended_type) @@ -306,45 +338,53 @@ int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr, PRINT_ATTRf(sample_type, p_sample_type); PRINT_ATTRf(read_format, p_read_format); - PRINT_ATTRf(disabled, p_unsigned); - PRINT_ATTRf(inherit, p_unsigned); - PRINT_ATTRf(pinned, p_unsigned); - PRINT_ATTRf(exclusive, p_unsigned); - PRINT_ATTRf(exclude_user, p_unsigned); - PRINT_ATTRf(exclude_kernel, p_unsigned); - PRINT_ATTRf(exclude_hv, p_unsigned); - PRINT_ATTRf(exclude_idle, p_unsigned); - PRINT_ATTRf(mmap, p_unsigned); - PRINT_ATTRf(comm, p_unsigned); - PRINT_ATTRf(freq, p_unsigned); - PRINT_ATTRf(inherit_stat, p_unsigned); - PRINT_ATTRf(enable_on_exec, p_unsigned); - PRINT_ATTRf(task, p_unsigned); - PRINT_ATTRf(watermark, p_unsigned); - PRINT_ATTRf(precise_ip, p_unsigned); - PRINT_ATTRf(mmap_data, p_unsigned); - PRINT_ATTRf(sample_id_all, p_unsigned); - PRINT_ATTRf(exclude_host, p_unsigned); - PRINT_ATTRf(exclude_guest, p_unsigned); - PRINT_ATTRf(exclude_callchain_kernel, p_unsigned); - PRINT_ATTRf(exclude_callchain_user, p_unsigned); - PRINT_ATTRf(mmap2, p_unsigned); - PRINT_ATTRf(comm_exec, p_unsigned); - PRINT_ATTRf(use_clockid, p_unsigned); - PRINT_ATTRf(context_switch, p_unsigned); - PRINT_ATTRf(write_backward, p_unsigned); - PRINT_ATTRf(namespaces, p_unsigned); - PRINT_ATTRf(ksymbol, p_unsigned); - PRINT_ATTRf(bpf_event, p_unsigned); - PRINT_ATTRf(aux_output, p_unsigned); - PRINT_ATTRf(cgroup, p_unsigned); - PRINT_ATTRf(text_poke, p_unsigned); - PRINT_ATTRf(build_id, p_unsigned); - PRINT_ATTRf(inherit_thread, p_unsigned); - PRINT_ATTRf(remove_on_exec, p_unsigned); - PRINT_ATTRf(sigtrap, p_unsigned); - PRINT_ATTRf(defer_callchain, p_unsigned); - PRINT_ATTRf(defer_output, p_unsigned); + /* + * All bitfields share a single __u64 right after read_format. + * BPF-captured attrs from perf trace may have a small size + * when the tracee passes a minimal struct, so skip the + * entire block when it's not covered. + */ + if (attr_size >= offsetof(struct perf_event_attr, wakeup_events)) { + PRINT_ATTRf_bf(disabled, p_unsigned); + PRINT_ATTRf_bf(inherit, p_unsigned); + PRINT_ATTRf_bf(pinned, p_unsigned); + PRINT_ATTRf_bf(exclusive, p_unsigned); + PRINT_ATTRf_bf(exclude_user, p_unsigned); + PRINT_ATTRf_bf(exclude_kernel, p_unsigned); + PRINT_ATTRf_bf(exclude_hv, p_unsigned); + PRINT_ATTRf_bf(exclude_idle, p_unsigned); + PRINT_ATTRf_bf(mmap, p_unsigned); + PRINT_ATTRf_bf(comm, p_unsigned); + PRINT_ATTRf_bf(freq, p_unsigned); + PRINT_ATTRf_bf(inherit_stat, p_unsigned); + PRINT_ATTRf_bf(enable_on_exec, p_unsigned); + PRINT_ATTRf_bf(task, p_unsigned); + PRINT_ATTRf_bf(watermark, p_unsigned); + PRINT_ATTRf_bf(precise_ip, p_unsigned); + PRINT_ATTRf_bf(mmap_data, p_unsigned); + PRINT_ATTRf_bf(sample_id_all, p_unsigned); + PRINT_ATTRf_bf(exclude_host, p_unsigned); + PRINT_ATTRf_bf(exclude_guest, p_unsigned); + PRINT_ATTRf_bf(exclude_callchain_kernel, p_unsigned); + PRINT_ATTRf_bf(exclude_callchain_user, p_unsigned); + PRINT_ATTRf_bf(mmap2, p_unsigned); + PRINT_ATTRf_bf(comm_exec, p_unsigned); + PRINT_ATTRf_bf(use_clockid, p_unsigned); + PRINT_ATTRf_bf(context_switch, p_unsigned); + PRINT_ATTRf_bf(write_backward, p_unsigned); + PRINT_ATTRf_bf(namespaces, p_unsigned); + PRINT_ATTRf_bf(ksymbol, p_unsigned); + PRINT_ATTRf_bf(bpf_event, p_unsigned); + PRINT_ATTRf_bf(aux_output, p_unsigned); + PRINT_ATTRf_bf(cgroup, p_unsigned); + PRINT_ATTRf_bf(text_poke, p_unsigned); + PRINT_ATTRf_bf(build_id, p_unsigned); + PRINT_ATTRf_bf(inherit_thread, p_unsigned); + PRINT_ATTRf_bf(remove_on_exec, p_unsigned); + PRINT_ATTRf_bf(sigtrap, p_unsigned); + PRINT_ATTRf_bf(defer_callchain, p_unsigned); + PRINT_ATTRf_bf(defer_output, p_unsigned); + } PRINT_ATTRn("{ wakeup_events, wakeup_watermark }", wakeup_events, p_unsigned, false); PRINT_ATTRf(bp_type, p_unsigned); @@ -359,9 +399,12 @@ int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr, PRINT_ATTRf(sample_max_stack, p_unsigned); PRINT_ATTRf(aux_sample_size, p_unsigned); PRINT_ATTRf(sig_data, p_unsigned); - PRINT_ATTRf(aux_start_paused, p_unsigned); - PRINT_ATTRf(aux_pause, p_unsigned); - PRINT_ATTRf(aux_resume, p_unsigned); + /* aux_{start_paused,pause,resume} are at byte 116, past VER0 */ + if (attr_size >= offsetof(struct perf_event_attr, sig_data)) { + PRINT_ATTRf_bf(aux_start_paused, p_unsigned); + PRINT_ATTRf_bf(aux_pause, p_unsigned); + PRINT_ATTRf_bf(aux_resume, p_unsigned); + } return ret; } diff --git a/tools/perf/util/perf_regs.c b/tools/perf/util/perf_regs.c index f52b0e1f7fc7..558c143abbab 100644 --- a/tools/perf/util/perf_regs.c +++ b/tools/perf/util/perf_regs.c @@ -19,6 +19,9 @@ int perf_sdt_arg_parse_op(uint16_t e_machine, char *old_op, char **new_op) case EM_PPC64: ret = __perf_sdt_arg_parse_op_powerpc(old_op, new_op); break; + case EM_RISCV: + ret = __perf_sdt_arg_parse_op_riscv(old_op, new_op); + break; case EM_386: case EM_X86_64: ret = __perf_sdt_arg_parse_op_x86(old_op, new_op); diff --git a/tools/perf/util/perf_regs.h b/tools/perf/util/perf_regs.h index 573f0d1dfe04..79be2b791509 100644 --- a/tools/perf/util/perf_regs.h +++ b/tools/perf/util/perf_regs.h @@ -53,6 +53,7 @@ const char *__perf_reg_name_powerpc(int id); uint64_t __perf_reg_ip_powerpc(void); uint64_t __perf_reg_sp_powerpc(void); +int __perf_sdt_arg_parse_op_riscv(char *old_op, char **new_op); uint64_t __perf_reg_mask_riscv(bool intr); const char *__perf_reg_name_riscv(int id); uint64_t __perf_reg_ip_riscv(void); diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 23337d2fa281..a550f030b85d 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -314,7 +314,7 @@ static int perf_pmu__parse_scale(struct perf_pmu *pmu, struct perf_pmu_alias *al goto error; sret = read(fd, scale, sizeof(scale)-1); - if (sret < 0) + if (sret <= 0) goto error; if (scale[sret - 1] == '\n') @@ -346,7 +346,7 @@ static int perf_pmu__parse_unit(struct perf_pmu *pmu, struct perf_pmu_alias *ali return -1; sret = read(fd, alias->unit, UNIT_MAX_LEN); - if (sret < 0) + if (sret <= 0) goto error; close(fd); @@ -865,6 +865,12 @@ static char *pmu_id(const char *name) if (filename__read_str(path, &str, &len) < 0) return NULL; + /* empty identifier file — nothing useful */ + if (len == 0) { + free(str); + return NULL; + } + str[len - 1] = 0; /* remove line feed */ return str; @@ -2029,9 +2035,26 @@ int perf_pmu__for_each_format(struct perf_pmu *pmu, void *state, pmu_format_call return 0; } +/** + * is_pmu_core() - Check if the given PMU name corresponds to a core CPU PMU. + * @name: The PMU name to check. + * + * Core PMUs can be identified by: + * 1. Exact name match: + * - "cpu": Typically used on x86 architectures. + * - "cpum_cf": Typically used on s390 architectures (CPU Measurement Counter Facility). + * - "default_core": A generic name used to refer to the default core PMU. + * 2. Sysfs file existence check (is_sysfs_pmu_core): + * - Typically used on ARM systems or Intel hybrid architectures (e.g., "cpu_atom", + * "cpu_core"). This approach checks if the sysfs directory for the PMU + * contains a "cpus" file. + */ bool is_pmu_core(const char *name) { - return !strcmp(name, "cpu") || !strcmp(name, "cpum_cf") || is_sysfs_pmu_core(name); + return !strcmp(name, "cpu") || + !strcmp(name, "cpum_cf") || + !strcmp(name, "default_core") || + is_sysfs_pmu_core(name); } bool perf_pmu__supports_legacy_cache(const struct perf_pmu *pmu) @@ -2117,7 +2140,7 @@ static char *format_alias(char *buf, int len, const struct perf_pmu *pmu, skip_duplicate_pmus); /* Paramemterized events have the parameters shown. */ - if (strstr(alias->terms, "=?")) { + if (!strstr(alias->terms, "=?")) { /* No parameters. */ snprintf(buf, len, "%.*s/%s/", (int)pmu_name_len, pmu->name, alias->name); return buf; @@ -2129,15 +2152,19 @@ static char *format_alias(char *buf, int len, const struct perf_pmu *pmu, pr_err("Failure to parse '%s' terms '%s': %d\n", alias->name, alias->terms, ret); parse_events_terms__exit(&terms); - snprintf(buf, len, "%.*s/%s/", (int)pmu_name_len, pmu->name, alias->name); + scnprintf(buf, len, "%.*s/%s/", (int)pmu_name_len, pmu->name, alias->name); return buf; } - used = snprintf(buf, len, "%.*s/%s", (int)pmu_name_len, pmu->name, alias->name); + used = scnprintf(buf, len, "%.*s/%s", (int)pmu_name_len, pmu->name, alias->name); list_for_each_entry(term, &terms.terms, list) { + const char *name = term->config; + + if (!name) + name = parse_events__term_type_str(term->type_term); if (term->type_val == PARSE_EVENTS__TERM_TYPE_STR) - used += snprintf(buf + used, sub_non_neg(len, used), - ",%s=%s", term->config, + used += scnprintf(buf + used, sub_non_neg(len, used), + ",%s=%s", name, term->val.str); } parse_events_terms__exit(&terms); @@ -2201,6 +2228,7 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, int ret = 0; struct hashmap_entry *entry; size_t bkt; + size_t size_rem; if (perf_pmu__is_tracepoint(pmu)) return tp_pmu__for_each_event(pmu, state, cb); @@ -2234,17 +2262,30 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, } buf_used = strlen(buf) + 1; } + info.scale_unit = NULL; if (strlen(event->unit) || event->scale != 1.0) { - info.scale_unit = buf + buf_used; - buf_used += snprintf(buf + buf_used, sizeof(buf) - buf_used, - "%G%s", event->scale, event->unit) + 1; + /* Check the remaining space */ + size_rem = sub_non_neg(sizeof(buf), buf_used); + + if (size_rem > 0) { + info.scale_unit = buf + buf_used; + buf_used += scnprintf(buf + buf_used, size_rem, "%G%s", + event->scale, event->unit) + 1; + } } info.desc = event->desc; info.long_desc = event->long_desc; - info.encoding_desc = buf + buf_used; - buf_used += snprintf(buf + buf_used, sizeof(buf) - buf_used, - "%.*s/%s/", (int)pmu_name_len, info.pmu_name, event->terms) + 1; + info.encoding_desc = NULL; + + /* Check the remaining space */ + size_rem = sub_non_neg(sizeof(buf), buf_used); + if (size_rem > 0) { + info.encoding_desc = buf + buf_used; + buf_used += scnprintf(buf + buf_used, size_rem, "%.*s/%s/", + (int)pmu_name_len, info.pmu_name, event->terms) + 1; + } + info.str = event->terms; info.topic = event->topic; info.deprecated = perf_pmu_alias__check_deprecated(pmu, event); @@ -2254,7 +2295,7 @@ int perf_pmu__for_each_event(struct perf_pmu *pmu, bool skip_duplicate_pmus, } if (pmu->selectable) { info.name = buf; - snprintf(buf, sizeof(buf), "%s//", pmu->name); + scnprintf(buf, sizeof(buf), "%s//", pmu->name); info.alias = NULL; info.scale_unit = NULL; info.desc = NULL; diff --git a/tools/perf/util/pmus.c b/tools/perf/util/pmus.c index 9a2023ceeefd..5e3f571450fe 100644 --- a/tools/perf/util/pmus.c +++ b/tools/perf/util/pmus.c @@ -409,7 +409,7 @@ struct perf_pmu *perf_pmus__scan_matching_wildcard(struct perf_pmu *pmu, const c if (!pmu) { /* * Core PMUs, other sysfs PMUs and tool PMU can have any name or - * aren't wother optimizing for. + * aren't worth optimizing for. */ unsigned int to_read_pmus = PERF_TOOL_PMU_TYPE_PE_CORE_MASK | PERF_TOOL_PMU_TYPE_PE_OTHER_MASK | @@ -486,6 +486,22 @@ static struct perf_pmu *perf_pmus__scan_skip_duplicates(struct perf_pmu *pmu) return NULL; } +struct perf_pmu *perf_pmus__scan_for_uncore_id(struct perf_pmu *pmu, const char *compat) +{ + if (!pmu) { + /* Only uncore PMUs can have identifiers. */ + unsigned int to_read_pmus = PERF_TOOL_PMU_TYPE_PE_OTHER_MASK; + + pmu_read_sysfs(to_read_pmus); + pmu = list_prepare_entry(pmu, &other_pmus, list); + } + list_for_each_entry_continue(pmu, &other_pmus, list) { + if (pmu->id && pmu_uncore_identifier_match(compat, pmu->id)) + return pmu; + } + return NULL; +} + const struct perf_pmu *perf_pmus__pmu_for_pmu_filter(const char *str) { struct perf_pmu *pmu = NULL; diff --git a/tools/perf/util/pmus.h b/tools/perf/util/pmus.h index 7cb36863711a..0d55edb3f2fc 100644 --- a/tools/perf/util/pmus.h +++ b/tools/perf/util/pmus.h @@ -23,6 +23,7 @@ struct perf_pmu *perf_pmus__scan(struct perf_pmu *pmu); struct perf_pmu *perf_pmus__scan_core(struct perf_pmu *pmu); struct perf_pmu *perf_pmus__scan_for_event(struct perf_pmu *pmu, const char *event); struct perf_pmu *perf_pmus__scan_matching_wildcard(struct perf_pmu *pmu, const char *wildcard); +struct perf_pmu *perf_pmus__scan_for_uncore_id(struct perf_pmu *pmu, const char *compat); const struct perf_pmu *perf_pmus__pmu_for_pmu_filter(const char *str); diff --git a/tools/perf/util/print_insn.c b/tools/perf/util/print_insn.c index 02e6fbb8ca04..4068436f26ea 100644 --- a/tools/perf/util/print_insn.c +++ b/tools/perf/util/print_insn.c @@ -4,19 +4,24 @@ * * Author(s): Changbin Du <changbin.du@huawei.com> */ +#include "print_insn.h" + #include <inttypes.h> -#include <string.h> #include <stdbool.h> +#include <string.h> + +#include <dwarf-regs.h> + #include "capstone.h" #include "debug.h" +#include "dso.h" +#include "dump-insn.h" +#include "env.h" +#include "machine.h" +#include "map.h" #include "sample.h" #include "symbol.h" -#include "machine.h" #include "thread.h" -#include "print_insn.h" -#include "dump-insn.h" -#include "map.h" -#include "dso.h" size_t sample__fprintf_insn_raw(struct perf_sample *sample, FILE *fp) { @@ -33,13 +38,13 @@ size_t sample__fprintf_insn_raw(struct perf_sample *sample, FILE *fp) static bool is64bitip(struct machine *machine, struct addr_location *al) { const struct dso *dso = al->map ? map__dso(al->map) : NULL; + uint16_t e_machine; if (dso) return dso__is_64_bit(dso); - return machine__is(machine, "x86_64") || - machine__normalized_is(machine, "arm64") || - machine__normalized_is(machine, "s390"); + e_machine = perf_env__e_machine(machine->env, /*e_flags=*/NULL); + return e_machine == EM_X86_64 || e_machine == EM_AARCH64 || e_machine == EM_S390; } ssize_t fprintf_insn_asm(struct machine *machine, struct thread *thread, u8 cpumode, diff --git a/tools/perf/util/print_insn.h b/tools/perf/util/print_insn.h index 07d11af3fc1c..cefa5c5f246e 100644 --- a/tools/perf/util/print_insn.h +++ b/tools/perf/util/print_insn.h @@ -5,10 +5,13 @@ #include <stddef.h> #include <stdio.h> -struct perf_sample; -struct thread; +#include <linux/types.h> + +struct addr_location; struct machine; struct perf_insn; +struct perf_sample; +struct thread; #define PRINT_INSN_IMM_HEX (1<<0) diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c index 34b4badd2c14..11ae4a09412c 100644 --- a/tools/perf/util/probe-event.c +++ b/tools/perf/util/probe-event.c @@ -416,7 +416,7 @@ static int find_alternative_probe_point(struct debuginfo *dinfo, map__for_each_symbol_by_name(map, pp->function, sym, idx) { if (uprobes) { address = sym->start; - if (sym->type == STT_GNU_IFUNC) + if (symbol__type(sym) == STT_GNU_IFUNC) pr_warning("Warning: The probe function (%s) is a GNU indirect function.\n" "Consider identifying the final function used at run time and set the probe directly on that.\n", pp->function); @@ -3189,7 +3189,7 @@ static int find_probe_trace_events_from_map(struct perf_probe_event *pev, for (j = 0; j < num_matched_functions; j++) { sym = syms[j]; - if (sym->type != STT_FUNC) + if (symbol__type(sym) != STT_FUNC) continue; /* There can be duplicated symbols in the map */ diff --git a/tools/perf/util/probe-finder.c b/tools/perf/util/probe-finder.c index 64328abeef8b..f3f9a1573502 100644 --- a/tools/perf/util/probe-finder.c +++ b/tools/perf/util/probe-finder.c @@ -79,7 +79,7 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, unsigned int regn; Dwarf_Word offs = 0; bool ref = false; - const char *regs; + const char *regs, *name; int ret, ret2 = 0; if (dwarf_attr(vr_die, DW_AT_external, &attr) != NULL) @@ -93,7 +93,8 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, if (!tvar) return 0; - dwarf_formsdata(&attr, &snum); + if (dwarf_formsdata(&attr, &snum) != 0) + return -ENOENT; ret = asprintf(&tvar->value, "\\%ld", (long)snum); return ret < 0 ? -ENOMEM : 0; @@ -103,8 +104,7 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, if (dwarf_attr(vr_die, DW_AT_location, &attr) == NULL) return -EINVAL; /* Broken DIE ? */ if (dwarf_getlocation_addr(&attr, addr, &op, &nops, 1) <= 0) { - ret = dwarf_entrypc(sp_die, &tmp); - if (ret) + if (dwarf_entrypc(sp_die, &tmp) != 0) return -ENOENT; if (probe_conf.show_location_range && @@ -115,8 +115,7 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, return -ENOENT; } - ret = dwarf_highpc(sp_die, &tmp); - if (ret) + if (dwarf_highpc(sp_die, &tmp) != 0) return -ENOENT; /* * This is fuzzed by fentry mcount. We try to find the @@ -138,12 +137,16 @@ found: static_var: if (!tvar) return ret2; + /* Static variables on memory (not stack), make @varname */ - ret = strlen(dwarf_diename(vr_die)); + name = dwarf_diename(vr_die); + if (!name) + return -ENOENT; + ret = strlen(name); tvar->value = zalloc(ret + 2); if (tvar->value == NULL) return -ENOMEM; - snprintf(tvar->value, ret + 2, "@%s", dwarf_diename(vr_die)); + snprintf(tvar->value, ret + 2, "@%s", name); tvar->ref = alloc_trace_arg_ref((long)offs); if (tvar->ref == NULL) return -ENOMEM; @@ -234,13 +237,14 @@ static int convert_variable_type(Dwarf_Die *vr_die, } if (die_get_real_type(vr_die, &type) == NULL) { - pr_warning("Failed to get a type information of %s.\n", - dwarf_diename(vr_die)); + const char *name = dwarf_diename(vr_die); + + pr_warning("Failed to get a type information of %s.\n", name ?: "<unknown>"); return -ENOENT; } pr_debug("%s type is %s.\n", - dwarf_diename(vr_die), dwarf_diename(&type)); + die_name(vr_die), die_name(&type)); if (cast && (!strcmp(cast, "string") || !strcmp(cast, "ustring"))) { /* String type */ @@ -249,7 +253,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, ret != DW_TAG_array_type) { pr_warning("Failed to cast into string: " "%s(%s) is not a pointer nor array.\n", - dwarf_diename(vr_die), dwarf_diename(&type)); + die_name(vr_die), die_name(&type)); return -EINVAL; } if (die_get_real_type(&type, &type) == NULL) { @@ -272,7 +276,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, !die_compare_name(&type, "unsigned char")) { pr_warning("Failed to cast into string: " "%s is not (unsigned) char *.\n", - dwarf_diename(vr_die)); + die_name(vr_die)); return -EINVAL; } tvar->type = strdup(cast); @@ -299,7 +303,7 @@ static int convert_variable_type(Dwarf_Die *vr_die, /* Check the bitwidth */ if (ret > MAX_BASIC_TYPE_BITS) { pr_info("%s exceeds max-bitwidth. Cut down to %d bits.\n", - dwarf_diename(&type), MAX_BASIC_TYPE_BITS); + die_name(&type), MAX_BASIC_TYPE_BITS); ret = MAX_BASIC_TYPE_BITS; } ret = snprintf(buf, 16, "%c%d", prefix, ret); @@ -333,12 +337,14 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, pr_warning("Failed to get the type of %s.\n", varname); return -ENOENT; } - pr_debug2("Var real type: %s (%x)\n", dwarf_diename(&type), + pr_debug2("Var real type: %s (%x)\n", die_name(&type), (unsigned)dwarf_dieoffset(&type)); tag = dwarf_tag(&type); if (field->name[0] == '[' && (tag == DW_TAG_array_type || tag == DW_TAG_pointer_type)) { + int bsize; + /* Save original type for next field or type */ memcpy(die_mem, &type, sizeof(*die_mem)); /* Get the type of this array */ @@ -346,7 +352,7 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, pr_warning("Failed to get the type of %s.\n", varname); return -ENOENT; } - pr_debug2("Array real type: %s (%x)\n", dwarf_diename(&type), + pr_debug2("Array real type: %s (%x)\n", die_name(&type), (unsigned)dwarf_dieoffset(&type)); if (tag == DW_TAG_pointer_type) { ref = zalloc(sizeof(struct probe_trace_arg_ref)); @@ -357,7 +363,15 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, else *ref_ptr = ref; } - ref->offset += dwarf_bytesize(&type) * field->index; + bsize = dwarf_bytesize(&type); + + if (bsize < 0) + return -EINVAL; + if (!ref) { + pr_warning("Array indexing not supported for variables in registers.\n"); + return -ENOTSUP; + } + ref->offset += bsize * field->index; ref->user_access = user_access; goto next; } else if (tag == DW_TAG_pointer_type) { @@ -414,7 +428,7 @@ static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, if (die_find_member(&type, field->name, die_mem) == NULL) { pr_warning("%s(type:%s) has no member %s.\n", varname, - dwarf_diename(&type), field->name); + die_name(&type), field->name); return -EINVAL; } @@ -461,7 +475,7 @@ static int convert_variable(Dwarf_Die *vr_die, struct probe_finder *pf) int ret; pr_debug("Converting variable %s into trace event.\n", - dwarf_diename(vr_die)); + die_name(vr_die)); ret = convert_variable_location(vr_die, pf->addr, pf->fb_ops, &pf->sp_die, pf, pf->tvar); @@ -542,7 +556,7 @@ static int convert_to_trace_point(Dwarf_Die *sp_die, Dwfl_Module *mod, /* Verify the address is correct */ if (!dwarf_haspc(sp_die, paddr)) { pr_warning("Specified offset is out of %s\n", - dwarf_diename(sp_die)); + die_name(sp_die)); return -EINVAL; } @@ -599,7 +613,7 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf) if (!die_find_realfunc(&pf->cu_die, pf->addr, &pf->sp_die)) { if (die_find_tailfunc(&pf->cu_die, pf->addr, &pf->sp_die)) { pr_warning("Ignoring tail call from %s\n", - dwarf_diename(&pf->sp_die)); + die_name(&pf->sp_die)); return 0; } else { pr_warning("Failed to find probe point in any " @@ -611,10 +625,16 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf) memcpy(&pf->sp_die, sc_die, sizeof(Dwarf_Die)); /* Get the frame base attribute/ops from subprogram */ - dwarf_attr(&pf->sp_die, DW_AT_frame_base, &fb_attr); - ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1); - if (ret <= 0 || nops == 0) { + if (dwarf_attr(&pf->sp_die, DW_AT_frame_base, &fb_attr) == NULL) { pf->fb_ops = NULL; + } else { + ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1); + if (ret <= 0 || nops == 0) + pf->fb_ops = NULL; + } + + if (pf->fb_ops == NULL) { + /* Not supported */ } else if (nops == 1 && pf->fb_ops[0].atom == DW_OP_call_frame_cfa && (pf->cfi_eh != NULL || pf->cfi_dbg != NULL)) { if ((dwarf_cfi_addrframe(pf->cfi_eh, pf->addr, &frame) != 0 && @@ -667,8 +687,8 @@ static int find_best_scope_cb(Dwarf_Die *fn_die, void *data) } } else { /* With the line number, find the nearest declared DIE */ - dwarf_decl_line(fn_die, &lno); - if (lno < fsp->line && fsp->diff > fsp->line - lno) { + if (dwarf_decl_line(fn_die, &lno) == 0 && lno < fsp->line && + fsp->diff > fsp->line - lno) { /* Keep a candidate and continue */ fsp->diff = fsp->line - lno; memcpy(fsp->die_mem, fn_die, sizeof(Dwarf_Die)); @@ -924,12 +944,12 @@ static int probe_point_inline_cb(Dwarf_Die *in_die, void *data) /* Get probe address */ if (die_entrypc(in_die, &addr) != 0) { pr_warning("Failed to get entry address of %s.\n", - dwarf_diename(in_die)); + die_name(in_die)); return -ENOENT; } if (addr == 0) { pr_debug("%s has no valid entry address. skipped.\n", - dwarf_diename(in_die)); + die_name(in_die)); return -ENOENT; } pf->addr = addr; @@ -971,12 +991,13 @@ static int probe_point_search_cb(Dwarf_Die *sp_die, void *data) if (pp->file && fname && strtailcmp(pp->file, fname)) return DWARF_CB_OK; - pr_debug("Matched function: %s [%lx]\n", dwarf_diename(sp_die), + pr_debug("Matched function: %s [%lx]\n", die_name(sp_die), (unsigned long)dwarf_dieoffset(sp_die)); pf->fname = fname; pf->abstrace_dieoffset = dwarf_dieoffset(sp_die); if (pp->line) { /* Function relative line */ - dwarf_decl_line(sp_die, &pf->lno); + if (dwarf_decl_line(sp_die, &pf->lno) != 0) + return DWARF_CB_OK; pf->lno += pp->line; param->retval = find_probe_point_by_line(pf); } else if (die_is_func_instance(sp_die)) { @@ -985,7 +1006,7 @@ static int probe_point_search_cb(Dwarf_Die *sp_die, void *data) /* But in some case the entry address is 0 */ if (pf->addr == 0) { pr_debug("%s has no entry PC. Skipped\n", - dwarf_diename(sp_die)); + die_name(sp_die)); param->retval = 0; /* Real function */ } else if (pp->lazy_line) @@ -1018,7 +1039,8 @@ static int find_probe_point_by_func(struct probe_finder *pf) { struct dwarf_callback_param _param = {.data = (void *)pf, .retval = 0}; - dwarf_getfuncs(&pf->cu_die, probe_point_search_cb, &_param, 0); + if (dwarf_getfuncs(&pf->cu_die, probe_point_search_cb, &_param, 0) < 0) + pr_debug("Failed to get functions from CU\n"); return _param.retval; } @@ -1207,7 +1229,8 @@ static int copy_variables_cb(Dwarf_Die *die_mem, void *data) * points to correct die. */ if (dwarf_attr(die_mem, DW_AT_abstract_origin, &attr)) { - dwarf_formref_die(&attr, &var_die); + if (dwarf_formref_die(&attr, &var_die) == NULL) + goto out; if (pf->abstrace_dieoffset != dwarf_dieoffset(&var_die)) goto out; } @@ -1293,13 +1316,16 @@ static int add_probe_trace_event(Dwarf_Die *sc_die, struct probe_finder *pf) if (ret < 0) goto end; - tev->point.realname = strdup(dwarf_diename(sc_die)); + tev->point.realname = strdup(die_name(sc_die)); if (!tev->point.realname) { ret = -ENOMEM; goto end; } - tev->lang = dwarf_srclang(dwarf_diecu(sc_die, &pf->cu_die, NULL, NULL)); + if (dwarf_diecu(sc_die, &pf->cu_die, NULL, NULL) != NULL) + tev->lang = dwarf_srclang(&pf->cu_die); + else + tev->lang = DW_LANG_C; /* Fallback */ pr_debug("Probe point found: %s+%lu\n", tev->point.symbol, tev->point.offset); @@ -1794,7 +1820,8 @@ static int line_range_search_cb(Dwarf_Die *sp_die, void *data) if (die_match_name(sp_die, lr->function) && die_is_func_def(sp_die)) { lf->fname = die_get_decl_file(sp_die); - dwarf_decl_line(sp_die, &lr->offset); + if (dwarf_decl_line(sp_die, &lr->offset) != 0) + lr->offset = 0; /* Fallback if no line info */ pr_debug("fname: %s, lineno:%d\n", lf->fname, lr->offset); lf->lno_s = lr->offset + lr->start; if (lf->lno_s < 0) /* Overflow */ @@ -1818,7 +1845,8 @@ static int line_range_search_cb(Dwarf_Die *sp_die, void *data) static int find_line_range_by_func(struct line_finder *lf) { struct dwarf_callback_param param = {.data = (void *)lf, .retval = 0}; - dwarf_getfuncs(&lf->cu_die, line_range_search_cb, ¶m, 0); + if (dwarf_getfuncs(&lf->cu_die, line_range_search_cb, ¶m, 0) < 0) + pr_debug("Failed to get functions from CU\n"); return param.retval; } diff --git a/tools/perf/util/s390-sample-raw.c b/tools/perf/util/s390-sample-raw.c index c6ae0ae8d86a..52bbca5c56c8 100644 --- a/tools/perf/util/s390-sample-raw.c +++ b/tools/perf/util/s390-sample-raw.c @@ -222,8 +222,9 @@ static char *get_counter_name(int set, int nr, struct perf_pmu *pmu) return result; } -static void s390_cpumcfdg_dump(struct perf_pmu *pmu, struct perf_sample *sample) +static void s390_cpumcfdg_dump(struct perf_sample *sample) { + struct perf_pmu *pmu = sample->evsel->pmu; size_t i, len = sample->raw_size, offset = 0; unsigned char *buf = sample->raw_data; const char *color = PERF_COLOR_BLUE; @@ -284,8 +285,9 @@ static bool s390_pai_all_test(struct perf_sample *sample) return true; } -static void s390_pai_all_dump(struct evsel *evsel, struct perf_sample *sample) +static void s390_pai_all_dump(struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; size_t len = sample->raw_size, offset = 0; unsigned char *p = sample->raw_data; const char *color = PERF_COLOR_BLUE; @@ -332,31 +334,32 @@ void evlist__s390_sample_raw(struct evlist *evlist, union perf_event *event, struct perf_sample *sample) { const char *pai_name; - struct evsel *evsel; if (event->header.type != PERF_RECORD_SAMPLE) return; - evsel = evlist__event2evsel(evlist, event); - if (!evsel) - return; + if (!sample->evsel) { + sample->evsel = evlist__event2evsel(evlist, event); + if (!sample->evsel) + return; + } /* Check for raw data in sample */ if (!sample->raw_size || !sample->raw_data) return; /* Display raw data on screen */ - if (evsel->core.attr.config == PERF_EVENT_CPUM_CF_DIAG) { - if (!evsel->pmu) - evsel->pmu = perf_pmus__find("cpum_cf"); + if (sample->evsel->core.attr.config == PERF_EVENT_CPUM_CF_DIAG) { + if (!sample->evsel->pmu) + sample->evsel->pmu = perf_pmus__find("cpum_cf"); if (!s390_cpumcfdg_testctr(sample)) pr_err("Invalid counter set data encountered\n"); else - s390_cpumcfdg_dump(evsel->pmu, sample); + s390_cpumcfdg_dump(sample); return; } - switch (evsel->core.attr.config) { + switch (sample->evsel->core.attr.config) { case PERF_EVENT_PAI_NNPA_ALL: pai_name = "NNPA_ALL"; break; @@ -370,8 +373,8 @@ void evlist__s390_sample_raw(struct evlist *evlist, union perf_event *event, if (!s390_pai_all_test(sample)) { pr_err("Invalid %s raw data encountered\n", pai_name); } else { - if (!evsel->pmu) - evsel->pmu = perf_pmus__find_by_type(evsel->core.attr.type); - s390_pai_all_dump(evsel, sample); + if (!sample->evsel->pmu) + sample->evsel->pmu = perf_pmus__find_by_type(sample->evsel->core.attr.type); + s390_pai_all_dump(sample); } } diff --git a/tools/perf/util/sample-raw.c b/tools/perf/util/sample-raw.c index bcf442574d6e..e20b73c0c5bd 100644 --- a/tools/perf/util/sample-raw.c +++ b/tools/perf/util/sample-raw.c @@ -1,11 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include "sample-raw.h" -#include <string.h> +#include <elf.h> #include <linux/string.h> -#include "evlist.h" + #include "env.h" +#include "evlist.h" #include "header.h" -#include "sample-raw.h" #include "session.h" /* @@ -14,14 +15,14 @@ */ void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env *env) { - const char *arch_pf = perf_env__arch(env); - const char *cpuid = perf_env__cpuid(env); + uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); - if (arch_pf && !strcmp("s390", arch_pf)) + if (e_machine == EM_S390) { evlist->trace_event_sample_raw = evlist__s390_sample_raw; - else if (arch_pf && !strcmp("x86", arch_pf) && - cpuid && strstarts(cpuid, "AuthenticAMD") && - evlist__has_amd_ibs(evlist)) { - evlist->trace_event_sample_raw = evlist__amd_sample_raw; + } else if (e_machine == EM_X86_64 || e_machine == EM_386) { + const char *cpuid = perf_env__cpuid(env); + + if (cpuid && strstarts(cpuid, "AuthenticAMD") && evlist__has_amd_ibs(evlist)) + evlist->trace_event_sample_raw = evlist__amd_sample_raw; } } diff --git a/tools/perf/util/sample-raw.h b/tools/perf/util/sample-raw.h index 896e9a87e373..c8d38c841c8c 100644 --- a/tools/perf/util/sample-raw.h +++ b/tools/perf/util/sample-raw.h @@ -2,7 +2,10 @@ #ifndef __SAMPLE_RAW_H #define __SAMPLE_RAW_H 1 +#include <stdbool.h> + struct evlist; +struct perf_env; union perf_event; struct perf_sample; @@ -12,4 +15,5 @@ bool evlist__has_amd_ibs(struct evlist *evlist); void evlist__amd_sample_raw(struct evlist *evlist, union perf_event *event, struct perf_sample *sample); void evlist__init_trace_event_sample_raw(struct evlist *evlist, struct perf_env *env); -#endif /* __PERF_EVLIST_H */ + +#endif /* __SAMPLE_RAW_H */ diff --git a/tools/perf/util/sample.h b/tools/perf/util/sample.h index e556c9b656ea..c4eae8b2fd06 100644 --- a/tools/perf/util/sample.h +++ b/tools/perf/util/sample.h @@ -158,6 +158,8 @@ struct perf_sample { u64 code_page_size; /** @cgroup: The sample event PERF_SAMPLE_CGROUP value. */ u64 cgroup; + /** @file_offset: Byte offset of this event in the perf.data file. */ + u64 file_offset; /** @flags: Extra flag data from auxiliary events like intel-pt. */ u32 flags; /** @machine_pid: The guest machine pid derived from the sample id. */ diff --git a/tools/perf/util/scripting-engines/trace-event-perl.c b/tools/perf/util/scripting-engines/trace-event-perl.c index e261a57b87d4..410dc4cd0600 100644 --- a/tools/perf/util/scripting-engines/trace-event-perl.c +++ b/tools/perf/util/scripting-engines/trace-event-perl.c @@ -257,7 +257,6 @@ static void define_event_symbols(struct tep_event *event, } static SV *perl_process_callchain(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al) { struct callchain_cursor *cursor; @@ -272,7 +271,7 @@ static SV *perl_process_callchain(struct perf_sample *sample, cursor = get_tls_callchain_cursor(); - if (thread__resolve_callchain(al->thread, cursor, evsel, + if (thread__resolve_callchain(al->thread, cursor, sample, NULL, NULL, scripting_max_stack) != 0) { pr_err("Failed to resolve callchain. Skipping\n"); goto exit; @@ -304,7 +303,7 @@ static SV *perl_process_callchain(struct perf_sample *sample, } if (!hv_stores(sym, "start", newSVuv(node->ms.sym->start)) || !hv_stores(sym, "end", newSVuv(node->ms.sym->end)) || - !hv_stores(sym, "binding", newSVuv(node->ms.sym->binding)) || + !hv_stores(sym, "binding", newSVuv(symbol__binding(node->ms.sym))) || !hv_stores(sym, "name", newSVpvn(node->ms.sym->name, node->ms.sym->namelen)) || !hv_stores(elem, "sym", newRV_noinc((SV*)sym))) { @@ -340,7 +339,6 @@ exit: } static void perl_process_tracepoint(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al) { struct thread *thread = al->thread; @@ -355,6 +353,7 @@ static void perl_process_tracepoint(struct perf_sample *sample, unsigned long long nsecs = sample->time; const char *comm = thread__comm_str(thread); DECLARE_BITMAP(events_defined, TRACE_EVENT_TYPE_MAX); + struct evsel *evsel = sample->evsel; bitmap_zero(events_defined, TRACE_EVENT_TYPE_MAX); dSP; @@ -389,7 +388,7 @@ static void perl_process_tracepoint(struct perf_sample *sample, XPUSHs(sv_2mortal(newSVuv(ns))); XPUSHs(sv_2mortal(newSViv(pid))); XPUSHs(sv_2mortal(newSVpv(comm, 0))); - XPUSHs(sv_2mortal(perl_process_callchain(sample, evsel, al))); + XPUSHs(sv_2mortal(perl_process_callchain(sample, al))); /* common fields other than pid can be accessed via xsub fns */ @@ -426,7 +425,7 @@ static void perl_process_tracepoint(struct perf_sample *sample, XPUSHs(sv_2mortal(newSVuv(nsecs))); XPUSHs(sv_2mortal(newSViv(pid))); XPUSHs(sv_2mortal(newSVpv(comm, 0))); - XPUSHs(sv_2mortal(perl_process_callchain(sample, evsel, al))); + XPUSHs(sv_2mortal(perl_process_callchain(sample, al))); call_pv("main::trace_unhandled", G_SCALAR); } SPAGAIN; @@ -435,9 +434,7 @@ static void perl_process_tracepoint(struct perf_sample *sample, LEAVE; } -static void perl_process_event_generic(union perf_event *event, - struct perf_sample *sample, - struct evsel *evsel) +static void perl_process_event_generic(union perf_event *event, struct perf_sample *sample) { dSP; @@ -448,7 +445,8 @@ static void perl_process_event_generic(union perf_event *event, SAVETMPS; PUSHMARK(SP); XPUSHs(sv_2mortal(newSVpvn((const char *)event, event->header.size))); - XPUSHs(sv_2mortal(newSVpvn((const char *)&evsel->core.attr, sizeof(evsel->core.attr)))); + XPUSHs(sv_2mortal(newSVpvn((const char *)&sample->evsel->core.attr, + sizeof(sample->evsel->core.attr)))); XPUSHs(sv_2mortal(newSVpvn((const char *)sample, sizeof(*sample)))); XPUSHs(sv_2mortal(newSVpvn((const char *)sample->raw_data, sample->raw_size))); PUTBACK; @@ -461,13 +459,12 @@ static void perl_process_event_generic(union perf_event *event, static void perl_process_event(union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { - scripting_context__update(scripting_context, event, sample, evsel, al, addr_al); - perl_process_tracepoint(sample, evsel, al); - perl_process_event_generic(event, sample, evsel); + scripting_context__update(scripting_context, event, sample, al, addr_al); + perl_process_tracepoint(sample, al); + perl_process_event_generic(event, sample); } static void run_start_sub(void) diff --git a/tools/perf/util/scripting-engines/trace-event-python.c b/tools/perf/util/scripting-engines/trace-event-python.c index 5a30caaec73e..8f832ae316ca 100644 --- a/tools/perf/util/scripting-engines/trace-event-python.c +++ b/tools/perf/util/scripting-engines/trace-event-python.c @@ -390,7 +390,6 @@ static unsigned long get_offset(struct symbol *sym, struct addr_location *al) } static PyObject *python_process_callchain(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al) { PyObject *pylist; @@ -404,7 +403,7 @@ static PyObject *python_process_callchain(struct perf_sample *sample, goto exit; cursor = get_tls_callchain_cursor(); - if (thread__resolve_callchain(al->thread, cursor, evsel, + if (thread__resolve_callchain(al->thread, cursor, sample, NULL, NULL, scripting_max_stack) != 0) { pr_err("Failed to resolve callchain. Skipping\n"); @@ -437,7 +436,7 @@ static PyObject *python_process_callchain(struct perf_sample *sample, pydict_set_item_string_decref(pysym, "end", PyLong_FromUnsignedLongLong(node->ms.sym->end)); pydict_set_item_string_decref(pysym, "binding", - _PyLong_FromLong(node->ms.sym->binding)); + _PyLong_FromLong(symbol__binding(node->ms.sym))); pydict_set_item_string_decref(pysym, "name", _PyUnicode_FromStringAndSize(node->ms.sym->name, node->ms.sym->namelen)); @@ -651,11 +650,9 @@ static PyObject *get_sample_value_as_tuple(struct sample_read_value *value, return t; } -static void set_sample_read_in_dict(PyObject *dict_sample, - struct perf_sample *sample, - struct evsel *evsel) +static void set_sample_read_in_dict(PyObject *dict_sample, struct perf_sample *sample) { - u64 read_format = evsel->core.attr.read_format; + u64 read_format = sample->evsel->core.attr.read_format; PyObject *values; unsigned int i; @@ -741,11 +738,10 @@ static void regs_map(struct regs_dump *regs, uint64_t mask, uint16_t e_machine, static int set_regs_in_dict(PyObject *dict, struct perf_sample *sample, - struct evsel *evsel, uint16_t e_machine, uint32_t e_flags) { - struct perf_event_attr *attr = &evsel->core.attr; + struct perf_event_attr *attr = &sample->evsel->core.attr; int size = (__sw_hweight64(attr->sample_regs_intr) * MAX_REG_SIZE) + 1; char *bf = NULL; @@ -831,7 +827,6 @@ static void python_process_sample_flags(struct perf_sample *sample, PyObject *di } static PyObject *get_perf_sample_dict(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al, PyObject *callchain) @@ -839,6 +834,7 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyObject *dict, *dict_sample, *brstack, *brstacksym; uint16_t e_machine = EM_HOST; uint32_t e_flags = EF_HOST; + struct evsel *evsel = sample->evsel; dict = PyDict_New(); if (!dict) @@ -871,7 +867,7 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyLong_FromUnsignedLongLong(sample->phys_addr)); pydict_set_item_string_decref(dict_sample, "addr", PyLong_FromUnsignedLongLong(sample->addr)); - set_sample_read_in_dict(dict_sample, sample, evsel); + set_sample_read_in_dict(dict_sample, sample); pydict_set_item_string_decref(dict_sample, "weight", PyLong_FromUnsignedLongLong(sample->weight)); pydict_set_item_string_decref(dict_sample, "ins_lat", @@ -928,7 +924,7 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, if (al->thread) e_machine = thread__e_machine(al->thread, /*machine=*/NULL, &e_flags); - if (set_regs_in_dict(dict, sample, evsel, e_machine, e_flags)) + if (set_regs_in_dict(dict, sample, e_machine, e_flags)) Py_FatalError("Failed to setting regs in dict"); return dict; @@ -936,7 +932,6 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, #ifdef HAVE_LIBTRACEEVENT static void python_process_tracepoint(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { @@ -954,6 +949,7 @@ static void python_process_tracepoint(struct perf_sample *sample, const char *comm = thread__comm_str(al->thread); const char *default_handler_name = "trace_unhandled"; DECLARE_BITMAP(events_defined, TRACE_EVENT_TYPE_MAX); + struct evsel *evsel = sample->evsel; bitmap_zero(events_defined, TRACE_EVENT_TYPE_MAX); @@ -995,7 +991,7 @@ static void python_process_tracepoint(struct perf_sample *sample, PyTuple_SetItem(t, n++, context); /* ip unwinding */ - callchain = python_process_callchain(sample, evsel, al); + callchain = python_process_callchain(sample, al); /* Need an additional reference for the perf_sample dict */ Py_INCREF(callchain); @@ -1051,7 +1047,7 @@ static void python_process_tracepoint(struct perf_sample *sample, PyTuple_SetItem(t, n++, dict); if (get_argument_count(handler) == (int) n + 1) { - all_entries_dict = get_perf_sample_dict(sample, evsel, al, addr_al, + all_entries_dict = get_perf_sample_dict(sample, al, addr_al, callchain); PyTuple_SetItem(t, n++, all_entries_dict); } else { @@ -1070,7 +1066,6 @@ static void python_process_tracepoint(struct perf_sample *sample, } #else static void python_process_tracepoint(struct perf_sample *sample __maybe_unused, - struct evsel *evsel __maybe_unused, struct addr_location *al __maybe_unused, struct addr_location *addr_al __maybe_unused) { @@ -1275,7 +1270,7 @@ static int python_export_symbol(struct db_export *dbe, struct symbol *sym, tuple_set_d64(t, 1, dso__db_id(dso)); tuple_set_d64(t, 2, sym->start); tuple_set_d64(t, 3, sym->end); - tuple_set_s32(t, 4, sym->binding); + tuple_set_s32(t, 4, symbol__binding(sym)); tuple_set_string(t, 5, sym->name); call_object(tables->symbol_handler, t, "symbol_table"); @@ -1312,7 +1307,7 @@ static void python_export_sample_table(struct db_export *dbe, t = tuple_new(28); tuple_set_d64(t, 0, es->db_id); - tuple_set_d64(t, 1, es->evsel->db_id); + tuple_set_d64(t, 1, es->sample->evsel->db_id); tuple_set_d64(t, 2, maps__machine(thread__maps(es->al->thread))->db_id); tuple_set_d64(t, 3, thread__db_id(es->al->thread)); tuple_set_d64(t, 4, es->comm_db_id); @@ -1353,7 +1348,7 @@ static void python_export_synth(struct db_export *dbe, struct export_sample *es) t = tuple_new(3); tuple_set_d64(t, 0, es->db_id); - tuple_set_d64(t, 1, es->evsel->core.attr.config); + tuple_set_d64(t, 1, es->sample->evsel->core.attr.config); tuple_set_bytes(t, 2, es->sample->raw_data, es->sample->raw_size); call_object(tables->synth_handler, t, "synth_data"); @@ -1368,7 +1363,7 @@ static int python_export_sample(struct db_export *dbe, python_export_sample_table(dbe, es); - if (es->evsel->core.attr.type == PERF_TYPE_SYNTH && tables->synth_handler) + if (es->sample->evsel->core.attr.type == PERF_TYPE_SYNTH && tables->synth_handler) python_export_synth(dbe, es); return 0; @@ -1465,7 +1460,6 @@ static int python_process_call_return(struct call_return *cr, u64 *parent_db_id, } static void python_process_general_event(struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { @@ -1488,8 +1482,8 @@ static void python_process_general_event(struct perf_sample *sample, Py_FatalError("couldn't create Python tuple"); /* ip unwinding */ - callchain = python_process_callchain(sample, evsel, al); - dict = get_perf_sample_dict(sample, evsel, al, addr_al, callchain); + callchain = python_process_callchain(sample, al); + dict = get_perf_sample_dict(sample, al, addr_al, callchain); PyTuple_SetItem(t, n++, dict); if (_PyTuple_Resize(&t, n) == -1) @@ -1502,24 +1496,23 @@ static void python_process_general_event(struct perf_sample *sample, static void python_process_event(union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { struct tables *tables = &tables_global; - scripting_context__update(scripting_context, event, sample, evsel, al, addr_al); + scripting_context__update(scripting_context, event, sample, al, addr_al); - switch (evsel->core.attr.type) { + switch (sample->evsel->core.attr.type) { case PERF_TYPE_TRACEPOINT: - python_process_tracepoint(sample, evsel, al, addr_al); + python_process_tracepoint(sample, al, addr_al); break; /* Reserve for future process_hw/sw/raw APIs */ default: if (tables->db_export_mode) - db_export__sample(&tables->dbe, event, sample, evsel, al, addr_al); + db_export__sample(&tables->dbe, event, sample, al, addr_al); else - python_process_general_event(sample, evsel, al, addr_al); + python_process_general_event(sample, al, addr_al); } } @@ -1614,6 +1607,9 @@ static void python_process_auxtrace_error(struct perf_session *session __maybe_u const char *handler_name = "auxtrace_error"; unsigned long long tm = e->time; const char *msg = e->msg; + s32 machine_pid = 0, vcpu = 0; + char msg_buf[MAX_AUXTRACE_ERROR_MSG + 1]; + int msg_max; PyObject *handler, *t; handler = get_handler(handler_name); @@ -1625,6 +1621,25 @@ static void python_process_auxtrace_error(struct perf_session *session __maybe_u msg = (const char *)&e->time; } + /* Bound msg to the bytes within the event, ensure NUL-termination */ + msg_max = (int)((void *)event + event->header.size - (void *)msg); + if (msg_max <= 0) { + msg_buf[0] = '\0'; + } else { + if (msg_max > (int)sizeof(msg_buf) - 1) + msg_max = sizeof(msg_buf) - 1; + memcpy(msg_buf, msg, msg_max); + msg_buf[msg_max] = '\0'; + } + + /* Only access fmt >= 2 fields if the event is large enough */ + if (e->fmt >= 2 && + event->header.size >= offsetof(typeof(event->auxtrace_error), vcpu) + + sizeof(event->auxtrace_error.vcpu)) { + machine_pid = e->machine_pid; + vcpu = e->vcpu; + } + t = tuple_new(11); tuple_set_u32(t, 0, e->type); @@ -1634,10 +1649,10 @@ static void python_process_auxtrace_error(struct perf_session *session __maybe_u tuple_set_s32(t, 4, e->tid); tuple_set_u64(t, 5, e->ip); tuple_set_u64(t, 6, tm); - tuple_set_string(t, 7, msg); + tuple_set_string(t, 7, msg_buf); tuple_set_u32(t, 8, cpumode); - tuple_set_s32(t, 9, e->machine_pid); - tuple_set_s32(t, 10, e->vcpu); + tuple_set_s32(t, 9, machine_pid); + tuple_set_s32(t, 10, vcpu); call_object(handler, t, handler_name); diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index fe0de2a0277f..f391a822480d 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -160,11 +160,12 @@ struct perf_session *__perf_session__new(struct perf_data *data, session->decomp_data.zstd_decomp = &session->zstd_data; session->active_decomp = &session->decomp_data; INIT_LIST_HEAD(&session->auxtrace_index); - machines__init(&session->machines); + perf_env__init(&session->header.env); + if (machines__init(&session->machines)) + goto out_delete; + ordered_events__init(&session->ordered_events, ordered_events__deliver_event, NULL); - - perf_env__init(&session->header.env); if (data) { ret = perf_data__open(data); if (ret < 0) @@ -276,34 +277,58 @@ void perf_session__delete(struct perf_session *session) static void swap_sample_id_all(union perf_event *event, void *data) { void *end = (void *) event + event->header.size; - int size = end - data; + int size; - BUG_ON(size % sizeof(u64)); - mem_bswap_64(data, size); + if (data >= end) + return; + + size = end - data; + if (size % sizeof(u64)) { + pr_warning("swap_sample_id_all: unaligned sample_id_all remainder (%d), skipping swap\n", size); + return; + } + if (size > 0) + mem_bswap_64(data, size); } -static void perf_event__all64_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__all64_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { struct perf_event_header *hdr = &event->header; - mem_bswap_64(hdr + 1, event->header.size - sizeof(*hdr)); + size_t size = event->header.size - sizeof(*hdr); + + /* mem_bswap_64 rounds up to 8-byte chunks — unaligned size overruns the buffer */ + if (size % sizeof(u64)) + return -1; + mem_bswap_64(hdr + 1, size); + return 0; } -static void perf_event__comm_swap(union perf_event *event, bool sample_id_all) +static int perf_event__comm_swap(union perf_event *event, bool sample_id_all) { event->comm.pid = bswap_32(event->comm.pid); event->comm.tid = bswap_32(event->comm.tid); if (sample_id_all) { void *data = &event->comm.comm; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); - data += PERF_ALIGN(strlen(data) + 1, sizeof(u64)); + /* + * No NUL within the event boundary — can't locate where + * sample_id_all starts. Reject so the event is skipped + * rather than swapping garbage. + */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } -static void perf_event__mmap_swap(union perf_event *event, - bool sample_id_all) +static int perf_event__mmap_swap(union perf_event *event, + bool sample_id_all) { event->mmap.pid = bswap_32(event->mmap.pid); event->mmap.tid = bswap_32(event->mmap.tid); @@ -313,13 +338,19 @@ static void perf_event__mmap_swap(union perf_event *event, if (sample_id_all) { void *data = &event->mmap.filename; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); - data += PERF_ALIGN(strlen(data) + 1, sizeof(u64)); + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } -static void perf_event__mmap2_swap(union perf_event *event, +static int perf_event__mmap2_swap(union perf_event *event, bool sample_id_all) { event->mmap2.pid = bswap_32(event->mmap2.pid); @@ -337,12 +368,19 @@ static void perf_event__mmap2_swap(union perf_event *event, if (sample_id_all) { void *data = &event->mmap2.filename; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); - data += PERF_ALIGN(strlen(data) + 1, sizeof(u64)); + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } -static void perf_event__task_swap(union perf_event *event, bool sample_id_all) + +static int perf_event__task_swap(union perf_event *event, bool sample_id_all) { event->fork.pid = bswap_32(event->fork.pid); event->fork.tid = bswap_32(event->fork.tid); @@ -352,22 +390,31 @@ static void perf_event__task_swap(union perf_event *event, bool sample_id_all) if (sample_id_all) swap_sample_id_all(event, &event->fork + 1); + return 0; } -static void perf_event__read_swap(union perf_event *event, bool sample_id_all) +static int perf_event__read_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { + size_t tail; + event->read.pid = bswap_32(event->read.pid); event->read.tid = bswap_32(event->read.tid); - event->read.value = bswap_64(event->read.value); - event->read.time_enabled = bswap_64(event->read.time_enabled); - event->read.time_running = bswap_64(event->read.time_running); - event->read.id = bswap_64(event->read.id); - - if (sample_id_all) - swap_sample_id_all(event, &event->read + 1); + /* + * Everything after pid/tid is u64: the read values (variable + * set determined by attr.read_format, which we don't have + * here) optionally followed by sample_id_all fields. + * Since all are u64, swap the entire remaining tail at once. + */ + tail = event->header.size - offsetof(struct perf_record_read, value); + /* mem_bswap_64 rounds up to 8-byte chunks — unaligned tail overruns the buffer */ + if (tail % sizeof(u64)) + return -1; + mem_bswap_64(&event->read.value, tail); + return 0; } -static void perf_event__aux_swap(union perf_event *event, bool sample_id_all) +static int perf_event__aux_swap(union perf_event *event, bool sample_id_all) { event->aux.aux_offset = bswap_64(event->aux.aux_offset); event->aux.aux_size = bswap_64(event->aux.aux_size); @@ -375,19 +422,21 @@ static void perf_event__aux_swap(union perf_event *event, bool sample_id_all) if (sample_id_all) swap_sample_id_all(event, &event->aux + 1); + return 0; } -static void perf_event__itrace_start_swap(union perf_event *event, - bool sample_id_all) +static int perf_event__itrace_start_swap(union perf_event *event, + bool sample_id_all) { event->itrace_start.pid = bswap_32(event->itrace_start.pid); event->itrace_start.tid = bswap_32(event->itrace_start.tid); if (sample_id_all) swap_sample_id_all(event, &event->itrace_start + 1); + return 0; } -static void perf_event__switch_swap(union perf_event *event, bool sample_id_all) +static int perf_event__switch_swap(union perf_event *event, bool sample_id_all) { if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE) { event->context_switch.next_prev_pid = @@ -396,30 +445,45 @@ static void perf_event__switch_swap(union perf_event *event, bool sample_id_all) bswap_32(event->context_switch.next_prev_tid); } - if (sample_id_all) - swap_sample_id_all(event, &event->context_switch + 1); + if (sample_id_all) { + /* + * PERF_RECORD_SWITCH has no fields beyond the header; + * SWITCH_CPU_WIDE adds pid/tid. Use the right offset + * so sample_id starts at the correct position. + */ + if (event->header.type == PERF_RECORD_SWITCH) + swap_sample_id_all(event, (void *)event + sizeof(event->header)); + else + swap_sample_id_all(event, &event->context_switch + 1); + } + return 0; } -static void perf_event__text_poke_swap(union perf_event *event, bool sample_id_all) +static int perf_event__text_poke_swap(union perf_event *event, bool sample_id_all) { event->text_poke.addr = bswap_64(event->text_poke.addr); event->text_poke.old_len = bswap_16(event->text_poke.old_len); event->text_poke.new_len = bswap_16(event->text_poke.new_len); if (sample_id_all) { + void *data = &event->text_poke.old_len; + void *end = (void *)event + event->header.size; size_t len = sizeof(event->text_poke.old_len) + sizeof(event->text_poke.new_len) + event->text_poke.old_len + event->text_poke.new_len; - void *data = &event->text_poke.old_len; + /* old_len + new_len exceeds event — can't find sample_id_all */ + if (data + len > end) + return -1; data += PERF_ALIGN(len, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } -static void perf_event__throttle_swap(union perf_event *event, - bool sample_id_all) +static int perf_event__throttle_swap(union perf_event *event, + bool sample_id_all) { event->throttle.time = bswap_64(event->throttle.time); event->throttle.id = bswap_64(event->throttle.id); @@ -427,18 +491,41 @@ static void perf_event__throttle_swap(union perf_event *event, if (sample_id_all) swap_sample_id_all(event, &event->throttle + 1); + return 0; } -static void perf_event__namespaces_swap(union perf_event *event, - bool sample_id_all) +static int perf_event__namespaces_swap(union perf_event *event, + bool sample_id_all) { - u64 i; + u64 i, nr, max_nr; event->namespaces.pid = bswap_32(event->namespaces.pid); event->namespaces.tid = bswap_32(event->namespaces.tid); event->namespaces.nr_namespaces = bswap_64(event->namespaces.nr_namespaces); - for (i = 0; i < event->namespaces.nr_namespaces; i++) { + nr = event->namespaces.nr_namespaces; + /* + * Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof. + * When sample_id_all is present max_nr slightly overestimates the + * array space because header.size includes the trailing sample_id. + * Harmless: both the per-element bswap_64 loop and swap_sample_id_all() + * perform the same u64 byte swap, so the result is correct regardless + * of where the boundary between array and sample_id falls. + */ + max_nr = (event->header.size - sizeof(event->namespaces)) / + sizeof(event->namespaces.link_info[0]); + /* + * Safe to clamp: each namespace entry is indexed by type; + * missing entries just won't be resolved. + */ + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_NAMESPACES: nr_namespaces %" PRIu64 " exceeds payload (max %" PRIu64 "), clamping\n", + nr, max_nr); + nr = max_nr; + event->namespaces.nr_namespaces = nr; + } + + for (i = 0; i < nr; i++) { struct perf_ns_link_info *ns = &event->namespaces.link_info[i]; ns->dev = bswap_64(ns->dev); @@ -447,18 +534,25 @@ static void perf_event__namespaces_swap(union perf_event *event, if (sample_id_all) swap_sample_id_all(event, &event->namespaces.link_info[i]); + return 0; } -static void perf_event__cgroup_swap(union perf_event *event, bool sample_id_all) +static int perf_event__cgroup_swap(union perf_event *event, bool sample_id_all) { event->cgroup.id = bswap_64(event->cgroup.id); if (sample_id_all) { void *data = &event->cgroup.path; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); - data += PERF_ALIGN(strlen(data) + 1, sizeof(u64)); + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); swap_sample_id_all(event, data); } + return 0; } static u8 revbyte(u8 b) @@ -499,9 +593,19 @@ void perf_event__attr_swap(struct perf_event_attr *attr) attr->type = bswap_32(attr->type); attr->size = bswap_32(attr->size); -#define bswap_safe(f, n) \ - (attr->size > (offsetof(struct perf_event_attr, f) + \ - sizeof(attr->f) * (n))) + /* + * ABI0: size == 0 means the producer didn't set it. + * Assume PERF_ATTR_SIZE_VER0 so bswap_safe() below + * correctly swaps the VER0 fields instead of skipping + * everything. Same convention as read_attr(). + */ + if (!attr->size) + attr->size = PERF_ATTR_SIZE_VER0; + +/* Verify the full field extent fits, not just its start offset */ +#define bswap_safe(f, n) \ + (attr->size >= (offsetof(struct perf_event_attr, f) + \ + sizeof(attr->f) * ((n) + 1))) #define bswap_field(f, sz) \ do { \ if (bswap_safe(f, 0)) \ @@ -539,40 +643,189 @@ do { \ #undef bswap_safe } -static void perf_event__hdr_attr_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__hdr_attr_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { + u32 attr_size, payload_size; size_t size; + /* + * Validate attr.size (still foreign-endian) before calling + * perf_event__attr_swap(), which uses it via bswap_safe() + * to decide which fields to swap. A crafted attr.size + * larger than the event payload would swap past the event + * boundary and corrupt adjacent memory. + * + * header.size alignment is already validated by + * perf_session__process_event(). The min_size table + * guarantees header.size >= sizeof(header) + + * PERF_ATTR_SIZE_VER0, so attr.size is safe to access. + */ + attr_size = bswap_32(event->attr.attr.size); + /* + * ABI0: size field not set. This only happens in pipe/inject + * mode where HEADER_ATTR events carry their own attr. For + * regular perf.data files, read_attr() uses f_header.attr_size + * from the file header instead. Assume PERF_ATTR_SIZE_VER0. + */ + if (!attr_size) + attr_size = PERF_ATTR_SIZE_VER0; + payload_size = event->header.size - sizeof(event->header); + + if (attr_size < PERF_ATTR_SIZE_VER0 || attr_size % sizeof(u64) || + attr_size > payload_size) { + pr_err("PERF_RECORD_HEADER_ATTR: invalid attr.size %u (min: %d, max: %u, 8-byte aligned)\n", + attr_size, PERF_ATTR_SIZE_VER0, payload_size); + return -1; + } + perf_event__attr_swap(&event->attr.attr); size = event->header.size; size -= perf_record_header_attr_id(event) - (void *)event; mem_bswap_64(perf_record_header_attr_id(event), size); + return 0; } -static void perf_event__event_update_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__build_id_swap(union perf_event *event, + bool sample_id_all) { - event->event_update.type = bswap_64(event->event_update.type); - event->event_update.id = bswap_64(event->event_update.id); + event->build_id.pid = bswap_32(event->build_id.pid); + + if (sample_id_all) { + void *data = &event->build_id.filename; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); + + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); + swap_sample_id_all(event, data); + } + return 0; } -static void perf_event__event_type_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__event_update_swap(union perf_event *event, + bool sample_id_all __maybe_unused) +{ + struct perf_record_event_update *ev = &event->event_update; + + ev->type = bswap_64(ev->type); + ev->id = bswap_64(ev->id); + + /* + * Swap variant-specific fields so the processing path + * sees native byte order. + */ + if (ev->type == PERF_EVENT_UPDATE__SCALE) { + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) + return -1; + mem_bswap_64(&ev->scale.scale, sizeof(ev->scale.scale)); + } else if (ev->type == PERF_EVENT_UPDATE__CPUS) { + u32 cpus_payload; + struct perf_record_cpu_map_data *data = &ev->cpus.cpus; + + /* CPUS fields start at the same offset as scale (union) */ + if (event->header.size < offsetof(struct perf_record_event_update, cpus) + + sizeof(__u16) + sizeof(struct perf_record_range_cpu_map)) + return -1; + cpus_payload = event->header.size - offsetof(struct perf_record_event_update, cpus); + data->type = bswap_16(data->type); + /* + * Full swap including array elements — same logic as + * perf_event__cpu_map_swap() but scoped to the + * embedded cpu_map_data within EVENT_UPDATE. + */ + switch (data->type) { + case PERF_CPU_MAP__CPUS: { + u16 nr, max_nr; + + data->cpus_data.nr = bswap_16(data->cpus_data.nr); + nr = data->cpus_data.nr; + max_nr = (cpus_payload - offsetof(struct perf_record_cpu_map_data, + cpus_data.cpu)) / + sizeof(data->cpus_data.cpu[0]); + if (nr > max_nr) { + nr = max_nr; + data->cpus_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) + data->cpus_data.cpu[i] = bswap_16(data->cpus_data.cpu[i]); + break; + } + case PERF_CPU_MAP__MASK: + data->mask32_data.long_size = bswap_16(data->mask32_data.long_size); + switch (data->mask32_data.long_size) { + case 4: { + u16 nr, max_nr; + + data->mask32_data.nr = bswap_16(data->mask32_data.nr); + nr = data->mask32_data.nr; + max_nr = (cpus_payload - offsetof(struct perf_record_cpu_map_data, + mask32_data.mask)) / + sizeof(data->mask32_data.mask[0]); + if (nr > max_nr) { + nr = max_nr; + data->mask32_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) + data->mask32_data.mask[i] = bswap_32(data->mask32_data.mask[i]); + break; + } + case 8: { + u16 nr, max_nr; + + data->mask64_data.nr = bswap_16(data->mask64_data.nr); + nr = data->mask64_data.nr; + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) { + data->mask64_data.nr = 0; + break; + } + max_nr = (cpus_payload - offsetof(struct perf_record_cpu_map_data, + mask64_data.mask)) / + sizeof(data->mask64_data.mask[0]); + if (nr > max_nr) { + nr = max_nr; + data->mask64_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) + data->mask64_data.mask[i] = bswap_64(data->mask64_data.mask[i]); + break; + } + default: + break; + } + break; + case PERF_CPU_MAP__RANGE_CPUS: + data->range_cpu_data.start_cpu = bswap_16(data->range_cpu_data.start_cpu); + data->range_cpu_data.end_cpu = bswap_16(data->range_cpu_data.end_cpu); + break; + default: + break; + } + } + return 0; +} + +static int perf_event__event_type_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->event_type.event_type.event_id = bswap_64(event->event_type.event_type.event_id); + return 0; } -static void perf_event__tracing_data_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__tracing_data_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->tracing_data.size = bswap_32(event->tracing_data.size); + return 0; } -static void perf_event__auxtrace_info_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__auxtrace_info_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { size_t size; @@ -581,10 +834,11 @@ static void perf_event__auxtrace_info_swap(union perf_event *event, size = event->header.size; size -= (void *)&event->auxtrace_info.priv - (void *)event; mem_bswap_64(event->auxtrace_info.priv, size); + return 0; } -static void perf_event__auxtrace_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__auxtrace_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->auxtrace.size = bswap_64(event->auxtrace.size); event->auxtrace.offset = bswap_64(event->auxtrace.offset); @@ -592,10 +846,11 @@ static void perf_event__auxtrace_swap(union perf_event *event, event->auxtrace.idx = bswap_32(event->auxtrace.idx); event->auxtrace.tid = bswap_32(event->auxtrace.tid); event->auxtrace.cpu = bswap_32(event->auxtrace.cpu); + return 0; } -static void perf_event__auxtrace_error_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__auxtrace_error_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->auxtrace_error.type = bswap_32(event->auxtrace_error.type); event->auxtrace_error.code = bswap_32(event->auxtrace_error.code); @@ -607,52 +862,128 @@ static void perf_event__auxtrace_error_swap(union perf_event *event, if (event->auxtrace_error.fmt) event->auxtrace_error.time = bswap_64(event->auxtrace_error.time); if (event->auxtrace_error.fmt >= 2) { - event->auxtrace_error.machine_pid = bswap_32(event->auxtrace_error.machine_pid); - event->auxtrace_error.vcpu = bswap_32(event->auxtrace_error.vcpu); + /* + * fmt >= 2 adds machine_pid and vcpu after msg[64]. + * Older files may have fmt >= 2 but an event size + * that doesn't include these fields — downgrade to + * avoid swapping out of bounds. + */ + if (event->header.size < offsetof(typeof(event->auxtrace_error), vcpu) + + sizeof(event->auxtrace_error.vcpu)) { + pr_warning("WARNING: PERF_RECORD_AUXTRACE_ERROR: fmt %u but event too small for machine_pid/vcpu (%u bytes), downgrading fmt\n", + event->auxtrace_error.fmt, + event->header.size); + event->auxtrace_error.fmt = 1; + } else { + event->auxtrace_error.machine_pid = bswap_32(event->auxtrace_error.machine_pid); + event->auxtrace_error.vcpu = bswap_32(event->auxtrace_error.vcpu); + } } + return 0; } -static void perf_event__thread_map_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__thread_map_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { - unsigned i; + unsigned int i; + u64 nr; event->thread_map.nr = bswap_64(event->thread_map.nr); - for (i = 0; i < event->thread_map.nr; i++) + /* + * Reject rather than clamp: unlike namespaces (indexed by type) + * or stat_config (self-describing tags), a truncated thread map + * is structurally broken — downstream would get a wrong map. + */ + /* Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof */ + nr = event->thread_map.nr; + if (nr > (event->header.size - sizeof(event->thread_map)) / + sizeof(event->thread_map.entries[0])) + return -1; + + for (i = 0; i < nr; i++) event->thread_map.entries[i].pid = bswap_64(event->thread_map.entries[i].pid); + return 0; } -static void perf_event__cpu_map_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__cpu_map_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { struct perf_record_cpu_map_data *data = &event->cpu_map.data; + u32 payload = event->header.size - sizeof(event->header); data->type = bswap_16(data->type); + /* + * Safe to clamp: a shorter CPU map just means some CPUs + * are absent; tools process the CPUs that are present. + */ switch (data->type) { - case PERF_CPU_MAP__CPUS: - data->cpus_data.nr = bswap_16(data->cpus_data.nr); + case PERF_CPU_MAP__CPUS: { + u16 nr, max_nr; - for (unsigned i = 0; i < data->cpus_data.nr; i++) + data->cpus_data.nr = bswap_16(data->cpus_data.nr); + nr = data->cpus_data.nr; + max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + cpus_data.cpu)) / + sizeof(data->cpus_data.cpu[0]); + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_CPU_MAP: nr %u exceeds payload (max %u), clamping\n", + nr, max_nr); + nr = max_nr; + data->cpus_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) data->cpus_data.cpu[i] = bswap_16(data->cpus_data.cpu[i]); break; + } case PERF_CPU_MAP__MASK: data->mask32_data.long_size = bswap_16(data->mask32_data.long_size); switch (data->mask32_data.long_size) { - case 4: + case 4: { + u16 nr, max_nr; + data->mask32_data.nr = bswap_16(data->mask32_data.nr); - for (unsigned i = 0; i < data->mask32_data.nr; i++) + nr = data->mask32_data.nr; + max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + mask32_data.mask)) / + sizeof(data->mask32_data.mask[0]); + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_CPU_MAP mask32: nr %u exceeds payload (max %u), clamping\n", + nr, max_nr); + nr = max_nr; + data->mask32_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) data->mask32_data.mask[i] = bswap_32(data->mask32_data.mask[i]); break; - case 8: + } + case 8: { + u16 nr, max_nr; + data->mask64_data.nr = bswap_16(data->mask64_data.nr); - for (unsigned i = 0; i < data->mask64_data.nr; i++) + nr = data->mask64_data.nr; + if (payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) { + data->mask64_data.nr = 0; + break; + } + max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + mask64_data.mask)) / + sizeof(data->mask64_data.mask[0]); + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_CPU_MAP mask64: nr %u exceeds payload (max %u), clamping\n", + nr, max_nr); + nr = max_nr; + data->mask64_data.nr = nr; + } + for (unsigned int i = 0; i < nr; i++) data->mask64_data.mask[i] = bswap_64(data->mask64_data.mask[i]); break; + } default: - pr_err("cpu_map swap: unsupported long size\n"); + pr_err("cpu_map swap: unsupported long size %u\n", + data->mask32_data.long_size); } break; case PERF_CPU_MAP__RANGE_CPUS: @@ -662,20 +993,38 @@ static void perf_event__cpu_map_swap(union perf_event *event, default: break; } + return 0; } -static void perf_event__stat_config_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__stat_config_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { - u64 size; + u64 nr, max_nr, size; - size = bswap_64(event->stat_config.nr) * sizeof(event->stat_config.data[0]); - size += 1; /* nr item itself */ + nr = bswap_64(event->stat_config.nr); + /* Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof */ + max_nr = (event->header.size - sizeof(event->stat_config)) / + sizeof(event->stat_config.data[0]); + /* + * Safe to clamp: each config entry is self-describing + * via its tag; missing entries keep their defaults. + */ + if (nr > max_nr) { + pr_warning("WARNING: PERF_RECORD_STAT_CONFIG: nr %" PRIu64 " exceeds payload (max %" PRIu64 "), clamping\n", + nr, max_nr); + nr = max_nr; + } + size = nr * sizeof(event->stat_config.data[0]); + /* The swap starts at &nr, so add its size to cover the full range */ + size += sizeof(event->stat_config.nr); mem_bswap_64(&event->stat_config.nr, size); + /* Persist the clamped value in native byte order */ + event->stat_config.nr = nr; + return 0; } -static void perf_event__stat_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__stat_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->stat.id = bswap_64(event->stat.id); event->stat.thread = bswap_32(event->stat.thread); @@ -683,44 +1032,140 @@ static void perf_event__stat_swap(union perf_event *event, event->stat.val = bswap_64(event->stat.val); event->stat.ena = bswap_64(event->stat.ena); event->stat.run = bswap_64(event->stat.run); + return 0; } -static void perf_event__stat_round_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__stat_round_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->stat_round.type = bswap_64(event->stat_round.type); event->stat_round.time = bswap_64(event->stat_round.time); + return 0; } -static void perf_event__time_conv_swap(union perf_event *event, - bool sample_id_all __maybe_unused) +static int perf_event__time_conv_swap(union perf_event *event, + bool sample_id_all __maybe_unused) { event->time_conv.time_shift = bswap_64(event->time_conv.time_shift); event->time_conv.time_mult = bswap_64(event->time_conv.time_mult); event->time_conv.time_zero = bswap_64(event->time_conv.time_zero); - if (event_contains(event->time_conv, time_cycles)) { + if (event_contains(event->time_conv, time_cycles)) event->time_conv.time_cycles = bswap_64(event->time_conv.time_cycles); + if (event_contains(event->time_conv, time_mask)) event->time_conv.time_mask = bswap_64(event->time_conv.time_mask); - } + return 0; } -static void +static int perf_event__compressed2_swap(union perf_event *event, + bool sample_id_all __maybe_unused) +{ + /* Only data_size needs swapping — compressed payload is a raw byte stream */ + event->pack2.data_size = bswap_64(event->pack2.data_size); + return 0; +} + +static int perf_event__bpf_metadata_swap(union perf_event *event, + bool sample_id_all __maybe_unused) +{ + u64 i, nr, max_nr; + + /* Fixed header must fit before accessing nr_entries or prog_name */ + if (event->header.size < sizeof(event->bpf_metadata)) + return -1; + + event->bpf_metadata.nr_entries = bswap_64(event->bpf_metadata.nr_entries); + + /* + * Ensure NUL-termination on the cross-endian path where the + * mapping is writable (MAP_PRIVATE + PROT_WRITE). Fixing + * the string in place is preferred over rejecting because it + * preserves the event for downstream processing — only the + * last byte is lost. + * + * The native-endian path (MAP_SHARED + PROT_READ) cannot + * write, so it validates and skips unterminated events in + * perf_session__process_user_event() instead. The two + * strategies produce different outcomes for the same + * malformed input (fix vs skip), which is inherent in the + * writable-vs-read-only mapping model. + */ + event->bpf_metadata.prog_name[BPF_PROG_NAME_LEN - 1] = '\0'; + + nr = event->bpf_metadata.nr_entries; + max_nr = (event->header.size - sizeof(event->bpf_metadata)) / + sizeof(event->bpf_metadata.entries[0]); + if (nr > max_nr) { + /* Persist clamped value so the native path processes entries, not skips */ + nr = max_nr; + event->bpf_metadata.nr_entries = nr; + } + + for (i = 0; i < nr; i++) { + event->bpf_metadata.entries[i].key[BPF_METADATA_KEY_LEN - 1] = '\0'; + event->bpf_metadata.entries[i].value[BPF_METADATA_VALUE_LEN - 1] = '\0'; + } + return 0; +} +static int perf_event__schedstat_cpu_swap(union perf_event *event __maybe_unused, bool sample_id_all __maybe_unused) { /* FIXME */ + return 0; } -static void +static int perf_event__schedstat_domain_swap(union perf_event *event __maybe_unused, bool sample_id_all __maybe_unused) { /* FIXME */ + return 0; +} + +static int perf_event__ksymbol_swap(union perf_event *event, + bool sample_id_all) +{ + event->ksymbol.addr = bswap_64(event->ksymbol.addr); + event->ksymbol.len = bswap_32(event->ksymbol.len); + event->ksymbol.ksym_type = bswap_16(event->ksymbol.ksym_type); + event->ksymbol.flags = bswap_16(event->ksymbol.flags); + + if (sample_id_all) { + void *data = &event->ksymbol.name; + void *end = (void *)event + event->header.size; + size_t len = strnlen(data, end - data); + + /* See comment in perf_event__comm_swap() */ + if (len == (size_t)(end - data)) + return -1; + data += PERF_ALIGN(len + 1, sizeof(u64)); + swap_sample_id_all(event, data); + } + return 0; +} + +static int perf_event__bpf_event_swap(union perf_event *event, + bool sample_id_all) +{ + event->bpf.type = bswap_16(event->bpf.type); + event->bpf.flags = bswap_16(event->bpf.flags); + event->bpf.id = bswap_32(event->bpf.id); + + if (sample_id_all) + swap_sample_id_all(event, &event->bpf + 1); + return 0; } -typedef void (*perf_event__swap_op)(union perf_event *event, - bool sample_id_all); +static int perf_event__header_feature_swap(union perf_event *event, + bool sample_id_all __maybe_unused) +{ + event->feat.feat_id = bswap_64(event->feat.feat_id); + return 0; +} + +typedef int (*perf_event__swap_op)(union perf_event *event, + bool sample_id_all); static perf_event__swap_op perf_event__swap_ops[] = { [PERF_RECORD_MMAP] = perf_event__mmap_swap, @@ -740,13 +1185,16 @@ static perf_event__swap_op perf_event__swap_ops[] = { [PERF_RECORD_SWITCH_CPU_WIDE] = perf_event__switch_swap, [PERF_RECORD_NAMESPACES] = perf_event__namespaces_swap, [PERF_RECORD_CGROUP] = perf_event__cgroup_swap, + [PERF_RECORD_KSYMBOL] = perf_event__ksymbol_swap, + [PERF_RECORD_BPF_EVENT] = perf_event__bpf_event_swap, [PERF_RECORD_TEXT_POKE] = perf_event__text_poke_swap, [PERF_RECORD_AUX_OUTPUT_HW_ID] = perf_event__all64_swap, [PERF_RECORD_CALLCHAIN_DEFERRED] = perf_event__all64_swap, [PERF_RECORD_HEADER_ATTR] = perf_event__hdr_attr_swap, [PERF_RECORD_HEADER_EVENT_TYPE] = perf_event__event_type_swap, [PERF_RECORD_HEADER_TRACING_DATA] = perf_event__tracing_data_swap, - [PERF_RECORD_HEADER_BUILD_ID] = NULL, + [PERF_RECORD_HEADER_BUILD_ID] = perf_event__build_id_swap, + [PERF_RECORD_HEADER_FEATURE] = perf_event__header_feature_swap, [PERF_RECORD_ID_INDEX] = perf_event__all64_swap, [PERF_RECORD_AUXTRACE_INFO] = perf_event__auxtrace_info_swap, [PERF_RECORD_AUXTRACE] = perf_event__auxtrace_swap, @@ -758,6 +1206,8 @@ static perf_event__swap_op perf_event__swap_ops[] = { [PERF_RECORD_STAT_ROUND] = perf_event__stat_round_swap, [PERF_RECORD_EVENT_UPDATE] = perf_event__event_update_swap, [PERF_RECORD_TIME_CONV] = perf_event__time_conv_swap, + [PERF_RECORD_COMPRESSED2] = perf_event__compressed2_swap, + [PERF_RECORD_BPF_METADATA] = perf_event__bpf_metadata_swap, [PERF_RECORD_SCHEDSTAT_CPU] = perf_event__schedstat_cpu_swap, [PERF_RECORD_SCHEDSTAT_DOMAIN] = perf_event__schedstat_domain_swap, [PERF_RECORD_HEADER_MAX] = NULL, @@ -1117,9 +1567,10 @@ char *get_page_size_name(u64 size, char *str) return str; } -static void dump_sample(struct machine *machine, struct evsel *evsel, union perf_event *event, +static void dump_sample(struct machine *machine, union perf_event *event, struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; u64 sample_type; char str[PAGE_SIZE_NAME_LEN]; uint16_t e_machine = EM_NONE; @@ -1183,9 +1634,10 @@ static void dump_sample(struct machine *machine, struct evsel *evsel, union perf sample_read__printf(sample, evsel->core.attr.read_format); } -static void dump_deferred_callchain(struct evsel *evsel, union perf_event *event, - struct perf_sample *sample) +static void dump_deferred_callchain(union perf_event *event, struct perf_sample *sample) { + struct evsel *evsel = sample->evsel; + if (!dump_trace) return; @@ -1198,8 +1650,9 @@ static void dump_deferred_callchain(struct evsel *evsel, union perf_event *event static void dump_read(struct evsel *evsel, union perf_event *event) { - struct perf_record_read *read_event = &event->read; u64 read_format; + __u64 *array; + void *end; if (!dump_trace) return; @@ -1211,18 +1664,37 @@ static void dump_read(struct evsel *evsel, union perf_event *event) return; read_format = evsel->core.attr.read_format; + /* + * The kernel packs only the enabled read_format fields + * after value, with no gaps. Walk the packed array + * instead of using fixed struct offsets. + */ + array = &event->read.value + 1; + end = (void *)event + event->header.size; - if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) - printf("... time enabled : %" PRI_lu64 "\n", read_event->time_enabled); + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { + if ((void *)(array + 1) > end) + return; + printf("... time enabled : %" PRI_lu64 "\n", *array++); + } - if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) - printf("... time running : %" PRI_lu64 "\n", read_event->time_running); + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { + if ((void *)(array + 1) > end) + return; + printf("... time running : %" PRI_lu64 "\n", *array++); + } - if (read_format & PERF_FORMAT_ID) - printf("... id : %" PRI_lu64 "\n", read_event->id); + if (read_format & PERF_FORMAT_ID) { + if ((void *)(array + 1) > end) + return; + printf("... id : %" PRI_lu64 "\n", *array++); + } - if (read_format & PERF_FORMAT_LOST) - printf("... lost : %" PRI_lu64 "\n", read_event->lost); + if (read_format & PERF_FORMAT_LOST) { + if ((void *)(array + 1) > end) + return; + printf("... lost : %" PRI_lu64 "\n", *array++); + } } static struct machine *machines__find_for_cpumode(struct machines *machines, @@ -1291,7 +1763,7 @@ static int deliver_sample_value(struct evlist *evlist, return 0; sample->evsel = container_of(sid->evsel, struct evsel, core); - ret = tool->sample(tool, event, sample, sample->evsel, machine); + ret = tool->sample(tool, event, sample, machine); sample->evsel = saved_evsel; return ret; } @@ -1323,8 +1795,9 @@ static int deliver_sample_group(struct evlist *evlist, static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine) + struct machine *machine) { + struct evsel *evsel = sample->evsel; /* We know evsel != NULL. */ u64 sample_type = evsel->core.attr.sample_type; u64 read_format = evsel->core.attr.read_format; @@ -1332,7 +1805,7 @@ static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool /* Standard sample delivery. */ if (!(sample_type & PERF_SAMPLE_READ)) - return tool->sample(tool, event, sample, evsel, machine); + return tool->sample(tool, event, sample, machine); /* For PERF_SAMPLE_READ we have either single or group mode. */ if (read_format & PERF_FORMAT_GROUP) @@ -1352,6 +1825,7 @@ static int evlist__deliver_sample(struct evlist *evlist, const struct perf_tool struct deferred_event { struct list_head list; union perf_event *event; + u64 file_offset; }; /* @@ -1371,8 +1845,7 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist, struct evsel *saved_evsel = sample->evsel; sample->evsel = evlist__id2evsel(evlist, sample->id); - ret = tool->callchain_deferred(tool, event, sample, - sample->evsel, machine); + ret = tool->callchain_deferred(tool, event, sample, machine); sample->evsel = saved_evsel; return ret; } @@ -1387,6 +1860,7 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist, perf_sample__exit(&orig_sample); break; } + orig_sample.file_offset = de->file_offset; if (sample->tid != orig_sample.tid) { perf_sample__exit(&orig_sample); @@ -1400,7 +1874,7 @@ static int evlist__deliver_deferred_callchain(struct evlist *evlist, orig_sample.evsel = evlist__id2evsel(evlist, orig_sample.id); ret = evlist__deliver_sample(evlist, tool, de->event, - &orig_sample, orig_sample.evsel, machine); + &orig_sample, machine); perf_sample__exit(&orig_sample); list_del(&de->list); @@ -1435,10 +1909,11 @@ static int session__flush_deferred_samples(struct perf_session *session, perf_sample__exit(&sample); break; } + sample.file_offset = de->file_offset; sample.evsel = evlist__id2evsel(evlist, sample.id); ret = evlist__deliver_sample(evlist, tool, de->event, - &sample, sample.evsel, machine); + &sample, machine); perf_sample__exit(&sample); list_del(&de->list); @@ -1451,6 +1926,26 @@ static int session__flush_deferred_samples(struct perf_session *session, return ret; } +/* + * Return true if the string field is properly null-terminated + * within the event boundary. Native-endian files are mapped + * read-only (MAP_SHARED + PROT_READ) so we cannot write a + * null byte in place; skip the event instead. + */ +static bool perf_event__check_nul(const char *str, const void *end, + const char *event_name, u64 file_offset) +{ + size_t max_len = (const char *)end - str; + + if (max_len == 0 || strnlen(str, max_len) == max_len) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_%s: string not null-terminated, skipping event\n", + file_offset, event_name); + return false; + } + + return true; +} + static int machines__deliver_event(struct machines *machines, struct evlist *evlist, union perf_event *event, @@ -1458,7 +1953,6 @@ static int machines__deliver_event(struct machines *machines, const struct perf_tool *tool, u64 file_offset, const char *file_path) { - struct evsel *evsel; struct machine *machine; dump_event(evlist, event, file_offset, sample, file_path); @@ -1468,21 +1962,20 @@ static int machines__deliver_event(struct machines *machines, else assert(sample->evsel == evlist__id2evsel(evlist, sample->id)); - evsel = sample->evsel; machine = machines__find_for_cpumode(machines, event, sample); switch (event->header.type) { case PERF_RECORD_SAMPLE: - if (evsel == NULL) { + if (sample->evsel == NULL) { ++evlist->stats.nr_unknown_id; return 0; } if (machine == NULL) { ++evlist->stats.nr_unprocessable_samples; - dump_sample(machine, evsel, event, sample); + dump_sample(machine, event, sample); return 0; } - dump_sample(machine, evsel, event, sample); + dump_sample(machine, event, sample); if (sample->deferred_callchain && tool->merge_deferred_callchains) { struct deferred_event *de = malloc(sizeof(*de)); size_t sz = event->header.size; @@ -1496,21 +1989,57 @@ static int machines__deliver_event(struct machines *machines, return -ENOMEM; } memcpy(de->event, event, sz); + de->file_offset = sample->file_offset; list_add_tail(&de->list, &evlist->deferred_samples); return 0; } - return evlist__deliver_sample(evlist, tool, event, sample, evsel, machine); + return evlist__deliver_sample(evlist, tool, event, sample, machine); case PERF_RECORD_MMAP: + if (!perf_event__check_nul(event->mmap.filename, + (void *)event + event->header.size, + "MMAP", file_offset)) + return 0; return tool->mmap(tool, event, sample, machine); case PERF_RECORD_MMAP2: if (event->header.misc & PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT) ++evlist->stats.nr_proc_map_timeout; + if (!perf_event__check_nul(event->mmap2.filename, + (void *)event + event->header.size, + "MMAP2", file_offset)) + return 0; return tool->mmap2(tool, event, sample, machine); case PERF_RECORD_COMM: + if (!perf_event__check_nul(event->comm.comm, + (void *)event + event->header.size, + "COMM", file_offset)) + return 0; return tool->comm(tool, event, sample, machine); - case PERF_RECORD_NAMESPACES: + case PERF_RECORD_NAMESPACES: { + /* + * Cannot underflow: perf_event__min_size[] guarantees header.size >= sizeof. + * Includes trailing sample_id space when present, but prevents OOB. + */ + u64 max_nr = (event->header.size - sizeof(event->namespaces)) / + sizeof(event->namespaces.link_info[0]); + + /* + * Native-endian events are mmap'd read-only, so we + * cannot clamp nr in place. Skip the event instead. + * The swap handler already clamps on the writable + * cross-endian path. + */ + if (event->namespaces.nr_namespaces > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_NAMESPACES: nr_namespaces %" PRIu64 " exceeds payload (max %" PRIu64 "), skipping\n", + file_offset, (u64)event->namespaces.nr_namespaces, max_nr); + return 0; + } return tool->namespaces(tool, event, sample, machine); + } case PERF_RECORD_CGROUP: + if (!perf_event__check_nul(event->cgroup.path, + (void *)event + event->header.size, + "CGROUP", file_offset)) + return 0; return tool->cgroup(tool, event, sample, machine); case PERF_RECORD_FORK: return tool->fork(tool, event, sample, machine); @@ -1527,8 +2056,8 @@ static int machines__deliver_event(struct machines *machines, evlist->stats.total_lost_samples += event->lost_samples.lost; return tool->lost_samples(tool, event, sample, machine); case PERF_RECORD_READ: - dump_read(evsel, event); - return tool->read(tool, event, sample, evsel, machine); + dump_read(sample->evsel, event); + return tool->read(tool, event, sample, machine); case PERF_RECORD_THROTTLE: return tool->throttle(tool, event, sample, machine); case PERF_RECORD_UNTHROTTLE: @@ -1549,15 +2078,30 @@ static int machines__deliver_event(struct machines *machines, case PERF_RECORD_SWITCH_CPU_WIDE: return tool->context_switch(tool, event, sample, machine); case PERF_RECORD_KSYMBOL: + if (!perf_event__check_nul(event->ksymbol.name, + (void *)event + event->header.size, + "KSYMBOL", file_offset)) + return 0; return tool->ksymbol(tool, event, sample, machine); case PERF_RECORD_BPF_EVENT: return tool->bpf(tool, event, sample, machine); - case PERF_RECORD_TEXT_POKE: + case PERF_RECORD_TEXT_POKE: { + /* offsetof(bytes), not sizeof — sizeof includes padding past the flexible array */ + size_t text_poke_len = offsetof(struct perf_record_text_poke_event, bytes) + + event->text_poke.old_len + + event->text_poke.new_len; + + if (event->header.size < text_poke_len) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_TEXT_POKE: old_len+new_len exceeds event, skipping\n", + file_offset); + return 0; + } return tool->text_poke(tool, event, sample, machine); + } case PERF_RECORD_AUX_OUTPUT_HW_ID: return tool->aux_output_hw_id(tool, event, sample, machine); case PERF_RECORD_CALLCHAIN_DEFERRED: - dump_deferred_callchain(evsel, event, sample); + dump_deferred_callchain(event, sample); return evlist__deliver_deferred_callchain(evlist, tool, event, sample, machine); default: @@ -1573,14 +2117,104 @@ static int perf_session__deliver_event(struct perf_session *session, const char *file_path) { struct perf_sample sample; + struct evsel *evsel; int ret; perf_sample__init(&sample, /*all=*/false); - ret = evlist__parse_sample(session->evlist, event, &sample); + evsel = evlist__event2evsel(session->evlist, event); + if (!evsel) { + pr_err("ERROR: at offset %#" PRIx64 ": no evsel found for %s (%u) event\n", + file_offset, perf_event__name(event->header.type), + event->header.type); + ret = -EFAULT; + goto out; + } + ret = evsel__parse_sample(evsel, event, &sample); if (ret) { - pr_err("Can't parse sample, err = %d\n", ret); + pr_err("ERROR: at offset %#" PRIx64 ": can't parse %s (%u) sample, err = %d\n", + file_offset, perf_event__name(event->header.type), + event->header.type, ret); goto out; } + sample.file_offset = file_offset; + /* + * evsel__parse_sample() doesn't populate machine_pid/vcpu, + * which are needed by machines__find_for_cpumode() to + * attribute samples to guest VMs. The SID table maps + * sample IDs to the guest that owns the event. + */ + if (perf_guest && sample.id) { + struct perf_sample_id *sid = evlist__id2sid(session->evlist, sample.id); + + if (sid) { + sample.machine_pid = sid->machine_pid; + sample.vcpu = sid->vcpu.cpu; + } + } + + /* + * Validate sample.cpu before any callback can use it as an + * array index (kwork cpus_runtime, timechart cpus_cstate_*, + * sched cpu_last_switched). + * + * When PERF_SAMPLE_CPU is absent, evsel__parse_sample() leaves + * sample.cpu as (u32)-1 — a sentinel that downstream tools + * (script, inject) check to identify events without CPU info. + * Only check when sample.cpu was actually populated from event + * data: PERF_RECORD_SAMPLE always has it when PERF_SAMPLE_CPU + * is set; non-sample events only have it when sample_id_all is + * enabled. Otherwise sample.cpu is the (u32)-1 sentinel from + * evsel__parse_sample() and must not be validated or clamped. + */ + if ((evsel->core.attr.sample_type & PERF_SAMPLE_CPU) && + (event->header.type == PERF_RECORD_SAMPLE || + evsel->core.attr.sample_id_all)) { + int nr_cpus_avail = perf_session__env(session)->nr_cpus_avail; + + /* + * For perf.data files the MAX_NR_CPUS fallback in + * perf_session__read_header() guarantees this is set. + * For pipe mode, HEADER_NRCPUS may arrive late or not + * at all (pre-2017 perf, third-party tools). Fall + * back to MAX_NR_CPUS so the bounds check still works + * against fixed-size downstream arrays. + * + * Do NOT write back to env: this function runs during + * recording (synthesized events) when nr_cpus_avail is + * legitimately 0. Writing MAX_NR_CPUS would cause + * write_cpu_topology() to emit 4096 core_id/socket_id + * pairs instead of the real CPU count, corrupting the + * topology section in the generated perf.data. + */ + if (nr_cpus_avail <= 0) + nr_cpus_avail = MAX_NR_CPUS; + /* + * Cap at MAX_NR_CPUS for the bounds check — downstream + * consumers use fixed-size arrays of that size. Keep + * the true nr_cpus_avail in env for header parsing + * (e.g. process_cpu_topology) which needs the real count. + */ + if (nr_cpus_avail > MAX_NR_CPUS) + nr_cpus_avail = MAX_NR_CPUS; + if (sample.cpu >= (u32)nr_cpus_avail && + sample.cpu != (u32)-1) { + /* + * Warn rather than abort: synthesized events + * (MMAP, COMM) lack sample_id_all data, so + * parse_id_sample reads garbage from the event + * payload. Clamping to 0 protects downstream + * array indexing while keeping the session alive. + * + * Preserve (u32)-1: perf script and perf inject + * use it as a sentinel for "CPU not applicable." + * Downstream array users (timechart, kwork) have + * their own per-callback bounds checks. + */ + pr_warning_once("WARNING: at offset %#" PRIx64 ": sample CPU %u >= nr_cpus_avail %u, clamping to 0\n", + file_offset, sample.cpu, nr_cpus_avail); + sample.cpu = 0; + } + } ret = auxtrace__process_event(session, event, &sample, tool); if (ret < 0) @@ -1607,6 +2241,7 @@ static s64 perf_session__process_user_event(struct perf_session *session, { struct ordered_events *oe = &session->ordered_events; const struct perf_tool *tool = session->tool; + const u32 event_size = READ_ONCE(event->header.size); struct perf_sample sample; int fd = perf_data__fd(session->data); s64 err; @@ -1647,6 +2282,12 @@ static s64 perf_session__process_user_event(struct perf_session *session, err = tool->tracing_data(tool, session, event); break; case PERF_RECORD_HEADER_BUILD_ID: + if (!perf_event__check_nul(event->build_id.filename, + (void *)event + event_size, + "HEADER_BUILD_ID", file_offset)) { + err = 0; + break; + } err = tool->build_id(tool, session, event); break; case PERF_RECORD_FINISHED_ROUND: @@ -1665,22 +2306,119 @@ static s64 perf_session__process_user_event(struct perf_session *session, * place already. */ if (!perf_data__is_pipe(session->data)) - lseek(fd, file_offset + event->header.size, SEEK_SET); + lseek(fd, file_offset + event_size, SEEK_SET); err = tool->auxtrace(tool, session, event); break; case PERF_RECORD_AUXTRACE_ERROR: perf_session__auxtrace_error_inc(session, event); err = tool->auxtrace_error(tool, session, event); break; - case PERF_RECORD_THREAD_MAP: + case PERF_RECORD_THREAD_MAP: { + u64 max_nr; + + if (event_size < sizeof(event->thread_map)) { + pr_err("ERROR: at offset %#" PRIx64 ": PERF_RECORD_THREAD_MAP: header.size (%u) too small\n", + file_offset, event_size); + err = -EINVAL; + break; + } + + max_nr = (event_size - sizeof(event->thread_map)) / + sizeof(event->thread_map.entries[0]); + if (event->thread_map.nr > max_nr) { + pr_err("ERROR: at offset %#" PRIx64 ": PERF_RECORD_THREAD_MAP: nr %" PRIu64 " exceeds max %" PRIu64 "\n", + file_offset, (u64)event->thread_map.nr, max_nr); + err = -EINVAL; + break; + } + err = tool->thread_map(tool, session, event); break; - case PERF_RECORD_CPU_MAP: + } + case PERF_RECORD_CPU_MAP: { + struct perf_record_cpu_map_data *data = &event->cpu_map.data; + u32 payload = event_size - sizeof(event->header); + + /* + * Native-endian events are mmap'd read-only, so we + * cannot clamp nr fields in place. Skip the event + * if any variant overflows. + */ + switch (data->type) { + case PERF_CPU_MAP__CPUS: { + u16 max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + cpus_data.cpu)) / + sizeof(data->cpus_data.cpu[0]); + + if (data->cpus_data.nr > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP: nr %u exceeds payload (max %u), skipping\n", + file_offset, data->cpus_data.nr, max_nr); + err = 0; + goto out; + } + break; + } + case PERF_CPU_MAP__MASK: + if (data->mask32_data.long_size == 4) { + u16 max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + mask32_data.mask)) / + sizeof(data->mask32_data.mask[0]); + + if (data->mask32_data.nr > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP mask32: nr %u exceeds payload (max %u), skipping\n", + file_offset, data->mask32_data.nr, max_nr); + err = 0; + goto out; + } + } else if (data->mask64_data.long_size == 8) { + u16 max_nr; + + if (payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) { + err = 0; + goto out; + } + max_nr = (payload - offsetof(struct perf_record_cpu_map_data, + mask64_data.mask)) / + sizeof(data->mask64_data.mask[0]); + if (data->mask64_data.nr > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP mask64: nr %u exceeds payload (max %u), skipping\n", + file_offset, data->mask64_data.nr, max_nr); + err = 0; + goto out; + } + } else { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_CPU_MAP: unsupported long_size %u, skipping\n", + file_offset, data->mask32_data.long_size); + err = 0; + goto out; + } + break; + default: + break; + } + err = tool->cpu_map(tool, session, event); break; - case PERF_RECORD_STAT_CONFIG: + } + case PERF_RECORD_STAT_CONFIG: { + /* Cannot underflow: perf_event__min_size[] guarantees event_size >= sizeof */ + u64 max_nr = (event_size - sizeof(event->stat_config)) / + sizeof(event->stat_config.data[0]); + + /* + * Native-endian events are mmap'd read-only, so we + * cannot clamp nr in place. Skip the event instead. + */ + if (event->stat_config.nr > max_nr) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_STAT_CONFIG: nr %" PRIu64 " exceeds payload (max %" PRIu64 "), skipping\n", + file_offset, (u64)event->stat_config.nr, max_nr); + err = 0; + goto out; + } + err = tool->stat_config(tool, session, event); break; + } case PERF_RECORD_STAT: err = tool->stat(tool, session, event); break; @@ -1688,7 +2426,14 @@ static s64 perf_session__process_user_event(struct perf_session *session, err = tool->stat_round(tool, session, event); break; case PERF_RECORD_TIME_CONV: - session->time_conv = event->time_conv; + /* + * Bounded copy: older kernels emit a shorter struct + * without time_cycles/time_mask/cap_user_time_*. + * Zero the rest so extended fields default to off. + */ + memset(&session->time_conv, 0, sizeof(session->time_conv)); + memcpy(&session->time_conv, &event->time_conv, + min((size_t)event_size, sizeof(session->time_conv))); err = tool->time_conv(tool, session, event); break; case PERF_RECORD_HEADER_FEATURE: @@ -1703,9 +2448,53 @@ static s64 perf_session__process_user_event(struct perf_session *session, case PERF_RECORD_FINISHED_INIT: err = tool->finished_init(tool, session, event); break; - case PERF_RECORD_BPF_METADATA: + case PERF_RECORD_BPF_METADATA: { + u64 nr_entries, max_entries; + + if (event_size < sizeof(event->bpf_metadata)) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: header.size (%u) too small, skipping\n", + file_offset, event_size); + err = 0; + break; + } + + /* + * Native-endian files are mmap'd read-only — validate + * NUL-termination instead of writing. + */ + if (strnlen(event->bpf_metadata.prog_name, + BPF_PROG_NAME_LEN) == BPF_PROG_NAME_LEN) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: prog_name not null-terminated, skipping\n", + file_offset); + err = 0; + break; + } + + nr_entries = READ_ONCE(event->bpf_metadata.nr_entries); + max_entries = (event_size - sizeof(event->bpf_metadata)) / + sizeof(event->bpf_metadata.entries[0]); + if (nr_entries > max_entries) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: nr_entries %" PRIu64 " exceeds max %" PRIu64 ", skipping\n", + file_offset, nr_entries, max_entries); + err = 0; + break; + } + + for (u64 i = 0; i < nr_entries; i++) { + if (strnlen(event->bpf_metadata.entries[i].key, + BPF_METADATA_KEY_LEN) == BPF_METADATA_KEY_LEN || + strnlen(event->bpf_metadata.entries[i].value, + BPF_METADATA_VALUE_LEN) == BPF_METADATA_VALUE_LEN) { + pr_warning("WARNING: at offset %#" PRIx64 ": PERF_RECORD_BPF_METADATA: entry %" PRIu64 " key/value not null-terminated, skipping\n", + file_offset, i); + err = 0; + goto out; + } + } + err = tool->bpf_metadata(tool, session, event); break; + } case PERF_RECORD_SCHEDSTAT_CPU: err = tool->schedstat_cpu(tool, session, event); break; @@ -1716,6 +2505,7 @@ static s64 perf_session__process_user_event(struct perf_session *session, err = -EINVAL; break; } +out: perf_sample__exit(&sample); return err; } @@ -1759,15 +2549,142 @@ int perf_session__deliver_synth_attr_event(struct perf_session *session, return perf_session__deliver_synth_event(session, &ev.ev, NULL); } -static void event_swap(union perf_event *event, bool sample_id_all) +/* Caller must ensure event->header.type < PERF_RECORD_HEADER_MAX */ +static int event_swap(union perf_event *event, bool sample_id_all) { - perf_event__swap_op swap; + perf_event__swap_op swap = perf_event__swap_ops[event->header.type]; - swap = perf_event__swap_ops[event->header.type]; if (swap) - swap(event, sample_id_all); + return swap(event, sample_id_all); + return 0; +} + +/* + * Minimum event sizes indexed by type. Checked before swap and + * processing so that both cross-endian and native-endian paths + * are protected from accessing fields past the event boundary. + * Zero means no minimum beyond the 8-byte header (already + * enforced by the reader). + * + * These values represent the smallest event the kernel has ever + * emitted for each type, so they do not reject legitimate legacy + * perf.data files from older kernels. Variable-length events + * use offsetof() to the first variable field; the variable + * content is validated separately (e.g., perf_event__check_nul). + */ +static const u32 perf_event__min_size[PERF_RECORD_HEADER_MAX] = { + /* + * offsetof() + 1 for types with a trailing variable-length + * string (filename, comm, path, name, msg): the +1 ensures + * room for at least a null terminator. Full null-termination + * within the event boundary is checked separately. + * + * PERF_RECORD_SAMPLE is omitted: all64_swap is bounded by + * header.size, and the internal layout varies by sample_type + * so a fixed minimum is not meaningful. + */ + [PERF_RECORD_MMAP] = offsetof(struct perf_record_mmap, filename) + 1, + [PERF_RECORD_LOST] = sizeof(struct perf_record_lost), + [PERF_RECORD_COMM] = offsetof(struct perf_record_comm, comm) + 1, + [PERF_RECORD_EXIT] = sizeof(struct perf_record_fork), + [PERF_RECORD_THROTTLE] = sizeof(struct perf_record_throttle), + [PERF_RECORD_UNTHROTTLE] = sizeof(struct perf_record_throttle), + [PERF_RECORD_FORK] = sizeof(struct perf_record_fork), + /* + * The kernel dynamically sizes PERF_RECORD_READ based on + * attr.read_format — only the enabled fields are emitted, + * packed with no gaps. The minimum valid event has just + * pid + tid + one u64 value (no optional fields). + */ + [PERF_RECORD_READ] = offsetof(struct perf_record_read, time_enabled), + [PERF_RECORD_MMAP2] = offsetof(struct perf_record_mmap2, filename) + 1, + [PERF_RECORD_LOST_SAMPLES] = sizeof(struct perf_record_lost_samples), + [PERF_RECORD_AUX] = sizeof(struct perf_record_aux), + [PERF_RECORD_ITRACE_START] = sizeof(struct perf_record_itrace_start), + [PERF_RECORD_SWITCH] = sizeof(struct perf_event_header), + [PERF_RECORD_SWITCH_CPU_WIDE] = sizeof(struct perf_record_switch), + [PERF_RECORD_NAMESPACES] = sizeof(struct perf_record_namespaces), + [PERF_RECORD_CGROUP] = offsetof(struct perf_record_cgroup, path) + 1, + [PERF_RECORD_TEXT_POKE] = sizeof(struct perf_record_text_poke_event), + [PERF_RECORD_KSYMBOL] = offsetof(struct perf_record_ksymbol, name) + 1, + [PERF_RECORD_BPF_EVENT] = sizeof(struct perf_record_bpf_event), + [PERF_RECORD_HEADER_ATTR] = sizeof(struct perf_event_header) + PERF_ATTR_SIZE_VER0, + [PERF_RECORD_HEADER_EVENT_TYPE] = sizeof(struct perf_record_header_event_type), + /* Legacy events predate the __u32 pad field, accept 12-byte records */ + [PERF_RECORD_HEADER_TRACING_DATA] = offsetof(struct perf_record_header_tracing_data, pad), + [PERF_RECORD_AUX_OUTPUT_HW_ID] = sizeof(struct perf_record_aux_output_hw_id), + [PERF_RECORD_AUXTRACE_INFO] = sizeof(struct perf_record_auxtrace_info), + [PERF_RECORD_AUXTRACE] = sizeof(struct perf_record_auxtrace), + [PERF_RECORD_AUXTRACE_ERROR] = offsetof(struct perf_record_auxtrace_error, msg) + 1, + [PERF_RECORD_THREAD_MAP] = sizeof(struct perf_record_thread_map), + /* + * sizeof(perf_record_cpu_map) is 20 because the outer struct + * isn't packed and GCC adds 2 bytes of trailing padding. + * The smallest valid variant (RANGE_CPUS) is only 16 bytes: + * header(8) + type(2) + range_cpu_data(6). Per-variant + * bounds are checked in the swap handler via payload. + */ + [PERF_RECORD_CPU_MAP] = sizeof(struct perf_event_header) + + sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map), + [PERF_RECORD_STAT_CONFIG] = sizeof(struct perf_record_stat_config), + [PERF_RECORD_STAT] = sizeof(struct perf_record_stat), + [PERF_RECORD_STAT_ROUND] = sizeof(struct perf_record_stat_round), + /* + * EVENT_UPDATE has a union whose largest member (cpus) + * inflates sizeof to 40, but SCALE events are only 32 + * and UNIT/NAME events can be even smaller. Use the + * fixed header fields (header + type + id) as minimum. + */ + [PERF_RECORD_EVENT_UPDATE] = offsetof(struct perf_record_event_update, scale), + [PERF_RECORD_TIME_CONV] = offsetof(struct perf_record_time_conv, time_cycles), + [PERF_RECORD_ID_INDEX] = sizeof(struct perf_record_id_index), + [PERF_RECORD_HEADER_BUILD_ID] = sizeof(struct perf_record_header_build_id), + [PERF_RECORD_HEADER_FEATURE] = sizeof(struct perf_record_header_feature), + [PERF_RECORD_COMPRESSED2] = sizeof(struct perf_record_compressed2), + [PERF_RECORD_BPF_METADATA] = sizeof(struct perf_record_bpf_metadata), + [PERF_RECORD_CALLCHAIN_DEFERRED] = sizeof(struct perf_event_header) + sizeof(__u64), + /* + * SCHEDSTAT events have a version-dependent union after the + * fixed header fields; the minimum is the base (pre-union) + * portion so old and new versions both pass. + */ + [PERF_RECORD_SCHEDSTAT_CPU] = offsetof(struct perf_record_schedstat_cpu, v15), + [PERF_RECORD_SCHEDSTAT_DOMAIN] = offsetof(struct perf_record_schedstat_domain, v15), +}; + +/* + * Return true if the event is too small for its declared type. + * Caller must ensure event->header.type < PERF_RECORD_HEADER_MAX. + * If min is non-NULL, stores the required minimum on failure. + */ +static bool perf_event__too_small(const union perf_event *event, u32 *min) +{ + u32 min_sz = perf_event__min_size[event->header.type]; + + if (min_sz && event->header.size < min_sz) { + if (min) + *min = min_sz; + return true; + } + + return false; } +/* + * Read and validate the event at @file_offset. + * + * Returns: + * 0 — success: *event_ptr is set and safe to access. + * -1 — error; check *event_ptr to decide whether to advance or abort: + * *event_ptr set — event header was read but the event is + * malformed (too small for its type, or byte-swap + * failed). header.size is still valid, so the + * caller can advance past the event. + * *event_ptr NULL — fatal: couldn't read the header at all + * (I/O error, offset out of range, pipe mode). + * Caller must abort. + */ int perf_session__peek_event(struct perf_session *session, off_t file_offset, void *buf, size_t buf_sz, union perf_event **event_ptr, @@ -1775,52 +2692,108 @@ int perf_session__peek_event(struct perf_session *session, off_t file_offset, { union perf_event *event; size_t hdr_sz, rest; + u32 min_sz; int fd; + *event_ptr = NULL; + if (session->one_mmap && !session->header.needs_swap) { - event = file_offset - session->one_mmap_offset + - session->one_mmap_addr; - goto out_parse_sample; - } + u64 offset_in_mmap; - if (perf_data__is_pipe(session->data)) - return -1; + /* Validate offset with integer arithmetic to avoid pointer UB */ + if ((u64)file_offset < session->one_mmap_offset) + return -1; - fd = perf_data__fd(session->data); - hdr_sz = sizeof(struct perf_event_header); + offset_in_mmap = (u64)file_offset - session->one_mmap_offset; - if (buf_sz < hdr_sz) - return -1; + /* Use subtraction to avoid addition overflow */ + if (offset_in_mmap >= session->one_mmap_size || + session->one_mmap_size - offset_in_mmap < sizeof(struct perf_event_header)) + return -1; - if (lseek(fd, file_offset, SEEK_SET) == (off_t)-1 || - readn(fd, buf, hdr_sz) != (ssize_t)hdr_sz) - return -1; + event = session->one_mmap_addr + offset_in_mmap; - event = (union perf_event *)buf; + if (event->header.size < sizeof(struct perf_event_header)) + return -1; - if (session->header.needs_swap) - perf_event_header__bswap(&event->header); + /* Ensure full event is within the mmap region */ + if (session->one_mmap_size - offset_in_mmap < event->header.size) + return -1; + } else { + if (perf_data__is_pipe(session->data)) + return -1; - if (event->header.size < hdr_sz || event->header.size > buf_sz) - return -1; + fd = perf_data__fd(session->data); + hdr_sz = sizeof(struct perf_event_header); + + if (buf_sz < hdr_sz) + return -1; + + if (lseek(fd, file_offset, SEEK_SET) == (off_t)-1 || + readn(fd, buf, hdr_sz) != (ssize_t)hdr_sz) + return -1; + + event = (union perf_event *)buf; + + if (session->header.needs_swap) + perf_event_header__bswap(&event->header); + + if (event->header.size < hdr_sz || event->header.size > buf_sz) + return -1; - buf += hdr_sz; - rest = event->header.size - hdr_sz; + buf += hdr_sz; + rest = event->header.size - hdr_sz; - if (readn(fd, buf, rest) != (ssize_t)rest) + if (readn(fd, buf, rest) != (ssize_t)rest) + return -1; + } + + /* Event data is fully loaded — expose so callers can advance */ + *event_ptr = event; + + /* + * Check alignment before type: an unaligned size misaligns the + * stream for all subsequent reads regardless of event type. + * Three legacy user events predate the 8-byte rule — exempt them. + */ + if (event->header.size % sizeof(u64) && + event->header.type != PERF_RECORD_HEADER_TRACING_DATA && + event->header.type != PERF_RECORD_COMPRESSED && + event->header.type != PERF_RECORD_HEADER_FEATURE) { + pr_warning("WARNING: at offset %#" PRIx64 ": %s (%u) event size %u not aligned to %zu\n", + (u64)file_offset, perf_event__name(event->header.type), + event->header.type, event->header.size, sizeof(u64)); return -1; + } - if (session->header.needs_swap) - event_swap(event, evlist__sample_id_all(session->evlist)); + if (event->header.type >= PERF_RECORD_HEADER_MAX) { + pr_warning("WARNING: at offset %#" PRIx64 ": unsupported event type %u, skipping\n", + (u64)file_offset, event->header.type); + return 0; + } + + if (perf_event__too_small(event, &min_sz)) { + pr_warning("WARNING: at offset %#" PRIx64 ": %s (%u) event size %u too small (min %u)\n", + (u64)file_offset, perf_event__name(event->header.type), + event->header.type, event->header.size, min_sz); + return -1; + } -out_parse_sample: + if (session->header.needs_swap && + event_swap(event, evlist__sample_id_all(session->evlist))) { + /* + * The header was already swapped so header.size is + * valid — expose the event so callers can advance + * past this malformed entry instead of aborting. + */ + *event_ptr = event; + return -1; + } if (sample && event->header.type < PERF_RECORD_USER_TYPE_START && evlist__parse_sample(session->evlist, event, sample)) return -1; - *event_ptr = event; - return 0; } @@ -1833,11 +2806,37 @@ int perf_session__peek_events(struct perf_session *session, u64 offset, int err; do { + event = NULL; err = perf_session__peek_event(session, offset, buf, PERF_SAMPLE_MAX_SIZE, &event, NULL); - if (err) - return err; + if (err) { + /* + * Recoverable error: peek_event returns -1 but + * sets event_ptr when the header was read + * successfully but the event is malformed (too + * small or swap failed). Skip past it using + * header.size — don't invoke the callback since + * type-specific fields may be truncated. + * + * Must abort if: event_ptr is NULL (I/O error), + * size is 0 (can't advance), type is AUXTRACE + * (payload extends beyond header.size), or size + * is unaligned (would misalign all subsequent reads). + * + * Direct callers (auxtrace, cs-etm) treat any + * non-zero return as fatal — only this loop skips. + */ + if (event && event->header.size && + event->header.type != PERF_RECORD_AUXTRACE && + event->header.size % sizeof(u64) == 0) { + offset += event->header.size; + err = 0; + } else { + return err; + } + continue; + } err = cb(session, event, offset, data); if (err) @@ -1858,21 +2857,74 @@ static s64 perf_session__process_event(struct perf_session *session, { struct evlist *evlist = session->evlist; const struct perf_tool *tool = session->tool; + u32 min_sz; int ret; - if (session->header.needs_swap) - event_swap(event, evlist__sample_id_all(evlist)); + /* + * The kernel aligns all event sizes to sizeof(u64) — see + * perf_event_comm_event() (ALIGN), perf_event_mmap_event(), + * perf_event_cgroup(), perf_event_ksymbol() (IS_ALIGNED loops), + * and perf_event_text_poke() (ALIGN) in kernel/events/core.c. + * + * An unaligned size means the file is corrupted or crafted. + * Abort: there is no point continuing to read unaligned records + * because the caller advances rd->head by event->header.size, + * so every subsequent read would start at a misaligned offset, + * producing garbage headers for the rest of the file. + * + * Exempt three legacy user events that predate the alignment rule: + * + * TRACING_DATA (66): struct tracing_data_event was 12 bytes before + * b39c915a4f36 ("libperf event: Ensure tracing data is multiple + * of 8 sized") added __u32 pad; old perf.data files still contain + * 12-byte records. + * TODO: introduce HEADER_TRACING_DATA2 with guaranteed alignment. + * + * COMPRESSED (81): raw ZSTD output, arbitrary length. Already + * superseded by COMPRESSED2 (83) with PERF_ALIGN. + * + * HEADER_FEATURE (80): do_write_string() uses a 4-byte length + * prefix with no padding to 8-byte total. + * TODO: introduce HEADER_FEATURE2 with guaranteed alignment. + */ + if (event->header.size % sizeof(u64) && + event->header.type != PERF_RECORD_HEADER_TRACING_DATA && + event->header.type != PERF_RECORD_COMPRESSED && + event->header.type != PERF_RECORD_HEADER_FEATURE) { + pr_err("ERROR: at offset %#" PRIx64 ": %s (%u) event size %u is not 8-byte aligned, aborting\n", + file_offset, perf_event__name(event->header.type), + event->header.type, event->header.size); + return -EINVAL; + } if (event->header.type >= PERF_RECORD_HEADER_MAX) { - /* perf should not support unaligned event, stop here. */ - if (event->header.size % sizeof(u64)) - return -EINVAL; - /* This perf is outdated and does not support the latest event type. */ ui__warning("Unsupported header type %u, please consider updating perf.\n", event->header.type); - /* Skip unsupported event by returning its size. */ - return event->header.size; + /* + * Return 0 to skip: the caller (reader__read_event) + * already advances by event->header.size. + */ + return 0; + } + + /* + * Skip rather than abort: a too-small-but-aligned event + * can be safely stepped over without misaligning the stream. + */ + if (perf_event__too_small(event, &min_sz)) { + pr_warning("WARNING: at offset %#" PRIx64 ": %s (%u) event size %u too small (min %u), skipping\n", + file_offset, perf_event__name(event->header.type), + event->header.type, event->header.size, min_sz); + return 0; + } + + if (session->header.needs_swap && + event_swap(event, evlist__sample_id_all(evlist))) { + pr_warning("WARNING: at offset %#" PRIx64 ": swap failed for %s (%u) event, skipping\n", + file_offset, perf_event__name(event->header.type), + event->header.type); + return 0; } events_stats__inc(&evlist->stats, event->header.type); @@ -2342,6 +3394,17 @@ reader__mmap(struct reader *rd, struct perf_session *session) char *buf, **mmaps = rd->mmaps; u64 page_offset; + /* + * Native-endian: MAP_SHARED + PROT_READ — the kernel + * guarantees page-level coherence but a concurrent writer + * could modify the file between validation and use. This + * is a theoretical TOCTOU that affects the entire perf.data + * processing pipeline; fixing it would require copying each + * event to a private buffer before processing. + * + * Cross-endian: MAP_PRIVATE + PROT_WRITE — swap handlers + * get a copy-on-write snapshot immune to concurrent writes. + */ mmap_prot = PROT_READ; mmap_flags = MAP_SHARED; @@ -2373,6 +3436,14 @@ reader__mmap(struct reader *rd, struct perf_session *session) if (session->one_mmap) { session->one_mmap_addr = buf; session->one_mmap_offset = rd->file_offset; + /* + * mmap_size was set to the full file extent (data_offset + + * data_size) but file_offset was shifted forward by + * page_offset for page alignment. Reduce by page_offset + * so the bounds check reflects the file-backed portion + * of the mapping — pages beyond the file cause SIGBUS. + */ + session->one_mmap_size = rd->mmap_size - page_offset; } return 0; @@ -3023,14 +4094,19 @@ uint16_t perf_session__e_machine(struct perf_session *session, uint32_t *e_flags return EM_HOST; } + /* + * Is the env caching an e_machine? If not we want to compute from the + * more accurate threads. + */ env = perf_session__env(session); - if (env && env->e_machine != EM_NONE) { - if (e_flags) - *e_flags = env->e_flags; - - return env->e_machine; - } + if (env && env->e_machine != EM_NONE) + return perf_env__e_machine(env, e_flags); + /* + * Compute from threads, note this is more accurate than + * perf_env__e_machine that falls back on EM_HOST and doesn't consider + * mixed 32-bit and 64-bit threads. + */ machines__for_each_thread(&session->machines, perf_session__e_machine_cb, &args); @@ -3048,10 +4124,9 @@ uint16_t perf_session__e_machine(struct perf_session *session, uint32_t *e_flags /* * Couldn't determine from the perf_env or current set of - * threads. Default to the host. + * threads. Potentially use logic that uses the arch string otherwise + * default to the host. Don't cache in the perf_env in case later + * threads indicate a better ELF machine type. */ - if (e_flags) - *e_flags = EF_HOST; - - return EM_HOST; + return perf_env__e_machine_nocache(env, e_flags); } diff --git a/tools/perf/util/session.h b/tools/perf/util/session.h index f05f0d4a6c23..d554e2a1a50e 100644 --- a/tools/perf/util/session.h +++ b/tools/perf/util/session.h @@ -71,6 +71,8 @@ struct perf_session { void *one_mmap_addr; /** @one_mmap_offset: File offset in perf.data file when mapped. */ u64 one_mmap_offset; + /** @one_mmap_size: Size of the single mmap in bytes. */ + u64 one_mmap_size; /** @ordered_events: Used to turn unordered events into ordered ones. */ struct ordered_events ordered_events; /** @data: Optional perf data file being read from. */ diff --git a/tools/perf/util/setup.py b/tools/perf/util/setup.py index b65b1792ca05..a0ce76624a23 100644 --- a/tools/perf/util/setup.py +++ b/tools/perf/util/setup.py @@ -74,18 +74,17 @@ class install_lib(_install_lib): self.build_dir = build_lib -cflags = getenv('CFLAGS', '').split() # switch off several checks (need to be at the end of cflags list) -cflags += ['-fno-strict-aliasing', '-Wno-write-strings', '-Wno-unused-parameter', '-Wno-redundant-decls' ] +extra_cflags = ['-fno-strict-aliasing', '-Wno-write-strings', '-Wno-unused-parameter', '-Wno-redundant-decls' ] if cc_is_clang: - cflags += ["-Wno-unused-command-line-argument" ] + extra_cflags += ["-Wno-unused-command-line-argument" ] if clang_has_option("-Wno-cast-function-type-mismatch"): - cflags += ["-Wno-cast-function-type-mismatch" ] + extra_cflags += ["-Wno-cast-function-type-mismatch" ] else: - cflags += ['-Wno-cast-function-type' ] + extra_cflags += ['-Wno-cast-function-type' ] # The python headers have mixed code with declarations (decls after asserts, for instance) -cflags += [ "-Wno-declaration-after-statement" ] +extra_cflags += [ "-Wno-declaration-after-statement" ] src_perf = f'{srctree}/tools/perf' build_lib = getenv('PYTHON_EXTBUILD_LIB') @@ -94,7 +93,7 @@ build_tmp = getenv('PYTHON_EXTBUILD_TMP') perf = Extension('perf', sources = [ src_perf + '/util/python.c' ], include_dirs = ['util/include'], - extra_compile_args = cflags, + extra_compile_args = extra_cflags, ) setup(name='perf', diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c index 0020089cb13c..005e7d85dc4a 100644 --- a/tools/perf/util/sort.c +++ b/tools/perf/util/sort.c @@ -1,40 +1,45 @@ // SPDX-License-Identifier: GPL-2.0 +#include "sort.h" + #include <ctype.h> #include <errno.h> #include <inttypes.h> -#include <regex.h> #include <stdlib.h> + +#include <elf.h> +#include <linux/kernel.h> #include <linux/mman.h> +#include <linux/string.h> #include <linux/time64.h> + +#include <regex.h> + +#include "annotate-data.h" +#include "annotate.h" +#include "branch.h" +#include "cacheline.h" +#include "cgroup.h" +#include "comm.h" #include "debug.h" #include "dso.h" -#include "sort.h" +#include "event.h" +#include "evlist.h" +#include "evsel.h" #include "hist.h" -#include "cacheline.h" -#include "comm.h" +#include "machine.h" #include "map.h" -#include "maps.h" -#include "symbol.h" #include "map_symbol.h" -#include "branch.h" -#include "thread.h" -#include "evsel.h" -#include "evlist.h" -#include "srcline.h" -#include "strlist.h" -#include "strbuf.h" +#include "maps.h" #include "mem-events.h" #include "mem-info.h" -#include "annotate.h" -#include "annotate-data.h" -#include "event.h" -#include "time-utils.h" -#include "cgroup.h" -#include "machine.h" #include "session.h" +#include "srcline.h" +#include "strbuf.h" +#include "strlist.h" +#include "symbol.h" +#include "thread.h" +#include "time-utils.h" #include "trace-event.h" -#include <linux/kernel.h> -#include <linux/string.h> #ifdef HAVE_LIBTRACEEVENT #include <event-parse.h> @@ -464,7 +469,7 @@ int64_t _sort__sym_cmp(struct symbol *sym_l, struct symbol *sym_r) if (sym_l == sym_r) return 0; - if (sym_l->inlined || sym_r->inlined) { + if (symbol__inlined(sym_l) || symbol__inlined(sym_r)) { int ret = strcmp(sym_l->name, sym_r->name); if (ret) @@ -531,7 +536,7 @@ static int _hist_entry__sym_snprintf(struct map_symbol *ms, ret += repsep_snprintf(bf + ret, size - ret, "[%c] ", level); if (sym && map) { - if (sym->type == STT_OBJECT) { + if (symbol__type(sym) == STT_OBJECT) { ret += repsep_snprintf(bf + ret, size - ret, "%s", sym->name); ret += repsep_snprintf(bf + ret, size - ret, "+0x%llx", ip - map__unmap_ip(map, sym->start)); @@ -539,7 +544,7 @@ static int _hist_entry__sym_snprintf(struct map_symbol *ms, ret += repsep_snprintf(bf + ret, size - ret, "%.*s", width - ret, sym->name); - if (sym->inlined) + if (symbol__inlined(sym)) ret += repsep_snprintf(bf + ret, size - ret, " (inlined)"); } @@ -1478,7 +1483,7 @@ static int _hist_entry__addr_snprintf(struct map_symbol *ms, ret += repsep_snprintf(bf + ret, size - ret, "[%c] ", level); if (sym && map) { - if (sym->type == STT_OBJECT) { + if (symbol__type(sym) == STT_OBJECT) { ret += repsep_snprintf(bf + ret, size - ret, "%s", sym->name); ret += repsep_snprintf(bf + ret, size - ret, "+0x%llx", ip - map__unmap_ip(map, sym->start)); @@ -2673,9 +2678,10 @@ struct sort_dimension { static int arch_support_sort_key(const char *sort_key, struct perf_env *env) { - const char *arch = perf_env__arch(env); + uint16_t e_machine = perf_env__e_machine(env, /*e_eflags=*/NULL); - if (!strcmp("x86", arch) || !strcmp("powerpc", arch)) { + if (e_machine == EM_X86_64 || e_machine == EM_386 || e_machine == EM_PPC64 || + e_machine == EM_PPC) { if (!strcmp(sort_key, "p_stage_cyc")) return 1; if (!strcmp(sort_key, "local_p_stage_cyc")) @@ -2686,14 +2692,14 @@ static int arch_support_sort_key(const char *sort_key, struct perf_env *env) static const char *arch_perf_header_entry(const char *se_header, struct perf_env *env) { - const char *arch = perf_env__arch(env); + uint16_t e_machine = perf_env__e_machine(env, /*e_eflags=*/NULL); - if (!strcmp("x86", arch)) { + if (e_machine == EM_X86_64 || e_machine == EM_386) { if (!strcmp(se_header, "Local Pipeline Stage Cycle")) return "Local Retire Latency"; else if (!strcmp(se_header, "Pipeline Stage Cycle")) return "Retire Latency"; - } else if (!strcmp("powerpc", arch)) { + } else if (e_machine == EM_PPC64 || e_machine == EM_PPC) { if (!strcmp(se_header, "Local INSTR Latency")) return "Finish Cyc"; else if (!strcmp(se_header, "INSTR Latency")) diff --git a/tools/perf/util/srcline.c b/tools/perf/util/srcline.c index db164d258163..b082178c279b 100644 --- a/tools/perf/util/srcline.c +++ b/tools/perf/util/srcline.c @@ -113,16 +113,16 @@ struct symbol *new_inline_sym(struct dso *dso, /* ensure that we don't alias an inlined symbol, which could * lead to double frees in inline_node__delete */ - assert(!base_sym->inlined); + assert(!symbol__inlined(base_sym)); } else { /* create a fake symbol for the inline frame */ inline_sym = symbol__new(base_sym ? base_sym->start : 0, base_sym ? (base_sym->end - base_sym->start) : 0, - base_sym ? base_sym->binding : 0, - base_sym ? base_sym->type : 0, + base_sym ? symbol__binding(base_sym) : 0, + base_sym ? symbol__type(base_sym) : 0, funcname); if (inline_sym) - inline_sym->inlined = 1; + symbol__set_inlined(inline_sym, true); } free(demangled); @@ -429,19 +429,26 @@ struct inline_node *dso__parse_addr_inlines(struct dso *dso, u64 addr, return addr2inlines(dso_name, addr, dso, sym); } -void inline_node__delete(struct inline_node *node) +void inline_node__clear_frames(struct inline_node *node) { struct inline_list *ilist, *tmp; + if (node == NULL) + return; + list_for_each_entry_safe(ilist, tmp, &node->val, list) { list_del_init(&ilist->list); zfree_srcline(&ilist->srcline); /* only the inlined symbols are owned by the list */ - if (ilist->symbol && ilist->symbol->inlined) + if (ilist->symbol && symbol__inlined(ilist->symbol)) symbol__delete(ilist->symbol); free(ilist); } +} +void inline_node__delete(struct inline_node *node) +{ + inline_node__clear_frames(node); free(node); } diff --git a/tools/perf/util/srcline.h b/tools/perf/util/srcline.h index 7c37b3bf9ce7..1018cbc886d6 100644 --- a/tools/perf/util/srcline.h +++ b/tools/perf/util/srcline.h @@ -47,6 +47,7 @@ struct inline_node *dso__parse_addr_inlines(struct dso *dso, u64 addr, struct symbol *sym); /* free resources associated to the inline node list */ void inline_node__delete(struct inline_node *node); +void inline_node__clear_frames(struct inline_node *node); /* insert the inline node list into the DSO, which will take ownership */ void inlines__tree_insert(struct rb_root_cached *tree, diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index 993f4c4b8f44..0a5750bb59fa 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -580,16 +580,13 @@ static void print_metricgroup_header_std(struct perf_stat_config *config, const char *metricgroup_name) { struct outstate *os = ctx; - int n; if (!metricgroup_name) { __new_line_std(config, os); return; } - n = fprintf(config->output, " %*s", EVNAME_LEN, metricgroup_name); - - fprintf(config->output, "%*s", MGROUP_LEN + config->unit_width + 2 - n, ""); + fprintf(config->output, " %*s", config->metric_only_len, metricgroup_name); } static void print_metric_only(struct perf_stat_config *config, @@ -599,19 +596,20 @@ static void print_metric_only(struct perf_stat_config *config, struct outstate *os = ctx; FILE *out = os->fh; char str[1024]; - unsigned mlen = config->metric_only_len; + unsigned mlen; const char *color = metric_threshold_classify__color(thresh); + int olen; - if (!unit) - unit = ""; - if (mlen < strlen(unit)) - mlen = strlen(unit) + 1; + if (!unit) { + os->first = false; + return; + } - if (color) - mlen += strlen(color) + sizeof(PERF_COLOR_RESET) - 1; + mlen = max_t(unsigned, strlen(unit), config->metric_only_len); + olen = snprintf(str, sizeof(str), fmt ?: "", val); color_snprintf(str, sizeof(str), color ?: "", fmt ?: "", val); - fprintf(out, "%*s ", mlen, str); + fprintf(out, "%*s%s", max_t(int, mlen - olen, 1), "", str); os->first = false; } @@ -823,9 +821,9 @@ static void printout(struct perf_stat_config *config, struct outstate *os, ok = false; if (counter->supported) { - if (!evlist__has_hybrid_pmus(counter->evlist)) { + if (!evlist__has_hybrid_pmus(counter->evlist) && + counter->pmu && counter->pmu->is_core) config->print_free_counters_hint = 1; - } } } diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c index bc2d44df7baf..c17373bb0e1e 100644 --- a/tools/perf/util/stat-shadow.c +++ b/tools/perf/util/stat-shadow.c @@ -53,6 +53,7 @@ static int prepare_metric(struct perf_stat_config *config, for (i = 0; metric_events[i]; i++) { int source_count = 0, tool_aggr_idx; + int aggr_nr = 1; bool is_tool_time = tool_pmu__is_time_event(config, metric_events[i], &tool_aggr_idx); struct perf_stat_evsel *ps = metric_events[i]->stats; @@ -89,6 +90,7 @@ static int prepare_metric(struct perf_stat_config *config, */ val = NAN; source_count = 0; + aggr_nr = 0; } else { struct perf_stat_aggr *aggr = &ps->aggr[is_tool_time ? tool_aggr_idx : aggr_idx]; @@ -96,6 +98,7 @@ static int prepare_metric(struct perf_stat_config *config, if (aggr->counts.run == 0) { val = NAN; source_count = 0; + aggr_nr = 0; } else { val = aggr->counts.val; if (is_tool_time) { @@ -104,13 +107,14 @@ static int prepare_metric(struct perf_stat_config *config, } if (!source_count) source_count = evsel__source_count(metric_events[i]); + aggr_nr = aggr->nr ?: 1; } } n = strdup(evsel__metric_id(metric_events[i])); if (!n) return -ENOMEM; - expr__add_id_val_source_count(pctx, n, val, source_count); + expr__add_id_val_source_count_aggr_nr(pctx, n, val, source_count, aggr_nr); } for (int j = 0; metric_refs && metric_refs[j].metric_name; j++) { diff --git a/tools/perf/util/svghelper.c b/tools/perf/util/svghelper.c index e360e7736c7b..826bd2577344 100644 --- a/tools/perf/util/svghelper.c +++ b/tools/perf/util/svghelper.c @@ -47,13 +47,13 @@ static double cpu2slot(int cpu) } static int *topology_map; +static int topology_map_size; static double cpu2y(int cpu) { - if (topology_map) + if (topology_map && cpu >= 0 && cpu < topology_map_size) return cpu2slot(topology_map[cpu]) * SLOT_MULT; - else - return cpu2slot(cpu) * SLOT_MULT; + return cpu2slot(cpu) * SLOT_MULT; } static double time2pixels(u64 __time) @@ -736,7 +736,8 @@ static int str_to_bitmap(char *s, cpumask_t *b, int nr_cpus) return -1; perf_cpu_map__for_each_cpu(cpu, idx, map) { - if (cpu.cpu >= nr_cpus) { + /* perf_cpu_map__new("") returns cpu.cpu == -1 */ + if (cpu.cpu < 0 || cpu.cpu >= nr_cpus) { ret = -1; break; } @@ -794,6 +795,7 @@ int svg_build_topology_map(struct perf_env *env) fprintf(stderr, "topology: no memory\n"); goto exit; } + topology_map_size = nr_cpus; for (i = 0; i < nr_cpus; i++) topology_map[i] = -1; diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 7afa8a117139..39562bdec8b9 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -217,7 +217,7 @@ bool filename__has_section(const char *filename, const char *sec) GElf_Shdr shdr; bool found = false; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) return false; @@ -350,7 +350,8 @@ static bool get_ifunc_name(Elf *elf, struct dso *dso, GElf_Ehdr *ehdr, sym = dso__find_symbol_nocache(dso, addr); /* Expecting the address to be an IFUNC or IFUNC alias */ - if (!sym || sym->start != addr || (sym->type != STT_GNU_IFUNC && !sym->ifunc_alias)) + if (!sym || sym->start != addr || + (symbol__type(sym) != STT_GNU_IFUNC && !symbol__ifunc_alias(sym))) return false; snprintf(buf, buf_sz, "%s@plt", sym->name); @@ -834,10 +835,24 @@ static int elf_read_build_id(Elf *elf, void *bf, size_t size) ptr = data->d_buf; while (ptr < (data->d_buf + data->d_size)) { GElf_Nhdr *nhdr = ptr; - size_t namesz = NOTE_ALIGN(nhdr->n_namesz), - descsz = NOTE_ALIGN(nhdr->n_descsz); + size_t namesz, descsz, remaining; const char *name; + /* ensure the note header fits within the section */ + if (ptr + sizeof(*nhdr) > data->d_buf + data->d_size) + break; + + namesz = NOTE_ALIGN(nhdr->n_namesz); + descsz = NOTE_ALIGN(nhdr->n_descsz); + + /* validate individually to avoid size_t overflow on 32-bit */ + remaining = data->d_buf + data->d_size - ptr - sizeof(*nhdr); + if (namesz > remaining || descsz > remaining - namesz) { + pr_warning("%s: oversized note: n_namesz=%u, n_descsz=%u\n", + __func__, nhdr->n_namesz, nhdr->n_descsz); + break; + } + ptr += sizeof(*nhdr); name = ptr; ptr += namesz; @@ -871,7 +886,7 @@ static int read_build_id(const char *filename, struct build_id *bid) if (size < BUILD_ID_SIZE) goto out; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) goto out; @@ -919,12 +934,14 @@ int filename__read_build_id(const char *filename, struct build_id *bid) return -1; } close(fd); - filename = path; + /* non-empty path means a temp file was created */ + if (path[0] != '\0') + filename = path; } err = read_build_id(filename, bid); - if (m.comp) + if (m.comp && filename == path) unlink(filename); return err; } @@ -934,7 +951,7 @@ int sysfs__read_build_id(const char *filename, struct build_id *bid) size_t size = sizeof(bid->data); int fd, err = -1; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) goto out; @@ -960,17 +977,28 @@ int sysfs__read_build_id(const char *filename, struct build_id *bid) err = 0; break; } - } else if (read(fd, bf, descsz) != (ssize_t)descsz) - break; + } else { + /* descsz from untrusted file — clamp to buffer */ + if (descsz > sizeof(bf)) + break; + if (read(fd, bf, descsz) != (ssize_t)descsz) + break; + } } else { - int n = namesz + descsz; + size_t n; - if (n > (int)sizeof(bf)) { + /* int sum of namesz+descsz can overflow negative, bypassing size check */ + if (namesz > sizeof(bf) || descsz > sizeof(bf) - namesz) { n = sizeof(bf); pr_debug("%s: truncating reading of build id in sysfs file %s: n_namesz=%u, n_descsz=%u.\n", __func__, filename, nhdr.n_namesz, nhdr.n_descsz); + } else { + n = namesz + descsz; } - if (read(fd, bf, n) != n) + /* no valid note has both namesz and descsz zero */ + if (n == 0) + break; + if (read(fd, bf, n) != (ssize_t)n) break; } } @@ -994,7 +1022,7 @@ int filename__read_debuglink(const char *filename, char *debuglink, if (err >= 0) goto out; - fd = open(filename, O_RDONLY); + fd = open(filename, O_RDONLY | O_CLOEXEC); if (fd < 0) goto out; @@ -1023,7 +1051,14 @@ int filename__read_debuglink(const char *filename, char *debuglink, goto out_elf_end; /* the start of this section is a zero-terminated string */ - strncpy(debuglink, data->d_buf, size); + if (data->d_size > 0) { + size_t len = min(size - 1, data->d_size); + + memcpy(debuglink, data->d_buf, len); + debuglink[len] = '\0'; + } else { + debuglink[0] = '\0'; + } err = 0; @@ -1108,9 +1143,9 @@ static Elf *read_gnu_debugdata(struct dso *dso, Elf *elf, const char *name, int return NULL; } - temp_fd = mkstemp(temp_filename); + temp_fd = mkostemp(temp_filename, O_CLOEXEC); if (temp_fd < 0) { - pr_debug("%s: mkstemp: %m\n", __func__); + pr_debug("%s: mkostemp: %m\n", __func__); *dso__load_errno(dso) = -errno; fclose(wrapped); return NULL; @@ -1152,7 +1187,7 @@ int symsrc__init(struct symsrc *ss, struct dso *dso, const char *name, type = dso__symtab_type(dso); } else { - fd = open(name, O_RDONLY); + fd = open(name, O_RDONLY | O_CLOEXEC); if (fd < 0) { *dso__load_errno(dso) = errno; return -1; @@ -1341,6 +1376,24 @@ static u64 ref_reloc(struct kmap *kmap) void __weak arch__sym_update(struct symbol *s __maybe_unused, GElf_Sym *sym __maybe_unused) { } +struct remap_kernel_ctx { + u64 sh_addr; + u64 sh_size; + u64 sh_offset; + struct kmap *kmap; +}; + +static int remap_kernel_cb(struct map *map, void *data) +{ + struct remap_kernel_ctx *ctx = data; + + map__set_start(map, ctx->sh_addr + ref_reloc(ctx->kmap)); + map__set_end(map, map__start(map) + ctx->sh_size); + map__set_pgoff(map, ctx->sh_offset); + map__set_mapping_type(map, MAPPING_TYPE__DSO); + return 0; +} + static int dso__process_kernel_symbol(struct dso *dso, struct map *map, GElf_Sym *sym, GElf_Shdr *shdr, struct maps *kmaps, struct kmap *kmap, @@ -1371,22 +1424,15 @@ static int dso__process_kernel_symbol(struct dso *dso, struct map *map, * map to the kernel dso. */ if (*remap_kernel && dso__kernel(dso) && !kmodule) { + struct remap_kernel_ctx ctx = { + .sh_addr = shdr->sh_addr, + .sh_size = shdr->sh_size, + .sh_offset = shdr->sh_offset, + .kmap = kmap + }; + *remap_kernel = false; - map__set_start(map, shdr->sh_addr + ref_reloc(kmap)); - map__set_end(map, map__start(map) + shdr->sh_size); - map__set_pgoff(map, shdr->sh_offset); - map__set_mapping_type(map, MAPPING_TYPE__DSO); - /* Ensure maps are correctly ordered */ - if (kmaps) { - int err; - struct map *tmp = map__get(map); - - maps__remove(kmaps, map); - err = maps__insert(kmaps, map); - map__put(tmp); - if (err) - return err; - } + maps__mutate_mapping(kmaps, map, remap_kernel_cb, &ctx); } /* @@ -1592,9 +1638,11 @@ dso__load_sym_internal(struct dso *dso, struct map *map, struct symsrc *syms_ss, if (!is_label && !elf_sym__filter(&sym)) continue; - /* Reject ARM ELF "mapping symbols": these aren't unique and + /* + * Reject ARM ELF "mapping symbols": these aren't unique and * don't identify functions, so will confuse the profile - * output: */ + * output: + */ if (ehdr.e_machine == EM_ARM || ehdr.e_machine == EM_AARCH64) { if (elf_name[0] == '$' && strchr("adtx", elf_name[1]) && (elf_name[2] == '\0' || elf_name[2] == '.')) @@ -1607,6 +1655,10 @@ dso__load_sym_internal(struct dso *dso, struct map *map, struct symsrc *syms_ss, continue; } + /* Reject kernel mapping symbols for kernel DSOs only */ + if (dso__kernel(dso) && is_ignored_kernel_symbol(elf_name)) + continue; + if (runtime_ss->opdsec && sym.st_shndx == runtime_ss->opdidx) { u32 offset = sym.st_value - syms_ss->opdshdr.sh_addr; u64 *opd = opddata->d_buf + offset; @@ -1727,7 +1779,7 @@ dso__load_sym_internal(struct dso *dso, struct map *map, struct symsrc *syms_ss, arch__sym_update(f, &sym); - __symbols__insert(dso__symbols(curr_dso), f, dso__kernel(dso)); + __symbols__insert(dso__symbols(curr_dso), f); nr++; } dso__put(curr_dso); @@ -1945,7 +1997,7 @@ static int kcore__open(struct kcore *kcore, const char *filename) { GElf_Ehdr *ehdr; - kcore->fd = open(filename, O_RDONLY); + kcore->fd = open(filename, O_RDONLY | O_CLOEXEC); if (kcore->fd == -1) return -1; @@ -1976,9 +2028,9 @@ static int kcore__init(struct kcore *kcore, char *filename, int elfclass, kcore->elfclass = elfclass; if (temp) - kcore->fd = mkstemp(filename); + kcore->fd = mkostemp(filename, O_CLOEXEC); else - kcore->fd = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0400); + kcore->fd = open(filename, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0400); if (kcore->fd == -1) return -1; @@ -2454,11 +2506,11 @@ static int kcore_copy__compare_files(const char *from_filename, { int from, to, err = -1; - from = open(from_filename, O_RDONLY); + from = open(from_filename, O_RDONLY | O_CLOEXEC); if (from < 0) return -1; - to = open(to_filename, O_RDONLY); + to = open(to_filename, O_RDONLY | O_CLOEXEC); if (to < 0) goto out_close_from; @@ -2876,7 +2928,7 @@ int get_sdt_note_list(struct list_head *head, const char *target) Elf *elf; int fd, ret; - fd = open(target, O_RDONLY); + fd = open(target, O_RDONLY | O_CLOEXEC); if (fd < 0) return -EBADF; diff --git a/tools/perf/util/symbol-minimal.c b/tools/perf/util/symbol-minimal.c index 8221dc9868f7..0a71d1463952 100644 --- a/tools/perf/util/symbol-minimal.c +++ b/tools/perf/util/symbol-minimal.c @@ -1,3 +1,4 @@ +#include "debug.h" #include "dso.h" #include "symbol.h" #include "symsrc.h" @@ -44,7 +45,7 @@ static int read_build_id(void *note_data, size_t note_len, struct build_id *bid, ptr = note_data; while ((ptr + sizeof(*nhdr)) < (note_data + note_len)) { const char *name; - size_t namesz, descsz; + size_t namesz, descsz, remaining; nhdr = ptr; if (need_swap) { @@ -56,6 +57,14 @@ static int read_build_id(void *note_data, size_t note_len, struct build_id *bid, namesz = NOTE_ALIGN(nhdr->n_namesz); descsz = NOTE_ALIGN(nhdr->n_descsz); + /* validate individually to avoid size_t overflow on 32-bit */ + remaining = note_data + note_len - ptr - sizeof(*nhdr); + if (namesz > remaining || descsz > remaining - namesz) { + pr_warning("%s: oversized note: n_namesz=%u, n_descsz=%u\n", + __func__, nhdr->n_namesz, nhdr->n_descsz); + break; + } + ptr += sizeof(*nhdr); name = ptr; ptr += namesz; @@ -166,7 +175,7 @@ int filename__read_build_id(const char *filename, struct build_id *bid) if (elf32) { hdrs.phdr32[i].p_type = bswap_32(hdrs.phdr32[i].p_type); hdrs.phdr32[i].p_offset = bswap_32(hdrs.phdr32[i].p_offset); - hdrs.phdr32[i].p_filesz = bswap_32(hdrs.phdr32[i].p_offset); + hdrs.phdr32[i].p_filesz = bswap_32(hdrs.phdr32[i].p_filesz); } else { hdrs.phdr64[i].p_type = bswap_32(hdrs.phdr64[i].p_type); hdrs.phdr64[i].p_offset = bswap_64(hdrs.phdr64[i].p_offset); @@ -177,6 +186,9 @@ int filename__read_build_id(const char *filename, struct build_id *bid) continue; p_filesz = elf32 ? hdrs.phdr32[i].p_filesz : hdrs.phdr64[i].p_filesz; + /* ssize_t can go negative with crafted ELF p_filesz values */ + if (p_filesz <= 0) + continue; if (p_filesz > buf_size) { void *tmp; diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index fcaeeddbbb6b..cd379ced19e5 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -48,9 +48,15 @@ #include <symbol/kallsyms.h> #include <sys/utsname.h> +static int map_fixup_cb(struct map *map, void *data __maybe_unused) +{ + map__fixup_start(map); + map__fixup_end(map); + return 0; +} + static int dso__load_kernel_sym(struct dso *dso, struct map *map); static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map); -static bool symbol__is_idle(const char *name); int vmlinux_path__nr_entries; char **vmlinux_path; @@ -163,24 +169,24 @@ static int choose_best_symbol(struct symbol *syma, struct symbol *symb) else if ((a == 0) && (b > 0)) return SYMBOL_B; - if (syma->type != symb->type) { - if (syma->type == STT_NOTYPE) + if (symbol__type(syma) != symbol__type(symb)) { + if (symbol__type(syma) == STT_NOTYPE) return SYMBOL_B; - if (symb->type == STT_NOTYPE) + if (symbol__type(symb) == STT_NOTYPE) return SYMBOL_A; } /* Prefer a non weak symbol over a weak one */ - a = syma->binding == STB_WEAK; - b = symb->binding == STB_WEAK; + a = symbol__binding(syma) == STB_WEAK; + b = symbol__binding(symb) == STB_WEAK; if (b && !a) return SYMBOL_A; if (a && !b) return SYMBOL_B; /* Prefer a global symbol over a non global one */ - a = syma->binding == STB_GLOBAL; - b = symb->binding == STB_GLOBAL; + a = symbol__binding(syma) == STB_GLOBAL; + b = symbol__binding(symb) == STB_GLOBAL; if (a && !b) return SYMBOL_A; if (b && !a) @@ -227,14 +233,14 @@ again: continue; if (choose_best_symbol(curr, next) == SYMBOL_A) { - if (next->type == STT_GNU_IFUNC) - curr->ifunc_alias = true; + if (symbol__type(next) == STT_GNU_IFUNC) + symbol__set_ifunc_alias(curr, true); rb_erase_cached(&next->rb_node, symbols); symbol__delete(next); goto again; } else { - if (curr->type == STT_GNU_IFUNC) - next->ifunc_alias = true; + if (symbol__type(curr) == STT_GNU_IFUNC) + symbol__set_ifunc_alias(next, true); nd = rb_next(&curr->rb_node); rb_erase_cached(&curr->rb_node, symbols); symbol__delete(curr); @@ -322,8 +328,8 @@ struct symbol *symbol__new(u64 start, u64 len, u8 binding, u8 type, const char * sym->start = start; sym->end = len ? start + len : start; - sym->type = type; - sym->binding = binding; + atomic_init(&sym->flags, (type << SYMBOL_FLAG_TYPE_SHIFT) | + (binding << SYMBOL_FLAG_BINDING_SHIFT)); sym->namelen = namelen - 1; pr_debug4("%s: %s %#" PRIx64 "-%#" PRIx64 "\n", @@ -345,6 +351,49 @@ void symbol__delete(struct symbol *sym) free(((void *)sym) - symbol_conf.priv_size); } +void symbol__set_ignore(struct symbol *sym, bool ignore) +{ + if (ignore) + atomic_fetch_or(&sym->flags, SYMBOL_FLAG_IGNORE); + else + atomic_fetch_and(&sym->flags, ~SYMBOL_FLAG_IGNORE); +} + +void symbol__set_annotate2(struct symbol *sym, bool annotate2) +{ + if (annotate2) + atomic_fetch_or(&sym->flags, SYMBOL_FLAG_ANNOTATE2); + else + atomic_fetch_and(&sym->flags, ~SYMBOL_FLAG_ANNOTATE2); +} + +void symbol__set_inlined(struct symbol *sym, bool inlined) +{ + if (inlined) + atomic_fetch_or(&sym->flags, SYMBOL_FLAG_INLINED); + else + atomic_fetch_and(&sym->flags, ~SYMBOL_FLAG_INLINED); +} + +void symbol__set_ifunc_alias(struct symbol *sym, bool ifunc_alias) +{ + if (ifunc_alias) + atomic_fetch_or(&sym->flags, SYMBOL_FLAG_IFUNC_ALIAS); + else + atomic_fetch_and(&sym->flags, ~SYMBOL_FLAG_IFUNC_ALIAS); +} + +static void symbol__set_idle(struct symbol *sym, bool idle) +{ + uint16_t old_flags = atomic_load_explicit(&sym->flags, memory_order_relaxed); + uint16_t new_flags; + uint16_t idle_val = idle ? SYMBOL_IDLE__IDLE : SYMBOL_IDLE__NOT_IDLE; + + do { + new_flags = old_flags & ~SYMBOL_FLAG_IDLE_MASK; + new_flags |= (idle_val << SYMBOL_FLAG_IDLE_SHIFT); + } while (!atomic_compare_exchange_weak(&sym->flags, &old_flags, new_flags)); +} void symbols__delete(struct rb_root_cached *symbols) { struct symbol *pos; @@ -358,8 +407,7 @@ void symbols__delete(struct rb_root_cached *symbols) } } -void __symbols__insert(struct rb_root_cached *symbols, - struct symbol *sym, bool kernel) +void __symbols__insert(struct rb_root_cached *symbols, struct symbol *sym) { struct rb_node **p = &symbols->rb_root.rb_node; struct rb_node *parent = NULL; @@ -367,17 +415,6 @@ void __symbols__insert(struct rb_root_cached *symbols, struct symbol *s; bool leftmost = true; - if (kernel) { - const char *name = sym->name; - /* - * ppc64 uses function descriptors and appends a '.' to the - * start of every instruction address. Remove it. - */ - if (name[0] == '.') - name++; - sym->idle = symbol__is_idle(name); - } - while (*p != NULL) { parent = *p; s = rb_entry(parent, struct symbol, rb_node); @@ -394,7 +431,7 @@ void __symbols__insert(struct rb_root_cached *symbols, void symbols__insert(struct rb_root_cached *symbols, struct symbol *sym) { - __symbols__insert(symbols, sym, false); + __symbols__insert(symbols, sym); } static struct symbol *symbols__find(struct rb_root_cached *symbols, u64 ip) @@ -555,7 +592,7 @@ void dso__reset_find_symbol_cache(struct dso *dso) void dso__insert_symbol(struct dso *dso, struct symbol *sym) { - __symbols__insert(dso__symbols(dso), sym, dso__kernel(dso)); + __symbols__insert(dso__symbols(dso), sym); /* update the symbol cache if necessary */ if (dso__last_find_result_addr(dso) >= sym->start && @@ -717,47 +754,120 @@ out: return err; } + /* * These are symbols in the kernel image, so make sure that * sym is from a kernel DSO. */ -static bool symbol__is_idle(const char *name) +static int sym_name_cmp(const void *a, const void *b) +{ + const char *name = a; + const char *const *sym = b; + + return strcmp(name, *sym); +} + +static bool match_x86_idle_routine(const char *name, const char *base) +{ + if (strstarts(name, base)) { + size_t len = strlen(base); + + if (name[len] == '\0' || name[len] == '.') + return true; + } + return false; +} + +bool symbol__is_idle(struct symbol *sym, const struct dso *dso, struct perf_env *env) { - const char * const idle_symbols[] = { + static const char * const idle_symbols[] = { "acpi_idle_do_entry", "acpi_processor_ffh_cstate_enter", "arch_cpu_idle", "cpu_idle", "cpu_startup_entry", - "idle_cpu", - "intel_idle", - "intel_idle_ibrs", "default_idle", - "native_safe_halt", "enter_idle", "exit_idle", - "mwait_idle", - "mwait_idle_with_hints", - "mwait_idle_with_hints.constprop.0", + "idle_cpu", + "native_safe_halt", "poll_idle", - "ppc64_runlatch_off", "pseries_dedicated_idle_sleep", - "psw_idle", - "psw_idle_exit", - NULL }; - int i; - static struct strlist *idle_symbols_list; + const char *name = sym->name; + uint16_t e_machine; + + { + uint16_t flags = atomic_load_explicit(&sym->flags, memory_order_relaxed); + uint16_t idle_val = (flags & SYMBOL_FLAG_IDLE_MASK) >> SYMBOL_FLAG_IDLE_SHIFT; + + if (idle_val != SYMBOL_IDLE__UNKNOWN) + return idle_val == SYMBOL_IDLE__IDLE; + } + + if (!dso || dso__kernel(dso) == DSO_SPACE__USER) { + symbol__set_idle(sym, /*idle=*/false); + return false; + } + + /* + * ppc64 uses function descriptors and appends a '.' to the + * start of every instruction address. Remove it. + */ + if (name[0] == '.') + name++; + + if (bsearch(name, idle_symbols, ARRAY_SIZE(idle_symbols), + sizeof(idle_symbols[0]), sym_name_cmp)) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } + + e_machine = (env && env->arch) ? perf_env__e_machine(env, NULL) : EM_NONE; + if (e_machine == EM_NONE && dso) + e_machine = dso__e_machine((struct dso *)dso, NULL, NULL); + if (e_machine == EM_NONE && env) + e_machine = perf_env__e_machine(env, NULL); + + if (e_machine == EM_386 || e_machine == EM_X86_64) { + if (match_x86_idle_routine(name, "intel_idle") || + match_x86_idle_routine(name, "intel_idle_irq") || + match_x86_idle_routine(name, "intel_idle_ibrs") || + match_x86_idle_routine(name, "mwait_idle") || + match_x86_idle_routine(name, "mwait_idle_with_hints")) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } + } - if (idle_symbols_list) - return strlist__has_entry(idle_symbols_list, name); + if (e_machine == EM_PPC64 && !strcmp(name, "ppc64_runlatch_off")) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } + + if (e_machine == EM_S390 && strstarts(name, "psw_idle")) { + int major = 0, minor = 0; + const char *release = env ? perf_env__os_release(env) : NULL; - idle_symbols_list = strlist__new(NULL, NULL); + /* + * If we can't determine the release (e.g. unpopulated guest traces), + * default to idle. + */ + if (!release) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } - for (i = 0; idle_symbols[i]; i++) - strlist__add(idle_symbols_list, idle_symbols[i]); + /* Before v6.10, s390 used psw_idle. */ + if (sscanf(release, "%d.%d", &major, &minor) == 2 && + (major < 6 || (major == 6 && minor < 10))) { + symbol__set_idle(sym, /*idle=*/true); + return true; + } + } - return strlist__has_entry(idle_symbols_list, name); + symbol__set_idle(sym, /*idle=*/false); + return false; } static int map__process_kallsym_symbol(void *arg, const char *name, @@ -770,8 +880,8 @@ static int map__process_kallsym_symbol(void *arg, const char *name, if (!symbol_type__filter(type)) return 0; - /* Ignore local symbols for ARM modules */ - if (name[0] == '$') + /* Ignore mapping symbols in kallsyms */ + if (is_ignored_kernel_symbol(name)) return 0; /* @@ -786,7 +896,7 @@ static int map__process_kallsym_symbol(void *arg, const char *name, * We will pass the symbols to the filter later, in * map__split_kallsyms, when we have split the maps per module */ - __symbols__insert(root, sym, !strchr(name, '[')); + __symbols__insert(root, sym); return 0; } @@ -851,6 +961,23 @@ static int maps__split_kallsyms_for_kcore(struct maps *kmaps, struct dso *dso) return count; } +static uint16_t machine_or_dso_e_machine(struct machine *machine, struct dso *dso) +{ + uint16_t e_machine = EM_NONE; + /* DSO should be most accurate */ + if (dso) + e_machine = dso__e_machine(dso, machine, /*e_flags=*/NULL); + + if (e_machine != EM_NONE) + return e_machine; + + /* Check the global environment next. */ + if (machine && machine->env && machine->env->e_machine != EM_NONE) + return machine->env->e_machine; + + return perf_env__e_machine(machine ? machine->env : NULL, /*e_flags=*/NULL); +} + /* * Split the symbols into maps, making sure there are no overlaps, i.e. the * kernel range is broken in several maps, named [kernel].N, as we don't have @@ -866,14 +993,13 @@ static int maps__split_kallsyms(struct maps *kmaps, struct dso *dso, u64 delta, struct rb_root_cached *root = dso__symbols(dso); struct rb_node *next = rb_first_cached(root); int kernel_range = 0; - bool x86_64; + uint16_t e_machine = EM_NONE; if (!kmaps) return -1; machine = maps__machine(kmaps); - - x86_64 = machine__is(machine, "x86_64"); + e_machine = machine_or_dso_e_machine(machine, dso); while (next) { char *module; @@ -925,7 +1051,7 @@ static int maps__split_kallsyms(struct maps *kmaps, struct dso *dso, u64 delta, */ pos->start = map__map_ip(curr_map, pos->start); pos->end = map__map_ip(curr_map, pos->end); - } else if (x86_64 && is_entry_trampoline(pos->name)) { + } else if (e_machine == EM_X86_64 && is_entry_trampoline(pos->name)) { /* * These symbols are not needed anymore since the * trampoline maps refer to the text section and it's @@ -1428,7 +1554,7 @@ static int dso__load_kcore(struct dso *dso, struct map *map, free(new_node); } - if (machine__is(machine, "x86_64")) { + if (machine_or_dso_e_machine(machine, dso) == EM_X86_64) { u64 addr; /* @@ -1716,7 +1842,7 @@ int dso__load(struct dso *dso, struct map *map) ret = dso__load_guest_kernel_sym(dso, map); machine = maps__machine(map__kmaps(map)); - if (machine__is(machine, "x86_64")) + if (machine && machine_or_dso_e_machine(machine, dso) == EM_X86_64) machine__map_x86_64_entry_trampolines(machine, dso); goto out; } @@ -2121,10 +2247,11 @@ do_kallsyms: free(kallsyms_allocated_filename); if (err > 0 && !dso__is_kcore(dso)) { + struct maps *kmaps = map__kmaps(map); + dso__set_binary_type(dso, DSO_BINARY_TYPE__KALLSYMS); dso__set_long_name(dso, DSO__NAME_KALLSYMS, false); - map__fixup_start(map); - map__fixup_end(map); + maps__mutate_mapping(kmaps, map, map_fixup_cb, NULL); } return err; @@ -2156,7 +2283,7 @@ static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map) if (!kallsyms_filename) return -1; } else { - sprintf(path, "%s/proc/kallsyms", machine->root_dir); + snprintf(path, sizeof(path), "%s/proc/kallsyms", machine->root_dir); kallsyms_filename = path; } @@ -2164,10 +2291,11 @@ static int dso__load_guest_kernel_sym(struct dso *dso, struct map *map) if (err > 0) pr_debug("Using %s for symbols\n", kallsyms_filename); if (err > 0 && !dso__is_kcore(dso)) { + struct maps *kmaps = map__kmaps(map); + dso__set_binary_type(dso, DSO_BINARY_TYPE__GUEST_KALLSYMS); dso__set_long_name(dso, machine->mmap_name, false); - map__fixup_start(map); - map__fixup_end(map); + maps__mutate_mapping(kmaps, map, map_fixup_cb, NULL); } return err; @@ -2209,7 +2337,7 @@ static int vmlinux_path__init(struct perf_env *env) { struct utsname uts; char bf[PATH_MAX]; - char *kernel_version; + const char *kernel_version; unsigned int i; vmlinux_path = malloc(sizeof(char *) * (ARRAY_SIZE(vmlinux_paths) + @@ -2226,7 +2354,7 @@ static int vmlinux_path__init(struct perf_env *env) return 0; if (env) { - kernel_version = env->os_release; + kernel_version = perf_env__os_release(env); } else { if (uname(&uts) < 0) goto out_fail; @@ -2476,6 +2604,7 @@ void symbol__exit(void) { if (!symbol_conf.initialized) return; + strlist__delete(symbol_conf.bt_stop_list); strlist__delete(symbol_conf.sym_list); strlist__delete(symbol_conf.dso_list); diff --git a/tools/perf/util/symbol.h b/tools/perf/util/symbol.h index bd6eb90c8668..a71525335703 100644 --- a/tools/perf/util/symbol.h +++ b/tools/perf/util/symbol.h @@ -6,6 +6,7 @@ #include <linux/refcount.h> #include <stdbool.h> #include <stdint.h> +#include <stdatomic.h> #include <linux/list.h> #include <linux/rbtree.h> #include <stdio.h> @@ -27,6 +28,22 @@ struct map; struct maps; struct option; struct build_id; +struct perf_env; + +/* + * Ignore kernel mapping symbols, matching kernel is_mapping_symbol() logic. + * This checks for '$' prefix (used by ARM, AArch64, RISC-V) and + * x86 local symbol prefixes (.L* and L0*). + * Only use this for kernel symbols (kallsyms, ksymbol events, kernel ELF DSOs). + */ +static inline bool is_ignored_kernel_symbol(const char *str) +{ + if (str[0] == '.' && str[1] == 'L') + return true; + if (str[0] == 'L' && str[1] == '0') + return true; + return str[0] == '$'; +} /* * libelf 0.8.x and earlier do not support ELF_C_READ_MMAP; @@ -43,6 +60,22 @@ Elf_Scn *elf_section_by_name(Elf *elf, GElf_Ehdr *ep, GElf_Shdr *shp, const char *name, size_t *idx); #endif +enum symbol_idle_kind { + SYMBOL_IDLE__UNKNOWN = 0, + SYMBOL_IDLE__NOT_IDLE = 1, + SYMBOL_IDLE__IDLE = 2, +}; + +#define SYMBOL_FLAG_TYPE_SHIFT 0 +#define SYMBOL_FLAG_TYPE_MASK (0xF << SYMBOL_FLAG_TYPE_SHIFT) +#define SYMBOL_FLAG_BINDING_SHIFT 4 +#define SYMBOL_FLAG_BINDING_MASK (0xF << SYMBOL_FLAG_BINDING_SHIFT) +#define SYMBOL_FLAG_IDLE_SHIFT 8 +#define SYMBOL_FLAG_IDLE_MASK (0x3 << SYMBOL_FLAG_IDLE_SHIFT) +#define SYMBOL_FLAG_IGNORE (1 << 10) +#define SYMBOL_FLAG_INLINED (1 << 11) +#define SYMBOL_FLAG_ANNOTATE2 (1 << 12) +#define SYMBOL_FLAG_IFUNC_ALIAS (1 << 13) /** * A symtab entry. When allocated this may be preceded by an annotation (see * symbol__annotation) and/or a browser_index (see symbol__browser_index). @@ -54,20 +87,7 @@ struct symbol { u64 end; /** Length of the string name. */ u16 namelen; - /** ELF symbol type as defined for st_info. E.g STT_OBJECT or STT_FUNC. */ - u8 type:4; - /** ELF binding type as defined for st_info. E.g. STB_WEAK or STB_GLOBAL. */ - u8 binding:4; - /** Set true for kernel symbols of idle routines. */ - u8 idle:1; - /** Resolvable but tools ignore it (e.g. idle routines). */ - u8 ignore:1; - /** Symbol for an inlined function. */ - u8 inlined:1; - /** Has symbol__annotate2 been performed. */ - u8 annotate2:1; - /** Symbol is an alias of an STT_GNU_IFUNC */ - u8 ifunc_alias:1; + _Atomic uint16_t flags; /** Architecture specific. Unused except on PPC where it holds st_other. */ u8 arch_sym; /** The name of length namelen associated with the symbol. */ @@ -77,6 +97,49 @@ struct symbol { void symbol__delete(struct symbol *sym); void symbols__delete(struct rb_root_cached *symbols); +static inline u8 symbol__type(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_TYPE_MASK) >> SYMBOL_FLAG_TYPE_SHIFT; +} + +static inline u8 symbol__binding(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_BINDING_MASK) >> SYMBOL_FLAG_BINDING_SHIFT; +} + +static inline bool symbol__ignore(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_IGNORE) != 0; +} + +static inline bool symbol__inlined(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_INLINED) != 0; +} + +static inline bool symbol__is_annotate2(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_ANNOTATE2) != 0; +} + +static inline bool symbol__ifunc_alias(const struct symbol *sym) +{ + return (atomic_load_explicit(&sym->flags, memory_order_relaxed) & + SYMBOL_FLAG_IFUNC_ALIAS) != 0; +} + +bool symbol__is_idle(struct symbol *sym, const struct dso *dso, struct perf_env *env); + +void symbol__set_ignore(struct symbol *sym, bool ignore); +void symbol__set_annotate2(struct symbol *sym, bool annotate2); +void symbol__set_inlined(struct symbol *sym, bool inlined); +void symbol__set_ifunc_alias(struct symbol *sym, bool ifunc_alias); + /* symbols__for_each_entry - iterate over symbols (rb_root) * * @symbols: the rb_root of symbols @@ -154,7 +217,6 @@ int filename__read_debuglink(const char *filename, char *debuglink, size_t size); bool filename__has_section(const char *filename, const char *sec); -struct perf_env; int symbol__init(struct perf_env *env); void symbol__exit(void); void symbol__elf_init(void); @@ -194,8 +256,7 @@ int dso__synthesize_plt_symbols(struct dso *dso, struct symsrc *ss); char *dso__demangle_sym(struct dso *dso, int kmodule, const char *elf_name); -void __symbols__insert(struct rb_root_cached *symbols, struct symbol *sym, - bool kernel); +void __symbols__insert(struct rb_root_cached *symbols, struct symbol *sym); void symbols__insert(struct rb_root_cached *symbols, struct symbol *sym); void symbols__fixup_duplicate(struct rb_root_cached *symbols); void symbols__fixup_end(struct rb_root_cached *symbols, bool is_kallsyms); diff --git a/tools/perf/util/symbol_conf.h b/tools/perf/util/symbol_conf.h index 6cd454d7c98e..0dee5aa6a534 100644 --- a/tools/perf/util/symbol_conf.h +++ b/tools/perf/util/symbol_conf.h @@ -9,6 +9,15 @@ struct strlist; struct intlist; +enum unwind_style { + UNWIND_STYLE_UNKNOWN = 0, + UNWIND_STYLE_LIBDW, + UNWIND_STYLE_LIBUNWIND, +}; + +#define MAX_UNWIND_STYLE (UNWIND_STYLE_LIBUNWIND + 1) + + enum a2l_style { A2L_STYLE_UNKNOWN = 0, A2L_STYLE_LIBDW, @@ -81,6 +90,7 @@ struct symbol_conf { const char *addr2line_path; enum a2l_style addr2line_style[MAX_A2L_STYLE]; int addr2line_timeout_ms; + enum unwind_style unwind_style[MAX_UNWIND_STYLE]; unsigned long time_quantum; struct strlist *dso_list, *comm_list, diff --git a/tools/perf/util/symbol_fprintf.c b/tools/perf/util/symbol_fprintf.c index 53e1af4ed9ac..4dc8d5761f52 100644 --- a/tools/perf/util/symbol_fprintf.c +++ b/tools/perf/util/symbol_fprintf.c @@ -11,8 +11,8 @@ size_t symbol__fprintf(struct symbol *sym, FILE *fp) { return fprintf(fp, " %" PRIx64 "-%" PRIx64 " %c %s\n", sym->start, sym->end, - sym->binding == STB_GLOBAL ? 'g' : - sym->binding == STB_LOCAL ? 'l' : 'w', + symbol__binding(sym) == STB_GLOBAL ? 'g' : + symbol__binding(sym) == STB_LOCAL ? 'l' : 'w', sym->name); } diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index 85bee747f4cd..5307d707711d 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -1455,7 +1455,8 @@ int perf_event__synthesize_stat_round(const struct perf_tool *tool, return process(tool, (union perf_event *) &event, NULL, machine); } -size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, u64 read_format) +size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, u64 read_format, + u64 branch_sample_type) { size_t sz, result = sizeof(struct perf_record_sample); @@ -1515,8 +1516,10 @@ size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, if (type & PERF_SAMPLE_BRANCH_STACK) { sz = sample->branch_stack->nr * sizeof(struct branch_entry); - /* nr, hw_idx */ - sz += 2 * sizeof(u64); + /* nr */ + sz += sizeof(u64); + if (branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) + sz += sizeof(u64); result += sz; } @@ -1605,7 +1608,7 @@ static __u64 *copy_read_group_values(__u64 *array, __u64 read_format, } int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format, - const struct perf_sample *sample) + u64 branch_sample_type, const struct perf_sample *sample) { __u64 *array; size_t sz; @@ -1719,9 +1722,17 @@ int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_fo if (type & PERF_SAMPLE_BRANCH_STACK) { sz = sample->branch_stack->nr * sizeof(struct branch_entry); - /* nr, hw_idx */ - sz += 2 * sizeof(u64); - memcpy(array, sample->branch_stack, sz); + + *array++ = sample->branch_stack->nr; + + if (branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) { + if (sample->no_hw_idx) + *array++ = 0; + else + *array++ = sample->branch_stack->hw_idx; + } + + memcpy(array, perf_sample__branch_entries((struct perf_sample *)sample), sz); array = (void *)array + sz; } @@ -2170,11 +2181,21 @@ int perf_event__synthesize_attr(const struct perf_tool *tool, struct perf_event_ u32 ids, u64 *id, perf_event__handler_t process) { union perf_event *ev; - size_t size; + size_t attr_size, size; int err; - size = sizeof(struct perf_event_attr); - size = PERF_ALIGN(size, sizeof(u64)); + /* + * Use attr->size for the event layout, not the compiled + * sizeof(struct perf_event_attr), so that synthesized events + * match the source perf.data layout. This matters for perf + * inject, which re-synthesizes attrs from a file that may + * have been recorded by a different version of perf. + * perf_record_header_attr_id() locates the ID array at + * attr->size bytes past the attr. + */ + attr_size = attr->size ?: sizeof(struct perf_event_attr); + + size = PERF_ALIGN(attr_size, sizeof(u64)); size += sizeof(struct perf_event_header); size += ids * sizeof(u64); @@ -2183,7 +2204,14 @@ int perf_event__synthesize_attr(const struct perf_tool *tool, struct perf_event_ if (ev == NULL) return -ENOMEM; - ev->attr.attr = *attr; + /* + * Copy only the bytes we understand; zalloc ensures that any + * extra bytes between sizeof(struct perf_event_attr) and + * attr_size are zero when the source file uses a newer, larger + * struct. + */ + memcpy(&ev->attr.attr, attr, min(sizeof(struct perf_event_attr), attr_size)); + ev->attr.attr.size = attr_size; memcpy(perf_record_header_attr_id(ev), id, ids * sizeof(u64)); ev->attr.header.type = PERF_RECORD_HEADER_ATTR; @@ -2252,17 +2280,25 @@ int perf_event__synthesize_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, perf_event__handler_t process, - const struct evsel *evsel, __u16 misc, const struct build_id *bid, const char *filename) { union perf_event ev; - size_t len; + size_t len, filename_len = strlen(filename); + u64 sample_type = sample->evsel ? sample->evsel->core.attr.sample_type : 0; + void *array = &ev; + int ret; - len = sizeof(ev.build_id) + strlen(filename) + 1; + if (filename_len >= PATH_MAX) + return -EINVAL; + + len = sizeof(ev.build_id) + filename_len + 1; len = PERF_ALIGN(len, sizeof(u64)); + if (len + MAX_ID_HDR_ENTRIES * sizeof(__u64) > sizeof(ev)) + return -E2BIG; + memset(&ev, 0, len); ev.build_id.size = bid->size; @@ -2275,23 +2311,17 @@ int perf_event__synthesize_build_id(const struct perf_tool *tool, ev.build_id.header.size = len; strcpy(ev.build_id.filename, filename); - if (evsel) { - void *array = &ev; - int ret; - - array += ev.header.size; - ret = perf_event__synthesize_id_sample(array, evsel->core.attr.sample_type, sample); - if (ret < 0) - return ret; - - if (ret & 7) { - pr_err("Bad id sample size %d\n", ret); - return -EINVAL; - } + array += ev.header.size; + ret = perf_event__synthesize_id_sample(array, sample_type, sample); + if (ret < 0) + return ret; - ev.header.size += ret; + if (ret & 7) { + pr_err("Bad id sample size %d\n", ret); + return -EINVAL; } + ev.header.size += ret; return process(tool, &ev, sample, machine); } @@ -2299,7 +2329,6 @@ int perf_event__synthesize_mmap2_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, perf_event__handler_t process, - const struct evsel *evsel, __u16 misc, __u32 pid, __u32 tid, __u64 start, __u64 len, __u64 pgoff, @@ -2308,13 +2337,21 @@ int perf_event__synthesize_mmap2_build_id(const struct perf_tool *tool, const char *filename) { union perf_event ev; + size_t filename_len = strlen(filename); size_t ev_len; + u64 sample_type = sample->evsel ? sample->evsel->core.attr.sample_type : 0; void *array; int ret; - ev_len = sizeof(ev.mmap2) - sizeof(ev.mmap2.filename) + strlen(filename) + 1; + if (filename_len >= sizeof(ev.mmap2.filename)) + return -EINVAL; + + ev_len = sizeof(ev.mmap2) - sizeof(ev.mmap2.filename) + filename_len + 1; ev_len = PERF_ALIGN(ev_len, sizeof(u64)); + if (ev_len + MAX_ID_HDR_ENTRIES * sizeof(__u64) > sizeof(ev)) + return -E2BIG; + memset(&ev, 0, ev_len); ev.mmap2.header.type = PERF_RECORD_MMAP2; @@ -2339,7 +2376,7 @@ int perf_event__synthesize_mmap2_build_id(const struct perf_tool *tool, array = &ev; array += ev.header.size; - ret = perf_event__synthesize_id_sample(array, evsel->core.attr.sample_type, sample); + ret = perf_event__synthesize_id_sample(array, sample_type, sample); if (ret < 0) return ret; diff --git a/tools/perf/util/synthetic-events.h b/tools/perf/util/synthetic-events.h index b0edad0c3100..243115ea61ae 100644 --- a/tools/perf/util/synthetic-events.h +++ b/tools/perf/util/synthetic-events.h @@ -50,7 +50,6 @@ int perf_event__synthesize_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, perf_event__handler_t process, - const struct evsel *evsel, __u16 misc, const struct build_id *bid, const char *filename); @@ -58,7 +57,6 @@ int perf_event__synthesize_mmap2_build_id(const struct perf_tool *tool, struct perf_sample *sample, struct machine *machine, perf_event__handler_t process, - const struct evsel *evsel, __u16 misc, __u32 pid, __u32 tid, __u64 start, __u64 len, __u64 pgoff, @@ -81,7 +79,8 @@ int perf_event__synthesize_mmap_events(const struct perf_tool *tool, union perf_ int perf_event__synthesize_modules(const struct perf_tool *tool, perf_event__handler_t process, struct machine *machine); int perf_event__synthesize_namespaces(const struct perf_tool *tool, union perf_event *event, pid_t pid, pid_t tgid, perf_event__handler_t process, struct machine *machine); int perf_event__synthesize_cgroups(const struct perf_tool *tool, perf_event__handler_t process, struct machine *machine); -int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format, const struct perf_sample *sample); +int perf_event__synthesize_sample(union perf_event *event, u64 type, u64 read_format, + u64 branch_sample_type, const struct perf_sample *sample); int perf_event__synthesize_stat_config(const struct perf_tool *tool, struct perf_stat_config *config, perf_event__handler_t process, struct machine *machine); int perf_event__synthesize_stat_events(struct perf_stat_config *config, const struct perf_tool *tool, struct evlist *evlist, perf_event__handler_t process, bool attrs); int perf_event__synthesize_stat_round(const struct perf_tool *tool, u64 time, u64 type, perf_event__handler_t process, struct machine *machine); @@ -97,7 +96,8 @@ void perf_event__synthesize_final_bpf_metadata(struct perf_session *session, int perf_tool__process_synth_event(const struct perf_tool *tool, union perf_event *event, struct machine *machine, perf_event__handler_t process); -size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, u64 read_format); +size_t perf_event__sample_event_size(const struct perf_sample *sample, u64 type, + u64 read_format, u64 branch_sample_type); int __machine__synthesize_threads(struct machine *machine, const struct perf_tool *tool, struct target *target, struct perf_thread_map *threads, diff --git a/tools/perf/util/syscalltbl.c b/tools/perf/util/syscalltbl.c deleted file mode 100644 index 67a8ec10e9e4..000000000000 --- a/tools/perf/util/syscalltbl.c +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * System call table mapper - * - * (C) 2016 Arnaldo Carvalho de Melo <acme@redhat.com> - */ - -#include "syscalltbl.h" -#include <stdlib.h> -#include <asm/bitsperlong.h> -#include <linux/compiler.h> -#include <linux/kernel.h> -#include <linux/zalloc.h> - -#include <string.h> -#include "string2.h" - -#include "trace/beauty/generated/syscalltbl.c" - -static const struct syscalltbl *find_table(int e_machine) -{ - static const struct syscalltbl *last_table; - static int last_table_machine = EM_NONE; - - /* Tables only exist for EM_SPARC. */ - if (e_machine == EM_SPARCV9) - e_machine = EM_SPARC; - - if (last_table_machine == e_machine && last_table != NULL) - return last_table; - - for (size_t i = 0; i < ARRAY_SIZE(syscalltbls); i++) { - const struct syscalltbl *entry = &syscalltbls[i]; - - if (entry->e_machine != e_machine && entry->e_machine != EM_NONE) - continue; - - last_table = entry; - last_table_machine = e_machine; - return entry; - } - return NULL; -} - -const char *syscalltbl__name(int e_machine, int id) -{ - const struct syscalltbl *table = find_table(e_machine); - - if (e_machine == EM_MIPS && id > 1000) { - /* - * MIPS may encode the N32/64/O32 type in the high part of - * syscall number. Mask this off if present. See the values of - * __NR_N32_Linux, __NR_64_Linux, __NR_O32_Linux and __NR_Linux. - */ - id = id % 1000; - } - if (table && id >= 0 && id < table->num_to_name_len) - return table->num_to_name[id]; - return NULL; -} - -struct syscall_cmp_key { - const char *name; - const char *const *tbl; -}; - -static int syscallcmpname(const void *vkey, const void *ventry) -{ - const struct syscall_cmp_key *key = vkey; - const uint16_t *entry = ventry; - - return strcmp(key->name, key->tbl[*entry]); -} - -int syscalltbl__id(int e_machine, const char *name) -{ - const struct syscalltbl *table = find_table(e_machine); - struct syscall_cmp_key key; - const uint16_t *id; - - if (!table) - return -1; - - key.name = name; - key.tbl = table->num_to_name; - id = bsearch(&key, table->sorted_names, table->sorted_names_len, - sizeof(table->sorted_names[0]), syscallcmpname); - - return id ? *id : -1; -} - -int syscalltbl__num_idx(int e_machine) -{ - const struct syscalltbl *table = find_table(e_machine); - - if (!table) - return 0; - - return table->sorted_names_len; -} - -int syscalltbl__id_at_idx(int e_machine, int idx) -{ - const struct syscalltbl *table = find_table(e_machine); - - if (!table) - return -1; - - assert(idx >= 0 && idx < table->sorted_names_len); - return table->sorted_names[idx]; -} - -int syscalltbl__strglobmatch_next(int e_machine, const char *syscall_glob, int *idx) -{ - const struct syscalltbl *table = find_table(e_machine); - - for (int i = *idx + 1; table && i < table->sorted_names_len; ++i) { - const char *name = table->num_to_name[table->sorted_names[i]]; - - if (strglobmatch(name, syscall_glob)) { - *idx = i; - return table->sorted_names[i]; - } - } - - return -1; -} - -int syscalltbl__strglobmatch_first(int e_machine, const char *syscall_glob, int *idx) -{ - *idx = -1; - return syscalltbl__strglobmatch_next(e_machine, syscall_glob, idx); -} diff --git a/tools/perf/util/syscalltbl.h b/tools/perf/util/syscalltbl.h deleted file mode 100644 index 2bb628eff367..000000000000 --- a/tools/perf/util/syscalltbl.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __PERF_SYSCALLTBL_H -#define __PERF_SYSCALLTBL_H - -const char *syscalltbl__name(int e_machine, int id); -int syscalltbl__id(int e_machine, const char *name); -int syscalltbl__num_idx(int e_machine); -int syscalltbl__id_at_idx(int e_machine, int idx); - -int syscalltbl__strglobmatch_first(int e_machine, const char *syscall_glob, int *idx); -int syscalltbl__strglobmatch_next(int e_machine, const char *syscall_glob, int *idx); - -#endif /* __PERF_SYSCALLTBL_H */ diff --git a/tools/perf/util/thread.c b/tools/perf/util/thread.c index 22be77225bb0..e483ffcb5d93 100644 --- a/tools/perf/util/thread.c +++ b/tools/perf/util/thread.c @@ -56,6 +56,7 @@ struct thread *thread__new(pid_t pid, pid_t tid) thread__set_cpu(thread, -1); thread__set_guest_cpu(thread, -1); thread__set_e_machine(thread, EM_NONE); + thread__set_e_is_big_endian(thread, false); thread__set_lbr_stitch_enable(thread, false); INIT_LIST_HEAD(thread__namespaces_list(thread)); INIT_LIST_HEAD(thread__comm_list(thread)); @@ -294,6 +295,11 @@ int thread__set_comm_from_proc(struct thread *thread) if (!(snprintf(path, sizeof(path), "%d/task/%d/comm", thread__pid(thread), thread__tid(thread)) >= (int)sizeof(path)) && procfs__read_str(path, &comm, &sz) == 0) { + /* sz==0: read got nothing, e.g. race during exit teardown */ + if (sz == 0) { + free(comm); + return -1; + } comm[sz - 1] = '\0'; err = thread__set_comm(thread, comm, 0); } @@ -358,41 +364,21 @@ size_t thread__fprintf(struct thread *thread, FILE *fp) int thread__insert_map(struct thread *thread, struct map *map) { int ret; + uint16_t e_machine; - ret = unwind__prepare_access(thread__maps(thread), map, NULL); + ret = maps__fixup_overlap_and_insert(thread__maps(thread), map); if (ret) return ret; - return maps__fixup_overlap_and_insert(thread__maps(thread), map); -} - -struct thread__prepare_access_maps_cb_args { - int err; - struct maps *maps; -}; - -static int thread__prepare_access_maps_cb(struct map *map, void *data) -{ - bool initialized = false; - struct thread__prepare_access_maps_cb_args *args = data; - - args->err = unwind__prepare_access(args->maps, map, &initialized); - - return (args->err || initialized) ? 1 : 0; + e_machine = thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL); + return unwind__prepare_access(thread__maps(thread), e_machine); } static int thread__prepare_access(struct thread *thread) { - struct thread__prepare_access_maps_cb_args args = { - .err = 0, - }; + uint16_t e_machine = thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL); - if (dwarf_callchain_users) { - args.maps = thread__maps(thread); - maps__for_each_map(thread__maps(thread), thread__prepare_access_maps_cb, &args); - } - - return args.err; + return unwind__prepare_access(thread__maps(thread), e_machine); } static int thread__clone_maps(struct thread *thread, struct thread *parent, bool do_maps_clone) @@ -449,7 +435,7 @@ void thread__find_cpumode_addr_location(struct thread *thread, u64 addr, } } -static uint16_t read_proc_e_machine_for_pid(pid_t pid, uint32_t *e_flags) +static uint16_t read_proc_e_machine_for_pid(pid_t pid, uint32_t *e_flags, bool *is_big_endian) { char path[6 /* "/proc/" */ + 11 /* max length of pid */ + 5 /* "/exe\0" */]; int fd; @@ -458,7 +444,8 @@ static uint16_t read_proc_e_machine_for_pid(pid_t pid, uint32_t *e_flags) snprintf(path, sizeof(path), "/proc/%d/exe", pid); fd = open(path, O_RDONLY); if (fd >= 0) { - e_machine = dso__read_e_machine(/*optional_dso=*/NULL, fd, e_flags); + e_machine = dso__read_e_machine_endian(/*optional_dso=*/NULL, fd, e_flags, + is_big_endian); close(fd); } return e_machine; @@ -468,6 +455,7 @@ struct thread__e_machine_callback_args { struct machine *machine; uint32_t e_flags; uint16_t e_machine; + bool is_big_endian; }; static int thread__e_machine_callback(struct map *map, void *_args) @@ -478,24 +466,38 @@ static int thread__e_machine_callback(struct map *map, void *_args) if (!dso) return 0; // No dso, continue search. - args->e_machine = dso__e_machine(dso, args->machine, &args->e_flags); + args->e_machine = + dso__e_machine_endian(dso, args->machine, &args->e_flags, &args->is_big_endian); return args->e_machine != EM_NONE ? 1 /* stop search */ : 0 /* continue search */; } -uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint32_t *e_flags) +uint16_t thread__e_machine_endian(struct thread *thread, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian) { pid_t tid, pid; - uint16_t e_machine = RC_CHK_ACCESS(thread)->e_machine; + uint16_t e_machine; uint32_t local_e_flags = 0; - struct thread__e_machine_callback_args args = { - .machine = machine, - .e_flags = 0, - .e_machine = EM_NONE, - }; + struct thread__e_machine_callback_args args; + + if (!thread) { + if (is_big_endian) { + *is_big_endian = perf_arch_is_big_endian( + machine && machine->env ? perf_env__arch(machine->env) : NULL); + } + return perf_env__e_machine(machine ? machine->env : NULL, e_flags); + } + + e_machine = RC_CHK_ACCESS(thread)->e_machine; + args.machine = machine; + args.e_flags = 0; + args.e_machine = EM_NONE; + args.is_big_endian = false; if (e_machine != EM_NONE) { if (e_flags) *e_flags = thread__e_flags(thread); + if (is_big_endian) + *is_big_endian = thread__e_is_big_endian(thread); return e_machine; } @@ -503,6 +505,7 @@ uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint3 struct maps *maps = thread__maps(thread); machine = maps__machine(maps); + args.machine = machine; } tid = thread__tid(thread); pid = thread__pid(thread); @@ -510,7 +513,8 @@ uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint3 struct thread *parent = machine__findnew_thread(machine, pid, pid); if (parent) { - e_machine = thread__e_machine(parent, machine, &local_e_flags); + e_machine = thread__e_machine_endian(parent, machine, &local_e_flags, + &args.is_big_endian); thread__put(parent); goto out; } @@ -535,16 +539,27 @@ uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint3 is_live = !!session->data; } /* Read from /proc/pid/exe if live. */ - if (is_live) - e_machine = read_proc_e_machine_for_pid(pid, &local_e_flags); + if (is_live) { + e_machine = read_proc_e_machine_for_pid(pid, &local_e_flags, + &args.is_big_endian); + } else if (machine && machine->env) { + /* Offline analysis: fallback to environment metadata. */ + e_machine = perf_env__e_machine(machine->env, &local_e_flags); + args.is_big_endian = perf_arch_is_big_endian(perf_env__arch(machine->env)); + } } out: if (e_machine != EM_NONE) { - thread__set_e_machine(thread, e_machine); thread__set_e_flags(thread, local_e_flags); + thread__set_e_is_big_endian(thread, args.is_big_endian); + thread__set_e_machine(thread, e_machine); + if (is_big_endian) + *is_big_endian = args.is_big_endian; } else { e_machine = EM_HOST; local_e_flags = EF_HOST; + if (is_big_endian) + *is_big_endian = (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__); } if (e_flags) *e_flags = local_e_flags; diff --git a/tools/perf/util/thread.h b/tools/perf/util/thread.h index f5792d3e8a16..d82fce8173ae 100644 --- a/tools/perf/util/thread.h +++ b/tools/perf/util/thread.h @@ -69,6 +69,11 @@ DECLARE_RC_STRUCT(thread) { * computed. */ uint16_t e_machine; + /** + * @e_is_big_endian: True if the ELF architecture of the thread is big endian. + * Valid if e_machine != EM_NONE. + */ + bool e_is_big_endian; /* LBR call stack stitch */ bool lbr_stitch_enable; struct lbr_stitch *lbr_stitch; @@ -311,7 +316,13 @@ static inline void thread__set_filter_entry_depth(struct thread *thread, int dep RC_CHK_ACCESS(thread)->filter_entry_depth = depth; } -uint16_t thread__e_machine(struct thread *thread, struct machine *machine, uint32_t *e_flags); +uint16_t thread__e_machine_endian(struct thread *thread, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t thread__e_machine(struct thread *thread, struct machine *machine, + uint32_t *e_flags) +{ + return thread__e_machine_endian(thread, machine, e_flags, NULL); +} static inline void thread__set_e_machine(struct thread *thread, uint16_t e_machine) { @@ -328,6 +339,16 @@ static inline void thread__set_e_flags(struct thread *thread, uint32_t e_flags) RC_CHK_ACCESS(thread)->e_flags = e_flags; } +static inline bool thread__e_is_big_endian(const struct thread *thread) +{ + return RC_CHK_ACCESS(thread)->e_is_big_endian; +} + +static inline void thread__set_e_is_big_endian(struct thread *thread, bool is_big_endian) +{ + RC_CHK_ACCESS(thread)->e_is_big_endian = is_big_endian; +} + static inline bool thread__lbr_stitch_enable(const struct thread *thread) { diff --git a/tools/perf/util/tool.c b/tools/perf/util/tool.c index 013c7839e2cf..25c9b378aa16 100644 --- a/tools/perf/util/tool.c +++ b/tools/perf/util/tool.c @@ -24,11 +24,32 @@ static int perf_session__process_compressed_event(const struct perf_tool *tool _ size_t mmap_len, decomp_len = perf_session__env(session)->comp_mmap_len; struct decomp *decomp, *decomp_last = session->active_decomp->decomp_last; + if (!decomp_len) { + pr_err("Compressed events found but HEADER_COMPRESSED not set\n"); + return -1; + } + if (decomp_last) { + /* Prevent u64 underflow in decomp_last_rem */ + if (decomp_last->head > decomp_last->size) + return -1; decomp_last_rem = decomp_last->size - decomp_last->head; + /* + * Check before adding: on 32-bit, size_t += u64 + * silently truncates, bypassing the overflow check + * below and producing an undersized buffer. + */ + if (decomp_last_rem > SIZE_MAX - decomp_len - sizeof(struct decomp)) { + pr_err("Decompression buffer size overflow\n"); + return -1; + } decomp_len += decomp_last_rem; } + if (decomp_len > SIZE_MAX - sizeof(struct decomp)) { + pr_err("Decompression buffer size overflow\n"); + return -1; + } mmap_len = sizeof(struct decomp) + decomp_len; decomp = mmap(NULL, mmap_len, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); @@ -47,14 +68,37 @@ static int perf_session__process_compressed_event(const struct perf_tool *tool _ decomp->size = decomp_last_rem; } + /* + * Events are read directly from the mmap'd file; fields could + * theoretically change via a FUSE-backed file, but that applies + * to the entire event processing pipeline, not just here. + */ if (event->header.type == PERF_RECORD_COMPRESSED) { + if (event->header.size < sizeof(struct perf_record_compressed)) + goto err_decomp; src = (void *)event + sizeof(struct perf_record_compressed); src_size = event->pack.header.size - sizeof(struct perf_record_compressed); } else if (event->header.type == PERF_RECORD_COMPRESSED2) { + /* + * prefetch_event() only guarantees that the 8-byte + * event header fits; validate that header.size covers + * the data_size field before accessing it, otherwise a + * crafted event reads data_size from adjacent memory. + */ + if (event->header.size < sizeof(struct perf_record_compressed2)) + goto err_decomp; src = (void *)event + sizeof(struct perf_record_compressed2); src_size = event->pack2.data_size; + /* + * data_size is independent of header.size (which + * includes padding); verify it doesn't exceed the + * actual payload to prevent out-of-bounds reads in + * zstd_decompress_stream(). + */ + if (src_size > event->header.size - sizeof(struct perf_record_compressed2)) + goto err_decomp; } else { - return -1; + goto err_decomp; } decomp_size = zstd_decompress_stream(session->active_decomp->zstd_decomp, src, src_size, @@ -77,6 +121,11 @@ static int perf_session__process_compressed_event(const struct perf_tool *tool _ pr_debug("decomp (B): %zd to %zd\n", src_size, decomp_size); return 0; + +err_decomp: + munmap(decomp, mmap_len); + pr_err("Couldn't decompress data\n"); + return -1; } #endif @@ -110,7 +159,6 @@ static int process_event_synth_event_update_stub(const struct perf_tool *tool __ int process_event_sample_stub(const struct perf_tool *tool __maybe_unused, union perf_event *event __maybe_unused, struct perf_sample *sample __maybe_unused, - struct evsel *evsel __maybe_unused, struct machine *machine __maybe_unused) { dump_printf(": unhandled!\n"); @@ -285,6 +333,7 @@ void perf_tool__init(struct perf_tool *tool, bool ordered_events) tool->no_warn = false; tool->show_feat_hdr = SHOW_FEAT_NO_HEADER; tool->merge_deferred_callchains = true; + tool->dont_split_sample_group = false; tool->sample = process_event_sample_stub; tool->mmap = process_event_stub; @@ -348,12 +397,11 @@ bool perf_tool__compressed_is_stub(const struct perf_tool *tool) static int delegate_ ## name(const struct perf_tool *tool, \ union perf_event *event, \ struct perf_sample *sample, \ - struct evsel *evsel, \ struct machine *machine) \ { \ struct delegate_tool *del_tool = container_of(tool, struct delegate_tool, tool); \ struct perf_tool *delegate = del_tool->delegate; \ - return delegate->name(delegate, event, sample, evsel, machine); \ + return delegate->name(delegate, event, sample, machine); \ } CREATE_DELEGATE_SAMPLE(read); CREATE_DELEGATE_SAMPLE(sample); @@ -433,6 +481,8 @@ CREATE_DELEGATE_OP2(stat_config); CREATE_DELEGATE_OP2(stat_round); CREATE_DELEGATE_OP2(thread_map); CREATE_DELEGATE_OP2(time_conv); +CREATE_DELEGATE_OP2(schedstat_cpu); +CREATE_DELEGATE_OP2(schedstat_domain); CREATE_DELEGATE_OP2(tracing_data); #define CREATE_DELEGATE_OP3(name) \ @@ -470,6 +520,7 @@ void delegate_tool__init(struct delegate_tool *tool, struct perf_tool *delegate) tool->tool.no_warn = delegate->no_warn; tool->tool.show_feat_hdr = delegate->show_feat_hdr; tool->tool.merge_deferred_callchains = delegate->merge_deferred_callchains; + tool->tool.dont_split_sample_group = delegate->dont_split_sample_group; tool->tool.sample = delegate_sample; tool->tool.read = delegate_read; @@ -516,4 +567,6 @@ void delegate_tool__init(struct delegate_tool *tool, struct perf_tool *delegate) tool->tool.bpf_metadata = delegate_bpf_metadata; tool->tool.compressed = delegate_compressed; tool->tool.auxtrace = delegate_auxtrace; + tool->tool.schedstat_cpu = delegate_schedstat_cpu; + tool->tool.schedstat_domain = delegate_schedstat_domain; } diff --git a/tools/perf/util/tool.h b/tools/perf/util/tool.h index 2d9a4b1ca9d0..2a4f124ffd8d 100644 --- a/tools/perf/util/tool.h +++ b/tools/perf/util/tool.h @@ -9,7 +9,6 @@ struct perf_session; union perf_event; struct evlist; -struct evsel; struct perf_sample; struct perf_tool; struct machine; @@ -17,7 +16,7 @@ struct ordered_events; typedef int (*event_sample)(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine); + struct machine *machine); typedef int (*event_op)(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, struct machine *machine); @@ -103,7 +102,6 @@ bool perf_tool__compressed_is_stub(const struct perf_tool *tool); int process_event_sample_stub(const struct perf_tool *tool, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct machine *machine); struct delegate_tool { diff --git a/tools/perf/util/tool_pmu.c b/tools/perf/util/tool_pmu.c index 6a9df3dc0e07..5c30854b4644 100644 --- a/tools/perf/util/tool_pmu.c +++ b/tools/perf/util/tool_pmu.c @@ -17,6 +17,8 @@ #include <fcntl.h> #include <strings.h> +#define INVALID_START_TIME ~0ULL + static const char *const tool_pmu__event_names[TOOL_PMU__EVENT_MAX] = { NULL, "duration_time", @@ -205,20 +207,57 @@ int evsel__tool_pmu_prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, int nthreads) { - if ((evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME || - evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) && - !evsel->start_times) { - evsel->start_times = xyarray__new(perf_cpu_map__nr(cpus), - nthreads, - sizeof(__u64)); - if (!evsel->start_times) - return -ENOMEM; + enum tool_pmu_event ev = evsel__tool_event(evsel); + + if (ev == TOOL_PMU__EVENT_SYSTEM_TIME || ev == TOOL_PMU__EVENT_USER_TIME) { + if (!evsel->process_time.start_times) { + evsel->process_time.start_times = + xyarray__new(perf_cpu_map__nr(cpus), nthreads, sizeof(__u64)); + if (!evsel->process_time.start_times) + return -ENOMEM; + } + if (!evsel->process_time.accumulated_times) { + evsel->process_time.accumulated_times = + xyarray__new(perf_cpu_map__nr(cpus), nthreads, sizeof(__u64)); + if (!evsel->process_time.accumulated_times) + return -ENOMEM; + } } return 0; } #define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) +static int tool_pmu__read_stat(struct evsel *evsel, int cpu_map_idx, int thread, __u64 *val) +{ + enum tool_pmu_event ev = evsel__tool_event(evsel); + bool system = ev == TOOL_PMU__EVENT_SYSTEM_TIME; + int fd = FD(evsel, cpu_map_idx, thread); + int err = 0; + + if (fd < 0) { + *val = 0; + return 0; + } + + lseek(fd, 0, SEEK_SET); + if (evsel->pid_stat) { + if (cpu_map_idx == 0) + err = read_pid_stat_field(fd, system ? 15 : 14, val); + else + *val = 0; + } else { + if (thread == 0) { + struct perf_cpu cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx); + + err = read_stat_field(fd, cpu, system ? 3 : 1, val); + } else { + *val = 0; + } + } + return err; +} + int evsel__tool_pmu_open(struct evsel *evsel, struct perf_thread_map *threads, int start_cpu_map_idx, int end_cpu_map_idx) @@ -232,7 +271,14 @@ int evsel__tool_pmu_open(struct evsel *evsel, if (ev == TOOL_PMU__EVENT_DURATION_TIME) { if (evsel->core.attr.sample_period) /* no sampling */ return -EINVAL; - evsel->start_time = rdclock(); + evsel->duration_time.accumulated_time = 0; + if (evsel->core.attr.disabled) { + evsel->disabled = true; + evsel->duration_time.start_time = INVALID_START_TIME; + } else { + evsel->disabled = false; + evsel->duration_time.start_time = rdclock(); + } return 0; } @@ -246,8 +292,8 @@ int evsel__tool_pmu_open(struct evsel *evsel, pid = perf_thread_map__pid(threads, thread); if (ev == TOOL_PMU__EVENT_USER_TIME || ev == TOOL_PMU__EVENT_SYSTEM_TIME) { - bool system = ev == TOOL_PMU__EVENT_SYSTEM_TIME; __u64 *start_time = NULL; + __u64 *accumulated_time = NULL; int fd; if (evsel->core.attr.sample_period) { @@ -269,21 +315,25 @@ int evsel__tool_pmu_open(struct evsel *evsel, err = -errno; goto out_close; } - start_time = xyarray__entry(evsel->start_times, idx, thread); - if (pid > -1) { - err = read_pid_stat_field(fd, system ? 15 : 14, - start_time); + start_time = xyarray__entry(evsel->process_time.start_times, idx, + thread); + accumulated_time = xyarray__entry( + evsel->process_time.accumulated_times, idx, thread); + *accumulated_time = 0; + + if (evsel->core.attr.disabled) { + evsel->disabled = true; + *start_time = INVALID_START_TIME; } else { - struct perf_cpu cpu; - - cpu = perf_cpu_map__cpu(evsel->core.cpus, idx); - err = read_stat_field(fd, cpu, system ? 3 : 1, - start_time); + evsel->disabled = false; + err = tool_pmu__read_stat(evsel, idx, thread, start_time); + if (err) { + close(fd); + FD(evsel, idx, thread) = -1; + goto out_close; + } } - if (err) - goto out_close; } - } } return 0; @@ -467,10 +517,111 @@ static void perf_counts__update(struct perf_counts_values *count, count->lost = 0; } } +int evsel__tool_pmu_enable_cpu(struct evsel *evsel, int cpu_map_idx) +{ + enum tool_pmu_event ev = evsel__tool_event(evsel); + int thread, nthreads; + + if (!evsel->disabled) + return 0; + + if (ev == TOOL_PMU__EVENT_DURATION_TIME) { + if (cpu_map_idx == 0) + evsel->duration_time.start_time = rdclock(); + return 0; + } + + if (ev == TOOL_PMU__EVENT_USER_TIME || ev == TOOL_PMU__EVENT_SYSTEM_TIME) { + nthreads = xyarray__max_y(evsel->process_time.start_times); + for (thread = 0; thread < nthreads; thread++) { + __u64 *start_time = xyarray__entry(evsel->process_time.start_times, + cpu_map_idx, thread); + __u64 val; + int err; + + err = tool_pmu__read_stat(evsel, cpu_map_idx, thread, &val); + if (!err) + *start_time = val; + else + *start_time = INVALID_START_TIME; + } + } + return 0; +} + +int evsel__tool_pmu_enable(struct evsel *evsel) +{ + unsigned int idx; + int err = 0; + + if (!evsel->disabled) + return 0; + + for (idx = 0; idx < perf_cpu_map__nr(evsel->core.cpus); idx++) { + err = evsel__tool_pmu_enable_cpu(evsel, idx); + if (err) + break; + } + return err; +} + +int evsel__tool_pmu_disable_cpu(struct evsel *evsel, int cpu_map_idx) +{ + enum tool_pmu_event ev = evsel__tool_event(evsel); + int thread, nthreads; + + if (evsel->disabled) + return 0; + + if (ev == TOOL_PMU__EVENT_DURATION_TIME) { + if (cpu_map_idx == 0) { + __u64 delta = rdclock() - evsel->duration_time.start_time; + + evsel->duration_time.accumulated_time += delta; + } + return 0; + } + + if (ev == TOOL_PMU__EVENT_USER_TIME || ev == TOOL_PMU__EVENT_SYSTEM_TIME) { + nthreads = xyarray__max_y(evsel->process_time.start_times); + for (thread = 0; thread < nthreads; thread++) { + __u64 *start_time = xyarray__entry(evsel->process_time.start_times, + cpu_map_idx, thread); + __u64 *accumulated_time = xyarray__entry( + evsel->process_time.accumulated_times, cpu_map_idx, thread); + __u64 val; + int err; + + err = tool_pmu__read_stat(evsel, cpu_map_idx, thread, &val); + if (!err) { + if (*start_time != INVALID_START_TIME && val >= *start_time) + *accumulated_time += (val - *start_time); + } + *start_time = INVALID_START_TIME; + } + } + return 0; +} + +int evsel__tool_pmu_disable(struct evsel *evsel) +{ + unsigned int idx; + int err = 0; + + if (evsel->disabled) + return 0; + + for (idx = 0; idx < perf_cpu_map__nr(evsel->core.cpus); idx++) { + err = evsel__tool_pmu_disable_cpu(evsel, idx); + if (err) + break; + } + return err; +} int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) { - __u64 *start_time, cur_time, delta_start; + __u64 delta_start = 0; int err = 0; struct perf_counts_values *count, *old_count = NULL; bool adjust = false; @@ -507,39 +658,33 @@ int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) return 0; } case TOOL_PMU__EVENT_DURATION_TIME: - /* - * Pretend duration_time is only on the first CPU and thread, or - * else aggregation will scale duration_time by the number of - * CPUs/threads. - */ - start_time = &evsel->start_time; - if (cpu_map_idx == 0 && thread == 0) - cur_time = rdclock(); - else - cur_time = *start_time; + if (cpu_map_idx == 0 && thread == 0) { + delta_start = evsel->duration_time.accumulated_time; + if (!evsel->disabled && + evsel->duration_time.start_time != INVALID_START_TIME) + delta_start += (rdclock() - evsel->duration_time.start_time); + } else { + delta_start = 0; + } break; case TOOL_PMU__EVENT_USER_TIME: case TOOL_PMU__EVENT_SYSTEM_TIME: { - bool system = evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME; - int fd = FD(evsel, cpu_map_idx, thread); - - start_time = xyarray__entry(evsel->start_times, cpu_map_idx, thread); - lseek(fd, SEEK_SET, 0); - if (evsel->pid_stat) { - /* The event exists solely on 1 CPU. */ - if (cpu_map_idx == 0) - err = read_pid_stat_field(fd, system ? 15 : 14, &cur_time); - else - cur_time = 0; - } else { - /* The event is for all threads. */ - if (thread == 0) { - struct perf_cpu cpu = perf_cpu_map__cpu(evsel->core.cpus, - cpu_map_idx); + __u64 accumulated = *(__u64 *)xyarray__entry(evsel->process_time.accumulated_times, + cpu_map_idx, thread); - err = read_stat_field(fd, cpu, system ? 3 : 1, &cur_time); - } else { - cur_time = 0; + if (evsel->disabled) { + delta_start = accumulated; + } else { + __u64 *start_time = xyarray__entry(evsel->process_time.start_times, + cpu_map_idx, thread); + __u64 cur_time; + + err = tool_pmu__read_stat(evsel, cpu_map_idx, thread, &cur_time); + if (!err) { + if (*start_time != INVALID_START_TIME && cur_time >= *start_time) + delta_start = accumulated + (cur_time - *start_time); + else + delta_start = accumulated; } } adjust = true; @@ -553,7 +698,6 @@ int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) if (err) return err; - delta_start = cur_time - *start_time; if (adjust) { __u64 ticks_per_sec = sysconf(_SC_CLK_TCK); diff --git a/tools/perf/util/tool_pmu.h b/tools/perf/util/tool_pmu.h index f1714001bc1d..f66d24cf3502 100644 --- a/tools/perf/util/tool_pmu.h +++ b/tools/perf/util/tool_pmu.h @@ -56,6 +56,10 @@ int evsel__tool_pmu_prepare_open(struct evsel *evsel, int evsel__tool_pmu_open(struct evsel *evsel, struct perf_thread_map *threads, int start_cpu_map_idx, int end_cpu_map_idx); +int evsel__tool_pmu_enable_cpu(struct evsel *evsel, int cpu_map_idx); +int evsel__tool_pmu_enable(struct evsel *evsel); +int evsel__tool_pmu_disable_cpu(struct evsel *evsel, int cpu_map_idx); +int evsel__tool_pmu_disable(struct evsel *evsel); int evsel__tool_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread); struct perf_pmu *tool_pmu__new(void); diff --git a/tools/perf/util/trace-event-scripting.c b/tools/perf/util/trace-event-scripting.c index fa850e44cb46..dc584ac316a3 100644 --- a/tools/perf/util/trace-event-scripting.c +++ b/tools/perf/util/trace-event-scripting.c @@ -103,12 +103,11 @@ int script_spec__for_each(int (*cb)(struct scripting_ops *ops, const char *spec) void scripting_context__update(struct scripting_context *c, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al) { #ifdef HAVE_LIBTRACEEVENT - const struct tep_event *tp_format = evsel__tp_format(evsel); + const struct tep_event *tp_format = evsel__tp_format(sample->evsel); c->pevent = tp_format ? tp_format->tep : NULL; #else @@ -117,7 +116,6 @@ void scripting_context__update(struct scripting_context *c, c->event_data = sample->raw_data; c->event = event; c->sample = sample; - c->evsel = evsel; c->al = al; c->addr_al = addr_al; } @@ -134,7 +132,6 @@ static int stop_script_unsupported(void) static void process_event_unsupported(union perf_event *event __maybe_unused, struct perf_sample *sample __maybe_unused, - struct evsel *evsel __maybe_unused, struct addr_location *al __maybe_unused, struct addr_location *addr_al __maybe_unused) { diff --git a/tools/perf/util/trace-event.h b/tools/perf/util/trace-event.h index 914d9b69ed62..720121c74f1d 100644 --- a/tools/perf/util/trace-event.h +++ b/tools/perf/util/trace-event.h @@ -94,7 +94,6 @@ struct scripting_ops { int (*stop_script) (void); void (*process_event) (union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al); void (*process_switch)(union perf_event *event, @@ -124,7 +123,6 @@ struct scripting_context { void *event_data; union perf_event *event; struct perf_sample *sample; - struct evsel *evsel; struct addr_location *al; struct addr_location *addr_al; struct perf_session *session; @@ -133,7 +131,6 @@ struct scripting_context { void scripting_context__update(struct scripting_context *scripting_context, union perf_event *event, struct perf_sample *sample, - struct evsel *evsel, struct addr_location *al, struct addr_location *addr_al); diff --git a/tools/perf/util/tsc.c b/tools/perf/util/tsc.c index 511a517ce613..ebf289bf6b9d 100644 --- a/tools/perf/util/tsc.c +++ b/tools/perf/util/tsc.c @@ -127,7 +127,7 @@ size_t perf_event__fprintf_time_conv(union perf_event *event, FILE *fp) * when supported cap_user_time_short, for backward compatibility, * prints the extended fields only if they are contained in the event. */ - if (event_contains(*tc, time_cycles)) { + if (event_contains(*tc, cap_user_time_short)) { ret += fprintf(fp, "... Time Cycles %" PRI_lu64 "\n", tc->time_cycles); ret += fprintf(fp, "... Time Mask %#" PRI_lx64 "\n", diff --git a/tools/perf/util/unwind-libdw.c b/tools/perf/util/unwind-libdw.c index 05e8e68bd49c..7f35042be567 100644 --- a/tools/perf/util/unwind-libdw.c +++ b/tools/perf/util/unwind-libdw.c @@ -339,7 +339,7 @@ frame_callback(Dwfl_Frame *state, void *arg) DWARF_CB_ABORT : DWARF_CB_OK; } -int unwind__get_entries(unwind_entry_cb_t cb, void *arg, +int libdw__get_entries(unwind_entry_cb_t cb, void *arg, struct thread *thread, struct perf_sample *data, int max_stack, @@ -353,10 +353,10 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, static struct unwind_info *ui; Dwfl *dwfl; Dwarf_Word ip; - int err = -EINVAL, i; + int err = -EINVAL, i, entries; if (!data->user_regs || !data->user_regs->regs) - return -EINVAL; + return 0; ui = zalloc(sizeof(*ui) + sizeof(ui->entries[0]) * max_stack); if (!ui) @@ -430,6 +430,18 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, map_symbol__exit(&ui->entries[i].ms); dwfl_ui_ti->ui = NULL; + entries = (int)ui->idx; free(ui); - return 0; + /* + * Unwinder return contract: + * > 0 : unwinding succeeded (stops fallback). If we found frames but hit an error + * (e.g. truncated stack), report success to preserve existing frames. + * 0 : unwinding failed without yielding frames. Ignore non-fatal errors + * (e.g. missing debug info, DWARF corruption) to allow fallback unwinder or + * kernel callchain resolution to proceed. + * < 0 : fatal error (e.g. -ENOMEM). Aborts unwinding entirely. + */ + if (err) + return (err == -ENOMEM) ? -ENOMEM : (entries > 0 ? 1 : 0); + return entries; } diff --git a/tools/perf/util/unwind-libunwind-local.c b/tools/perf/util/unwind-libunwind-local.c deleted file mode 100644 index 87d496e9dfa6..000000000000 --- a/tools/perf/util/unwind-libunwind-local.c +++ /dev/null @@ -1,831 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Post mortem Dwarf CFI based unwinding on top of regs and stack dumps. - * - * Lots of this code have been borrowed or heavily inspired from parts of - * the libunwind 0.99 code which are (amongst other contributors I may have - * forgotten): - * - * Copyright (C) 2002-2007 Hewlett-Packard Co - * Contributed by David Mosberger-Tang <davidm@hpl.hp.com> - * - * And the bugs have been added by: - * - * Copyright (C) 2010, Frederic Weisbecker <fweisbec@gmail.com> - * Copyright (C) 2012, Jiri Olsa <jolsa@redhat.com> - * - */ - -#include <elf.h> -#include <errno.h> -#include <gelf.h> -#include <fcntl.h> -#include <inttypes.h> -#include <string.h> -#include <unistd.h> -#include <sys/mman.h> -#include <linux/list.h> -#ifndef REMOTE_UNWIND_LIBUNWIND -#include <libunwind.h> -#include <libunwind-ptrace.h> -#endif -#include "callchain.h" -#include "thread.h" -#include "session.h" -#include "perf_regs.h" -#include "unwind.h" -#include "map.h" -#include "symbol.h" -#include "debug.h" -#include "asm/bug.h" -#include "dso.h" - -extern int -UNW_OBJ(dwarf_search_unwind_table) (unw_addr_space_t as, - unw_word_t ip, - unw_dyn_info_t *di, - unw_proc_info_t *pi, - int need_unwind_info, void *arg); - -#define dwarf_search_unwind_table UNW_OBJ(dwarf_search_unwind_table) - -extern int -UNW_OBJ(dwarf_find_debug_frame) (int found, unw_dyn_info_t *di_debug, - unw_word_t ip, - unw_word_t segbase, - const char *obj_name, unw_word_t start, - unw_word_t end); - -#define dwarf_find_debug_frame UNW_OBJ(dwarf_find_debug_frame) - -#define DW_EH_PE_FORMAT_MASK 0x0f /* format of the encoded value */ -#define DW_EH_PE_APPL_MASK 0x70 /* how the value is to be applied */ - -/* Pointer-encoding formats: */ -#define DW_EH_PE_omit 0xff -#define DW_EH_PE_ptr 0x00 /* pointer-sized unsigned value */ -#define DW_EH_PE_udata4 0x03 /* unsigned 32-bit value */ -#define DW_EH_PE_udata8 0x04 /* unsigned 64-bit value */ -#define DW_EH_PE_sdata4 0x0b /* signed 32-bit value */ -#define DW_EH_PE_sdata8 0x0c /* signed 64-bit value */ - -/* Pointer-encoding application: */ -#define DW_EH_PE_absptr 0x00 /* absolute value */ -#define DW_EH_PE_pcrel 0x10 /* rel. to addr. of encoded value */ - -/* - * The following are not documented by LSB v1.3, yet they are used by - * GCC, presumably they aren't documented by LSB since they aren't - * used on Linux: - */ -#define DW_EH_PE_funcrel 0x40 /* start-of-procedure-relative */ -#define DW_EH_PE_aligned 0x50 /* aligned pointer */ - -/* Flags intentionally not handled, since they're not needed: - * #define DW_EH_PE_indirect 0x80 - * #define DW_EH_PE_uleb128 0x01 - * #define DW_EH_PE_udata2 0x02 - * #define DW_EH_PE_sleb128 0x09 - * #define DW_EH_PE_sdata2 0x0a - * #define DW_EH_PE_textrel 0x20 - * #define DW_EH_PE_datarel 0x30 - */ - -struct unwind_info { - struct perf_sample *sample; - struct machine *machine; - struct thread *thread; - bool best_effort; -}; - -#define dw_read(ptr, type, end) ({ \ - type *__p = (type *) ptr; \ - type __v; \ - if ((__p + 1) > (type *) end) \ - return -EINVAL; \ - __v = *__p++; \ - ptr = (typeof(ptr)) __p; \ - __v; \ - }) - -static int __dw_read_encoded_value(u8 **p, u8 *end, u64 *val, - u8 encoding) -{ - u8 *cur = *p; - *val = 0; - - switch (encoding) { - case DW_EH_PE_omit: - *val = 0; - goto out; - case DW_EH_PE_ptr: - *val = dw_read(cur, unsigned long, end); - goto out; - default: - break; - } - - switch (encoding & DW_EH_PE_APPL_MASK) { - case DW_EH_PE_absptr: - break; - case DW_EH_PE_pcrel: - *val = (unsigned long) cur; - break; - default: - return -EINVAL; - } - - if ((encoding & 0x07) == 0x00) - encoding |= DW_EH_PE_udata4; - - switch (encoding & DW_EH_PE_FORMAT_MASK) { - case DW_EH_PE_sdata4: - *val += dw_read(cur, s32, end); - break; - case DW_EH_PE_udata4: - *val += dw_read(cur, u32, end); - break; - case DW_EH_PE_sdata8: - *val += dw_read(cur, s64, end); - break; - case DW_EH_PE_udata8: - *val += dw_read(cur, u64, end); - break; - default: - return -EINVAL; - } - - out: - *p = cur; - return 0; -} - -#define dw_read_encoded_value(ptr, end, enc) ({ \ - u64 __v; \ - if (__dw_read_encoded_value(&ptr, end, &__v, enc)) { \ - return -EINVAL; \ - } \ - __v; \ - }) - -static int elf_section_address_and_offset(int fd, const char *name, u64 *address, u64 *offset) -{ - Elf *elf; - GElf_Ehdr ehdr; - GElf_Shdr shdr; - int ret = -1; - - elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); - if (elf == NULL) - return -1; - - if (gelf_getehdr(elf, &ehdr) == NULL) - goto out_err; - - if (!elf_section_by_name(elf, &ehdr, &shdr, name, NULL)) - goto out_err; - - *address = shdr.sh_addr; - *offset = shdr.sh_offset; - ret = 0; -out_err: - elf_end(elf); - return ret; -} - -#ifndef NO_LIBUNWIND_DEBUG_FRAME -static u64 elf_section_offset(int fd, const char *name) -{ - u64 address, offset = 0; - - if (elf_section_address_and_offset(fd, name, &address, &offset)) - return 0; - - return offset; -} -#endif - -static u64 elf_base_address(int fd) -{ - Elf *elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); - GElf_Phdr phdr; - u64 retval = 0; - size_t i, phdrnum = 0; - - if (elf == NULL) - return 0; - (void)elf_getphdrnum(elf, &phdrnum); - /* PT_LOAD segments are sorted by p_vaddr, so the first has the minimum p_vaddr. */ - for (i = 0; i < phdrnum; i++) { - if (gelf_getphdr(elf, i, &phdr) && phdr.p_type == PT_LOAD) { - retval = phdr.p_vaddr & -getpagesize(); - break; - } - } - - elf_end(elf); - return retval; -} - -#ifndef NO_LIBUNWIND_DEBUG_FRAME -static int elf_is_exec(int fd, const char *name) -{ - Elf *elf; - GElf_Ehdr ehdr; - int retval = 0; - - elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); - if (elf == NULL) - return 0; - if (gelf_getehdr(elf, &ehdr) == NULL) - goto out; - - retval = (ehdr.e_type == ET_EXEC); - -out: - elf_end(elf); - pr_debug("unwind: elf_is_exec(%s): %d\n", name, retval); - return retval; -} -#endif - -struct table_entry { - u32 start_ip_offset; - u32 fde_offset; -}; - -struct eh_frame_hdr { - unsigned char version; - unsigned char eh_frame_ptr_enc; - unsigned char fde_count_enc; - unsigned char table_enc; - - /* - * The rest of the header is variable-length and consists of the - * following members: - * - * encoded_t eh_frame_ptr; - * encoded_t fde_count; - */ - - /* A single encoded pointer should not be more than 8 bytes. */ - u64 enc[2]; - - /* - * struct { - * encoded_t start_ip; - * encoded_t fde_addr; - * } binary_search_table[fde_count]; - */ - char data[]; -} __packed; - -static int unwind_spec_ehframe(struct dso *dso, struct machine *machine, - u64 offset, u64 *table_data_offset, u64 *fde_count) -{ - struct eh_frame_hdr hdr; - u8 *enc = (u8 *) &hdr.enc; - u8 *end = (u8 *) &hdr.data; - ssize_t r; - - r = dso__data_read_offset(dso, machine, offset, - (u8 *) &hdr, sizeof(hdr)); - if (r != sizeof(hdr)) - return -EINVAL; - - /* We dont need eh_frame_ptr, just skip it. */ - dw_read_encoded_value(enc, end, hdr.eh_frame_ptr_enc); - - *fde_count = dw_read_encoded_value(enc, end, hdr.fde_count_enc); - *table_data_offset = enc - (u8 *) &hdr; - return 0; -} - -struct read_unwind_spec_eh_frame_maps_cb_args { - struct dso *dso; - u64 base_addr; -}; - -static int read_unwind_spec_eh_frame_maps_cb(struct map *map, void *data) -{ - - struct read_unwind_spec_eh_frame_maps_cb_args *args = data; - - if (map__dso(map) == args->dso && map__start(map) - map__pgoff(map) < args->base_addr) - args->base_addr = map__start(map) - map__pgoff(map); - - return 0; -} - - -static int read_unwind_spec_eh_frame(struct dso *dso, struct unwind_info *ui, - u64 *table_data, u64 *segbase, - u64 *fde_count) -{ - struct read_unwind_spec_eh_frame_maps_cb_args args = { - .dso = dso, - .base_addr = UINT64_MAX, - }; - int ret, fd; - - if (dso__data(dso)->eh_frame_hdr_offset == 0) { - if (!dso__data_get_fd(dso, ui->machine, &fd)) - return -EINVAL; - - /* Check the .eh_frame section for unwinding info */ - ret = elf_section_address_and_offset(fd, ".eh_frame_hdr", - &dso__data(dso)->eh_frame_hdr_addr, - &dso__data(dso)->eh_frame_hdr_offset); - dso__data(dso)->elf_base_addr = elf_base_address(fd); - dso__data_put_fd(dso); - if (ret || dso__data(dso)->eh_frame_hdr_offset == 0) - return -EINVAL; - } - - maps__for_each_map(thread__maps(ui->thread), read_unwind_spec_eh_frame_maps_cb, &args); - - args.base_addr -= dso__data(dso)->elf_base_addr; - /* Address of .eh_frame_hdr */ - *segbase = args.base_addr + dso__data(dso)->eh_frame_hdr_addr; - ret = unwind_spec_ehframe(dso, ui->machine, dso__data(dso)->eh_frame_hdr_offset, - table_data, fde_count); - if (ret) - return ret; - /* binary_search_table offset plus .eh_frame_hdr address */ - *table_data += *segbase; - return 0; -} - -#ifndef NO_LIBUNWIND_DEBUG_FRAME -static int read_unwind_spec_debug_frame(struct dso *dso, - struct machine *machine, u64 *offset) -{ - int fd; - u64 ofs = dso__data(dso)->debug_frame_offset; - - /* debug_frame can reside in: - * - dso - * - debug pointed by symsrc_filename - * - gnu_debuglink, which doesn't necessary - * has to be pointed by symsrc_filename - */ - if (ofs == 0) { - if (dso__data_get_fd(dso, machine, &fd)) { - ofs = elf_section_offset(fd, ".debug_frame"); - dso__data_put_fd(dso); - } - - if (ofs <= 0) { - fd = open(dso__symsrc_filename(dso), O_RDONLY); - if (fd >= 0) { - ofs = elf_section_offset(fd, ".debug_frame"); - close(fd); - } - } - - if (ofs <= 0) { - char *debuglink = malloc(PATH_MAX); - int ret = 0; - - if (debuglink == NULL) { - pr_err("unwind: Can't read unwind spec debug frame.\n"); - return -ENOMEM; - } - - ret = dso__read_binary_type_filename( - dso, DSO_BINARY_TYPE__DEBUGLINK, - machine->root_dir, debuglink, PATH_MAX); - if (!ret) { - fd = open(debuglink, O_RDONLY); - if (fd >= 0) { - ofs = elf_section_offset(fd, - ".debug_frame"); - close(fd); - } - } - if (ofs > 0) { - if (dso__symsrc_filename(dso) != NULL) { - pr_warning( - "%s: overwrite symsrc(%s,%s)\n", - __func__, - dso__symsrc_filename(dso), - debuglink); - dso__free_symsrc_filename(dso); - } - dso__set_symsrc_filename(dso, debuglink); - } else { - free(debuglink); - } - } - - dso__data(dso)->debug_frame_offset = ofs; - } - - *offset = ofs; - if (*offset) - return 0; - - return -EINVAL; -} -#endif - -static struct map *find_map(unw_word_t ip, struct unwind_info *ui) -{ - struct addr_location al; - struct map *ret; - - addr_location__init(&al); - thread__find_map(ui->thread, PERF_RECORD_MISC_USER, ip, &al); - ret = map__get(al.map); - addr_location__exit(&al); - return ret; -} - -static int -find_proc_info(unw_addr_space_t as, unw_word_t ip, unw_proc_info_t *pi, - int need_unwind_info, void *arg) -{ - struct unwind_info *ui = arg; - struct map *map; - struct dso *dso; - unw_dyn_info_t di; - u64 table_data, segbase, fde_count; - int ret = -EINVAL; - - map = find_map(ip, ui); - if (!map) - return -EINVAL; - - dso = map__dso(map); - if (!dso) { - map__put(map); - return -EINVAL; - } - - pr_debug("unwind: find_proc_info dso %s\n", dso__name(dso)); - - /* Check the .eh_frame section for unwinding info */ - if (!read_unwind_spec_eh_frame(dso, ui, &table_data, &segbase, &fde_count)) { - memset(&di, 0, sizeof(di)); - di.format = UNW_INFO_FORMAT_REMOTE_TABLE; - di.start_ip = map__start(map); - di.end_ip = map__end(map); - di.u.rti.segbase = segbase; - di.u.rti.table_data = table_data; - di.u.rti.table_len = fde_count * sizeof(struct table_entry) - / sizeof(unw_word_t); - ret = dwarf_search_unwind_table(as, ip, &di, pi, - need_unwind_info, arg); - } - -#ifndef NO_LIBUNWIND_DEBUG_FRAME - /* Check the .debug_frame section for unwinding info */ - if (ret < 0 && - !read_unwind_spec_debug_frame(dso, ui->machine, &segbase)) { - int fd; - u64 start = map__start(map); - unw_word_t base = start; - const char *symfile; - - if (dso__data_get_fd(dso, ui->machine, &fd)) { - if (elf_is_exec(fd, dso__name(dso))) - base = 0; - dso__data_put_fd(dso); - } - - symfile = dso__symsrc_filename(dso) ?: dso__name(dso); - - memset(&di, 0, sizeof(di)); - if (dwarf_find_debug_frame(0, &di, ip, base, symfile, start, map__end(map))) - ret = dwarf_search_unwind_table(as, ip, &di, pi, - need_unwind_info, arg); - } -#endif - map__put(map); - return ret; -} - -static int access_fpreg(unw_addr_space_t __maybe_unused as, - unw_regnum_t __maybe_unused num, - unw_fpreg_t __maybe_unused *val, - int __maybe_unused __write, - void __maybe_unused *arg) -{ - pr_err("unwind: access_fpreg unsupported\n"); - return -UNW_EINVAL; -} - -static int get_dyn_info_list_addr(unw_addr_space_t __maybe_unused as, - unw_word_t __maybe_unused *dil_addr, - void __maybe_unused *arg) -{ - return -UNW_ENOINFO; -} - -static int resume(unw_addr_space_t __maybe_unused as, - unw_cursor_t __maybe_unused *cu, - void __maybe_unused *arg) -{ - pr_err("unwind: resume unsupported\n"); - return -UNW_EINVAL; -} - -static int -get_proc_name(unw_addr_space_t __maybe_unused as, - unw_word_t __maybe_unused addr, - char __maybe_unused *bufp, size_t __maybe_unused buf_len, - unw_word_t __maybe_unused *offp, void __maybe_unused *arg) -{ - pr_err("unwind: get_proc_name unsupported\n"); - return -UNW_EINVAL; -} - -static int access_dso_mem(struct unwind_info *ui, unw_word_t addr, - unw_word_t *data) -{ - struct map *map; - struct dso *dso; - ssize_t size; - - map = find_map(addr, ui); - if (!map) { - pr_debug("unwind: no map for %lx\n", (unsigned long)addr); - return -1; - } - - dso = map__dso(map); - - if (!dso) { - map__put(map); - return -1; - } - - size = dso__data_read_addr(dso, map, ui->machine, - addr, (u8 *) data, sizeof(*data)); - map__put(map); - return !(size == sizeof(*data)); -} - -static int access_mem(unw_addr_space_t __maybe_unused as, - unw_word_t addr, unw_word_t *valp, - int __write, void *arg) -{ - struct unwind_info *ui = arg; - struct stack_dump *stack = &ui->sample->user_stack; - u64 start, end; - int offset; - int ret; - - /* Don't support write, probably not needed. */ - if (__write || !stack || !ui->sample->user_regs || !ui->sample->user_regs->regs) { - *valp = 0; - return 0; - } - - ret = perf_reg_value(&start, perf_sample__user_regs(ui->sample), - perf_arch_reg_sp(thread__e_machine(ui->thread, - ui->machine, - /*e_flags=*/NULL))); - if (ret) - return ret; - - end = start + stack->size; - - /* Check overflow. */ - if (addr + sizeof(unw_word_t) < addr) - return -EINVAL; - - if (addr < start || addr + sizeof(unw_word_t) >= end) { - ret = access_dso_mem(ui, addr, valp); - if (ret) { - pr_debug("unwind: access_mem %p not inside range" - " 0x%" PRIx64 "-0x%" PRIx64 "\n", - (void *) (uintptr_t) addr, start, end); - *valp = 0; - return ret; - } - return 0; - } - - offset = addr - start; - *valp = *(unw_word_t *)&stack->data[offset]; - pr_debug("unwind: access_mem addr %p val %lx, offset %d\n", - (void *) (uintptr_t) addr, (unsigned long)*valp, offset); - return 0; -} - -static int access_reg(unw_addr_space_t __maybe_unused as, - unw_regnum_t regnum, unw_word_t *valp, - int __write, void *arg) -{ - struct unwind_info *ui = arg; - int id, ret; - u64 val; - - /* Don't support write, I suspect we don't need it. */ - if (__write) { - pr_err("unwind: access_reg w %d\n", regnum); - return 0; - } - - if (!ui->sample->user_regs || !ui->sample->user_regs->regs) { - *valp = 0; - return 0; - } - - id = LIBUNWIND__ARCH_REG_ID(regnum); - if (id < 0) - return -EINVAL; - - ret = perf_reg_value(&val, perf_sample__user_regs(ui->sample), id); - if (ret) { - if (!ui->best_effort) - pr_err("unwind: can't read reg %d\n", regnum); - return ret; - } - - *valp = (unw_word_t) val; - pr_debug("unwind: reg %d, val %lx\n", regnum, (unsigned long)*valp); - return 0; -} - -static void put_unwind_info(unw_addr_space_t __maybe_unused as, - unw_proc_info_t *pi __maybe_unused, - void *arg __maybe_unused) -{ - pr_debug("unwind: put_unwind_info called\n"); -} - -static int entry(u64 ip, struct thread *thread, - unwind_entry_cb_t cb, void *arg) -{ - struct unwind_entry e; - struct addr_location al; - int ret; - - addr_location__init(&al); - e.ms.sym = thread__find_symbol(thread, PERF_RECORD_MISC_USER, ip, &al); - e.ip = ip; - e.ms.map = al.map; - e.ms.thread = thread__get(al.thread); - - pr_debug("unwind: %s:ip = 0x%" PRIx64 " (0x%" PRIx64 ")\n", - al.sym ? al.sym->name : "''", - ip, - al.map ? map__map_ip(al.map, ip) : (u64) 0); - - ret = cb(&e, arg); - addr_location__exit(&al); - return ret; -} - -static void display_error(int err) -{ - switch (err) { - case UNW_EINVAL: - pr_err("unwind: Only supports local.\n"); - break; - case UNW_EUNSPEC: - pr_err("unwind: Unspecified error.\n"); - break; - case UNW_EBADREG: - pr_err("unwind: Register unavailable.\n"); - break; - default: - break; - } -} - -static unw_accessors_t accessors = { - .find_proc_info = find_proc_info, - .put_unwind_info = put_unwind_info, - .get_dyn_info_list_addr = get_dyn_info_list_addr, - .access_mem = access_mem, - .access_reg = access_reg, - .access_fpreg = access_fpreg, - .resume = resume, - .get_proc_name = get_proc_name, -}; - -static int _unwind__prepare_access(struct maps *maps) -{ - void *addr_space = unw_create_addr_space(&accessors, 0); - - maps__set_addr_space(maps, addr_space); - if (!addr_space) { - pr_err("unwind: Can't create unwind address space.\n"); - return -ENOMEM; - } - - unw_set_caching_policy(addr_space, UNW_CACHE_GLOBAL); - return 0; -} - -static void _unwind__flush_access(struct maps *maps) -{ - unw_flush_cache(maps__addr_space(maps), 0, 0); -} - -static void _unwind__finish_access(struct maps *maps) -{ - unw_destroy_addr_space(maps__addr_space(maps)); -} - -static int get_entries(struct unwind_info *ui, unwind_entry_cb_t cb, - void *arg, int max_stack) -{ - uint16_t e_machine = thread__e_machine(ui->thread, ui->machine, /*e_flags=*/NULL); - u64 val; - unw_word_t ips[max_stack]; - unw_addr_space_t addr_space; - unw_cursor_t c; - int ret, i = 0; - - ret = perf_reg_value(&val, perf_sample__user_regs(ui->sample), - perf_arch_reg_ip(e_machine)); - if (ret) - return ret; - - ips[i++] = (unw_word_t) val; - - /* - * If we need more than one entry, do the DWARF - * unwind itself. - */ - if (max_stack - 1 > 0) { - WARN_ONCE(!ui->thread, "WARNING: ui->thread is NULL"); - addr_space = maps__addr_space(thread__maps(ui->thread)); - - if (addr_space == NULL) - return -1; - - ret = unw_init_remote(&c, addr_space, ui); - if (ret && !ui->best_effort) - display_error(ret); - - while (!ret && (unw_step(&c) > 0) && i < max_stack) { - unw_get_reg(&c, UNW_REG_IP, &ips[i]); - - /* - * Decrement the IP for any non-activation frames. - * this is required to properly find the srcline - * for caller frames. - * See also the documentation for dwfl_frame_pc(), - * which this code tries to replicate. - */ - if (unw_is_signal_frame(&c) <= 0) - --ips[i]; - - ++i; - } - - max_stack = i; - } - - /* - * Display what we got based on the order setup. - */ - for (i = 0; i < max_stack && !ret; i++) { - int j = i; - - if (callchain_param.order == ORDER_CALLER) - j = max_stack - i - 1; - ret = ips[j] ? entry(ips[j], ui->thread, cb, arg) : 0; - } - - return ret; -} - -static int _unwind__get_entries(unwind_entry_cb_t cb, void *arg, - struct thread *thread, - struct perf_sample *data, int max_stack, - bool best_effort) -{ - struct unwind_info ui = { - .sample = data, - .thread = thread, - .machine = maps__machine(thread__maps(thread)), - .best_effort = best_effort - }; - - if (!data->user_regs || !data->user_regs->regs) - return -EINVAL; - - if (max_stack <= 0) - return -EINVAL; - - return get_entries(&ui, cb, arg, max_stack); -} - -static struct unwind_libunwind_ops -_unwind_libunwind_ops = { - .prepare_access = _unwind__prepare_access, - .flush_access = _unwind__flush_access, - .finish_access = _unwind__finish_access, - .get_entries = _unwind__get_entries, -}; - -#ifndef REMOTE_UNWIND_LIBUNWIND -struct unwind_libunwind_ops * -local_unwind_libunwind_ops = &_unwind_libunwind_ops; -#endif diff --git a/tools/perf/util/unwind-libunwind.c b/tools/perf/util/unwind-libunwind.c index cb8be6acfb6f..73d191ce51a5 100644 --- a/tools/perf/util/unwind-libunwind.c +++ b/tools/perf/util/unwind-libunwind.c @@ -1,92 +1,683 @@ // SPDX-License-Identifier: GPL-2.0 -#include "unwind.h" +#include "callchain.h" +#include "debug.h" #include "dso.h" +#include "env.h" #include "map.h" -#include "thread.h" +#include "perf_regs.h" #include "session.h" -#include "debug.h" -#include "env.h" -#include "callchain.h" +#include "symbol.h" +#include "thread.h" +#include "unwind.h" +#include "libunwind-arch/libunwind-arch.h" +#include <dwarf-regs.h> +#include <elf.h> +#include <fcntl.h> +#include <gelf.h> +#include <inttypes.h> + +#define DW_EH_PE_FORMAT_MASK 0x0f /* format of the encoded value */ +#define DW_EH_PE_APPL_MASK 0x70 /* how the value is to be applied */ -struct unwind_libunwind_ops __weak *local_unwind_libunwind_ops; -struct unwind_libunwind_ops __weak *x86_32_unwind_libunwind_ops; -struct unwind_libunwind_ops __weak *arm64_unwind_libunwind_ops; +/* Pointer-encoding formats: */ +#define DW_EH_PE_omit 0xff +#define DW_EH_PE_ptr 0x00 /* pointer-sized unsigned value */ +#define DW_EH_PE_udata4 0x03 /* unsigned 32-bit value */ +#define DW_EH_PE_udata8 0x04 /* unsigned 64-bit value */ +#define DW_EH_PE_sdata4 0x0b /* signed 32-bit value */ +#define DW_EH_PE_sdata8 0x0c /* signed 64-bit value */ -int unwind__prepare_access(struct maps *maps, struct map *map, bool *initialized) +/* Pointer-encoding application: */ +#define DW_EH_PE_absptr 0x00 /* absolute value */ +#define DW_EH_PE_pcrel 0x10 /* rel. to addr. of encoded value */ + +/* + * The following are not documented by LSB v1.3, yet they are used by + * GCC, presumably they aren't documented by LSB since they aren't + * used on Linux: + */ +#define DW_EH_PE_funcrel 0x40 /* start-of-procedure-relative */ +#define DW_EH_PE_aligned 0x50 /* aligned pointer */ + +/* Flags intentionally not handled, since they're not needed: + * #define DW_EH_PE_indirect 0x80 + * #define DW_EH_PE_uleb128 0x01 + * #define DW_EH_PE_udata2 0x02 + * #define DW_EH_PE_sleb128 0x09 + * #define DW_EH_PE_sdata2 0x0a + * #define DW_EH_PE_textrel 0x20 + * #define DW_EH_PE_datarel 0x30 + */ + +#define dw_read(ptr, type, end) ({ \ + type *__p = (type *) ptr; \ + type __v; \ + if ((__p + 1) > (type *) end) \ + return -EINVAL; \ + __v = *__p++; \ + ptr = (typeof(ptr)) __p; \ + __v; \ + }) + +static int __dw_read_encoded_value(u8 **p, u8 *end, u64 *val, + u8 encoding) { - const char *arch; - enum dso_type dso_type; - struct unwind_libunwind_ops *ops = local_unwind_libunwind_ops; - struct dso *dso = map__dso(map); - struct machine *machine; - int err; + u8 *cur = *p; + *val = 0; - if (!dwarf_callchain_users) + switch (encoding) { + case DW_EH_PE_omit: + *val = 0; + goto out; + case DW_EH_PE_ptr: + *val = dw_read(cur, unsigned long, end); + goto out; + default: + break; + } + + switch (encoding & DW_EH_PE_APPL_MASK) { + case DW_EH_PE_absptr: + break; + case DW_EH_PE_pcrel: + *val = (unsigned long) cur; + break; + default: + return -EINVAL; + } + + if ((encoding & 0x07) == 0x00) + encoding |= DW_EH_PE_udata4; + + switch (encoding & DW_EH_PE_FORMAT_MASK) { + case DW_EH_PE_sdata4: + *val += dw_read(cur, s32, end); + break; + case DW_EH_PE_udata4: + *val += dw_read(cur, u32, end); + break; + case DW_EH_PE_sdata8: + *val += dw_read(cur, s64, end); + break; + case DW_EH_PE_udata8: + *val += dw_read(cur, u64, end); + break; + default: + return -EINVAL; + } + + out: + *p = cur; + return 0; +} + +#define dw_read_encoded_value(ptr, end, enc) ({ \ + u64 __v; \ + if (__dw_read_encoded_value(&ptr, end, &__v, enc)) { \ + return -EINVAL; \ + } \ + __v; \ + }) + +static u64 elf_base_address(int fd) +{ + Elf *elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); + GElf_Phdr phdr; + u64 retval = 0; + size_t i, phdrnum = 0; + + if (elf == NULL) return 0; + (void)elf_getphdrnum(elf, &phdrnum); + /* PT_LOAD segments are sorted by p_vaddr, so the first has the minimum p_vaddr. */ + for (i = 0; i < phdrnum; i++) { + if (gelf_getphdr(elf, i, &phdr) && phdr.p_type == PT_LOAD) { + retval = phdr.p_vaddr & -getpagesize(); + break; + } + } - if (maps__addr_space(maps)) { - pr_debug("unwind: thread map already set, dso=%s\n", dso__name(dso)); - if (initialized) - *initialized = true; + elf_end(elf); + return retval; +} + +static int unwind_spec_ehframe(struct dso *dso, struct machine *machine, + u64 offset, u64 *table_data_offset, u64 *fde_count) +{ + struct eh_frame_hdr { + unsigned char version; + unsigned char eh_frame_ptr_enc; + unsigned char fde_count_enc; + unsigned char table_enc; + + /* + * The rest of the header is variable-length and consists of the + * following members: + * + * encoded_t eh_frame_ptr; + * encoded_t fde_count; + */ + + /* A single encoded pointer should not be more than 8 bytes. */ + u64 enc[2]; + + /* + * struct { + * encoded_t start_ip; + * encoded_t fde_addr; + * } binary_search_table[fde_count]; + */ + char data[]; + } __packed hdr; + u8 *enc = (u8 *) &hdr.enc; + u8 *end = (u8 *) &hdr.data; + ssize_t r; + + r = dso__data_read_offset(dso, machine, offset, (u8 *) &hdr, sizeof(hdr)); + if (r != sizeof(hdr)) + return -EINVAL; + + /* We dont need eh_frame_ptr, just skip it. */ + dw_read_encoded_value(enc, end, hdr.eh_frame_ptr_enc); + + *fde_count = dw_read_encoded_value(enc, end, hdr.fde_count_enc); + *table_data_offset = enc - (u8 *) &hdr; + return 0; +} + +struct read_unwind_spec_eh_frame_maps_cb_args { + struct dso *dso; + u64 base_addr; +}; + +static int read_unwind_spec_eh_frame_maps_cb(struct map *map, void *data) +{ + + struct read_unwind_spec_eh_frame_maps_cb_args *args = data; + + if (map__dso(map) == args->dso && map__start(map) - map__pgoff(map) < args->base_addr) + args->base_addr = map__start(map) - map__pgoff(map); + + return 0; +} + +static int elf_section_address_and_offset(int fd, const char *name, u64 *address, u64 *offset) +{ + Elf *elf; + GElf_Ehdr ehdr; + GElf_Shdr shdr; + int ret = -1; + + elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); + if (elf == NULL) + return -1; + + if (gelf_getehdr(elf, &ehdr) == NULL) + goto out_err; + + if (!elf_section_by_name(elf, &ehdr, &shdr, name, NULL)) + goto out_err; + + *address = shdr.sh_addr; + *offset = shdr.sh_offset; + ret = 0; +out_err: + elf_end(elf); + return ret; +} + +static int read_unwind_spec_eh_frame(struct dso *dso, struct unwind_info *ui, + u64 *table_data, u64 *segbase, + u64 *fde_count) +{ + struct read_unwind_spec_eh_frame_maps_cb_args args = { + .dso = dso, + .base_addr = UINT64_MAX, + }; + int ret, fd; + + if (dso__data(dso)->eh_frame_hdr_offset == 0) { + if (!dso__data_get_fd(dso, ui->machine, &fd)) + return -EINVAL; + + /* Check the .eh_frame section for unwinding info */ + ret = elf_section_address_and_offset(fd, ".eh_frame_hdr", + &dso__data(dso)->eh_frame_hdr_addr, + &dso__data(dso)->eh_frame_hdr_offset); + dso__data(dso)->elf_base_addr = elf_base_address(fd); + dso__data_put_fd(dso); + if (ret || dso__data(dso)->eh_frame_hdr_offset == 0) + return -EINVAL; + } + + maps__for_each_map(thread__maps(ui->thread), read_unwind_spec_eh_frame_maps_cb, &args); + + args.base_addr -= dso__data(dso)->elf_base_addr; + /* Address of .eh_frame_hdr */ + *segbase = args.base_addr + dso__data(dso)->eh_frame_hdr_addr; + ret = unwind_spec_ehframe(dso, ui->machine, dso__data(dso)->eh_frame_hdr_offset, + table_data, fde_count); + if (ret) + return ret; + /* binary_search_table offset plus .eh_frame_hdr address */ + *table_data += *segbase; + return 0; +} + +static u64 elf_section_offset(int fd, const char *name) +{ + u64 address, offset = 0; + + if (elf_section_address_and_offset(fd, name, &address, &offset)) + return 0; + + return offset; +} + +static int read_unwind_spec_debug_frame(struct dso *dso, + struct machine *machine, u64 *offset) +{ + int fd; + u64 ofs = dso__data(dso)->debug_frame_offset; + + /* debug_frame can reside in: + * - dso + * - debug pointed by symsrc_filename + * - gnu_debuglink, which doesn't necessary + * has to be pointed by symsrc_filename + */ + if (ofs == 0) { + if (dso__data_get_fd(dso, machine, &fd)) { + ofs = elf_section_offset(fd, ".debug_frame"); + dso__data_put_fd(dso); + } + + if (ofs <= 0) { + fd = open(dso__symsrc_filename(dso), O_RDONLY); + if (fd >= 0) { + ofs = elf_section_offset(fd, ".debug_frame"); + close(fd); + } + } + + if (ofs <= 0) { + char *debuglink = malloc(PATH_MAX); + int ret = 0; + + if (debuglink == NULL) { + pr_err("unwind: Can't read unwind spec debug frame.\n"); + return -ENOMEM; + } + + ret = dso__read_binary_type_filename( + dso, DSO_BINARY_TYPE__DEBUGLINK, + machine->root_dir, debuglink, PATH_MAX); + if (!ret) { + fd = open(debuglink, O_RDONLY); + if (fd >= 0) { + ofs = elf_section_offset(fd, + ".debug_frame"); + close(fd); + } + } + if (ofs > 0) { + if (dso__symsrc_filename(dso) != NULL) { + pr_warning( + "%s: overwrite symsrc(%s,%s)\n", + __func__, + dso__symsrc_filename(dso), + debuglink); + dso__free_symsrc_filename(dso); + } + dso__set_symsrc_filename(dso, debuglink); + } else { + free(debuglink); + } + } + + dso__data(dso)->debug_frame_offset = ofs; + } + + *offset = ofs; + if (*offset) + return 0; + + return -EINVAL; +} + +static struct map *find_map(uint64_t ip, struct unwind_info *ui) +{ + struct addr_location al; + struct map *ret; + + addr_location__init(&al); + thread__find_map(ui->thread, PERF_RECORD_MISC_USER, ip, &al); + ret = map__get(al.map); + addr_location__exit(&al); + return ret; +} + +static int elf_is_exec(int fd, const char *name) +{ + Elf *elf; + GElf_Ehdr ehdr; + int retval = 0; + + elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); + if (elf == NULL) + return 0; + if (gelf_getehdr(elf, &ehdr) == NULL) + goto out; + + retval = (ehdr.e_type == ET_EXEC); + +out: + elf_end(elf); + pr_debug3("unwind: elf_is_exec(%s): %d\n", name, retval); + return retval; +} + +int __libunwind__find_proc_info(void *as, uint64_t ip, void *pi, int need_unwind_info, void *arg) +{ + struct unwind_info *ui = arg; + struct map *map; + struct dso *dso; + u64 table_data, segbase, fde_count; + int ret = -EINVAL; + + map = find_map(ip, ui); + if (!map) + return -EINVAL; + + dso = map__dso(map); + if (!dso) { + map__put(map); + return -EINVAL; + } + + pr_debug3("unwind: find_proc_info dso %s\n", dso__name(dso)); + + /* Check the .eh_frame section for unwinding info */ + if (!read_unwind_spec_eh_frame(dso, ui, &table_data, &segbase, &fde_count)) { + struct table_entry { + u32 start_ip_offset; + u32 fde_offset; + }; + struct libarch_unwind__dyn_info di = { + .start_ip = map__start(map), + .end_ip = map__end(map), + .segbase = segbase, + .table_data = table_data, + .table_len = fde_count * sizeof(struct table_entry) / ui->unw_word_t_size, + }; + + ret = libunwind_arch__dwarf_search_unwind_table(ui->e_machine, as, ip, &di, pi, + need_unwind_info, arg); + } + + /* Check the .debug_frame section for unwinding info */ + if (ret < 0 && !read_unwind_spec_debug_frame(dso, ui->machine, &segbase)) { + int fd; + u64 start = map__start(map); + u64 base = start; + const char *symfile; + struct libarch_unwind__dyn_info di = {}; + + if (dso__data_get_fd(dso, ui->machine, &fd)) { + if (elf_is_exec(fd, dso__name(dso))) + base = 0; + dso__data_put_fd(dso); + } + + symfile = dso__symsrc_filename(dso) ?: dso__name(dso); + + if (libunwind_arch__dwarf_find_debug_frame(ui->e_machine, /*found=*/0, &di, ip, + base, symfile, start, map__end(map))) { + ret = libunwind_arch__dwarf_search_unwind_table(ui->e_machine, as, ip, &di, pi, + need_unwind_info, arg); + } + } + map__put(map); + return ret; +} + +static int access_dso_mem(struct unwind_info *ui, uint64_t addr, void *data_word) +{ + struct map *map; + struct dso *dso; + ssize_t size; + + map = find_map(addr, ui); + if (!map) { + pr_debug("unwind: no map for %lx\n", (unsigned long)addr); + return -1; + } + + dso = map__dso(map); + + if (!dso) { + map__put(map); + return -1; + } + + size = dso__data_read_addr(dso, map, ui->machine, + addr, + (u8 *) data_word, + ui->unw_word_t_size); + map__put(map); + return !((size_t)size == ui->unw_word_t_size); +} + +int __libunwind__access_mem(void *as __maybe_unused, uint64_t addr, void *valp_word, + int __write, void *arg) +{ + struct unwind_info *ui = arg; + struct stack_dump *stack = &ui->sample->user_stack; + u64 start, end; + int offset; + int ret; + + /* Don't support write, probably not needed. */ + if (__write || !stack || !ui->sample->user_regs || !ui->sample->user_regs->regs) { + uint64_t zero = 0; + + memcpy(valp_word, &zero, ui->unw_word_t_size); return 0; } - machine = maps__machine(maps); - /* env->arch is NULL for live-mode (i.e. perf top) */ - if (!machine->env || !machine->env->arch) - goto out_register; + ret = perf_reg_value(&start, perf_sample__user_regs(ui->sample), + perf_arch_reg_sp(ui->e_machine)); + if (ret) + return ret; + + end = start + stack->size; + + /* Check overflow. */ + if (addr + ui->unw_word_t_size < addr) + return -EINVAL; + + if (addr < start || addr + ui->unw_word_t_size >= end) { + ret = access_dso_mem(ui, addr, valp_word); + if (ret) { + pr_debug3("unwind: access_mem %p not inside range" + " 0x%" PRIx64 "-0x%" PRIx64 "\n", + (void *) (uintptr_t) addr, start, end); + memset(valp_word, 0, ui->unw_word_t_size); + return ret; + } + return 0; + } + + offset = addr - start; + memcpy(valp_word, &stack->data[offset], ui->unw_word_t_size); + pr_debug3("unwind: access_mem addr %p val %lx, offset %d\n", + (void *) (uintptr_t) addr, *((unsigned long *)valp_word), offset); + return 0; +} + +int __libunwind__access_reg(void *as __maybe_unused, int regnum, void *valp_word, int __write, + void *arg) +{ + struct unwind_info *ui = arg; + int id, ret; + u64 val; + + /* Don't support write, I suspect we don't need it. */ + if (__write) { + pr_err("unwind: access_reg w %d\n", regnum); + return 0; + } - dso_type = dso__type(dso, machine); - if (dso_type == DSO__TYPE_UNKNOWN) + if (!ui->sample->user_regs || !ui->sample->user_regs->regs) { + memset(valp_word, 0, ui->unw_word_t_size); return 0; + } + + id = get_perf_regnum_for_unw_regnum(ui->e_machine, regnum); + if (id < 0) + return -EINVAL; + + ret = perf_reg_value(&val, perf_sample__user_regs(ui->sample), id); + if (ret) { + if (!ui->best_effort) + pr_err("unwind: can't read reg %d\n", regnum); + return ret; + } + + if (ui->unw_word_t_size == 8) + *(uint64_t *)valp_word = val; + else + *(uint32_t *)valp_word = (uint32_t)val; + pr_debug3("unwind: reg %d, val %lx\n", regnum, val); + return 0; +} + +int unwind__prepare_access(struct maps *maps, uint16_t e_machine) +{ + void *addr_space; - arch = perf_env__arch(machine->env); + if (!dwarf_callchain_users) + return 0; - if (!strcmp(arch, "x86")) { - if (dso_type != DSO__TYPE_64BIT) - ops = x86_32_unwind_libunwind_ops; - } else if (!strcmp(arch, "arm64") || !strcmp(arch, "arm")) { - if (dso_type == DSO__TYPE_64BIT) - ops = arm64_unwind_libunwind_ops; + if (maps__addr_space(maps)) { + pr_debug3("unwind: thread map already set\n"); + return 0; } - if (!ops) { - pr_warning_once("unwind: target platform=%s is not supported\n", arch); + if (e_machine == EM_NONE) return 0; + + maps__set_e_machine(maps, e_machine); + addr_space = libunwind_arch__create_addr_space(e_machine); + + maps__set_addr_space(maps, addr_space); + if (!addr_space) { + pr_err("unwind: Can't create unwind address space.\n"); + return -ENOMEM; } -out_register: - maps__set_unwind_libunwind_ops(maps, ops); - err = maps__unwind_libunwind_ops(maps)->prepare_access(maps); - if (initialized) - *initialized = err ? false : true; - return err; + return 0; } void unwind__flush_access(struct maps *maps) { - const struct unwind_libunwind_ops *ops = maps__unwind_libunwind_ops(maps); - - if (ops) - ops->flush_access(maps); + libunwind_arch__flush_access(maps); } void unwind__finish_access(struct maps *maps) { - const struct unwind_libunwind_ops *ops = maps__unwind_libunwind_ops(maps); + libunwind_arch__finish_access(maps); +} + +static int entry(uint64_t ip, struct thread *thread, unwind_entry_cb_t cb, void *arg) +{ + struct unwind_entry e; + struct addr_location al; + int ret; - if (ops) - ops->finish_access(maps); + addr_location__init(&al); + e.ms.sym = thread__find_symbol(thread, PERF_RECORD_MISC_USER, ip, &al); + e.ip = ip; + e.ms.map = al.map; + e.ms.thread = thread__get(al.thread); + + pr_debug("unwind: %s:ip = 0x%" PRIx64 " (0x%" PRIx64 ")\n", + al.sym ? al.sym->name : "''", + ip, + al.map ? map__map_ip(al.map, ip) : (u64) 0); + + ret = cb(&e, arg); + addr_location__exit(&al); + return ret; } -int unwind__get_entries(unwind_entry_cb_t cb, void *arg, +int libunwind__get_entries(unwind_entry_cb_t cb, void *arg, struct thread *thread, - struct perf_sample *data, int max_stack, + struct perf_sample *sample, int max_stack, bool best_effort) { - const struct unwind_libunwind_ops *ops = maps__unwind_libunwind_ops(thread__maps(thread)); + struct unwind_info *ui; + uint64_t first_ip; + int ret, i = 0, entries = 0; + uint16_t e_machine; - if (ops) - return ops->get_entries(cb, arg, thread, data, max_stack, best_effort); - return 0; + if (!sample->user_regs || !sample->user_regs->regs) + return 0; + + if (max_stack <= 0) + return 0; + + if (!thread) { + pr_warning_once("WARNING: thread is NULL"); + return 0; + } + + e_machine = thread__e_machine(thread, /*machine=*/NULL, /*e_flags=*/NULL); + ret = perf_reg_value(&first_ip, perf_sample__user_regs(sample), + perf_arch_reg_ip(e_machine)); + if (ret) + return 0; + + if (max_stack == 1) { + /* Special case for a single entry. */ + ret = entry(first_ip, thread, cb, arg); + return ret ? (ret == -ENOMEM ? -ENOMEM : 0) : 1; + } + + ui = libunwind_arch_unwind_info__new(thread, sample, max_stack, best_effort, e_machine, first_ip); + if (!ui) + return -ENOMEM; + + do { + ret = libunwind_arch__unwind_step(ui); + if (ret < 0) + goto out; + + } while (ret); + + /* + * Display what we got based on the order setup. + */ + for (i = 0; i < ui->cur_ip; i++) { + int j = callchain_param.order == ORDER_CALLEE ? i : ui->cur_ip - i - 1; + + if (ui->ips[j]) { + ret = entry(ui->ips[j], thread, cb, arg); + if (ret) + break; + entries++; + } + } +out: + libunwind_arch_unwind_info__delete(ui); + /* + * Unwinder return contract: + * > 0 : unwinding succeeded (stops fallback). + * 0 : unwinding failed without yielding frames. Ignore non-fatal errors + * (e.g. stepping failure) to allow fallback unwinder or kernel callchains. + * < 0 : fatal error (e.g. -ENOMEM). Aborts unwinding entirely. + */ + if (ret == -ENOMEM) + return -ENOMEM; + return (entries > 0 || ret == 0) ? entries : 0; } diff --git a/tools/perf/util/unwind.c b/tools/perf/util/unwind.c new file mode 100644 index 000000000000..4ed4b1d55c69 --- /dev/null +++ b/tools/perf/util/unwind.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "debug.h" +#include "symbol_conf.h" +#include "unwind.h" +#include <linux/string.h> +#include <string.h> +#include <stdlib.h> + +int unwind__get_entries(unwind_entry_cb_t cb __maybe_unused, void *arg __maybe_unused, + struct thread *thread __maybe_unused, + struct perf_sample *data __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused) +{ + int ret = 0; + +#if defined(HAVE_LIBDW_SUPPORT) || defined(HAVE_LIBUNWIND_SUPPORT) + if (symbol_conf.unwind_style[0] == UNWIND_STYLE_UNKNOWN) { + int i = 0; +#ifdef HAVE_LIBDW_SUPPORT + symbol_conf.unwind_style[i++] = UNWIND_STYLE_LIBDW; +#endif +#ifdef HAVE_LIBUNWIND_SUPPORT + symbol_conf.unwind_style[i++] = UNWIND_STYLE_LIBUNWIND; +#endif + } +#endif //defined(HAVE_LIBDW_SUPPORT) || defined(HAVE_LIBUNWIND_SUPPORT) + + for (size_t i = 0; i < ARRAY_SIZE(symbol_conf.unwind_style); i++) { + switch (symbol_conf.unwind_style[i]) { + case UNWIND_STYLE_LIBDW: + ret = libdw__get_entries(cb, arg, thread, data, max_stack, best_effort); + break; + case UNWIND_STYLE_LIBUNWIND: + ret = libunwind__get_entries(cb, arg, thread, data, max_stack, best_effort); + break; + case UNWIND_STYLE_UNKNOWN: + default: +#if !defined(HAVE_LIBDW_SUPPORT) && !defined(HAVE_LIBUNWIND_SUPPORT) + pr_warning_once( + "Error: dwarf unwinding not supported, build perf with libdw or libunwind.\n"); +#endif + ret = 0; + break; + } + if (ret > 0) { + ret = 0; + break; + } + if (ret < 0) + break; + } + return ret; +} + +int unwind__configure(const char *var, const char *value, void *cb __maybe_unused) +{ + static const char * const unwind_style_names[] = { + [UNWIND_STYLE_LIBDW] = "libdw", + [UNWIND_STYLE_LIBUNWIND] = "libunwind", + NULL + }; + char *s, *p, *saveptr; + size_t i = 0; + + if (strcmp(var, "unwind.style")) + return 0; + + if (!value) + return -1; + + s = strdup(value); + if (!s) + return -1; + + memset(symbol_conf.unwind_style, 0, sizeof(symbol_conf.unwind_style)); + + p = strtok_r(s, ",", &saveptr); + while (p && i < ARRAY_SIZE(symbol_conf.unwind_style)) { + bool found = false; + char *q = strim(p); + + for (size_t j = UNWIND_STYLE_LIBDW; j < MAX_UNWIND_STYLE; j++) { + if (!strcasecmp(q, unwind_style_names[j])) { + symbol_conf.unwind_style[i++] = j; + found = true; + break; + } + } + if (!found) + pr_warning("Unknown unwind style: %s\n", q); + p = strtok_r(NULL, ",", &saveptr); + } + + free(s); + return 0; +} + +int unwind__option(const struct option *opt __maybe_unused, + const char *arg, + int unset __maybe_unused) +{ + return unwind__configure("unwind.style", arg, NULL); +} diff --git a/tools/perf/util/unwind.h b/tools/perf/util/unwind.h index 9f7164c6d9aa..a2d4af694fde 100644 --- a/tools/perf/util/unwind.h +++ b/tools/perf/util/unwind.h @@ -2,11 +2,13 @@ #ifndef __UNWIND_H #define __UNWIND_H +#include <stdint.h> #include <linux/compiler.h> #include <linux/types.h> -#include "util/map_symbol.h" +#include "map_symbol.h" struct maps; +struct option; struct perf_sample; struct thread; @@ -17,16 +19,9 @@ struct unwind_entry { typedef int (*unwind_entry_cb_t)(struct unwind_entry *entry, void *arg); -struct unwind_libunwind_ops { - int (*prepare_access)(struct maps *maps); - void (*flush_access)(struct maps *maps); - void (*finish_access)(struct maps *maps); - int (*get_entries)(unwind_entry_cb_t cb, void *arg, - struct thread *thread, - struct perf_sample *data, int max_stack, bool best_effort); -}; +int unwind__configure(const char *var, const char *value, void *cb); +int unwind__option(const struct option *opt, const char *arg, int unset); -#ifdef HAVE_DWARF_UNWIND_SUPPORT /* * When best_effort is set, don't report errors and fail silently. This could * be expanded in the future to be more permissive about things other than @@ -36,47 +31,55 @@ int unwind__get_entries(unwind_entry_cb_t cb, void *arg, struct thread *thread, struct perf_sample *data, int max_stack, bool best_effort); -/* libunwind specific */ -#ifdef HAVE_LIBUNWIND_SUPPORT -#ifndef LIBUNWIND__ARCH_REG_ID -#define LIBUNWIND__ARCH_REG_ID(regnum) libunwind__arch_reg_id(regnum) -#endif -int LIBUNWIND__ARCH_REG_ID(int regnum); -int unwind__prepare_access(struct maps *maps, struct map *map, bool *initialized); -void unwind__flush_access(struct maps *maps); -void unwind__finish_access(struct maps *maps); +#ifdef HAVE_LIBDW_SUPPORT +int libdw__get_entries(unwind_entry_cb_t cb, void *arg, + struct thread *thread, + struct perf_sample *data, int max_stack, + bool best_effort); #else -static inline int unwind__prepare_access(struct maps *maps __maybe_unused, - struct map *map __maybe_unused, - bool *initialized __maybe_unused) +#include "debug.h" +static inline int libdw__get_entries(unwind_entry_cb_t cb __maybe_unused, void *arg __maybe_unused, + struct thread *thread __maybe_unused, + struct perf_sample *data __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused) { + pr_warning_once("Error: libdw dwarf unwinding not built into perf\n"); return 0; } - -static inline void unwind__flush_access(struct maps *maps __maybe_unused) {} -static inline void unwind__finish_access(struct maps *maps __maybe_unused) {} #endif + +#ifdef HAVE_LIBUNWIND_SUPPORT +/* libunwind specific */ +int libunwind__get_entries(unwind_entry_cb_t cb, void *arg, + struct thread *thread, + struct perf_sample *data, int max_stack, + bool best_effort); +int unwind__prepare_access(struct maps *maps, uint16_t e_machine); +void unwind__flush_access(struct maps *maps); +void unwind__finish_access(struct maps *maps); #else -static inline int -unwind__get_entries(unwind_entry_cb_t cb __maybe_unused, - void *arg __maybe_unused, - struct thread *thread __maybe_unused, - struct perf_sample *data __maybe_unused, - int max_stack __maybe_unused, - bool best_effort __maybe_unused) +#include "debug.h" +static inline int libunwind__get_entries(unwind_entry_cb_t cb __maybe_unused, + void *arg __maybe_unused, + struct thread *thread __maybe_unused, + struct perf_sample *data __maybe_unused, + int max_stack __maybe_unused, + bool best_effort __maybe_unused) { + pr_warning_once("Error: libunwind dwarf unwinding not built into perf\n"); return 0; } static inline int unwind__prepare_access(struct maps *maps __maybe_unused, - struct map *map __maybe_unused, - bool *initialized __maybe_unused) + uint16_t e_machine __maybe_unused) { return 0; } static inline void unwind__flush_access(struct maps *maps __maybe_unused) {} static inline void unwind__finish_access(struct maps *maps __maybe_unused) {} -#endif /* HAVE_DWARF_UNWIND_SUPPORT */ +#endif + #endif /* __UNWIND_H */ diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c index 25849434f0a4..2c2a5c449ffd 100644 --- a/tools/perf/util/util.c +++ b/tools/perf/util/util.c @@ -419,11 +419,21 @@ out: char *perf_exe(char *buf, int len) { - int n = readlink("/proc/self/exe", buf, len); + int n; + + if (len <= 0) + return buf; + + n = readlink("/proc/self/exe", buf, len - 1); if (n > 0) { buf[n] = 0; return buf; } + if (len < (int)sizeof("perf")) { + buf[0] = '\0'; + return buf; + } + return strcpy(buf, "perf"); } diff --git a/tools/perf/util/zstd.c b/tools/perf/util/zstd.c index 57027e0ac7b6..21a0eb58597c 100644 --- a/tools/perf/util/zstd.c +++ b/tools/perf/util/zstd.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <string.h> +#include <linux/perf_event.h> #include "util/compress.h" #include "util/debug.h" @@ -54,7 +55,13 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ while (input.pos < input.size) { record = dst; + /* process_header writes the event header into record */ + if (dst_size < sizeof(struct perf_event_header)) + goto reset; size = process_header(record, 0); + /* Output buffer full — cannot fit even the record header */ + if (size > dst_size) + goto reset; compressed += size; dst += size; dst_size -= size; @@ -65,10 +72,18 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ if (ZSTD_isError(ret)) { pr_err("failed to compress %ld bytes: %s\n", (long)src_size, ZSTD_getErrorName(ret)); - memcpy(dst, src, src_size); - return src_size; + goto reset; } size = output.pos; + /* + * No progress: ZSTD couldn't emit any bytes into the + * remaining output buffer. Calling process_header + * with size=0 would re-trigger header initialization, + * double-subtracting the header size from dst_size and + * underflowing the unsigned counter. + */ + if (size == 0) + goto reset; size = process_header(record, size); compressed += size; dst += size; @@ -76,6 +91,14 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ } return compressed; + +reset: + /* Reset so the context is usable if the caller retries */ + ret = ZSTD_initCStream(data->cstream, data->comp_level); + if (ZSTD_isError(ret)) + pr_err("failed to reset compression context: %s\n", + ZSTD_getErrorName(ret)); + return -1; } size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size, @@ -100,14 +123,26 @@ size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size } } while (input.pos < input.size) { + size_t prev_in = input.pos; + size_t prev_out = output.pos; + ret = ZSTD_decompressStream(data->dstream, &output, &input); if (ZSTD_isError(ret)) { pr_err("failed to decompress (B): %zd -> %zd, dst_size %zd : %s\n", - src_size, output.size, dst_size, ZSTD_getErrorName(ret)); - break; + src_size, output.pos, dst_size, ZSTD_getErrorName(ret)); + return 0; } - output.dst = dst + output.pos; - output.size = dst_size - output.pos; + /* + * Neither stream advanced — decompression is stuck. + * Return 0 (error) rather than partial output: perf + * uses ZSTD_flushStream (not ZSTD_endStream), so the + * stream is continuous across compressed events. + * Discarding unconsumed input would desynchronize the + * decompressor, causing the next call to produce + * garbage that could be misinterpreted as valid events. + */ + if (input.pos == prev_in && output.pos == prev_out) + return 0; } return output.pos; |