From b6c00fb9949fbd073e651a77aa75faca978cf2a6 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Wed, 4 Jan 2023 12:13:41 -0800 Subject: perf: Add PMU_FORMAT_ATTR_SHOW The macro PMU_FORMAT_ATTR facilitates the definition of both the "show" function and "format_attr". But it only works for a non-hybrid platform. For a hybrid platform, the name "format_attr_hybrid_" is used. The definition of the "show" function can be shared between a non-hybrid platform and a hybrid platform. Add a new macro PMU_FORMAT_ATTR_SHOW. No functional change. The PMU_FORMAT_ATTR_SHOW will be used in the following patch. Signed-off-by: Kan Liang Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230104201349.1451191-1-kan.liang@linux.intel.com --- include/linux/perf_event.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index c6a3bac76966..ad92ad37600e 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1724,7 +1724,7 @@ static struct perf_pmu_events_attr _var = { \ .id = _id, } \ })[0].attr.attr) -#define PMU_FORMAT_ATTR(_name, _format) \ +#define PMU_FORMAT_ATTR_SHOW(_name, _format) \ static ssize_t \ _name##_show(struct device *dev, \ struct device_attribute *attr, \ @@ -1733,6 +1733,9 @@ _name##_show(struct device *dev, \ BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ return sprintf(page, _format "\n"); \ } \ + +#define PMU_FORMAT_ATTR(_name, _format) \ + PMU_FORMAT_ATTR_SHOW(_name, _format) \ \ static struct device_attribute format_attr_##_name = __ATTR_RO(_name) -- cgit v1.2.3 From 7bdb1767bf011c7f6065ac483ad2f00e434c3979 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Thu, 29 Dec 2022 12:40:59 -0800 Subject: perf/core: Change the layout of perf_sample_data The layout of perf_sample_data is designed to minimize cache-line access. The perf_sample_data_init() used to initialize a couple of fields unconditionally so they were placed together at the head. But it's changed now to set the fields according to the actual sample_type flags. The main user (the perf tools) sets the IP, TID, TIME, PERIOD always. Also group relevant fields like addr, phys_addr and data_page_size. Suggested-by: Peter Zijlstra Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20221229204101.1099430-1-namhyung@kernel.org --- include/linux/perf_event.h | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index ad92ad37600e..03949d017ac9 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1098,47 +1098,51 @@ extern u64 perf_event_read_value(struct perf_event *event, struct perf_sample_data { /* - * Fields set by perf_sample_data_init(), group so as to - * minimize the cachelines touched. + * Fields set by perf_sample_data_init() unconditionally, + * group so as to minimize the cachelines touched. */ u64 sample_flags; u64 period; /* - * The other fields, optionally {set,used} by - * perf_{prepare,output}_sample(). + * Fields commonly set by __perf_event_header__init_id(), + * group so as to minimize the cachelines touched. */ - struct perf_branch_stack *br_stack; - union perf_sample_weight weight; - union perf_mem_data_src data_src; - u64 txn; - u64 addr; - struct perf_raw_record *raw; - u64 type; - u64 ip; struct { u32 pid; u32 tid; } tid_entry; u64 time; u64 id; - u64 stream_id; struct { u32 cpu; u32 reserved; } cpu_entry; + + /* + * The other fields, optionally {set,used} by + * perf_{prepare,output}_sample(). + */ + u64 ip; struct perf_callchain_entry *callchain; - u64 aux_size; + struct perf_raw_record *raw; + struct perf_branch_stack *br_stack; + union perf_sample_weight weight; + union perf_mem_data_src data_src; + u64 txn; struct perf_regs regs_user; struct perf_regs regs_intr; u64 stack_user_size; - u64 phys_addr; + u64 stream_id; u64 cgroup; + u64 addr; + u64 phys_addr; u64 data_page_size; u64 code_page_size; + u64 aux_size; } ____cacheline_aligned; /* default value for data source */ -- cgit v1.2.3 From 4cf7a136115e96241f9f1089d2b53c47accf3823 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:52 -0800 Subject: perf/core: Save the dynamic parts of sample data size The perf sample data can be divided into parts. The event->header_size and event->id_header_size keep the static part of the sample data which is determined by the sample_type flags. But other parts like CALLCHAIN and BRANCH_STACK are changing dynamically so it needs to see the actual data. In preparation of handling repeated calls for perf_prepare_sample(), it can save the dynamic size to the perf sample data to avoid the duplicate work. Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Song Liu Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-2-namhyung@kernel.org --- include/linux/perf_event.h | 2 ++ kernel/events/core.c | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 03949d017ac9..16b980014449 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1103,6 +1103,7 @@ struct perf_sample_data { */ u64 sample_flags; u64 period; + u64 dyn_size; /* * Fields commonly set by __perf_event_header__init_id(), @@ -1158,6 +1159,7 @@ static inline void perf_sample_data_init(struct perf_sample_data *data, /* remaining struct members initialized in perf_prepare_sample() */ data->sample_flags = PERF_SAMPLE_PERIOD; data->period = period; + data->dyn_size = 0; if (addr) { data->addr = addr; diff --git a/kernel/events/core.c b/kernel/events/core.c index d56328e5080e..827082d1e7f9 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7586,7 +7586,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += data->callchain->nr; - header->size += size * sizeof(u64); + data->dyn_size += size * sizeof(u64); } if (sample_type & PERF_SAMPLE_RAW) { @@ -7612,7 +7612,7 @@ void perf_prepare_sample(struct perf_event_header *header, data->raw = NULL; } - header->size += size; + data->dyn_size += size; } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { @@ -7624,7 +7624,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += data->br_stack->nr * sizeof(struct perf_branch_entry); } - header->size += size; + data->dyn_size += size; } if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) @@ -7639,7 +7639,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += hweight64(mask) * sizeof(u64); } - header->size += size; + data->dyn_size += size; } if (sample_type & PERF_SAMPLE_STACK_USER) { @@ -7664,7 +7664,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += sizeof(u64) + stack_size; data->stack_user_size = stack_size; - header->size += size; + data->dyn_size += size; } if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) @@ -7693,7 +7693,7 @@ void perf_prepare_sample(struct perf_event_header *header, size += hweight64(mask) * sizeof(u64); } - header->size += size; + data->dyn_size += size; } if (sample_type & PERF_SAMPLE_PHYS_ADDR && @@ -7738,8 +7738,11 @@ void perf_prepare_sample(struct perf_event_header *header, size = perf_prepare_sample_aux(event, data, size); WARN_ON_ONCE(size + header->size > U16_MAX); - header->size += size; + data->dyn_size += size + sizeof(u64); /* size above */ } + + header->size += data->dyn_size; + /* * If you're adding more sample types here, you likely need to do * something about the overflowing header::size, like repurpose the -- cgit v1.2.3 From 31046500c1864b8ab25d1b9846ad10aa3f7b1821 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:53 -0800 Subject: perf/core: Add perf_sample_save_callchain() helper When we save the callchain to the perf sample data, we need to update the sample flags and the dynamic size. To ensure this is done consistently, add the perf_sample_save_callchain() helper and convert all call sites. Suggested-by: Peter Zijlstra Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Song Liu Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-3-namhyung@kernel.org --- arch/x86/events/amd/ibs.c | 6 ++---- arch/x86/events/intel/ds.c | 12 ++++-------- include/linux/perf_event.h | 16 +++++++++++++++- kernel/events/core.c | 12 ++---------- 4 files changed, 23 insertions(+), 23 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index da3f5ebac4e1..417c80bd3274 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -1122,10 +1122,8 @@ fail: * recorded as part of interrupt regs. Thus we need to use rip from * interrupt regs while unwinding call stack. */ - if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { - data.callchain = perf_callchain(event, iregs); - data.sample_flags |= PERF_SAMPLE_CALLCHAIN; - } + if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) + perf_sample_save_callchain(&data, event, iregs); throttle = perf_event_overflow(event, &data, ®s); out: diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 6ec326b47e2e..158cf845fc80 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1617,10 +1617,8 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, * previous PMI context or an (I)RET happened between the record and * PMI. */ - if (sample_type & PERF_SAMPLE_CALLCHAIN) { - data->callchain = perf_callchain(event, iregs); - data->sample_flags |= PERF_SAMPLE_CALLCHAIN; - } + if (sample_type & PERF_SAMPLE_CALLCHAIN) + perf_sample_save_callchain(data, event, iregs); /* * We use the interrupt regs as a base because the PEBS record does not @@ -1795,10 +1793,8 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, * previous PMI context or an (I)RET happened between the record and * PMI. */ - if (sample_type & PERF_SAMPLE_CALLCHAIN) { - data->callchain = perf_callchain(event, iregs); - data->sample_flags |= PERF_SAMPLE_CALLCHAIN; - } + if (sample_type & PERF_SAMPLE_CALLCHAIN) + perf_sample_save_callchain(data, event, iregs); *regs = *iregs; /* The ip in basic is EventingIP */ diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 16b980014449..a9419608402b 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1095,6 +1095,8 @@ int perf_event_read_local(struct perf_event *event, u64 *value, extern u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running); +extern struct perf_callchain_entry *perf_callchain(struct perf_event *event, struct pt_regs *regs); + struct perf_sample_data { /* @@ -1167,6 +1169,19 @@ static inline void perf_sample_data_init(struct perf_sample_data *data, } } +static inline void perf_sample_save_callchain(struct perf_sample_data *data, + struct perf_event *event, + struct pt_regs *regs) +{ + int size = 1; + + data->callchain = perf_callchain(event, regs); + size += data->callchain->nr; + + data->dyn_size += size * sizeof(u64); + data->sample_flags |= PERF_SAMPLE_CALLCHAIN; +} + /* * Clear all bitfields in the perf_branch_entry. * The to and from fields are not cleared because they are @@ -1408,7 +1423,6 @@ extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct extern struct perf_callchain_entry * get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, u32 max_stack, bool crosstask, bool add_mark); -extern struct perf_callchain_entry *perf_callchain(struct perf_event *event, struct pt_regs *regs); extern int get_callchain_buffers(int max_stack); extern void put_callchain_buffers(void); extern struct perf_callchain_entry *get_callchain_entry(int *rctx); diff --git a/kernel/events/core.c b/kernel/events/core.c index 827082d1e7f9..12b7d5114b06 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7578,16 +7578,8 @@ void perf_prepare_sample(struct perf_event_header *header, if (sample_type & (PERF_SAMPLE_IP | PERF_SAMPLE_CODE_PAGE_SIZE)) data->ip = perf_instruction_pointer(regs); - if (sample_type & PERF_SAMPLE_CALLCHAIN) { - int size = 1; - - if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) - data->callchain = perf_callchain(event, regs); - - size += data->callchain->nr; - - data->dyn_size += size * sizeof(u64); - } + if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) + perf_sample_save_callchain(data, event, regs); if (sample_type & PERF_SAMPLE_RAW) { struct perf_raw_record *raw = data->raw; -- cgit v1.2.3 From 0a9081cf0a11770f6b0affd377db8caa3ec4c793 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:54 -0800 Subject: perf/core: Add perf_sample_save_raw_data() helper When we save the raw_data to the perf sample data, we need to update the sample flags and the dynamic size. To make sure this is done consistently, add the perf_sample_save_raw_data() helper and convert all call sites. Suggested-by: Peter Zijlstra Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-4-namhyung@kernel.org --- arch/s390/kernel/perf_cpum_cf.c | 4 +--- arch/s390/kernel/perf_pai_crypto.c | 4 +--- arch/s390/kernel/perf_pai_ext.c | 4 +--- arch/x86/events/amd/ibs.c | 3 +-- include/linux/perf_event.h | 33 ++++++++++++++++++++++++++++----- kernel/events/core.c | 31 +++++-------------------------- kernel/trace/bpf_trace.c | 6 ++---- 7 files changed, 39 insertions(+), 46 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index f043a7ff220b..aa38649c7c27 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -662,9 +662,7 @@ static int cfdiag_push_sample(struct perf_event *event, if (event->attr.sample_type & PERF_SAMPLE_RAW) { raw.frag.size = cpuhw->usedss; raw.frag.data = cpuhw->stop; - raw.size = raw.frag.size; - data.raw = &raw; - data.sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(&data, &raw); } overflow = perf_event_overflow(event, &data, ®s); diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index 985e243a2ed8..a7b339c4fd7c 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -362,9 +362,7 @@ static int paicrypt_push_sample(void) if (event->attr.sample_type & PERF_SAMPLE_RAW) { raw.frag.size = rawsize; raw.frag.data = cpump->save; - raw.size = raw.frag.size; - data.raw = &raw; - data.sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(&data, &raw); } overflow = perf_event_overflow(event, &data, ®s); diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index 1138f57baae3..555597222bad 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -451,9 +451,7 @@ static int paiext_push_sample(void) if (event->attr.sample_type & PERF_SAMPLE_RAW) { raw.frag.size = rawsize; raw.frag.data = cpump->save; - raw.size = raw.frag.size; - data.raw = &raw; - data.sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(&data, &raw); } overflow = perf_event_overflow(event, &data, ®s); diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 417c80bd3274..64582954b5f6 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -1110,8 +1110,7 @@ fail: .data = ibs_data.data, }, }; - data.raw = &raw; - data.sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(&data, &raw); } if (perf_ibs == &perf_ibs_op) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index a9419608402b..569dfac5887f 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -95,6 +95,11 @@ struct perf_raw_record { u32 size; }; +static __always_inline bool perf_raw_frag_last(const struct perf_raw_frag *frag) +{ + return frag->pad < sizeof(u64); +} + /* * branch stack layout: * nr: number of taken branches stored in entries[] @@ -1182,6 +1187,29 @@ static inline void perf_sample_save_callchain(struct perf_sample_data *data, data->sample_flags |= PERF_SAMPLE_CALLCHAIN; } +static inline void perf_sample_save_raw_data(struct perf_sample_data *data, + struct perf_raw_record *raw) +{ + struct perf_raw_frag *frag = &raw->frag; + u32 sum = 0; + int size; + + do { + sum += frag->size; + if (perf_raw_frag_last(frag)) + break; + frag = frag->next; + } while (1); + + size = round_up(sum + sizeof(u32), sizeof(u64)); + raw->size = size - sizeof(u32); + frag->pad = raw->size - sum; + + data->raw = raw; + data->dyn_size += size; + data->sample_flags |= PERF_SAMPLE_RAW; +} + /* * Clear all bitfields in the perf_branch_entry. * The to and from fields are not cleared because they are @@ -1690,11 +1718,6 @@ extern void perf_restore_debug_store(void); static inline void perf_restore_debug_store(void) { } #endif -static __always_inline bool perf_raw_frag_last(const struct perf_raw_frag *frag) -{ - return frag->pad < sizeof(u64); -} - #define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x)) struct perf_pmu_events_attr { diff --git a/kernel/events/core.c b/kernel/events/core.c index 12b7d5114b06..17108a23b3dd 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7581,30 +7581,10 @@ void perf_prepare_sample(struct perf_event_header *header, if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) perf_sample_save_callchain(data, event, regs); - if (sample_type & PERF_SAMPLE_RAW) { - struct perf_raw_record *raw = data->raw; - int size; - - if (raw && (data->sample_flags & PERF_SAMPLE_RAW)) { - struct perf_raw_frag *frag = &raw->frag; - u32 sum = 0; - - do { - sum += frag->size; - if (perf_raw_frag_last(frag)) - break; - frag = frag->next; - } while (1); - - size = round_up(sum + sizeof(u32), sizeof(u64)); - raw->size = size - sizeof(u32); - frag->pad = raw->size - sum; - } else { - size = sizeof(u64); - data->raw = NULL; - } - - data->dyn_size += size; + if (filtered_sample_type & PERF_SAMPLE_RAW) { + data->raw = NULL; + data->dyn_size += sizeof(u64); + data->sample_flags |= PERF_SAMPLE_RAW; } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { @@ -10120,8 +10100,7 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, }; perf_sample_data_init(&data, 0, 0); - data.raw = &raw; - data.sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(&data, &raw); perf_trace_buf_update(record, event_type); diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 3bbd3f0c810c..ad37608afc35 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -687,8 +687,7 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, } perf_sample_data_init(sd, 0, 0); - sd->raw = &raw; - sd->sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(sd, &raw); err = __bpf_perf_event_output(regs, map, flags, sd); @@ -746,8 +745,7 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, perf_fetch_caller_regs(regs); perf_sample_data_init(sd, 0, 0); - sd->raw = &raw; - sd->sample_flags |= PERF_SAMPLE_RAW; + perf_sample_save_raw_data(sd, &raw); ret = __bpf_perf_event_output(regs, map, flags, sd); out: -- cgit v1.2.3 From eb55b455ef9c7123bdfa7e8a7f1ebeaa8034eb83 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:55 -0800 Subject: perf/core: Add perf_sample_save_brstack() helper When we saves the branch stack to the perf sample data, we needs to update the sample flags and the dynamic size. To make sure this is done consistently, add the perf_sample_save_brstack() helper and convert all call sites. Suggested-by: Peter Zijlstra Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Athira Rajeev Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-5-namhyung@kernel.org --- arch/powerpc/perf/core-book3s.c | 3 +- arch/x86/events/amd/core.c | 6 ++-- arch/x86/events/intel/core.c | 6 ++-- arch/x86/events/intel/ds.c | 9 ++---- include/linux/perf_event.h | 66 +++++++++++++++++++++++++---------------- kernel/events/core.c | 16 ++++------ 6 files changed, 53 insertions(+), 53 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index bf318dd9b709..8c1f7def596e 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -2313,8 +2313,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, struct cpu_hw_events *cpuhw; cpuhw = this_cpu_ptr(&cpu_hw_events); power_pmu_bhrb_read(event, cpuhw); - data.br_stack = &cpuhw->bhrb_stack; - data.sample_flags |= PERF_SAMPLE_BRANCH_STACK; + perf_sample_save_brstack(&data, event, &cpuhw->bhrb_stack); } if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC && diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 4386b10682ce..8c45b198b62f 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -928,10 +928,8 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) if (!x86_perf_event_set_period(event)) continue; - if (has_branch_stack(event)) { - data.br_stack = &cpuc->lbr_stack; - data.sample_flags |= PERF_SAMPLE_BRANCH_STACK; - } + if (has_branch_stack(event)) + perf_sample_save_brstack(&data, event, &cpuc->lbr_stack); if (perf_event_overflow(event, &data, regs)) x86_pmu_stop(event, 0); diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 29d2d0411caf..14f0a746257d 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3036,10 +3036,8 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) perf_sample_data_init(&data, 0, event->hw.last_period); - if (has_branch_stack(event)) { - data.br_stack = &cpuc->lbr_stack; - data.sample_flags |= PERF_SAMPLE_BRANCH_STACK; - } + if (has_branch_stack(event)) + perf_sample_save_brstack(&data, event, &cpuc->lbr_stack); if (perf_event_overflow(event, &data, regs)) x86_pmu_stop(event, 0); diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 158cf845fc80..07c8a2cdc3ee 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1720,10 +1720,8 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, data->sample_flags |= PERF_SAMPLE_TIME; } - if (has_branch_stack(event)) { - data->br_stack = &cpuc->lbr_stack; - data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; - } + if (has_branch_stack(event)) + perf_sample_save_brstack(data, event, &cpuc->lbr_stack); } static void adaptive_pebs_save_regs(struct pt_regs *regs, @@ -1883,8 +1881,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, if (has_branch_stack(event)) { intel_pmu_store_pebs_lbrs(lbr); - data->br_stack = &cpuc->lbr_stack; - data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; + perf_sample_save_brstack(data, event, &cpuc->lbr_stack); } } diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 569dfac5887f..7db0e9cc2682 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1102,6 +1102,31 @@ extern u64 perf_event_read_value(struct perf_event *event, extern struct perf_callchain_entry *perf_callchain(struct perf_event *event, struct pt_regs *regs); +static inline bool branch_sample_no_flags(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_FLAGS; +} + +static inline bool branch_sample_no_cycles(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_CYCLES; +} + +static inline bool branch_sample_type(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_TYPE_SAVE; +} + +static inline bool branch_sample_hw_index(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX; +} + +static inline bool branch_sample_priv(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE; +} + struct perf_sample_data { /* @@ -1210,6 +1235,21 @@ static inline void perf_sample_save_raw_data(struct perf_sample_data *data, data->sample_flags |= PERF_SAMPLE_RAW; } +static inline void perf_sample_save_brstack(struct perf_sample_data *data, + struct perf_event *event, + struct perf_branch_stack *brs) +{ + int size = sizeof(u64); /* nr */ + + if (branch_sample_hw_index(event)) + size += sizeof(u64); + size += brs->nr * sizeof(struct perf_branch_entry); + + data->br_stack = brs; + data->dyn_size += size; + data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; +} + /* * Clear all bitfields in the perf_branch_entry. * The to and from fields are not cleared because they are @@ -1827,30 +1867,4 @@ static inline void perf_lopwr_cb(bool mode) } #endif -#ifdef CONFIG_PERF_EVENTS -static inline bool branch_sample_no_flags(const struct perf_event *event) -{ - return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_FLAGS; -} - -static inline bool branch_sample_no_cycles(const struct perf_event *event) -{ - return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_CYCLES; -} - -static inline bool branch_sample_type(const struct perf_event *event) -{ - return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_TYPE_SAVE; -} - -static inline bool branch_sample_hw_index(const struct perf_event *event) -{ - return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX; -} - -static inline bool branch_sample_priv(const struct perf_event *event) -{ - return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE; -} -#endif /* CONFIG_PERF_EVENTS */ #endif /* _LINUX_PERF_EVENT_H */ diff --git a/kernel/events/core.c b/kernel/events/core.c index 17108a23b3dd..bd20705a23bd 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7310,7 +7310,7 @@ void perf_output_sample(struct perf_output_handle *handle, } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { - if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) { + if (data->br_stack) { size_t size; size = data->br_stack->nr @@ -7587,16 +7587,10 @@ void perf_prepare_sample(struct perf_event_header *header, data->sample_flags |= PERF_SAMPLE_RAW; } - if (sample_type & PERF_SAMPLE_BRANCH_STACK) { - int size = sizeof(u64); /* nr */ - if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) { - if (branch_sample_hw_index(event)) - size += sizeof(u64); - - size += data->br_stack->nr - * sizeof(struct perf_branch_entry); - } - data->dyn_size += size; + if (filtered_sample_type & PERF_SAMPLE_BRANCH_STACK) { + data->br_stack = NULL; + data->dyn_size += sizeof(u64); + data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; } if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) -- cgit v1.2.3 From f6e707156e1d5d150f288823987bee1ba0104c4c Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Tue, 17 Jan 2023 22:05:58 -0800 Subject: perf/core: Introduce perf_prepare_header() Factor out perf_prepare_header() so that it can call perf_prepare_sample() without a header if not needed. Also it checks the filtered_sample_type to avoid duplicate work when perf_prepare_sample() is called twice (or more). Suggested-by: Peter Zijlstr Signed-off-by: Namhyung Kim Signed-off-by: Ingo Molnar Tested-by: Jiri Olsa Acked-by: Jiri Olsa Acked-by: Song Liu Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20230118060559.615653-8-namhyung@kernel.org --- arch/s390/kernel/perf_cpum_sf.c | 3 ++- arch/x86/events/intel/ds.c | 3 ++- include/linux/perf_event.h | 16 +++++++++++++++- kernel/events/core.c | 38 ++++++++++++++++++++++++-------------- 4 files changed, 43 insertions(+), 17 deletions(-) (limited to 'include/linux') diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c index ce886a03545a..ead6eea48be8 100644 --- a/arch/s390/kernel/perf_cpum_sf.c +++ b/arch/s390/kernel/perf_cpum_sf.c @@ -672,7 +672,8 @@ static void cpumsf_output_event_pid(struct perf_event *event, /* Protect callchain buffers, tasks */ rcu_read_lock(); - perf_prepare_sample(&header, data, event, regs); + perf_prepare_sample(data, event, regs); + perf_prepare_header(&header, data, event, regs); if (perf_output_begin(&handle, data, event, header.size)) goto out; diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 07c8a2cdc3ee..183efa914b99 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -807,7 +807,8 @@ int intel_pmu_drain_bts_buffer(void) * the sample. */ rcu_read_lock(); - perf_prepare_sample(&header, &data, event, ®s); + perf_prepare_sample(&data, event, ®s); + perf_prepare_header(&header, &data, event, ®s); if (perf_output_begin(&handle, &data, event, header.size * (top - base - skip))) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 7db0e9cc2682..d5628a7b5eaa 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1250,6 +1250,17 @@ static inline void perf_sample_save_brstack(struct perf_sample_data *data, data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; } +static inline u32 perf_sample_data_size(struct perf_sample_data *data, + struct perf_event *event) +{ + u32 size = sizeof(struct perf_event_header); + + size += event->header_size + event->id_header_size; + size += data->dyn_size; + + return size; +} + /* * Clear all bitfields in the perf_branch_entry. * The to and from fields are not cleared because they are @@ -1271,7 +1282,10 @@ extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data, struct perf_event *event); -extern void perf_prepare_sample(struct perf_event_header *header, +extern void perf_prepare_sample(struct perf_sample_data *data, + struct perf_event *event, + struct pt_regs *regs); +extern void perf_prepare_header(struct perf_event_header *header, struct perf_sample_data *data, struct perf_event *event, struct pt_regs *regs); diff --git a/kernel/events/core.c b/kernel/events/core.c index 47bfd9985813..4aa73edbaf4c 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7568,20 +7568,13 @@ static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d) return d * !!(flags & s); } -void perf_prepare_sample(struct perf_event_header *header, - struct perf_sample_data *data, +void perf_prepare_sample(struct perf_sample_data *data, struct perf_event *event, struct pt_regs *regs) { u64 sample_type = event->attr.sample_type; u64 filtered_sample_type; - header->type = PERF_RECORD_SAMPLE; - header->size = sizeof(*header) + event->header_size + event->id_header_size; - - header->misc = 0; - header->misc |= perf_misc_flags(regs); - /* * Add the sample flags that are dependent to others. And clear the * sample flags that have already been done by the PMU driver. @@ -7595,6 +7588,12 @@ void perf_prepare_sample(struct perf_event_header *header, PERF_SAMPLE_REGS_USER); filtered_sample_type &= ~data->sample_flags; + if (filtered_sample_type == 0) { + /* Make sure it has the correct data->type for output */ + data->type = event->attr.sample_type; + return; + } + __perf_event_header__init_id(data, event, filtered_sample_type); if (filtered_sample_type & PERF_SAMPLE_IP) { @@ -7646,9 +7645,10 @@ void perf_prepare_sample(struct perf_event_header *header, * up the rest of the sample size. */ u16 stack_size = event->attr.sample_stack_user; + u16 header_size = perf_sample_data_size(data, event); u16 size = sizeof(u64); - stack_size = perf_sample_ustack_size(stack_size, header->size, + stack_size = perf_sample_ustack_size(stack_size, header_size, data->regs_user.regs); /* @@ -7733,8 +7733,9 @@ void perf_prepare_sample(struct perf_event_header *header, if (filtered_sample_type & PERF_SAMPLE_AUX) { u64 size; + u16 header_size = perf_sample_data_size(data, event); - header->size += sizeof(u64); /* size */ + header_size += sizeof(u64); /* size */ /* * Given the 16bit nature of header::size, an AUX sample can @@ -7742,17 +7743,25 @@ void perf_prepare_sample(struct perf_event_header *header, * Make sure this doesn't happen by using up to U16_MAX bytes * per sample in total (rounded down to 8 byte boundary). */ - size = min_t(size_t, U16_MAX - header->size, + size = min_t(size_t, U16_MAX - header_size, event->attr.aux_sample_size); size = rounddown(size, 8); size = perf_prepare_sample_aux(event, data, size); - WARN_ON_ONCE(size + header->size > U16_MAX); + WARN_ON_ONCE(size + header_size > U16_MAX); data->dyn_size += size + sizeof(u64); /* size above */ data->sample_flags |= PERF_SAMPLE_AUX; } +} - header->size += data->dyn_size; +void perf_prepare_header(struct perf_event_header *header, + struct perf_sample_data *data, + struct perf_event *event, + struct pt_regs *regs) +{ + header->type = PERF_RECORD_SAMPLE; + header->size = perf_sample_data_size(data, event); + header->misc = perf_misc_flags(regs); /* * If you're adding more sample types here, you likely need to do @@ -7781,7 +7790,8 @@ __perf_event_output(struct perf_event *event, /* protect the callchain buffers */ rcu_read_lock(); - perf_prepare_sample(&header, data, event, regs); + perf_prepare_sample(data, event, regs); + perf_prepare_header(&header, data, event, regs); err = output_begin(&handle, data, event, header.size); if (err) -- cgit v1.2.3