diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-20 13:00:41 -1000 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-10-20 13:00:41 -1000 |
| commit | 4cbdd58dfa7aaad679bf62f81cdbc96a6cb7c6ae (patch) | |
| tree | e18f6f4b0ee07a3e43d815b6ab3f29af04bf88fa | |
| parent | 5b6dca02b3e2254d2fd5706d76d7ab4610a67296 (diff) | |
| parent | f5a78ecccd98a04c47cea1cd63c2a2fdce8a4b48 (diff) | |
| download | linux-next-4cbdd58dfa7aaad679bf62f81cdbc96a6cb7c6ae.tar.gz linux-next-4cbdd58dfa7aaad679bf62f81cdbc96a6cb7c6ae.zip | |
Merge tag 'perf-core-2026-10-18' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull perf events updates from Ingo Molnar:
- Support SIMD/eGPRs/SSP register sampling in perf
(Dapeng Mi, Kan Liang)
- Turn amd_uncore_ctx events into a flexible array (Rosen Penev)
- Finish the work related to fixing the guest PEBS path to not
set unwanted PERF_GLOBAL_CTRL bits, for which urgent fixes
are upstream already (Sean Christopherson)
- Drop redundant counter slot searches in the AMD uncore
driver (Usama Arif)
- Don't WARN when the ext_regs_buf allocation fails (vineash)
- Annotate x86_pmu::hybrid_pmu with __counted_by_ptr (Thorsten Blum)
* tag 'perf-core-2026-10-18' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (34 commits)
perf/x86: Don't WARN when the ext_regs_buf allocation fails
perf/x86/amd/uncore: Free counter slot by index
perf/x86/amd/uncore: Remove redundant event slot scan
perf/x86/intel: Check only PMC bits in PEBS_ENABLED when detecting host PEBS usage
KVM: VMX: Only tell perf to enable PEBS counters for fully enabled PMCs
KVM: VMX: Drop a redundant pmu->global_ctrl check when processing pebs_enable
perf/x86/intel: KVM: Handle cross-mapped PEBS PMCs entirely within KVM
perf/x86: KVM: Have perf define a dedicated struct for getting guest PEBS data
perf/x86/intel: Invert names of intel_ctrl_{guest,host}_mask
perf/x86/intel: Annotate x86_pmu::hybrid_pmu with __counted_by_ptr
perf/x86/amd/uncore: Turn amd_uncore_ctx events into a flexible array
perf/x86/intel: Add sanity check for PEBS record/fragment size
perf/x86: Activate back-to-back NMI detection for arch-PEBS induced NMIs
perf/x86/intel: Advertise PERF_PMU_CAP_SIMD_REGS capability
perf/x86/intel: Support arch-PEBS based SIMD/eGPRs sampling
perf/x86: Support SSP sampling using sample_regs_* fields
perf/x86: Support eGPRs sampling using sample_regs_* fields
perf: Enhance perf_reg_validate() with simd_enabled argument
perf/x86: Support OPMASK sampling using sample_simd_pred_reg_* fields
perf/x86: Support ZMM sampling using sample_simd_vec_reg_* fields
...
32 files changed, 1576 insertions, 310 deletions
diff --git a/arch/arm/kernel/perf_regs.c b/arch/arm/kernel/perf_regs.c index 0529f90395c9..838d701adf4d 100644 --- a/arch/arm/kernel/perf_regs.c +++ b/arch/arm/kernel/perf_regs.c @@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_ARM_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; @@ -31,9 +31,3 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_32; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/arm64/kernel/perf_regs.c b/arch/arm64/kernel/perf_regs.c index b4eece3eb17d..71a3e0238de4 100644 --- a/arch/arm64/kernel/perf_regs.c +++ b/arch/arm64/kernel/perf_regs.c @@ -77,7 +77,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_ARM64_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { u64 reserved_mask = REG_RESERVED; @@ -98,9 +98,3 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_64; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/csky/kernel/perf_regs.c b/arch/csky/kernel/perf_regs.c index 09b7f88a2d6a..c932a96afc56 100644 --- a/arch/csky/kernel/perf_regs.c +++ b/arch/csky/kernel/perf_regs.c @@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_CSKY_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; @@ -31,9 +31,3 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_32; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/loongarch/kernel/perf_regs.c b/arch/loongarch/kernel/perf_regs.c index 263ac4ab5af6..164514f40ae0 100644 --- a/arch/loongarch/kernel/perf_regs.c +++ b/arch/loongarch/kernel/perf_regs.c @@ -25,7 +25,7 @@ u64 perf_reg_abi(struct task_struct *tsk) } #endif /* CONFIG_32BIT */ -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask) return -EINVAL; @@ -45,9 +45,3 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return regs->regs[idx]; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/mips/kernel/perf_regs.c b/arch/mips/kernel/perf_regs.c index e686780d1647..00a5201dbd5d 100644 --- a/arch/mips/kernel/perf_regs.c +++ b/arch/mips/kernel/perf_regs.c @@ -28,7 +28,7 @@ u64 perf_reg_abi(struct task_struct *tsk) } #endif /* CONFIG_32BIT */ -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask) return -EINVAL; @@ -60,9 +60,3 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return (s64)v; /* Sign extend if 32-bit. */ } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/parisc/kernel/perf_regs.c b/arch/parisc/kernel/perf_regs.c index 10a1a5f06a18..4f21aab5405c 100644 --- a/arch/parisc/kernel/perf_regs.c +++ b/arch/parisc/kernel/perf_regs.c @@ -34,7 +34,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_PARISC_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; @@ -53,9 +53,3 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_64; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/powerpc/perf/perf_regs.c b/arch/powerpc/perf/perf_regs.c index 350dccb0143c..a01d8a903640 100644 --- a/arch/powerpc/perf/perf_regs.c +++ b/arch/powerpc/perf/perf_regs.c @@ -125,7 +125,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return regs_get_register(regs, pt_regs_offset[idx]); } -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; diff --git a/arch/riscv/kernel/perf_regs.c b/arch/riscv/kernel/perf_regs.c index fd304a248de6..1ecc8760b88b 100644 --- a/arch/riscv/kernel/perf_regs.c +++ b/arch/riscv/kernel/perf_regs.c @@ -18,7 +18,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1ULL << PERF_REG_RISCV_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; @@ -35,9 +35,3 @@ u64 perf_reg_abi(struct task_struct *task) #endif } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} diff --git a/arch/s390/kernel/perf_regs.c b/arch/s390/kernel/perf_regs.c index 7b305f1456f8..6496fd23c540 100644 --- a/arch/s390/kernel/perf_regs.c +++ b/arch/s390/kernel/perf_regs.c @@ -34,7 +34,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) #define REG_RESERVED (~((1UL << PERF_REG_S390_MAX) - 1)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { if (!mask || mask & REG_RESERVED) return -EINVAL; diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 7181973b5b12..7aa5a5b652a2 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -39,11 +39,11 @@ static int pmu_version; struct amd_uncore_ctx { int refcnt; int cpu; - struct perf_event **events; unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)]; int nr_active; struct hrtimer hrtimer; u64 hrtimer_duration; + struct perf_event *events[]; }; struct amd_uncore_pmu { @@ -206,17 +206,15 @@ static int amd_uncore_add(struct perf_event *event, int flags) struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu); struct hw_perf_event *hwc = &event->hw; - /* are we already assigned? */ + /* + * Perf serializes ->add() and ->del() for an event. A successful + * ->add() records the claimed slot in hwc->idx before returning, and + * ->del() clears that slot before resetting hwc->idx. Therefore, an + * existing assignment must be at hwc->idx. + */ if (hwc->idx != -1 && ctx->events[hwc->idx] == event) goto out; - for (i = 0; i < pmu->num_counters; i++) { - if (ctx->events[i] == event) { - hwc->idx = i; - goto out; - } - } - /* if not, take the first available counter */ hwc->idx = -1; for (i = 0; i < pmu->num_counters; i++) { @@ -248,19 +246,15 @@ out: static void amd_uncore_del(struct perf_event *event, int flags) { - int i; struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu); struct hw_perf_event *hwc = &event->hw; + struct perf_event *old = event; event->pmu->stop(event, PERF_EF_UPDATE); - for (i = 0; i < pmu->num_counters; i++) { - struct perf_event *tmp = event; - - if (try_cmpxchg(&ctx->events[i], &tmp, NULL)) - break; - } + /* ->del() follows a successful ->add(), so hwc->idx owns this slot. */ + WARN_ON_ONCE(!try_cmpxchg(&ctx->events[hwc->idx], &old, NULL)); hwc->idx = -1; } @@ -519,10 +513,8 @@ static void amd_uncore_ctx_free(struct amd_uncore *uncore, unsigned int cpu) if (cpu == ctx->cpu) cpumask_clear_cpu(cpu, &pmu->active_mask); - if (!--ctx->refcnt) { - kfree(ctx->events); + if (!--ctx->refcnt) kfree(ctx); - } *per_cpu_ptr(pmu->ctx, cpu) = NULL; } @@ -567,18 +559,11 @@ static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu) /* Allocate context if sibling does not exist */ if (!curr) { node = cpu_to_node(cpu); - curr = kzalloc_node(sizeof(*curr), GFP_KERNEL, node); + curr = kzalloc_node(struct_size(curr, events, pmu->num_counters), GFP_KERNEL, node); if (!curr) goto fail; curr->cpu = cpu; - curr->events = kzalloc_node(sizeof(*curr->events) * - pmu->num_counters, - GFP_KERNEL, node); - if (!curr->events) { - kfree(curr); - goto fail; - } amd_uncore_init_hrtimer(curr); curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC; diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 8b3ea0adb965..4df792b139e0 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -93,7 +93,7 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling, *x86_pmu.stop_scheduling); DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task, *x86_pmu.sched_task); -DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs, *x86_pmu.drain_pebs); +DEFINE_STATIC_CALL_RET0(x86_pmu_drain_pebs, *x86_pmu.drain_pebs); DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases); DEFINE_STATIC_CALL_NULL(x86_pmu_filter, *x86_pmu.filter); @@ -408,6 +408,53 @@ set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) return x86_pmu_extra_regs(val, event); } +static DEFINE_PER_CPU(struct xregs_state *, ext_regs_buf); + +static void release_ext_regs_buffers(void) +{ + int cpu; + + if (!x86_pmu.ext_regs_mask) + return; + + for_each_possible_cpu(cpu) { + kfree(per_cpu(ext_regs_buf, cpu)); + per_cpu(ext_regs_buf, cpu) = NULL; + } +} + +static void reserve_ext_regs_buffers(void) +{ + bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); + unsigned int size; + int cpu; + + if (!x86_pmu.ext_regs_mask) + return; + + /* Add 64 bytes to satisfy the XSAVE area's 64-byte alignment. */ + size = xstate_calculate_size(x86_pmu.ext_regs_mask, compacted) + 64; + + for_each_possible_cpu(cpu) { + per_cpu(ext_regs_buf, cpu) = kzalloc_node(size, GFP_KERNEL, + cpu_to_node(cpu)); + if (!per_cpu(ext_regs_buf, cpu)) + goto err; + } + + return; + +err: + release_ext_regs_buffers(); +} + +static inline struct xregs_state *get_ext_regs_buf(int cpu) +{ + void *buf = per_cpu(ext_regs_buf, cpu); + + return buf ? PTR_ALIGN(buf, 64) : NULL; +} + int x86_reserve_hardware(void) { int err = 0; @@ -420,6 +467,7 @@ int x86_reserve_hardware(void) } else { reserve_ds_buffers(); reserve_lbr_buffers(); + reserve_ext_regs_buffers(); } } if (!err) @@ -436,6 +484,7 @@ void x86_release_hardware(void) release_pmc_hardware(); release_ds_buffers(); release_lbr_buffers(); + release_ext_regs_buffers(); mutex_unlock(&pmc_reserve_mutex); } } @@ -583,6 +632,79 @@ int x86_pmu_max_precise(struct pmu *pmu) return precise; } +static int pebs_simd_regs_validate(struct perf_event *event) +{ + u64 caps = hybrid(event->pmu, arch_pebs_cap).caps; + + if (event_needs_xmm(event) && + !x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline) + return -EINVAL; + if (event_needs_xmm(event) && + x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) + return -EINVAL; + + if (event_needs_ssp(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_GPR))) + return -EINVAL; + if (event_needs_ymm(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_YMMH))) + return -EINVAL; + if (event_needs_egprs(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_EGPRS))) + return -EINVAL; + if (event_needs_opmask(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_OPMASK))) + return -EINVAL; + if (event_needs_low16_zmm(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_ZMMH))) + return -EINVAL; + if (event_needs_high16_zmm(event) && + !(x86_pmu.arch_pebs && (caps & ARCH_PEBS_VECR_H16ZMM))) + return -EINVAL; + + return 0; +} + +static int event_simd_regs_validate(struct perf_event *event) +{ + u64 reserved = ~GENMASK_ULL(PERF_REG_MISC_MAX - 1, 0); + + if (!get_ext_regs_buf(raw_smp_processor_id())) + return -ENOMEM; + /* + * The XMM space in the perf_event_x86_regs is reclaimed + * for eGPRs and other general registers. + */ + if (((event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & reserved)) || + ((event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & reserved))) + return -EINVAL; + if (event_needs_egprs(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_APX)) + return -EINVAL; + if (event_needs_ssp(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_CET_USER)) + return -EINVAL; + if (event_needs_xmm(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE)) + return -EINVAL; + if (event_needs_ymm(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_YMM)) + return -EINVAL; + if (event_needs_low16_zmm(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_ZMM_Hi256)) + return -EINVAL; + if (event_needs_high16_zmm(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_Hi16_ZMM)) + return -EINVAL; + if (event_needs_opmask(event) && + !(x86_pmu.ext_regs_mask & XFEATURE_MASK_OPMASK)) + return -EINVAL; + + return 0; +} + int x86_pmu_hw_config(struct perf_event *event) { if (event->attr.precise_ip) { @@ -653,19 +775,38 @@ int x86_pmu_hw_config(struct perf_event *event) return -EINVAL; } - /* sample_regs_user never support XMM registers */ - if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) - return -EINVAL; - /* - * Besides the general purpose registers, XMM registers may - * be collected in PEBS on some platforms, e.g. Icelake - */ - if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) { - if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) - return -EINVAL; + if (event->attr.sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) { + int ret; - if (!event->attr.precise_ip) - return -EINVAL; + if (event->attr.sample_simd_regs_enabled) { + if (!(event->pmu->capabilities & PERF_PMU_CAP_SIMD_REGS)) + return -EINVAL; + + if (event->attr.precise_ip) { + ret = pebs_simd_regs_validate(event); + if (ret) + return ret; + } + ret = event_simd_regs_validate(event); + if (ret) + return ret; + } else if (event_has_extended_regs(event)) { + if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) + return -EINVAL; + + if (event->attr.precise_ip) { + u64 caps = hybrid(event->pmu, arch_pebs_cap).caps; + + if (x86_pmu.arch_pebs && !(caps & ARCH_PEBS_VECR_XMM)) + return -EINVAL; + if (!x86_pmu.arch_pebs && !x86_pmu.intel_cap.pebs_baseline) + return -EINVAL; + } + if (!get_ext_regs_buf(raw_smp_processor_id())) + return -ENOMEM; + if (!(x86_pmu.ext_regs_mask & XFEATURE_MASK_SSE)) + return -EINVAL; + } } return x86_setup_perfctr(event); @@ -721,9 +862,10 @@ void x86_pmu_disable_all(void) } } -struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data) +struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs) { - return static_call(x86_pmu_guest_get_msrs)(nr, data); + return static_call(x86_pmu_guest_get_msrs)(nr, guest_pebs); } EXPORT_SYMBOL_FOR_KVM(perf_guest_get_msrs); @@ -1717,6 +1859,302 @@ do_del: static_call_cond(x86_pmu_del)(event); } +void x86_pmu_clear_perf_regs(struct pt_regs *regs) +{ + struct x86_perf_regs *perf_regs = container_of(regs, struct x86_perf_regs, regs); + + perf_regs->abi = PERF_SAMPLE_REGS_ABI_NONE; + perf_regs->xmm_regs = NULL; + perf_regs->ymmh_regs = NULL; + perf_regs->zmmh_regs = NULL; + perf_regs->h16zmm_regs = NULL; + perf_regs->opmask_regs = NULL; + perf_regs->egpr_regs = NULL; + perf_regs->ssp = NULL; +} + +static void update_perf_regs(struct x86_perf_regs *perf_regs, + struct xregs_state *xsave, u64 bitmap) +{ + struct cet_user_state *cet; + u64 mask; + + if (!xsave) + return; + + /* Restrict to features actually saved by XSAVES */ + mask = bitmap & xsave->header.xfeatures; + + if (mask & XFEATURE_MASK_SSE) + perf_regs->xmm_space = xsave->i387.xmm_space; + if (mask & XFEATURE_MASK_YMM) + perf_regs->ymmh = get_xsave_addr(xsave, XFEATURE_YMM); + if (mask & XFEATURE_MASK_ZMM_Hi256) + perf_regs->zmmh = get_xsave_addr(xsave, XFEATURE_ZMM_Hi256); + if (mask & XFEATURE_MASK_Hi16_ZMM) + perf_regs->h16zmm = get_xsave_addr(xsave, XFEATURE_Hi16_ZMM); + if (mask & XFEATURE_MASK_OPMASK) + perf_regs->opmask = get_xsave_addr(xsave, XFEATURE_OPMASK); + if (mask & XFEATURE_MASK_APX) + perf_regs->egpr = get_xsave_addr(xsave, XFEATURE_APX); + if (mask & XFEATURE_MASK_CET_USER) { + cet = get_xsave_addr(xsave, XFEATURE_CET_USER); + perf_regs->ssp = cet ? &cet->user_ssp : NULL; + } +} + +/* + * The x86 specific variant of perf_sample_regs_intr(). + * Update data->regs_intr fields for extended registers (e.g., SIMD). + */ +static void x86_pmu_update_regs_intr(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs, + bool exclude_kernel) +{ + struct x86_perf_regs *perf_regs; + + if (exclude_kernel && !user_mode(regs)) { + data->regs_intr.regs = NULL; + data->regs_intr.abi = PERF_SAMPLE_REGS_ABI_NONE; + } else { + data->regs_intr.regs = regs; + data->regs_intr.abi = perf_reg_abi(current); + } + + data->dyn_size += sizeof(u64); + if (data->regs_intr.regs) { + data->dyn_size += hweight64(event->attr.sample_regs_intr) * + sizeof(u64); + if (event_has_simd_regs(event)) { + data->dyn_size += perf_update_xregs_size(event, true); + data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } + + perf_regs = container_of(data->regs_intr.regs, + struct x86_perf_regs, regs); + perf_regs->abi = data->regs_intr.abi; + } + + /* + * Set PERF_SAMPLE_REGS_INTR to bypass perf_sample_regs_intr() call + * in perf_prepare_sample() function. + */ + data->sample_flags |= PERF_SAMPLE_REGS_INTR; +} + +static DEFINE_PER_CPU(struct x86_perf_regs, x86_user_regs); + +static void x86_pmu_get_regs_user(struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs); + struct perf_regs regs_user; + + x86_pmu_clear_perf_regs(&x86_regs_user->regs); + + perf_get_regs_user(®s_user, regs); + data->regs_user.abi = regs_user.abi; + if (regs_user.regs) { + x86_regs_user->regs = *regs_user.regs; + data->regs_user.regs = &x86_regs_user->regs; + } else { + data->regs_user.regs = NULL; + } +} + +/* + * The x86 specific variant of perf_sample_regs_user(). + * Update data->regs_user fields for extended registers (e.g., SIMD). + */ +static void x86_pmu_update_regs_user(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct x86_perf_regs *x86_regs_user = this_cpu_ptr(&x86_user_regs); + struct perf_event_attr *attr = &event->attr; + struct x86_perf_regs *perf_regs; + + /* + * PERF_SAMPLE_REGS_INTR and PERF_SAMPLE_REGS_USER can both be + * requested for one event. Keep user regs in a separate x86_perf_regs + * instance, so intr-reg collection does not overwrite user-reg data. + */ + if (user_mode(regs)) { + x86_pmu_clear_perf_regs(&x86_regs_user->regs); + perf_regs = container_of(regs, struct x86_perf_regs, regs); + /* Copy all sampled regs data to x86_regs_user. */ + *x86_regs_user = *perf_regs; + data->regs_user.regs = &x86_regs_user->regs; + data->regs_user.abi = perf_reg_abi(current); + } else if (is_user_task(current)) { + x86_pmu_get_regs_user(data, regs); + } else { + data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE; + data->regs_user.regs = NULL; + } + + data->dyn_size += sizeof(u64); + if (data->regs_user.regs) { + data->dyn_size += hweight64(attr->sample_regs_user) * sizeof(u64); + if (event_has_simd_regs(event)) { + data->dyn_size += perf_update_xregs_size(event, false); + data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } + + x86_regs_user->abi = data->regs_user.abi; + } + + /* + * Set PERF_SAMPLE_REGS_USER to bypass perf_sample_regs_user() call + * in perf_prepare_sample() function. + */ + data->sample_flags |= PERF_SAMPLE_REGS_USER; +} + +/* + * This function retrieves cached user-space fpu registers (XMM/YMM/ZMM). + * If TIF_NEED_FPU_LOAD is set or PMI hits into guest, it indicates that + * the user-space FPU state is cached. Otherwise, the data should be read + * directly from the hardware registers. + */ +static inline u64 x86_pmu_update_user_xregs(struct perf_sample_data *data, + struct pt_regs *regs, + u64 mask, bool from_pebs) +{ + struct x86_perf_regs *perf_regs; + struct xregs_state *xsave; + struct fpu *fpu; + struct fpstate *fps; + u64 user_mask = mask; + + if (!is_user_task(current)) + return 0; + + if (data->regs_user.abi == PERF_SAMPLE_REGS_ABI_NONE) + return 0; + + /* + * If PEBS hits kernel space, need to re-sample extended + * registers for user space. + */ + if (user_mode(regs)) + user_mask = from_pebs ? 0 : mask; + + fpu = x86_task_fpu(current); + fps = READ_ONCE(fpu->__task_fpstate); + /* + * If fpu->__task_fpstate is set, it points to the cached user + * FPU state (e.g. with KVM guest-state swapping). Otherwise, + * when TIF_NEED_FPU_LOAD is set, fpu->fpstate holds the cached + * user state. If neither is true, the user state is live in hardware. + */ + if (user_mask && (test_thread_flag(TIF_NEED_FPU_LOAD) || fps)) { + perf_regs = container_of(data->regs_user.regs, + struct x86_perf_regs, regs); + if (!fps) + fps = fpu->fpstate; + xsave = &fps->regs.xsave; + + update_perf_regs(perf_regs, xsave, user_mask); + return 0; + } + + return user_mask; +} + +static u64 get_simd_sample_mask(struct perf_event *event, u64 sample_type) +{ + u64 mask = 0; + + if (__event_needs_xmm(event, sample_type)) + mask |= XFEATURE_MASK_SSE; + if (__event_needs_ymm(event, sample_type)) + mask |= XFEATURE_MASK_YMM; + if (__event_needs_low16_zmm(event, sample_type)) + mask |= XFEATURE_MASK_ZMM_Hi256; + if (__event_needs_high16_zmm(event, sample_type)) + mask |= XFEATURE_MASK_Hi16_ZMM; + if (__event_needs_opmask(event, sample_type)) + mask |= XFEATURE_MASK_OPMASK; + if (__event_needs_egprs(event, sample_type)) + mask |= XFEATURE_MASK_APX; + if (__event_needs_ssp(event, sample_type)) + mask |= XFEATURE_MASK_CET_USER; + + return mask; +} + +static void x86_pmu_sample_xregs(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs, + bool from_pebs) +{ + struct xregs_state *xsave = get_ext_regs_buf(smp_processor_id()); + u64 sample_type = event->attr.sample_type; + struct x86_perf_regs *perf_regs; + u64 intr_mask = 0; + u64 user_mask = 0; + + if (WARN_ON_ONCE(!xsave) || !in_nmi()) + return; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && data->regs_intr.regs) { + intr_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_INTR); + intr_mask &= x86_pmu.ext_regs_mask; + intr_mask = from_pebs ? 0 : intr_mask; + } + + if ((sample_type & PERF_SAMPLE_REGS_USER) && data->regs_user.regs) { + user_mask |= get_simd_sample_mask(event, PERF_SAMPLE_REGS_USER); + user_mask &= x86_pmu.ext_regs_mask; + user_mask = x86_pmu_update_user_xregs(data, regs, + user_mask, from_pebs); + } + + if (user_mask | intr_mask) { + xsave->header.xfeatures = 0; + xsaves_nmi(xsave, user_mask | intr_mask); + } + + if (intr_mask) { + perf_regs = container_of(data->regs_intr.regs, + struct x86_perf_regs, regs); + update_perf_regs(perf_regs, xsave, intr_mask); + } + + if (user_mask) { + perf_regs = container_of(data->regs_user.regs, + struct x86_perf_regs, regs); + update_perf_regs(perf_regs, xsave, user_mask); + } +} + +void x86_pmu_update_perf_regs(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs, + bool from_pebs) +{ + u64 sample_type = event->attr.sample_type; + + if (!(sample_type & + (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))) + return; + + if (!event_needs_xmm(event) && + !event_has_simd_regs(event)) + return; + + if (sample_type & PERF_SAMPLE_REGS_INTR) { + x86_pmu_update_regs_intr(event, data, regs, + event->attr.exclude_kernel); + } + if (sample_type & PERF_SAMPLE_REGS_USER) + x86_pmu_update_regs_user(event, data, regs); + + x86_pmu_sample_xregs(event, data, regs, from_pebs); +} + int x86_pmu_handle_irq(struct pt_regs *regs) { struct perf_sample_data data; @@ -1800,9 +2238,11 @@ void perf_put_guest_lvtpc(void) EXPORT_SYMBOL_FOR_KVM(perf_put_guest_lvtpc); #endif /* CONFIG_PERF_GUEST_MEDIATED_PMU */ +static DEFINE_PER_CPU(struct x86_perf_regs, x86_intr_regs); static int perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) { + struct x86_perf_regs *x86_regs = this_cpu_ptr(&x86_intr_regs); u64 start_clock; u64 finish_clock; int ret; @@ -1826,7 +2266,8 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) return NMI_DONE; start_clock = sched_clock(); - ret = static_call(x86_pmu_handle_irq)(regs); + x86_regs->regs = *regs; + ret = static_call(x86_pmu_handle_irq)(&x86_regs->regs); finish_clock = sched_clock(); perf_sample_event_took(finish_clock - start_clock); @@ -2218,8 +2659,20 @@ static int __init init_hw_perf_events(void) pmu.attr_update = x86_pmu.attr_update; - if (!is_hybrid()) + if (!is_hybrid()) { x86_pmu_show_pmu_cap(NULL); + } else { + int i; + + /* + * Init default ops. + * Must be called before registering x86_pmu_starting_cpu(), + * otherwise some key PMU fields, e.g., capabilities + * initialized in x86_pmu_starting_cpu(), would be overwritten. + */ + for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) + x86_pmu.hybrid_pmu[i].pmu = pmu; + } if (!x86_pmu.read) x86_pmu.read = _x86_pmu_read; @@ -2266,7 +2719,6 @@ static int __init init_hw_perf_events(void) for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { hybrid_pmu = &x86_pmu.hybrid_pmu[i]; - hybrid_pmu->pmu = pmu; hybrid_pmu->pmu.type = -1; hybrid_pmu->pmu.attr_update = x86_pmu.attr_update; hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 3ef80882843e..0a34d674df59 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -14,7 +14,6 @@ #include <linux/slab.h> #include <linux/export.h> #include <linux/nmi.h> -#include <linux/kvm_host.h> #include <asm/cpufeature.h> #include <asm/cpuid/api.h> @@ -2797,7 +2796,7 @@ static void __intel_pmu_enable_all(int added, bool pmi) } wrmsrq(MSR_CORE_PERF_GLOBAL_CTRL, - intel_ctrl & ~cpuc->intel_ctrl_guest_mask); + intel_ctrl & ~cpuc->intel_ctrl_exclude_host_mask); if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) { struct perf_event *event = @@ -2995,9 +2994,9 @@ static inline void intel_set_masks(struct perf_event *event, int idx) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); if (event->attr.exclude_host) - __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask); + __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask); if (event->attr.exclude_guest) - __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask); + __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_guest_mask); if (event_is_checkpointed(event)) __set_bit(idx, (unsigned long *)&cpuc->intel_cp_status); } @@ -3006,8 +3005,8 @@ static inline void intel_clear_masks(struct perf_event *event, int idx) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask); - __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask); + __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask); + __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_exclude_guest_mask); __clear_bit(idx, (unsigned long *)&cpuc->intel_cp_status); } @@ -3499,6 +3498,21 @@ static void intel_pmu_enable_event_ext(struct perf_event *event) if (pebs_data_cfg & PEBS_DATACFG_XMMS) ext |= ARCH_PEBS_VECR_XMM & cap.caps; + if (pebs_data_cfg & PEBS_DATACFG_YMMHS) + ext |= ARCH_PEBS_VECR_YMMH & cap.caps; + + if (pebs_data_cfg & PEBS_DATACFG_EGPRS) + ext |= ARCH_PEBS_VECR_EGPRS & cap.caps; + + if (pebs_data_cfg & PEBS_DATACFG_OPMASKS) + ext |= ARCH_PEBS_VECR_OPMASK & cap.caps; + + if (pebs_data_cfg & PEBS_DATACFG_ZMMHS) + ext |= ARCH_PEBS_VECR_ZMMH & cap.caps; + + if (pebs_data_cfg & PEBS_DATACFG_H16ZMMS) + ext |= ARCH_PEBS_VECR_H16ZMM & cap.caps; + if (pebs_data_cfg & PEBS_DATACFG_LBRS) ext |= ARCH_PEBS_LBR & cap.caps; @@ -3770,20 +3784,20 @@ static void intel_pmu_reset(void) * * The contents and other behavior of the guest event do not matter. */ -static void x86_pmu_handle_guest_pebs(struct pt_regs *regs, - struct perf_sample_data *data) +static int x86_pmu_handle_guest_pebs(struct pt_regs *regs, + struct perf_sample_data *data) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - u64 guest_pebs_idxs = cpuc->pebs_enabled & ~cpuc->intel_ctrl_host_mask; + u64 guest_pebs_idxs = cpuc->pebs_enabled & ~cpuc->intel_ctrl_exclude_guest_mask; struct perf_event *event = NULL; int bit; if (!unlikely(perf_guest_state())) - return; + return 0; if (!x86_pmu.pebs_ept || !x86_pmu.pebs_active || !guest_pebs_idxs) - return; + return 0; for_each_set_bit(bit, (unsigned long *)&guest_pebs_idxs, X86_PMC_IDX_MAX) { event = cpuc->events[bit]; @@ -3793,9 +3807,14 @@ static void x86_pmu_handle_guest_pebs(struct pt_regs *regs, perf_sample_data_init(data, 0, event->hw.last_period); perf_event_overflow(event, data, regs); - /* Inject one fake event is enough. */ - break; + /* + * Inject one fake event is enough. + * Returning 1 to inform PMI is handled. + */ + return 1; } + + return 0; } static int handle_pmi_common(struct pt_regs *regs, u64 status) @@ -3844,9 +3863,11 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) if (__test_and_clear_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, (unsigned long *)&status)) { u64 pebs_enabled = cpuc->pebs_enabled; - handled++; - x86_pmu_handle_guest_pebs(regs, &data); - static_call(x86_pmu_drain_pebs)(regs, &data); + handled += x86_pmu_handle_guest_pebs(regs, &data); + handled += static_call(x86_pmu_drain_pebs)(regs, &data); + /* Ensure no "suspicious NMI" warning for empty PEBS buffer. */ + if (!handled) + handled++; /* * PMI throttle may be triggered, which stops the PEBS event. @@ -3873,8 +3894,10 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) */ if (__test_and_clear_bit(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT, (unsigned long *)&status)) { - handled++; - static_call(x86_pmu_drain_pebs)(regs, &data); + handled += static_call(x86_pmu_drain_pebs)(regs, &data); + /* Ensure no "suspicious NMI" warning for empty PEBS buffer. */ + if (!handled) + handled++; if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS] && is_pebs_counter_event_group(cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS])) @@ -3950,6 +3973,9 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) if (has_branch_stack(event)) intel_pmu_lbr_save_brstack(&data, cpuc, event); + x86_pmu_clear_perf_regs(regs); + x86_pmu_update_perf_regs(event, &data, regs, false); + perf_event_overflow(event, &data, regs); } @@ -4717,14 +4743,20 @@ static void intel_pebs_aliases_skl(struct perf_event *event) static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event) { unsigned long flags = x86_pmu.large_pebs_flags; + u64 gprs_mask = event->attr.sample_simd_regs_enabled ? + PEBS_GP_REGS | PERF_X86_EGPRS_MASK | + BIT_ULL(PERF_REG_X86_SSP) : + PEBS_GP_REGS | PERF_REG_EXTENDED_MASK; if (event->attr.use_clockid) flags &= ~PERF_SAMPLE_TIME; if (!event->attr.exclude_kernel) flags &= ~PERF_SAMPLE_REGS_USER; - if (event->attr.sample_regs_user & ~PEBS_GP_REGS) + if ((event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & ~gprs_mask)) flags &= ~PERF_SAMPLE_REGS_USER; - if (event->attr.sample_regs_intr & ~PEBS_GP_REGS) + if ((event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & ~gprs_mask)) flags &= ~PERF_SAMPLE_REGS_INTR; return flags; } @@ -5295,13 +5327,13 @@ static int intel_pmu_hw_config(struct perf_event *event) * when it uses {RD,WR}MSR, which should be handled by the KVM context, * specifically in the intel_pmu_{get,set}_msr(). */ -static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) +static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs; - struct kvm_pmu *kvm_pmu = (struct kvm_pmu *)data; u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); - u64 pebs_mask = cpuc->pebs_enabled & x86_pmu.pebs_capable; + u64 pebs_mask = intel_ctrl & cpuc->pebs_enabled & x86_pmu.pebs_capable; u64 guest_pebs_mask; int global_ctrl; @@ -5316,8 +5348,8 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) global_ctrl = (*nr)++; arr[global_ctrl] = (struct perf_guest_switch_msr){ .msr = MSR_CORE_PERF_GLOBAL_CTRL, - .host = intel_ctrl & ~cpuc->intel_ctrl_guest_mask, - .guest = intel_ctrl & ~cpuc->intel_ctrl_host_mask & ~pebs_mask, + .host = intel_ctrl & ~cpuc->intel_ctrl_exclude_host_mask, + .guest = intel_ctrl & ~cpuc->intel_ctrl_exclude_guest_mask & ~pebs_mask, }; if (!x86_pmu.ds_pebs) @@ -5353,17 +5385,9 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) * the guest wants to use for PEBS, (c) are not excluded from counting * in the guest, and (d) _are_ excluded from counting in the host. */ - guest_pebs_mask = pebs_mask & intel_ctrl & kvm_pmu->pebs_enable & - ~cpuc->intel_ctrl_host_mask & - cpuc->intel_ctrl_guest_mask; - - /* - * Disable counters where the guest PMC is different than the host PMC - * being used on behalf of the guest, as the PEBS record includes - * PERF_GLOBAL_STATUS, i.e. the guest will see overflow status for the - * wrong counter(s). - */ - guest_pebs_mask &= ~kvm_pmu->host_cross_mapped_mask; + guest_pebs_mask = pebs_mask & guest_pebs->enable & + ~cpuc->intel_ctrl_exclude_guest_mask & + cpuc->intel_ctrl_exclude_host_mask; /* * FIXME: Allow guest and host usage of PEBS events to co-exist instead @@ -5371,7 +5395,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) * What exactly goes wrong if guest and host are using PEBS is * unknown. */ - if (pebs_mask & ~cpuc->intel_ctrl_guest_mask) + if (pebs_mask & ~cpuc->intel_ctrl_exclude_host_mask) guest_pebs_mask = 0; /* @@ -5382,14 +5406,14 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) arr[(*nr)++] = (struct perf_guest_switch_msr){ .msr = MSR_IA32_DS_AREA, .host = (unsigned long)cpuc->ds, - .guest = guest_pebs_mask ? kvm_pmu->ds_area : (unsigned long)cpuc->ds, + .guest = guest_pebs_mask ? guest_pebs->ds_area : (unsigned long)cpuc->ds, }; if (x86_pmu.intel_cap.pebs_baseline) { arr[(*nr)++] = (struct perf_guest_switch_msr){ .msr = MSR_PEBS_DATA_CFG, .host = cpuc->active_pebs_data_cfg, - .guest = guest_pebs_mask ? kvm_pmu->pebs_data_cfg : + .guest = guest_pebs_mask ? guest_pebs->data_cfg : cpuc->active_pebs_data_cfg, }; } @@ -5405,7 +5429,8 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) return arr; } -static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr, void *data) +static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs; @@ -6216,12 +6241,46 @@ static inline bool intel_pmu_broken_perf_cap(void) return false; } -static inline void __intel_update_pmu_caps(struct pmu *pmu) +static inline void __intel_update_pmu_xregs_caps(struct pmu *pmu) { struct pmu *dest_pmu = pmu ? pmu : x86_get_pmu(smp_processor_id()); - if (hybrid(pmu, arch_pebs_cap).caps & ARCH_PEBS_VECR_XMM) - dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS; + /* Only support the extension when XSAVES is available. */ + if (!boot_cpu_has(X86_FEATURE_XSAVES)) + return; + + if (!boot_cpu_has(X86_FEATURE_XMM) || + !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) + return; + + /* + * On current hybrid platforms, P-cores and E-cores expose the same + * XSAVE feature set. Therefore, using the global x86_pmu.ext_regs_mask + * is sufficient to represent the hardware-supported XSAVE features. + */ + x86_pmu.ext_regs_mask |= XFEATURE_MASK_SSE; + + if (boot_cpu_has(X86_FEATURE_AVX) && + cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_YMM; + if (boot_cpu_has(X86_FEATURE_APX) && + cpu_has_xfeatures(XFEATURE_MASK_APX, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_APX; + if (boot_cpu_has(X86_FEATURE_AVX512F)) { + if (cpu_has_xfeatures(XFEATURE_MASK_OPMASK, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_OPMASK; + if (cpu_has_xfeatures(XFEATURE_MASK_ZMM_Hi256, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_ZMM_Hi256; + if (cpu_has_xfeatures(XFEATURE_MASK_Hi16_ZMM, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_Hi16_ZMM; + } + if (cpu_feature_enabled(X86_FEATURE_USER_SHSTK) && + cpu_has_xfeatures(XFEATURE_MASK_CET_USER, NULL)) + x86_pmu.ext_regs_mask |= XFEATURE_MASK_CET_USER; + + dest_pmu->capabilities |= PERF_PMU_CAP_EXTENDED_REGS; + if (x86_pmu.ext_regs_mask > XFEATURE_MASK_SSE) + dest_pmu->capabilities |= PERF_PMU_CAP_SIMD_REGS; } static inline void __intel_update_large_pebs_flags(struct pmu *pmu) @@ -6292,12 +6351,10 @@ static void update_pmu_cap_from_perfmonext(struct pmu *pmu) hybrid(pmu, arch_pebs_cap).counters = pebs_mask; hybrid(pmu, arch_pebs_cap).pdists = pdists_mask; - if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask)) { + if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask)) x86_pmu.arch_pebs = 0; - } else { - __intel_update_pmu_caps(pmu); + else __intel_update_large_pebs_flags(pmu); - } } else { WARN_ON(x86_pmu.arch_pebs == 1); x86_pmu.arch_pebs = 0; @@ -6321,6 +6378,7 @@ static void intel_update_pmu_caps(struct pmu *pmu) hybrid_pmu(pmu)->pmu_type == hybrid_big) hybrid(pmu, intel_cap).perf_metrics = 1; } + __intel_update_pmu_xregs_caps(pmu); } static void intel_pmu_check_hybrid_pmus(struct x86_hybrid_pmu *pmu) @@ -6474,8 +6532,6 @@ static void intel_pmu_cpu_starting(int cpu) } } - __intel_update_pmu_caps(cpuc->pmu); - if (!cpuc->shared_regs) return; diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index d0329a7eb8a5..8444670cee4a 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1704,6 +1704,7 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event) u64 sample_type = attr->sample_type; u64 pebs_data_cfg = 0; bool gprs, tsx_weight; + u64 xgprs_mask; if (!(sample_type & ~(PERF_SAMPLE_IP|PERF_SAMPLE_TIME)) && attr->precise_ip > 1) @@ -1718,10 +1719,13 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event) * + precise_ip < 2 for the non event IP * + For RTM TSX weight we need GPRs for the abort code. */ + xgprs_mask = event->attr.sample_simd_regs_enabled ? + PEBS_GP_REGS | BIT_ULL(PERF_REG_X86_SSP) : + PEBS_GP_REGS; gprs = ((sample_type & PERF_SAMPLE_REGS_INTR) && - (attr->sample_regs_intr & PEBS_GP_REGS)) || + (attr->sample_regs_intr & xgprs_mask)) || ((sample_type & PERF_SAMPLE_REGS_USER) && - (attr->sample_regs_user & PEBS_GP_REGS)); + (attr->sample_regs_user & xgprs_mask)); tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT_TYPE) && ((attr->config & INTEL_ARCH_EVENT_MASK) == @@ -1730,9 +1734,20 @@ static u64 pebs_update_adaptive_cfg(struct perf_event *event) if (gprs || (attr->precise_ip < 2) || tsx_weight) pebs_data_cfg |= PEBS_DATACFG_GP; - if ((sample_type & PERF_SAMPLE_REGS_INTR) && - (attr->sample_regs_intr & PERF_REG_EXTENDED_MASK)) - pebs_data_cfg |= PEBS_DATACFG_XMMS; + if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) { + if (event_needs_xmm(event)) + pebs_data_cfg |= PEBS_DATACFG_XMMS; + if (x86_pmu.arch_pebs && event_needs_ymm(event)) + pebs_data_cfg |= PEBS_DATACFG_YMMHS; + if (x86_pmu.arch_pebs && event_needs_low16_zmm(event)) + pebs_data_cfg |= PEBS_DATACFG_ZMMHS; + if (x86_pmu.arch_pebs && event_needs_high16_zmm(event)) + pebs_data_cfg |= PEBS_DATACFG_H16ZMMS; + if (x86_pmu.arch_pebs && event_needs_opmask(event)) + pebs_data_cfg |= PEBS_DATACFG_OPMASKS; + if (x86_pmu.arch_pebs && event_needs_egprs(event)) + pebs_data_cfg |= PEBS_DATACFG_EGPRS; + } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { /* @@ -2523,7 +2538,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, return; perf_regs = container_of(regs, struct x86_perf_regs, regs); - perf_regs->xmm_regs = NULL; + x86_pmu_clear_perf_regs(regs); format_group = basic->format_group; @@ -2608,6 +2623,8 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, next_record += nr * sizeof(u64); } + x86_pmu_update_perf_regs(event, data, regs, true); + WARN_ONCE(next_record != __pebs + basic->format_size, "PEBS record size %u, expected %llu, config %llx\n", basic->format_size, @@ -2640,7 +2657,7 @@ static void setup_arch_pebs_sample_data(struct perf_event *event, return; perf_regs = container_of(regs, struct x86_perf_regs, regs); - perf_regs->xmm_regs = NULL; + x86_pmu_clear_perf_regs(regs); __setup_perf_sample_data(event, iregs, data); @@ -2648,6 +2665,9 @@ static void setup_arch_pebs_sample_data(struct perf_event *event, again: header = at; + if (!header->size) + return; + next_record = at + sizeof(struct arch_pebs_header); if (header->basic) { struct arch_pebs_basic *basic = next_record; @@ -2678,6 +2698,11 @@ again: __setup_pebs_gpr_group(event, regs, (struct pebs_gprs *)gprs, sample_type); + + /* Currently only user space mode enables SSP. */ + if (user_mode(regs) && (sample_type & + (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))) + perf_regs->ssp = &gprs->ssp; } if (header->aux) { @@ -2690,14 +2715,63 @@ again: meminfo->tsx_tuning, ax); } - if (header->xmm) { + if (header->xmm || header->ymmh || header->egpr || + header->opmask || header->zmmh || header->h16zmm) { + struct arch_pebs_xer_header *xer_header = next_record; struct pebs_xmm *xmm; + struct ymmh_struct *ymmh; + struct avx_512_zmm_uppers_state *zmmh; + struct avx_512_hi16_state *h16zmm; + struct avx_512_opmask_state *opmask; + struct apx_state *egpr; next_record += sizeof(struct arch_pebs_xer_header); - xmm = next_record; - perf_regs->xmm_regs = xmm->xmm; - next_record = xmm + 1; + if (header->xmm) { + xmm = next_record; + /* + * Only output XMM regs to user space when arch-PEBS + * really writes data into xstate area. + */ + if (xer_header->xstate & XFEATURE_MASK_SSE) + perf_regs->xmm_regs = xmm->xmm; + next_record = xmm + 1; + } + + if (header->ymmh) { + ymmh = next_record; + if (xer_header->xstate & XFEATURE_MASK_YMM) + perf_regs->ymmh = ymmh; + next_record = ymmh + 1; + } + + if (header->egpr) { + egpr = next_record; + if (xer_header->xstate & XFEATURE_MASK_APX) + perf_regs->egpr = egpr; + next_record = egpr + 1; + } + + if (header->opmask) { + opmask = next_record; + if (xer_header->xstate & XFEATURE_MASK_OPMASK) + perf_regs->opmask = opmask; + next_record = opmask + 1; + } + + if (header->zmmh) { + zmmh = next_record; + if (xer_header->xstate & XFEATURE_MASK_ZMM_Hi256) + perf_regs->zmmh = zmmh; + next_record = zmmh + 1; + } + + if (header->h16zmm) { + h16zmm = next_record; + if (xer_header->xstate & XFEATURE_MASK_Hi16_ZMM) + perf_regs->h16zmm = h16zmm; + next_record = h16zmm + 1; + } } if (header->lbr) { @@ -2742,6 +2816,8 @@ again: at = at + header->size; goto again; } + + x86_pmu_update_perf_regs(event, data, regs, true); } static inline void * @@ -2865,13 +2941,21 @@ __intel_pmu_pebs_last_event(struct perf_event *event, struct pt_regs *iregs, struct pt_regs *regs, struct perf_sample_data *data, - void *at, - int count, + void *at, int count, bool corrupted, setup_fn setup_sample) { struct hw_perf_event *hwc = &event->hw; - setup_sample(event, iregs, at, data, regs); + /* Skip parsing corrupted PEBS record. */ + if (corrupted) { + /* Clear stale register states in previous records. */ + memset(regs, 0, sizeof(*regs)); + x86_pmu_clear_perf_regs(regs); + perf_sample_data_init(data, 0, event->hw.last_period); + } else { + setup_sample(event, iregs, at, data, regs); + } + if (iregs == &dummy_iregs) { /* * The PEBS records may be drained in the non-overflow context, @@ -2889,12 +2973,16 @@ __intel_pmu_pebs_last_event(struct perf_event *event, } if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) { - if ((is_pebs_counter_event_group(event))) { - /* - * The value of each sample has been updated when setup - * the corresponding sample data. - */ - perf_event_update_userpage(event); + if (is_pebs_counter_event_group(event)) { + if (corrupted) { + intel_pmu_save_and_restart_reload(event, 1); + } else { + /* + * The value of each sample has been updated + * when setup the corresponding sample data. + */ + perf_event_update_userpage(event); + } } else { /* * Now, auto-reload is only enabled in fixed period mode. @@ -2918,13 +3006,15 @@ __intel_pmu_pebs_last_event(struct perf_event *event, * counters-snapshotting record, only needs to set the new * period for the counter. */ - if (is_pebs_counter_event_group(event)) + if (is_pebs_counter_event_group(event) && !corrupted) static_call(x86_pmu_set_period)(event); else intel_pmu_save_and_restart(event); } } +static DEFINE_PER_CPU(struct x86_perf_regs, x86_pebs_regs); + static __always_inline void __intel_pmu_pebs_events(struct perf_event *event, struct pt_regs *iregs, @@ -2934,25 +3024,29 @@ __intel_pmu_pebs_events(struct perf_event *event, setup_fn setup_sample) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - struct x86_perf_regs perf_regs; - struct pt_regs *regs = &perf_regs.regs; + struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs); + struct pt_regs *regs = &perf_regs->regs; void *at = get_next_pebs_record_by_bit(base, top, bit); int cnt = count; + x86_pmu_clear_perf_regs(regs); + if (!iregs) iregs = &dummy_iregs; while (cnt > 1) { - __intel_pmu_pebs_event(event, iregs, regs, data, at, setup_sample); + __intel_pmu_pebs_event(event, iregs, regs, data, + at, setup_sample); at += cpuc->pebs_record_size; at = get_next_pebs_record_by_bit(at, top, bit); cnt--; } - __intel_pmu_pebs_last_event(event, iregs, regs, data, at, count, setup_sample); + __intel_pmu_pebs_last_event(event, iregs, regs, data, at, + count, false, setup_sample); } -static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data) +static int intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct debug_store *ds = cpuc->ds; @@ -2961,7 +3055,7 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_ int n; if (!x86_pmu.pebs_active) - return; + return 0; at = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base; top = (struct pebs_record_core *)(unsigned long)ds->pebs_index; @@ -2972,22 +3066,25 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_ ds->pebs_index = ds->pebs_buffer_base; if (!test_bit(0, cpuc->active_mask)) - return; + return 0; WARN_ON_ONCE(!event); if (!event->attr.precise_ip) - return; + return 0; n = top - at; if (n <= 0) { if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD) intel_pmu_save_and_restart_reload(event, 0); - return; + return 0; } __intel_pmu_pebs_events(event, iregs, data, at, top, 0, n, setup_pebs_fixed_sample_data); + + /* PMC0 only */ + return 1; } static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64 mask) @@ -3010,7 +3107,7 @@ static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64 } } -static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data) +static int intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct debug_store *ds = cpuc->ds; @@ -3019,11 +3116,12 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; short error[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; int max_pebs_events = intel_pmu_max_num_pebs(NULL); + u64 events_bitmap = 0; int bit, i, size; u64 mask; if (!x86_pmu.pebs_active) - return; + return 0; base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base; top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index; @@ -3039,7 +3137,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d if (unlikely(base >= top)) { intel_pmu_pebs_event_update_no_drain(cpuc, mask); - return; + return 0; } for (at = base; at < top; at += x86_pmu.pebs_record_size) { @@ -3103,6 +3201,7 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d if ((counts[bit] == 0) && (error[bit] == 0)) continue; + events_bitmap |= BIT_ULL(bit); event = cpuc->events[bit]; if (WARN_ON_ONCE(!event)) continue; @@ -3124,6 +3223,8 @@ static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_d setup_pebs_fixed_sample_data); } } + + return hweight64(events_bitmap); } static __always_inline void @@ -3158,39 +3259,46 @@ static __always_inline void __intel_pmu_handle_last_pebs_record(struct pt_regs *iregs, struct pt_regs *regs, struct perf_sample_data *data, - u64 mask, short *counts, void **last, + u64 mask, short *counts, + void **last, bool corrupted, setup_fn setup_sample) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_event *event; + bool handled = false; int bit; for_each_set_bit(bit, (unsigned long *)&mask, X86_PMC_IDX_MAX) { if (!counts[bit]) continue; + handled = true; event = cpuc->events[bit]; - __intel_pmu_pebs_last_event(event, iregs, regs, data, last[bit], - counts[bit], setup_sample); + counts[bit], corrupted, setup_sample); } + /* All records are corrupted, reset sampling period. */ + if (!handled) + intel_pmu_pebs_event_update_no_drain(cpuc, mask); } -static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data) +static int intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data) { short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS]; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct debug_store *ds = cpuc->ds; - struct x86_perf_regs perf_regs; - struct pt_regs *regs = &perf_regs.regs; + struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs); + struct pt_regs *regs = &perf_regs->regs; struct pebs_basic *basic; void *base, *at, *top; + u64 events_bitmap = 0; + bool corrupted = false; u64 mask; if (!x86_pmu.pebs_active) - return; + return 0; base = (struct pebs_basic *)(unsigned long)ds->pebs_buffer_base; top = (struct pebs_basic *)(unsigned long)ds->pebs_index; @@ -3203,7 +3311,7 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d if (unlikely(base >= top)) { intel_pmu_pebs_event_update_no_drain(cpuc, mask); - return; + return 0; } if (!iregs) @@ -3214,36 +3322,45 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d u64 pebs_status; basic = at; + if (WARN_ON_ONCE(!basic->format_size)) { + corrupted = true; + break; + } if (basic->format_size != cpuc->pebs_record_size) continue; pebs_status = mask & basic->applicable_counters; + events_bitmap |= pebs_status; __intel_pmu_handle_pebs_record(iregs, regs, data, at, pebs_status, counts, last, setup_pebs_adaptive_sample_data); } __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, counts, last, - setup_pebs_adaptive_sample_data); + corrupted, setup_pebs_adaptive_sample_data); + + return hweight64(events_bitmap); } -static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs, - struct perf_sample_data *data) +static int intel_pmu_drain_arch_pebs(struct pt_regs *iregs, + struct perf_sample_data *data) { short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {}; void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS]; struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); union arch_pebs_index index; - struct x86_perf_regs perf_regs; - struct pt_regs *regs = &perf_regs.regs; + struct x86_perf_regs *perf_regs = this_cpu_ptr(&x86_pebs_regs); + struct pt_regs *regs = &perf_regs->regs; void *base, *at, *top; + u64 events_bitmap = 0; + bool corrupted = false; u64 mask; rdmsrq(MSR_IA32_PEBS_INDEX, index.whole); if (unlikely(!index.wr)) { intel_pmu_pebs_event_update_no_drain(cpuc, X86_PMC_IDX_MAX); - return; + return 0; } base = cpuc->pebs_vaddr; @@ -3271,8 +3388,10 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs, header = at; - if (WARN_ON_ONCE(!header->size)) - break; + if (WARN_ON_ONCE(!header->size)) { + corrupted = true; + goto done; + } /* 1st fragment or single record must have basic group */ if (!header->basic) { @@ -3282,6 +3401,7 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs, basic = at + sizeof(struct arch_pebs_header); pebs_status = mask & basic->applicable_counters; + events_bitmap |= pebs_status; __intel_pmu_handle_pebs_record(iregs, regs, data, at, pebs_status, counts, last, setup_arch_pebs_sample_data); @@ -3291,16 +3411,27 @@ static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs, if (!header->size) break; at += header->size; + if (WARN_ON_ONCE(at >= top)) { + corrupted = true; + goto done; + } header = at; } /* Skip last fragment or the single record */ at += header->size; + if (WARN_ON_ONCE(at > top)) { + corrupted = true; + goto done; + } } +done: __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, - counts, last, + counts, last, corrupted, setup_arch_pebs_sample_data); + + return hweight64(events_bitmap); } static void __init intel_arch_pebs_init(void) @@ -3402,7 +3533,6 @@ static void __init intel_ds_pebs_init(void) x86_pmu.flags |= PMU_FL_PEBS_ALL; x86_pmu.pebs_capable = ~0ULL; pebs_qual = "-baseline"; - x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_EXTENDED_REGS; } else { /* Only basic record supported */ x86_pmu.large_pebs_flags &= diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 22e2a06d5786..0b446d8912e4 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -714,7 +714,7 @@ static inline bool vlbr_exclude_host(void) struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); return test_bit(INTEL_PMC_IDX_FIXED_VLBR, - (unsigned long *)&cpuc->intel_ctrl_guest_mask); + (unsigned long *)&cpuc->intel_ctrl_exclude_host_mask); } void intel_pmu_lbr_enable_all(bool pmi) diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index fab9da78a5c7..e274802ef062 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -147,6 +147,199 @@ static inline bool is_acr_self_reload_event(struct perf_event *event) return test_bit(hwc->idx, (unsigned long *)&hwc->config1); } +static inline bool __event_needs_xmm(struct perf_event *event, u64 sample_type) +{ + if (event->attr.sample_simd_regs_enabled) { + if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_XMM_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_simd_vec_reg_user > 0)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_simd_vec_reg_intr > 0)) + return true; + } else { + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) + return true; + } + + return false; +} + +static inline bool event_needs_xmm(struct perf_event *event) +{ + return __event_needs_xmm(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + +static inline bool __event_needs_ymm(struct perf_event *event, u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_YMM_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_simd_vec_reg_user > 0)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_simd_vec_reg_intr > 0)) + return true; + + return false; +} + +static inline bool event_needs_ymm(struct perf_event *event) +{ + return __event_needs_ymm(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + +static inline bool __event_needs_low16_zmm(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_ZMM_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_simd_vec_reg_user > 0)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_simd_vec_reg_intr > 0)) + return true; + + return false; +} + +static inline bool event_needs_low16_zmm(struct perf_event *event) +{ + return __event_needs_low16_zmm(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + +static inline bool __event_needs_high16_zmm(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + if (event->attr.sample_simd_vec_reg_qwords < PERF_X86_ZMM_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (fls64(event->attr.sample_simd_vec_reg_user) > PERF_X86_H16ZMM_BASE)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (fls64(event->attr.sample_simd_vec_reg_intr) > PERF_X86_H16ZMM_BASE)) + return true; + + return false; +} + +static inline bool event_needs_high16_zmm(struct perf_event *event) +{ + return __event_needs_high16_zmm(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + +static inline bool __event_needs_opmask(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + if (event->attr.sample_simd_pred_reg_qwords != PERF_X86_OPMASK_QWORDS) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_simd_pred_reg_user > 0)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_simd_pred_reg_intr > 0)) + return true; + + return false; +} + +static inline bool event_needs_opmask(struct perf_event *event) +{ + return __event_needs_opmask(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + +static inline bool __event_needs_egprs(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & PERF_X86_EGPRS_MASK)) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & PERF_X86_EGPRS_MASK)) + return true; + + return false; +} + +static inline bool event_needs_egprs(struct perf_event *event) +{ + return __event_needs_egprs(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + +static inline bool __event_needs_ssp(struct perf_event *event, + u64 sample_type) +{ + if (!event->attr.sample_simd_regs_enabled) + return false; + + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_type & PERF_SAMPLE_REGS_USER) && + (event->attr.sample_regs_user & BIT_ULL(PERF_REG_X86_SSP))) + return true; + + if ((sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_type & PERF_SAMPLE_REGS_INTR) && + (event->attr.sample_regs_intr & BIT_ULL(PERF_REG_X86_SSP))) + return true; + + return false; +} + +static inline bool event_needs_ssp(struct perf_event *event) +{ + return __event_needs_ssp(event, + PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER); +} + struct amd_nb { int nb_id; /* NorthBridge id */ int refcnt; /* reference count */ @@ -347,8 +540,8 @@ struct cpu_hw_events { /* * Intel host/guest exclude bits */ - u64 intel_ctrl_guest_mask; - u64 intel_ctrl_host_mask; + u64 intel_ctrl_exclude_host_mask; + u64 intel_ctrl_exclude_guest_mask; struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX]; /* @@ -948,7 +1141,7 @@ struct x86_pmu { int pebs_record_size; int pebs_buffer_size; u64 pebs_events_mask; - void (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data); + int (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data); struct event_constraint *pebs_constraints; void (*pebs_aliases)(struct perf_event *event); u64 (*pebs_latency_data)(struct perf_event *event, u64 status); @@ -1025,9 +1218,16 @@ struct x86_pmu { unsigned int flags; /* + * Extended regs, e.g., vector registers + * Utilize the same format as the XFEATURE_MASK_* + */ + u64 ext_regs_mask; + + /* * Intel host/guest support (KVM) */ - struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, void *data); + struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, + struct x86_guest_pebs *guest_pebs); /* * Check period value for PERF_EVENT_IOC_PERIOD ioctl. @@ -1046,7 +1246,7 @@ struct x86_pmu { * unique capabilities. */ int num_hybrid_pmus; - struct x86_hybrid_pmu *hybrid_pmu; + struct x86_hybrid_pmu *hybrid_pmu __counted_by_ptr(num_hybrid_pmus); enum intel_cpu_type (*get_hybrid_cpu_type) (void); }; @@ -1311,6 +1511,13 @@ void x86_pmu_enable_event(struct perf_event *event); int x86_pmu_handle_irq(struct pt_regs *regs); +void x86_pmu_clear_perf_regs(struct pt_regs *regs); + +void x86_pmu_update_perf_regs(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs, + bool from_pebs); + void x86_pmu_show_pmu_cap(struct pmu *pmu); static inline int x86_pmu_num_counters(struct pmu *pmu) diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h index 48d74371a75a..67b0dfa3530f 100644 --- a/arch/x86/include/asm/fpu/sched.h +++ b/arch/x86/include/asm/fpu/sched.h @@ -10,6 +10,8 @@ #include <asm/trace/fpu.h> extern void save_fpregs_to_fpstate(struct fpu *fpu); +extern void update_fpu_state_and_flag(struct fpu *fpu, + struct task_struct *task); extern void fpu__drop(struct task_struct *tsk); extern int fpu_clone(struct task_struct *dst, u64 clone_flags, bool minimal, unsigned long shstk_addr); @@ -35,8 +37,7 @@ static inline void switch_fpu(struct task_struct *old, int cpu) !(old->flags & (PF_KTHREAD | PF_USER_WORKER))) { struct fpu *old_fpu = x86_task_fpu(old); - set_tsk_thread_flag(old, TIF_NEED_FPU_LOAD); - save_fpregs_to_fpstate(old_fpu); + update_fpu_state_and_flag(old_fpu, old); /* * The save operation preserved register state, so the * fpu_fpregs_owner_ctx is still @old_fpu. Store the diff --git a/arch/x86/include/asm/fpu/xstate.h b/arch/x86/include/asm/fpu/xstate.h index 33343e5d54b0..19dec5f0b1c7 100644 --- a/arch/x86/include/asm/fpu/xstate.h +++ b/arch/x86/include/asm/fpu/xstate.h @@ -110,6 +110,7 @@ int xfeature_size(int xfeature_nr); void xsaves(struct xregs_state *xsave, u64 mask); void xrstors(struct xregs_state *xsave, u64 mask); +void xsaves_nmi(struct xregs_state *xsave, u64 mask); unsigned int xstate_calculate_size(u64 xfeatures, bool compacted); diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 57d37491c7c0..20b9db3f5203 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -609,15 +609,6 @@ struct kvm_pmu { u64 pebs_data_cfg_rsvd; /* - * If a guest counter is cross-mapped to host counter with different - * index, its PEBS capability will be temporarily disabled. - * - * The user should make sure that this mask is updated - * after disabling interrupts and before perf_guest_get_msrs(); - */ - u64 host_cross_mapped_mask; - - /* * The gate to release perf_events not marked in * pmc_in_use only once in a vcpu time slice. */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index fbbfd4bddf97..c0914829a925 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -352,6 +352,13 @@ #define ARCH_PEBS_LBR_SHIFT 40 #define ARCH_PEBS_LBR (0x3ull << ARCH_PEBS_LBR_SHIFT) #define ARCH_PEBS_VECR_XMM BIT_ULL(49) +#define ARCH_PEBS_VECR_YMMH BIT_ULL(50) +#define ARCH_PEBS_VECR_EGPRS BIT_ULL(51) +#define ARCH_PEBS_VECR_OPMASK BIT_ULL(53) +#define ARCH_PEBS_VECR_ZMMH BIT_ULL(54) +#define ARCH_PEBS_VECR_H16ZMM BIT_ULL(55) +#define ARCH_PEBS_VECR_EXT_SHIFT 49 +#define ARCH_PEBS_VECR_EXT (0x7full << ARCH_PEBS_VECR_EXT_SHIFT) #define ARCH_PEBS_GPR BIT_ULL(61) #define ARCH_PEBS_AUX BIT_ULL(62) #define ARCH_PEBS_EN BIT_ULL(63) diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index 1eb13673e889..5c92d43bbef8 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -150,6 +150,11 @@ #define PEBS_DATACFG_LBRS BIT_ULL(3) #define PEBS_DATACFG_CNTR BIT_ULL(4) #define PEBS_DATACFG_METRICS BIT_ULL(5) +#define PEBS_DATACFG_YMMHS BIT_ULL(6) +#define PEBS_DATACFG_OPMASKS BIT_ULL(7) +#define PEBS_DATACFG_ZMMHS BIT_ULL(8) +#define PEBS_DATACFG_H16ZMMS BIT_ULL(9) +#define PEBS_DATACFG_EGPRS BIT_ULL(10) #define PEBS_DATACFG_LBR_SHIFT 24 #define PEBS_DATACFG_CNTR_SHIFT 32 #define PEBS_DATACFG_CNTR_MASK GENMASK_ULL(15, 0) @@ -547,7 +552,8 @@ struct arch_pebs_header { rsvd3:7, xmm:1, ymmh:1, - rsvd4:2, + egpr:1, + rsvd4:1, opmask:1, zmmh:1, h16zmm:1, @@ -728,7 +734,32 @@ extern void perf_events_lapic_init(void); struct pt_regs; struct x86_perf_regs { struct pt_regs regs; - u64 *xmm_regs; + u64 abi; + union { + u64 *xmm_regs; + u32 *xmm_space; /* for xsaves */ + }; + union { + u64 *ymmh_regs; + struct ymmh_struct *ymmh; + }; + union { + u64 *zmmh_regs; + struct avx_512_zmm_uppers_state *zmmh; + }; + union { + u64 *h16zmm_regs; + struct avx_512_hi16_state *h16zmm; + }; + union { + u64 *opmask_regs; + struct avx_512_opmask_state *opmask; + }; + union { + u64 *egpr_regs; + struct apx_state *egpr; + }; + u64 *ssp; }; extern unsigned long perf_arch_instruction_pointer(struct pt_regs *regs); @@ -788,11 +819,18 @@ extern void perf_load_guest_lvtpc(u32 guest_lvtpc); extern void perf_put_guest_lvtpc(void); #endif +struct x86_guest_pebs { + u64 enable; + u64 ds_area; + u64 data_cfg; +}; #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) -extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data); +extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs); extern void x86_perf_get_lbr(struct x86_pmu_lbr *lbr); #else -struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data); +struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, + struct x86_guest_pebs *guest_pebs); static inline void x86_perf_get_lbr(struct x86_pmu_lbr *lbr) { memset(lbr, 0, sizeof(*lbr)); diff --git a/arch/x86/include/uapi/asm/perf_regs.h b/arch/x86/include/uapi/asm/perf_regs.h index 7c9d2bb3833b..faaa82df688d 100644 --- a/arch/x86/include/uapi/asm/perf_regs.h +++ b/arch/x86/include/uapi/asm/perf_regs.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_PERF_REGS_H #define _ASM_X86_PERF_REGS_H +#include <linux/bits.h> + enum perf_event_x86_regs { PERF_REG_X86_AX, PERF_REG_X86_BX, @@ -27,9 +29,35 @@ enum perf_event_x86_regs { PERF_REG_X86_R13, PERF_REG_X86_R14, PERF_REG_X86_R15, + /* + * The eGPRs/SSP and XMM have overlaps. Only one can be used + * at a time. The ABI PERF_SAMPLE_REGS_ABI_SIMD is used to + * distinguish which one is used. If PERF_SAMPLE_REGS_ABI_SIMD + * is set, then eGPRs/SSP is used, otherwise, XMM is used. + * + * Extended GPRs (eGPRs) + */ + PERF_REG_X86_R16, + PERF_REG_X86_R17, + PERF_REG_X86_R18, + PERF_REG_X86_R19, + PERF_REG_X86_R20, + PERF_REG_X86_R21, + PERF_REG_X86_R22, + PERF_REG_X86_R23, + PERF_REG_X86_R24, + PERF_REG_X86_R25, + PERF_REG_X86_R26, + PERF_REG_X86_R27, + PERF_REG_X86_R28, + PERF_REG_X86_R29, + PERF_REG_X86_R30, + PERF_REG_X86_R31, + PERF_REG_X86_SSP, /* These are the limits for the GPRs. */ PERF_REG_X86_32_MAX = PERF_REG_X86_GS + 1, PERF_REG_X86_64_MAX = PERF_REG_X86_R15 + 1, + PERF_REG_MISC_MAX = PERF_REG_X86_SSP + 1, /* These all need two bits set because they are 128bit */ PERF_REG_X86_XMM0 = 32, @@ -54,5 +82,30 @@ enum perf_event_x86_regs { }; #define PERF_REG_EXTENDED_MASK (~((1ULL << PERF_REG_X86_XMM0) - 1)) +#define PERF_X86_EGPRS_MASK __GENMASK_ULL(PERF_REG_X86_R31, PERF_REG_X86_R16) + +enum { + PERF_X86_SIMD_XMM_REGS = 16, + PERF_X86_SIMD_YMM_REGS = 16, + PERF_X86_SIMD_ZMM_REGS = 32, + PERF_X86_SIMD_VEC_REGS_MAX = PERF_X86_SIMD_ZMM_REGS, + + PERF_X86_SIMD_OPMASK_REGS = 8, + PERF_X86_SIMD_PRED_REGS_MAX = PERF_X86_SIMD_OPMASK_REGS, +}; + +#define PERF_X86_SIMD_PRED_MASK __GENMASK(PERF_X86_SIMD_PRED_REGS_MAX - 1, 0) +#define PERF_X86_SIMD_VEC_MASK __GENMASK_ULL(PERF_X86_SIMD_VEC_REGS_MAX - 1, 0) + +#define PERF_X86_H16ZMM_BASE 16 + +enum { + /* 1 qword = 8 bytes */ + PERF_X86_OPMASK_QWORDS = 1, + PERF_X86_XMM_QWORDS = 2, + PERF_X86_YMM_QWORDS = 4, + PERF_X86_ZMM_QWORDS = 8, + PERF_X86_SIMD_QWORDS_MAX = PERF_X86_ZMM_QWORDS, +}; #endif /* _ASM_X86_PERF_REGS_H */ diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 67d4db38c4cb..80a7f41cf3ad 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -213,6 +213,19 @@ void restore_fpregs_from_fpstate(struct fpstate *fpstate, u64 mask) } } +/* + * Save the FPU register state in fpu->fpstate->regs and set + * TIF_NEED_FPU_LOAD subsequently. + * + * Must be called with fpregs_lock() held, ensuring flag + * TIF_NEED_FPU_LOAD is set last. + */ +void update_fpu_state_and_flag(struct fpu *fpu, struct task_struct *task) +{ + save_fpregs_to_fpstate(fpu); + set_tsk_thread_flag(task, TIF_NEED_FPU_LOAD); +} + void fpu_reset_from_exception_fixup(void) { restore_fpregs_from_fpstate(&init_fpstate, XFEATURE_MASK_FPSTATE); @@ -383,13 +396,13 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest) /* Swap fpstate */ if (enter_guest) { - fpu->__task_fpstate = cur_fps; + WRITE_ONCE(fpu->__task_fpstate, cur_fps); + barrier(); fpu->fpstate = guest_fps; guest_fps->in_use = true; } else { guest_fps->in_use = false; fpu->fpstate = fpu->__task_fpstate; - fpu->__task_fpstate = NULL; } cur_fps = fpu->fpstate; @@ -406,6 +419,16 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest) xfd_update_state(cur_fps); } + /* + * Clear fpu->__task_fpstate after switching back to host state. + * A non-NULL __task_fpstate means guest state is still resident in + * hardware; reset it only once host state has been restored. + */ + if (!enter_guest) { + barrier(); + WRITE_ONCE(fpu->__task_fpstate, NULL); + } + fpregs_mark_activate(); fpregs_unlock(); return 0; @@ -481,10 +504,8 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask) this_cpu_write(kernel_fpu_allowed, false); if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) && - !test_thread_flag(TIF_NEED_FPU_LOAD)) { - set_thread_flag(TIF_NEED_FPU_LOAD); - save_fpregs_to_fpstate(x86_task_fpu(current)); - } + !test_thread_flag(TIF_NEED_FPU_LOAD)) + update_fpu_state_and_flag(x86_task_fpu(current), current); __cpu_invalidate_fpregs_state(); /* Put sane initial values into the control registers. */ diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 561eec713dfc..da303714379c 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -1491,6 +1491,29 @@ void xrstors(struct xregs_state *xstate, u64 mask) WARN_ON_ONCE(err); } +/** + * xsaves_nmi - Save selected components to a kernel xstate buffer in NMI + * @xstate: Pointer to the buffer + * @mask: Feature mask to select the components to save + * + * This function is similar to xsaves(), but should only be called within + * the NMI handler. This function returns the actual register contents at + * the moment the NMI occurs. + * + * Currently, the perf subsystem is the sole user of this helper. It uses + * the function to snapshot SIMD (XMM/YMM/ZMM) and APX eGPRs registers. + */ +void xsaves_nmi(struct xregs_state *xstate, u64 mask) +{ + int err; + + if (!in_nmi()) + return; + + XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err); + WARN_ON_ONCE(err); +} + #if IS_ENABLED(CONFIG_KVM) void fpstate_clear_xstate_component(struct fpstate *fpstate, unsigned int xfeature) { diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index 624703af80a1..f8952f7c36cc 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -61,11 +61,29 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) { struct x86_perf_regs *perf_regs; - if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) { + if (idx > PERF_REG_X86_R15) { perf_regs = container_of(regs, struct x86_perf_regs, regs); - if (!perf_regs->xmm_regs) + if (perf_regs->abi == PERF_SAMPLE_REGS_ABI_NONE) return 0; - return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0]; + + if (perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD) { + if (idx <= PERF_REG_X86_R31) { + if (!perf_regs->egpr_regs) + return 0; + return perf_regs->egpr_regs[idx - PERF_REG_X86_R16]; + } + if (idx == PERF_REG_X86_SSP) { + if (!perf_regs->ssp) + return 0; + return *perf_regs->ssp; + } + } else { + if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) { + if (!perf_regs->xmm_regs) + return 0; + return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0]; + } + } } if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset))) @@ -74,22 +92,130 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return regs_get_register(regs, pt_regs_offset[idx]); } -#define PERF_REG_X86_RESERVED (((1ULL << PERF_REG_X86_XMM0) - 1) & \ - ~((1ULL << PERF_REG_X86_MAX) - 1)) +#define PERF_X86_YMMH_QWORDS (PERF_X86_YMM_QWORDS / 2) +#define PERF_X86_ZMMH_QWORDS (PERF_X86_ZMM_QWORDS / 2) + +u64 perf_simd_reg_value(struct pt_regs *regs, int idx, + u16 qwords_idx, bool pred) +{ + struct x86_perf_regs *perf_regs = + container_of(regs, struct x86_perf_regs, regs); + + if (!(perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD)) + return 0; + + if (pred) { + if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_PRED_REGS_MAX || + qwords_idx >= PERF_X86_OPMASK_QWORDS)) + return 0; + if (!perf_regs->opmask_regs) + return 0; + return perf_regs->opmask_regs[idx]; + } + + if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_VEC_REGS_MAX || + qwords_idx >= PERF_X86_SIMD_QWORDS_MAX)) + return 0; + + if (idx >= PERF_X86_H16ZMM_BASE) { + if (!perf_regs->h16zmm_regs) + return 0; + return perf_regs->h16zmm_regs[(idx - PERF_X86_H16ZMM_BASE) * + PERF_X86_ZMM_QWORDS + qwords_idx]; + } + + if (qwords_idx < PERF_X86_XMM_QWORDS) { + if (!perf_regs->xmm_regs) + return 0; + return perf_regs->xmm_regs[idx * PERF_X86_XMM_QWORDS + + qwords_idx]; + } else if (qwords_idx < PERF_X86_YMM_QWORDS) { + if (!perf_regs->ymmh_regs) + return 0; + return perf_regs->ymmh_regs[idx * PERF_X86_YMMH_QWORDS + + qwords_idx - PERF_X86_XMM_QWORDS]; + } else if (qwords_idx < PERF_X86_ZMM_QWORDS) { + if (!perf_regs->zmmh_regs) + return 0; + return perf_regs->zmmh_regs[idx * PERF_X86_ZMMH_QWORDS + + qwords_idx - PERF_X86_YMM_QWORDS]; + } + + return 0; +} + +int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, + u16 pred_qwords, u32 pred_mask) +{ + unsigned long mask; + u64 size; + + if (!vec_qwords && !pred_qwords) { + if (vec_mask || pred_mask) + return -EINVAL; + } + + if (vec_qwords) { + if (vec_qwords != PERF_X86_XMM_QWORDS && + vec_qwords != PERF_X86_YMM_QWORDS && + vec_qwords != PERF_X86_ZMM_QWORDS) + return -EINVAL; + if (vec_mask & ~PERF_X86_SIMD_VEC_MASK) + return -EINVAL; + /* Only full-register sampling is allowed. */ + mask = vec_mask; + if (vec_qwords == PERF_X86_XMM_QWORDS && mask && + !bitmap_full(&mask, PERF_X86_SIMD_XMM_REGS)) + return -EINVAL; + if (vec_qwords == PERF_X86_YMM_QWORDS && mask && + !bitmap_full(&mask, PERF_X86_SIMD_YMM_REGS)) + return -EINVAL; + if (vec_qwords == PERF_X86_ZMM_QWORDS && mask && + !bitmap_full(&mask, PERF_X86_SIMD_ZMM_REGS)) + return -EINVAL; + } + + if (pred_qwords) { + if (pred_qwords != PERF_X86_OPMASK_QWORDS) + return -EINVAL; + if (pred_mask & ~PERF_X86_SIMD_PRED_MASK) + return -EINVAL; + /* Only full-register sampling is allowed. */ + mask = pred_mask; + if (pred_qwords == PERF_X86_OPMASK_QWORDS && mask && + !bitmap_full(&mask, PERF_X86_SIMD_OPMASK_REGS)) + return -EINVAL; + } + + size = sizeof(u64) * 4; + size += (hweight64(vec_mask) * vec_qwords + + hweight32(pred_mask) * pred_qwords) * sizeof(u64); + /* + * INTR_REGS and USR_REGS could be sampled simultaneously, + * so roughly restrict the size to half of U16_MAX. + */ + if (size >= U16_MAX / 2) + return -EINVAL; + + return 0; +} + +#define PERF_REG_X86_RESERVED (GENMASK_ULL(PERF_REG_X86_XMM0 - 1, PERF_REG_X86_AX) & \ + ~GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_AX)) +#define PERF_REG_X86_EXT_RESERVED (~GENMASK_ULL(PERF_REG_MISC_MAX - 1, PERF_REG_X86_AX)) #ifdef CONFIG_X86_32 -#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_R8) | \ - (1ULL << PERF_REG_X86_R9) | \ - (1ULL << PERF_REG_X86_R10) | \ - (1ULL << PERF_REG_X86_R11) | \ - (1ULL << PERF_REG_X86_R12) | \ - (1ULL << PERF_REG_X86_R13) | \ - (1ULL << PERF_REG_X86_R14) | \ - (1ULL << PERF_REG_X86_R15)) - -int perf_reg_validate(u64 mask) +#define REG_NOSUPPORT GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_R8) + +int perf_reg_validate(u64 mask, bool simd_enabled) { - if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED))) + if (!simd_enabled && + (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))) + return -EINVAL; + + /* The mask could be 0 if only the SIMD registers are interested */ + if (simd_enabled && + (mask & ~GENMASK_ULL(PERF_REG_X86_GS, PERF_REG_X86_AX))) return -EINVAL; return 0; @@ -100,21 +226,21 @@ u64 perf_reg_abi(struct task_struct *task) return PERF_SAMPLE_REGS_ABI_32; } -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} #else /* CONFIG_X86_64 */ #define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \ (1ULL << PERF_REG_X86_ES) | \ (1ULL << PERF_REG_X86_FS) | \ (1ULL << PERF_REG_X86_GS)) -int perf_reg_validate(u64 mask) +int perf_reg_validate(u64 mask, bool simd_enabled) { - if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED))) + if (!simd_enabled && + (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))) + return -EINVAL; + + /* The mask could be 0 if only the SIMD registers are interested */ + if (simd_enabled && + (mask & (REG_NOSUPPORT | PERF_REG_X86_EXT_RESERVED))) return -EINVAL; return 0; diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 70a8c4816135..fb7ad53855f1 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -750,24 +750,36 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) intel_pmu_release_guest_lbr_event(vcpu); } -void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu) +u64 __intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu) { - struct kvm_pmc *pmc = NULL; + u64 guest_pebs_enable = pmu->pebs_enable & pmu->global_ctrl; + u64 pebs_enable = 0; + struct kvm_pmc *pmc; int bit, hw_idx; - kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&pmu->global_ctrl) { - if (!pmc_is_locally_enabled(pmc) || - !pmc_is_globally_enabled(pmc) || !pmc->perf_event) + /* + * Omit counters that are locally disabled, don't have a perf event, or + * ended up with a perf event that is using a different counter than + * the guest, i.e. where the guest PMC is different than the host PMC + * being used on behalf of the guest. PEBS records include + * PERF_GLOBAL_STATUS, and so using a counter with a different index + * means the guest will see overflow status for the wrong counter(s). + */ + kvm_for_each_pmc(pmu, pmc, bit, (unsigned long *)&guest_pebs_enable) { + if (!pmc_is_locally_enabled(pmc) || !pmc->perf_event) continue; /* - * A negative index indicates the event isn't mapped to a + * Note, a negative index indicates the event isn't mapped to a * physical counter in the host, e.g. due to contention. */ hw_idx = pmc->perf_event->hw.idx; - if (hw_idx != pmc->idx && hw_idx > -1) - pmu->host_cross_mapped_mask |= BIT_ULL(hw_idx); + if (hw_idx != pmc->idx) + continue; + + pebs_enable |= BIT_ULL(pmc->idx); } + return pebs_enable; } static bool intel_pmu_is_mediated_pmu_supported(struct x86_pmu_capability *host_pmu) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 612ab07d4100..2d0443562a16 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7363,12 +7363,14 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx) if (kvm_vcpu_has_mediated_pmu(&vmx->vcpu)) return; - pmu->host_cross_mapped_mask = 0; - if (pmu->pebs_enable & pmu->global_ctrl) - intel_pmu_cross_mapped_check(pmu); + struct x86_guest_pebs guest_pebs = { + .enable = intel_pmu_compute_pebs_enable(pmu), + .ds_area = pmu->ds_area, + .data_cfg = pmu->pebs_data_cfg, + }; /* Note, nr_msrs may be garbage if perf_guest_get_msrs() returns NULL. */ - msrs = perf_guest_get_msrs(&nr_msrs, (void *)pmu); + msrs = perf_guest_get_msrs(&nr_msrs, &guest_pebs); if (!msrs) return; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index dc8517f15bc4..1db461060c7e 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -664,7 +664,20 @@ static __always_inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu) return container_of(vcpu, struct vcpu_vmx, vcpu); } -void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu); +u64 __intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu); + +static inline u64 intel_pmu_compute_pebs_enable(struct kvm_pmu *pmu) +{ + /* + * Avoid the function call overhead in the common case that the guest + * isn't using PEBS. + */ + if (!(pmu->pebs_enable & pmu->global_ctrl)) + return 0; + + return __intel_pmu_compute_pebs_enable(pmu); +} + int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu); void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu); diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 5f50a3ee08f5..3f4dd3f50f56 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -456,12 +456,14 @@ static void xen_convert_regs(const struct xen_pmu_regs *xen_regs, } } +static DEFINE_PER_CPU(struct x86_perf_regs, x86_xen_intr_regs); irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id) { int err, ret = IRQ_NONE; struct pt_regs regs = {0}; const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); uint8_t xenpmu_flags = get_xenpmu_flags(); + struct x86_perf_regs *x86_regs = this_cpu_ptr(&x86_xen_intr_regs); if (!xenpmu_data) { pr_warn_once("%s: pmudata not initialized\n", __func__); @@ -472,7 +474,8 @@ irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id) xenpmu_flags | XENPMU_IRQ_PROCESSING; xen_convert_regs(&xenpmu_data->pmu.r.regs, ®s, xenpmu_data->pmu.pmu_flags); - if (x86_pmu.handle_irq(®s)) + x86_regs->regs = regs; + if (x86_pmu.handle_irq(&x86_regs->regs)) ret = IRQ_HANDLED; /* Write out cached context to HW */ diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 2d5e7687875b..eb3f7bab498c 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -306,6 +306,7 @@ struct perf_event_pmu_context; #define PERF_PMU_CAP_AUX_PAUSE 0x0200 #define PERF_PMU_CAP_AUX_PREFER_LARGE 0x0400 #define PERF_PMU_CAP_MEDIATED_VPMU 0x0800 +#define PERF_PMU_CAP_SIMD_REGS 0x1000 /** * pmu::scope @@ -1467,6 +1468,7 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data, return size; } +extern u64 perf_update_xregs_size(struct perf_event *event, bool intr); extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data, @@ -1518,6 +1520,27 @@ perf_event__output_id_sample(struct perf_event *event, extern void perf_log_lost_samples(struct perf_event *event, u64 lost); +static inline bool event_has_simd_regs(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + + if (!(event->attr.sample_type & + (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))) + return false; + + return attr->sample_simd_regs_enabled != 0; +} + +static inline bool event_has_extended_regs(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + + return ((attr->sample_type & PERF_SAMPLE_REGS_USER) && + (attr->sample_regs_user & PERF_REG_EXTENDED_MASK)) || + ((attr->sample_type & PERF_SAMPLE_REGS_INTR) && + (attr->sample_regs_intr & PERF_REG_EXTENDED_MASK)); +} + static inline bool event_has_any_exclude_flag(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; diff --git a/include/linux/perf_regs.h b/include/linux/perf_regs.h index f632c5725f16..09dbc2fc3859 100644 --- a/include/linux/perf_regs.h +++ b/include/linux/perf_regs.h @@ -9,6 +9,16 @@ struct perf_regs { struct pt_regs *regs; }; +u64 perf_reg_value(struct pt_regs *regs, int idx); +int perf_reg_validate(u64 mask, bool simd_enabled); +u64 perf_reg_abi(struct task_struct *task); +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs); +int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, + u16 pred_qwords, u32 pred_mask); +u64 perf_simd_reg_value(struct pt_regs *regs, int idx, + u16 qwords_idx, bool pred); + #ifdef CONFIG_HAVE_PERF_REGS #include <asm/perf_regs.h> @@ -16,35 +26,9 @@ struct perf_regs { #define PERF_REG_EXTENDED_MASK 0 #endif -u64 perf_reg_value(struct pt_regs *regs, int idx); -int perf_reg_validate(u64 mask); -u64 perf_reg_abi(struct task_struct *task); -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs); #else #define PERF_REG_EXTENDED_MASK 0 -static inline u64 perf_reg_value(struct pt_regs *regs, int idx) -{ - return 0; -} - -static inline int perf_reg_validate(u64 mask) -{ - return mask ? -ENOSYS : 0; -} - -static inline u64 perf_reg_abi(struct task_struct *task) -{ - return PERF_SAMPLE_REGS_ABI_NONE; -} - -static inline void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} #endif /* CONFIG_HAVE_PERF_REGS */ #endif /* _LINUX_PERF_REGS_H */ diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index fd10aa8d697f..c49fc76292f7 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -314,8 +314,9 @@ enum { */ enum perf_sample_regs_abi { PERF_SAMPLE_REGS_ABI_NONE = 0, - PERF_SAMPLE_REGS_ABI_32 = 1, - PERF_SAMPLE_REGS_ABI_64 = 2, + PERF_SAMPLE_REGS_ABI_32 = (1 << 0), + PERF_SAMPLE_REGS_ABI_64 = (1 << 1), + PERF_SAMPLE_REGS_ABI_SIMD = (1 << 2), }; /* @@ -383,6 +384,7 @@ enum perf_event_read_format { #define PERF_ATTR_SIZE_VER7 128 /* Add: sig_data */ #define PERF_ATTR_SIZE_VER8 136 /* Add: config3 */ #define PERF_ATTR_SIZE_VER9 144 /* add: config4 */ +#define PERF_ATTR_SIZE_VER10 176 /* Add: sample_simd_{vec|pred}_reg_* */ /* * 'struct perf_event_attr' contains various attributes that define @@ -547,6 +549,29 @@ struct perf_event_attr { __u64 config3; /* extension of config2 */ __u64 config4; /* extension of config3 */ + + /* + * Defines the sampling SIMD/PRED(predicate) registers bitmap and + * qwords (8 bytes) length. + * + * sample_simd_regs_enabled != 0 indicates there are SIMD/PRED + * registers to be sampled, the SIMD/PRED registers bitmap and + * qwords length are represented in + * sample_simd_{vec|pred}_reg_{intr|user} and + * sample_simd_{vec|pred}_reg_qwords fields separately. + * + * sample_simd_regs_enabled == 0 indicates no SIMD/PRED registers + * are sampled. + */ + __u16 sample_simd_regs_enabled; + __u16 sample_simd_pred_reg_qwords; + __u16 sample_simd_vec_reg_qwords; + __u16 __reserved_4; + + __u32 sample_simd_pred_reg_intr; + __u32 sample_simd_pred_reg_user; + __u64 sample_simd_vec_reg_intr; + __u64 sample_simd_vec_reg_user; }; /* @@ -1020,7 +1045,15 @@ enum perf_event_type { * } && PERF_SAMPLE_BRANCH_STACK * * { u64 abi; # enum perf_sample_regs_abi - * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER + * u64 regs[weight(mask)]; + * struct { + * u64 nr_vectors; # 0 ... weight(sample_simd_vec_reg_user) + * u64 vector_qwords; # 0 ... sample_simd_vec_reg_qwords + * u64 nr_pred; # 0 ... weight(sample_simd_pred_reg_user) + * u64 pred_qwords; # 0 ... sample_simd_pred_reg_qwords + * u64 data[nr_vectors * vector_qwords + nr_pred * pred_qwords]; + * } && (abi & PERF_SAMPLE_REGS_ABI_SIMD) + * } && PERF_SAMPLE_REGS_USER * * { u64 size; * char data[size]; @@ -1047,7 +1080,15 @@ enum perf_event_type { * { u64 data_src; } && PERF_SAMPLE_DATA_SRC * { u64 transaction; } && PERF_SAMPLE_TRANSACTION * { u64 abi; # enum perf_sample_regs_abi - * u64 regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR + * u64 regs[weight(mask)]; + * struct { + * u64 nr_vectors; # 0 ... weight(sample_simd_vec_reg_intr) + * u64 vector_qwords; # 0 ... sample_simd_vec_reg_qwords + * u64 nr_pred; # 0 ... weight(sample_simd_pred_reg_intr) + * u64 pred_qwords; # 0 ... sample_simd_pred_reg_qwords + * u64 data[nr_vectors * vector_qwords + nr_pred * pred_qwords]; + * } && (abi & PERF_SAMPLE_REGS_ABI_SIMD) + * } && PERF_SAMPLE_REGS_INTR * { u64 phys_addr;} && PERF_SAMPLE_PHYS_ADDR * { u64 cgroup;} && PERF_SAMPLE_CGROUP * { u64 data_page_size;} && PERF_SAMPLE_DATA_PAGE_SIZE diff --git a/kernel/events/core.c b/kernel/events/core.c index 2b38e9a0c834..a9ee91151b2d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7831,22 +7831,82 @@ unsigned long perf_instruction_pointer(struct perf_event *event, 0 : perf_arch_instruction_pointer(regs); } +u64 __weak perf_reg_value(struct pt_regs *regs, int idx) +{ + return 0; +} + +int __weak perf_reg_validate(u64 mask, bool simd_enabled) +{ + return mask ? -ENOSYS : 0; +} + +u64 __weak perf_reg_abi(struct task_struct *task) +{ + return PERF_SAMPLE_REGS_ABI_NONE; +} + +void __weak perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} + +#define word_for_each_set_bit(bit, val) \ + for (unsigned long long __v = (val); \ + __v && ((bit = __builtin_ctzll(__v)), 1); \ + __v &= __v - 1) + static void perf_output_sample_regs(struct perf_output_handle *handle, struct pt_regs *regs, u64 mask) { int bit; - DECLARE_BITMAP(_mask, 64); - bitmap_from_u64(_mask, mask); - for_each_set_bit(bit, _mask, sizeof(mask) * BITS_PER_BYTE) { - u64 val; - - val = perf_reg_value(regs, bit); + word_for_each_set_bit(bit, mask) { + u64 val = perf_reg_value(regs, bit); perf_output_put(handle, val); } } +static void +perf_output_sample_simd_regs(struct perf_output_handle *handle, + struct perf_event *event, + struct pt_regs *regs, + u64 mask, u32 pred_mask) +{ + u64 pred_qwords = event->attr.sample_simd_pred_reg_qwords; + u64 vec_qwords = event->attr.sample_simd_vec_reg_qwords; + u64 nr_vectors = hweight64(mask); + u64 nr_pred = hweight32(pred_mask); + int bit; + + perf_output_put(handle, nr_vectors); + perf_output_put(handle, vec_qwords); + perf_output_put(handle, nr_pred); + perf_output_put(handle, pred_qwords); + + if (nr_vectors) { + word_for_each_set_bit(bit, mask) { + for (int i = 0; i < vec_qwords; i++) { + u64 val = perf_simd_reg_value(regs, bit, + i, false); + perf_output_put(handle, val); + } + } + } + if (nr_pred) { + word_for_each_set_bit(bit, pred_mask) { + for (int i = 0; i < pred_qwords; i++) { + u64 val = perf_simd_reg_value(regs, bit, + i, true); + perf_output_put(handle, val); + } + } + } +} + static void perf_sample_regs_user(struct perf_regs *regs_user, struct pt_regs *regs) { @@ -7880,6 +7940,17 @@ static void perf_sample_regs_intr(struct perf_regs *regs_intr, } } +int __weak perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, + u16 pred_qwords, u32 pred_mask) +{ + return -EINVAL; +} + +u64 __weak perf_simd_reg_value(struct pt_regs *regs, int idx, + u16 qwords_idx, bool pred) +{ + return 0; +} /* * Get remaining task size from user stack pointer. @@ -8415,10 +8486,17 @@ void perf_output_sample(struct perf_output_handle *handle, perf_output_put(handle, abi); if (abi) { - u64 mask = event->attr.sample_regs_user; + struct perf_event_attr *attr = &event->attr; + u64 mask = attr->sample_regs_user; perf_output_sample_regs(handle, data->regs_user.regs, mask); + if (abi & PERF_SAMPLE_REGS_ABI_SIMD) { + perf_output_sample_simd_regs(handle, event, + data->regs_user.regs, + attr->sample_simd_vec_reg_user, + attr->sample_simd_pred_reg_user); + } } } @@ -8446,11 +8524,18 @@ void perf_output_sample(struct perf_output_handle *handle, perf_output_put(handle, abi); if (abi) { - u64 mask = event->attr.sample_regs_intr; + struct perf_event_attr *attr = &event->attr; + u64 mask = attr->sample_regs_intr; perf_output_sample_regs(handle, data->regs_intr.regs, mask); + if (abi & PERF_SAMPLE_REGS_ABI_SIMD) { + perf_output_sample_simd_regs(handle, event, + data->regs_intr.regs, + attr->sample_simd_vec_reg_intr, + attr->sample_simd_pred_reg_intr); + } } } @@ -8653,6 +8738,29 @@ static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d) return d * !!(flags & s); } +u64 perf_update_xregs_size(struct perf_event *event, bool intr) +{ + u16 pred_qwords = event->attr.sample_simd_pred_reg_qwords; + u16 vec_qwords = event->attr.sample_simd_vec_reg_qwords; + u64 pred_mask; + u64 mask; + int size; + + if (intr) { + mask = event->attr.sample_simd_vec_reg_intr; + pred_mask = event->attr.sample_simd_pred_reg_intr; + } else { + mask = event->attr.sample_simd_vec_reg_user; + pred_mask = event->attr.sample_simd_pred_reg_user; + } + + size = sizeof(u64) * 4; + size += (hweight64(mask) * vec_qwords + + hweight64(pred_mask) * pred_qwords) * sizeof(u64); + + return size; +} + void perf_prepare_sample(struct perf_sample_data *data, struct perf_event *event, struct pt_regs *regs) @@ -8715,7 +8823,12 @@ void perf_prepare_sample(struct perf_sample_data *data, if (data->regs_user.regs) { u64 mask = event->attr.sample_regs_user; + size += hweight64(mask) * sizeof(u64); + if (event_has_simd_regs(event)) { + size += perf_update_xregs_size(event, false); + data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } } data->dyn_size += size; @@ -8780,6 +8893,10 @@ void perf_prepare_sample(struct perf_sample_data *data, u64 mask = event->attr.sample_regs_intr; size += hweight64(mask) * sizeof(u64); + if (event_has_simd_regs(event)) { + size += perf_update_xregs_size(event, true); + data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD; + } } data->dyn_size += size; @@ -13071,12 +13188,6 @@ int perf_pmu_unregister(struct pmu *pmu) } EXPORT_SYMBOL_GPL(perf_pmu_unregister); -static inline bool has_extended_regs(struct perf_event *event) -{ - return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) || - (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK); -} - static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) { struct perf_event_context *ctx = NULL; @@ -13110,8 +13221,14 @@ static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) if (ret) goto err_pmu; + if (!(pmu->capabilities & PERF_PMU_CAP_SIMD_REGS) && + event_has_simd_regs(event)) { + ret = -EOPNOTSUPP; + goto err_destroy; + } + if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) && - has_extended_regs(event)) { + event_has_extended_regs(event)) { ret = -EOPNOTSUPP; goto err_destroy; } @@ -13605,7 +13722,8 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, attr->size = size; - if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) + if (attr->__reserved_1 || attr->__reserved_2 || + attr->__reserved_3 || attr->__reserved_4) return -EINVAL; if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) @@ -13651,9 +13769,18 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, } if (attr->sample_type & PERF_SAMPLE_REGS_USER) { - ret = perf_reg_validate(attr->sample_regs_user); + ret = perf_reg_validate(attr->sample_regs_user, + attr->sample_simd_regs_enabled); if (ret) return ret; + if (attr->sample_simd_regs_enabled) { + ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords, + attr->sample_simd_vec_reg_user, + attr->sample_simd_pred_reg_qwords, + attr->sample_simd_pred_reg_user); + if (ret) + return ret; + } } if (attr->sample_type & PERF_SAMPLE_STACK_USER) { @@ -13674,8 +13801,20 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr, if (!attr->sample_max_stack) attr->sample_max_stack = sysctl_perf_event_max_stack; - if (attr->sample_type & PERF_SAMPLE_REGS_INTR) - ret = perf_reg_validate(attr->sample_regs_intr); + if (attr->sample_type & PERF_SAMPLE_REGS_INTR) { + ret = perf_reg_validate(attr->sample_regs_intr, + attr->sample_simd_regs_enabled); + if (ret) + return ret; + if (attr->sample_simd_regs_enabled) { + ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords, + attr->sample_simd_vec_reg_intr, + attr->sample_simd_pred_reg_qwords, + attr->sample_simd_pred_reg_intr); + if (ret) + return ret; + } + } #ifndef CONFIG_CGROUP_PERF if (attr->sample_type & PERF_SAMPLE_CGROUP) |