diff options
Diffstat (limited to 'include')
44 files changed, 479 insertions, 162 deletions
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index 714d111d8053..32cac3a6f362 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -108,7 +108,6 @@ enum acpi_bus_device_type { ACPI_BUS_DEVICE_TYPE_COUNT }; -struct acpi_driver; struct acpi_device; /* @@ -159,32 +158,6 @@ struct acpi_hotplug_context { }; /* - * ACPI Driver - * ----------- - */ - -typedef int (*acpi_op_add) (struct acpi_device * device); -typedef void (*acpi_op_remove) (struct acpi_device *device); -typedef void (*acpi_op_notify) (struct acpi_device * device, u32 event); - -struct acpi_device_ops { - acpi_op_add add; - acpi_op_remove remove; - acpi_op_notify notify; -}; - -#define ACPI_DRIVER_ALL_NOTIFY_EVENTS 0x1 /* system AND device events */ - -struct acpi_driver { - char name[80]; - char class[80]; - const struct acpi_device_id *ids; /* Supported Hardware IDs */ - unsigned int flags; - struct acpi_device_ops ops; - struct device_driver drv; -}; - -/* * ACPI Device * ----------- */ @@ -211,7 +184,6 @@ struct acpi_device_flags { u32 removable:1; u32 ejectable:1; u32 power_manageable:1; - u32 match_driver:1; u32 initialized:1; u32 visited:1; u32 hotplug_notify:1; @@ -221,7 +193,7 @@ struct acpi_device_flags { u32 cca_seen:1; u32 enumeration_by_parent:1; u32 honor_deps:1; - u32 reserved:18; + u32 reserved:19; }; /* File System */ @@ -438,7 +410,7 @@ enum acpi_device_swnode_ep_props { * @lane_polarities: "lane-polarities" property values. * @link_frequencies: "link_frequencies" property values. * @port_nr: Port number. - * @crs_crs2_local: _CRS CSI2 record present (i.e. this is a transmitter one). + * @crs_csi2_local: _CRS CSI2 record present (i.e. this is a transmitter one). * @port_props: Port properties. * @ep_props: Endpoint properties. * @remote_ep: Reference to the remote endpoint. @@ -461,7 +433,7 @@ struct acpi_device_software_node_port { * struct acpi_device_software_nodes - Software nodes for an ACPI device * @dev_props: Device properties. * @nodes: Software nodes for root as well as ports and endpoints. - * @nodeprts: Array of software node pointers, for (un)registering them. + * @nodeptrs: Array of software node pointers, for (un)registering them. * @ports: Information related to each port and endpoint within a port. * @num_ports: The number of ports. */ @@ -570,7 +542,6 @@ static inline void *acpi_driver_data(struct acpi_device *d) } #define to_acpi_device(d) container_of(d, struct acpi_device, dev) -#define to_acpi_driver(d) container_of_const(d, struct acpi_driver, drv) static inline struct acpi_device *acpi_dev_parent(struct acpi_device *adev) { @@ -676,13 +647,6 @@ void acpi_scan_lock_release(void); void acpi_lock_hp_context(void); void acpi_unlock_hp_context(void); int acpi_scan_add_handler(struct acpi_scan_handler *handler); -/* - * use a macro to avoid include chaining to get THIS_MODULE - */ -#define acpi_bus_register_driver(drv) \ - __acpi_bus_register_driver(drv, THIS_MODULE) -int __acpi_bus_register_driver(struct acpi_driver *driver, struct module *owner); -void acpi_bus_unregister_driver(struct acpi_driver *driver); int acpi_bus_scan(acpi_handle handle); void acpi_bus_trim(struct acpi_device *start); acpi_status acpi_bus_get_ejd(acpi_handle handle, acpi_handle * ejd); @@ -696,18 +660,6 @@ static inline bool acpi_device_enumerated(struct acpi_device *adev) return adev && adev->flags.initialized && adev->flags.visited; } -/** - * module_acpi_driver(acpi_driver) - Helper macro for registering an ACPI driver - * @__acpi_driver: acpi_driver struct - * - * Helper macro for ACPI drivers which do not do anything special in module - * init/exit. This eliminates a lot of boilerplate. Each module may only - * use this macro once, and calling it replaces module_init() and module_exit() - */ -#define module_acpi_driver(__acpi_driver) \ - module_driver(__acpi_driver, acpi_bus_register_driver, \ - acpi_bus_unregister_driver) - /* * Bind physical devices with ACPI devices */ diff --git a/include/acpi/acpi_drivers.h b/include/acpi/acpi_drivers.h index 402b97d12138..bff2ca035fcf 100644 --- a/include/acpi/acpi_drivers.h +++ b/include/acpi/acpi_drivers.h @@ -59,14 +59,19 @@ int acpi_pci_link_free_irq(acpi_handle handle); struct pci_bus; #ifdef CONFIG_PCI -struct pci_dev *acpi_get_pci_dev(acpi_handle); +struct pci_dev *acpi_dev_get_pci_dev(struct acpi_device *adev); #else -static inline struct pci_dev *acpi_get_pci_dev(acpi_handle handle) +static inline struct pci_dev *acpi_dev_get_pci_dev(struct acpi_device *adev) { return NULL; } #endif +static inline struct pci_dev *acpi_get_pci_dev(acpi_handle handle) +{ + return acpi_dev_get_pci_dev(acpi_fetch_acpi_dev(handle)); +} + /* Arch-defined function to add a bus to the system */ struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root); diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 8c191b9ac18f..94a6277edab2 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -69,7 +69,10 @@ struct cpc_register_resource { acpi_object_type type; u64 __iomem *sys_mem_vaddr; union { - struct cpc_reg reg; + struct { + struct cpc_reg reg; + bool use_rmw_lock; + }; u64 int_value; } cpc_entry; }; @@ -176,6 +179,8 @@ extern int cppc_get_transition_latency(int cpu); extern bool cpc_ffh_supported(void); extern bool cpc_supported_by_cpu(void); extern int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val); +extern int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1, + struct cpc_reg *reg2, u64 *val2); extern int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val); extern int cppc_get_epp_perf(int cpunum, u64 *epp_perf); extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable); @@ -251,6 +256,11 @@ static inline int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val) { return -EOPNOTSUPP; } +static inline int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1, + struct cpc_reg *reg2, u64 *val2) +{ + return -EOPNOTSUPP; +} static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) { return -EOPNOTSUPP; diff --git a/include/acpi/nhlt.h b/include/acpi/nhlt.h index 2108aa6d0207..370958d93706 100644 --- a/include/acpi/nhlt.h +++ b/include/acpi/nhlt.h @@ -10,7 +10,6 @@ #define __ACPI_NHLT_H__ #include <linux/acpi.h> -#include <linux/kconfig.h> #include <linux/overflow.h> #include <linux/types.h> diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index 51f8f3881523..c8683c046615 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -59,6 +59,20 @@ static __always_inline void __preempt_count_sub(int val) *preempt_count_ptr() -= val; } +static __always_inline int __preempt_count_add_return(int val) +{ + *preempt_count_ptr() += val; + + return *preempt_count_ptr(); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + *preempt_count_ptr() -= val; + + return *preempt_count_ptr(); +} + static __always_inline bool __preempt_count_dec_and_test(void) { /* diff --git a/include/asm-generic/qspinlock.h b/include/asm-generic/qspinlock.h index bf47cca2c375..2ca94e41823b 100644 --- a/include/asm-generic/qspinlock.h +++ b/include/asm-generic/qspinlock.h @@ -41,6 +41,7 @@ #include <asm-generic/qspinlock_types.h> #include <linux/atomic.h> +#include <linux/tracepoint-defs.h> #ifndef queued_spin_is_locked /** @@ -115,12 +116,12 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock) } #endif -#ifndef queued_spin_unlock +#ifndef queued_spin_release /** - * queued_spin_unlock - release a queued spinlock + * queued_spin_release - release a queued spinlock * @lock : Pointer to queued spinlock structure */ -static __always_inline void queued_spin_unlock(struct qspinlock *lock) +static __always_inline void queued_spin_release(struct qspinlock *lock) { /* * unlock() needs release semantics: @@ -129,6 +130,37 @@ static __always_inline void queued_spin_unlock(struct qspinlock *lock) } #endif +#ifndef queued_spin_unlock + +DECLARE_TRACEPOINT(contended_release); + +extern void queued_spin_release_traced(struct qspinlock *lock); + +/** + * queued_spin_unlock - unlock a queued spinlock + * @lock : Pointer to queued spinlock structure + * + * Generic tracing wrapper around the arch-overridable + * queued_spin_release(). + */ +static __always_inline void queued_spin_unlock(struct qspinlock *lock) +{ + /* + * Trace and release are combined in queued_spin_release_traced() so + * the compiler does not need to preserve the lock pointer across the + * function call, avoiding callee-saved register save/restore on the + * hot path. queued_spin_release() is therefore called both here and in + * queued_spin_release_traced(). Keep the two in sync. + */ + if (IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE) && + tracepoint_enabled(contended_release)) { + queued_spin_release_traced(lock); + return; + } + queued_spin_release(lock); +} +#endif + #ifndef virt_spin_lock static __always_inline bool virt_spin_lock(struct qspinlock *lock) { diff --git a/include/asm-generic/runtime-const.h b/include/asm-generic/runtime-const.h index 670499459514..03e6e3e02401 100644 --- a/include/asm-generic/runtime-const.h +++ b/include/asm-generic/runtime-const.h @@ -10,6 +10,7 @@ */ #define runtime_const_ptr(sym) (sym) #define runtime_const_shift_right_32(val, sym) ((u32)(val)>>(sym)) +#define runtime_const_mask_32(val, sym) ((u32)(val)&(sym)) #define runtime_const_init(type,sym) do { } while (0) #endif diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 5659f4b5a125..b2988aa12f66 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -839,12 +839,20 @@ .stab.index 0 : { *(.stab.index) } \ .stab.indexstr 0 : { *(.stab.indexstr) } +#ifdef CONFIG_KLP_BUILD +#define KLP_SYMID \ + .klp.symid 0 : { *(.klp.symid) } +#else +#define KLP_SYMID +#endif + /* Required sections not related to debugging. */ #define ELF_DETAILS \ .comment 0 : { *(.comment) } \ .symtab 0 : { *(.symtab) } \ .strtab 0 : { *(.strtab) } \ - .shstrtab 0 : { *(.shstrtab) } + .shstrtab 0 : { *(.shstrtab) } \ + KLP_SYMID #define MODINFO \ .modinfo : { *(.modinfo) . = ALIGN(8); } @@ -970,7 +978,10 @@ RUNTIME_CONST(ptr, __dentry_cache) \ RUNTIME_CONST(ptr, __names_cache) \ RUNTIME_CONST(ptr, __filp_cache) \ - RUNTIME_CONST(ptr, __bfilp_cache) + RUNTIME_CONST(ptr, __bfilp_cache) \ + RUNTIME_CONST(shift, __futex_shift) \ + RUNTIME_CONST(mask, __futex_mask) \ + RUNTIME_CONST(ptr, __futex_queues) /* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */ #define KUNIT_TABLE() \ diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 60ab50cb8930..ddacac812094 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -94,6 +94,12 @@ static inline void acpi_preset_companion(struct device *dev, ACPI_COMPANION_SET(dev, acpi_find_child_device(parent, addr, false)); } +static inline void acpi_device_clear_deps(struct device *dev) +{ + if (has_acpi_companion(dev)) + acpi_dev_clear_dependencies(ACPI_COMPANION(dev)); +} + static inline const char *acpi_dev_name(struct acpi_device *adev) { return dev_name(&adev->dev); @@ -315,6 +321,19 @@ static inline int acpi_processor_evaluate_cst(acpi_handle handle, u32 cpu, } #endif +#ifdef CONFIG_ACPI_PROCESSOR_IDLE +int acpi_processor_extract_lpi_info(acpi_handle pr_handle, + struct acpi_processor_power *pr_power, + bool strict); +#else +static inline int acpi_processor_extract_lpi_info(acpi_handle pr_handle, + struct acpi_processor_power *pr_power, + bool strict) +{ + return -ENODEV; +} +#endif + #ifdef CONFIG_ACPI_HOTPLUG_CPU /* Arch dependent functions for cpu hotplug support */ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id, @@ -360,9 +379,10 @@ int acpi_gsi_to_irq (u32 gsi, unsigned int *irq); int acpi_isa_irq_to_gsi (unsigned isa_irq, u32 *gsi); typedef struct fwnode_handle *(*acpi_gsi_domain_disp_fn)(u32); +typedef acpi_handle (*acpi_gsi_handle_disp_fn)(u32); void acpi_set_irq_model(enum acpi_irq_model_id model, - acpi_gsi_domain_disp_fn fn); + acpi_gsi_domain_disp_fn fn, acpi_gsi_handle_disp_fn gsi_dep_fn); acpi_gsi_domain_disp_fn acpi_get_gsi_dispatcher(void); void acpi_set_gsi_to_irq_fallback(u32 (*)(u32)); @@ -372,6 +392,8 @@ struct irq_domain *acpi_irq_create_hierarchy(unsigned int flags, const struct irq_domain_ops *ops, void *host_data); +u32 acpi_irq_add_auto_dep(acpi_handle handle); + #ifdef CONFIG_X86_IO_APIC extern int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity); #else @@ -448,6 +470,7 @@ extern char *wmi_get_acpi_device_uid(const char *guid); #define ACPI_VIDEO_OUTPUT_SWITCHING_DMI_VIDEO 0x0800 extern char acpi_video_backlight_string[]; +extern bool acpi_dev_is_video_device(struct acpi_device *adev); extern long acpi_is_video_device(acpi_handle handle); extern void acpi_osi_setup(char *str); @@ -908,6 +931,8 @@ static inline void acpi_preset_companion(struct device *dev, { } +static inline void acpi_device_clear_deps(struct device *dev) {} + static inline const char *acpi_dev_name(struct acpi_device *adev) { return NULL; diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index 17bb3374f4ca..931eaa7bbf6a 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -27,7 +27,8 @@ int iort_register_domain_token(int trans_id, phys_addr_t base, struct fwnode_handle *fw_node); void iort_deregister_domain_token(int trans_id); struct fwnode_handle *iort_find_domain_token(int trans_id); -struct fwnode_handle *iort_iwb_handle(u32 iwb_id); +acpi_handle iort_iwb_handle(u32 iwb_id); +struct fwnode_handle *iort_iwb_handle_fwnode(u32 iwb_id); #ifdef CONFIG_ACPI_IORT u32 iort_msi_map_id(struct device *dev, u32 id); diff --git a/include/linux/acpi_pmtmr.h b/include/linux/acpi_pmtmr.h index 0ded9220d379..cceed294d0b5 100644 --- a/include/linux/acpi_pmtmr.h +++ b/include/linux/acpi_pmtmr.h @@ -27,15 +27,17 @@ static inline u32 acpi_pm_read_early(void) } /** - * Register callback for suspend and resume event + * acpi_pmtmr_register_suspend_resume_callback - Register callback for + * suspend and resume event * - * @cb Callback triggered on suspend and resume - * @data Data passed with the callback + * @cb: Callback triggered on suspend and resume + * @data: Data passed with the callback */ void acpi_pmtmr_register_suspend_resume_callback(void (*cb)(void *data, bool suspend), void *data); /** - * Remove registered callback for suspend and resume event + * acpi_pmtmr_unregister_suspend_resume_callback - Remove registered callback + * for suspend and resume event */ void acpi_pmtmr_unregister_suspend_resume_callback(void); diff --git a/include/linux/arm_sdei.h b/include/linux/arm_sdei.h index f652a5028b59..b9dc21c241be 100644 --- a/include/linux/arm_sdei.h +++ b/include/linux/arm_sdei.h @@ -37,6 +37,15 @@ int sdei_event_unregister(u32 event_num); int sdei_event_enable(u32 event_num); int sdei_event_disable(u32 event_num); +/* + * Signal the software-signalled event (event 0) to another PE, NMI-like. + * @mpidr is the target's MPIDR affinity. + */ +int sdei_event_signal(u32 event_num, u64 mpidr); + +/* Was SDEI firmware probed and usable? */ +bool sdei_is_present(void); + /* GHES register/unregister helpers */ int sdei_register_ghes(struct ghes *ghes, sdei_event_callback *normal_cb, sdei_event_callback *critical_cb); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index ae9d1ce4f49c..35ce665edfd8 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -379,6 +379,7 @@ struct cpufreq_driver { void (*adjust_perf)(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, + unsigned long max_perf, unsigned long capacity); /* @@ -624,6 +625,7 @@ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, + unsigned long max_perf, unsigned long capacity); bool cpufreq_driver_has_adjust_perf(void); int cpufreq_driver_target(struct cpufreq_policy *policy, diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 65d76a38974b..9db2d4fcead1 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -80,6 +80,7 @@ extern void lockdep_assert_cpuset_lock_held(void); extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); +extern int cpuset_num_cpus(struct cgroup *cgroup); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); @@ -216,6 +217,11 @@ static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p) return false; } +static inline int cpuset_num_cpus(struct cgroup *cgroup) +{ + return num_online_cpus(); +} + static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) { return node_possible_map; diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h index c1dee3f971a9..bc087124cd78 100644 --- a/include/linux/crash_core.h +++ b/include/linux/crash_core.h @@ -59,6 +59,9 @@ extern int crash_exclude_mem_range(struct crash_mem *mem, unsigned long long mend); extern int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, void **addr, unsigned long *sz); +extern int crash_prepare_headers(int need_kernel_map, void **addr, + unsigned long *sz, unsigned long *nr_mem_ranges); +extern int crash_exclude_core_ranges(struct crash_mem **cmem); struct kimage; struct kexec_segment; @@ -76,6 +79,12 @@ int kexec_should_crash(struct task_struct *p); int kexec_crash_loaded(void); void crash_save_cpu(struct pt_regs *regs, int cpu); extern int kimage_crash_copy_vmcoreinfo(struct kimage *image); +extern unsigned int arch_get_system_nr_ranges(void); +extern int arch_crash_populate_cmem(struct crash_mem *cmem); +extern int arch_crash_exclude_ranges(struct crash_mem *cmem); +extern int arch_crash_exclude_mem_range(struct crash_mem **mem, + unsigned long long mstart, + unsigned long long mend); #else /* !CONFIG_CRASH_DUMP*/ struct pt_regs; diff --git a/include/linux/crash_reserve.h b/include/linux/crash_reserve.h index f0dc03d94ca2..30864d90d7f5 100644 --- a/include/linux/crash_reserve.h +++ b/include/linux/crash_reserve.h @@ -14,9 +14,11 @@ extern struct resource crashk_res; extern struct resource crashk_low_res; extern struct range crashk_cma_ranges[]; + +#define CRASHK_CMA_RANGES_MAX 4 #if defined(CONFIG_CMA) && defined(CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION) #define CRASHKERNEL_CMA -#define CRASHKERNEL_CMA_RANGES_MAX 4 +#define CRASHKERNEL_CMA_RANGES_MAX (CRASHK_CMA_RANGES_MAX) extern int crashk_cma_cnt; #else #define crashk_cma_cnt 0 diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index d57cab4d4c06..73b48dd9f135 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -92,6 +92,37 @@ void irq_exit_rcu(void); #define arch_nmi_exit() do { } while (0) #endif +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +static __always_inline void __preempt_count_nmi_enter(void) +{ + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); +} + +static __always_inline void __preempt_count_nmi_exit(void) +{ + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); +} +#else +DECLARE_PER_CPU(unsigned int, nmi_nesting); + +#define __preempt_count_nmi_enter() \ + do { \ + __preempt_count_add(HARDIRQ_OFFSET); \ + /* Maximum NMI nesting is 15. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \ + __this_cpu_inc(nmi_nesting); \ + preempt_count_set(preempt_count() | NMI_MASK); \ + } while (0) + +#define __preempt_count_nmi_exit() \ + do { \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + if (!__this_cpu_dec_return(nmi_nesting)) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ + } while (0) + +#endif + /* * NMI vs Tracing * -------------- @@ -102,21 +133,20 @@ void irq_exit_rcu(void); */ /* - * nmi_enter() can nest up to 15 times; see NMI_BITS. + * nmi_enter() can nest - nesting is tracked in a per-CPU counter. */ #define __nmi_enter() \ do { \ lockdep_off(); \ arch_nmi_enter(); \ - BUG_ON(in_nmi() == NMI_MASK); \ - __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ + __preempt_count_nmi_enter(); \ } while (0) #define nmi_enter() \ do { \ __nmi_enter(); \ lockdep_hardirq_enter(); \ - ct_nmi_enter(); \ + ct_nmi_enter(); \ instrumentation_begin(); \ ftrace_nmi_enter(); \ instrumentation_end(); \ @@ -125,7 +155,7 @@ void irq_exit_rcu(void); #define __nmi_exit() \ do { \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ + __preempt_count_nmi_exit(); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h new file mode 100644 index 000000000000..b9a7f05ecf42 --- /dev/null +++ b/include/linux/interrupt_rc.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_INTERRUPT_RC_H +#define __LINUX_INTERRUPT_RC_H + +/* + * include/linux/interrupt_rc.h - refcounted local processor interrupt + * management. + * + * Since the implementation of this API currently depends on + * local_irq_save()/local_irq_restore(), we split this into its own header to + * make it easier to include without hitting circular header dependencies. + */ + +#include <linux/irqflags.h> +#include <linux/preempt.h> +#include <linux/processor.h> +#include <linux/smp.h> + +#ifndef MODULE +/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */ +DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); + +static __always_inline void __local_interrupt_disable(void) +{ + unsigned long flags; + + local_irq_save(flags); + raw_cpu_write(local_interrupt_disable_state, flags); +} + +static __always_inline void __local_interrupt_enable(void) +{ + unsigned long flags = raw_cpu_read(local_interrupt_disable_state); + + local_irq_restore(flags); +} + +#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE +static __always_inline void _local_interrupt_disable(void) +{ + __local_interrupt_disable(); +} + +static __always_inline void _local_interrupt_enable(void) +{ + __local_interrupt_enable(); +} +#else +extern void _local_interrupt_disable(void); +extern void _local_interrupt_enable(void); +#endif + +#else /* !MODULE */ +extern void _local_interrupt_disable(void); +extern void _local_interrupt_enable(void); +#endif /* !MODULE */ + +static inline void local_interrupt_disable(void) +{ + int new_count; + + WARN_ON_ONCE(in_nmi()); + + new_count = hardirq_disable_enter(); + + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ + + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) + _local_interrupt_disable(); +} + +static inline void local_interrupt_enable(void) +{ + int new_count; + + new_count = hardirq_disable_exit(); + + if ((new_count & HARDIRQ_DISABLE_MASK) == 0) + _local_interrupt_enable(); +} + +#endif /* !__LINUX_INTERRUPT_RC_H */ diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index b9c7b0ebf7b9..c95350a10987 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -71,7 +71,7 @@ * Additional babbling in: Documentation/staging/static-keys.rst */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <linux/compiler.h> @@ -100,12 +100,12 @@ struct static_key { #endif /* CONFIG_JUMP_LABEL */ }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #ifdef CONFIG_JUMP_LABEL #include <asm/jump_label.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE struct jump_entry { @@ -180,7 +180,7 @@ static inline int jump_entry_size(struct jump_entry *entry) #endif #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ enum jump_label_type { JUMP_LABEL_NOP = 0, @@ -524,6 +524,6 @@ extern bool ____wrong_branch_error(void); #define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key) #define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _LINUX_JUMP_LABEL_H */ diff --git a/include/linux/kernel.h b/include/linux/kernel.h index e5570a16cbb1..24414c79e59a 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -72,7 +72,7 @@ extern int dynamic_might_resched(void); #ifdef CONFIG_DEBUG_ATOMIC_SLEEP extern void __might_resched(const char *file, int line, unsigned int offsets); extern void __might_sleep(const char *file, int line); -extern void __cant_sleep(const char *file, int line, int preempt_offset); +extern void __cant_sleep(const char *file, int line); extern void __cant_migrate(const char *file, int line); /** @@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line); * this macro will print a stack trace if it is executed with preemption enabled */ # define cant_sleep() \ - do { __cant_sleep(__FILE__, __LINE__, 0); } while (0) + do { __cant_sleep(__FILE__, __LINE__); } while (0) # define sched_annotate_sleep() (current->task_state_change = 0) /** diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 8a22bc9b8c6c..0af8ae4fdd08 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -13,7 +13,7 @@ #define IND_SOURCE (1 << IND_SOURCE_BIT) #define IND_FLAGS (IND_DESTINATION | IND_INDIRECTION | IND_DONE | IND_SOURCE) -#if !defined(__ASSEMBLY__) +#if !defined(__ASSEMBLER__) #include <linux/vmcore_info.h> #include <linux/crash_reserve.h> diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h index 88722e5caf02..63ea5417de84 100644 --- a/include/linux/liveupdate.h +++ b/include/linux/liveupdate.h @@ -173,7 +173,9 @@ struct liveupdate_flb_ops { * @lock: A mutex that protects all fields within this structure, providing * the synchronization service for the FLB's ops. * @finished: True once the FLB's finish() callback has run. - * @retrieved: True once the FLB's retrieve() callback has run. + * @retrieve_status: Status code indicating whether retrieve() has been + * attempted. 0 means not attempted, 1 means successful, + * and negative value means it failed with that error code. */ struct luo_flb_private_state { refcount_t count; @@ -181,7 +183,7 @@ struct luo_flb_private_state { void *obj; struct mutex lock; bool finished; - bool retrieved; + int retrieve_status; }; /* @@ -243,6 +245,7 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp); void liveupdate_flb_put_incoming(struct liveupdate_flb *flb); int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp); +void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb); #else /* CONFIG_LIVEUPDATE */ @@ -292,5 +295,9 @@ static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, return -EOPNOTSUPP; } +static inline void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb) +{ +} + #endif /* CONFIG_LIVEUPDATE */ #endif /* _LINUX_LIVEUPDATE_H */ diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 7c9d66729c60..06c58cb05779 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -289,7 +289,7 @@ static inline void __remove_memory(u64 start, u64 size) {} /* Default online_type (MMOP_*) when new memory blocks are added. */ extern enum mmop mhp_get_default_online_type(void); extern void mhp_set_default_online_type(enum mmop online_type); -extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat); +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat); extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 9a00e701454c..af2e68e496e5 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -7,7 +7,7 @@ #ifdef CONFIG_OBJTOOL -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define UNWIND_HINT(type, sp_reg, sp_offset, signal) \ "987: \n\t" \ @@ -53,7 +53,7 @@ #define __ASM_BREF(label) label ## b -#else /* __ASSEMBLY__ */ +#else /* __ASSEMBLER__ */ /* * In asm, there are two kinds of code: normal C-type callable functions and @@ -102,11 +102,11 @@ #endif .endm -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #else /* !CONFIG_OBJTOOL */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define UNWIND_HINT(type, sp_reg, sp_offset, signal) "\n\t" #define STACK_FRAME_NON_STANDARD(func) diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h index c6def4049b1a..c24e9ea39269 100644 --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -2,7 +2,7 @@ #ifndef _LINUX_OBJTOOL_TYPES_H #define _LINUX_OBJTOOL_TYPES_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> @@ -18,7 +18,7 @@ struct unwind_hint { u8 signal; }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* * UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 1c9d40e09107..2c17239aacea 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2640,6 +2640,8 @@ #define PCI_VENDOR_ID_SUNIX 0x1fd4 #define PCI_DEVICE_ID_SUNIX_1999 0x1999 +#define PCI_VENDOR_ID_PICOHEART 0x20fa + #define PCI_VENDOR_ID_HINT 0x3388 #define PCI_DEVICE_ID_HINT_VXPROII_IDE 0x8013 diff --git a/include/linux/pcie-dwc.h b/include/linux/pcie-dwc.h index 8ff778e7aec0..d3cd701a58e3 100644 --- a/include/linux/pcie-dwc.h +++ b/include/linux/pcie-dwc.h @@ -26,6 +26,8 @@ static const struct dwc_pcie_vsec_id dwc_pcie_rasdes_vsec_ids[] = { .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_AMPERE, .vsec_id = 0x02, .vsec_rev = 0x4 }, + { .vendor_id = PCI_VENDOR_ID_PICOHEART, + .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_QCOM, .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_ROCKCHIP, diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h index 52b37f7bdbf9..02d2c7f45b52 100644 --- a/include/linux/perf/arm_pmu.h +++ b/include/linux/perf/arm_pmu.h @@ -119,7 +119,7 @@ struct arm_pmu { /* PMUv3 only */ int pmuver; - bool has_smt; + bool avoid_pmccntr; u64 reg_pmmir; u64 reg_brbidr; #define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40 diff --git a/include/linux/preempt.h b/include/linux/preempt.h index d964f965c8ff..8299657f0f86 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,6 +17,9 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) + * - bits 16-23 are the hardirq disable count (max # of hardirq disable: 256) + * - bits 24-27 are the hardirq count (max # of hardirqs: 16) + * - bit 28 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -24,31 +27,56 @@ * there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * + * NMI nesting depth is tracked in a separate per-CPU variable + * (nmi_nesting) to save bits in preempt_count. + * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 - * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00f00000 + * HARDIRQ_DISABLE_MASK: 0x00ff0000 + * HARDIRQ_MASK: 0x0f000000 + * + * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=y, PREEMPT_NEED_RESCHED is put in a + * separate word and that allows 64bit load-store architectures to 'set' + * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric + * modifications used on preempt_count and still load the whole thing + * (single-copy) atomically, without having to resort to full atomic + * operations. + * + * Because of the above, NMI_MASK bits are different depending on + * HAS_SEPARATE_PREEMPT_RESCHED_BITS: + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=n: + * + * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=y: + * NMI_MASK: 0xf0000000 + * (PREEMPT_NEED_RESCHED is in a different word) */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS)) #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) #define __IRQ_MASK(x) ((1UL << (x))-1) #define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT) #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) #define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT) #define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_OFFSET (1UL << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) #define NMI_OFFSET (1UL << NMI_SHIFT) @@ -105,8 +133,8 @@ static __always_inline unsigned char interrupt_context_level(void) * preempt_count() is commonly implemented with READ_ONCE(). */ -#define nmi_count() (preempt_count() & NMI_MASK) -#define hardirq_count() (preempt_count() & HARDIRQ_MASK) +#define nmi_count() (preempt_count() & NMI_MASK) +#define hardirq_count() (preempt_count() & HARDIRQ_MASK) #ifdef CONFIG_PREEMPT_RT # define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK) # define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count()) @@ -140,6 +168,10 @@ static __always_inline unsigned char interrupt_context_level(void) #define in_softirq() (softirq_count()) #define in_interrupt() (irq_count()) +#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT) +#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET) +#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET) + /* * The preempt_count offset after preempt_disable(); */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 1e4136c2b2a3..3188fc78ac9b 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -550,7 +550,6 @@ struct sched_statistics { s64 exec_max; u64 slice_max; - u64 nr_migrations_cold; u64 nr_failed_migrations_affine; u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; @@ -563,8 +562,6 @@ struct sched_statistics { u64 nr_wakeups_remote; u64 nr_wakeups_affine; u64 nr_wakeups_affine_attempts; - u64 nr_wakeups_passive; - u64 nr_wakeups_idle; #ifdef CONFIG_SCHED_CORE u64 core_forceidle_sum; @@ -575,6 +572,7 @@ struct sched_statistics { struct sched_entity { /* For load-balancing: */ struct load_weight load; + struct load_weight h_load; struct rb_node run_node; u64 deadline; u64 min_vruntime; diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 42839cfa2778..dc3ec2452ee1 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -146,8 +146,7 @@ SD_FLAG(SD_ASYM_PACKING, SDF_NEEDS_GROUPS) /* * Prefer to place tasks in a sibling domain * - * Set up until domains start spanning NUMA nodes. Close to being a SHARED_CHILD - * flag, but cleared below domains with SD_ASYM_CPUCAPACITY. + * Set up until domains start spanning NUMA nodes. * * NEEDS_GROUPS: Load balancing flag. */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 241277cd34cf..799a8f7d2741 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -57,6 +57,7 @@ #include <linux/linkage.h> #include <linux/compiler.h> #include <linux/irqflags.h> +#include <linux/interrupt_rc.h> #include <linux/thread_info.h> #include <linux/stringify.h> #include <linux/bottom_half.h> @@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #endif #define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock) +#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock) #define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock) #define raw_spin_unlock(lock) _raw_spin_unlock(lock) #define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock) +#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock) #define raw_spin_unlock_irqrestore(lock, flags) \ do { \ @@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lock, &(flags)) +#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(lock) + #ifndef CONFIG_PREEMPT_RT /* Include rwlock functions for !RT */ #include <linux/rwlock.h> @@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock) raw_spin_lock_irq(&lock->rlock); } +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + raw_spin_lock_irq_disable(&lock->rlock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ raw_spin_lock_irqsave(spinlock_check(lock), flags); \ @@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock) raw_spin_unlock_irq(&lock->rlock); } +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) __no_context_analysis +{ + raw_spin_unlock_irq_enable(&lock->rlock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock) __no_context_analysis { @@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned lon } #define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags)) +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) __no_context_analysis +{ + return raw_spin_trylock_irq_disable(&lock->rlock); +} + /** * spin_is_locked() - Check whether a spinlock is locked. * @lock: Pointer to the spinlock. @@ -549,12 +572,12 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acquires(_T), __releases(*(raw #define class_raw_spinlock_nested_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, _T) DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t, - raw_spin_lock_irq(_T->lock), - raw_spin_unlock_irq(_T->lock)) + raw_spin_lock_irq_disable(_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, _T) -DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->lock)) +DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, _T) @@ -569,14 +592,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, __acquires(_T), __releases(*(raw #define class_raw_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, _T) DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t, - raw_spin_lock_irqsave(_T->lock, _T->flags), - raw_spin_unlock_irqrestore(_T->lock, _T->flags), - unsigned long flags) + raw_spin_lock_irq_disable(_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, _T) DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try, - raw_spin_trylock_irqsave(_T->lock, _T->flags)) + raw_spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, _T) @@ -595,13 +617,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_T), __releases(*(spinlock_t #define class_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_try, _T) DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t, - spin_lock_irq(_T->lock), - spin_unlock_irq(_T->lock)) + spin_lock_irq_disable(_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spinlock_t **)_T)) #define class_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq, _T) DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try, - spin_trylock_irq(_T->lock)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(spinlock_t **)_T)) #define class_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq_try, _T) @@ -617,14 +639,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh_try, __acquires(_T), __releases(*(spinloc #define class_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_bh_try, _T) DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t, - spin_lock_irqsave(_T->lock, _T->flags), - spin_unlock_irqrestore(_T->lock, _T->flags), - unsigned long flags) + spin_lock_irq_disable(_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave, __acquires(_T), __releases(*(spinlock_t **)_T)) #define class_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave, _T) DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try, - spin_trylock_irqsave(_T->lock, _T->flags)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __releases(*(spinlock_t **)_T)) #define class_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, _T) diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index bda5e7a390cd..90909d933dab 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map) void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock); void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) __acquires(lock); +void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock); unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) __acquires(lock); @@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(true, void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock); +void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) __releases(lock); @@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) #define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock) #endif +/* Use the same config as spin_lock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ +#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock) +#endif + #ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE #define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock) #endif @@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) #define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock) #endif +/* Use the same config as spin_unlock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ +#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(lock) +#endif + #ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE #define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrestore(lock, flags) #endif @@ -105,6 +118,16 @@ static __always_inline bool _raw_spin_trylock_irq(raw_spinlock_t *lock) return false; } +static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + local_interrupt_disable(); + if (_raw_spin_trylock(lock)) + return true; + local_interrupt_enable(); + return false; +} + static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags) __cond_acquires(true, lock) { @@ -143,6 +166,15 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock) LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); } +static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + local_interrupt_disable(); + preempt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); + LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); +} + static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock) __no_context_analysis { @@ -188,6 +220,15 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock) preempt_enable(); } +static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock) + __releases(lock) +{ + spin_release(&lock->dep_map, _RET_IP_); + do_raw_spin_unlock(lock); + local_interrupt_enable(); + preempt_enable(); +} + static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock) { diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h index a9d5c7c66e03..d03d3065ee04 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -42,6 +42,9 @@ #define __LOCK_IRQSAVE(lock, flags, ...) \ do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0) +#define __LOCK_IRQ_DISABLE(lock, ...) \ + do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0) + #define ___UNLOCK_(lock) \ do { __release(lock); (void)(lock); } while (0) @@ -61,6 +64,9 @@ #define __UNLOCK_IRQRESTORE(lock, flags, ...) \ do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0) +#define __UNLOCK_IRQ_ENABLE(lock, ...) \ + do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0) + #define _raw_spin_lock(lock) __LOCK(lock) #define _raw_spin_lock_nested(lock, subclass) __LOCK(lock) #define _raw_read_lock(lock) __LOCK(lock, shared) @@ -70,6 +76,7 @@ #define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared) #define _raw_write_lock_bh(lock) __LOCK_BH(lock) #define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock) +#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock) #define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared) #define _raw_write_lock_irq(lock) __LOCK_IRQ(lock) #define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) @@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock) return 1; } +static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + __LOCK_IRQ_DISABLE(lock); + return 1; +} + static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags) __cond_acquires(true, lock) { @@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned l #define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock) #define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared) #define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock) +#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock) #define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared) #define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock) #define _raw_spin_unlock_irqrestore(lock, flags) \ diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 373618a4243c..560d06384e0c 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock) rt_spin_lock(lock); } +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) +{ + rt_spin_lock(lock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ @@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock) rt_spin_unlock(lock); } +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) +{ + rt_spin_unlock(lock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock) @@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, #define spin_trylock(lock) rt_spin_trylock(lock) +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) +{ + return rt_spin_trylock(lock); +} + #define spin_trylock_bh(lock) rt_spin_trylock_bh(lock) #define spin_trylock_irq(lock) rt_spin_trylock(lock) diff --git a/include/linux/srcu.h b/include/linux/srcu.h index a54ce9e808b9..72c86d3b23f2 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -638,4 +638,11 @@ DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct, DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) #define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T) +DEFINE_LOCK_GUARD_1(srcu_fast_updown, struct srcu_struct, + _T->scp = srcu_read_lock_fast_updown(_T->lock), + srcu_read_unlock_fast_updown(_T->lock, _T->scp), + struct srcu_ctr __percpu *scp) +DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_updown, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) +#define class_srcu_fast_updown_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_updown, _T) + #endif diff --git a/include/linux/static_call_types.h b/include/linux/static_call_types.h index cfb6ddeb292b..ac55bc966a56 100644 --- a/include/linux/static_call_types.h +++ b/include/linux/static_call_types.h @@ -25,7 +25,7 @@ #define STATIC_CALL_SITE_INIT 2UL /* init section */ #define STATIC_CALL_SITE_FLAGS 3UL -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * The static call site table needs to be created by external tooling (objtool @@ -102,6 +102,6 @@ struct static_call_key { #endif /* CONFIG_HAVE_STATIC_CALL */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _STATIC_CALL_TYPES_H */ diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h index 01011113d226..84e7fb627ba4 100644 --- a/include/linux/stop_machine.h +++ b/include/linux/stop_machine.h @@ -2,6 +2,7 @@ #ifndef _LINUX_STOP_MACHINE #define _LINUX_STOP_MACHINE +#include <linux/bug.h> #include <linux/cpu.h> #include <linux/cpumask_types.h> #include <linux/smp.h> @@ -31,7 +32,7 @@ struct cpu_stop_work { int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg); int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg); -bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, +void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, struct cpu_stop_work *work_buf); void stop_machine_park(int cpu); void stop_machine_unpark(int cpu); @@ -68,19 +69,17 @@ static void stop_one_cpu_nowait_workfn(struct work_struct *work) preempt_enable(); } -static inline bool stop_one_cpu_nowait(unsigned int cpu, +static inline void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, struct cpu_stop_work *work_buf) { - if (cpu == smp_processor_id()) { - INIT_WORK(&work_buf->work, stop_one_cpu_nowait_workfn); - work_buf->fn = fn; - work_buf->arg = arg; - schedule_work(&work_buf->work); - return true; - } - - return false; + if (WARN_ON_ONCE(cpu != smp_processor_id())) + return; + + INIT_WORK(&work_buf->work, stop_one_cpu_nowait_workfn); + work_buf->fn = fn; + work_buf->arg = arg; + schedule_work(&work_buf->work); } static inline void print_stop_info(const char *log_lvl, struct task_struct *task) { } diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec..d34dbc0fbbfe 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -25,6 +25,7 @@ struct mm_struct; struct inode; struct notifier_block; struct page; +struct srcu_ctr; /* * Allowed return values from uprobe consumer's handler callback @@ -106,7 +107,7 @@ enum hprobe_state { * underlying uprobe is not guaranteed anymore. __UPROBE_DEAD is just an * internal marker and is handled transparently by hprobe_fetch() helper. * - * When uprobe is SRCU-protected, we also record srcu_idx value, necessary for + * When uprobe is SRCU-protected, we also record srcu_scp value, necessary for * SRCU unlocking. * * See hprobe_expire() and hprobe_fetch() for details of race-free uprobe @@ -115,7 +116,7 @@ enum hprobe_state { */ struct hprobe { enum hprobe_state state; - int srcu_idx; + struct srcu_ctr __percpu *srcu_scp; struct uprobe *uprobe; }; @@ -186,9 +187,6 @@ struct xol_area; struct uprobes_state { struct xol_area *xol_area; -#ifdef CONFIG_X86_64 - struct hlist_head head_tramps; -#endif }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, @@ -238,8 +236,6 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs); extern void *arch_uretprobe_trampoline(unsigned long *psize); extern unsigned long uprobe_get_trampoline_vaddr(void); extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len); -extern void arch_uprobe_clear_state(struct mm_struct *mm); -extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); extern unsigned long arch_uprobe_get_xol_area(void); diff --git a/include/trace/events/lock.h b/include/trace/events/lock.h index 1ded869cd619..b1d5b18c4514 100644 --- a/include/trace/events/lock.h +++ b/include/trace/events/lock.h @@ -137,7 +137,11 @@ TRACE_EVENT(contention_end, TP_printk("%p (ret=%d)", __entry->lock_addr, __entry->ret) ); -TRACE_EVENT(contended_release, +/* kernel/locking/mutex.c */ +int arch_contended_release_trace_reg(void); +void arch_contended_release_trace_unreg(void); + +TRACE_EVENT_FN(contended_release, TP_PROTO(void *lock), @@ -151,7 +155,9 @@ TRACE_EVENT(contended_release, __entry->lock_addr = lock; ), - TP_printk("%p", __entry->lock_addr) + TP_printk("%p", __entry->lock_addr), + + arch_contended_release_trace_reg, arch_contended_release_trace_unreg ); #endif /* _TRACE_LOCK_H */ diff --git a/include/uapi/linux/arm_sdei.h b/include/uapi/linux/arm_sdei.h index af0630ba5437..22eb61612673 100644 --- a/include/uapi/linux/arm_sdei.h +++ b/include/uapi/linux/arm_sdei.h @@ -22,6 +22,7 @@ #define SDEI_1_0_FN_SDEI_PE_UNMASK SDEI_1_0_FN(0x0C) #define SDEI_1_0_FN_SDEI_INTERRUPT_BIND SDEI_1_0_FN(0x0D) #define SDEI_1_0_FN_SDEI_INTERRUPT_RELEASE SDEI_1_0_FN(0x0E) +#define SDEI_1_0_FN_SDEI_EVENT_SIGNAL SDEI_1_0_FN(0x0F) #define SDEI_1_0_FN_SDEI_PRIVATE_RESET SDEI_1_0_FN(0x11) #define SDEI_1_0_FN_SDEI_SHARED_RESET SDEI_1_0_FN(0x12) diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 419011097fa8..70e36e6a0ad4 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -997,6 +997,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_KEYOP 247 #define KVM_CAP_S390_VSIE_ESAMODE 248 #define KVM_CAP_S390_HPAGE_2G 249 +#define KVM_CAP_PPC_COMPAT_CAPS 250 struct kvm_irq_routing_irqchip { __u32 irqchip; @@ -1341,6 +1342,8 @@ struct kvm_s390_keyop { /* Available with KVM_CAP_COUNTER_OFFSET */ #define KVM_ARM_SET_COUNTER_OFFSET _IOW(KVMIO, 0xb5, struct kvm_arm_counter_offset) #define KVM_ARM_GET_REG_WRITABLE_MASKS _IOR(KVMIO, 0xb6, struct reg_mask_range) +/* Available with KVM_CAP_PPC_COMPAT_CAPS */ +#define KVM_PPC_GET_COMPAT_CAPS _IO(KVMIO, 0xb8) /* ioctl for vm fd */ #define KVM_CREATE_DEVICE _IOWR(KVMIO, 0xe0, struct kvm_create_device) diff --git a/include/xen/platform_pci.h b/include/xen/platform_pci.h index e51e7cb71a85..267040c1f504 100644 --- a/include/xen/platform_pci.h +++ b/include/xen/platform_pci.h @@ -30,7 +30,7 @@ static inline int xen_must_unplug_nics(void) { #if (defined(CONFIG_XEN_NETDEV_FRONTEND) || \ defined(CONFIG_XEN_NETDEV_FRONTEND_MODULE)) && \ - defined(CONFIG_XEN_PVHVM) + defined(CONFIG_X86) return 1; #else return 0; @@ -40,14 +40,14 @@ static inline int xen_must_unplug_nics(void) { static inline int xen_must_unplug_disks(void) { #if (defined(CONFIG_XEN_BLKDEV_FRONTEND) || \ defined(CONFIG_XEN_BLKDEV_FRONTEND_MODULE)) && \ - defined(CONFIG_XEN_PVHVM) + defined(CONFIG_X86) return 1; #else return 0; #endif } -#if defined(CONFIG_XEN_PVHVM) +#if defined(CONFIG_X86) extern bool xen_has_pv_devices(void); extern bool xen_has_pv_disk_devices(void); extern bool xen_has_pv_nic_devices(void); diff --git a/include/xen/xen-ops.h b/include/xen/xen-ops.h index 496e6013c689..15e0c3f4b7bb 100644 --- a/include/xen/xen-ops.h +++ b/include/xen/xen-ops.h @@ -59,7 +59,6 @@ static inline int xen_remap_pfn(struct vm_area_struct *vma, unsigned long addr, struct vm_area_struct; -#ifdef CONFIG_XEN_AUTO_XLATE int xen_xlate_remap_gfn_array(struct vm_area_struct *vma, unsigned long addr, xen_pfn_t *gfn, int nr, @@ -68,27 +67,6 @@ int xen_xlate_remap_gfn_array(struct vm_area_struct *vma, struct page **pages); int xen_xlate_unmap_gfn_range(struct vm_area_struct *vma, int nr, struct page **pages); -#else -/* - * These two functions are called from arch/x86/xen/mmu.c and so stubs - * are needed for a configuration not specifying CONFIG_XEN_AUTO_XLATE. - */ -static inline int xen_xlate_remap_gfn_array(struct vm_area_struct *vma, - unsigned long addr, - xen_pfn_t *gfn, int nr, - int *err_ptr, pgprot_t prot, - unsigned int domid, - struct page **pages) -{ - return -EOPNOTSUPP; -} - -static inline int xen_xlate_unmap_gfn_range(struct vm_area_struct *vma, - int nr, struct page **pages) -{ - return -EOPNOTSUPP; -} -#endif int xen_remap_vma_range(struct vm_area_struct *vma, unsigned long addr, unsigned long len); |