diff options
49 files changed, 690 insertions, 569 deletions
@@ -75,6 +75,9 @@ Andreas Herrmann <aherrman@de.ibm.com> Andreas Hindborg <a.hindborg@kernel.org> <a.hindborg@samsung.com> Andrej Shadura <andrew.shadura@collabora.co.uk> Andrej Shadura <andrew@shadura.me> <andrew@beldisplaytech.com> +Andrew Donnellan <andrew+kernel@donnellan.id.au> <andrew@donnellan.id.au> +Andrew Donnellan <andrew+kernel@donnellan.id.au> <ajd@linux.ibm.com> +Andrew Donnellan <andrew+kernel@donnellan.id.au> <andrew.donnellan@au1.ibm.com> Andrew Morton <akpm@linux-foundation.org> Andrew Murray <amurray@thegoodpenguin.co.uk> <amurray@embedded-bits.co.uk> Andrew Murray <amurray@thegoodpenguin.co.uk> <andrew.murray@arm.com> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 10240cb80904..60b9862d530e 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -172,6 +172,7 @@ config PPC select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx + select ARCH_SUPPORTS_HUGE_PFNMAP if PPC_BOOK3S_64 && TRANSPARENT_HUGEPAGE select ARCH_SUPPORTS_PAGE_TABLE_CHECK if !HUGETLB_PAGE select ARCH_SUPPORTS_SCHED_MC if SMP select ARCH_SUPPORTS_SCHED_SMT if PPC64 && SMP @@ -188,6 +189,7 @@ config PPC select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP if PPC_RADIX_MMU select ARCH_WANTS_MODULES_DATA_IN_VMALLOC if PPC_BOOK3S_32 || PPC_8xx select ARCH_WEAK_RELEASE_ACQUIRE + select AUDIT_ARCH_COMPAT_GENERIC select BINFMT_ELF select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS @@ -370,10 +372,6 @@ config GENERIC_TBSYNC bool default y if PPC32 && SMP -config AUDIT_ARCH - bool - default y - config GENERIC_BUG bool default y diff --git a/arch/powerpc/boot/dts/wii.dts b/arch/powerpc/boot/dts/wii.dts index e46143c32308..473d9feb9cdb 100644 --- a/arch/powerpc/boot/dts/wii.dts +++ b/arch/powerpc/boot/dts/wii.dts @@ -29,7 +29,7 @@ bootargs = "root=/dev/mmcblk0p2 rootwait udbg-immortal"; }; - memory { + memory@0 { device_type = "memory"; reg = <0x00000000 0x01800000 /* MEM1 24MB 1T-SRAM */ 0x10000000 0x04000000>; /* MEM2 64MB GDDR3 */ @@ -246,7 +246,7 @@ compatible = "gpio-leds"; /* This is the blue LED in the disk drive slot */ - drive-slot { + led-0 { label = "wii:blue:drive_slot"; gpios = <&GPIO 5 GPIO_ACTIVE_HIGH>; panic-indicator; @@ -256,13 +256,13 @@ gpio-keys { compatible = "gpio-keys"; - power { + button-power { label = "Power Button"; gpios = <&GPIO 0 GPIO_ACTIVE_HIGH>; linux,code = <KEY_POWER>; }; - eject { + button-eject { label = "Eject Button"; gpios = <&GPIO 6 GPIO_ACTIVE_HIGH>; linux,code = <KEY_EJECTCD>; diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 1a91762b455d..1b8916618f89 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -107,8 +107,8 @@ * in here, on radix we expect them to be zero. */ #define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \ - _PAGE_ACCESSED | H_PAGE_THP_HUGE | _PAGE_PTE | \ - _PAGE_SOFT_DIRTY) + _PAGE_ACCESSED | H_PAGE_THP_HUGE | _PAGE_SPECIAL | \ + _PAGE_PTE | _PAGE_SOFT_DIRTY) /* * user access blocked by key */ @@ -1289,6 +1289,29 @@ static inline pud_t pud_mkhuge(pud_t pud) return pud; } +#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP +static inline bool pmd_special(pmd_t pmd) +{ + return pte_special(pmd_pte(pmd)); +} + +static inline pmd_t pmd_mkspecial(pmd_t pmd) +{ + return pte_pmd(pte_mkspecial(pmd_pte(pmd))); +} +#endif + +#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP +static inline bool pud_special(pud_t pud) +{ + return pte_special(pud_pte(pud)); +} + +static inline pud_t pud_mkspecial(pud_t pud) +{ + return pte_pud(pte_mkspecial(pud_pte(pud))); +} +#endif #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS extern int pmdp_set_access_flags(struct vm_area_struct *vma, @@ -1313,12 +1336,27 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, { pmd_t old_pmd; + /* + * Non-present PMDs can be migration entries or device-private THP + * entries. This can happen at 2 places: + * - When the address space is being unmapped zap_huge_pmd(), and we + * encounter non-present pmds. + * - migrate_vma_collect_huge_pmd() could calls this during migration + * of device-private pmd entries. + */ + if (!pmd_present(*pmdp)) { + old_pmd = READ_ONCE(*pmdp); + pmd_clear(pmdp); + goto out; + } + if (radix_enabled()) { old_pmd = radix__pmdp_huge_get_and_clear(mm, addr, pmdp); } else { old_pmd = hash__pmdp_huge_get_and_clear(mm, addr, pmdp); } +out: page_table_check_pmd_clear(mm, addr, old_pmd); return old_pmd; @@ -1400,7 +1438,6 @@ static inline bool arch_needs_pgtable_deposit(void) return false; return true; } -extern void serialize_against_pte_lookup(struct mm_struct *mm); #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index a38542259fab..de9b96660582 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -92,7 +92,6 @@ extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad #define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr) #define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p) #endif -extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr); extern void radix__flush_tlb_all(void); diff --git a/arch/powerpc/include/asm/checksum.h b/arch/powerpc/include/asm/checksum.h index 4b573a3b7e17..52921ea2494a 100644 --- a/arch/powerpc/include/asm/checksum.h +++ b/arch/powerpc/include/asm/checksum.h @@ -8,6 +8,7 @@ #include <linux/bitops.h> #include <linux/in6.h> +#include <linux/uaccess.h> /* * Computes the checksum of a memory block at src, length len, * and adds in "sum" (32-bit), while copying the block to dst. @@ -21,11 +22,24 @@ extern __wsum csum_partial_copy_generic(const void *src, void *dst, int len); #define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER -extern __wsum csum_and_copy_from_user(const void __user *src, void *dst, - int len); +static inline __wsum csum_and_copy_from_user(const void __user *src, void *dst, int len) +{ + scoped_user_read_access_size(src, len, efault) + return csum_partial_copy_generic((void __force *)src, dst, len); + +efault: + return 0; +} + #define HAVE_CSUM_COPY_USER -extern __wsum csum_and_copy_to_user(const void *src, void __user *dst, - int len); +static inline __wsum csum_and_copy_to_user(const void *src, void __user *dst, int len) +{ + scoped_user_write_access_size(dst, len, efault) + return csum_partial_copy_generic(src, (void __force *)dst, len); + +efault: + return 0; +} #define _HAVE_ARCH_CSUM_AND_COPY #define csum_partial_copy_nocheck(src, dst, len) \ diff --git a/arch/powerpc/include/asm/futex.h b/arch/powerpc/include/asm/futex.h index b3001f8b2c1e..8cf3b2e97e17 100644 --- a/arch/powerpc/include/asm/futex.h +++ b/arch/powerpc/include/asm/futex.h @@ -33,8 +33,7 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, { int oldval = 0, ret; - if (!user_access_begin(uaddr, sizeof(u32))) - return -EFAULT; + uaddr = masked_user_access_begin(uaddr); switch (op) { case FUTEX_OP_SET: @@ -69,8 +68,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, int ret = 0; u32 prev; - if (!user_access_begin(uaddr, sizeof(u32))) - return -EFAULT; + uaddr = masked_user_access_begin(uaddr); __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index bd4a6c42a5f3..e02710d6a2e1 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -66,11 +66,9 @@ void relocate_new_kernel(unsigned long indirection_page, unsigned long reboot_co unsigned long start_address) __noreturn; void kexec_copy_flush(struct kimage *image); -#ifdef CONFIG_KEXEC_FILE -extern const struct kexec_file_ops kexec_elf64_ops; +#if defined(CONFIG_KEXEC_FILE) || defined(CONFIG_CRASH_DUMP) #define ARCH_HAS_KIMAGE_ARCH - struct kimage_arch { struct crash_mem *exclude_ranges; @@ -78,6 +76,10 @@ struct kimage_arch { void *backup_buf; void *fdt; }; +#endif + +#ifdef CONFIG_KEXEC_FILE +extern const struct kexec_file_ops kexec_elf64_ops; char *setup_kdump_cmdline(struct kimage *image, char *cmdline, unsigned long cmdline_len); @@ -145,6 +147,10 @@ int arch_crash_hotplug_support(struct kimage *image, unsigned long kexec_flags); unsigned int arch_crash_get_elfcorehdr_size(void); #define crash_get_elfcorehdr_size arch_crash_get_elfcorehdr_size + +int machine_kexec_post_load(struct kimage *image); +#define machine_kexec_post_load machine_kexec_post_load + #endif /* CONFIG_CRASH_HOTPLUG */ extern int crashing_cpu; @@ -159,6 +165,8 @@ extern void default_machine_crash_shutdown(struct pt_regs *regs); extern void crash_kexec_prepare(void); extern void crash_kexec_secondary(struct pt_regs *regs); +extern void sync_backup_region_phdr(struct kimage *image, Elf64_Ehdr *ehdr, + bool phdr_to_kimage); static inline bool kdump_in_progress(void) { return crashing_cpu >= 0; diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index dcd3a88caaf6..97ccfa6e3dde 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -63,6 +63,20 @@ static inline pgprot_t pte_pgprot(pte_t pte) return __pgprot(pte_flags); } +#ifdef CONFIG_PPC64 +#define pmd_pgprot pmd_pgprot +static inline pgprot_t pmd_pgprot(pmd_t pmd) +{ + return pte_pgprot(pmd_pte(pmd)); +} + +#define pud_pgprot pud_pgprot +static inline pgprot_t pud_pgprot(pud_t pud) +{ + return pte_pgprot(pud_pte(pud)); +} +#endif /* CONFIG_PPC64 */ + static inline pgprot_t pgprot_nx(pgprot_t prot) { return pte_pgprot(pte_exprotect(__pte(pgprot_val(prot)))); diff --git a/arch/powerpc/include/asm/ps3.h b/arch/powerpc/include/asm/ps3.h index 987e23a2bd28..b090ceb32a69 100644 --- a/arch/powerpc/include/asm/ps3.h +++ b/arch/powerpc/include/asm/ps3.h @@ -65,6 +65,7 @@ struct ps3_dma_region_ops; /** * struct ps3_dma_region - A per device dma state variables structure + * @dev: device structure * @did: The HV device id. * @page_size: The ioc pagesize. * @region_type: The HV region type. @@ -108,15 +109,15 @@ struct ps3_dma_region_ops { dma_addr_t bus_addr, unsigned long len); }; + +struct ps3_system_bus_device; + /** * struct ps3_dma_region_init - Helper to initialize structure variables * * Helper to properly initialize variables prior to calling * ps3_system_bus_device_register. */ - -struct ps3_system_bus_device; - int ps3_dma_region_init(struct ps3_system_bus_device *dev, struct ps3_dma_region *r, enum ps3_dma_page_size page_size, enum ps3_dma_region_type region_type, void *addr, unsigned long len); @@ -156,10 +157,12 @@ struct ps3_mmio_region_ops { int (*free)(struct ps3_mmio_region *); }; /** - * struct ps3_mmio_region_init - Helper to initialize structure variables + * ps3_mmio_region_init - Helper to initialize structure variables * * Helper to properly initialize variables prior to calling * ps3_system_bus_device_register. + * + * Returns: %0 on success, %-errno on error (or BUG()) */ int ps3_mmio_region_init(struct ps3_system_bus_device *dev, @@ -405,7 +408,7 @@ static inline struct ps3_system_bus_driver * } /** - * ps3_system_bus_set_drvdata - + * ps3_system_bus_set_drvdata - set driver's private data for this device * @dev: device structure * @data: Data to set */ @@ -464,7 +467,7 @@ enum ps3_lpm_rights { * enum ps3_lpm_tb_type - Type of trace buffer lv1 should use. * * @PS3_LPM_TB_TYPE_NONE: Do not use a trace buffer. - * @PS3_LPM_RIGHTS_USE_TB: Use the lv1 internal trace buffer. Must have + * @PS3_LPM_TB_TYPE_INTERNAL: Use the lv1 internal trace buffer. Must have * rights @PS3_LPM_RIGHTS_USE_TB. */ diff --git a/arch/powerpc/include/asm/unistd32.h b/arch/powerpc/include/asm/unistd32.h new file mode 100644 index 000000000000..07689897d206 --- /dev/null +++ b/arch/powerpc/include/asm/unistd32.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_UNISTD32_H_ +#define _ASM_POWERPC_UNISTD32_H_ + +#include <asm/unistd_32.h> + +#endif /* _ASM_POWERPC_UNISTD32_H_ */ diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 2f0a2e69c607..7bf6b16b2d93 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -149,9 +149,6 @@ obj-$(CONFIG_PCI) += pci_$(BITS).o $(pci64-y) \ pci-common.o pci_of_scan.o obj-$(CONFIG_PCI_MSI) += msi.o -obj-$(CONFIG_AUDIT) += audit.o -obj64-$(CONFIG_AUDIT) += compat_audit.o - obj-y += trace/ ifneq ($(CONFIG_PPC_INDIRECT_PIO),y) diff --git a/arch/powerpc/kernel/align.c b/arch/powerpc/kernel/align.c index 3e37ece06739..61409431138f 100644 --- a/arch/powerpc/kernel/align.c +++ b/arch/powerpc/kernel/align.c @@ -165,25 +165,23 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg, temp.ll = data.ll = 0; p = addr; - if (!user_read_access_begin(addr, nb)) - return -EFAULT; - - switch (nb) { - case 8: - unsafe_get_user(temp.v[0], p++, Efault_read); - unsafe_get_user(temp.v[1], p++, Efault_read); - unsafe_get_user(temp.v[2], p++, Efault_read); - unsafe_get_user(temp.v[3], p++, Efault_read); - fallthrough; - case 4: - unsafe_get_user(temp.v[4], p++, Efault_read); - unsafe_get_user(temp.v[5], p++, Efault_read); - fallthrough; - case 2: - unsafe_get_user(temp.v[6], p++, Efault_read); - unsafe_get_user(temp.v[7], p++, Efault_read); + scoped_user_read_access_size(addr, nb, efault) { + switch (nb) { + case 8: + unsafe_get_user(temp.v[0], p++, efault); + unsafe_get_user(temp.v[1], p++, efault); + unsafe_get_user(temp.v[2], p++, efault); + unsafe_get_user(temp.v[3], p++, efault); + fallthrough; + case 4: + unsafe_get_user(temp.v[4], p++, efault); + unsafe_get_user(temp.v[5], p++, efault); + fallthrough; + case 2: + unsafe_get_user(temp.v[6], p++, efault); + unsafe_get_user(temp.v[7], p++, efault); + } } - user_read_access_end(); switch (instr) { case EVLDD: @@ -252,25 +250,23 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg, if (flags & ST) { p = addr; - if (!user_write_access_begin(addr, nb)) - return -EFAULT; - - switch (nb) { - case 8: - unsafe_put_user(data.v[0], p++, Efault_write); - unsafe_put_user(data.v[1], p++, Efault_write); - unsafe_put_user(data.v[2], p++, Efault_write); - unsafe_put_user(data.v[3], p++, Efault_write); - fallthrough; - case 4: - unsafe_put_user(data.v[4], p++, Efault_write); - unsafe_put_user(data.v[5], p++, Efault_write); - fallthrough; - case 2: - unsafe_put_user(data.v[6], p++, Efault_write); - unsafe_put_user(data.v[7], p++, Efault_write); + scoped_user_write_access_size(addr, nb, efault) { + switch (nb) { + case 8: + unsafe_put_user(data.v[0], p++, efault); + unsafe_put_user(data.v[1], p++, efault); + unsafe_put_user(data.v[2], p++, efault); + unsafe_put_user(data.v[3], p++, efault); + fallthrough; + case 4: + unsafe_put_user(data.v[4], p++, efault); + unsafe_put_user(data.v[5], p++, efault); + fallthrough; + case 2: + unsafe_put_user(data.v[6], p++, efault); + unsafe_put_user(data.v[7], p++, efault); + } } - user_write_access_end(); } else { *evr = data.w[0]; regs->gpr[reg] = data.w[1]; @@ -278,12 +274,7 @@ static int emulate_spe(struct pt_regs *regs, unsigned int reg, return 1; -Efault_read: - user_read_access_end(); - return -EFAULT; - -Efault_write: - user_write_access_end(); +efault: return -EFAULT; } #endif /* CONFIG_SPE */ diff --git a/arch/powerpc/kernel/audit.c b/arch/powerpc/kernel/audit.c deleted file mode 100644 index 92298d6a3a37..000000000000 --- a/arch/powerpc/kernel/audit.c +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <linux/init.h> -#include <linux/types.h> -#include <linux/audit.h> -#include <asm/unistd.h> - -#include "audit_32.h" - -static unsigned dir_class[] = { -#include <asm-generic/audit_dir_write.h> -~0U -}; - -static unsigned read_class[] = { -#include <asm-generic/audit_read.h> -~0U -}; - -static unsigned write_class[] = { -#include <asm-generic/audit_write.h> -~0U -}; - -static unsigned chattr_class[] = { -#include <asm-generic/audit_change_attr.h> -~0U -}; - -static unsigned signal_class[] = { -#include <asm-generic/audit_signal.h> -~0U -}; - -int audit_classify_arch(int arch) -{ -#ifdef CONFIG_PPC64 - if (arch == AUDIT_ARCH_PPC) - return 1; -#endif - return 0; -} - -int audit_classify_syscall(int abi, unsigned syscall) -{ -#ifdef CONFIG_PPC64 - if (abi == AUDIT_ARCH_PPC) - return ppc32_classify_syscall(syscall); -#endif - switch(syscall) { - case __NR_open: - return AUDITSC_OPEN; - case __NR_openat: - return AUDITSC_OPENAT; - case __NR_socketcall: - return AUDITSC_SOCKETCALL; - case __NR_execve: - return AUDITSC_EXECVE; - case __NR_openat2: - return AUDITSC_OPENAT2; - default: - return AUDITSC_NATIVE; - } -} - -static int __init audit_classes_init(void) -{ -#ifdef CONFIG_PPC64 - extern __u32 ppc32_dir_class[]; - extern __u32 ppc32_write_class[]; - extern __u32 ppc32_read_class[]; - extern __u32 ppc32_chattr_class[]; - extern __u32 ppc32_signal_class[]; - audit_register_class(AUDIT_CLASS_WRITE_32, ppc32_write_class); - audit_register_class(AUDIT_CLASS_READ_32, ppc32_read_class); - audit_register_class(AUDIT_CLASS_DIR_WRITE_32, ppc32_dir_class); - audit_register_class(AUDIT_CLASS_CHATTR_32, ppc32_chattr_class); - audit_register_class(AUDIT_CLASS_SIGNAL_32, ppc32_signal_class); -#endif - audit_register_class(AUDIT_CLASS_WRITE, write_class); - audit_register_class(AUDIT_CLASS_READ, read_class); - audit_register_class(AUDIT_CLASS_DIR_WRITE, dir_class); - audit_register_class(AUDIT_CLASS_CHATTR, chattr_class); - audit_register_class(AUDIT_CLASS_SIGNAL, signal_class); - return 0; -} - -__initcall(audit_classes_init); diff --git a/arch/powerpc/kernel/compat_audit.c b/arch/powerpc/kernel/compat_audit.c deleted file mode 100644 index b4d81a57b2d9..000000000000 --- a/arch/powerpc/kernel/compat_audit.c +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <linux/audit_arch.h> -#include <asm/unistd_32.h> - -#include "audit_32.h" - -unsigned ppc32_dir_class[] = { -#include <asm-generic/audit_dir_write.h> -~0U -}; - -unsigned ppc32_chattr_class[] = { -#include <asm-generic/audit_change_attr.h> -~0U -}; - -unsigned ppc32_write_class[] = { -#include <asm-generic/audit_write.h> -~0U -}; - -unsigned ppc32_read_class[] = { -#include <asm-generic/audit_read.h> -~0U -}; - -unsigned ppc32_signal_class[] = { -#include <asm-generic/audit_signal.h> -~0U -}; - -int ppc32_classify_syscall(unsigned syscall) -{ - switch(syscall) { - case __NR_open: - return AUDITSC_OPEN; - case __NR_openat: - return AUDITSC_OPENAT; - case __NR_socketcall: - return AUDITSC_SOCKETCALL; - case __NR_execve: - return AUDITSC_EXECVE; - case __NR_openat2: - return AUDITSC_OPENAT2; - default: - return AUDITSC_COMPAT; - } -} diff --git a/arch/powerpc/kernel/kgdb.c b/arch/powerpc/kernel/kgdb.c index 5081334b7bd2..861db2334db8 100644 --- a/arch/powerpc/kernel/kgdb.c +++ b/arch/powerpc/kernel/kgdb.c @@ -101,7 +101,6 @@ static int computeSignal(unsigned int tt) } /** - * * kgdb_skipexception - Bail out of KGDB when we've been triggered. * @exception: Exception vector number * @regs: Current &struct pt_regs. @@ -109,6 +108,8 @@ static int computeSignal(unsigned int tt) * On some architectures we need to skip a breakpoint exception when * it occurs after a breakpoint has been removed. * + * Return: return %1 if the breakpoint for this address has been removed, + * otherwise return %0 */ int kgdb_skipexception(int exception, struct pt_regs *regs) { diff --git a/arch/powerpc/kernel/ptrace/ptrace-view.c b/arch/powerpc/kernel/ptrace/ptrace-view.c index 0310f9097e39..eb5f2091bb59 100644 --- a/arch/powerpc/kernel/ptrace/ptrace-view.c +++ b/arch/powerpc/kernel/ptrace/ptrace-view.c @@ -758,38 +758,37 @@ static int gpr32_set_common_user(struct task_struct *target, const void *kbuf = NULL; compat_ulong_t reg; - if (!user_read_access_begin(u, count)) - return -EFAULT; - - pos /= sizeof(reg); - count /= sizeof(reg); - - for (; count > 0 && pos < PT_MSR; --count) { - unsafe_get_user(reg, u++, Efault); - regs[pos++] = reg; - } - - if (count > 0 && pos == PT_MSR) { - unsafe_get_user(reg, u++, Efault); - set_user_msr(target, reg); - ++pos; - --count; - } - - for (; count > 0 && pos <= PT_MAX_PUT_REG; --count) { - unsafe_get_user(reg, u++, Efault); - regs[pos++] = reg; - } - for (; count > 0 && pos < PT_TRAP; --count, ++pos) - unsafe_get_user(reg, u++, Efault); - - if (count > 0 && pos == PT_TRAP) { - unsafe_get_user(reg, u++, Efault); - set_user_trap(target, reg); - ++pos; - --count; + scoped_user_read_access_size(ubuf, count, efault) { + u = ubuf; + pos /= sizeof(reg); + count /= sizeof(reg); + + for (; count > 0 && pos < PT_MSR; --count) { + unsafe_get_user(reg, u++, efault); + regs[pos++] = reg; + } + + if (count > 0 && pos == PT_MSR) { + unsafe_get_user(reg, u++, efault); + set_user_msr(target, reg); + ++pos; + --count; + } + + for (; count > 0 && pos <= PT_MAX_PUT_REG; --count) { + unsafe_get_user(reg, u++, efault); + regs[pos++] = reg; + } + for (; count > 0 && pos < PT_TRAP; --count, ++pos) + unsafe_get_user(reg, u++, efault); + + if (count > 0 && pos == PT_TRAP) { + unsafe_get_user(reg, u++, efault); + set_user_trap(target, reg); + ++pos; + --count; + } } - user_read_access_end(); ubuf = u; pos *= sizeof(reg); @@ -798,8 +797,7 @@ static int gpr32_set_common_user(struct task_struct *target, (PT_TRAP + 1) * sizeof(reg), -1); return 0; -Efault: - user_read_access_end(); +efault: return -EFAULT; } diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index b1761909c23f..8a86b0efcb1c 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -865,6 +865,10 @@ static __init void print_system_info(void) cur_cpu_spec->cpu_user_features, cur_cpu_spec->cpu_user_features2); pr_info("mmu_features = 0x%08x\n", cur_cpu_spec->mmu_features); + pr_info(" possible = 0x%016lx\n", + (unsigned long)MMU_FTRS_POSSIBLE); + pr_info(" always = 0x%016lx\n", + (unsigned long)MMU_FTRS_ALWAYS); #ifdef CONFIG_PPC64 pr_info("firmware_features = 0x%016lx\n", powerpc_firmware_features); #ifdef CONFIG_PPC_BOOK3S diff --git a/arch/powerpc/kexec/crash.c b/arch/powerpc/kexec/crash.c index a325c1c02f96..e6539f213b3d 100644 --- a/arch/powerpc/kexec/crash.c +++ b/arch/powerpc/kexec/crash.c @@ -27,6 +27,7 @@ #include <asm/debug.h> #include <asm/interrupt.h> #include <asm/kexec_ranges.h> +#include <asm/crashdump-ppc64.h> /* * The primary CPU waits a while for all secondary CPUs to enter. This is to @@ -399,7 +400,68 @@ void default_machine_crash_shutdown(struct pt_regs *regs) ppc_md.kexec_cpu_down(1, 0); } +#ifdef CONFIG_CRASH_DUMP +/** + * sync_backup_region_phdr - synchronize backup region offset between + * kexec image and ELF core header. + * @image: Kexec image. + * @ehdr: ELF core header. + * @phdr_to_kimage: If true, read the offset from the ELF program header + * and update the kimage backup region. If false, update + * the ELF program header offset from the kimage backup + * region. + * + * Note: During kexec_load, this is called with phdr_to_kimage = true. For + * kexec_file_load and ELF core header recreation during memory hotplug + * events, it is called with phdr_to_kimage = false. + * + * Returns nothing. + */ +void sync_backup_region_phdr(struct kimage *image, Elf64_Ehdr *ehdr, bool phdr_to_kimage) +{ + Elf64_Phdr *phdr; + unsigned int i; + + phdr = (Elf64_Phdr *)(ehdr + 1); + for (i = 0; i < ehdr->e_phnum; i++, phdr++) { + if (phdr->p_paddr == BACKUP_SRC_START) { + if (phdr_to_kimage) + image->arch.backup_start = phdr->p_offset; + else + phdr->p_offset = image->arch.backup_start; + + kexec_dprintk("Backup region offset updated to 0x%lx\n", + image->arch.backup_start); + return; + } + } +} +#endif /* CONFIG_CRASH_DUMP */ + #ifdef CONFIG_CRASH_HOTPLUG + +int machine_kexec_post_load(struct kimage *image) +{ + int i; + unsigned long mem; + unsigned char *ptr; + + if (image->type != KEXEC_TYPE_CRASH) + return 0; + + if (image->file_mode) + return 0; + + for (i = 0; i < image->nr_segments; i++) { + mem = image->segment[i].mem; + ptr = (char *)__va(mem); + + if (ptr && memcmp(ptr, ELFMAG, SELFMAG) == 0) + sync_backup_region_phdr(image, (Elf64_Ehdr *) ptr, true); + } + return 0; +} + #undef pr_fmt #define pr_fmt(fmt) "crash hp: " fmt @@ -474,6 +536,8 @@ static void update_crash_elfcorehdr(struct kimage *image, struct memory_notify * goto out; } + sync_backup_region_phdr(image, (Elf64_Ehdr *) elfbuf, false); + ptr = __va(mem); if (ptr) { /* Temporarily invalidate the crash image while it is replaced */ diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index 5f6d50e4c3d4..8c72e12ea44e 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -374,33 +374,6 @@ static int load_backup_segment(struct kimage *image, struct kexec_buf *kbuf) return 0; } -/** - * update_backup_region_phdr - Update backup region's offset for the core to - * export the region appropriately. - * @image: Kexec image. - * @ehdr: ELF core header. - * - * Assumes an exclusive program header is setup for the backup region - * in the ELF headers - * - * Returns nothing. - */ -static void update_backup_region_phdr(struct kimage *image, Elf64_Ehdr *ehdr) -{ - Elf64_Phdr *phdr; - unsigned int i; - - phdr = (Elf64_Phdr *)(ehdr + 1); - for (i = 0; i < ehdr->e_phnum; i++) { - if (phdr->p_paddr == BACKUP_SRC_START) { - phdr->p_offset = image->arch.backup_start; - kexec_dprintk("Backup region offset updated to 0x%lx\n", - image->arch.backup_start); - return; - } - } -} - static unsigned int kdump_extra_elfcorehdr_size(struct crash_mem *cmem) { #if defined(CONFIG_CRASH_HOTPLUG) && defined(CONFIG_MEMORY_HOTPLUG) @@ -445,7 +418,7 @@ static int load_elfcorehdr_segment(struct kimage *image, struct kexec_buf *kbuf) } /* Fix the offset for backup region in the ELF header */ - update_backup_region_phdr(image, headers); + sync_backup_region_phdr(image, headers, false); kbuf->buffer = headers; kbuf->mem = KEXEC_BUF_MEM_UNKNOWN; diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index f14ecab674a3..bcdf387f3998 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -62,8 +62,7 @@ obj64-$(CONFIG_ALTIVEC) += vmx-helper.o obj64-$(CONFIG_KPROBES_SANITY_TEST) += test_emulate_step.o \ test_emulate_step_exec_instr.o -obj-y += checksum_$(BITS).o checksum_wrappers.o \ - string_$(BITS).o +obj-y += checksum_$(BITS).o string_$(BITS).o obj-y += sstep.o obj-$(CONFIG_PPC_FPU) += ldstfp.o diff --git a/arch/powerpc/lib/checksum_wrappers.c b/arch/powerpc/lib/checksum_wrappers.c deleted file mode 100644 index 1a14c8780278..000000000000 --- a/arch/powerpc/lib/checksum_wrappers.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * - * Copyright (C) IBM Corporation, 2010 - * - * Author: Anton Blanchard <anton@au.ibm.com> - */ -#include <linux/export.h> -#include <linux/compiler.h> -#include <linux/types.h> -#include <asm/checksum.h> -#include <linux/uaccess.h> - -__wsum csum_and_copy_from_user(const void __user *src, void *dst, - int len) -{ - __wsum csum; - - if (unlikely(!user_read_access_begin(src, len))) - return 0; - - csum = csum_partial_copy_generic((void __force *)src, dst, len); - - user_read_access_end(); - return csum; -} - -__wsum csum_and_copy_to_user(const void *src, void __user *dst, int len) -{ - __wsum csum; - - if (unlikely(!user_write_access_begin(dst, len))) - return 0; - - csum = csum_partial_copy_generic(src, (void __force *)dst, len); - - user_write_access_end(); - return csum; -} diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c index ac3ee19531d8..f0d6aa657c1a 100644 --- a/arch/powerpc/lib/sstep.c +++ b/arch/powerpc/lib/sstep.c @@ -329,20 +329,17 @@ Efault: static nokprobe_inline int read_mem_aligned(unsigned long *dest, unsigned long ea, int nb, struct pt_regs *regs) { - int err; + void __user *uea = (void __user *)ea; if (is_kernel_addr(ea)) return __read_mem_aligned(dest, ea, nb, regs); - if (user_read_access_begin((void __user *)ea, nb)) { - err = __read_mem_aligned(dest, ea, nb, regs); - user_read_access_end(); - } else { - err = -EFAULT; - regs->dar = ea; - } + scoped_user_read_access_size(uea, nb, efault) + return __read_mem_aligned(dest, (unsigned long)uea, nb, regs); - return err; +efault: + regs->dar = ea; + return -EFAULT; } /* @@ -385,20 +382,17 @@ Efault: static nokprobe_inline int copy_mem_in(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs) { - int err; + void __user *uea = (void __user *)ea; if (is_kernel_addr(ea)) return __copy_mem_in(dest, ea, nb, regs); - if (user_read_access_begin((void __user *)ea, nb)) { - err = __copy_mem_in(dest, ea, nb, regs); - user_read_access_end(); - } else { - err = -EFAULT; - regs->dar = ea; - } + scoped_user_read_access_size(uea, nb, efault) + return __copy_mem_in(dest, (unsigned long)uea, nb, regs); - return err; +efault: + regs->dar = ea; + return -EFAULT; } static nokprobe_inline int read_mem_unaligned(unsigned long *dest, @@ -465,20 +459,17 @@ Efault: static nokprobe_inline int write_mem_aligned(unsigned long val, unsigned long ea, int nb, struct pt_regs *regs) { - int err; + void __user *uea = (void __user *)ea; if (is_kernel_addr(ea)) return __write_mem_aligned(val, ea, nb, regs); - if (user_write_access_begin((void __user *)ea, nb)) { - err = __write_mem_aligned(val, ea, nb, regs); - user_write_access_end(); - } else { - err = -EFAULT; - regs->dar = ea; - } + scoped_user_write_access_size(uea, nb, efault) + return __write_mem_aligned(val, (unsigned long)uea, nb, regs); - return err; +efault: + regs->dar = ea; + return -EFAULT; } /* @@ -521,20 +512,17 @@ Efault: static nokprobe_inline int copy_mem_out(u8 *dest, unsigned long ea, int nb, struct pt_regs *regs) { - int err; + void __user *uea = (void __user *)ea; if (is_kernel_addr(ea)) return __copy_mem_out(dest, ea, nb, regs); - if (user_write_access_begin((void __user *)ea, nb)) { - err = __copy_mem_out(dest, ea, nb, regs); - user_write_access_end(); - } else { - err = -EFAULT; - regs->dar = ea; - } + scoped_user_write_access_size(uea, nb, efault) + return __copy_mem_out(dest, (unsigned long)uea, nb, regs); - return err; +efault: + regs->dar = ea; + return -EFAULT; } static nokprobe_inline int write_mem_unaligned(unsigned long val, @@ -1065,6 +1053,7 @@ Efault: int emulate_dcbz(unsigned long ea, struct pt_regs *regs) { + void __user *uea = (void __user *)ea; int err; unsigned long size = l1_dcache_bytes(); @@ -1073,20 +1062,20 @@ int emulate_dcbz(unsigned long ea, struct pt_regs *regs) if (!address_ok(regs, ea, size)) return -EFAULT; - if (is_kernel_addr(ea)) { - err = __emulate_dcbz(ea); - } else if (user_write_access_begin((void __user *)ea, size)) { + if (is_kernel_addr(ea)) err = __emulate_dcbz(ea); - user_write_access_end(); - } else { - err = -EFAULT; - } + else + scoped_user_write_access_size(uea, size, efault) + err = __emulate_dcbz((unsigned long)uea); if (err) regs->dar = ea; - return err; + +efault: + regs->dar = ea; + return -EFAULT; } NOKPROBE_SYMBOL(emulate_dcbz); diff --git a/arch/powerpc/mm/book3s64/hash_pgtable.c b/arch/powerpc/mm/book3s64/hash_pgtable.c index ac2a24d15d2e..d9b5b751d7b7 100644 --- a/arch/powerpc/mm/book3s64/hash_pgtable.c +++ b/arch/powerpc/mm/book3s64/hash_pgtable.c @@ -221,6 +221,27 @@ unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr return old; } +static void do_nothing(void *arg) +{ + +} + +/* + * Serialize against __find_linux_pte() which does lock-less + * lookup in page tables with local interrupts disabled. For huge pages + * it casts pmd_t to pte_t. Since format of pte_t is different from + * pmd_t we want to prevent transit from pmd pointing to page table + * to pmd pointing to huge page (and back) while interrupts are disabled. + * We clear pmd to possibly replace it with page table pointer in + * different code paths. So make sure we wait for the parallel + * __find_linux_pte() to finish. + */ +static void serialize_against_pte_lookup(struct mm_struct *mm) +{ + smp_mb(); + smp_call_function_many(mm_cpumask(mm), do_nothing, mm, 1); +} + pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { diff --git a/arch/powerpc/mm/book3s64/internal.h b/arch/powerpc/mm/book3s64/internal.h index cad08d83369c..f7055251c8b7 100644 --- a/arch/powerpc/mm/book3s64/internal.h +++ b/arch/powerpc/mm/book3s64/internal.h @@ -31,6 +31,4 @@ static inline bool slb_preload_disabled(void) void hpt_do_stress(unsigned long ea, unsigned long hpte_group); -void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush); - #endif /* ARCH_POWERPC_MM_BOOK3S64_INTERNAL_H */ diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 4b09c04654a8..d32197d3298a 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -23,8 +23,6 @@ #include <mm/mmu_decl.h> #include <trace/events/thp.h> -#include "internal.h" - struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; EXPORT_SYMBOL_GPL(mmu_psize_defs); @@ -150,31 +148,6 @@ void set_pud_at(struct mm_struct *mm, unsigned long addr, return set_pte_at_unchecked(mm, addr, pudp_ptep(pudp), pud_pte(pud)); } -static void do_serialize(void *arg) -{ - /* We've taken the IPI, so try to trim the mask while here */ - if (radix_enabled()) { - struct mm_struct *mm = arg; - exit_lazy_flush_tlb(mm, false); - } -} - -/* - * Serialize against __find_linux_pte() which does lock-less - * lookup in page tables with local interrupts disabled. For huge pages - * it casts pmd_t to pte_t. Since format of pte_t is different from - * pmd_t we want to prevent transit from pmd pointing to page table - * to pmd pointing to huge page (and back) while interrupts are disabled. - * We clear pmd to possibly replace it with page table pointer in - * different code paths. So make sure we wait for the parallel - * __find_linux_pte() to finish. - */ -void serialize_against_pte_lookup(struct mm_struct *mm) -{ - smp_mb(); - smp_call_function_many(mm_cpumask(mm), do_serialize, mm, 1); -} - /* * We use this to invalidate a pmdp entry before switching from a * hugepte to regular pmd entry. @@ -209,16 +182,21 @@ pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp, int full) { pmd_t pmd; + bool was_present = pmd_present(*pmdp); + VM_BUG_ON(addr & ~HPAGE_PMD_MASK); - VM_BUG_ON((pmd_present(*pmdp) && !pmd_trans_huge(*pmdp)) || - !pmd_present(*pmdp)); + VM_BUG_ON(was_present && !pmd_trans_huge(*pmdp)); + /* + * Check pmdp_huge_get_and_clear() for non-present pmd case. + */ pmd = pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp); /* * if it not a fullmm flush, then we can possibly end up converting * this PMD pte entry to a regular level 0 PTE by a parallel page fault. - * Make sure we flush the tlb in this case. + * Make sure we flush the tlb in this case. TLB flush not needed for + * non-present case. */ - if (!full) + if (was_present && !full) flush_pmd_tlb_range(vma, addr, addr + HPAGE_PMD_SIZE); return pmd; } diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index 9e1f6558d026..7de5760164a9 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -19,8 +19,6 @@ #include <asm/cputhreads.h> #include <asm/plpar_wrappers.h> -#include "internal.h" - /* * tlbiel instruction for radix, set invalidation * i.e., r=1 and is=01 or is=10 or is=11 @@ -187,7 +185,7 @@ static __always_inline void __tlbie_va(unsigned long va, unsigned long pid, trace_tlbie(0, 0, rb, rs, ric, prs, r); } -static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid, +static __always_inline void __tlbie_va_lpid(unsigned long va, unsigned long lpid, unsigned long ap, unsigned long ric) { unsigned long rb,rs,prs,r; @@ -251,17 +249,17 @@ static inline void fixup_tlbie_pid(unsigned long pid) } } -static inline void fixup_tlbie_lpid_va(unsigned long va, unsigned long lpid, +static inline void fixup_tlbie_va_lpid(unsigned long va, unsigned long lpid, unsigned long ap) { if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { asm volatile("ptesync": : :"memory"); - __tlbie_lpid_va(va, 0, ap, RIC_FLUSH_TLB); + __tlbie_va_lpid(va, 0, ap, RIC_FLUSH_TLB); } if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { asm volatile("ptesync": : :"memory"); - __tlbie_lpid_va(va, lpid, ap, RIC_FLUSH_TLB); + __tlbie_va_lpid(va, lpid, ap, RIC_FLUSH_TLB); } } @@ -280,7 +278,7 @@ static inline void fixup_tlbie_lpid(unsigned long lpid) if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { asm volatile("ptesync": : :"memory"); - __tlbie_lpid_va(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); + __tlbie_va_lpid(va, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); } } @@ -531,14 +529,14 @@ static void do_tlbiel_va_range(void *info) t->psize, t->also_pwc); } -static __always_inline void _tlbie_lpid_va(unsigned long va, unsigned long lpid, +static __always_inline void _tlbie_va_lpid(unsigned long va, unsigned long lpid, unsigned long psize, unsigned long ric) { unsigned long ap = mmu_get_ap(psize); asm volatile("ptesync": : :"memory"); - __tlbie_lpid_va(va, lpid, ap, ric); - fixup_tlbie_lpid_va(va, lpid, ap); + __tlbie_va_lpid(va, lpid, ap, ric); + fixup_tlbie_va_lpid(va, lpid, ap); asm volatile("eieio; tlbsync; ptesync": : :"memory"); } @@ -660,7 +658,7 @@ static bool mm_needs_flush_escalation(struct mm_struct *mm) * If always_flush is true, then flush even if this CPU can't be removed * from mm_cpumask. */ -void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush) +static void exit_lazy_flush_tlb(struct mm_struct *mm) { unsigned long pid = mm->context.id; int cpu = smp_processor_id(); @@ -703,19 +701,17 @@ void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush) if (cpumask_test_cpu(cpu, mm_cpumask(mm))) { dec_mm_active_cpus(mm); cpumask_clear_cpu(cpu, mm_cpumask(mm)); - always_flush = true; } out: - if (always_flush) - _tlbiel_pid(pid, RIC_FLUSH_ALL); + _tlbiel_pid(pid, RIC_FLUSH_ALL); } #ifdef CONFIG_SMP static void do_exit_flush_lazy_tlb(void *arg) { struct mm_struct *mm = arg; - exit_lazy_flush_tlb(mm, true); + exit_lazy_flush_tlb(mm); } static void exit_flush_lazy_tlbs(struct mm_struct *mm) @@ -777,7 +773,7 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm) * to trim. */ if (tick_and_test_trim_clock()) { - exit_lazy_flush_tlb(mm, true); + exit_lazy_flush_tlb(mm); return FLUSH_TYPE_NONE; } } @@ -823,7 +819,7 @@ static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm) if (current->mm == mm) return FLUSH_TYPE_LOCAL; if (cpumask_test_cpu(cpu, mm_cpumask(mm))) - exit_lazy_flush_tlb(mm, true); + exit_lazy_flush_tlb(mm); return FLUSH_TYPE_NONE; } @@ -889,8 +885,7 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) } else if (type == FLUSH_TYPE_GLOBAL) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt = H_RPTI_TARGET_CMMU; - unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | - H_RPTI_TYPE_PRT; + unsigned long type = H_RPTI_TYPE_ALL; if (atomic_read(&mm->context.copros) > 0) tgt |= H_RPTI_TARGET_NMMU; @@ -986,8 +981,7 @@ void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt = H_RPTI_TARGET_CMMU | H_RPTI_TARGET_NMMU; - unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | - H_RPTI_TYPE_PRT; + unsigned long type = H_RPTI_TYPE_ALL; pseries_rpt_invalidate(0, tgt, type, H_RPTI_PAGE_ALL, start, end); @@ -1151,7 +1145,7 @@ void radix__flush_tlb_lpid_page(unsigned int lpid, { int psize = radix_get_mmu_psize(page_size); - _tlbie_lpid_va(addr, lpid, psize, RIC_FLUSH_TLB); + _tlbie_va_lpid(addr, lpid, psize, RIC_FLUSH_TLB); } EXPORT_SYMBOL_GPL(radix__flush_tlb_lpid_page); @@ -1341,8 +1335,7 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) unsigned long tgt, type, pg_sizes; tgt = H_RPTI_TARGET_CMMU; - type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | - H_RPTI_TYPE_PRT; + type = H_RPTI_TYPE_ALL; pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize); if (atomic_read(&mm->context.copros) > 0) @@ -1415,7 +1408,7 @@ static __always_inline void __tlbie_pid_lpid(unsigned long pid, trace_tlbie(0, 0, rb, rs, ric, prs, r); } -static __always_inline void __tlbie_va_lpid(unsigned long va, unsigned long pid, +static __always_inline void __tlbie_va_pid_lpid(unsigned long va, unsigned long pid, unsigned long lpid, unsigned long ap, unsigned long ric) { @@ -1447,7 +1440,7 @@ static inline void fixup_tlbie_pid_lpid(unsigned long pid, unsigned long lpid) if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { asm volatile("ptesync" : : : "memory"); - __tlbie_va_lpid(va, pid, lpid, mmu_get_ap(MMU_PAGE_64K), + __tlbie_va_pid_lpid(va, pid, lpid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB); } } @@ -1478,7 +1471,7 @@ static inline void _tlbie_pid_lpid(unsigned long pid, unsigned long lpid, asm volatile("eieio; tlbsync; ptesync" : : : "memory"); } -static inline void fixup_tlbie_va_range_lpid(unsigned long va, +static inline void fixup_tlbie_va_range_pid_lpid(unsigned long va, unsigned long pid, unsigned long lpid, unsigned long ap) @@ -1490,11 +1483,11 @@ static inline void fixup_tlbie_va_range_lpid(unsigned long va, if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { asm volatile("ptesync" : : : "memory"); - __tlbie_va_lpid(va, pid, lpid, ap, RIC_FLUSH_TLB); + __tlbie_va_pid_lpid(va, pid, lpid, ap, RIC_FLUSH_TLB); } } -static inline void __tlbie_va_range_lpid(unsigned long start, unsigned long end, +static inline void __tlbie_va_range_pid_lpid(unsigned long start, unsigned long end, unsigned long pid, unsigned long lpid, unsigned long page_size, unsigned long psize) @@ -1503,12 +1496,12 @@ static inline void __tlbie_va_range_lpid(unsigned long start, unsigned long end, unsigned long ap = mmu_get_ap(psize); for (addr = start; addr < end; addr += page_size) - __tlbie_va_lpid(addr, pid, lpid, ap, RIC_FLUSH_TLB); + __tlbie_va_pid_lpid(addr, pid, lpid, ap, RIC_FLUSH_TLB); - fixup_tlbie_va_range_lpid(addr - page_size, pid, lpid, ap); + fixup_tlbie_va_range_pid_lpid(addr - page_size, pid, lpid, ap); } -static inline void _tlbie_va_range_lpid(unsigned long start, unsigned long end, +static inline void _tlbie_va_range_pid_lpid(unsigned long start, unsigned long end, unsigned long pid, unsigned long lpid, unsigned long page_size, unsigned long psize, bool also_pwc) @@ -1516,7 +1509,7 @@ static inline void _tlbie_va_range_lpid(unsigned long start, unsigned long end, asm volatile("ptesync" : : : "memory"); if (also_pwc) __tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC); - __tlbie_va_range_lpid(start, end, pid, lpid, page_size, psize); + __tlbie_va_range_pid_lpid(start, end, pid, lpid, page_size, psize); asm volatile("eieio; tlbsync; ptesync" : : : "memory"); } @@ -1567,7 +1560,7 @@ void do_h_rpt_invalidate_prt(unsigned long pid, unsigned long lpid, _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB); return; } - _tlbie_va_range_lpid(start, end, pid, lpid, + _tlbie_va_range_pid_lpid(start, end, pid, lpid, (1UL << def->shift), psize, false); } } diff --git a/arch/powerpc/mm/pgtable-frag.c b/arch/powerpc/mm/pgtable-frag.c index 77e55eac16e4..ae742564a3d5 100644 --- a/arch/powerpc/mm/pgtable-frag.c +++ b/arch/powerpc/mm/pgtable-frag.c @@ -25,6 +25,7 @@ void pte_frag_destroy(void *pte_frag) count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT; /* We allow PTE_FRAG_NR fragments from a PTE page */ if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { + folio_clear_active(ptdesc_folio(ptdesc)); pagetable_dtor(ptdesc); pagetable_free(ptdesc); } diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 7354e1d72f79..f32de8704d4d 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -178,8 +178,14 @@ struct codegen_context { bool is_subprog; bool exception_boundary; bool exception_cb; + void __percpu *priv_sp; + unsigned int priv_stack_size; }; +/* Memory size & magic-value to detect private stack overflow/underflow */ +#define PRIV_STACK_GUARD_SZ 16 +#define PRIV_STACK_GUARD_VAL 0xEB9F12345678eb9fULL + #define bpf_to_ppc(r) (ctx->b2p[r]) #ifdef CONFIG_PPC32 @@ -212,7 +218,9 @@ void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx); void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx); void bpf_jit_realloc_regs(struct codegen_context *ctx); int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr); - +void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int cookie_cnt, + int cookie_off, int retval_off); +void store_func_meta(u32 *image, struct codegen_context *ctx, u64 func_meta, int func_meta_off); int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, struct codegen_context *ctx, int insn_idx, int jmp_off, int dst_reg, u32 code); diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index a62a9a92b7b5..50103b3794fb 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -129,25 +129,60 @@ bool bpf_jit_needs_zext(void) return true; } +static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size) +{ + int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; + u64 *stack_ptr; + + for_each_possible_cpu(cpu) { + stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu); + stack_ptr[0] = PRIV_STACK_GUARD_VAL; + stack_ptr[1] = PRIV_STACK_GUARD_VAL; + stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL; + stack_ptr[underflow_idx + 1] = PRIV_STACK_GUARD_VAL; + } +} + +static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size, + struct bpf_prog *fp) +{ + int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; + u64 *stack_ptr; + + for_each_possible_cpu(cpu) { + stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu); + if (stack_ptr[0] != PRIV_STACK_GUARD_VAL || + stack_ptr[1] != PRIV_STACK_GUARD_VAL || + stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL || + stack_ptr[underflow_idx + 1] != PRIV_STACK_GUARD_VAL) { + pr_err("BPF private stack overflow/underflow detected for prog %s\n", + bpf_jit_get_prog_name(fp)); + break; + } + } +} + struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) { u32 proglen; u32 alloclen; u8 *image = NULL; - u32 *code_base; - u32 *addrs; - struct powerpc_jit_data *jit_data; + u32 *code_base = NULL; + u32 *addrs = NULL; + struct powerpc_jit_data *jit_data = NULL; struct codegen_context cgctx; int pass; int flen; + int priv_stack_alloc_size; + void __percpu *priv_stack_ptr = NULL; struct bpf_binary_header *fhdr = NULL; struct bpf_binary_header *hdr = NULL; struct bpf_prog *org_fp = fp; - struct bpf_prog *tmp_fp; + struct bpf_prog *tmp_fp = NULL; bool bpf_blinded = false; bool extra_pass = false; u8 *fimage = NULL; - u32 *fcode_base; + u32 *fcode_base = NULL; u32 extable_len; u32 fixup_len; @@ -173,6 +208,26 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) fp->aux->jit_data = jit_data; } + priv_stack_ptr = fp->aux->priv_stack_ptr; + if (!priv_stack_ptr && fp->aux->jits_use_priv_stack) { + /* + * Allocate private stack of size equivalent to + * verifier-calculated stack size plus two memory + * guard regions to detect private stack overflow + * and underflow. + */ + priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + + 2 * PRIV_STACK_GUARD_SZ; + priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL); + if (!priv_stack_ptr) { + fp = org_fp; + goto out_priv_stack; + } + + priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size); + fp->aux->priv_stack_ptr = priv_stack_ptr; + } + flen = fp->len; addrs = jit_data->addrs; if (addrs) { @@ -209,6 +264,19 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) cgctx.is_subprog = bpf_is_subprog(fp); cgctx.exception_boundary = fp->aux->exception_boundary; cgctx.exception_cb = fp->aux->exception_cb; + cgctx.priv_sp = priv_stack_ptr; + cgctx.priv_stack_size = 0; + if (priv_stack_ptr) { + /* + * priv_stack_size required for setting bpf FP inside + * percpu allocation. + * stack_size is marked 0 to prevent allocation on + * general stack and offset calculation don't go for + * a toss in bpf_jit_stack_offsetof() & bpf_jit_stack_local() + */ + cgctx.priv_stack_size = cgctx.stack_size; + cgctx.stack_size = 0; + } /* Scouting faux-generate pass 0 */ if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { @@ -305,7 +373,19 @@ skip_init_ctx: goto out_addrs; } bpf_prog_fill_jited_linfo(fp, addrs); + /* + * On ABI V1, executable code starts after the function + * descriptor, so adjust base accordingly. + */ + bpf_prog_update_insn_ptrs(fp, addrs, + (void *)fimage + FUNCTION_DESCR_SIZE); + out_addrs: + if (!image && priv_stack_ptr) { + fp->aux->priv_stack_ptr = NULL; + free_percpu(priv_stack_ptr); + } +out_priv_stack: kfree(addrs); kfree(jit_data); fp->aux->jit_data = NULL; @@ -419,6 +499,8 @@ void bpf_jit_free(struct bpf_prog *fp) if (fp->jited) { struct powerpc_jit_data *jit_data = fp->aux->jit_data; struct bpf_binary_header *hdr; + void __percpu *priv_stack_ptr; + int priv_stack_alloc_size; /* * If we fail the final pass of JIT (from jit_subprogs), @@ -432,6 +514,13 @@ void bpf_jit_free(struct bpf_prog *fp) } hdr = bpf_jit_binary_pack_hdr(fp); bpf_jit_binary_pack_free(hdr, NULL); + priv_stack_ptr = fp->aux->priv_stack_ptr; + if (priv_stack_ptr) { + priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + + 2 * PRIV_STACK_GUARD_SZ; + priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_size, fp); + free_percpu(priv_stack_ptr); + } WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); } @@ -453,6 +542,22 @@ bool bpf_jit_supports_kfunc_call(void) return IS_ENABLED(CONFIG_PPC64); } +bool bpf_jit_supports_private_stack(void) +{ + return IS_ENABLED(CONFIG_PPC64); +} + +bool bpf_jit_supports_fsession(void) +{ + /* + * TODO: Remove after validating support + * for fsession and trampoline on ppc32. + */ + if (IS_ENABLED(CONFIG_PPC32)) + return -EOPNOTSUPP; + return true; +} + bool bpf_jit_supports_arena(void) { return IS_ENABLED(CONFIG_PPC64); @@ -725,12 +830,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im struct bpf_tramp_links *tlinks, void *func_addr) { - int regs_off, nregs_off, ip_off, run_ctx_off, retval_off, nvr_off, alt_lr_off, r4_off = 0; + int regs_off, func_meta_off, ip_off, run_ctx_off, retval_off; + int nvr_off, alt_lr_off, r4_off = 0; struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; int i, ret, nr_regs, retaddr_off, bpf_frame_size = 0; struct codegen_context codegen_ctx, *ctx; + int cookie_off, cookie_cnt, cookie_ctx_off; + int fsession_cnt = bpf_fsession_cnt(tlinks); + u64 func_meta; u32 *image = (u32 *)rw_image; ppc_inst_t branch_insn; u32 *branches = NULL; @@ -766,9 +875,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * [ reg argN ] * [ ... ] * regs_off [ reg_arg1 ] prog_ctx - * nregs_off [ args count ] ((u64 *)prog_ctx)[-1] + * func_meta_off [ args count ] ((u64 *)prog_ctx)[-1] * ip_off [ traced function ] ((u64 *)prog_ctx)[-2] + * [ stack cookieN ] * [ ... ] + * cookie_off [ stack cookie1 ] * run_ctx_off [ bpf_tramp_run_ctx ] * [ reg argN ] * [ ... ] @@ -800,16 +911,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im run_ctx_off = bpf_frame_size; bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL); + /* room for session cookies */ + cookie_off = bpf_frame_size; + cookie_cnt = bpf_fsession_cookie_cnt(tlinks); + bpf_frame_size += cookie_cnt * 8; + /* Room for IP address argument */ ip_off = bpf_frame_size; if (flags & BPF_TRAMP_F_IP_ARG) bpf_frame_size += SZL; - /* Room for args count */ - nregs_off = bpf_frame_size; + /* Room for function metadata, arg regs count */ + func_meta_off = bpf_frame_size; bpf_frame_size += SZL; - /* Room for args */ + /* Room for arg regs */ regs_off = bpf_frame_size; bpf_frame_size += nr_regs * SZL; @@ -908,9 +1024,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off)); } - /* Save function arg count -- see bpf_get_func_arg_cnt() */ - EMIT(PPC_RAW_LI(_R3, nr_regs)); - EMIT(PPC_RAW_STL(_R3, _R1, nregs_off)); + /* Save function arg regs count -- see bpf_get_func_arg_cnt() */ + func_meta = nr_regs; + store_func_meta(image, ctx, func_meta, func_meta_off); /* Save nv regs */ EMIT(PPC_RAW_STL(_R25, _R1, nvr_off)); @@ -924,10 +1040,28 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im return ret; } - for (i = 0; i < fentry->nr_links; i++) + if (fsession_cnt) { + /* + * Clear all the session cookies' values + * Clear the return value to make sure fentry always get 0 + */ + prepare_for_fsession_fentry(image, ctx, cookie_cnt, cookie_off, retval_off); + } + + cookie_ctx_off = (regs_off - cookie_off) / 8; + + for (i = 0; i < fentry->nr_links; i++) { + if (bpf_prog_calls_session_cookie(fentry->links[i])) { + u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); + + store_func_meta(image, ctx, meta, func_meta_off); + cookie_ctx_off--; + } + if (invoke_bpf_prog(image, ro_image, ctx, fentry->links[i], regs_off, retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) return -EINVAL; + } if (fmod_ret->nr_links) { branches = kcalloc(fmod_ret->nr_links, sizeof(u32), GFP_KERNEL); @@ -989,12 +1123,27 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im image[branches[i]] = ppc_inst_val(branch_insn); } - for (i = 0; i < fexit->nr_links; i++) + /* set the "is_return" flag for fsession */ + func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT); + if (fsession_cnt) + store_func_meta(image, ctx, func_meta, func_meta_off); + + cookie_ctx_off = (regs_off - cookie_off) / 8; + + for (i = 0; i < fexit->nr_links; i++) { + if (bpf_prog_calls_session_cookie(fexit->links[i])) { + u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT); + + store_func_meta(image, ctx, meta, func_meta_off); + cookie_ctx_off--; + } + if (invoke_bpf_prog(image, ro_image, ctx, fexit->links[i], regs_off, retval_off, run_ctx_off, false)) { ret = -EINVAL; goto cleanup; } + } if (flags & BPF_TRAMP_F_CALL_ORIG) { if (ro_image) /* image is NULL for dummy pass */ diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c index 3087e744fb25..bfdc50740da8 100644 --- a/arch/powerpc/net/bpf_jit_comp32.c +++ b/arch/powerpc/net/bpf_jit_comp32.c @@ -123,6 +123,41 @@ void bpf_jit_realloc_regs(struct codegen_context *ctx) } } +void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int cookie_cnt, + int cookie_off, int retval_off) +{ + /* + * Set session cookies value + * Clear cookies field on stack + * Ensure retval to be cleared on fentry + */ + EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG), 0)); + + for (int i = 0; i < cookie_cnt; i++) { + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, cookie_off + 4 * i)); + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, cookie_off + 4 * i + 4)); + } + + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, retval_off)); + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, retval_off + 4)); +} + +void store_func_meta(u32 *image, struct codegen_context *ctx, + u64 func_meta, int func_meta_off) +{ + /* + * Store func_meta to stack: [R1 + func_meta_off] = func_meta + * func_meta := argument count in first byte + cookie value + */ + /* Store lower word */ + PPC_LI32(bpf_to_ppc(TMP_REG), (u32)func_meta); + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, func_meta_off)); + + /* Store upper word */ + PPC_LI32(bpf_to_ppc(TMP_REG), (u32)(func_meta >> 32)); + EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, func_meta_off + 4)); +} + void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) { int i; diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index c5e26d231cd5..db364d9083e7 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -179,10 +179,53 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) BUG(); } +void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int cookie_cnt, + int cookie_off, int retval_off) +{ + EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0)); + + for (int i = 0; i < cookie_cnt; i++) + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, cookie_off + 8 * i)); + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, retval_off)); +} + +void store_func_meta(u32 *image, struct codegen_context *ctx, + u64 func_meta, int func_meta_off) +{ + /* + * Store func_meta to stack at [R1 + func_meta_off] = func_meta + * + * func_meta : + * bit[63]: is_return flag + * byte[1]: cookie offset from ctx + * byte[0]: args count + */ + PPC_LI64(bpf_to_ppc(TMP_REG_1), func_meta); + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, func_meta_off)); +} + void bpf_jit_realloc_regs(struct codegen_context *ctx) { } +static void emit_fp_priv_stack(u32 *image, struct codegen_context *ctx) +{ + PPC_LI64(bpf_to_ppc(BPF_REG_FP), (__force long)ctx->priv_sp); + /* + * Load base percpu pointer of private stack allocation. + * Runtime per-cpu address = (base + data_offset) + (guard + stack_size) + */ +#ifdef CONFIG_SMP + /* Load percpu data offset */ + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R13, + offsetof(struct paca_struct, data_offset))); + EMIT(PPC_RAW_ADD(bpf_to_ppc(BPF_REG_FP), + bpf_to_ppc(TMP_REG_1), bpf_to_ppc(BPF_REG_FP))); +#endif + EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), bpf_to_ppc(BPF_REG_FP), + PRIV_STACK_GUARD_SZ + round_up(ctx->priv_stack_size, 16))); +} + /* * For exception boundary & exception_cb progs: * return increased size to accommodate additional NVRs. @@ -307,9 +350,16 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) * Exception_cb not restricted from using stack area or arena. * Setup frame pointer to point to the bpf stack area */ - if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) - EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1, - STACK_FRAME_MIN_SIZE + ctx->stack_size)); + if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) { + if (ctx->priv_sp) { + /* Set up fp in private stack */ + emit_fp_priv_stack(image, ctx); + } else { + /* Setup frame pointer to point to the bpf stack area */ + EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1, + STACK_FRAME_MIN_SIZE + ctx->stack_size)); + } + } if (ctx->arena_vm_start) PPC_LI64(bpf_to_ppc(ARENA_VM_START), ctx->arena_vm_start); @@ -1659,6 +1709,14 @@ emit_clear: break; /* + * JUMP reg + */ + case BPF_JMP | BPF_JA | BPF_X: + EMIT(PPC_RAW_MTCTR(dst_reg)); + EMIT(PPC_RAW_BCTR()); + break; + + /* * Return/Exit */ case BPF_JMP | BPF_EXIT: diff --git a/arch/powerpc/platforms/44x/uic.c b/arch/powerpc/platforms/44x/uic.c index cf4fc5263c89..3f90126a9056 100644 --- a/arch/powerpc/platforms/44x/uic.c +++ b/arch/powerpc/platforms/44x/uic.c @@ -309,8 +309,8 @@ void __init uic_init_tree(void) cascade_virq = irq_of_parse_and_map(np, 0); - irq_set_handler_data(cascade_virq, uic); - irq_set_chained_handler(cascade_virq, uic_irq_cascade); + irq_set_chained_handler_and_data(cascade_virq, + uic_irq_cascade, uic); /* FIXME: setup critical cascade?? */ } diff --git a/arch/powerpc/platforms/52xx/media5200.c b/arch/powerpc/platforms/52xx/media5200.c index bc7f83cfec1d..c20ac8010f6d 100644 --- a/arch/powerpc/platforms/52xx/media5200.c +++ b/arch/powerpc/platforms/52xx/media5200.c @@ -176,8 +176,8 @@ static void __init media5200_init_irq(void) of_node_put(fpga_np); - irq_set_handler_data(cascade_virq, &media5200_irq); - irq_set_chained_handler(cascade_virq, media5200_irq_cascade); + irq_set_chained_handler_and_data(cascade_virq, media5200_irq_cascade, + &media5200_irq); return; diff --git a/arch/powerpc/platforms/52xx/mpc52xx_gpt.c b/arch/powerpc/platforms/52xx/mpc52xx_gpt.c index 7748b6641a3c..e8163fdee69a 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_gpt.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_gpt.c @@ -253,8 +253,7 @@ mpc52xx_gpt_irq_setup(struct mpc52xx_gpt_priv *gpt, struct device_node *node) return; } - irq_set_handler_data(cascade_virq, gpt); - irq_set_chained_handler(cascade_virq, mpc52xx_gpt_irq_cascade); + irq_set_chained_handler_and_data(cascade_virq, mpc52xx_gpt_irq_cascade, gpt); /* If the GPT is currently disabled, then change it to be in Input * Capture mode. If the mode is non-zero, then the pin could be diff --git a/arch/powerpc/platforms/embedded6xx/Kconfig b/arch/powerpc/platforms/embedded6xx/Kconfig index c6adff216fe6..f406b3c7936b 100644 --- a/arch/powerpc/platforms/embedded6xx/Kconfig +++ b/arch/powerpc/platforms/embedded6xx/Kconfig @@ -51,6 +51,22 @@ config MVME5100 This option enables support for the Motorola (now Emerson) MVME5100 board. +config GAMECUBE + bool "Nintendo-GameCube" + depends on EMBEDDED6xx + select GAMECUBE_COMMON + help + Select GAMECUBE if configuring for the Nintendo GameCube. + More information at: <http://gc-linux.sourceforge.net/> + +config WII + bool "Nintendo-Wii" + depends on EMBEDDED6xx + select GAMECUBE_COMMON + help + Select WII if configuring for the Nintendo Wii. + More information at: <http://gc-linux.sourceforge.net/> + config TSI108_BRIDGE bool select FORCE_PCI @@ -77,18 +93,3 @@ config USBGECKO_UDBG If in doubt, say N here. -config GAMECUBE - bool "Nintendo-GameCube" - depends on EMBEDDED6xx - select GAMECUBE_COMMON - help - Select GAMECUBE if configuring for the Nintendo GameCube. - More information at: <http://gc-linux.sourceforge.net/> - -config WII - bool "Nintendo-Wii" - depends on EMBEDDED6xx - select GAMECUBE_COMMON - help - Select WII if configuring for the Nintendo Wii. - More information at: <http://gc-linux.sourceforge.net/> diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 885392f4cd94..32ecbc46e74b 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -292,18 +292,16 @@ static void pnv_ioda_reserve_m64_pe(struct pci_bus *bus, static struct pnv_ioda_pe *pnv_ioda_pick_m64_pe(struct pci_bus *bus, bool all) { + unsigned long *pe_alloc __free(bitmap) = NULL; struct pnv_phb *phb = pci_bus_to_pnvhb(bus); struct pnv_ioda_pe *master_pe, *pe; - unsigned long size, *pe_alloc; - int i; + unsigned int i; /* Root bus shouldn't use M64 */ if (pci_is_root_bus(bus)) return NULL; - /* Allocate bitmap */ - size = ALIGN(phb->ioda.total_pe_num / 8, sizeof(unsigned long)); - pe_alloc = kzalloc(size, GFP_KERNEL); + pe_alloc = bitmap_zalloc(phb->ioda.total_pe_num, GFP_KERNEL); if (!pe_alloc) { pr_warn("%s: Out of memory !\n", __func__); @@ -314,23 +312,15 @@ static struct pnv_ioda_pe *pnv_ioda_pick_m64_pe(struct pci_bus *bus, bool all) pnv_ioda_reserve_m64_pe(bus, pe_alloc, all); /* - * the current bus might not own M64 window and that's all - * contributed by its child buses. For the case, we needn't - * pick M64 dependent PE#. - */ - if (bitmap_empty(pe_alloc, phb->ioda.total_pe_num)) { - kfree(pe_alloc); - return NULL; - } - - /* * Figure out the master PE and put all slave PEs to master * PE's list to form compound PE. + * + * The current bus might not own M64 window and that's all + * contributed by its child buses. For the case, we needn't + * pick M64 dependent PE#. */ master_pe = NULL; - i = -1; - while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe_num, i + 1)) < - phb->ioda.total_pe_num) { + for_each_set_bit(i, pe_alloc, phb->ioda.total_pe_num) { pe = &phb->ioda.pe_array[i]; phb->ioda.m64_segmap[pe->pe_number] = pe->pe_number; @@ -345,7 +335,6 @@ static struct pnv_ioda_pe *pnv_ioda_pick_m64_pe(struct pci_bus *bus, bool all) } } - kfree(pe_alloc); return master_pe; } diff --git a/arch/powerpc/platforms/ps3/spu.c b/arch/powerpc/platforms/ps3/spu.c index 10ab256b675c..e4e0b45e1b9d 100644 --- a/arch/powerpc/platforms/ps3/spu.c +++ b/arch/powerpc/platforms/ps3/spu.c @@ -26,7 +26,7 @@ /** * enum spe_type - Type of spe to create. - * @spe_type_logical: Standard logical spe. + * @SPE_TYPE_LOGICAL: Standard logical spe. * * For use with lv1_construct_logical_spe(). The current HV does not support * any types other than those listed. @@ -64,9 +64,9 @@ struct spe_shadow { /** * enum spe_ex_state - Logical spe execution state. - * @spe_ex_state_unexecutable: Uninitialized. - * @spe_ex_state_executable: Enabled, not ready. - * @spe_ex_state_executed: Ready for use. + * @SPE_EX_STATE_UNEXECUTABLE: Uninitialized. + * @SPE_EX_STATE_EXECUTABLE: Enabled, not ready. + * @SPE_EX_STATE_EXECUTED: Ready for use. * * The execution state (status) of the logical spe as reported in * struct spe_shadow:spe_execution_status. @@ -185,6 +185,8 @@ static void spu_unmap(struct spu *spu) * * The current HV requires the spu shadow regs to be mapped with the * PTE page protection bits set as read-only. + * + * Returns: %0 on success or -errno on error. */ static int __init setup_areas(struct spu *spu) diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c index e1a4f8a97393..c120be73d149 100644 --- a/arch/powerpc/sysdev/xive/common.c +++ b/arch/powerpc/sysdev/xive/common.c @@ -548,40 +548,21 @@ static void xive_dec_target_count(int cpu) static int xive_find_target_in_mask(const struct cpumask *mask, unsigned int fuzz) { - int cpu, first, num, i; + int cpu, first; /* Pick up a starting point CPU in the mask based on fuzz */ - num = min_t(int, cpumask_weight(mask), nr_cpu_ids); - first = fuzz % num; - - /* Locate it */ - cpu = cpumask_first(mask); - for (i = 0; i < first && cpu < nr_cpu_ids; i++) - cpu = cpumask_next(cpu, mask); - - /* Sanity check */ - if (WARN_ON(cpu >= nr_cpu_ids)) - cpu = cpumask_first(cpu_online_mask); - - /* Remember first one to handle wrap-around */ - first = cpu; + fuzz %= cpumask_weight(mask); + first = cpumask_nth(fuzz, mask); + WARN_ON(first >= nr_cpu_ids); /* * Now go through the entire mask until we find a valid * target. */ - do { - /* - * We re-check online as the fallback case passes us - * an untested affinity mask - */ + for_each_cpu_wrap(cpu, mask, first) { if (cpu_online(cpu) && xive_try_pick_target(cpu)) return cpu; - cpu = cpumask_next(cpu, mask); - /* Wrap around */ - if (cpu >= nr_cpu_ids) - cpu = cpumask_first(mask); - } while (cpu != first); + } return -1; } @@ -1038,13 +1019,19 @@ static struct xive_irq_data *xive_irq_alloc_data(unsigned int virq, irq_hw_numbe return xd; } -static void xive_irq_free_data(unsigned int virq) +static void xive_irq_free_data(struct irq_domain *domain, unsigned int virq) { - struct xive_irq_data *xd = irq_get_chip_data(virq); + struct xive_irq_data *xd; + struct irq_data *data = irq_domain_get_irq_data(domain, virq); + if (!data) + return; + + xd = irq_data_get_irq_chip_data(data); if (!xd) return; - irq_set_chip_data(virq, NULL); + + irq_domain_reset_irq_data(data); xive_cleanup_irq_data(xd); kfree(xd); } @@ -1305,7 +1292,7 @@ static int xive_irq_domain_map(struct irq_domain *h, unsigned int virq, static void xive_irq_domain_unmap(struct irq_domain *d, unsigned int virq) { - xive_irq_free_data(virq); + xive_irq_free_data(d, virq); } static int xive_irq_domain_xlate(struct irq_domain *h, struct device_node *ct, @@ -1443,7 +1430,7 @@ static void xive_irq_domain_free(struct irq_domain *domain, pr_debug("%s %d #%d\n", __func__, virq, nr_irqs); for (i = 0; i < nr_irqs; i++) - xive_irq_free_data(virq + i); + xive_irq_free_data(domain, virq + i); } #endif diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c index 9ebedd972df0..b89e7111e7b8 100644 --- a/drivers/cpuidle/cpuidle-powernv.c +++ b/drivers/cpuidle/cpuidle-powernv.c @@ -95,7 +95,10 @@ static int snooze_loop(struct cpuidle_device *dev, HMT_medium(); ppc64_runlatch_on(); - clear_thread_flag(TIF_POLLING_NRFLAG); + + /* Avoid double clear when breaking */ + if (!dev->poll_time_limit) + clear_thread_flag(TIF_POLLING_NRFLAG); local_irq_disable(); diff --git a/drivers/cpuidle/cpuidle-pseries.c b/drivers/cpuidle/cpuidle-pseries.c index f68c65f1d023..864dd5d6e627 100644 --- a/drivers/cpuidle/cpuidle-pseries.c +++ b/drivers/cpuidle/cpuidle-pseries.c @@ -64,7 +64,10 @@ int snooze_loop(struct cpuidle_device *dev, struct cpuidle_driver *drv, } HMT_medium(); - clear_thread_flag(TIF_POLLING_NRFLAG); + + /* Avoid double clear when breaking */ + if (!dev->poll_time_limit) + clear_thread_flag(TIF_POLLING_NRFLAG); raw_local_irq_disable(); diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 460852f79f29..3fea064d00de 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -1670,21 +1670,16 @@ vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) return VM_FAULT_SIGBUS; - switch (order) { - case 0: + if (!order) return vmf_insert_pfn(vmf->vma, vmf->address, pfn); -#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP - case PMD_ORDER: + + if (IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_PFNMAP) && order == PMD_ORDER) return vmf_insert_pfn_pmd(vmf, pfn, false); -#endif -#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP - case PUD_ORDER: + + if (IS_ENABLED(CONFIG_ARCH_SUPPORTS_PUD_PFNMAP) && order == PUD_ORDER) return vmf_insert_pfn_pud(vmf, pfn, false); - break; -#endif - default: - return VM_FAULT_FALLBACK; - } + + return VM_FAULT_FALLBACK; } EXPORT_SYMBOL_GPL(vfio_pci_vmf_insert_pfn); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c index 269870bec941..482d38b9c29e 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c @@ -3,7 +3,7 @@ #include <bpf/bpf.h> #include <test_progs.h> -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(__powerpc__) static int map_create(__u32 map_type, __u32 max_entries) { const char *map_name = "insn_array"; diff --git a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c index d42123a0fb16..98db9bafa44b 100644 --- a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c +++ b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c @@ -5,6 +5,7 @@ #include "struct_ops_private_stack_fail.skel.h" #include "struct_ops_private_stack_recur.skel.h" +#if defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__) static void test_private_stack(void) { struct struct_ops_private_stack *skel; @@ -15,11 +16,6 @@ static void test_private_stack(void) if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack__open")) return; - if (skel->data->skip) { - test__skip(); - goto cleanup; - } - err = struct_ops_private_stack__load(skel); if (!ASSERT_OK(err, "struct_ops_private_stack__load")) goto cleanup; @@ -48,15 +44,9 @@ static void test_private_stack_fail(void) if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_fail__open")) return; - if (skel->data->skip) { - test__skip(); - goto cleanup; - } - err = struct_ops_private_stack_fail__load(skel); ASSERT_ERR(err, "struct_ops_private_stack_fail__load"); -cleanup: struct_ops_private_stack_fail__destroy(skel); } @@ -70,11 +60,6 @@ static void test_private_stack_recur(void) if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_recur__open")) return; - if (skel->data->skip) { - test__skip(); - goto cleanup; - } - err = struct_ops_private_stack_recur__load(skel); if (!ASSERT_OK(err, "struct_ops_private_stack_recur__load")) goto cleanup; @@ -93,7 +78,7 @@ cleanup: struct_ops_private_stack_recur__destroy(skel); } -void test_struct_ops_private_stack(void) +static void __test_struct_ops_private_stack(void) { if (test__start_subtest("private_stack")) test_private_stack(); @@ -102,3 +87,14 @@ void test_struct_ops_private_stack(void) if (test__start_subtest("private_stack_recur")) test_private_stack_recur(); } +#else +static void __test_struct_ops_private_stack(void) +{ + test__skip(); +} +#endif + +void test_struct_ops_private_stack(void) +{ + __test_struct_ops_private_stack(); +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack.c index dbe646013811..3cd0c1a55cbd 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_private_stack.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack.c @@ -7,12 +7,6 @@ char _license[] SEC("license") = "GPL"; -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) -bool skip __attribute((__section__(".data"))) = false; -#else -bool skip = true; -#endif - void bpf_testmod_ops3_call_test_2(void) __ksym; int val_i, val_j; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c index 3d89ad7cbe2a..1442728f5604 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c @@ -7,12 +7,6 @@ char _license[] SEC("license") = "GPL"; -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) -bool skip __attribute((__section__(".data"))) = false; -#else -bool skip = true; -#endif - void bpf_testmod_ops3_call_test_2(void) __ksym; int val_i, val_j; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c index b1f6d7e5a8e5..faaa0f8d65a4 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c @@ -7,12 +7,6 @@ char _license[] SEC("license") = "GPL"; -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) -bool skip __attribute((__section__(".data"))) = false; -#else -bool skip = true; -#endif - void bpf_testmod_ops3_call_test_1(void) __ksym; int val_i, val_j; diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index 607dad058ca1..0f43b56ec2bc 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -6,7 +6,7 @@ #include "bpf_misc.h" #include "../../../include/linux/filter.h" -#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) +#if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_powerpc) #define DEFINE_SIMPLE_JUMP_TABLE_PROG(NAME, SRC_REG, OFF, IMM, OUTCOME) \ \ @@ -384,6 +384,6 @@ jt0_%=: \ : __clobber_all); } -#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 */ +#endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/ char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/powerpc/vphn/Makefile b/tools/testing/selftests/powerpc/vphn/Makefile index 61d519a076c6..778fc396340d 100644 --- a/tools/testing/selftests/powerpc/vphn/Makefile +++ b/tools/testing/selftests/powerpc/vphn/Makefile @@ -5,7 +5,7 @@ top_srcdir = ../../../../.. include ../../lib.mk include ../flags.mk -CFLAGS += -m64 -I$(CURDIR) +CFLAGS += -m64 -I$(CURDIR) -fno-strict-aliasing $(TEST_GEN_PROGS): ../harness.c |