diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2024-01-18 15:01:28 -0800 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2024-01-18 15:01:28 -0800 |
commit | e7ded27593bf0aff08d18258251e3de0a2697f47 (patch) | |
tree | 183225b3019d1e2db5bb34c670d5e8b0e9c6ea4f /arch | |
parent | 24f3a63e1fc36d5d240c1b3973c75618c20cf458 (diff) | |
parent | 6b9f29b81b155af023da95f560f738f29722b306 (diff) | |
download | lwn-e7ded27593bf0aff08d18258251e3de0a2697f47.tar.gz lwn-e7ded27593bf0aff08d18258251e3de0a2697f47.zip |
Merge tag 'percpu-for-6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu
Pull percpu updates from Dennis Zhou:
"Enable percpu page allocator for RISC-V.
There are RISC-V configurations with sparse NUMA configurations and
small vmalloc space causing dynamic percpu allocations to fail as the
backing chunk stride is too far apart"
* tag 'percpu-for-6.8' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu:
riscv: Enable pcpu page first chunk allocator
mm: Introduce flush_cache_vmap_early()
Diffstat (limited to 'arch')
-rw-r--r-- | arch/arc/include/asm/cacheflush.h | 1 | ||||
-rw-r--r-- | arch/arm/include/asm/cacheflush.h | 2 | ||||
-rw-r--r-- | arch/csky/abiv1/inc/abi/cacheflush.h | 1 | ||||
-rw-r--r-- | arch/csky/abiv2/inc/abi/cacheflush.h | 1 | ||||
-rw-r--r-- | arch/m68k/include/asm/cacheflush_mm.h | 1 | ||||
-rw-r--r-- | arch/mips/include/asm/cacheflush.h | 2 | ||||
-rw-r--r-- | arch/nios2/include/asm/cacheflush.h | 1 | ||||
-rw-r--r-- | arch/parisc/include/asm/cacheflush.h | 1 | ||||
-rw-r--r-- | arch/riscv/Kconfig | 2 | ||||
-rw-r--r-- | arch/riscv/include/asm/cacheflush.h | 3 | ||||
-rw-r--r-- | arch/riscv/include/asm/tlbflush.h | 1 | ||||
-rw-r--r-- | arch/riscv/mm/kasan_init.c | 8 | ||||
-rw-r--r-- | arch/riscv/mm/tlbflush.c | 5 | ||||
-rw-r--r-- | arch/sh/include/asm/cacheflush.h | 1 | ||||
-rw-r--r-- | arch/sparc/include/asm/cacheflush_32.h | 1 | ||||
-rw-r--r-- | arch/sparc/include/asm/cacheflush_64.h | 1 | ||||
-rw-r--r-- | arch/xtensa/include/asm/cacheflush.h | 6 |
17 files changed, 35 insertions, 3 deletions
diff --git a/arch/arc/include/asm/cacheflush.h b/arch/arc/include/asm/cacheflush.h index 563af3e75f01..329c94cd45d8 100644 --- a/arch/arc/include/asm/cacheflush.h +++ b/arch/arc/include/asm/cacheflush.h @@ -40,6 +40,7 @@ void dma_cache_wback(phys_addr_t start, unsigned long sz); /* TBD: optimize this */ #define flush_cache_vmap(start, end) flush_cache_all() +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) flush_cache_all() #define flush_cache_dup_mm(mm) /* called on fork (VIVT only) */ diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index f6181f69577f..1075534b0a2e 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h @@ -340,6 +340,8 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end) dsb(ishst); } +#define flush_cache_vmap_early(start, end) do { } while (0) + static inline void flush_cache_vunmap(unsigned long start, unsigned long end) { if (!cache_is_vipt_nonaliasing()) diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h index 908d8b0bc4fd..d011a81575d2 100644 --- a/arch/csky/abiv1/inc/abi/cacheflush.h +++ b/arch/csky/abiv1/inc/abi/cacheflush.h @@ -43,6 +43,7 @@ static inline void flush_anon_page(struct vm_area_struct *vma, */ extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); #define flush_cache_vmap(start, end) cache_wbinv_all() +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) cache_wbinv_all() #define flush_icache_range(start, end) cache_wbinv_range(start, end) diff --git a/arch/csky/abiv2/inc/abi/cacheflush.h b/arch/csky/abiv2/inc/abi/cacheflush.h index 40be16907267..6513ac5d2578 100644 --- a/arch/csky/abiv2/inc/abi/cacheflush.h +++ b/arch/csky/abiv2/inc/abi/cacheflush.h @@ -41,6 +41,7 @@ void flush_icache_mm_range(struct mm_struct *mm, void flush_icache_deferred(struct mm_struct *mm); #define flush_cache_vmap(start, end) do { } while (0) +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) do { } while (0) #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ diff --git a/arch/m68k/include/asm/cacheflush_mm.h b/arch/m68k/include/asm/cacheflush_mm.h index ed12358c4783..9a71b0148461 100644 --- a/arch/m68k/include/asm/cacheflush_mm.h +++ b/arch/m68k/include/asm/cacheflush_mm.h @@ -191,6 +191,7 @@ extern void cache_push_v(unsigned long vaddr, int len); #define flush_cache_all() __flush_cache_all() #define flush_cache_vmap(start, end) flush_cache_all() +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) flush_cache_all() static inline void flush_cache_mm(struct mm_struct *mm) diff --git a/arch/mips/include/asm/cacheflush.h b/arch/mips/include/asm/cacheflush.h index f36c2519ed97..1f14132b3fc9 100644 --- a/arch/mips/include/asm/cacheflush.h +++ b/arch/mips/include/asm/cacheflush.h @@ -97,6 +97,8 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end) __flush_cache_vmap(); } +#define flush_cache_vmap_early(start, end) do { } while (0) + extern void (*__flush_cache_vunmap)(void); static inline void flush_cache_vunmap(unsigned long start, unsigned long end) diff --git a/arch/nios2/include/asm/cacheflush.h b/arch/nios2/include/asm/cacheflush.h index 348cea097792..81484a776b33 100644 --- a/arch/nios2/include/asm/cacheflush.h +++ b/arch/nios2/include/asm/cacheflush.h @@ -38,6 +38,7 @@ void flush_icache_pages(struct vm_area_struct *vma, struct page *page, #define flush_icache_pages flush_icache_pages #define flush_cache_vmap(start, end) flush_dcache_range(start, end) +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) flush_dcache_range(start, end) extern void copy_to_user_page(struct vm_area_struct *vma, struct page *page, diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h index b4006f2a9705..ba4c05bc24d6 100644 --- a/arch/parisc/include/asm/cacheflush.h +++ b/arch/parisc/include/asm/cacheflush.h @@ -41,6 +41,7 @@ void flush_kernel_vmap_range(void *vaddr, int size); void invalidate_kernel_vmap_range(void *vaddr, int size); #define flush_cache_vmap(start, end) flush_cache_all() +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) flush_cache_all() void flush_dcache_folio(struct folio *folio); diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c729f60eaa00..b549499eb363 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -416,7 +416,9 @@ config NUMA depends on SMP && MMU select ARCH_SUPPORTS_NUMA_BALANCING select GENERIC_ARCH_NUMA + select HAVE_SETUP_PER_CPU_AREA select NEED_PER_CPU_EMBED_FIRST_CHUNK + select NEED_PER_CPU_PAGE_FIRST_CHUNK select OF_NUMA select USE_PERCPU_NUMA_NODE_ID help diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index 3cb53c4df27c..a129dac4521d 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -37,7 +37,8 @@ static inline void flush_dcache_page(struct page *page) flush_icache_mm(vma->vm_mm, 0) #ifdef CONFIG_64BIT -#define flush_cache_vmap(start, end) flush_tlb_kernel_range(start, end) +#define flush_cache_vmap(start, end) flush_tlb_kernel_range(start, end) +#define flush_cache_vmap_early(start, end) local_flush_tlb_kernel_range(start, end) #endif #ifndef CONFIG_SMP diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index 8f3418c5f172..a60416bbe190 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -41,6 +41,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr); void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); void flush_tlb_kernel_range(unsigned long start, unsigned long end); +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end); #ifdef CONFIG_TRANSPARENT_HUGEPAGE #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index e96251853037..c301c8d291d2 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -441,6 +441,14 @@ static void __init kasan_shallow_populate(void *start, void *end) kasan_shallow_populate_pgd(vaddr, vend); } +#ifdef CONFIG_KASAN_VMALLOC +void __init kasan_populate_early_vm_area_shadow(void *start, unsigned long size) +{ + kasan_populate(kasan_mem_to_shadow(start), + kasan_mem_to_shadow(start + size)); +} +#endif + static void __init create_tmp_mapping(void) { void *ptr; diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index e6659d7368b3..8aadc5f71c93 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -66,6 +66,11 @@ static inline void local_flush_tlb_range_asid(unsigned long start, local_flush_tlb_range_threshold_asid(start, size, stride, asid); } +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + local_flush_tlb_range_asid(start, end, PAGE_SIZE, FLUSH_TLB_NO_ASID); +} + static void __ipi_flush_tlb_all(void *info) { local_flush_tlb_all(); diff --git a/arch/sh/include/asm/cacheflush.h b/arch/sh/include/asm/cacheflush.h index 878b6b551bd2..51112f54552b 100644 --- a/arch/sh/include/asm/cacheflush.h +++ b/arch/sh/include/asm/cacheflush.h @@ -90,6 +90,7 @@ extern void copy_from_user_page(struct vm_area_struct *vma, unsigned long len); #define flush_cache_vmap(start, end) local_flush_cache_all(NULL) +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) local_flush_cache_all(NULL) #define flush_dcache_mmap_lock(mapping) do { } while (0) diff --git a/arch/sparc/include/asm/cacheflush_32.h b/arch/sparc/include/asm/cacheflush_32.h index f3b7270bf71b..9fee0ccfccb8 100644 --- a/arch/sparc/include/asm/cacheflush_32.h +++ b/arch/sparc/include/asm/cacheflush_32.h @@ -48,6 +48,7 @@ static inline void flush_dcache_page(struct page *page) #define flush_dcache_mmap_unlock(mapping) do { } while (0) #define flush_cache_vmap(start, end) flush_cache_all() +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) flush_cache_all() /* When a context switch happens we must flush all user windows so that diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h index 0e879004efff..2b1261b77ecd 100644 --- a/arch/sparc/include/asm/cacheflush_64.h +++ b/arch/sparc/include/asm/cacheflush_64.h @@ -75,6 +75,7 @@ void flush_ptrace_access(struct vm_area_struct *, struct page *, #define flush_dcache_mmap_unlock(mapping) do { } while (0) #define flush_cache_vmap(start, end) do { } while (0) +#define flush_cache_vmap_early(start, end) do { } while (0) #define flush_cache_vunmap(start, end) do { } while (0) #endif /* !__ASSEMBLY__ */ diff --git a/arch/xtensa/include/asm/cacheflush.h b/arch/xtensa/include/asm/cacheflush.h index 785a00ce83c1..38bcecb0e457 100644 --- a/arch/xtensa/include/asm/cacheflush.h +++ b/arch/xtensa/include/asm/cacheflush.h @@ -116,8 +116,9 @@ void flush_cache_page(struct vm_area_struct*, #define flush_cache_mm(mm) flush_cache_all() #define flush_cache_dup_mm(mm) flush_cache_mm(mm) -#define flush_cache_vmap(start,end) flush_cache_all() -#define flush_cache_vunmap(start,end) flush_cache_all() +#define flush_cache_vmap(start,end) flush_cache_all() +#define flush_cache_vmap_early(start,end) do { } while (0) +#define flush_cache_vunmap(start,end) flush_cache_all() void flush_dcache_folio(struct folio *folio); #define flush_dcache_folio flush_dcache_folio @@ -140,6 +141,7 @@ void local_flush_cache_page(struct vm_area_struct *vma, #define flush_cache_dup_mm(mm) do { } while (0) #define flush_cache_vmap(start,end) do { } while (0) +#define flush_cache_vmap_early(start,end) do { } while (0) #define flush_cache_vunmap(start,end) do { } while (0) #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0 |