From 7c674700098c87b305b99652e3c694c4ef195866 Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Sat, 27 Dec 2014 18:19:12 -0700 Subject: PCI: Move domain assignment from arm64 to generic code The current logic in arm64 pci_bus_assign_domain_nr() is flawed in that depending on the host controller configuration for a platform and the initialization sequence, core code may end up allocating PCI domain numbers from both DT and the generic domain counter, which would result in PCI domain allocation aliases/errors. Fix the logic behind the PCI domain number assignment and move the resulting code to the PCI core so the same domain allocation logic is used on all platforms that select CONFIG_PCI_DOMAINS_GENERIC. [bhelgaas: tidy changelog] Signed-off-by: Lorenzo Pieralisi Signed-off-by: Bjorn Helgaas Acked-by: Liviu Dudau Acked-by: Arnd Bergmann CC: Rob Herring CC: Catalin Marinas --- arch/arm64/kernel/pci.c | 22 ---------------------- 1 file changed, 22 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c index ce5836c14ec1..6f93c24ca801 100644 --- a/arch/arm64/kernel/pci.c +++ b/arch/arm64/kernel/pci.c @@ -46,25 +46,3 @@ int pcibios_add_device(struct pci_dev *dev) return 0; } - - -#ifdef CONFIG_PCI_DOMAINS_GENERIC -static bool dt_domain_found = false; - -void pci_bus_assign_domain_nr(struct pci_bus *bus, struct device *parent) -{ - int domain = of_get_pci_domain_nr(parent->of_node); - - if (domain >= 0) { - dt_domain_found = true; - } else if (dt_domain_found == true) { - dev_err(parent, "Node %s is missing \"linux,pci-domain\" property in DT\n", - parent->of_node->full_name); - return; - } else { - domain = pci_get_new_domain_nr(); - } - - bus->domain_nr = domain; -} -#endif -- cgit v1.2.3 From f3cdfd239da56a4cea75a2920dc326f0f45f67e3 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Oct 2014 16:27:26 +0200 Subject: arm64/efi: move SetVirtualAddressMap() to UEFI stub In order to support kexec, the kernel needs to be able to deal with the state of the UEFI firmware after SetVirtualAddressMap() has been called. To avoid having separate code paths for non-kexec and kexec, let's move the call to SetVirtualAddressMap() to the stub: this will guarantee us that it will only be called once (since the stub is not executed during kexec), and ensures that the UEFI state is identical between kexec and normal boot. This implies that the layout of the virtual mapping needs to be created by the stub as well. All regions are rounded up to a naturally aligned multiple of 64 KB (for compatibility with 64k pages kernels) and recorded in the UEFI memory map. The kernel proper reads those values and installs the mappings in a dedicated set of page tables that are swapped in during UEFI Runtime Services calls. Acked-by: Leif Lindholm Acked-by: Matt Fleming Tested-by: Leif Lindholm Signed-off-by: Ard Biesheuvel --- arch/arm64/include/asm/efi.h | 34 ++++- arch/arm64/kernel/efi.c | 230 ++++++++++++++++++-------------- arch/arm64/kernel/setup.c | 1 + drivers/firmware/efi/libstub/arm-stub.c | 59 ++++++++ drivers/firmware/efi/libstub/efistub.h | 4 + drivers/firmware/efi/libstub/fdt.c | 62 ++++++++- 6 files changed, 282 insertions(+), 108 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index 71291253114f..effef3713c5a 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -7,28 +7,36 @@ #ifdef CONFIG_EFI extern void efi_init(void); extern void efi_idmap_init(void); +extern void efi_virtmap_init(void); #else #define efi_init() #define efi_idmap_init() +#define efi_virtmap_init() #endif #define efi_call_virt(f, ...) \ ({ \ - efi_##f##_t *__f = efi.systab->runtime->f; \ + efi_##f##_t *__f; \ efi_status_t __s; \ \ kernel_neon_begin(); \ + efi_virtmap_load(); \ + __f = efi.systab->runtime->f; \ __s = __f(__VA_ARGS__); \ + efi_virtmap_unload(); \ kernel_neon_end(); \ __s; \ }) #define __efi_call_virt(f, ...) \ ({ \ - efi_##f##_t *__f = efi.systab->runtime->f; \ + efi_##f##_t *__f; \ \ kernel_neon_begin(); \ + efi_virtmap_load(); \ + __f = efi.systab->runtime->f; \ __f(__VA_ARGS__); \ + efi_virtmap_unload(); \ kernel_neon_end(); \ }) @@ -46,4 +54,26 @@ extern void efi_idmap_init(void); #define EFI_ALLOC_ALIGN SZ_64K +/* + * On ARM systems, virtually remapped UEFI runtime services are set up in three + * distinct stages: + * - The stub retrieves the final version of the memory map from UEFI, populates + * the virt_addr fields and calls the SetVirtualAddressMap() [SVAM] runtime + * service to communicate the new mapping to the firmware (Note that the new + * mapping is not live at this time) + * - During early boot, the page tables are allocated and populated based on the + * virt_addr fields in the memory map, but only if all descriptors with the + * EFI_MEMORY_RUNTIME attribute have a non-zero value for virt_addr. If this + * succeeds, the EFI_VIRTMAP flag is set to indicate that the virtual mappings + * have been installed successfully. + * - During an early initcall(), the UEFI Runtime Services are enabled and the + * EFI_RUNTIME_SERVICES bit set if some conditions are met, i.e., we need a + * non-early mapping of the UEFI system table, and we need to have the virtmap + * installed. + */ +#define EFI_VIRTMAP EFI_ARCH_1 + +void efi_virtmap_load(void); +void efi_virtmap_unload(void); + #endif /* _ASM_EFI_H */ diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 2bb4347d0edf..755e545144ea 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -11,25 +11,31 @@ * */ +#include #include #include #include #include +#include #include #include #include +#include +#include +#include #include #include +#include #include #include #include #include +#include +#include struct efi_memory_map memmap; -static efi_runtime_services_t *runtime; - static u64 efi_system_table; static int uefi_debug __initdata; @@ -69,9 +75,33 @@ static void __init efi_setup_idmap(void) } } +/* + * Translate a EFI virtual address into a physical address: this is necessary, + * as some data members of the EFI system table are virtually remapped after + * SetVirtualAddressMap() has been called. + */ +static phys_addr_t efi_to_phys(unsigned long addr) +{ + efi_memory_desc_t *md; + + for_each_efi_memory_desc(&memmap, md) { + if (!(md->attribute & EFI_MEMORY_RUNTIME)) + continue; + if (md->virt_addr == 0) + /* no virtual mapping has been installed by the stub */ + break; + if (md->virt_addr <= addr && + (addr - md->virt_addr) < (md->num_pages << EFI_PAGE_SHIFT)) + return md->phys_addr + addr - md->virt_addr; + } + return addr; +} + static int __init uefi_init(void) { efi_char16_t *c16; + void *config_tables; + u64 table_size; char vendor[100] = "unknown"; int i, retval; @@ -99,7 +129,7 @@ static int __init uefi_init(void) efi.systab->hdr.revision & 0xffff); /* Show what we know for posterity */ - c16 = early_memremap(efi.systab->fw_vendor, + c16 = early_memremap(efi_to_phys(efi.systab->fw_vendor), sizeof(vendor)); if (c16) { for (i = 0; i < (int) sizeof(vendor) - 1 && *c16; ++i) @@ -112,8 +142,14 @@ static int __init uefi_init(void) efi.systab->hdr.revision >> 16, efi.systab->hdr.revision & 0xffff, vendor); - retval = efi_config_init(NULL); + table_size = sizeof(efi_config_table_64_t) * efi.systab->nr_tables; + config_tables = early_memremap(efi_to_phys(efi.systab->tables), + table_size); + + retval = efi_config_parse_tables(config_tables, efi.systab->nr_tables, + sizeof(efi_config_table_64_t), NULL); + early_memunmap(config_tables, table_size); out: early_memunmap(efi.systab, sizeof(efi_system_table_t)); return retval; @@ -329,51 +365,14 @@ void __init efi_idmap_init(void) early_memunmap(memmap.map, memmap.map_end - memmap.map); } -static int __init remap_region(efi_memory_desc_t *md, void **new) -{ - u64 paddr, vaddr, npages, size; - - paddr = md->phys_addr; - npages = md->num_pages; - memrange_efi_to_native(&paddr, &npages); - size = npages << PAGE_SHIFT; - - if (is_normal_ram(md)) - vaddr = (__force u64)ioremap_cache(paddr, size); - else - vaddr = (__force u64)ioremap(paddr, size); - - if (!vaddr) { - pr_err("Unable to remap 0x%llx pages @ %p\n", - npages, (void *)paddr); - return 0; - } - - /* adjust for any rounding when EFI and system pagesize differs */ - md->virt_addr = vaddr + (md->phys_addr - paddr); - - if (uefi_debug) - pr_info(" EFI remap 0x%012llx => %p\n", - md->phys_addr, (void *)md->virt_addr); - - memcpy(*new, md, memmap.desc_size); - *new += memmap.desc_size; - - return 1; -} - /* - * Switch UEFI from an identity map to a kernel virtual map + * Enable the UEFI Runtime Services if all prerequisites are in place, i.e., + * non-early mapping of the UEFI system table and virtual mappings for all + * EFI_MEMORY_RUNTIME regions. */ -static int __init arm64_enter_virtual_mode(void) +static int __init arm64_enable_runtime_services(void) { - efi_memory_desc_t *md; - phys_addr_t virtmap_phys; - void *virtmap, *virt_md; - efi_status_t status; u64 mapsize; - int count = 0; - unsigned long flags; if (!efi_enabled(EFI_BOOT)) { pr_info("EFI services will not be available.\n"); @@ -395,81 +394,30 @@ static int __init arm64_enter_virtual_mode(void) efi.memmap = &memmap; - /* Map the runtime regions */ - virtmap = kmalloc(mapsize, GFP_KERNEL); - if (!virtmap) { - pr_err("Failed to allocate EFI virtual memmap\n"); - return -1; - } - virtmap_phys = virt_to_phys(virtmap); - virt_md = virtmap; - - for_each_efi_memory_desc(&memmap, md) { - if (!(md->attribute & EFI_MEMORY_RUNTIME)) - continue; - if (!remap_region(md, &virt_md)) - goto err_unmap; - ++count; - } - - efi.systab = (__force void *)efi_lookup_mapped_addr(efi_system_table); + efi.systab = (__force void *)ioremap_cache(efi_system_table, + sizeof(efi_system_table_t)); if (!efi.systab) { - /* - * If we have no virtual mapping for the System Table at this - * point, the memory map doesn't cover the physical offset where - * it resides. This means the System Table will be inaccessible - * to Runtime Services themselves once the virtual mapping is - * installed. - */ - pr_err("Failed to remap EFI System Table -- buggy firmware?\n"); - goto err_unmap; + pr_err("Failed to remap EFI System Table\n"); + return -1; } set_bit(EFI_SYSTEM_TABLES, &efi.flags); - local_irq_save(flags); - cpu_switch_mm(idmap_pg_dir, &init_mm); - - /* Call SetVirtualAddressMap with the physical address of the map */ - runtime = efi.systab->runtime; - efi.set_virtual_address_map = runtime->set_virtual_address_map; - - status = efi.set_virtual_address_map(count * memmap.desc_size, - memmap.desc_size, - memmap.desc_version, - (efi_memory_desc_t *)virtmap_phys); - cpu_set_reserved_ttbr0(); - flush_tlb_all(); - local_irq_restore(flags); - - kfree(virtmap); - free_boot_services(); - if (status != EFI_SUCCESS) { - pr_err("Failed to set EFI virtual address map! [%lx]\n", - status); + if (!efi_enabled(EFI_VIRTMAP)) { + pr_err("No UEFI virtual mapping was installed -- runtime services will not be available\n"); return -1; } /* Set up runtime services function pointers */ - runtime = efi.systab->runtime; efi_native_runtime_setup(); set_bit(EFI_RUNTIME_SERVICES, &efi.flags); efi.runtime_version = efi.systab->hdr.revision; return 0; - -err_unmap: - /* unmap all mappings that succeeded: there are 'count' of those */ - for (virt_md = virtmap; count--; virt_md += memmap.desc_size) { - md = virt_md; - iounmap((__force void __iomem *)md->virt_addr); - } - kfree(virtmap); - return -1; } -early_initcall(arm64_enter_virtual_mode); +early_initcall(arm64_enable_runtime_services); static int __init arm64_dmi_init(void) { @@ -484,3 +432,79 @@ static int __init arm64_dmi_init(void) return 0; } core_initcall(arm64_dmi_init); + +static pgd_t efi_pgd[PTRS_PER_PGD] __page_aligned_bss; + +static struct mm_struct efi_mm = { + .mm_rb = RB_ROOT, + .pgd = efi_pgd, + .mm_users = ATOMIC_INIT(2), + .mm_count = ATOMIC_INIT(1), + .mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem), + .page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock), + .mmlist = LIST_HEAD_INIT(efi_mm.mmlist), + INIT_MM_CONTEXT(efi_mm) +}; + +static void efi_set_pgd(struct mm_struct *mm) +{ + cpu_switch_mm(mm->pgd, mm); + flush_tlb_all(); + if (icache_is_aivivt()) + __flush_icache_all(); +} + +void efi_virtmap_load(void) +{ + preempt_disable(); + efi_set_pgd(&efi_mm); +} + +void efi_virtmap_unload(void) +{ + efi_set_pgd(current->active_mm); + preempt_enable(); +} + +void __init efi_virtmap_init(void) +{ + efi_memory_desc_t *md; + + if (!efi_enabled(EFI_BOOT)) + return; + + for_each_efi_memory_desc(&memmap, md) { + u64 paddr, npages, size; + pgprot_t prot; + + if (!(md->attribute & EFI_MEMORY_RUNTIME)) + continue; + if (WARN(md->virt_addr == 0, + "UEFI virtual mapping incomplete or missing -- no entry found for 0x%llx\n", + md->phys_addr)) + return; + + paddr = md->phys_addr; + npages = md->num_pages; + memrange_efi_to_native(&paddr, &npages); + size = npages << PAGE_SHIFT; + + pr_info(" EFI remap 0x%016llx => %p\n", + md->phys_addr, (void *)md->virt_addr); + + /* + * Only regions of type EFI_RUNTIME_SERVICES_CODE need to be + * executable, everything else can be mapped with the XN bits + * set. + */ + if (!is_normal_ram(md)) + prot = __pgprot(PROT_DEVICE_nGnRE); + else if (md->type == EFI_RUNTIME_SERVICES_CODE) + prot = PAGE_KERNEL_EXEC; + else + prot = PAGE_KERNEL; + + create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot); + } + set_bit(EFI_VIRTMAP, &efi.flags); +} diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 20fe2932ad0c..beac8188fdbd 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -401,6 +401,7 @@ void __init setup_arch(char **cmdline_p) paging_init(); request_standard_resources(); + efi_virtmap_init(); efi_idmap_init(); early_ioremap_reset(); diff --git a/drivers/firmware/efi/libstub/arm-stub.c b/drivers/firmware/efi/libstub/arm-stub.c index eb48a1a1a576..e2432b39b6df 100644 --- a/drivers/firmware/efi/libstub/arm-stub.c +++ b/drivers/firmware/efi/libstub/arm-stub.c @@ -295,3 +295,62 @@ fail_free_image: fail: return EFI_ERROR; } + +/* + * This is the base address at which to start allocating virtual memory ranges + * for UEFI Runtime Services. This is in the low TTBR0 range so that we can use + * any allocation we choose, and eliminate the risk of a conflict after kexec. + * The value chosen is the largest non-zero power of 2 suitable for this purpose + * both on 32-bit and 64-bit ARM CPUs, to maximize the likelihood that it can + * be mapped efficiently. + */ +#define EFI_RT_VIRTUAL_BASE 0x40000000 + +/* + * efi_get_virtmap() - create a virtual mapping for the EFI memory map + * + * This function populates the virt_addr fields of all memory region descriptors + * in @memory_map whose EFI_MEMORY_RUNTIME attribute is set. Those descriptors + * are also copied to @runtime_map, and their total count is returned in @count. + */ +void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size, + unsigned long desc_size, efi_memory_desc_t *runtime_map, + int *count) +{ + u64 efi_virt_base = EFI_RT_VIRTUAL_BASE; + efi_memory_desc_t *out = runtime_map; + int l; + + for (l = 0; l < map_size; l += desc_size) { + efi_memory_desc_t *in = (void *)memory_map + l; + u64 paddr, size; + + if (!(in->attribute & EFI_MEMORY_RUNTIME)) + continue; + + /* + * Make the mapping compatible with 64k pages: this allows + * a 4k page size kernel to kexec a 64k page size kernel and + * vice versa. + */ + paddr = round_down(in->phys_addr, SZ_64K); + size = round_up(in->num_pages * EFI_PAGE_SIZE + + in->phys_addr - paddr, SZ_64K); + + /* + * Avoid wasting memory on PTEs by choosing a virtual base that + * is compatible with section mappings if this region has the + * appropriate size and physical alignment. (Sections are 2 MB + * on 4k granule kernels) + */ + if (IS_ALIGNED(in->phys_addr, SZ_2M) && size >= SZ_2M) + efi_virt_base = round_up(efi_virt_base, SZ_2M); + + in->virt_addr = efi_virt_base + in->phys_addr - paddr; + efi_virt_base += size; + + memcpy(out, in, desc_size); + out = (void *)out + desc_size; + ++*count; + } +} diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index 304ab295ca1a..2be10984a67a 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -39,4 +39,8 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, void *get_fdt(efi_system_table_t *sys_table); +void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size, + unsigned long desc_size, efi_memory_desc_t *runtime_map, + int *count); + #endif diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c index c846a9608cbd..91da56c4fd54 100644 --- a/drivers/firmware/efi/libstub/fdt.c +++ b/drivers/firmware/efi/libstub/fdt.c @@ -14,6 +14,8 @@ #include #include +#include "efistub.h" + efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt, unsigned long orig_fdt_size, void *fdt, int new_fdt_size, char *cmdline_ptr, @@ -193,9 +195,26 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, unsigned long map_size, desc_size; u32 desc_ver; unsigned long mmap_key; - efi_memory_desc_t *memory_map; + efi_memory_desc_t *memory_map, *runtime_map; unsigned long new_fdt_size; efi_status_t status; + int runtime_entry_count = 0; + + /* + * Get a copy of the current memory map that we will use to prepare + * the input for SetVirtualAddressMap(). We don't have to worry about + * subsequent allocations adding entries, since they could not affect + * the number of EFI_MEMORY_RUNTIME regions. + */ + status = efi_get_memory_map(sys_table, &runtime_map, &map_size, + &desc_size, &desc_ver, &mmap_key); + if (status != EFI_SUCCESS) { + pr_efi_err(sys_table, "Unable to retrieve UEFI memory map.\n"); + return status; + } + + pr_efi(sys_table, + "Exiting boot services and installing virtual address map...\n"); /* * Estimate size of new FDT, and allocate memory for it. We @@ -248,12 +267,48 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table, } } + /* + * Update the memory map with virtual addresses. The function will also + * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME + * entries so that we can pass it straight into SetVirtualAddressMap() + */ + efi_get_virtmap(memory_map, map_size, desc_size, runtime_map, + &runtime_entry_count); + /* Now we are ready to exit_boot_services.*/ status = sys_table->boottime->exit_boot_services(handle, mmap_key); + if (status == EFI_SUCCESS) { + efi_set_virtual_address_map_t *svam; - if (status == EFI_SUCCESS) - return status; + /* Install the new virtual address map */ + svam = sys_table->runtime->set_virtual_address_map; + status = svam(runtime_entry_count * desc_size, desc_size, + desc_ver, runtime_map); + + /* + * We are beyond the point of no return here, so if the call to + * SetVirtualAddressMap() failed, we need to signal that to the + * incoming kernel but proceed normally otherwise. + */ + if (status != EFI_SUCCESS) { + int l; + + /* + * Set the virtual address field of all + * EFI_MEMORY_RUNTIME entries to 0. This will signal + * the incoming kernel that no virtual translation has + * been installed. + */ + for (l = 0; l < map_size; l += desc_size) { + efi_memory_desc_t *p = (void *)memory_map + l; + + if (p->attribute & EFI_MEMORY_RUNTIME) + p->virt_addr = 0; + } + } + return EFI_SUCCESS; + } pr_efi_err(sys_table, "Exit boot services failed.\n"); @@ -264,6 +319,7 @@ fail_free_new_fdt: efi_free(sys_table, new_fdt_size, *new_fdt_addr); fail: + sys_table->boottime->free_pool(runtime_map); return EFI_LOAD_ERROR; } -- cgit v1.2.3 From 3033b84596eaec0093b68c5711c265738eb0745d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Oct 2014 16:35:21 +0200 Subject: arm64/efi: remove free_boot_services() and friends Now that we are calling SetVirtualAddressMap() from the stub, there is no need to reserve boot-only memory regions, which implies that there is also no reason to free them again later. Acked-by: Leif Lindholm Acked-by: Will Deacon Tested-by: Leif Lindholm Signed-off-by: Ard Biesheuvel --- arch/arm64/kernel/efi.c | 123 +----------------------------------------------- 1 file changed, 1 insertion(+), 122 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 755e545144ea..4a5d7343dddd 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -199,9 +199,7 @@ static __init void reserve_regions(void) if (is_normal_ram(md)) early_init_dt_add_memory_arch(paddr, size); - if (is_reserve_region(md) || - md->type == EFI_BOOT_SERVICES_CODE || - md->type == EFI_BOOT_SERVICES_DATA) { + if (is_reserve_region(md)) { memblock_reserve(paddr, size); if (uefi_debug) pr_cont("*"); @@ -214,123 +212,6 @@ static __init void reserve_regions(void) set_bit(EFI_MEMMAP, &efi.flags); } - -static u64 __init free_one_region(u64 start, u64 end) -{ - u64 size = end - start; - - if (uefi_debug) - pr_info(" EFI freeing: 0x%012llx-0x%012llx\n", start, end - 1); - - free_bootmem_late(start, size); - return size; -} - -static u64 __init free_region(u64 start, u64 end) -{ - u64 map_start, map_end, total = 0; - - if (end <= start) - return total; - - map_start = (u64)memmap.phys_map; - map_end = PAGE_ALIGN(map_start + (memmap.map_end - memmap.map)); - map_start &= PAGE_MASK; - - if (start < map_end && end > map_start) { - /* region overlaps UEFI memmap */ - if (start < map_start) - total += free_one_region(start, map_start); - - if (map_end < end) - total += free_one_region(map_end, end); - } else - total += free_one_region(start, end); - - return total; -} - -static void __init free_boot_services(void) -{ - u64 total_freed = 0; - u64 keep_end, free_start, free_end; - efi_memory_desc_t *md; - - /* - * If kernel uses larger pages than UEFI, we have to be careful - * not to inadvertantly free memory we want to keep if there is - * overlap at the kernel page size alignment. We do not want to - * free is_reserve_region() memory nor the UEFI memmap itself. - * - * The memory map is sorted, so we keep track of the end of - * any previous region we want to keep, remember any region - * we want to free and defer freeing it until we encounter - * the next region we want to keep. This way, before freeing - * it, we can clip it as needed to avoid freeing memory we - * want to keep for UEFI. - */ - - keep_end = 0; - free_start = 0; - - for_each_efi_memory_desc(&memmap, md) { - u64 paddr, npages, size; - - if (is_reserve_region(md)) { - /* - * We don't want to free any memory from this region. - */ - if (free_start) { - /* adjust free_end then free region */ - if (free_end > md->phys_addr) - free_end -= PAGE_SIZE; - total_freed += free_region(free_start, free_end); - free_start = 0; - } - keep_end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT); - continue; - } - - if (md->type != EFI_BOOT_SERVICES_CODE && - md->type != EFI_BOOT_SERVICES_DATA) { - /* no need to free this region */ - continue; - } - - /* - * We want to free memory from this region. - */ - paddr = md->phys_addr; - npages = md->num_pages; - memrange_efi_to_native(&paddr, &npages); - size = npages << PAGE_SHIFT; - - if (free_start) { - if (paddr <= free_end) - free_end = paddr + size; - else { - total_freed += free_region(free_start, free_end); - free_start = paddr; - free_end = paddr + size; - } - } else { - free_start = paddr; - free_end = paddr + size; - } - if (free_start < keep_end) { - free_start += PAGE_SIZE; - if (free_start >= free_end) - free_start = 0; - } - } - if (free_start) - total_freed += free_region(free_start, free_end); - - if (total_freed) - pr_info("Freed 0x%llx bytes of EFI boot services memory", - total_freed); -} - void __init efi_init(void) { struct efi_fdt_params params; @@ -402,8 +283,6 @@ static int __init arm64_enable_runtime_services(void) } set_bit(EFI_SYSTEM_TABLES, &efi.flags); - free_boot_services(); - if (!efi_enabled(EFI_VIRTMAP)) { pr_err("No UEFI virtual mapping was installed -- runtime services will not be available\n"); return -1; -- cgit v1.2.3 From 9679be103108926cfe9e6fd2f6829cefa77e47b0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 20 Oct 2014 16:41:38 +0200 Subject: arm64/efi: remove idmap manipulations from UEFI code Now that we have moved the call to SetVirtualAddressMap() to the stub, UEFI has no use for the ID map, so we can drop the code that installs ID mappings for UEFI memory regions. Acked-by: Leif Lindholm Acked-by: Will Deacon Tested-by: Leif Lindholm Signed-off-by: Ard Biesheuvel --- arch/arm64/include/asm/efi.h | 2 -- arch/arm64/include/asm/mmu.h | 2 -- arch/arm64/kernel/efi.c | 32 +------------------------------- arch/arm64/kernel/setup.c | 1 - arch/arm64/mm/mmu.c | 12 ------------ 5 files changed, 1 insertion(+), 48 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index effef3713c5a..7baf2cc04e1e 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -6,11 +6,9 @@ #ifdef CONFIG_EFI extern void efi_init(void); -extern void efi_idmap_init(void); extern void efi_virtmap_init(void); #else #define efi_init() -#define efi_idmap_init() #define efi_virtmap_init() #endif diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 5fd40c43be80..3d311761e3c2 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -31,8 +31,6 @@ extern void paging_init(void); extern void setup_mm_for_reboot(void); extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt); extern void init_mem_pgprot(void); -/* create an identity mapping for memory (or io if map_io is true) */ -extern void create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io); extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot); diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 4a5d7343dddd..a98415b5979c 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -54,27 +54,6 @@ static int __init is_normal_ram(efi_memory_desc_t *md) return 0; } -static void __init efi_setup_idmap(void) -{ - struct memblock_region *r; - efi_memory_desc_t *md; - u64 paddr, npages, size; - - for_each_memblock(memory, r) - create_id_mapping(r->base, r->size, 0); - - /* map runtime io spaces */ - for_each_efi_memory_desc(&memmap, md) { - if (!(md->attribute & EFI_MEMORY_RUNTIME) || is_normal_ram(md)) - continue; - paddr = md->phys_addr; - npages = md->num_pages; - memrange_efi_to_native(&paddr, &npages); - size = npages << PAGE_SHIFT; - create_id_mapping(paddr, size, 1); - } -} - /* * Translate a EFI virtual address into a physical address: this is necessary, * as some data members of the EFI system table are virtually remapped after @@ -236,16 +215,6 @@ void __init efi_init(void) reserve_regions(); } -void __init efi_idmap_init(void) -{ - if (!efi_enabled(EFI_BOOT)) - return; - - /* boot time idmap_pg_dir is incomplete, so fill in missing parts */ - efi_setup_idmap(); - early_memunmap(memmap.map, memmap.map_end - memmap.map); -} - /* * Enable the UEFI Runtime Services if all prerequisites are in place, i.e., * non-early mapping of the UEFI system table and virtual mappings for all @@ -386,4 +355,5 @@ void __init efi_virtmap_init(void) create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot); } set_bit(EFI_VIRTMAP, &efi.flags); + early_memunmap(memmap.map, memmap.map_end - memmap.map); } diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index beac8188fdbd..199d1b7809d7 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -402,7 +402,6 @@ void __init setup_arch(char **cmdline_p) request_standard_resources(); efi_virtmap_init(); - efi_idmap_init(); early_ioremap_reset(); unflatten_device_tree(); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 3f3d5aa4a8b1..328638548871 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -271,18 +271,6 @@ static void __init create_mapping(phys_addr_t phys, unsigned long virt, size, PAGE_KERNEL_EXEC); } -void __init create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io) -{ - if ((addr >> PGDIR_SHIFT) >= ARRAY_SIZE(idmap_pg_dir)) { - pr_warn("BUG: not creating id mapping for %pa\n", &addr); - return; - } - __create_mapping(&init_mm, &idmap_pg_dir[pgd_index(addr)], - addr, addr, size, - map_io ? __pgprot(PROT_DEVICE_nGnRE) - : PAGE_KERNEL_EXEC); -} - void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot) -- cgit v1.2.3 From 5d425c18653731af62831d30a4fa023d532657a9 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Thu, 8 Jan 2015 10:42:34 +0000 Subject: arm64: kernel: add support for cpu cache information This patch adds support for cacheinfo on ARM64. On ARMv8, the cache hierarchy can be identified through Cache Level ID (CLIDR) register while the cache geometry is provided by Cache Size ID (CCSIDR) register. Since the architecture doesn't provide any way of detecting the cpus sharing particular cache, device tree is used for the same purpose. Signed-off-by: Sudeep Holla Cc: Will Deacon Cc: Mark Rutland Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/cachetype.h | 29 +++++++-- arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/cacheinfo.c | 128 +++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/cpuinfo.c | 12 ---- 4 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 arch/arm64/kernel/cacheinfo.c (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/cachetype.h b/arch/arm64/include/asm/cachetype.h index 4c631a0a3609..da2fc9e3cedd 100644 --- a/arch/arm64/include/asm/cachetype.h +++ b/arch/arm64/include/asm/cachetype.h @@ -39,24 +39,41 @@ extern unsigned long __icache_flags; +/* + * NumSets, bits[27:13] - (Number of sets in cache) - 1 + * Associativity, bits[12:3] - (Associativity of cache) - 1 + * LineSize, bits[2:0] - (Log2(Number of words in cache line)) - 2 + */ +#define CCSIDR_EL1_WRITE_THROUGH BIT(31) +#define CCSIDR_EL1_WRITE_BACK BIT(30) +#define CCSIDR_EL1_READ_ALLOCATE BIT(29) +#define CCSIDR_EL1_WRITE_ALLOCATE BIT(28) #define CCSIDR_EL1_LINESIZE_MASK 0x7 #define CCSIDR_EL1_LINESIZE(x) ((x) & CCSIDR_EL1_LINESIZE_MASK) - +#define CCSIDR_EL1_ASSOCIATIVITY_SHIFT 3 +#define CCSIDR_EL1_ASSOCIATIVITY_MASK 0x3ff +#define CCSIDR_EL1_ASSOCIATIVITY(x) \ + (((x) >> CCSIDR_EL1_ASSOCIATIVITY_SHIFT) & CCSIDR_EL1_ASSOCIATIVITY_MASK) #define CCSIDR_EL1_NUMSETS_SHIFT 13 -#define CCSIDR_EL1_NUMSETS_MASK (0x7fff << CCSIDR_EL1_NUMSETS_SHIFT) +#define CCSIDR_EL1_NUMSETS_MASK 0x7fff #define CCSIDR_EL1_NUMSETS(x) \ - (((x) & CCSIDR_EL1_NUMSETS_MASK) >> CCSIDR_EL1_NUMSETS_SHIFT) + (((x) >> CCSIDR_EL1_NUMSETS_SHIFT) & CCSIDR_EL1_NUMSETS_MASK) + +#define CACHE_LINESIZE(x) (16 << CCSIDR_EL1_LINESIZE(x)) +#define CACHE_NUMSETS(x) (CCSIDR_EL1_NUMSETS(x) + 1) +#define CACHE_ASSOCIATIVITY(x) (CCSIDR_EL1_ASSOCIATIVITY(x) + 1) -extern u64 __attribute_const__ icache_get_ccsidr(void); +extern u64 __attribute_const__ cache_get_ccsidr(u64 csselr); +/* Helpers for Level 1 Instruction cache csselr = 1L */ static inline int icache_get_linesize(void) { - return 16 << CCSIDR_EL1_LINESIZE(icache_get_ccsidr()); + return CACHE_LINESIZE(cache_get_ccsidr(1L)); } static inline int icache_get_numsets(void) { - return 1 + CCSIDR_EL1_NUMSETS(icache_get_ccsidr()); + return CACHE_NUMSETS(cache_get_ccsidr(1L)); } /* diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index eaa77ed7766a..77d3d9568333 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -16,7 +16,7 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \ - cpuinfo.o cpu_errata.o alternative.o + cpuinfo.o cpu_errata.o alternative.o cacheinfo.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o \ diff --git a/arch/arm64/kernel/cacheinfo.c b/arch/arm64/kernel/cacheinfo.c new file mode 100644 index 000000000000..b8629d52fba9 --- /dev/null +++ b/arch/arm64/kernel/cacheinfo.c @@ -0,0 +1,128 @@ +/* + * ARM64 cacheinfo support + * + * Copyright (C) 2015 ARM Ltd. + * All Rights Reserved + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include + +#include +#include + +#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */ +/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */ +#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1)) +#define CLIDR_CTYPE_MASK(level) (7 << CLIDR_CTYPE_SHIFT(level)) +#define CLIDR_CTYPE(clidr, level) \ + (((clidr) & CLIDR_CTYPE_MASK(level)) >> CLIDR_CTYPE_SHIFT(level)) + +static inline enum cache_type get_cache_type(int level) +{ + u64 clidr; + + if (level > MAX_CACHE_LEVEL) + return CACHE_TYPE_NOCACHE; + asm volatile ("mrs %x0, clidr_el1" : "=r" (clidr)); + return CLIDR_CTYPE(clidr, level); +} + +/* + * Cache Size Selection Register(CSSELR) selects which Cache Size ID + * Register(CCSIDR) is accessible by specifying the required cache + * level and the cache type. We need to ensure that no one else changes + * CSSELR by calling this in non-preemtible context + */ +u64 __attribute_const__ cache_get_ccsidr(u64 csselr) +{ + u64 ccsidr; + + WARN_ON(preemptible()); + + /* Put value into CSSELR */ + asm volatile("msr csselr_el1, %x0" : : "r" (csselr)); + isb(); + /* Read result out of CCSIDR */ + asm volatile("mrs %x0, ccsidr_el1" : "=r" (ccsidr)); + + return ccsidr; +} + +static void ci_leaf_init(struct cacheinfo *this_leaf, + enum cache_type type, unsigned int level) +{ + bool is_icache = type & CACHE_TYPE_INST; + u64 tmp = cache_get_ccsidr((level - 1) << 1 | is_icache); + + this_leaf->level = level; + this_leaf->type = type; + this_leaf->coherency_line_size = CACHE_LINESIZE(tmp); + this_leaf->number_of_sets = CACHE_NUMSETS(tmp); + this_leaf->ways_of_associativity = CACHE_ASSOCIATIVITY(tmp); + this_leaf->size = this_leaf->number_of_sets * + this_leaf->coherency_line_size * this_leaf->ways_of_associativity; + this_leaf->attributes = + ((tmp & CCSIDR_EL1_WRITE_THROUGH) ? CACHE_WRITE_THROUGH : 0) | + ((tmp & CCSIDR_EL1_WRITE_BACK) ? CACHE_WRITE_BACK : 0) | + ((tmp & CCSIDR_EL1_READ_ALLOCATE) ? CACHE_READ_ALLOCATE : 0) | + ((tmp & CCSIDR_EL1_WRITE_ALLOCATE) ? CACHE_WRITE_ALLOCATE : 0); +} + +static int __init_cache_level(unsigned int cpu) +{ + unsigned int ctype, level, leaves; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + + for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) { + ctype = get_cache_type(level); + if (ctype == CACHE_TYPE_NOCACHE) { + level--; + break; + } + /* Separate instruction and data caches */ + leaves += (ctype == CACHE_TYPE_SEPARATE) ? 2 : 1; + } + + this_cpu_ci->num_levels = level; + this_cpu_ci->num_leaves = leaves; + return 0; +} + +static int __populate_cache_leaves(unsigned int cpu) +{ + unsigned int level, idx; + enum cache_type type; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf = this_cpu_ci->info_list; + + for (idx = 0, level = 1; level <= this_cpu_ci->num_levels && + idx < this_cpu_ci->num_leaves; idx++, level++) { + type = get_cache_type(level); + if (type == CACHE_TYPE_SEPARATE) { + ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level); + ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level); + } else { + ci_leaf_init(this_leaf++, type, level); + } + } + return 0; +} + +DEFINE_SMP_CALL_CACHE_FUNCTION(init_cache_level) +DEFINE_SMP_CALL_CACHE_FUNCTION(populate_cache_leaves) diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 07d435cf2eea..49782282a027 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -231,15 +231,3 @@ void __init cpuinfo_store_boot_cpu(void) boot_cpu_data = *info; } - -u64 __attribute_const__ icache_get_ccsidr(void) -{ - u64 ccsidr; - - WARN_ON(preemptible()); - - /* Select L1 I-cache and read its size ID register */ - asm("msr csselr_el1, %1; isb; mrs %0, ccsidr_el1" - : "=r"(ccsidr) : "r"(1L)); - return ccsidr; -} -- cgit v1.2.3 From aed40e0144bdbdadb45f2b623e07aa3157eb74c9 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 24 Nov 2014 12:31:40 +0000 Subject: arm64: move to ESR_ELx macros Now that we have common ESR_ELx_* macros, move the core arm64 code over to them. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Reviewed-by: Christoffer Dall Cc: Marc Zyngier Cc: Peter Maydell Cc: Will Deacon --- arch/arm64/kernel/entry.S | 64 ++++++++++++++++++++++---------------------- arch/arm64/kernel/signal32.c | 2 +- arch/arm64/mm/fault.c | 2 +- 3 files changed, 34 insertions(+), 34 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index fd4fa374e5d2..02e6af117762 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -269,18 +269,18 @@ ENDPROC(el1_error_invalid) el1_sync: kernel_entry 1 mrs x1, esr_el1 // read the syndrome register - lsr x24, x1, #ESR_EL1_EC_SHIFT // exception class - cmp x24, #ESR_EL1_EC_DABT_EL1 // data abort in EL1 + lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class + cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1 b.eq el1_da - cmp x24, #ESR_EL1_EC_SYS64 // configurable trap + cmp x24, #ESR_ELx_EC_SYS64 // configurable trap b.eq el1_undef - cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception + cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception b.eq el1_sp_pc - cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception + cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception b.eq el1_sp_pc - cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL1 + cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1 b.eq el1_undef - cmp x24, #ESR_EL1_EC_BREAKPT_EL1 // debug exception in EL1 + cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1 b.ge el1_dbg b el1_inv el1_da: @@ -318,7 +318,7 @@ el1_dbg: /* * Debug exception handling */ - cmp x24, #ESR_EL1_EC_BRK64 // if BRK64 + cmp x24, #ESR_ELx_EC_BRK64 // if BRK64 cinc x24, x24, eq // set bit '0' tbz x24, #0, el1_inv // EL1 only mrs x0, far_el1 @@ -375,26 +375,26 @@ el1_preempt: el0_sync: kernel_entry 0 mrs x25, esr_el1 // read the syndrome register - lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class - cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state + lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class + cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state b.eq el0_svc - cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 + cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0 b.eq el0_da - cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 + cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0 b.eq el0_ia - cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access + cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access b.eq el0_fpsimd_acc - cmp x24, #ESR_EL1_EC_FP_EXC64 // FP/ASIMD exception + cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception b.eq el0_fpsimd_exc - cmp x24, #ESR_EL1_EC_SYS64 // configurable trap + cmp x24, #ESR_ELx_EC_SYS64 // configurable trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception + cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception b.eq el0_sp_pc - cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception + cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception b.eq el0_sp_pc - cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 + cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0 b.eq el0_undef - cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 + cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0 b.ge el0_dbg b el0_inv @@ -403,30 +403,30 @@ el0_sync: el0_sync_compat: kernel_entry 0, 32 mrs x25, esr_el1 // read the syndrome register - lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class - cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state + lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class + cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state b.eq el0_svc_compat - cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 + cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0 b.eq el0_da - cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 + cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0 b.eq el0_ia - cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access + cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access b.eq el0_fpsimd_acc - cmp x24, #ESR_EL1_EC_FP_EXC32 // FP/ASIMD exception + cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception b.eq el0_fpsimd_exc - cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 + cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0 b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP15_32 // CP15 MRC/MCR trap + cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP15_64 // CP15 MRRC/MCRR trap + cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP14_MR // CP14 MRC/MCR trap + cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP14_LS // CP14 LDC/STC trap + cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP14_64 // CP14 MRRC/MCRR trap + cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 + cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0 b.ge el0_dbg b el0_inv el0_svc_compat: diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c index 5a1ba6e80d4e..192d900c058f 100644 --- a/arch/arm64/kernel/signal32.c +++ b/arch/arm64/kernel/signal32.c @@ -501,7 +501,7 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf, __put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err); /* set the compat FSR WnR */ - __put_user_error(!!(current->thread.fault_code & ESR_EL1_WRITE) << + __put_user_error(!!(current->thread.fault_code & ESR_ELx_WNR) << FSR_WRITE_SHIFT, &sf->uc.uc_mcontext.error_code, err); __put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err); __put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index c11cd27ca8f5..96da13167d4a 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -219,7 +219,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, if (esr & ESR_LNX_EXEC) { vm_flags = VM_EXEC; - } else if ((esr & ESR_EL1_WRITE) && !(esr & ESR_EL1_CM)) { + } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { vm_flags = VM_WRITE; mm_flags |= FAULT_FLAG_WRITE; } -- cgit v1.2.3 From 60a1f02c9e91e0796b54e83b14fb8a07f7a568b6 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 18 Nov 2014 12:16:30 +0000 Subject: arm64: decode ESR_ELx.EC when reporting exceptions To aid the developer when something triggers an unexpected exception, decode the ESR_ELx.EC field when logging an ESR_ELx value. This doesn't tell the developer the specifics of the exception encoded in the remaining IL and ISS bits, but it can be helpful to distinguish between exception classes (e.g. SError and a data abort) without having to manually decode the field, which can be tiresome. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Reviewed-by: Christoffer Dall Cc: Marc Zyngier Cc: Peter Maydell Cc: Will Deacon --- arch/arm64/include/asm/esr.h | 6 ++++++ arch/arm64/kernel/traps.c | 50 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 2 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 0fd1b0e15ea8..c315543d50f9 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -133,4 +133,10 @@ #define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT) #define ESR_ELx_WFx_ISS_WFE (UL(1) << 0) +#ifndef __ASSEMBLY__ +#include + +const char *esr_get_class_string(u32 esr); +#endif /* __ASSEMBLY */ + #endif /* __ASM_ESR_H */ diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 0a801e3743d5..1ef2940df13c 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -373,6 +374,51 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs) return sys_ni_syscall(); } +static const char *esr_class_str[] = { + [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", + [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", + [ESR_ELx_EC_WFx] = "WFI/WFE", + [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC", + [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC", + [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC", + [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC", + [ESR_ELx_EC_FP_ASIMD] = "ASIMD", + [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS", + [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC", + [ESR_ELx_EC_ILL] = "PSTATE.IL", + [ESR_ELx_EC_SVC32] = "SVC (AArch32)", + [ESR_ELx_EC_HVC32] = "HVC (AArch32)", + [ESR_ELx_EC_SMC32] = "SMC (AArch32)", + [ESR_ELx_EC_SVC64] = "SVC (AArch64)", + [ESR_ELx_EC_HVC64] = "HVC (AArch64)", + [ESR_ELx_EC_SMC64] = "SMC (AArch64)", + [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", + [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", + [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", + [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", + [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", + [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", + [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", + [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", + [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)", + [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", + [ESR_ELx_EC_SERROR] = "SError", + [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", + [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", + [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", + [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", + [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", + [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", + [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)", + [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)", + [ESR_ELx_EC_BRK64] = "BRK (AArch64)", +}; + +const char *esr_get_class_string(u32 esr) +{ + return esr_class_str[esr >> ESR_ELx_EC_SHIFT]; +} + /* * bad_mode handles the impossible case in the exception vector. */ @@ -382,8 +428,8 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) void __user *pc = (void __user *)instruction_pointer(regs); console_verbose(); - pr_crit("Bad mode in %s handler detected, code 0x%08x\n", - handler[reason], esr); + pr_crit("Bad mode in %s handler detected, code 0x%08x -- %s\n", + handler[reason], esr, esr_get_class_string(esr)); __show_regs(regs); info.si_signo = SIGILL; -- cgit v1.2.3 From ddeeefe2dfbe1fa6b116b9362b1bec465b64c873 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 12 Jan 2015 20:28:20 +0000 Subject: arm64/efi: efistub: Apply __init annotation This ensures all stub component are freed when the kernel proper is done booting, by prefixing the names of all ELF sections that have the SHF_ALLOC attribute with ".init". This approach ensures that even implicitly emitted allocated data (like initializer values and string literals) are covered. At the same time, remove some __init annotations in the stub that have now become redundant, and add the __init annotation to handle_kernel_image which will now trigger a section mismatch warning without it. Signed-off-by: Ard Biesheuvel Signed-off-by: Matt Fleming --- arch/arm64/kernel/efi-stub.c | 14 +++++++------- drivers/firmware/efi/libstub/Makefile | 14 ++++++++++++++ drivers/firmware/efi/libstub/arm-stub.c | 8 ++++---- drivers/firmware/efi/libstub/efi-stub-helper.c | 2 +- 4 files changed, 26 insertions(+), 12 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/efi-stub.c b/arch/arm64/kernel/efi-stub.c index d27dd982ff26..f5374065ad53 100644 --- a/arch/arm64/kernel/efi-stub.c +++ b/arch/arm64/kernel/efi-stub.c @@ -13,13 +13,13 @@ #include #include -efi_status_t handle_kernel_image(efi_system_table_t *sys_table, - unsigned long *image_addr, - unsigned long *image_size, - unsigned long *reserve_addr, - unsigned long *reserve_size, - unsigned long dram_base, - efi_loaded_image_t *image) +efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table, + unsigned long *image_addr, + unsigned long *image_size, + unsigned long *reserve_addr, + unsigned long *reserve_size, + unsigned long dram_base, + efi_loaded_image_t *image) { efi_status_t status; unsigned long kernel_size, kernel_memsize = 0; diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index b14bc2b9fb4d..8902f52e0998 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -24,3 +24,17 @@ lib-y := efi-stub-helper.o lib-$(CONFIG_EFI_ARMSTUB) += arm-stub.o fdt.o CFLAGS_fdt.o += -I$(srctree)/scripts/dtc/libfdt/ + +# +# arm64 puts the stub in the kernel proper, which will unnecessarily retain all +# code indefinitely unless it is annotated as __init/__initdata/__initconst etc. +# So let's apply the __init annotations at the section level, by prefixing +# the section names directly. This will ensure that even all the inline string +# literals are covered. +# +extra-$(CONFIG_ARM64) := $(lib-y) +lib-$(CONFIG_ARM64) := $(patsubst %.o,%.init.o,$(lib-y)) + +OBJCOPYFLAGS := --prefix-alloc-sections=.init +$(obj)/%.init.o: $(obj)/%.o FORCE + $(call if_changed,objcopy) diff --git a/drivers/firmware/efi/libstub/arm-stub.c b/drivers/firmware/efi/libstub/arm-stub.c index 75ee05964cbc..a1fda71c425a 100644 --- a/drivers/firmware/efi/libstub/arm-stub.c +++ b/drivers/firmware/efi/libstub/arm-stub.c @@ -17,10 +17,10 @@ #include "efistub.h" -static int __init efi_secureboot_enabled(efi_system_table_t *sys_table_arg) +static int efi_secureboot_enabled(efi_system_table_t *sys_table_arg) { - static efi_guid_t const var_guid __initconst = EFI_GLOBAL_VARIABLE_GUID; - static efi_char16_t const var_name[] __initconst = { + static efi_guid_t const var_guid = EFI_GLOBAL_VARIABLE_GUID; + static efi_char16_t const var_name[] = { 'S', 'e', 'c', 'u', 'r', 'e', 'B', 'o', 'o', 't', 0 }; efi_get_variable_t *f_getvar = sys_table_arg->runtime->get_variable; @@ -164,7 +164,7 @@ efi_status_t handle_kernel_image(efi_system_table_t *sys_table, * for both archictectures, with the arch-specific code provided in the * handle_kernel_image() function. */ -unsigned long __init efi_entry(void *handle, efi_system_table_t *sys_table, +unsigned long efi_entry(void *handle, efi_system_table_t *sys_table, unsigned long *image_addr) { efi_loaded_image_t *image; diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c index a920fec8fe88..9bd9fbb5bea8 100644 --- a/drivers/firmware/efi/libstub/efi-stub-helper.c +++ b/drivers/firmware/efi/libstub/efi-stub-helper.c @@ -101,7 +101,7 @@ fail: } -unsigned long __init get_dram_base(efi_system_table_t *sys_table_arg) +unsigned long get_dram_base(efi_system_table_t *sys_table_arg) { efi_status_t status; unsigned long map_size; -- cgit v1.2.3 From 7fe5d2b1daf6fd82857a8d0ee47547d7852ebe7b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 15 Jan 2015 12:01:06 +0000 Subject: arm64/efi: handle potential failure to remap memory map When remapping the UEFI memory map using ioremap_cache(), we have to deal with potential failure. Note that, even if the common case is for ioremap_cache() to return the existing linear mapping of the memory map, we cannot rely on that to be always the case, e.g., in the presence of a mem= kernel parameter. At the same time, remove a stale comment and move the memmap code together. Signed-off-by: Ard Biesheuvel Acked-by: Mark Rutland Acked-by: Mark Salter Signed-off-by: Catalin Marinas --- arch/arm64/kernel/efi.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index a98415b5979c..c9cb0fbe7aa4 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -229,19 +229,21 @@ static int __init arm64_enable_runtime_services(void) return -1; } - mapsize = memmap.map_end - memmap.map; - if (efi_runtime_disabled()) { pr_info("EFI runtime services will be disabled.\n"); return -1; } pr_info("Remapping and enabling EFI services.\n"); - /* replace early memmap mapping with permanent mapping */ + + mapsize = memmap.map_end - memmap.map; memmap.map = (__force void *)ioremap_cache((phys_addr_t)memmap.phys_map, mapsize); + if (!memmap.map) { + pr_err("Failed to remap EFI memory map\n"); + return -1; + } memmap.map_end = memmap.map + mapsize; - efi.memmap = &memmap; efi.systab = (__force void *)ioremap_cache(efi_system_table, -- cgit v1.2.3 From 6083fe74b7bfffc2c7be8c711596608bda0cda6e Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 15 Jan 2015 16:42:14 +0000 Subject: arm64: respect mem= for EFI When booting with EFI, we acquire the EFI memory map after parsing the early params. This unfortuantely renders the option useless as we call memblock_enforce_memory_limit (which uses memblock_remove_range behind the scenes) before we've added any memblocks. We end up removing nothing, then adding all of memory later when efi_init calls reserve_regions. Instead, we can log the limit and apply this later when we do the rest of the memblock work in memblock_init, which should work regardless of the presence of EFI. At the same time we may as well move the early parameter into arm64's mm/init.c, close to arm64_memblock_init. Any memory which must be mapped (e.g. for use by EFI runtime services) must be mapped explicitly reather than relying on the linear mapping, which may be truncated as a result of a mem= option passed on the kernel command line. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Acked-by: Ard Biesheuvel Tested-by: Ard Biesheuvel Cc: Leif Lindholm Cc: Will Deacon Signed-off-by: Catalin Marinas --- arch/arm64/kernel/setup.c | 19 ------------------- arch/arm64/mm/init.c | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 199d1b7809d7..207413fe08a0 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -322,25 +322,6 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys) dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name()); } -/* - * Limit the memory size that was specified via FDT. - */ -static int __init early_mem(char *p) -{ - phys_addr_t limit; - - if (!p) - return 1; - - limit = memparse(p, &p) & PAGE_MASK; - pr_notice("Memory limited to %lldMB\n", limit >> 20); - - memblock_enforce_memory_limit(limit); - - return 0; -} -early_param("mem", early_mem); - static void __init request_standard_resources(void) { struct memblock_region *region; diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index bac492c12fcc..11c7b701b681 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -136,10 +136,29 @@ static void arm64_memory_present(void) } #endif +static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX; + +/* + * Limit the memory size that was specified via FDT. + */ +static int __init early_mem(char *p) +{ + if (!p) + return 1; + + memory_limit = memparse(p, &p) & PAGE_MASK; + pr_notice("Memory limited to %lldMB\n", memory_limit >> 20); + + return 0; +} +early_param("mem", early_mem); + void __init arm64_memblock_init(void) { phys_addr_t dma_phys_limit = 0; + memblock_enforce_memory_limit(memory_limit); + /* * Register the kernel text, kernel data, initrd, and initial * pagetables with memblock. -- cgit v1.2.3 From 2f896d5866107e2926dcdec34a7d40bc56dd2951 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Thu, 22 Jan 2015 01:36:05 +0000 Subject: arm64: use fixmap for text patching When kernel text is marked as read only, it cannot be modified directly. Use a fixmap to modify the text instead in a similar manner to x86 and arm. Reviewed-by: Kees Cook Reviewed-by: Mark Rutland Tested-by: Kees Cook Tested-by: Mark Rutland Signed-off-by: Laura Abbott Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/fixmap.h | 1 + arch/arm64/kernel/insn.c | 47 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 9ef6eca905ca..defa0ff98250 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -49,6 +49,7 @@ enum fixed_addresses { FIX_BTMAP_END = __end_of_permanent_fixed_addresses, FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1, + FIX_TEXT_POKE0, __end_of_fixed_addresses }; diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c index 7e9327a0986d..27d4864577e5 100644 --- a/arch/arm64/kernel/insn.c +++ b/arch/arm64/kernel/insn.c @@ -17,14 +17,19 @@ * along with this program. If not, see . */ #include +#include #include #include +#include #include +#include #include +#include #include #include #include +#include #include #define AARCH64_INSN_SF_BIT BIT(31) @@ -72,6 +77,29 @@ bool __kprobes aarch64_insn_is_nop(u32 insn) } } +static DEFINE_SPINLOCK(patch_lock); + +static void __kprobes *patch_map(void *addr, int fixmap) +{ + unsigned long uintaddr = (uintptr_t) addr; + bool module = !core_kernel_text(uintaddr); + struct page *page; + + if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX)) + page = vmalloc_to_page(addr); + else + page = virt_to_page(addr); + + BUG_ON(!page); + set_fixmap(fixmap, page_to_phys(page)); + + return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK)); +} + +static void __kprobes patch_unmap(int fixmap) +{ + clear_fixmap(fixmap); +} /* * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always * little-endian. @@ -88,10 +116,27 @@ int __kprobes aarch64_insn_read(void *addr, u32 *insnp) return ret; } +static int __kprobes __aarch64_insn_write(void *addr, u32 insn) +{ + void *waddr = addr; + unsigned long flags = 0; + int ret; + + spin_lock_irqsave(&patch_lock, flags); + waddr = patch_map(addr, FIX_TEXT_POKE0); + + ret = probe_kernel_write(waddr, &insn, AARCH64_INSN_SIZE); + + patch_unmap(FIX_TEXT_POKE0); + spin_unlock_irqrestore(&patch_lock, flags); + + return ret; +} + int __kprobes aarch64_insn_write(void *addr, u32 insn) { insn = cpu_to_le32(insn); - return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE); + return __aarch64_insn_write(addr, insn); } static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn) -- cgit v1.2.3 From da141706aea52c1a9fbd28cb8d289b78819f5436 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Wed, 21 Jan 2015 17:36:06 -0800 Subject: arm64: add better page protections to arm64 Add page protections for arm64 similar to those in arm. This is for security reasons to prevent certain classes of exploits. The current method: - Map all memory as either RWX or RW. We round to the nearest section to avoid creating page tables before everything is mapped - Once everything is mapped, if either end of the RWX section should not be X, we split the PMD and remap as necessary - When initmem is to be freed, we change the permissions back to RW (using stop machine if necessary to flush the TLB) - If CONFIG_DEBUG_RODATA is set, the read only sections are set read only. Acked-by: Ard Biesheuvel Tested-by: Kees Cook Tested-by: Ard Biesheuvel Signed-off-by: Laura Abbott Signed-off-by: Catalin Marinas --- arch/arm64/Kconfig.debug | 23 ++++ arch/arm64/include/asm/cacheflush.h | 5 + arch/arm64/kernel/vmlinux.lds.S | 17 ++- arch/arm64/mm/init.c | 1 + arch/arm64/mm/mm.h | 2 + arch/arm64/mm/mmu.c | 211 ++++++++++++++++++++++++++++++++---- 6 files changed, 233 insertions(+), 26 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 5fdd6dce8061..4a8741073c90 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -66,4 +66,27 @@ config DEBUG_SET_MODULE_RONX against certain classes of kernel exploits. If in doubt, say "N". +config DEBUG_RODATA + bool "Make kernel text and rodata read-only" + help + If this is set, kernel text and rodata will be made read-only. This + is to help catch accidental or malicious attempts to change the + kernel's executable code. Additionally splits rodata from kernel + text so it can be made explicitly non-executable. + + If in doubt, say Y + +config DEBUG_ALIGN_RODATA + depends on DEBUG_RODATA && !ARM64_64K_PAGES + bool "Align linker sections up to SECTION_SIZE" + help + If this option is enabled, sections that may potentially be marked as + read only or non-executable will be aligned up to the section size of + the kernel. This prevents sections from being split into pages and + avoids a potential TLB penalty. The downside is an increase in + alignment and potentially wasted space. Turn on this option if + performance is more important than memory pressure. + + If in doubt, say N + endmenu diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index 7ae31a2cc6c0..67d309cc3b6b 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -152,4 +152,9 @@ int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); int set_memory_x(unsigned long addr, int numpages); int set_memory_nx(unsigned long addr, int numpages); + +#ifdef CONFIG_DEBUG_RODATA +void mark_rodata_ro(void); +#endif + #endif diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 9965ec87cbec..5d9d2dca530d 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -8,6 +8,7 @@ #include #include #include +#include #include "image.h" @@ -49,6 +50,14 @@ PECOFF_FILE_ALIGNMENT = 0x200; #define PECOFF_EDATA_PADDING #endif +#ifdef CONFIG_DEBUG_ALIGN_RODATA +#define ALIGN_DEBUG_RO . = ALIGN(1< #include #include +#include #include #include @@ -59,21 +60,43 @@ EXPORT_SYMBOL(phys_mem_access_prot); static void __init *early_alloc(unsigned long sz) { void *ptr = __va(memblock_alloc(sz, sz)); + BUG_ON(!ptr); memset(ptr, 0, sz); return ptr; } -static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, +/* + * remap a PMD into pages + */ +static void split_pmd(pmd_t *pmd, pte_t *pte) +{ + unsigned long pfn = pmd_pfn(*pmd); + int i = 0; + + do { + /* + * Need to have the least restrictive permissions available + * permissions will be fixed up later + */ + set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC)); + pfn++; + } while (pte++, i++, i < PTRS_PER_PTE); +} + +static void alloc_init_pte(pmd_t *pmd, unsigned long addr, unsigned long end, unsigned long pfn, - pgprot_t prot) + pgprot_t prot, + void *(*alloc)(unsigned long size)) { pte_t *pte; - if (pmd_none(*pmd)) { - pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t)); + if (pmd_none(*pmd) || pmd_bad(*pmd)) { + pte = alloc(PTRS_PER_PTE * sizeof(pte_t)); + if (pmd_sect(*pmd)) + split_pmd(pmd, pte); __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE); + flush_tlb_all(); } - BUG_ON(pmd_bad(*pmd)); pte = pte_offset_kernel(pmd, addr); do { @@ -82,9 +105,22 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, } while (pte++, addr += PAGE_SIZE, addr != end); } -static void __init alloc_init_pmd(struct mm_struct *mm, pud_t *pud, +void split_pud(pud_t *old_pud, pmd_t *pmd) +{ + unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT; + pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr); + int i = 0; + + do { + set_pmd(pmd, __pmd(addr | prot)); + addr += PMD_SIZE; + } while (pmd++, i++, i < PTRS_PER_PMD); +} + +static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, unsigned long addr, unsigned long end, - phys_addr_t phys, pgprot_t prot) + phys_addr_t phys, pgprot_t prot, + void *(*alloc)(unsigned long size)) { pmd_t *pmd; unsigned long next; @@ -93,8 +129,16 @@ static void __init alloc_init_pmd(struct mm_struct *mm, pud_t *pud, * Check for initial section mappings in the pgd/pud and remove them. */ if (pud_none(*pud) || pud_bad(*pud)) { - pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t)); + pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t)); + if (pud_sect(*pud)) { + /* + * need to have the 1G of mappings continue to be + * present + */ + split_pud(pud, pmd); + } pud_populate(mm, pud, pmd); + flush_tlb_all(); } pmd = pmd_offset(pud, addr); @@ -113,21 +157,34 @@ static void __init alloc_init_pmd(struct mm_struct *mm, pud_t *pud, flush_tlb_all(); } else { alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), - prot); + prot, alloc); } phys += next - addr; } while (pmd++, addr = next, addr != end); } -static void __init alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, +static inline bool use_1G_block(unsigned long addr, unsigned long next, + unsigned long phys) +{ + if (PAGE_SHIFT != 12) + return false; + + if (((addr | next | phys) & ~PUD_MASK) != 0) + return false; + + return true; +} + +static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, unsigned long addr, unsigned long end, - phys_addr_t phys, pgprot_t prot) + phys_addr_t phys, pgprot_t prot, + void *(*alloc)(unsigned long size)) { pud_t *pud; unsigned long next; if (pgd_none(*pgd)) { - pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t)); + pud = alloc(PTRS_PER_PUD * sizeof(pud_t)); pgd_populate(mm, pgd, pud); } BUG_ON(pgd_bad(*pgd)); @@ -139,8 +196,7 @@ static void __init alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, /* * For 4K granule only, attempt to put down a 1GB block */ - if ((PAGE_SHIFT == 12) && - ((addr | next | phys) & ~PUD_MASK) == 0) { + if (use_1G_block(addr, next, phys)) { pud_t old_pud = *pud; set_pud(pud, __pud(phys | pgprot_val(mk_sect_prot(prot)))); @@ -158,7 +214,7 @@ static void __init alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, flush_tlb_all(); } } else { - alloc_init_pmd(mm, pud, addr, next, phys, prot); + alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc); } phys += next - addr; } while (pud++, addr = next, addr != end); @@ -168,9 +224,10 @@ static void __init alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, * Create the page directory entries and any necessary page tables for the * mapping specified by 'md'. */ -static void __init __create_mapping(struct mm_struct *mm, pgd_t *pgd, +static void __create_mapping(struct mm_struct *mm, pgd_t *pgd, phys_addr_t phys, unsigned long virt, - phys_addr_t size, pgprot_t prot) + phys_addr_t size, pgprot_t prot, + void *(*alloc)(unsigned long size)) { unsigned long addr, length, end, next; @@ -180,13 +237,23 @@ static void __init __create_mapping(struct mm_struct *mm, pgd_t *pgd, end = addr + length; do { next = pgd_addr_end(addr, end); - alloc_init_pud(mm, pgd, addr, next, phys, prot); + alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc); phys += next - addr; } while (pgd++, addr = next, addr != end); } -static void __init create_mapping(phys_addr_t phys, unsigned long virt, - phys_addr_t size) +static void *late_alloc(unsigned long size) +{ + void *ptr; + + BUG_ON(size > PAGE_SIZE); + ptr = (void *)__get_free_page(PGALLOC_GFP); + BUG_ON(!ptr); + return ptr; +} + +static void __ref create_mapping(phys_addr_t phys, unsigned long virt, + phys_addr_t size, pgprot_t prot) { if (virt < VMALLOC_START) { pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", @@ -194,15 +261,71 @@ static void __init create_mapping(phys_addr_t phys, unsigned long virt, return; } __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt, - size, PAGE_KERNEL_EXEC); + size, prot, early_alloc); } void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot) { - __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot); + __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot, + early_alloc); +} + +static void create_mapping_late(phys_addr_t phys, unsigned long virt, + phys_addr_t size, pgprot_t prot) +{ + if (virt < VMALLOC_START) { + pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", + &phys, virt); + return; + } + + return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), + phys, virt, size, prot, late_alloc); +} + +#ifdef CONFIG_DEBUG_RODATA +static void __init __map_memblock(phys_addr_t start, phys_addr_t end) +{ + /* + * Set up the executable regions using the existing section mappings + * for now. This will get more fine grained later once all memory + * is mapped + */ + unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE); + unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); + + if (end < kernel_x_start) { + create_mapping(start, __phys_to_virt(start), + end - start, PAGE_KERNEL); + } else if (start >= kernel_x_end) { + create_mapping(start, __phys_to_virt(start), + end - start, PAGE_KERNEL); + } else { + if (start < kernel_x_start) + create_mapping(start, __phys_to_virt(start), + kernel_x_start - start, + PAGE_KERNEL); + create_mapping(kernel_x_start, + __phys_to_virt(kernel_x_start), + kernel_x_end - kernel_x_start, + PAGE_KERNEL_EXEC); + if (kernel_x_end < end) + create_mapping(kernel_x_end, + __phys_to_virt(kernel_x_end), + end - kernel_x_end, + PAGE_KERNEL); + } + +} +#else +static void __init __map_memblock(phys_addr_t start, phys_addr_t end) +{ + create_mapping(start, __phys_to_virt(start), end - start, + PAGE_KERNEL_EXEC); } +#endif static void __init map_mem(void) { @@ -248,14 +371,53 @@ static void __init map_mem(void) memblock_set_current_limit(limit); } #endif - - create_mapping(start, __phys_to_virt(start), end - start); + __map_memblock(start, end); } /* Limit no longer required. */ memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); } +void __init fixup_executable(void) +{ +#ifdef CONFIG_DEBUG_RODATA + /* now that we are actually fully mapped, make the start/end more fine grained */ + if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) { + unsigned long aligned_start = round_down(__pa(_stext), + SECTION_SIZE); + + create_mapping(aligned_start, __phys_to_virt(aligned_start), + __pa(_stext) - aligned_start, + PAGE_KERNEL); + } + + if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) { + unsigned long aligned_end = round_up(__pa(__init_end), + SECTION_SIZE); + create_mapping(__pa(__init_end), (unsigned long)__init_end, + aligned_end - __pa(__init_end), + PAGE_KERNEL); + } +#endif +} + +#ifdef CONFIG_DEBUG_RODATA +void mark_rodata_ro(void) +{ + create_mapping_late(__pa(_stext), (unsigned long)_stext, + (unsigned long)_etext - (unsigned long)_stext, + PAGE_KERNEL_EXEC | PTE_RDONLY); + +} +#endif + +void fixup_init(void) +{ + create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin, + (unsigned long)__init_end - (unsigned long)__init_begin, + PAGE_KERNEL); +} + /* * paging_init() sets up the page tables, initialises the zone memory * maps and sets up the zero page. @@ -265,6 +427,7 @@ void __init paging_init(void) void *zero_page; map_mem(); + fixup_executable(); /* * Finally flush the caches and tlb to ensure that we're in a -- cgit v1.2.3 From 60305db9884515ca063474e262b454f6da04e4e2 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 22 Jan 2015 10:01:40 +0000 Subject: arm64/efi: move virtmap init to early initcall Now that the create_mapping() code in mm/mmu.c is able to support setting up kernel page tables at initcall time, we can move the whole virtmap creation to arm64_enable_runtime_services() instead of having a distinct stage during early boot. This also allows us to drop the arm64-specific EFI_VIRTMAP flag. Signed-off-by: Ard Biesheuvel Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/efi.h | 18 ++----- arch/arm64/kernel/efi.c | 111 +++++++++++++++++++++---------------------- arch/arm64/kernel/setup.c | 1 - arch/arm64/mm/mmu.c | 2 +- 4 files changed, 59 insertions(+), 73 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index 7baf2cc04e1e..ef572206f1c3 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -6,10 +6,8 @@ #ifdef CONFIG_EFI extern void efi_init(void); -extern void efi_virtmap_init(void); #else #define efi_init() -#define efi_virtmap_init() #endif #define efi_call_virt(f, ...) \ @@ -53,23 +51,17 @@ extern void efi_virtmap_init(void); #define EFI_ALLOC_ALIGN SZ_64K /* - * On ARM systems, virtually remapped UEFI runtime services are set up in three + * On ARM systems, virtually remapped UEFI runtime services are set up in two * distinct stages: * - The stub retrieves the final version of the memory map from UEFI, populates * the virt_addr fields and calls the SetVirtualAddressMap() [SVAM] runtime * service to communicate the new mapping to the firmware (Note that the new * mapping is not live at this time) - * - During early boot, the page tables are allocated and populated based on the - * virt_addr fields in the memory map, but only if all descriptors with the - * EFI_MEMORY_RUNTIME attribute have a non-zero value for virt_addr. If this - * succeeds, the EFI_VIRTMAP flag is set to indicate that the virtual mappings - * have been installed successfully. - * - During an early initcall(), the UEFI Runtime Services are enabled and the - * EFI_RUNTIME_SERVICES bit set if some conditions are met, i.e., we need a - * non-early mapping of the UEFI system table, and we need to have the virtmap - * installed. + * - During an early initcall(), the EFI system table is permanently remapped + * and the virtual remapping of the UEFI Runtime Services regions is loaded + * into a private set of page tables. If this all succeeds, the Runtime + * Services are enabled and the EFI_RUNTIME_SERVICES bit set. */ -#define EFI_VIRTMAP EFI_ARCH_1 void efi_virtmap_load(void); void efi_virtmap_unload(void); diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index c9cb0fbe7aa4..b42c7b480e1e 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -38,6 +38,19 @@ struct efi_memory_map memmap; static u64 efi_system_table; +static pgd_t efi_pgd[PTRS_PER_PGD] __page_aligned_bss; + +static struct mm_struct efi_mm = { + .mm_rb = RB_ROOT, + .pgd = efi_pgd, + .mm_users = ATOMIC_INIT(2), + .mm_count = ATOMIC_INIT(1), + .mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem), + .page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock), + .mmlist = LIST_HEAD_INIT(efi_mm.mmlist), + INIT_MM_CONTEXT(efi_mm) +}; + static int uefi_debug __initdata; static int __init uefi_debug_setup(char *str) { @@ -213,6 +226,45 @@ void __init efi_init(void) return; reserve_regions(); + early_memunmap(memmap.map, params.mmap_size); +} + +static bool __init efi_virtmap_init(void) +{ + efi_memory_desc_t *md; + + for_each_efi_memory_desc(&memmap, md) { + u64 paddr, npages, size; + pgprot_t prot; + + if (!(md->attribute & EFI_MEMORY_RUNTIME)) + continue; + if (md->virt_addr == 0) + return false; + + paddr = md->phys_addr; + npages = md->num_pages; + memrange_efi_to_native(&paddr, &npages); + size = npages << PAGE_SHIFT; + + pr_info(" EFI remap 0x%016llx => %p\n", + md->phys_addr, (void *)md->virt_addr); + + /* + * Only regions of type EFI_RUNTIME_SERVICES_CODE need to be + * executable, everything else can be mapped with the XN bits + * set. + */ + if (!is_normal_ram(md)) + prot = __pgprot(PROT_DEVICE_nGnRE); + else if (md->type == EFI_RUNTIME_SERVICES_CODE) + prot = PAGE_KERNEL_EXEC; + else + prot = PAGE_KERNEL; + + create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot); + } + return true; } /* @@ -254,7 +306,7 @@ static int __init arm64_enable_runtime_services(void) } set_bit(EFI_SYSTEM_TABLES, &efi.flags); - if (!efi_enabled(EFI_VIRTMAP)) { + if (!efi_virtmap_init()) { pr_err("No UEFI virtual mapping was installed -- runtime services will not be available\n"); return -1; } @@ -283,19 +335,6 @@ static int __init arm64_dmi_init(void) } core_initcall(arm64_dmi_init); -static pgd_t efi_pgd[PTRS_PER_PGD] __page_aligned_bss; - -static struct mm_struct efi_mm = { - .mm_rb = RB_ROOT, - .pgd = efi_pgd, - .mm_users = ATOMIC_INIT(2), - .mm_count = ATOMIC_INIT(1), - .mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem), - .page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock), - .mmlist = LIST_HEAD_INIT(efi_mm.mmlist), - INIT_MM_CONTEXT(efi_mm) -}; - static void efi_set_pgd(struct mm_struct *mm) { cpu_switch_mm(mm->pgd, mm); @@ -315,47 +354,3 @@ void efi_virtmap_unload(void) efi_set_pgd(current->active_mm); preempt_enable(); } - -void __init efi_virtmap_init(void) -{ - efi_memory_desc_t *md; - - if (!efi_enabled(EFI_BOOT)) - return; - - for_each_efi_memory_desc(&memmap, md) { - u64 paddr, npages, size; - pgprot_t prot; - - if (!(md->attribute & EFI_MEMORY_RUNTIME)) - continue; - if (WARN(md->virt_addr == 0, - "UEFI virtual mapping incomplete or missing -- no entry found for 0x%llx\n", - md->phys_addr)) - return; - - paddr = md->phys_addr; - npages = md->num_pages; - memrange_efi_to_native(&paddr, &npages); - size = npages << PAGE_SHIFT; - - pr_info(" EFI remap 0x%016llx => %p\n", - md->phys_addr, (void *)md->virt_addr); - - /* - * Only regions of type EFI_RUNTIME_SERVICES_CODE need to be - * executable, everything else can be mapped with the XN bits - * set. - */ - if (!is_normal_ram(md)) - prot = __pgprot(PROT_DEVICE_nGnRE); - else if (md->type == EFI_RUNTIME_SERVICES_CODE) - prot = PAGE_KERNEL_EXEC; - else - prot = PAGE_KERNEL; - - create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot); - } - set_bit(EFI_VIRTMAP, &efi.flags); - early_memunmap(memmap.map, memmap.map_end - memmap.map); -} diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 207413fe08a0..bb10903887d4 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -382,7 +382,6 @@ void __init setup_arch(char **cmdline_p) paging_init(); request_standard_resources(); - efi_virtmap_init(); early_ioremap_reset(); unflatten_device_tree(); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 91d55b6efd8a..155cbb0a74b6 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -269,7 +269,7 @@ void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, pgprot_t prot) { __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot, - early_alloc); + late_alloc); } static void create_mapping_late(phys_addr_t phys, unsigned long virt, -- cgit v1.2.3 From 78d51e0b8b57728099a3da74f4a10b6f8c71b764 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 12 Jan 2015 20:48:54 +0000 Subject: arm64: implement generic IOMMU configuration Add the necessary call to of_iommu_init. Acked-by: Will Deacon Signed-off-by: Robin Murphy Signed-off-by: Catalin Marinas --- arch/arm64/kernel/setup.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index bb10903887d4..e8420f635bd4 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -405,6 +406,7 @@ void __init setup_arch(char **cmdline_p) static int __init arm64_device_init(void) { + of_iommu_init(); of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); return 0; } -- cgit v1.2.3 From 04597a65c5efc207257a736d339c6f2f5b00250f Mon Sep 17 00:00:00 2001 From: "Suzuki K. Poulose" Date: Wed, 21 Jan 2015 12:43:09 +0000 Subject: arm64: Track system support for mixed endian EL0 This patch keeps track of the mixed endian EL0 support across the system and provides helper functions to export it. The status is a boolean indicating whether all the CPUs on the system supports mixed endian at EL0. Signed-off-by: Suzuki K. Poulose Cc: Will Deacon Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/cpufeature.h | 2 ++ arch/arm64/include/asm/cputype.h | 14 ++++++++++++++ arch/arm64/kernel/cpuinfo.c | 22 ++++++++++++++++++++++ 3 files changed, 38 insertions(+) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 07547ccc1f2b..b6c16d5f622f 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -52,6 +52,8 @@ static inline void cpus_set_cap(unsigned int num) } void check_local_cpu_errata(void); +bool cpu_supports_mixed_endian_el0(void); +bool system_supports_mixed_endian_el0(void); #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 8adb986a3086..ee6403df9fe4 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -72,6 +72,15 @@ #define APM_CPU_PART_POTENZA 0x000 +#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16 +#define ID_AA64MMFR0_BIGENDEL0_MASK (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT) +#define ID_AA64MMFR0_BIGENDEL0(mmfr0) \ + (((mmfr0) & ID_AA64MMFR0_BIGENDEL0_MASK) >> ID_AA64MMFR0_BIGENDEL0_SHIFT) +#define ID_AA64MMFR0_BIGEND_SHIFT 8 +#define ID_AA64MMFR0_BIGEND_MASK (0xf << ID_AA64MMFR0_BIGEND_SHIFT) +#define ID_AA64MMFR0_BIGEND(mmfr0) \ + (((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT) + #ifndef __ASSEMBLY__ /* @@ -104,6 +113,11 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void) return read_cpuid(CTR_EL0); } +static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0) +{ + return (ID_AA64MMFR0_BIGEND(mmfr0) == 0x1) || + (ID_AA64MMFR0_BIGENDEL0(mmfr0) == 0x1); +} #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 49782282a027..929855691dae 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -35,6 +35,7 @@ */ DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data); static struct cpuinfo_arm64 boot_cpu_data; +static bool mixed_endian_el0 = true; static char *icache_policy_str[] = { [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN", @@ -68,6 +69,26 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); } +bool cpu_supports_mixed_endian_el0(void) +{ + return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); +} + +bool system_supports_mixed_endian_el0(void) +{ + return mixed_endian_el0; +} + +static void update_mixed_endian_el0_support(struct cpuinfo_arm64 *info) +{ + mixed_endian_el0 &= id_aa64mmfr0_mixed_endian_el0(info->reg_id_aa64mmfr0); +} + +static void update_cpu_features(struct cpuinfo_arm64 *info) +{ + update_mixed_endian_el0_support(info); +} + static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu) { if ((boot & mask) == (cur & mask)) @@ -215,6 +236,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) cpuinfo_detect_icache_policy(info); check_local_cpu_errata(); + update_cpu_features(info); } void cpuinfo_store_cpu(void) -- cgit v1.2.3 From 736d474f0fafd1486f178570bc47660ee9dfdef8 Mon Sep 17 00:00:00 2001 From: "Suzuki K. Poulose" Date: Wed, 21 Jan 2015 12:43:10 +0000 Subject: arm64: Consolidate hotplug notifier for instruction emulation As of now each insn_emulation has a cpu hotplug notifier that enables/disables the CPU feature bit for the functionality. This patch re-arranges the code, such that there is only one notifier that runs through the list of registered emulation hooks and runs their corresponding set_hw_mode. We do nothing when a CPU is dying as we will set the appropriate bits as it comes back online based on the state of the hooks. Signed-off-by: Mark Rutland Signed-off-by: Suzuki K. Poulose Cc: Will Deacon Cc: Punit Agrawal [catalin.marinas@arm.com: fix pr_warn compilation error] [catalin.marinas@arm.com: remove unnecessary "insn" check] Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/cputype.h | 2 + arch/arm64/kernel/armv8_deprecated.c | 125 +++++++++++++++++++++-------------- 2 files changed, 79 insertions(+), 48 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index ee6403df9fe4..68732e9a02fb 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -81,6 +81,8 @@ #define ID_AA64MMFR0_BIGEND(mmfr0) \ (((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT) +#define SCTLR_EL1_CP15BEN (0x1 << 5) + #ifndef __ASSEMBLY__ /* diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c index c363671d7509..68b955e1fd99 100644 --- a/arch/arm64/kernel/armv8_deprecated.c +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -19,6 +19,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include "trace-events-emulation.h" @@ -85,6 +86,57 @@ static void remove_emulation_hooks(struct insn_emulation_ops *ops) pr_notice("Removed %s emulation handler\n", ops->name); } +static void enable_insn_hw_mode(void *data) +{ + struct insn_emulation *insn = (struct insn_emulation *)data; + if (insn->ops->set_hw_mode) + insn->ops->set_hw_mode(true); +} + +static void disable_insn_hw_mode(void *data) +{ + struct insn_emulation *insn = (struct insn_emulation *)data; + if (insn->ops->set_hw_mode) + insn->ops->set_hw_mode(false); +} + +/* Run set_hw_mode(mode) on all active CPUs */ +static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable) +{ + if (!insn->ops->set_hw_mode) + return -EINVAL; + if (enable) + on_each_cpu(enable_insn_hw_mode, (void *)insn, true); + else + on_each_cpu(disable_insn_hw_mode, (void *)insn, true); + return 0; +} + +/* + * Run set_hw_mode for all insns on a starting CPU. + * Returns: + * 0 - If all the hooks ran successfully. + * -EINVAL - At least one hook is not supported by the CPU. + */ +static int run_all_insn_set_hw_mode(unsigned long cpu) +{ + int rc = 0; + unsigned long flags; + struct insn_emulation *insn; + + raw_spin_lock_irqsave(&insn_emulation_lock, flags); + list_for_each_entry(insn, &insn_emulation, node) { + bool enable = (insn->current_mode == INSN_HW); + if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) { + pr_warn("CPU[%ld] cannot support the emulation of %s", + cpu, insn->ops->name); + rc = -EINVAL; + } + } + raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); + return rc; +} + static int update_insn_emulation_mode(struct insn_emulation *insn, enum insn_emulation_mode prev) { @@ -97,10 +149,8 @@ static int update_insn_emulation_mode(struct insn_emulation *insn, remove_emulation_hooks(insn->ops); break; case INSN_HW: - if (insn->ops->set_hw_mode) { - insn->ops->set_hw_mode(false); + if (!run_all_cpu_set_hw_mode(insn, false)) pr_notice("Disabled %s support\n", insn->ops->name); - } break; } @@ -111,10 +161,9 @@ static int update_insn_emulation_mode(struct insn_emulation *insn, register_emulation_hooks(insn->ops); break; case INSN_HW: - if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true)) + ret = run_all_cpu_set_hw_mode(insn, true); + if (!ret) pr_notice("Enabled %s support\n", insn->ops->name); - else - ret = -EINVAL; break; } @@ -133,6 +182,8 @@ static void register_insn_emulation(struct insn_emulation_ops *ops) switch (ops->status) { case INSN_DEPRECATED: insn->current_mode = INSN_EMULATE; + /* Disable the HW mode if it was turned on at early boot time */ + run_all_cpu_set_hw_mode(insn, false); insn->max = INSN_HW; break; case INSN_OBSOLETE: @@ -453,8 +504,6 @@ ret: return 0; } -#define SCTLR_EL1_CP15BEN (1 << 5) - static inline void config_sctlr_el1(u32 clear, u32 set) { u32 val; @@ -465,48 +514,13 @@ static inline void config_sctlr_el1(u32 clear, u32 set) asm volatile("msr sctlr_el1, %0" : : "r" (val)); } -static void enable_cp15_ben(void *info) -{ - config_sctlr_el1(0, SCTLR_EL1_CP15BEN); -} - -static void disable_cp15_ben(void *info) -{ - config_sctlr_el1(SCTLR_EL1_CP15BEN, 0); -} - -static int cpu_hotplug_notify(struct notifier_block *b, - unsigned long action, void *hcpu) -{ - switch (action) { - case CPU_STARTING: - case CPU_STARTING_FROZEN: - enable_cp15_ben(NULL); - return NOTIFY_DONE; - case CPU_DYING: - case CPU_DYING_FROZEN: - disable_cp15_ben(NULL); - return NOTIFY_DONE; - } - - return NOTIFY_OK; -} - -static struct notifier_block cpu_hotplug_notifier = { - .notifier_call = cpu_hotplug_notify, -}; - static int cp15_barrier_set_hw_mode(bool enable) { - if (enable) { - register_cpu_notifier(&cpu_hotplug_notifier); - on_each_cpu(enable_cp15_ben, NULL, true); - } else { - unregister_cpu_notifier(&cpu_hotplug_notifier); - on_each_cpu(disable_cp15_ben, NULL, true); - } - - return true; + if (enable) + config_sctlr_el1(0, SCTLR_EL1_CP15BEN); + else + config_sctlr_el1(SCTLR_EL1_CP15BEN, 0); + return 0; } static struct undef_hook cp15_barrier_hooks[] = { @@ -534,6 +548,20 @@ static struct insn_emulation_ops cp15_barrier_ops = { .set_hw_mode = cp15_barrier_set_hw_mode, }; +static int insn_cpu_hotplug_notify(struct notifier_block *b, + unsigned long action, void *hcpu) +{ + int rc = 0; + if ((action & ~CPU_TASKS_FROZEN) == CPU_STARTING) + rc = run_all_insn_set_hw_mode((unsigned long)hcpu); + + return notifier_from_errno(rc); +} + +static struct notifier_block insn_cpu_hotplug_notifier = { + .notifier_call = insn_cpu_hotplug_notify, +}; + /* * Invoked as late_initcall, since not needed before init spawned. */ @@ -545,6 +573,7 @@ static int __init armv8_deprecated_init(void) if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) register_insn_emulation(&cp15_barrier_ops); + register_cpu_notifier(&insn_cpu_hotplug_notifier); register_insn_emulation_sysctl(ctl_abi); return 0; -- cgit v1.2.3 From 2d888f48e056119495847a269a435d5c3d9df349 Mon Sep 17 00:00:00 2001 From: "Suzuki K. Poulose" Date: Wed, 21 Jan 2015 12:43:11 +0000 Subject: arm64: Emulate SETEND for AArch32 tasks Emulate deprecated 'setend' instruction for AArch32 bit tasks. setend [le/be] - Sets the endianness of EL0 On systems with CPUs which support mixed endian at EL0, the hardware support for the instruction can be enabled by setting the SCTLR_EL1.SED bit. Like the other emulated instructions it is controlled by an entry in /proc/sys/abi/. For more information see : Documentation/arm64/legacy_instructions.txt The instruction is emulated by setting/clearing the SPSR_EL1.E bit, which will be reflected in the PSTATE.E in AArch32 context. This patch also restores the native endianness for the execution of signal handlers, since the process could have changed the endianness. Note: All CPUs on the system must have mixed endian support at EL0. Once the handler is registered, hotplugging a CPU which doesn't support mixed endian, could lead to unexpected results/behavior in applications. Signed-off-by: Suzuki K. Poulose Cc: Will Deacon Cc: Punit Agrawal Signed-off-by: Catalin Marinas --- Documentation/arm64/legacy_instructions.txt | 12 +++++ arch/arm64/Kconfig | 15 ++++++ arch/arm64/include/asm/cputype.h | 1 + arch/arm64/include/asm/ptrace.h | 7 +++ arch/arm64/kernel/armv8_deprecated.c | 80 +++++++++++++++++++++++++++++ arch/arm64/kernel/signal32.c | 5 +- 6 files changed, 119 insertions(+), 1 deletion(-) (limited to 'arch/arm64/kernel') diff --git a/Documentation/arm64/legacy_instructions.txt b/Documentation/arm64/legacy_instructions.txt index a3b3da2ec6ed..01bf3d9fac85 100644 --- a/Documentation/arm64/legacy_instructions.txt +++ b/Documentation/arm64/legacy_instructions.txt @@ -32,6 +32,9 @@ The default mode depends on the status of the instruction in the architecture. Deprecated instructions should default to emulation while obsolete instructions must be undefined by default. +Note: Instruction emulation may not be possible in all cases. See +individual instruction notes for further information. + Supported legacy instructions ----------------------------- * SWP{B} @@ -43,3 +46,12 @@ Default: Undef (0) Node: /proc/sys/abi/cp15_barrier Status: Deprecated Default: Emulate (1) + +* SETEND +Node: /proc/sys/abi/setend +Status: Deprecated +Default: Emulate (1)* +Note: All the cpus on the system must have mixed endian support at EL0 +for this feature to be enabled. If a new CPU - which doesn't support mixed +endian - is hotplugged in after this feature has been enabled, there could +be unexpected results in the application. diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b1f9a20a3677..21a59bf37145 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -540,6 +540,21 @@ config CP15_BARRIER_EMULATION If unsure, say Y +config SETEND_EMULATION + bool "Emulate SETEND instruction" + help + The SETEND instruction alters the data-endianness of the + AArch32 EL0, and is deprecated in ARMv8. + + Say Y here to enable software emulation of the instruction + for AArch32 userspace code. + + Note: All the cpus on the system must have mixed endian support at EL0 + for this feature to be enabled. If a new CPU - which doesn't support mixed + endian - is hotplugged in after this feature has been enabled, there could + be unexpected results in the applications. + + If unsure, say Y endif endmenu diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 68732e9a02fb..a84ec605bed8 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -82,6 +82,7 @@ (((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT) #define SCTLR_EL1_CP15BEN (0x1 << 5) +#define SCTLR_EL1_SED (0x1 << 8) #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index 41ed9e13795e..d6dd9fdbc3be 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -58,6 +58,13 @@ #define COMPAT_PSR_Z_BIT 0x40000000 #define COMPAT_PSR_N_BIT 0x80000000 #define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */ + +#ifdef CONFIG_CPU_BIG_ENDIAN +#define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT +#else +#define COMPAT_PSR_ENDSTATE 0 +#endif + /* * These are 'magic' values for PTRACE_PEEKUSR that return info about where a * process is located in memory. diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c index 68b955e1fd99..7922c2e710ca 100644 --- a/arch/arm64/kernel/armv8_deprecated.c +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -548,6 +548,79 @@ static struct insn_emulation_ops cp15_barrier_ops = { .set_hw_mode = cp15_barrier_set_hw_mode, }; +static int setend_set_hw_mode(bool enable) +{ + if (!cpu_supports_mixed_endian_el0()) + return -EINVAL; + + if (enable) + config_sctlr_el1(SCTLR_EL1_SED, 0); + else + config_sctlr_el1(0, SCTLR_EL1_SED); + return 0; +} + +static int compat_setend_handler(struct pt_regs *regs, u32 big_endian) +{ + char *insn; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); + + if (big_endian) { + insn = "setend be"; + regs->pstate |= COMPAT_PSR_E_BIT; + } else { + insn = "setend le"; + regs->pstate &= ~COMPAT_PSR_E_BIT; + } + + trace_instruction_emulation(insn, regs->pc); + pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n", + current->comm, (unsigned long)current->pid, regs->pc); + + return 0; +} + +static int a32_setend_handler(struct pt_regs *regs, u32 instr) +{ + int rc = compat_setend_handler(regs, (instr >> 9) & 1); + regs->pc += 4; + return rc; +} + +static int t16_setend_handler(struct pt_regs *regs, u32 instr) +{ + int rc = compat_setend_handler(regs, (instr >> 3) & 1); + regs->pc += 2; + return rc; +} + +static struct undef_hook setend_hooks[] = { + { + .instr_mask = 0xfffffdff, + .instr_val = 0xf1010000, + .pstate_mask = COMPAT_PSR_MODE_MASK, + .pstate_val = COMPAT_PSR_MODE_USR, + .fn = a32_setend_handler, + }, + { + /* Thumb mode */ + .instr_mask = 0x0000fff7, + .instr_val = 0x0000b650, + .pstate_mask = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_MASK), + .pstate_val = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_USR), + .fn = t16_setend_handler, + }, + {} +}; + +static struct insn_emulation_ops setend_ops = { + .name = "setend", + .status = INSN_DEPRECATED, + .hooks = setend_hooks, + .set_hw_mode = setend_set_hw_mode, +}; + static int insn_cpu_hotplug_notify(struct notifier_block *b, unsigned long action, void *hcpu) { @@ -573,6 +646,13 @@ static int __init armv8_deprecated_init(void) if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) register_insn_emulation(&cp15_barrier_ops); + if (IS_ENABLED(CONFIG_SETEND_EMULATION)) { + if(system_supports_mixed_endian_el0()) + register_insn_emulation(&setend_ops); + else + pr_info("setend instruction emulation is not supported on the system"); + } + register_cpu_notifier(&insn_cpu_hotplug_notifier); register_insn_emulation_sysctl(ctl_abi); diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c index 192d900c058f..e299de396e9b 100644 --- a/arch/arm64/kernel/signal32.c +++ b/arch/arm64/kernel/signal32.c @@ -440,7 +440,7 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka, { compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler); compat_ulong_t retcode; - compat_ulong_t spsr = regs->pstate & ~PSR_f; + compat_ulong_t spsr = regs->pstate & ~(PSR_f | COMPAT_PSR_E_BIT); int thumb; /* Check if the handler is written for ARM or Thumb */ @@ -454,6 +454,9 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka, /* The IT state must be cleared for both ARM and Thumb-2 */ spsr &= ~COMPAT_PSR_IT_MASK; + /* Restore the original endianness */ + spsr |= COMPAT_PSR_ENDSTATE; + if (ka->sa.sa_flags & SA_RESTORER) { retcode = ptr_to_compat(ka->sa.sa_restorer); } else { -- cgit v1.2.3 From 0aaf0dae81b586134faeb52e28b7ad567629dd68 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Fri, 23 Jan 2015 05:36:42 +0000 Subject: smp, ARM64: Kill SMP single function call interrupt Commit 9a46ad6d6df3b54 "smp: make smp_call_function_many() use logic similar to smp_call_function_single()" has unified the way to handle single and multiple cross-CPU function calls. Now only one interrupt is needed for architecture specific code to support generic SMP function call interfaces, so kill the redundant single function call interrupt. Signed-off-by: Jiang Liu Acked-by: Will Deacon Cc: linux-arm-kernel@lists.infradead.org Cc: Arnd Bergmann Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/hardirq.h | 2 +- arch/arm64/kernel/smp.c | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h index e8a3268a891c..6aae421f4d73 100644 --- a/arch/arm64/include/asm/hardirq.h +++ b/arch/arm64/include/asm/hardirq.h @@ -20,7 +20,7 @@ #include #include -#define NR_IPI 6 +#define NR_IPI 5 typedef struct { unsigned int __softirq_pending; diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 7ae6ee085261..328b8ce4b007 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -65,7 +65,6 @@ struct secondary_data secondary_data; enum ipi_msg_type { IPI_RESCHEDULE, IPI_CALL_FUNC, - IPI_CALL_FUNC_SINGLE, IPI_CPU_STOP, IPI_TIMER, IPI_IRQ_WORK, @@ -483,7 +482,6 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { #define S(x,s) [x] = s S(IPI_RESCHEDULE, "Rescheduling interrupts"), S(IPI_CALL_FUNC, "Function call interrupts"), - S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"), S(IPI_CPU_STOP, "CPU stop interrupts"), S(IPI_TIMER, "Timer broadcast interrupts"), S(IPI_IRQ_WORK, "IRQ work interrupts"), @@ -527,7 +525,7 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask) void arch_send_call_function_single_ipi(int cpu) { - smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); + smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC); } #ifdef CONFIG_IRQ_WORK @@ -585,12 +583,6 @@ void handle_IPI(int ipinr, struct pt_regs *regs) irq_exit(); break; - case IPI_CALL_FUNC_SINGLE: - irq_enter(); - generic_smp_call_function_single_interrupt(); - irq_exit(); - break; - case IPI_CPU_STOP: irq_enter(); ipi_cpu_stop(cpu); -- cgit v1.2.3 From 0156411b1828771c09c92f034e7b81f702b84f07 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 6 Jan 2015 16:42:32 +0000 Subject: arm64: Implement the compat_sys_call_table in C Unlike the sys_call_table[], the compat one was implemented in sys32.S making it impossible to notice discrepancies between the number of compat syscalls and the __NR_compat_syscalls macro, the latter having to be defined in asm/unistd.h as including asm/unistd32.h would cause conflicts on __NR_* definitions. With this patch, incorrect __NR_compat_syscalls values will result in a build-time error. Signed-off-by: Catalin Marinas Suggested-by: Mark Rutland Acked-by: Mark Rutland --- arch/arm64/include/asm/unistd.h | 3 ++ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/entry.S | 2 +- arch/arm64/kernel/entry32.S | 105 ++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/sys32.S | 115 ---------------------------------------- arch/arm64/kernel/sys32.c | 51 ++++++++++++++++++ 6 files changed, 161 insertions(+), 117 deletions(-) create mode 100644 arch/arm64/kernel/entry32.S delete mode 100644 arch/arm64/kernel/sys32.S create mode 100644 arch/arm64/kernel/sys32.c (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index b780c6c76eec..159b44fff258 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -48,6 +48,9 @@ #endif #define __ARCH_WANT_SYS_CLONE + +#ifndef __COMPAT_SYSCALL_NR #include +#endif #define NR_syscalls (__NR_syscalls) diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 77d3d9568333..3b3b1cd3e7f0 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -19,7 +19,7 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ cpuinfo.o cpu_errata.o alternative.o cacheinfo.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ - sys_compat.o \ + sys_compat.o entry32.o \ ../../arm/kernel/opcodes.o arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 02e6af117762..cf21bb3bf752 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -433,7 +433,7 @@ el0_svc_compat: /* * AArch32 syscall handling */ - adr stbl, compat_sys_call_table // load compat syscall table pointer + adrp stbl, compat_sys_call_table // load compat syscall table pointer uxtw scno, w7 // syscall number in w7 (r7) mov sc_nr, #__NR_compat_syscalls b el0_svc_naked diff --git a/arch/arm64/kernel/entry32.S b/arch/arm64/kernel/entry32.S new file mode 100644 index 000000000000..9a8f6ae2530e --- /dev/null +++ b/arch/arm64/kernel/entry32.S @@ -0,0 +1,105 @@ +/* + * Compat system call wrappers + * + * Copyright (C) 2012 ARM Ltd. + * Authors: Will Deacon + * Catalin Marinas + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include + +/* + * System call wrappers for the AArch32 compatibility layer. + */ + +ENTRY(compat_sys_sigreturn_wrapper) + mov x0, sp + mov x27, #0 // prevent syscall restart handling (why) + b compat_sys_sigreturn +ENDPROC(compat_sys_sigreturn_wrapper) + +ENTRY(compat_sys_rt_sigreturn_wrapper) + mov x0, sp + mov x27, #0 // prevent syscall restart handling (why) + b compat_sys_rt_sigreturn +ENDPROC(compat_sys_rt_sigreturn_wrapper) + +ENTRY(compat_sys_statfs64_wrapper) + mov w3, #84 + cmp w1, #88 + csel w1, w3, w1, eq + b compat_sys_statfs64 +ENDPROC(compat_sys_statfs64_wrapper) + +ENTRY(compat_sys_fstatfs64_wrapper) + mov w3, #84 + cmp w1, #88 + csel w1, w3, w1, eq + b compat_sys_fstatfs64 +ENDPROC(compat_sys_fstatfs64_wrapper) + +/* + * Wrappers for AArch32 syscalls that either take 64-bit parameters + * in registers or that take 32-bit parameters which require sign + * extension. + */ +ENTRY(compat_sys_pread64_wrapper) + regs_to_64 x3, x4, x5 + b sys_pread64 +ENDPROC(compat_sys_pread64_wrapper) + +ENTRY(compat_sys_pwrite64_wrapper) + regs_to_64 x3, x4, x5 + b sys_pwrite64 +ENDPROC(compat_sys_pwrite64_wrapper) + +ENTRY(compat_sys_truncate64_wrapper) + regs_to_64 x1, x2, x3 + b sys_truncate +ENDPROC(compat_sys_truncate64_wrapper) + +ENTRY(compat_sys_ftruncate64_wrapper) + regs_to_64 x1, x2, x3 + b sys_ftruncate +ENDPROC(compat_sys_ftruncate64_wrapper) + +ENTRY(compat_sys_readahead_wrapper) + regs_to_64 x1, x2, x3 + mov w2, w4 + b sys_readahead +ENDPROC(compat_sys_readahead_wrapper) + +ENTRY(compat_sys_fadvise64_64_wrapper) + mov w6, w1 + regs_to_64 x1, x2, x3 + regs_to_64 x2, x4, x5 + mov w3, w6 + b sys_fadvise64_64 +ENDPROC(compat_sys_fadvise64_64_wrapper) + +ENTRY(compat_sys_sync_file_range2_wrapper) + regs_to_64 x2, x2, x3 + regs_to_64 x3, x4, x5 + b sys_sync_file_range2 +ENDPROC(compat_sys_sync_file_range2_wrapper) + +ENTRY(compat_sys_fallocate_wrapper) + regs_to_64 x2, x2, x3 + regs_to_64 x3, x4, x5 + b sys_fallocate +ENDPROC(compat_sys_fallocate_wrapper) diff --git a/arch/arm64/kernel/sys32.S b/arch/arm64/kernel/sys32.S deleted file mode 100644 index 423a5b3fc2be..000000000000 --- a/arch/arm64/kernel/sys32.S +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Compat system call wrappers - * - * Copyright (C) 2012 ARM Ltd. - * Authors: Will Deacon - * Catalin Marinas - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include - -#include -#include - -/* - * System call wrappers for the AArch32 compatibility layer. - */ - -compat_sys_sigreturn_wrapper: - mov x0, sp - mov x27, #0 // prevent syscall restart handling (why) - b compat_sys_sigreturn -ENDPROC(compat_sys_sigreturn_wrapper) - -compat_sys_rt_sigreturn_wrapper: - mov x0, sp - mov x27, #0 // prevent syscall restart handling (why) - b compat_sys_rt_sigreturn -ENDPROC(compat_sys_rt_sigreturn_wrapper) - -compat_sys_statfs64_wrapper: - mov w3, #84 - cmp w1, #88 - csel w1, w3, w1, eq - b compat_sys_statfs64 -ENDPROC(compat_sys_statfs64_wrapper) - -compat_sys_fstatfs64_wrapper: - mov w3, #84 - cmp w1, #88 - csel w1, w3, w1, eq - b compat_sys_fstatfs64 -ENDPROC(compat_sys_fstatfs64_wrapper) - -/* - * Wrappers for AArch32 syscalls that either take 64-bit parameters - * in registers or that take 32-bit parameters which require sign - * extension. - */ -compat_sys_pread64_wrapper: - regs_to_64 x3, x4, x5 - b sys_pread64 -ENDPROC(compat_sys_pread64_wrapper) - -compat_sys_pwrite64_wrapper: - regs_to_64 x3, x4, x5 - b sys_pwrite64 -ENDPROC(compat_sys_pwrite64_wrapper) - -compat_sys_truncate64_wrapper: - regs_to_64 x1, x2, x3 - b sys_truncate -ENDPROC(compat_sys_truncate64_wrapper) - -compat_sys_ftruncate64_wrapper: - regs_to_64 x1, x2, x3 - b sys_ftruncate -ENDPROC(compat_sys_ftruncate64_wrapper) - -compat_sys_readahead_wrapper: - regs_to_64 x1, x2, x3 - mov w2, w4 - b sys_readahead -ENDPROC(compat_sys_readahead_wrapper) - -compat_sys_fadvise64_64_wrapper: - mov w6, w1 - regs_to_64 x1, x2, x3 - regs_to_64 x2, x4, x5 - mov w3, w6 - b sys_fadvise64_64 -ENDPROC(compat_sys_fadvise64_64_wrapper) - -compat_sys_sync_file_range2_wrapper: - regs_to_64 x2, x2, x3 - regs_to_64 x3, x4, x5 - b sys_sync_file_range2 -ENDPROC(compat_sys_sync_file_range2_wrapper) - -compat_sys_fallocate_wrapper: - regs_to_64 x2, x2, x3 - regs_to_64 x3, x4, x5 - b sys_fallocate -ENDPROC(compat_sys_fallocate_wrapper) - -#undef __SYSCALL -#define __SYSCALL(x, y) .quad y // x - -/* - * The system calls table must be 4KB aligned. - */ - .align 12 -ENTRY(compat_sys_call_table) -#include diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c new file mode 100644 index 000000000000..2d5ab3c90b82 --- /dev/null +++ b/arch/arm64/kernel/sys32.c @@ -0,0 +1,51 @@ +/* + * arch/arm64/kernel/sys32.c + * + * Copyright (C) 2015 ARM Ltd. + * + * This program is free software(void); you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * Needed to avoid conflicting __NR_* macros between uapi/asm/unistd.h and + * asm/unistd32.h. + */ +#define __COMPAT_SYSCALL_NR + +#include +#include + +asmlinkage long compat_sys_sigreturn_wrapper(void); +asmlinkage long compat_sys_rt_sigreturn_wrapper(void); +asmlinkage long compat_sys_statfs64_wrapper(void); +asmlinkage long compat_sys_fstatfs64_wrapper(void); +asmlinkage long compat_sys_pread64_wrapper(void); +asmlinkage long compat_sys_pwrite64_wrapper(void); +asmlinkage long compat_sys_truncate64_wrapper(void); +asmlinkage long compat_sys_ftruncate64_wrapper(void); +asmlinkage long compat_sys_readahead_wrapper(void); +asmlinkage long compat_sys_fadvise64_64_wrapper(void); +asmlinkage long compat_sys_sync_file_range2_wrapper(void); +asmlinkage long compat_sys_fallocate_wrapper(void); + +#undef __SYSCALL +#define __SYSCALL(nr, sym) [nr] = sym, + +/* + * The sys_call_table array must be 4K aligned to be accessible from + * kernel/entry.S. + */ +void * const compat_sys_call_table[__NR_compat_syscalls] __aligned(4096) = { + [0 ... __NR_compat_syscalls - 1] = sys_ni_syscall, +#include +}; -- cgit v1.2.3 From 9648606946b3b0c608846dddb30482b48a6f5c68 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 6 Jan 2015 17:01:56 +0000 Subject: arm64: Remove asm/syscalls.h This patch moves the sys_rt_sigreturn_wrapper prototype to arch/arm64/kernel/sys.c and removes the asm/syscalls.h header. Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/syscalls.h | 30 ------------------------------ arch/arm64/kernel/sys.c | 3 +-- 2 files changed, 1 insertion(+), 32 deletions(-) delete mode 100644 arch/arm64/include/asm/syscalls.h (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/include/asm/syscalls.h b/arch/arm64/include/asm/syscalls.h deleted file mode 100644 index 48fe7c600e98..000000000000 --- a/arch/arm64/include/asm/syscalls.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2012 ARM Ltd. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#ifndef __ASM_SYSCALLS_H -#define __ASM_SYSCALLS_H - -#include -#include -#include - -/* - * System call wrappers implemented in kernel/entry.S. - */ -asmlinkage long sys_rt_sigreturn_wrapper(void); - -#include - -#endif /* __ASM_SYSCALLS_H */ diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c index 3fa98ff14f0e..dec351a9f3d6 100644 --- a/arch/arm64/kernel/sys.c +++ b/arch/arm64/kernel/sys.c @@ -39,10 +39,9 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len, /* * Wrappers to pass the pt_regs argument. */ +asmlinkage long sys_rt_sigreturn_wrapper(void); #define sys_rt_sigreturn sys_rt_sigreturn_wrapper -#include - #undef __SYSCALL #define __SYSCALL(nr, sym) [nr] = sym, -- cgit v1.2.3 From c623b33b4e9599c6ac5076f7db7369eb9869aa04 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 8 Jan 2015 11:42:59 +0000 Subject: arm64: make sys_call_table const As with x86, mark the sys_call_table const such that it will be placed in the .rodata section. This will cause attempts to modify the table (accidental or deliberate) to fail when strict page permissions are in place. In the absence of strict page permissions, there should be no functional change. Signed-off-by: Mark Rutland Acked-by: Will Deacon Signed-off-by: Catalin Marinas --- arch/arm64/kernel/sys.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c index dec351a9f3d6..75151aaf1a52 100644 --- a/arch/arm64/kernel/sys.c +++ b/arch/arm64/kernel/sys.c @@ -49,7 +49,7 @@ asmlinkage long sys_rt_sigreturn_wrapper(void); * The sys_call_table array must be 4K aligned to be accessible from * kernel/entry.S. */ -void *sys_call_table[__NR_syscalls] __aligned(4096) = { +void * const sys_call_table[__NR_syscalls] __aligned(4096) = { [0 ... __NR_syscalls - 1] = sys_ni_syscall, #include }; -- cgit v1.2.3 From af3cfdbf56b91785650f54e7c9a899d814b4b9fb Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Mon, 26 Jan 2015 18:33:44 +0000 Subject: arm64: kernel: remove ARM64_CPU_SUSPEND config option ARM64_CPU_SUSPEND config option was introduced to make code providing context save/restore selectable only on platforms requiring power management capabilities. Currently ARM64_CPU_SUSPEND depends on the PM_SLEEP config option which in turn is set by the SUSPEND config option. The introduction of CPU_IDLE for arm64 requires that code configured by ARM64_CPU_SUSPEND (context save/restore) should be compiled in in order to enable the CPU idle driver to rely on CPU operations carrying out context save/restore. The ARM64_CPUIDLE config option (ARM64 generic idle driver) is therefore forced to select ARM64_CPU_SUSPEND, even if there may be (ie PM_SLEEP) failed dependencies, which is not a clean way of handling the kernel configuration option. For these reasons, this patch removes the ARM64_CPU_SUSPEND config option and makes the context save/restore dependent on CPU_PM, which is selected whenever either SUSPEND or CPU_IDLE are configured, cleaning up dependencies in the process. This way, code previously configured through ARM64_CPU_SUSPEND is compiled in whenever a power management subsystem requires it to be present in the kernel (SUSPEND || CPU_IDLE), which is the behaviour expected on ARM64 kernels. The cpu_suspend and cpu_init_idle CPU operations are added only if CPU_IDLE is selected, since they are CPU_IDLE specific methods and should be grouped and defined accordingly. PSCI CPU operations are updated to reflect the introduced changes. Signed-off-by: Lorenzo Pieralisi Cc: Arnd Bergmann Cc: Will Deacon Cc: Krzysztof Kozlowski Cc: Daniel Lezcano Cc: Mark Rutland Signed-off-by: Catalin Marinas --- arch/arm64/Kconfig | 3 --- arch/arm64/include/asm/cpu_ops.h | 8 ++++---- arch/arm64/include/asm/cpuidle.h | 6 ++++++ arch/arm64/include/asm/suspend.h | 2 -- arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/asm-offsets.c | 2 +- arch/arm64/kernel/cpuidle.c | 20 ++++++++++++++++++++ arch/arm64/kernel/hw_breakpoint.c | 2 +- arch/arm64/kernel/psci.c | 2 -- arch/arm64/kernel/suspend.c | 21 --------------------- arch/arm64/mm/proc.S | 2 +- drivers/cpuidle/Kconfig.arm64 | 1 - drivers/cpuidle/cpuidle-arm64.c | 1 - 13 files changed, 34 insertions(+), 38 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 21a59bf37145..d3f7e4941231 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -642,9 +642,6 @@ source "kernel/power/Kconfig" config ARCH_SUSPEND_POSSIBLE def_bool y -config ARM64_CPU_SUSPEND - def_bool PM_SLEEP - endmenu menu "CPU Power Management" diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index 6f8e2ef9094a..da301ee7395c 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -28,8 +28,6 @@ struct device_node; * enable-method property. * @cpu_init: Reads any data necessary for a specific enable-method from the * devicetree, for a given cpu node and proposed logical id. - * @cpu_init_idle: Reads any data necessary to initialize CPU idle states from - * devicetree, for a given cpu node and proposed logical id. * @cpu_prepare: Early one-time preparation step for a cpu. If there is a * mechanism for doing so, tests whether it is possible to boot * the given CPU. @@ -42,6 +40,8 @@ struct device_node; * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the * cpu being killed. * @cpu_kill: Ensures a cpu has left the kernel. Called from another cpu. + * @cpu_init_idle: Reads any data necessary to initialize CPU idle states from + * devicetree, for a given cpu node and proposed logical id. * @cpu_suspend: Suspends a cpu and saves the required context. May fail owing * to wrong parameters or error conditions. Called from the * CPU being suspended. Must be called with IRQs disabled. @@ -49,7 +49,6 @@ struct device_node; struct cpu_operations { const char *name; int (*cpu_init)(struct device_node *, unsigned int); - int (*cpu_init_idle)(struct device_node *, unsigned int); int (*cpu_prepare)(unsigned int); int (*cpu_boot)(unsigned int); void (*cpu_postboot)(void); @@ -58,7 +57,8 @@ struct cpu_operations { void (*cpu_die)(unsigned int cpu); int (*cpu_kill)(unsigned int cpu); #endif -#ifdef CONFIG_ARM64_CPU_SUSPEND +#ifdef CONFIG_CPU_IDLE + int (*cpu_init_idle)(struct device_node *, unsigned int); int (*cpu_suspend)(unsigned long); #endif }; diff --git a/arch/arm64/include/asm/cpuidle.h b/arch/arm64/include/asm/cpuidle.h index b52a9932e2b1..0710654631e7 100644 --- a/arch/arm64/include/asm/cpuidle.h +++ b/arch/arm64/include/asm/cpuidle.h @@ -3,11 +3,17 @@ #ifdef CONFIG_CPU_IDLE extern int cpu_init_idle(unsigned int cpu); +extern int cpu_suspend(unsigned long arg); #else static inline int cpu_init_idle(unsigned int cpu) { return -EOPNOTSUPP; } + +static inline int cpu_suspend(unsigned long arg) +{ + return -EOPNOTSUPP; +} #endif #endif diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h index 456d67c1f0fa..003802f58963 100644 --- a/arch/arm64/include/asm/suspend.h +++ b/arch/arm64/include/asm/suspend.h @@ -23,6 +23,4 @@ struct sleep_save_sp { extern int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)); extern void cpu_resume(void); -extern int cpu_suspend(unsigned long); - #endif diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 3b3b1cd3e7f0..bef04afd6031 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -27,7 +27,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o -arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o +arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o arm64-obj-$(CONFIG_KGDB) += kgdb.o diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 9a9fce090d58..a2ae19403abb 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -152,7 +152,7 @@ int main(void) DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); #endif -#ifdef CONFIG_ARM64_CPU_SUSPEND +#ifdef CONFIG_CPU_PM DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx)); DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask)); diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c index 19d17f51db37..5c0896647fd1 100644 --- a/arch/arm64/kernel/cpuidle.c +++ b/arch/arm64/kernel/cpuidle.c @@ -29,3 +29,23 @@ int cpu_init_idle(unsigned int cpu) of_node_put(cpu_node); return ret; } + +/** + * cpu_suspend() - function to enter a low-power idle state + * @arg: argument to pass to CPU suspend operations + * + * Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU + * operations back-end error code otherwise. + */ +int cpu_suspend(unsigned long arg) +{ + int cpu = smp_processor_id(); + + /* + * If cpu_ops have not been registered or suspend + * has not been initialized, cpu_suspend call fails early. + */ + if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend) + return -EOPNOTSUPP; + return cpu_ops[cpu]->cpu_suspend(arg); +} diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index df1cf15377b4..98bbe06e469c 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -894,7 +894,7 @@ static struct notifier_block hw_breakpoint_reset_nb = { .notifier_call = hw_breakpoint_reset_notify, }; -#ifdef CONFIG_ARM64_CPU_SUSPEND +#ifdef CONFIG_CPU_PM extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)); #else static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index f1dbca7d5c96..3425f311c49e 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -540,8 +540,6 @@ const struct cpu_operations cpu_psci_ops = { .name = "psci", #ifdef CONFIG_CPU_IDLE .cpu_init_idle = cpu_psci_cpu_init_idle, -#endif -#ifdef CONFIG_ARM64_CPU_SUSPEND .cpu_suspend = cpu_psci_cpu_suspend, #endif #ifdef CONFIG_SMP diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index 2d6b6065fe7f..d7daf45ae7a2 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -51,26 +50,6 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) hw_breakpoint_restore = hw_bp_restore; } -/** - * cpu_suspend() - function to enter a low-power state - * @arg: argument to pass to CPU suspend operations - * - * Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU - * operations back-end error code otherwise. - */ -int cpu_suspend(unsigned long arg) -{ - int cpu = smp_processor_id(); - - /* - * If cpu_ops have not been registered or suspend - * has not been initialized, cpu_suspend call fails early. - */ - if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend) - return -EOPNOTSUPP; - return cpu_ops[cpu]->cpu_suspend(arg); -} - /* * __cpu_suspend * diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index b98fc8ce6a16..28eebfb6af76 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -102,7 +102,7 @@ ENTRY(cpu_do_idle) ret ENDPROC(cpu_do_idle) -#ifdef CONFIG_ARM64_CPU_SUSPEND +#ifdef CONFIG_CPU_PM /** * cpu_do_suspend - save CPU registers context * diff --git a/drivers/cpuidle/Kconfig.arm64 b/drivers/cpuidle/Kconfig.arm64 index d0a08ed1b2ee..6effb3656735 100644 --- a/drivers/cpuidle/Kconfig.arm64 +++ b/drivers/cpuidle/Kconfig.arm64 @@ -4,7 +4,6 @@ config ARM64_CPUIDLE bool "Generic ARM64 CPU idle Driver" - select ARM64_CPU_SUSPEND select DT_IDLE_STATES help Select this to enable generic cpuidle driver for ARM64. diff --git a/drivers/cpuidle/cpuidle-arm64.c b/drivers/cpuidle/cpuidle-arm64.c index 80704b931ba4..39a2c62716c3 100644 --- a/drivers/cpuidle/cpuidle-arm64.c +++ b/drivers/cpuidle/cpuidle-arm64.c @@ -19,7 +19,6 @@ #include #include -#include #include "dt_idle_states.h" -- cgit v1.2.3