From df049672dddde4a2fdacf63fb32eb80146e26841 Mon Sep 17 00:00:00 2001 From: Tero Roponen Date: Tue, 31 May 2011 10:24:39 +0300 Subject: x86: tsc: Remove unneeded DMI-based blacklisting The blacklist was added in response to my bug report (http://lkml.org/lkml/2006/1/19/362) and has never contained more than the one entry describing my old now dead ThinkPad 380XD laptop. As found out later (http://lkml.org/lkml/2007/11/29/50), this special treatment has been unnecessary for a long time, so it can be removed. Signed-off-by: Tero Roponen Signed-off-by: John Stultz --- arch/x86/kernel/tsc.c | 24 ------------------------ 1 file changed, 24 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 6cc6922262af..5c45c62cce5d 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -800,27 +799,6 @@ void mark_tsc_unstable(char *reason) EXPORT_SYMBOL_GPL(mark_tsc_unstable); -static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d) -{ - printk(KERN_NOTICE "%s detected: marking TSC unstable.\n", - d->ident); - tsc_unstable = 1; - return 0; -} - -/* List of systems that have known TSC problems */ -static struct dmi_system_id __initdata bad_tsc_dmi_table[] = { - { - .callback = dmi_mark_tsc_unstable, - .ident = "IBM Thinkpad 380XD", - .matches = { - DMI_MATCH(DMI_BOARD_VENDOR, "IBM"), - DMI_MATCH(DMI_BOARD_NAME, "2635FA0"), - }, - }, - {} -}; - static void __init check_system_tsc_reliable(void) { #ifdef CONFIG_MGEODE_LX @@ -1010,8 +988,6 @@ void __init tsc_init(void) lpj_fine = lpj; use_tsc_delay(); - /* Check and install the TSC clocksource */ - dmi_check_system(bad_tsc_dmi_table); if (unsynchronized_tsc()) mark_tsc_unstable("TSCs unsynchronized"); -- cgit v1.2.3 From cac0e0a78f722abd85b7f8d614ee0820f7672f58 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Tue, 31 May 2011 22:21:52 +0200 Subject: x86, asm: Flip SAVE_ARGS arguments logic This saves us the else part of the conditional statement in the macro. No functionality change. Signed-off-by: Borislav Petkov Link: http://lkml.kernel.org/r/1306873314-32523-3-git-send-email-bp@alien8.de Signed-off-by: H. Peter Anvin --- arch/x86/ia32/ia32entry.S | 6 +++--- arch/x86/include/asm/calling.h | 8 +++----- arch/x86/kernel/entry_64.S | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S index c1870dddd322..c5435dcea15c 100644 --- a/arch/x86/ia32/ia32entry.S +++ b/arch/x86/ia32/ia32entry.S @@ -143,7 +143,7 @@ ENTRY(ia32_sysenter_target) CFI_REL_OFFSET rip,0 pushq_cfi %rax cld - SAVE_ARGS 0,0,1 + SAVE_ARGS 0,1,0 /* no need to do an access_ok check here because rbp has been 32bit zero extended */ 1: movl (%rbp),%ebp @@ -289,7 +289,7 @@ ENTRY(ia32_cstar_target) * disabled irqs and here we enable it straight after entry: */ ENABLE_INTERRUPTS(CLBR_NONE) - SAVE_ARGS 8,1,1 + SAVE_ARGS 8,0,0 movl %eax,%eax /* zero extension */ movq %rax,ORIG_RAX-ARGOFFSET(%rsp) movq %rcx,RIP-ARGOFFSET(%rsp) @@ -419,7 +419,7 @@ ENTRY(ia32_syscall) cld /* note the registers are not zero extended to the sf. this could be a problem. */ - SAVE_ARGS 0,0,1 + SAVE_ARGS 0,1,0 GET_THREAD_INFO(%r10) orl $TS_COMPAT,TI_status(%r10) testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10) diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h index b67e06c4710e..b0b7d90d3054 100644 --- a/arch/x86/include/asm/calling.h +++ b/arch/x86/include/asm/calling.h @@ -85,22 +85,20 @@ For 32-bit we have the following conventions - kernel is built with #define ARGOFFSET R11 #define SWFRAME ORIG_RAX - .macro SAVE_ARGS addskip=0, norcx=0, nor891011=0 + .macro SAVE_ARGS addskip=0, save_rcx=1, save_r891011=1 subq $9*8+\addskip, %rsp CFI_ADJUST_CFA_OFFSET 9*8+\addskip movq_cfi rdi, 8*8 movq_cfi rsi, 7*8 movq_cfi rdx, 6*8 - .if \norcx - .else + .if \save_rcx movq_cfi rcx, 5*8 .endif movq_cfi rax, 4*8 - .if \nor891011 - .else + .if \save_r891011 movq_cfi r8, 3*8 movq_cfi r9, 2*8 movq_cfi r10, 1*8 diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 8a445a0c989e..e5ece6b6e716 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -473,7 +473,7 @@ ENTRY(system_call_after_swapgs) * and short: */ ENABLE_INTERRUPTS(CLBR_NONE) - SAVE_ARGS 8,1 + SAVE_ARGS 8,0 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) movq %rcx,RIP-ARGOFFSET(%rsp) CFI_REL_OFFSET rip,RIP-ARGOFFSET -- cgit v1.2.3 From 838feb47549a9b73534c6c1d7da4a9639a0750f4 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Tue, 31 May 2011 22:21:53 +0200 Subject: x86, asm: Flip RESTORE_ARGS arguments logic ... thus getting rid of the "else" part of the conditional statement in the macro. No functionality change. Signed-off-by: Borislav Petkov Link: http://lkml.kernel.org/r/1306873314-32523-4-git-send-email-bp@alien8.de Signed-off-by: H. Peter Anvin --- arch/x86/ia32/ia32entry.S | 4 ++-- arch/x86/include/asm/calling.h | 21 ++++++++------------- arch/x86/kernel/entry_64.S | 4 ++-- 3 files changed, 12 insertions(+), 17 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S index c5435dcea15c..a0e866d233ee 100644 --- a/arch/x86/ia32/ia32entry.S +++ b/arch/x86/ia32/ia32entry.S @@ -173,7 +173,7 @@ sysexit_from_sys_call: andl $~0x200,EFLAGS-R11(%rsp) movl RIP-R11(%rsp),%edx /* User %eip */ CFI_REGISTER rip,rdx - RESTORE_ARGS 1,24,1,1,1,1 + RESTORE_ARGS 0,24,0,0,0,0 xorq %r8,%r8 xorq %r9,%r9 xorq %r10,%r10 @@ -328,7 +328,7 @@ cstar_dispatch: jnz sysretl_audit sysretl_from_sys_call: andl $~TS_COMPAT,TI_status(%r10) - RESTORE_ARGS 1,-ARG_SKIP,1,1,1 + RESTORE_ARGS 0,-ARG_SKIP,0,0,0 movl RIP-ARGOFFSET(%rsp),%ecx CFI_REGISTER rip,rcx movl EFLAGS-ARGOFFSET(%rsp),%r11d diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h index b0b7d90d3054..a9e3a740f697 100644 --- a/arch/x86/include/asm/calling.h +++ b/arch/x86/include/asm/calling.h @@ -109,32 +109,27 @@ For 32-bit we have the following conventions - kernel is built with #define ARG_SKIP (9*8) - .macro RESTORE_ARGS skiprax=0, addskip=0, skiprcx=0, skipr11=0, \ - skipr8910=0, skiprdx=0 - .if \skipr11 - .else + .macro RESTORE_ARGS rstor_rax=1, addskip=0, rstor_rcx=1, rstor_r11=1, \ + rstor_r8910=1, rstor_rdx=1 + .if \rstor_r11 movq_cfi_restore 0*8, r11 .endif - .if \skipr8910 - .else + .if \rstor_r8910 movq_cfi_restore 1*8, r10 movq_cfi_restore 2*8, r9 movq_cfi_restore 3*8, r8 .endif - .if \skiprax - .else + .if \rstor_rax movq_cfi_restore 4*8, rax .endif - .if \skiprcx - .else + .if \rstor_rcx movq_cfi_restore 5*8, rcx .endif - .if \skiprdx - .else + .if \rstor_rdx movq_cfi_restore 6*8, rdx .endif @@ -193,7 +188,7 @@ For 32-bit we have the following conventions - kernel is built with .macro RESTORE_ALL addskip=0 RESTORE_REST - RESTORE_ARGS 0, \addskip + RESTORE_ARGS 1, \addskip .endm .macro icebp diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index e5ece6b6e716..0412bcbe171c 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -508,7 +508,7 @@ sysret_check: TRACE_IRQS_ON movq RIP-ARGOFFSET(%rsp),%rcx CFI_REGISTER rip,rcx - RESTORE_ARGS 0,-ARG_SKIP,1 + RESTORE_ARGS 1,-ARG_SKIP,0 /*CFI_REGISTER rflags,r11*/ movq PER_CPU_VAR(old_rsp), %rsp USERGS_SYSRET64 @@ -858,7 +858,7 @@ retint_restore_args: /* return to kernel space */ */ TRACE_IRQS_IRETQ restore_args: - RESTORE_ARGS 0,8,0 + RESTORE_ARGS 1,8,1 irq_return: INTERRUPT_RETURN -- cgit v1.2.3 From 8b4777a4b50cb0c84c1152eac85d24415fb6ff7d Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Sun, 5 Jun 2011 13:50:18 -0400 Subject: x86-64: Document some of entry_64.S Signed-off-by: Andy Lutomirski Cc: Jesper Juhl Cc: Borislav Petkov Cc: Linus Torvalds Cc: Arjan van de Ven Cc: Jan Beulich Cc: richard -rw- weinberger Cc: Mikael Pettersson Cc: Andi Kleen Cc: Brian Gerst Cc: Louis Rilling Cc: Valdis.Kletnieks@vt.edu Cc: pageexec@freemail.hu Link: http://lkml.kernel.org/r/fc134867cc550977cc996866129e11a16ba0f9ea.1307292171.git.luto@mit.edu Signed-off-by: Ingo Molnar --- Documentation/x86/entry_64.txt | 98 ++++++++++++++++++++++++++++++++++++++++++ arch/x86/kernel/entry_64.S | 2 + 2 files changed, 100 insertions(+) create mode 100644 Documentation/x86/entry_64.txt (limited to 'arch/x86/kernel') diff --git a/Documentation/x86/entry_64.txt b/Documentation/x86/entry_64.txt new file mode 100644 index 000000000000..7869f14d055c --- /dev/null +++ b/Documentation/x86/entry_64.txt @@ -0,0 +1,98 @@ +This file documents some of the kernel entries in +arch/x86/kernel/entry_64.S. A lot of this explanation is adapted from +an email from Ingo Molnar: + +http://lkml.kernel.org/r/<20110529191055.GC9835%40elte.hu> + +The x86 architecture has quite a few different ways to jump into +kernel code. Most of these entry points are registered in +arch/x86/kernel/traps.c and implemented in arch/x86/kernel/entry_64.S +and arch/x86/ia32/ia32entry.S. + +The IDT vector assignments are listed in arch/x86/include/irq_vectors.h. + +Some of these entries are: + + - system_call: syscall instruction from 64-bit code. + + - ia32_syscall: int 0x80 from 32-bit or 64-bit code; compat syscall + either way. + + - ia32_syscall, ia32_sysenter: syscall and sysenter from 32-bit + code + + - interrupt: An array of entries. Every IDT vector that doesn't + explicitly point somewhere else gets set to the corresponding + value in interrupts. These point to a whole array of + magically-generated functions that make their way to do_IRQ with + the interrupt number as a parameter. + + - emulate_vsyscall: int 0xcc, a special non-ABI entry used by + vsyscall emulation. + + - APIC interrupts: Various special-purpose interrupts for things + like TLB shootdown. + + - Architecturally-defined exceptions like divide_error. + +There are a few complexities here. The different x86-64 entries +have different calling conventions. The syscall and sysenter +instructions have their own peculiar calling conventions. Some of +the IDT entries push an error code onto the stack; others don't. +IDT entries using the IST alternative stack mechanism need their own +magic to get the stack frames right. (You can find some +documentation in the AMD APM, Volume 2, Chapter 8 and the Intel SDM, +Volume 3, Chapter 6.) + +Dealing with the swapgs instruction is especially tricky. Swapgs +toggles whether gs is the kernel gs or the user gs. The swapgs +instruction is rather fragile: it must nest perfectly and only in +single depth, it should only be used if entering from user mode to +kernel mode and then when returning to user-space, and precisely +so. If we mess that up even slightly, we crash. + +So when we have a secondary entry, already in kernel mode, we *must +not* use SWAPGS blindly - nor must we forget doing a SWAPGS when it's +not switched/swapped yet. + +Now, there's a secondary complication: there's a cheap way to test +which mode the CPU is in and an expensive way. + +The cheap way is to pick this info off the entry frame on the kernel +stack, from the CS of the ptregs area of the kernel stack: + + xorl %ebx,%ebx + testl $3,CS+8(%rsp) + je error_kernelspace + SWAPGS + +The expensive (paranoid) way is to read back the MSR_GS_BASE value +(which is what SWAPGS modifies): + + movl $1,%ebx + movl $MSR_GS_BASE,%ecx + rdmsr + testl %edx,%edx + js 1f /* negative -> in kernel */ + SWAPGS + xorl %ebx,%ebx +1: ret + +and the whole paranoid non-paranoid macro complexity is about whether +to suffer that RDMSR cost. + +If we are at an interrupt or user-trap/gate-alike boundary then we can +use the faster check: the stack will be a reliable indicator of +whether SWAPGS was already done: if we see that we are a secondary +entry interrupting kernel mode execution, then we know that the GS +base has already been switched. If it says that we interrupted +user-space execution then we must do the SWAPGS. + +But if we are in an NMI/MCE/DEBUG/whatever super-atomic entry context, +which might have triggered right after a normal entry wrote CS to the +stack but before we executed SWAPGS, then the only safe way to check +for GS is the slower method: the RDMSR. + +So we try only to mark those entry methods 'paranoid' that absolutely +need the more expensive check for the GS base - and we generate all +'normal' entry points with the regular (faster) entry macros. diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 8a445a0c989e..72c4a777bb91 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -9,6 +9,8 @@ /* * entry.S contains the system-call and fault low-level handling routines. * + * Some of this is documented in Documentation/x86/entry_64.txt + * * NOTE: This code handles signal-recognition, which happens every time * after an interrupt and after each system call. * -- cgit v1.2.3 From 9fd67b4ed0714ab718f1f9bd14c344af336a6df7 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Sun, 5 Jun 2011 13:50:19 -0400 Subject: x86-64: Give vvars their own page Move vvars out of the vsyscall page into their own page and mark it NX. Without this patch, an attacker who can force a daemon to call some fixed address could wait until the time contains, say, 0xCD80, and then execute the current time. Signed-off-by: Andy Lutomirski Cc: Jesper Juhl Cc: Borislav Petkov Cc: Linus Torvalds Cc: Arjan van de Ven Cc: Jan Beulich Cc: richard -rw- weinberger Cc: Mikael Pettersson Cc: Andi Kleen Cc: Brian Gerst Cc: Louis Rilling Cc: Valdis.Kletnieks@vt.edu Cc: pageexec@freemail.hu Link: http://lkml.kernel.org/r/b1460f81dc4463d66ea3f2b5ce240f58d48effec.1307292171.git.luto@mit.edu Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fixmap.h | 1 + arch/x86/include/asm/pgtable_types.h | 2 ++ arch/x86/include/asm/vvar.h | 22 ++++++++++------------ arch/x86/kernel/vmlinux.lds.S | 28 +++++++++++++++++----------- arch/x86/kernel/vsyscall_64.c | 5 +++++ 5 files changed, 35 insertions(+), 23 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h index 4729b2b63117..460c74e4852c 100644 --- a/arch/x86/include/asm/fixmap.h +++ b/arch/x86/include/asm/fixmap.h @@ -78,6 +78,7 @@ enum fixed_addresses { VSYSCALL_LAST_PAGE, VSYSCALL_FIRST_PAGE = VSYSCALL_LAST_PAGE + ((VSYSCALL_END-VSYSCALL_START) >> PAGE_SHIFT) - 1, + VVAR_PAGE, VSYSCALL_HPET, #endif FIX_DBGP_BASE, diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index d56187c6b838..6a29aed65902 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -108,6 +108,7 @@ #define __PAGE_KERNEL_UC_MINUS (__PAGE_KERNEL | _PAGE_PCD) #define __PAGE_KERNEL_VSYSCALL (__PAGE_KERNEL_RX | _PAGE_USER) #define __PAGE_KERNEL_VSYSCALL_NOCACHE (__PAGE_KERNEL_VSYSCALL | _PAGE_PCD | _PAGE_PWT) +#define __PAGE_KERNEL_VVAR (__PAGE_KERNEL_RO | _PAGE_USER) #define __PAGE_KERNEL_LARGE (__PAGE_KERNEL | _PAGE_PSE) #define __PAGE_KERNEL_LARGE_NOCACHE (__PAGE_KERNEL | _PAGE_CACHE_UC | _PAGE_PSE) #define __PAGE_KERNEL_LARGE_EXEC (__PAGE_KERNEL_EXEC | _PAGE_PSE) @@ -130,6 +131,7 @@ #define PAGE_KERNEL_LARGE_EXEC __pgprot(__PAGE_KERNEL_LARGE_EXEC) #define PAGE_KERNEL_VSYSCALL __pgprot(__PAGE_KERNEL_VSYSCALL) #define PAGE_KERNEL_VSYSCALL_NOCACHE __pgprot(__PAGE_KERNEL_VSYSCALL_NOCACHE) +#define PAGE_KERNEL_VVAR __pgprot(__PAGE_KERNEL_VVAR) #define PAGE_KERNEL_IO __pgprot(__PAGE_KERNEL_IO) #define PAGE_KERNEL_IO_NOCACHE __pgprot(__PAGE_KERNEL_IO_NOCACHE) diff --git a/arch/x86/include/asm/vvar.h b/arch/x86/include/asm/vvar.h index a4eaca4a6133..de656ac2af41 100644 --- a/arch/x86/include/asm/vvar.h +++ b/arch/x86/include/asm/vvar.h @@ -10,15 +10,14 @@ * In normal kernel code, they are used like any other variable. * In user code, they are accessed through the VVAR macro. * - * Each of these variables lives in the vsyscall page, and each - * one needs a unique offset within the little piece of the page - * reserved for vvars. Specify that offset in DECLARE_VVAR. - * (There are 896 bytes available. If you mess up, the linker will - * catch it.) + * These variables live in a page of kernel data that has an extra RO + * mapping for userspace. Each variable needs a unique offset within + * that page; specify that offset with the DECLARE_VVAR macro. (If + * you mess up, the linker will catch it.) */ -/* Offset of vars within vsyscall page */ -#define VSYSCALL_VARS_OFFSET (3072 + 128) +/* Base address of vvars. This is not ABI. */ +#define VVAR_ADDRESS (-10*1024*1024 - 4096) #if defined(__VVAR_KERNEL_LDS) @@ -26,17 +25,17 @@ * right place. */ #define DECLARE_VVAR(offset, type, name) \ - EMIT_VVAR(name, VSYSCALL_VARS_OFFSET + offset) + EMIT_VVAR(name, offset) #else #define DECLARE_VVAR(offset, type, name) \ static type const * const vvaraddr_ ## name = \ - (void *)(VSYSCALL_START + VSYSCALL_VARS_OFFSET + (offset)); + (void *)(VVAR_ADDRESS + (offset)); #define DEFINE_VVAR(type, name) \ - type __vvar_ ## name \ - __attribute__((section(".vsyscall_var_" #name), aligned(16))) + type name \ + __attribute__((section(".vvar_" #name), aligned(16))) #define VVAR(name) (*vvaraddr_ ## name) @@ -49,4 +48,3 @@ DECLARE_VVAR(16, int, vgetcpu_mode) DECLARE_VVAR(128, struct vsyscall_gtod_data, vsyscall_gtod_data) #undef DECLARE_VVAR -#undef VSYSCALL_VARS_OFFSET diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 89aed99aafce..98b378dc7f86 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -161,12 +161,6 @@ SECTIONS #define VVIRT_OFFSET (VSYSCALL_ADDR - __vsyscall_0) #define VVIRT(x) (ADDR(x) - VVIRT_OFFSET) -#define EMIT_VVAR(x, offset) .vsyscall_var_ ## x \ - ADDR(.vsyscall_0) + offset \ - : AT(VLOAD(.vsyscall_var_ ## x)) { \ - *(.vsyscall_var_ ## x) \ - } \ - x = VVIRT(.vsyscall_var_ ## x); . = ALIGN(4096); __vsyscall_0 = .; @@ -192,19 +186,31 @@ SECTIONS *(.vsyscall_3) } -#define __VVAR_KERNEL_LDS -#include -#undef __VVAR_KERNEL_LDS - - . = __vsyscall_0 + PAGE_SIZE; + . = ALIGN(__vsyscall_0 + PAGE_SIZE, PAGE_SIZE); #undef VSYSCALL_ADDR #undef VLOAD_OFFSET #undef VLOAD #undef VVIRT_OFFSET #undef VVIRT + + __vvar_page = .; + + .vvar : AT(ADDR(.vvar) - LOAD_OFFSET) { + + /* Place all vvars at the offsets in asm/vvar.h. */ +#define EMIT_VVAR(name, offset) \ + . = offset; \ + *(.vvar_ ## name) +#define __VVAR_KERNEL_LDS +#include +#undef __VVAR_KERNEL_LDS #undef EMIT_VVAR + } :data + + . = ALIGN(__vvar_page + PAGE_SIZE, PAGE_SIZE); + #endif /* CONFIG_X86_64 */ /* Init code and data - will be freed after init */ diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index 3e682184d76c..3cf1cef75a6a 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -284,9 +284,14 @@ void __init map_vsyscall(void) { extern char __vsyscall_0; unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0); + extern char __vvar_page; + unsigned long physaddr_vvar_page = __pa_symbol(&__vvar_page); /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */ __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL); + __set_fixmap(VVAR_PAGE, physaddr_vvar_page, PAGE_KERNEL_VVAR); + BUILD_BUG_ON((unsigned long)__fix_to_virt(VVAR_PAGE) != + (unsigned long)VVAR_ADDRESS); } static int __init vsyscall_init(void) -- cgit v1.2.3 From 0d7b8547fb67d5c2a7d954c56b3715b0e708be4a Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Sun, 5 Jun 2011 13:50:20 -0400 Subject: x86-64: Remove kernel.vsyscall64 sysctl It's unnecessary overhead in code that's supposed to be highly optimized. Removing it allows us to remove one of the two syscall instructions in the vsyscall page. The only sensible use for it is for UML users, and it doesn't fully address inconsistent vsyscall results on UML. The real fix for UML is to stop using vsyscalls entirely. Signed-off-by: Andy Lutomirski Cc: Jesper Juhl Cc: Borislav Petkov Cc: Linus Torvalds Cc: Arjan van de Ven Cc: Jan Beulich Cc: richard -rw- weinberger Cc: Mikael Pettersson Cc: Andi Kleen Cc: Brian Gerst Cc: Louis Rilling Cc: Valdis.Kletnieks@vt.edu Cc: pageexec@freemail.hu Link: http://lkml.kernel.org/r/973ae803fe76f712da4b2740e66dccf452d3b1e4.1307292171.git.luto@mit.edu Signed-off-by: Ingo Molnar --- arch/x86/include/asm/vgtod.h | 1 - arch/x86/kernel/vsyscall_64.c | 34 +------------------------- arch/x86/vdso/vclock_gettime.c | 55 ++++++++++++++++-------------------------- 3 files changed, 22 insertions(+), 68 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h index 646b4c1ca695..aa5add855a91 100644 --- a/arch/x86/include/asm/vgtod.h +++ b/arch/x86/include/asm/vgtod.h @@ -11,7 +11,6 @@ struct vsyscall_gtod_data { time_t wall_time_sec; u32 wall_time_nsec; - int sysctl_enabled; struct timezone sys_tz; struct { /* extract of a clocksource struct */ cycle_t (*vread)(void); diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index 3cf1cef75a6a..9b2f3f51bc91 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -53,7 +53,6 @@ DEFINE_VVAR(int, vgetcpu_mode); DEFINE_VVAR(struct vsyscall_gtod_data, vsyscall_gtod_data) = { .lock = __SEQLOCK_UNLOCKED(__vsyscall_gtod_data.lock), - .sysctl_enabled = 1, }; void update_vsyscall_tz(void) @@ -103,15 +102,6 @@ static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz) return ret; } -static __always_inline long time_syscall(long *t) -{ - long secs; - asm volatile("syscall" - : "=a" (secs) - : "0" (__NR_time),"D" (t) : __syscall_clobber); - return secs; -} - static __always_inline void do_vgettimeofday(struct timeval * tv) { cycle_t now, base, mask, cycle_delta; @@ -122,8 +112,7 @@ static __always_inline void do_vgettimeofday(struct timeval * tv) seq = read_seqbegin(&VVAR(vsyscall_gtod_data).lock); vread = VVAR(vsyscall_gtod_data).clock.vread; - if (unlikely(!VVAR(vsyscall_gtod_data).sysctl_enabled || - !vread)) { + if (unlikely(!vread)) { gettimeofday(tv,NULL); return; } @@ -165,8 +154,6 @@ time_t __vsyscall(1) vtime(time_t *t) { unsigned seq; time_t result; - if (unlikely(!VVAR(vsyscall_gtod_data).sysctl_enabled)) - return time_syscall(t); do { seq = read_seqbegin(&VVAR(vsyscall_gtod_data).lock); @@ -227,22 +214,6 @@ static long __vsyscall(3) venosys_1(void) return -ENOSYS; } -#ifdef CONFIG_SYSCTL -static ctl_table kernel_table2[] = { - { .procname = "vsyscall64", - .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec }, - {} -}; - -static ctl_table kernel_root_table2[] = { - { .procname = "kernel", .mode = 0555, - .child = kernel_table2 }, - {} -}; -#endif - /* Assume __initcall executes before all user space. Hopefully kmod doesn't violate that. We'll find out if it does. */ static void __cpuinit vsyscall_set_cpu(int cpu) @@ -301,9 +272,6 @@ static int __init vsyscall_init(void) BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime)); BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE))); BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu)); -#ifdef CONFIG_SYSCTL - register_sysctl_table(kernel_root_table2); -#endif on_each_cpu(cpu_vsyscall_init, NULL, 1); /* notifier priority > KVM */ hotcpu_notifier(cpu_vsyscall_notifier, 30); diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c index a724905fdae7..cf54813ac527 100644 --- a/arch/x86/vdso/vclock_gettime.c +++ b/arch/x86/vdso/vclock_gettime.c @@ -116,21 +116,21 @@ notrace static noinline int do_monotonic_coarse(struct timespec *ts) notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) { - if (likely(gtod->sysctl_enabled)) - switch (clock) { - case CLOCK_REALTIME: - if (likely(gtod->clock.vread)) - return do_realtime(ts); - break; - case CLOCK_MONOTONIC: - if (likely(gtod->clock.vread)) - return do_monotonic(ts); - break; - case CLOCK_REALTIME_COARSE: - return do_realtime_coarse(ts); - case CLOCK_MONOTONIC_COARSE: - return do_monotonic_coarse(ts); - } + switch (clock) { + case CLOCK_REALTIME: + if (likely(gtod->clock.vread)) + return do_realtime(ts); + break; + case CLOCK_MONOTONIC: + if (likely(gtod->clock.vread)) + return do_monotonic(ts); + break; + case CLOCK_REALTIME_COARSE: + return do_realtime_coarse(ts); + case CLOCK_MONOTONIC_COARSE: + return do_monotonic_coarse(ts); + } + return vdso_fallback_gettime(clock, ts); } int clock_gettime(clockid_t, struct timespec *) @@ -139,7 +139,7 @@ int clock_gettime(clockid_t, struct timespec *) notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) { long ret; - if (likely(gtod->sysctl_enabled && gtod->clock.vread)) { + if (likely(gtod->clock.vread)) { if (likely(tv != NULL)) { BUILD_BUG_ON(offsetof(struct timeval, tv_usec) != offsetof(struct timespec, tv_nsec) || @@ -161,27 +161,14 @@ notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) int gettimeofday(struct timeval *, struct timezone *) __attribute__((weak, alias("__vdso_gettimeofday"))); -/* This will break when the xtime seconds get inaccurate, but that is - * unlikely */ - -static __always_inline long time_syscall(long *t) -{ - long secs; - asm volatile("syscall" - : "=a" (secs) - : "0" (__NR_time), "D" (t) : "cc", "r11", "cx", "memory"); - return secs; -} - +/* + * This will break when the xtime seconds get inaccurate, but that is + * unlikely + */ notrace time_t __vdso_time(time_t *t) { - time_t result; - - if (unlikely(!VVAR(vsyscall_gtod_data).sysctl_enabled)) - return time_syscall(t); - /* This is atomic on x86_64 so we don't need any locks. */ - result = ACCESS_ONCE(VVAR(vsyscall_gtod_data).wall_time_sec); + time_t result = ACCESS_ONCE(VVAR(vsyscall_gtod_data).wall_time_sec); if (t) *t = result; -- cgit v1.2.3 From d319bb79afa4039bda6f85661d6bf0c13299ce93 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Sun, 5 Jun 2011 13:50:21 -0400 Subject: x86-64: Map the HPET NX Currently the HPET mapping is a user-accessible syscall instruction at a fixed address some of the time. A sufficiently determined hacker might be able to guess when. Signed-off-by: Andy Lutomirski Cc: Jesper Juhl Cc: Borislav Petkov Cc: Linus Torvalds Cc: Arjan van de Ven Cc: Jan Beulich Cc: richard -rw- weinberger Cc: Mikael Pettersson Cc: Andi Kleen Cc: Brian Gerst Cc: Louis Rilling Cc: Valdis.Kletnieks@vt.edu Cc: pageexec@freemail.hu Link: http://lkml.kernel.org/r/ab41b525a4ca346b1ca1145d16fb8d181861a8aa.1307292171.git.luto@mit.edu Signed-off-by: Ingo Molnar --- arch/x86/include/asm/pgtable_types.h | 4 ++-- arch/x86/kernel/hpet.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h index 6a29aed65902..013286a10c2c 100644 --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -107,8 +107,8 @@ #define __PAGE_KERNEL_NOCACHE (__PAGE_KERNEL | _PAGE_PCD | _PAGE_PWT) #define __PAGE_KERNEL_UC_MINUS (__PAGE_KERNEL | _PAGE_PCD) #define __PAGE_KERNEL_VSYSCALL (__PAGE_KERNEL_RX | _PAGE_USER) -#define __PAGE_KERNEL_VSYSCALL_NOCACHE (__PAGE_KERNEL_VSYSCALL | _PAGE_PCD | _PAGE_PWT) #define __PAGE_KERNEL_VVAR (__PAGE_KERNEL_RO | _PAGE_USER) +#define __PAGE_KERNEL_VVAR_NOCACHE (__PAGE_KERNEL_VVAR | _PAGE_PCD | _PAGE_PWT) #define __PAGE_KERNEL_LARGE (__PAGE_KERNEL | _PAGE_PSE) #define __PAGE_KERNEL_LARGE_NOCACHE (__PAGE_KERNEL | _PAGE_CACHE_UC | _PAGE_PSE) #define __PAGE_KERNEL_LARGE_EXEC (__PAGE_KERNEL_EXEC | _PAGE_PSE) @@ -130,8 +130,8 @@ #define PAGE_KERNEL_LARGE_NOCACHE __pgprot(__PAGE_KERNEL_LARGE_NOCACHE) #define PAGE_KERNEL_LARGE_EXEC __pgprot(__PAGE_KERNEL_LARGE_EXEC) #define PAGE_KERNEL_VSYSCALL __pgprot(__PAGE_KERNEL_VSYSCALL) -#define PAGE_KERNEL_VSYSCALL_NOCACHE __pgprot(__PAGE_KERNEL_VSYSCALL_NOCACHE) #define PAGE_KERNEL_VVAR __pgprot(__PAGE_KERNEL_VVAR) +#define PAGE_KERNEL_VVAR_NOCACHE __pgprot(__PAGE_KERNEL_VVAR_NOCACHE) #define PAGE_KERNEL_IO __pgprot(__PAGE_KERNEL_IO) #define PAGE_KERNEL_IO_NOCACHE __pgprot(__PAGE_KERNEL_IO_NOCACHE) diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 6781765b3a0d..e9f5605e4748 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -71,7 +71,7 @@ static inline void hpet_set_mapping(void) { hpet_virt_address = ioremap_nocache(hpet_address, HPET_MMAP_SIZE); #ifdef CONFIG_X86_64 - __set_fixmap(VSYSCALL_HPET, hpet_address, PAGE_KERNEL_VSYSCALL_NOCACHE); + __set_fixmap(VSYSCALL_HPET, hpet_address, PAGE_KERNEL_VVAR_NOCACHE); #endif } -- cgit v1.2.3 From bb5fe2f78eadf5a52d8dcbf9a57728fd107af97b Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Sun, 5 Jun 2011 13:50:22 -0400 Subject: x86-64: Remove vsyscall number 3 (venosys) It just segfaults since April 2008 (a4928cff), so I'm pretty sure that nothing uses it. And having an empty section makes the linker script a bit fragile. Signed-off-by: Andy Lutomirski Cc: Jesper Juhl Cc: Borislav Petkov Cc: Linus Torvalds Cc: Arjan van de Ven Cc: Jan Beulich Cc: richard -rw- weinberger Cc: Mikael Pettersson Cc: Andi Kleen Cc: Brian Gerst Cc: Louis Rilling Cc: Valdis.Kletnieks@vt.edu Cc: pageexec@freemail.hu Link: http://lkml.kernel.org/r/4a4abcf47ecadc269f2391a313576fe6d06acef7.1307292171.git.luto@mit.edu Signed-off-by: Ingo Molnar --- arch/x86/kernel/vmlinux.lds.S | 4 ---- arch/x86/kernel/vsyscall_64.c | 5 ----- 2 files changed, 9 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 98b378dc7f86..4f90082fd640 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -182,10 +182,6 @@ SECTIONS *(.vsyscall_2) } - .vsyscall_3 ADDR(.vsyscall_0) + 3072: AT(VLOAD(.vsyscall_3)) { - *(.vsyscall_3) - } - . = ALIGN(__vsyscall_0 + PAGE_SIZE, PAGE_SIZE); #undef VSYSCALL_ADDR diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index 9b2f3f51bc91..70a5f6eebd6c 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -209,11 +209,6 @@ vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache) return 0; } -static long __vsyscall(3) venosys_1(void) -{ - return -ENOSYS; -} - /* Assume __initcall executes before all user space. Hopefully kmod doesn't violate that. We'll find out if it does. */ static void __cpuinit vsyscall_set_cpu(int cpu) -- cgit v1.2.3 From 5dfcea629a08b4684a019cd0cb59d0c9129a6c02 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Sun, 5 Jun 2011 13:50:23 -0400 Subject: x86-64: Fill unused parts of the vsyscall page with 0xcc Jumping to 0x00 might do something depending on the following bytes. Jumping to 0xcc is a trap. So fill the unused parts of the vsyscall page with 0xcc to make it useless for exploits to jump there. Signed-off-by: Andy Lutomirski Cc: Jesper Juhl Cc: Borislav Petkov Cc: Linus Torvalds Cc: Arjan van de Ven Cc: Jan Beulich Cc: richard -rw- weinberger Cc: Mikael Pettersson Cc: Andi Kleen Cc: Brian Gerst Cc: Louis Rilling Cc: Valdis.Kletnieks@vt.edu Cc: pageexec@freemail.hu Link: http://lkml.kernel.org/r/ed54bfcfbe50a9070d20ec1edbe0d149e22a4568.1307292171.git.luto@mit.edu Signed-off-by: Ingo Molnar --- arch/x86/kernel/vmlinux.lds.S | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 4f90082fd640..80174719910c 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -166,22 +166,20 @@ SECTIONS __vsyscall_0 = .; . = VSYSCALL_ADDR; - .vsyscall_0 : AT(VLOAD(.vsyscall_0)) { + .vsyscall : AT(VLOAD(.vsyscall)) { *(.vsyscall_0) - } :user - . = ALIGN(L1_CACHE_BYTES); - .vsyscall_fn : AT(VLOAD(.vsyscall_fn)) { + . = ALIGN(L1_CACHE_BYTES); *(.vsyscall_fn) - } - .vsyscall_1 ADDR(.vsyscall_0) + 1024: AT(VLOAD(.vsyscall_1)) { + . = 1024; *(.vsyscall_1) - } - .vsyscall_2 ADDR(.vsyscall_0) + 2048: AT(VLOAD(.vsyscall_2)) { + + . = 2048; *(.vsyscall_2) - } + . = 4096; /* Pad the whole page. */ + } :user =0xcc . = ALIGN(__vsyscall_0 + PAGE_SIZE, PAGE_SIZE); #undef VSYSCALL_ADDR -- cgit v1.2.3 From 5cec93c216db77c45f7ce970d46283bcb1933884 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Sun, 5 Jun 2011 13:50:24 -0400 Subject: x86-64: Emulate legacy vsyscalls There's a fair amount of code in the vsyscall page. It contains a syscall instruction (in the gettimeofday fallback) and who knows what will happen if an exploit jumps into the middle of some other code. Reduce the risk by replacing the vsyscalls with short magic incantations that cause the kernel to emulate the real vsyscalls. These incantations are useless if entered in the middle. This causes vsyscalls to be a little more expensive than real syscalls. Fortunately sensible programs don't use them. The only exception is time() which is still called by glibc through the vsyscall - but calling time() millions of times per second is not sensible. glibc has this fixed in the development tree. This patch is not perfect: the vread_tsc and vread_hpet functions are still at a fixed address. Fixing that might involve making alternative patching work in the vDSO. Signed-off-by: Andy Lutomirski Acked-by: Linus Torvalds Cc: Jesper Juhl Cc: Borislav Petkov Cc: Arjan van de Ven Cc: Jan Beulich Cc: richard -rw- weinberger Cc: Mikael Pettersson Cc: Andi Kleen Cc: Brian Gerst Cc: Louis Rilling Cc: Valdis.Kletnieks@vt.edu Cc: pageexec@freemail.hu Link: http://lkml.kernel.org/r/e64e1b3c64858820d12c48fa739efbd1485e79d5.1307292171.git.luto@mit.edu [ Removed the CONFIG option - it's simpler to just do it unconditionally. Tidied up the code as well. ] Signed-off-by: Ingo Molnar --- arch/x86/include/asm/irq_vectors.h | 6 +- arch/x86/include/asm/traps.h | 4 + arch/x86/include/asm/vsyscall.h | 12 ++ arch/x86/kernel/Makefile | 1 + arch/x86/kernel/entry_64.S | 2 + arch/x86/kernel/traps.c | 6 + arch/x86/kernel/vsyscall_64.c | 261 +++++++++++++++++-------------------- arch/x86/kernel/vsyscall_emu_64.S | 27 ++++ include/linux/seccomp.h | 10 ++ 9 files changed, 189 insertions(+), 140 deletions(-) create mode 100644 arch/x86/kernel/vsyscall_emu_64.S (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h index 6e976ee3b3ef..a563c509edcb 100644 --- a/arch/x86/include/asm/irq_vectors.h +++ b/arch/x86/include/asm/irq_vectors.h @@ -17,7 +17,8 @@ * Vectors 0 ... 31 : system traps and exceptions - hardcoded events * Vectors 32 ... 127 : device interrupts * Vector 128 : legacy int80 syscall interface - * Vectors 129 ... INVALIDATE_TLB_VECTOR_START-1 : device interrupts + * Vector 204 : legacy x86_64 vsyscall emulation + * Vectors 129 ... INVALIDATE_TLB_VECTOR_START-1 except 204 : device interrupts * Vectors INVALIDATE_TLB_VECTOR_START ... 255 : special interrupts * * 64-bit x86 has per CPU IDT tables, 32-bit has one shared IDT table. @@ -50,6 +51,9 @@ #ifdef CONFIG_X86_32 # define SYSCALL_VECTOR 0x80 #endif +#ifdef CONFIG_X86_64 +# define VSYSCALL_EMU_VECTOR 0xcc +#endif /* * Vectors 0x30-0x3f are used for ISA interrupts. diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 0310da67307f..2bae0a513b40 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -1,6 +1,8 @@ #ifndef _ASM_X86_TRAPS_H #define _ASM_X86_TRAPS_H +#include + #include #include /* TRAP_TRACE, ... */ @@ -38,6 +40,7 @@ asmlinkage void alignment_check(void); asmlinkage void machine_check(void); #endif /* CONFIG_X86_MCE */ asmlinkage void simd_coprocessor_error(void); +asmlinkage void emulate_vsyscall(void); dotraplinkage void do_divide_error(struct pt_regs *, long); dotraplinkage void do_debug(struct pt_regs *, long); @@ -64,6 +67,7 @@ dotraplinkage void do_alignment_check(struct pt_regs *, long); dotraplinkage void do_machine_check(struct pt_regs *, long); #endif dotraplinkage void do_simd_coprocessor_error(struct pt_regs *, long); +dotraplinkage void do_emulate_vsyscall(struct pt_regs *, long); #ifdef CONFIG_X86_32 dotraplinkage void do_iret_error(struct pt_regs *, long); #endif diff --git a/arch/x86/include/asm/vsyscall.h b/arch/x86/include/asm/vsyscall.h index d55597351f6a..bb710cb0cdc1 100644 --- a/arch/x86/include/asm/vsyscall.h +++ b/arch/x86/include/asm/vsyscall.h @@ -31,6 +31,18 @@ extern struct timezone sys_tz; extern void map_vsyscall(void); +/* Emulation */ + +static inline bool is_vsyscall_entry(unsigned long addr) +{ + return (addr & ~0xC00UL) == VSYSCALL_START; +} + +static inline int vsyscall_entry_nr(unsigned long addr) +{ + return (addr & 0xC00UL) >> 10; +} + #endif /* __KERNEL__ */ #endif /* _ASM_X86_VSYSCALL_H */ diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 90b06d4daee2..cc0469a65120 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -44,6 +44,7 @@ obj-y += probe_roms.o obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o obj-$(CONFIG_X86_64) += sys_x86_64.o x8664_ksyms_64.o obj-$(CONFIG_X86_64) += syscall_64.o vsyscall_64.o vread_tsc_64.o +obj-$(CONFIG_X86_64) += vsyscall_emu_64.o obj-y += bootflag.o e820.o obj-y += pci-dma.o quirks.o topology.o kdebugfs.o obj-y += alternative.o i8253.o pci-nommu.o hw_breakpoint.o diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 72c4a777bb91..e949793d6b93 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -1123,6 +1123,8 @@ zeroentry spurious_interrupt_bug do_spurious_interrupt_bug zeroentry coprocessor_error do_coprocessor_error errorentry alignment_check do_alignment_check zeroentry simd_coprocessor_error do_simd_coprocessor_error +zeroentry emulate_vsyscall do_emulate_vsyscall + /* Reload gs selector with exception handling */ /* edi: new selector */ diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index b9b67166f9de..fbc097a085ca 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -872,6 +872,12 @@ void __init trap_init(void) set_bit(SYSCALL_VECTOR, used_vectors); #endif +#ifdef CONFIG_X86_64 + BUG_ON(test_bit(VSYSCALL_EMU_VECTOR, used_vectors)); + set_system_intr_gate(VSYSCALL_EMU_VECTOR, &emulate_vsyscall); + set_bit(VSYSCALL_EMU_VECTOR, used_vectors); +#endif + /* * Should be a barrier for any external CPU state: */ diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index 70a5f6eebd6c..10cd8ac3395a 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -2,6 +2,8 @@ * Copyright (C) 2001 Andrea Arcangeli SuSE * Copyright 2003 Andi Kleen, SuSE Labs. * + * [ NOTE: this mechanism is now deprecated in favor of the vDSO. ] + * * Thanks to hpa@transmeta.com for some useful hint. * Special thanks to Ingo Molnar for his early experience with * a different vsyscall implementation for Linux/IA32 and for the name. @@ -11,10 +13,9 @@ * vsyscalls. One vsyscall can reserve more than 1 slot to avoid * jumping out of line if necessary. We cannot add more with this * mechanism because older kernels won't return -ENOSYS. - * If we want more than four we need a vDSO. * - * Note: the concept clashes with user mode linux. If you use UML and - * want per guest time just set the kernel.vsyscall64 sysctl to 0. + * Note: the concept clashes with user mode linux. UML users should + * use the vDSO. */ /* Disable profiling for userspace code: */ @@ -32,6 +33,8 @@ #include #include #include +#include +#include #include #include @@ -44,10 +47,7 @@ #include #include #include - -#define __vsyscall(nr) \ - __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace -#define __syscall_clobber "r11","cx","memory" +#include DEFINE_VVAR(int, vgetcpu_mode); DEFINE_VVAR(struct vsyscall_gtod_data, vsyscall_gtod_data) = @@ -71,146 +71,129 @@ void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, unsigned long flags; write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags); + /* copy vsyscall data */ - vsyscall_gtod_data.clock.vread = clock->vread; - vsyscall_gtod_data.clock.cycle_last = clock->cycle_last; - vsyscall_gtod_data.clock.mask = clock->mask; - vsyscall_gtod_data.clock.mult = mult; - vsyscall_gtod_data.clock.shift = clock->shift; - vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec; - vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec; - vsyscall_gtod_data.wall_to_monotonic = *wtm; - vsyscall_gtod_data.wall_time_coarse = __current_kernel_time(); + vsyscall_gtod_data.clock.vread = clock->vread; + vsyscall_gtod_data.clock.cycle_last = clock->cycle_last; + vsyscall_gtod_data.clock.mask = clock->mask; + vsyscall_gtod_data.clock.mult = mult; + vsyscall_gtod_data.clock.shift = clock->shift; + vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec; + vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec; + vsyscall_gtod_data.wall_to_monotonic = *wtm; + vsyscall_gtod_data.wall_time_coarse = __current_kernel_time(); + write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags); } -/* RED-PEN may want to readd seq locking, but then the variable should be - * write-once. - */ -static __always_inline void do_get_tz(struct timezone * tz) +static void warn_bad_vsyscall(const char *level, struct pt_regs *regs, + const char *message) { - *tz = VVAR(vsyscall_gtod_data).sys_tz; -} + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); + struct task_struct *tsk; -static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz) -{ - int ret; - asm volatile("syscall" - : "=a" (ret) - : "0" (__NR_gettimeofday),"D" (tv),"S" (tz) - : __syscall_clobber ); - return ret; -} + if (!show_unhandled_signals || !__ratelimit(&rs)) + return; -static __always_inline void do_vgettimeofday(struct timeval * tv) -{ - cycle_t now, base, mask, cycle_delta; - unsigned seq; - unsigned long mult, shift, nsec; - cycle_t (*vread)(void); - do { - seq = read_seqbegin(&VVAR(vsyscall_gtod_data).lock); - - vread = VVAR(vsyscall_gtod_data).clock.vread; - if (unlikely(!vread)) { - gettimeofday(tv,NULL); - return; - } - - now = vread(); - base = VVAR(vsyscall_gtod_data).clock.cycle_last; - mask = VVAR(vsyscall_gtod_data).clock.mask; - mult = VVAR(vsyscall_gtod_data).clock.mult; - shift = VVAR(vsyscall_gtod_data).clock.shift; - - tv->tv_sec = VVAR(vsyscall_gtod_data).wall_time_sec; - nsec = VVAR(vsyscall_gtod_data).wall_time_nsec; - } while (read_seqretry(&VVAR(vsyscall_gtod_data).lock, seq)); - - /* calculate interval: */ - cycle_delta = (now - base) & mask; - /* convert to nsecs: */ - nsec += (cycle_delta * mult) >> shift; - - while (nsec >= NSEC_PER_SEC) { - tv->tv_sec += 1; - nsec -= NSEC_PER_SEC; - } - tv->tv_usec = nsec / NSEC_PER_USEC; -} + tsk = current; -int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz) -{ - if (tv) - do_vgettimeofday(tv); - if (tz) - do_get_tz(tz); - return 0; + printk("%s%s[%d] %s ip:%lx sp:%lx ax:%lx si:%lx di:%lx\n", + level, tsk->comm, task_pid_nr(tsk), + message, regs->ip - 2, regs->sp, regs->ax, regs->si, regs->di); } -/* This will break when the xtime seconds get inaccurate, but that is - * unlikely */ -time_t __vsyscall(1) vtime(time_t *t) +void dotraplinkage do_emulate_vsyscall(struct pt_regs *regs, long error_code) { - unsigned seq; - time_t result; + const char *vsyscall_name; + struct task_struct *tsk; + unsigned long caller; + int vsyscall_nr; + long ret; + + /* Kernel code must never get here. */ + BUG_ON(!user_mode(regs)); + + local_irq_enable(); + + /* + * x86-ism here: regs->ip points to the instruction after the int 0xcc, + * and int 0xcc is two bytes long. + */ + if (!is_vsyscall_entry(regs->ip - 2)) { + warn_bad_vsyscall(KERN_WARNING, regs, "illegal int 0xcc (exploit attempt?)"); + goto sigsegv; + } + vsyscall_nr = vsyscall_entry_nr(regs->ip - 2); - do { - seq = read_seqbegin(&VVAR(vsyscall_gtod_data).lock); + if (get_user(caller, (unsigned long __user *)regs->sp) != 0) { + warn_bad_vsyscall(KERN_WARNING, regs, "int 0xcc with bad stack (exploit attempt?)"); + goto sigsegv; + } - result = VVAR(vsyscall_gtod_data).wall_time_sec; + tsk = current; + if (seccomp_mode(&tsk->seccomp)) + do_exit(SIGKILL); + + switch (vsyscall_nr) { + case 0: + vsyscall_name = "gettimeofday"; + ret = sys_gettimeofday( + (struct timeval __user *)regs->di, + (struct timezone __user *)regs->si); + break; + + case 1: + vsyscall_name = "time"; + ret = sys_time((time_t __user *)regs->di); + break; + + case 2: + vsyscall_name = "getcpu"; + ret = sys_getcpu((unsigned __user *)regs->di, + (unsigned __user *)regs->si, + 0); + break; + + default: + /* + * If we get here, then vsyscall_nr indicates that int 0xcc + * happened at an address in the vsyscall page that doesn't + * contain int 0xcc. That can't happen. + */ + BUG(); + } - } while (read_seqretry(&VVAR(vsyscall_gtod_data).lock, seq)); + if (ret == -EFAULT) { + /* + * Bad news -- userspace fed a bad pointer to a vsyscall. + * + * With a real vsyscall, that would have caused SIGSEGV. + * To make writing reliable exploits using the emulated + * vsyscalls harder, generate SIGSEGV here as well. + */ + warn_bad_vsyscall(KERN_INFO, regs, + "vsyscall fault (exploit attempt?)"); + goto sigsegv; + } - if (t) - *t = result; - return result; -} + regs->ax = ret; -/* Fast way to get current CPU and node. - This helps to do per node and per CPU caches in user space. - The result is not guaranteed without CPU affinity, but usually - works out because the scheduler tries to keep a thread on the same - CPU. + /* Emulate a ret instruction. */ + regs->ip = caller; + regs->sp += 8; - tcache must point to a two element sized long array. - All arguments can be NULL. */ -long __vsyscall(2) -vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache) -{ - unsigned int p; - unsigned long j = 0; - - /* Fast cache - only recompute value once per jiffies and avoid - relatively costly rdtscp/cpuid otherwise. - This works because the scheduler usually keeps the process - on the same CPU and this syscall doesn't guarantee its - results anyways. - We do this here because otherwise user space would do it on - its own in a likely inferior way (no access to jiffies). - If you don't like it pass NULL. */ - if (tcache && tcache->blob[0] == (j = VVAR(jiffies))) { - p = tcache->blob[1]; - } else if (VVAR(vgetcpu_mode) == VGETCPU_RDTSCP) { - /* Load per CPU data from RDTSCP */ - native_read_tscp(&p); - } else { - /* Load per CPU data from GDT */ - asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG)); - } - if (tcache) { - tcache->blob[0] = j; - tcache->blob[1] = p; - } - if (cpu) - *cpu = p & 0xfff; - if (node) - *node = p >> 12; - return 0; + local_irq_disable(); + return; + +sigsegv: + regs->ip -= 2; /* The faulting instruction should be the int 0xcc. */ + force_sig(SIGSEGV, current); } -/* Assume __initcall executes before all user space. Hopefully kmod - doesn't violate that. We'll find out if it does. */ +/* + * Assume __initcall executes before all user space. Hopefully kmod + * doesn't violate that. We'll find out if it does. + */ static void __cpuinit vsyscall_set_cpu(int cpu) { unsigned long d; @@ -221,13 +204,15 @@ static void __cpuinit vsyscall_set_cpu(int cpu) if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP)) write_rdtscp_aux((node << 12) | cpu); - /* Store cpu number in limit so that it can be loaded quickly - in user space in vgetcpu. - 12 bits for the CPU and 8 bits for the node. */ + /* + * Store cpu number in limit so that it can be loaded quickly + * in user space in vgetcpu. (12 bits for the CPU and 8 bits for the node) + */ d = 0x0f40000000000ULL; d |= cpu; d |= (node & 0xf) << 12; d |= (node >> 4) << 48; + write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S); } @@ -241,8 +226,10 @@ static int __cpuinit cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg) { long cpu = (long)arg; + if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1); + return NOTIFY_DONE; } @@ -256,21 +243,17 @@ void __init map_vsyscall(void) /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */ __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL); __set_fixmap(VVAR_PAGE, physaddr_vvar_page, PAGE_KERNEL_VVAR); - BUILD_BUG_ON((unsigned long)__fix_to_virt(VVAR_PAGE) != - (unsigned long)VVAR_ADDRESS); + BUILD_BUG_ON((unsigned long)__fix_to_virt(VVAR_PAGE) != (unsigned long)VVAR_ADDRESS); } static int __init vsyscall_init(void) { - BUG_ON(((unsigned long) &vgettimeofday != - VSYSCALL_ADDR(__NR_vgettimeofday))); - BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime)); - BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE))); - BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu)); + BUG_ON(VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)); + on_each_cpu(cpu_vsyscall_init, NULL, 1); /* notifier priority > KVM */ hotcpu_notifier(cpu_vsyscall_notifier, 30); + return 0; } - __initcall(vsyscall_init); diff --git a/arch/x86/kernel/vsyscall_emu_64.S b/arch/x86/kernel/vsyscall_emu_64.S new file mode 100644 index 000000000000..ffa845eae5ca --- /dev/null +++ b/arch/x86/kernel/vsyscall_emu_64.S @@ -0,0 +1,27 @@ +/* + * vsyscall_emu_64.S: Vsyscall emulation page + * + * Copyright (c) 2011 Andy Lutomirski + * + * Subject to the GNU General Public License, version 2 + */ + +#include +#include + +/* The unused parts of the page are filled with 0xcc by the linker script. */ + +.section .vsyscall_0, "a" +ENTRY(vsyscall_0) + int $VSYSCALL_EMU_VECTOR +END(vsyscall_0) + +.section .vsyscall_1, "a" +ENTRY(vsyscall_1) + int $VSYSCALL_EMU_VECTOR +END(vsyscall_1) + +.section .vsyscall_2, "a" +ENTRY(vsyscall_2) + int $VSYSCALL_EMU_VECTOR +END(vsyscall_2) diff --git a/include/linux/seccomp.h b/include/linux/seccomp.h index 167c33361d9c..cc7a4e9cc7ad 100644 --- a/include/linux/seccomp.h +++ b/include/linux/seccomp.h @@ -19,6 +19,11 @@ static inline void secure_computing(int this_syscall) extern long prctl_get_seccomp(void); extern long prctl_set_seccomp(unsigned long); +static inline int seccomp_mode(seccomp_t *s) +{ + return s->mode; +} + #else /* CONFIG_SECCOMP */ #include @@ -37,6 +42,11 @@ static inline long prctl_set_seccomp(unsigned long arg2) return -EINVAL; } +static inline int seccomp_mode(seccomp_t *s) +{ + return 0; +} + #endif /* CONFIG_SECCOMP */ #endif /* _LINUX_SECCOMP_H */ -- cgit v1.2.3 From 3d5fe5a65af9c0b609d6e26b8d63fe5923c4e512 Mon Sep 17 00:00:00 2001 From: Benjamin Herrenschmidt Date: Mon, 11 Apr 2011 11:19:09 +1000 Subject: x86/devicetree: Use generic PCI <-> OF matching Instead of walking the whole PCI tree to update the of_node's for PCI busses and devices after the fact, enable the new generic core code for doing so by providing the proper device nodes for the PCI host bridges Signed-off-by: Benjamin Herrenschmidt Acked-by: Grant Likely Tested-by: Sebastian Andrzej Siewior --- arch/x86/kernel/devicetree.c | 60 +++++++++++++------------------------------- 1 file changed, 18 insertions(+), 42 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c index 690bc8461835..d23f7af43679 100644 --- a/arch/x86/kernel/devicetree.c +++ b/arch/x86/kernel/devicetree.c @@ -123,6 +123,24 @@ static int __init add_bus_probe(void) module_init(add_bus_probe); #ifdef CONFIG_PCI +struct device_node *pcibios_get_phb_of_node(struct pci_bus *bus) +{ + struct device_node *np; + + for_each_node_by_type(np, "pci") { + const void *prop; + unsigned int bus_min; + + prop = of_get_property(np, "bus-range", NULL); + if (!prop) + continue; + bus_min = be32_to_cpup(prop); + if (bus->number == bus_min) + return np; + } + return NULL; +} + static int x86_of_pci_irq_enable(struct pci_dev *dev) { struct of_irq oirq; @@ -154,50 +172,8 @@ static void x86_of_pci_irq_disable(struct pci_dev *dev) void __cpuinit x86_of_pci_init(void) { - struct device_node *np; - pcibios_enable_irq = x86_of_pci_irq_enable; pcibios_disable_irq = x86_of_pci_irq_disable; - - for_each_node_by_type(np, "pci") { - const void *prop; - struct pci_bus *bus; - unsigned int bus_min; - struct device_node *child; - - prop = of_get_property(np, "bus-range", NULL); - if (!prop) - continue; - bus_min = be32_to_cpup(prop); - - bus = pci_find_bus(0, bus_min); - if (!bus) { - printk(KERN_ERR "Can't find a node for bus %s.\n", - np->full_name); - continue; - } - - if (bus->self) - bus->self->dev.of_node = np; - else - bus->dev.of_node = np; - - for_each_child_of_node(np, child) { - struct pci_dev *dev; - u32 devfn; - - prop = of_get_property(child, "reg", NULL); - if (!prop) - continue; - - devfn = (be32_to_cpup(prop) >> 8) & 0xff; - dev = pci_get_slot(bus, devfn); - if (!dev) - continue; - dev->dev.of_node = child; - pci_dev_put(dev); - } - } } #endif -- cgit v1.2.3 From 334955ef964bee9d3b1e20966847eee28cfd05f6 Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Wed, 1 Jun 2011 19:04:57 +0100 Subject: i8253: Create linux/i8253.h and use it in all 8253 related files Signed-off-by: Ralf Baechle Cc: linux-mips@linux-mips.org Link: http://lkml.kernel.org/r/20110601180610.054254048@duck.linux-mips.net Signed-off-by: Thomas Gleixner arch/arm/mach-footbridge/isa-timer.c | 2 +- arch/mips/cobalt/time.c | 2 +- arch/mips/jazz/irq.c | 2 +- arch/mips/kernel/i8253.c | 2 +- arch/mips/mti-malta/malta-time.c | 2 +- arch/mips/sgi-ip22/ip22-time.c | 2 +- arch/mips/sni/time.c | 2 +- arch/x86/kernel/apic/apic.c | 2 +- arch/x86/kernel/apm_32.c | 2 +- arch/x86/kernel/hpet.c | 2 +- arch/x86/kernel/i8253.c | 2 +- arch/x86/kernel/time.c | 2 +- drivers/block/hd.c | 2 +- drivers/clocksource/i8253.c | 2 +- drivers/input/gameport/gameport.c | 2 +- drivers/input/joystick/analog.c | 2 +- drivers/input/misc/pcspkr.c | 2 +- include/linux/i8253.h | 11 +++++++++++ sound/drivers/pcsp/pcsp.h | 2 +- 19 files changed, 29 insertions(+), 18 deletions(-) --- arch/arm/mach-footbridge/isa-timer.c | 2 +- arch/mips/cobalt/time.c | 2 +- arch/mips/jazz/irq.c | 2 +- arch/mips/kernel/i8253.c | 2 +- arch/mips/mti-malta/malta-time.c | 2 +- arch/mips/sgi-ip22/ip22-time.c | 2 +- arch/mips/sni/time.c | 2 +- arch/x86/kernel/apic/apic.c | 2 +- arch/x86/kernel/apm_32.c | 2 +- arch/x86/kernel/hpet.c | 2 +- arch/x86/kernel/i8253.c | 2 +- arch/x86/kernel/time.c | 2 +- drivers/block/hd.c | 2 +- drivers/clocksource/i8253.c | 2 +- drivers/input/gameport/gameport.c | 2 +- drivers/input/joystick/analog.c | 2 +- drivers/input/misc/pcspkr.c | 2 +- include/linux/i8253.h | 11 +++++++++++ sound/drivers/pcsp/pcsp.h | 2 +- 19 files changed, 29 insertions(+), 18 deletions(-) create mode 100644 include/linux/i8253.h (limited to 'arch/x86/kernel') diff --git a/arch/arm/mach-footbridge/isa-timer.c b/arch/arm/mach-footbridge/isa-timer.c index 7020f1a3feca..3c7367be5459 100644 --- a/arch/arm/mach-footbridge/isa-timer.c +++ b/arch/arm/mach-footbridge/isa-timer.c @@ -6,6 +6,7 @@ */ #include #include +#include #include #include #include @@ -14,7 +15,6 @@ #include #include -#include #include #include "common.h" diff --git a/arch/mips/cobalt/time.c b/arch/mips/cobalt/time.c index 0162f9edc693..3bff3b820baf 100644 --- a/arch/mips/cobalt/time.c +++ b/arch/mips/cobalt/time.c @@ -17,10 +17,10 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ +#include #include #include -#include #include #define GT641XX_BASE_CLOCK 50000000 /* 50MHz */ diff --git a/arch/mips/jazz/irq.c b/arch/mips/jazz/irq.c index 260df4750949..ca9bd2069142 100644 --- a/arch/mips/jazz/irq.c +++ b/arch/mips/jazz/irq.c @@ -7,6 +7,7 @@ * Copyright (C) 1994 - 2001, 2003, 07 Ralf Baechle */ #include +#include #include #include #include @@ -15,7 +16,6 @@ #include #include -#include #include #include #include diff --git a/arch/mips/kernel/i8253.c b/arch/mips/kernel/i8253.c index 391221b6a6aa..82b5a9f037e7 100644 --- a/arch/mips/kernel/i8253.c +++ b/arch/mips/kernel/i8253.c @@ -3,6 +3,7 @@ * */ #include +#include #include #include #include @@ -12,7 +13,6 @@ #include #include -#include #include #include diff --git a/arch/mips/mti-malta/malta-time.c b/arch/mips/mti-malta/malta-time.c index 1620b83cd13e..f8ee945ee411 100644 --- a/arch/mips/mti-malta/malta-time.c +++ b/arch/mips/mti-malta/malta-time.c @@ -19,6 +19,7 @@ */ #include +#include #include #include #include @@ -31,7 +32,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/mips/sgi-ip22/ip22-time.c b/arch/mips/sgi-ip22/ip22-time.c index 1a94c9894188..607192449335 100644 --- a/arch/mips/sgi-ip22/ip22-time.c +++ b/arch/mips/sgi-ip22/ip22-time.c @@ -10,6 +10,7 @@ * Copyright (C) 2003, 06 Ralf Baechle (ralf@linux-mips.org) */ #include +#include #include #include #include @@ -20,7 +21,6 @@ #include #include -#include #include #include #include diff --git a/arch/mips/sni/time.c b/arch/mips/sni/time.c index 0904d4d30cb3..ec0be14996a4 100644 --- a/arch/mips/sni/time.c +++ b/arch/mips/sni/time.c @@ -1,11 +1,11 @@ #include +#include #include #include #include #include #include -#include #include #include #include diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index b961af86bfea..f3c37841bb33 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index 965a7666c283..a30740e6890e 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c @@ -229,11 +229,11 @@ #include #include #include +#include #include #include #include -#include #include #include #include diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 6781765b3a0d..85b8a8a76c21 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -12,7 +13,6 @@ #include #include -#include #include #define HPET_MASK CLOCKSOURCE_MASK(32) diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c index fb66dc9e36cb..323555050f89 100644 --- a/arch/x86/kernel/i8253.c +++ b/arch/x86/kernel/i8253.c @@ -9,10 +9,10 @@ #include #include #include +#include #include #include -#include #include #include diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index 00cbb272627f..5a64d057be57 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -11,13 +11,13 @@ #include #include +#include #include #include #include #include #include -#include #include #include #include diff --git a/drivers/block/hd.c b/drivers/block/hd.c index 007c630904c1..b52c9ca146fc 100644 --- a/drivers/block/hd.c +++ b/drivers/block/hd.c @@ -155,7 +155,7 @@ else \ #if (HD_DELAY > 0) -#include +#include unsigned long last_req; diff --git a/drivers/clocksource/i8253.c b/drivers/clocksource/i8253.c index 225c1761b372..33735ebd2dcc 100644 --- a/drivers/clocksource/i8253.c +++ b/drivers/clocksource/i8253.c @@ -7,7 +7,7 @@ #include #include -#include +#include /* * Since the PIT overflows every tick, its not very useful diff --git a/drivers/input/gameport/gameport.c b/drivers/input/gameport/gameport.c index 5b8f59d6c3e8..c351aa421f8f 100644 --- a/drivers/input/gameport/gameport.c +++ b/drivers/input/gameport/gameport.c @@ -47,7 +47,7 @@ static void gameport_disconnect_port(struct gameport *gameport); #if defined(__i386__) -#include +#include #define DELTA(x,y) ((y)-(x)+((y)<(x)?1193182/HZ:0)) #define GET_TIME(x) do { x = get_time_pit(); } while (0) diff --git a/drivers/input/joystick/analog.c b/drivers/input/joystick/analog.c index 4afe0a3b4884..9882971827e6 100644 --- a/drivers/input/joystick/analog.c +++ b/drivers/input/joystick/analog.c @@ -136,7 +136,7 @@ struct analog_port { #ifdef __i386__ -#include +#include #define GET_TIME(x) do { if (cpu_has_tsc) rdtscl(x); else x = get_time_pit(); } while (0) #define DELTA(x,y) (cpu_has_tsc ? ((y) - (x)) : ((x) - (y) + ((x) < (y) ? CLOCK_TICK_RATE / HZ : 0))) diff --git a/drivers/input/misc/pcspkr.c b/drivers/input/misc/pcspkr.c index f080dd31499b..2aa9d9020179 100644 --- a/drivers/input/misc/pcspkr.c +++ b/drivers/input/misc/pcspkr.c @@ -27,7 +27,7 @@ MODULE_ALIAS("platform:pcspkr"); #if defined(CONFIG_MIPS) || defined(CONFIG_X86) /* Use the global PIT lock ! */ -#include +#include #else #include static DEFINE_RAW_SPINLOCK(i8253_lock); diff --git a/include/linux/i8253.h b/include/linux/i8253.h new file mode 100644 index 000000000000..d2cba88f75d6 --- /dev/null +++ b/include/linux/i8253.h @@ -0,0 +1,11 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#ifndef __LINUX_I8253_H +#define __LINUX_I8253_H + +#include + +#endif /* __LINUX_I8253_H */ diff --git a/sound/drivers/pcsp/pcsp.h b/sound/drivers/pcsp/pcsp.h index 4ff6c8cc5077..6757c304ee46 100644 --- a/sound/drivers/pcsp/pcsp.h +++ b/sound/drivers/pcsp/pcsp.h @@ -13,7 +13,7 @@ #include #if defined(CONFIG_MIPS) || defined(CONFIG_X86) /* Use the global PIT lock ! */ -#include +#include #else #include static DEFINE_RAW_SPINLOCK(i8253_lock); -- cgit v1.2.3 From 15f304b664c0d0a3e76ed3a9ce3615a86908babe Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Wed, 1 Jun 2011 19:04:59 +0100 Subject: i8253: Consolidate all kernel definitions of i8253_lock Move them to drivers/clocksource/i8253.c and remove the implementations in arch/ [ tglx: Avoid the extra file in lib - folded arch patches in. The export will become conditional in a later step ] Signed-off-by: Ralf Baechle Link: http://lkml.kernel.org/r/20110601180610.221426078@duck.linux-mips.net Cc: Russell King Signed-off-by: Thomas Gleixner --- arch/arm/mach-footbridge/isa-timer.c | 2 -- arch/mips/kernel/i8253.c | 3 --- arch/x86/Kconfig | 1 + arch/x86/kernel/i8253.c | 3 --- drivers/clocksource/Kconfig | 6 ++++++ drivers/clocksource/Makefile | 2 +- drivers/clocksource/i8253.c | 13 ++++++++++++- init/Kconfig | 1 + 8 files changed, 21 insertions(+), 10 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/arm/mach-footbridge/isa-timer.c b/arch/arm/mach-footbridge/isa-timer.c index 3c7367be5459..71fd96df7f73 100644 --- a/arch/arm/mach-footbridge/isa-timer.c +++ b/arch/arm/mach-footbridge/isa-timer.c @@ -19,8 +19,6 @@ #include "common.h" -DEFINE_RAW_SPINLOCK(i8253_lock); - static void pit_set_mode(enum clock_event_mode mode, struct clock_event_device *evt) { diff --git a/arch/mips/kernel/i8253.c b/arch/mips/kernel/i8253.c index 82b5a9f037e7..3d2ff57fa69a 100644 --- a/arch/mips/kernel/i8253.c +++ b/arch/mips/kernel/i8253.c @@ -16,9 +16,6 @@ #include #include -DEFINE_RAW_SPINLOCK(i8253_lock); -EXPORT_SYMBOL(i8253_lock); - /* * Initialize the PIT timer. * diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index da349723d411..e6060a1c1b58 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -70,6 +70,7 @@ config X86 select IRQ_FORCED_THREADING select USE_GENERIC_SMP_HELPERS if SMP select HAVE_BPF_JIT if (X86_64 && NET) + select I8253_LOCK config INSTRUCTION_DECODER def_bool (KPROBES || PERF_EVENTS) diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c index 323555050f89..9c92b6ff571f 100644 --- a/arch/x86/kernel/i8253.c +++ b/arch/x86/kernel/i8253.c @@ -16,9 +16,6 @@ #include #include -DEFINE_RAW_SPINLOCK(i8253_lock); -EXPORT_SYMBOL(i8253_lock); - /* * HPET replaces the PIT, when enabled. So we need to know, which of * the two timers is used diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 96c921910469..330343bcb67e 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -1,5 +1,11 @@ config CLKSRC_I8253 bool +config I8253_LOCK + bool + +config CLKBLD_I8253 + def_bool y if CLKSRC_I8253 || I8253_LOCK + config CLKSRC_MMIO bool diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index b995942a5060..7922a0cfc99f 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -6,5 +6,5 @@ obj-$(CONFIG_CS5535_CLOCK_EVENT_SRC) += cs5535-clockevt.o obj-$(CONFIG_SH_TIMER_CMT) += sh_cmt.o obj-$(CONFIG_SH_TIMER_MTU2) += sh_mtu2.o obj-$(CONFIG_SH_TIMER_TMU) += sh_tmu.o -obj-$(CONFIG_CLKSRC_I8253) += i8253.o +obj-$(CONFIG_CLKBLD_I8253) += i8253.o obj-$(CONFIG_CLKSRC_MMIO) += mmio.o diff --git a/drivers/clocksource/i8253.c b/drivers/clocksource/i8253.c index 33735ebd2dcc..e594f52eb88e 100644 --- a/drivers/clocksource/i8253.c +++ b/drivers/clocksource/i8253.c @@ -6,9 +6,19 @@ #include #include #include - +#include #include +/* + * Protects access to I/O ports + * + * 0040-0043 : timer0, i8253 / i8254 + * 0061-0061 : NMI Control Register which contains two speaker control bits. + */ +DEFINE_RAW_SPINLOCK(i8253_lock); +EXPORT_SYMBOL(i8253_lock); + +#ifdef CONFIG_CLKSRC_I8253 /* * Since the PIT overflows every tick, its not very useful * to just read by itself. So use jiffies to emulate a free @@ -86,3 +96,4 @@ int __init clocksource_i8253_init(void) { return clocksource_register_hz(&i8253_cs, PIT_TICK_RATE); } +#endif diff --git a/init/Kconfig b/init/Kconfig index ebafac4231ee..2d2ef447a9ab 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1002,6 +1002,7 @@ config ELF_CORE config PCSPKR_PLATFORM bool "Enable PC-Speaker support" if EXPERT depends on ALPHA || X86 || MIPS || PPC_PREP || PPC_CHRP || PPC_PSERIES + select I8253_LOCK default y help This option allows to disable the internal PC-Speaker -- cgit v1.2.3 From 16f871bc30f86560017b9d34520593a28e08f373 Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Wed, 1 Jun 2011 19:05:06 +0100 Subject: x86: i8253: Consolidate definitions of global_clock_event There are multiple declarations of global_clock_event in header files specific to particular clock event implementations. Consolidate them in and make sure all users include that header. Signed-off-by: Ralf Baechle Cc: Venkatesh Pallipadi (Venki) Link: http://lkml.kernel.org/r/20110601180610.762763451@duck.linux-mips.net Signed-off-by: Thomas Gleixner --- arch/x86/include/asm/apb_timer.h | 1 - arch/x86/include/asm/i8253.h | 2 -- arch/x86/include/asm/time.h | 6 ++++-- arch/x86/kernel/apb_timer.c | 1 + arch/x86/kernel/apic/apic.c | 1 + arch/x86/kernel/hpet.c | 1 + arch/x86/kernel/i8253.c | 1 + 7 files changed, 8 insertions(+), 5 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/apb_timer.h b/arch/x86/include/asm/apb_timer.h index 2fefa501d3ba..963f7d487781 100644 --- a/arch/x86/include/asm/apb_timer.h +++ b/arch/x86/include/asm/apb_timer.h @@ -50,7 +50,6 @@ #define APBT_DEV_USED 1 extern void apbt_time_init(void); -extern struct clock_event_device *global_clock_event; extern unsigned long apbt_quick_calibrate(void); extern int arch_setup_apbt_irqs(int irq, int trigger, int mask, int cpu); extern void apbt_setup_secondary_clock(void); diff --git a/arch/x86/include/asm/i8253.h b/arch/x86/include/asm/i8253.h index 20480cef7547..3d5f5eea5da4 100644 --- a/arch/x86/include/asm/i8253.h +++ b/arch/x86/include/asm/i8253.h @@ -3,8 +3,6 @@ #define PIT_LATCH LATCH -extern struct clock_event_device *global_clock_event; - extern void setup_pit_timer(void); #endif /* _ASM_X86_I8253_H */ diff --git a/arch/x86/include/asm/time.h b/arch/x86/include/asm/time.h index 7bdec4e9b739..92b8aec06970 100644 --- a/arch/x86/include/asm/time.h +++ b/arch/x86/include/asm/time.h @@ -1,10 +1,12 @@ #ifndef _ASM_X86_TIME_H #define _ASM_X86_TIME_H -extern void hpet_time_init(void); - +#include #include +extern void hpet_time_init(void); extern void time_init(void); +extern struct clock_event_device *global_clock_event; + #endif /* _ASM_X86_TIME_H */ diff --git a/arch/x86/kernel/apb_timer.c b/arch/x86/kernel/apb_timer.c index 289e92862fd9..2b6630d75e17 100644 --- a/arch/x86/kernel/apb_timer.c +++ b/arch/x86/kernel/apb_timer.c @@ -44,6 +44,7 @@ #include #include #include +#include #define APBT_MASK CLOCKSOURCE_MASK(32) #define APBT_SHIFT 22 diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index f3c37841bb33..9da0dc02bea4 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 85b8a8a76c21..0f4b0651cd3f 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -14,6 +14,7 @@ #include #include +#include #define HPET_MASK CLOCKSOURCE_MASK(32) diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c index 9c92b6ff571f..5783e6de2079 100644 --- a/arch/x86/kernel/i8253.c +++ b/arch/x86/kernel/i8253.c @@ -14,6 +14,7 @@ #include #include +#include #include /* -- cgit v1.2.3 From 28f65c11f2ffb3957259dece647a24f8ad2e241b Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Thu, 9 Jun 2011 09:13:32 -0700 Subject: treewide: Convert uses of struct resource to resource_size(ptr) Several fixes as well where the +1 was missing. Done via coccinelle scripts like: @@ struct resource *ptr; @@ - ptr->end - ptr->start + 1 + resource_size(ptr) and some grep and typing. Mostly uncompiled, no cross-compilers. Signed-off-by: Joe Perches Signed-off-by: Jiri Kosina --- arch/arm/common/scoop.c | 2 +- arch/arm/mach-at91/at91sam9261_devices.c | 2 +- arch/arm/mach-mv78xx0/pcie.c | 8 ++++---- arch/arm/mach-u300/core.c | 2 +- arch/arm/plat-mxc/pwm.c | 8 ++++---- arch/arm/plat-s5p/sysmmu.c | 6 +++--- arch/arm/plat-samsung/pm-check.c | 2 +- arch/avr32/kernel/setup.c | 10 +++++----- arch/avr32/mach-at32ap/extint.c | 2 +- arch/avr32/mach-at32ap/hsmc.c | 2 +- arch/avr32/mach-at32ap/intc.c | 2 +- arch/avr32/mach-at32ap/pio.c | 2 +- arch/microblaze/pci/pci-common.c | 2 +- arch/mips/pci/pci-rc32434.c | 2 +- arch/mips/pci/pci-vr41xx.c | 2 +- arch/mips/powertv/asic/asic_devices.c | 10 ++++------ arch/powerpc/include/asm/macio.h | 2 +- arch/powerpc/kernel/machine_kexec.c | 4 ++-- arch/powerpc/kernel/pci-common.c | 2 +- arch/powerpc/platforms/52xx/mpc52xx_pci.c | 8 ++++---- arch/powerpc/platforms/83xx/km83xx.c | 2 +- arch/powerpc/platforms/83xx/mpc832x_mds.c | 2 +- arch/powerpc/platforms/83xx/mpc834x_mds.c | 2 +- arch/powerpc/platforms/83xx/mpc836x_mds.c | 2 +- arch/powerpc/platforms/83xx/usb.c | 2 +- arch/powerpc/platforms/85xx/sbc8560.c | 2 +- arch/powerpc/platforms/85xx/xes_mpc85xx.c | 2 +- arch/powerpc/platforms/cell/celleb_scc_epci.c | 8 ++++---- arch/powerpc/platforms/cell/celleb_scc_pciex.c | 2 +- arch/powerpc/platforms/cell/spu_manage.c | 2 +- arch/powerpc/platforms/chrp/pci.c | 2 +- arch/powerpc/platforms/pasemi/dma_lib.c | 2 +- arch/powerpc/platforms/powermac/nvram.c | 4 ++-- arch/powerpc/platforms/powermac/pci.c | 6 ++---- arch/powerpc/platforms/powermac/time.c | 2 +- arch/powerpc/sysdev/axonram.c | 2 +- arch/powerpc/sysdev/cpm1.c | 2 +- arch/powerpc/sysdev/cpm_common.c | 2 +- arch/powerpc/sysdev/dart_iommu.c | 2 +- arch/powerpc/sysdev/fsl_msi.c | 2 +- arch/powerpc/sysdev/fsl_pci.c | 12 ++++++------ arch/powerpc/sysdev/fsl_rio.c | 2 +- arch/powerpc/sysdev/ipic.c | 2 +- arch/powerpc/sysdev/mmio_nvram.c | 2 +- arch/powerpc/sysdev/mpc8xx_pic.c | 2 +- arch/powerpc/sysdev/mv64x60_udbg.c | 4 ++-- arch/powerpc/sysdev/ppc4xx_pci.c | 16 ++++++++-------- arch/powerpc/sysdev/qe_lib/qe_ic.c | 2 +- arch/powerpc/sysdev/qe_lib/qe_io.c | 2 +- arch/powerpc/sysdev/xics/icp-native.c | 2 +- arch/sh/kernel/io_trapped.c | 8 ++++---- arch/sh/kernel/machine_kexec.c | 2 +- arch/sparc/kernel/ioport.c | 12 ++++++------ arch/sparc/kernel/pci.c | 6 ++---- arch/tile/kernel/setup.c | 3 +-- arch/x86/kernel/pci-calgary_64.c | 2 +- arch/x86/kernel/probe_roms.c | 2 +- drivers/char/bsr.c | 2 +- drivers/char/xilinx_hwicap/xilinx_hwicap.c | 2 +- drivers/dma/mv_xor.c | 5 ++--- drivers/edac/cell_edac.c | 2 +- drivers/edac/mpc85xx_edac.c | 12 ++++++------ drivers/gpio/gpio-u300.c | 8 ++++---- drivers/ide/palm_bk3710.c | 2 +- drivers/ide/tx4939ide.c | 4 ++-- drivers/input/serio/sa1111ps2.c | 6 ++---- drivers/media/video/davinci/vpif.c | 2 +- drivers/media/video/omap24xxcam.c | 5 ++--- drivers/message/i2o/iop.c | 8 ++++---- drivers/mfd/tc6387xb.c | 2 +- drivers/misc/atmel-ssc.c | 2 +- drivers/misc/atmel_pwm.c | 2 +- drivers/mmc/host/dw_mmc.c | 2 +- drivers/mtd/maps/bfin-async-flash.c | 2 +- drivers/mtd/maps/ixp2000.c | 11 ++++++----- drivers/mtd/maps/pxa2xx-flash.c | 2 +- drivers/mtd/nand/atmel_nand.c | 4 ++-- drivers/mtd/nand/bcm_umi_nand.c | 2 +- drivers/mtd/nand/mpc5121_nfc.c | 2 +- drivers/net/bcm63xx_enet.c | 8 ++++---- drivers/net/can/softing/softing_main.c | 2 +- drivers/net/davinci_emac.c | 6 +++--- drivers/net/ethoc.c | 2 +- drivers/net/fec_mpc52xx.c | 7 ++++--- drivers/net/fs_enet/mii-bitbang.c | 4 ++-- drivers/net/fs_enet/mii-fec.c | 2 +- drivers/net/gianfar_ptp.c | 2 +- drivers/net/ibm_newemac/core.c | 2 +- drivers/net/macb.c | 2 +- drivers/net/mv643xx_eth.c | 2 +- drivers/net/pxa168_eth.c | 2 +- drivers/net/sb1250-mac.c | 2 +- drivers/parport/parport_ax88796.c | 2 +- drivers/pci/hotplug/shpchp_sysfs.c | 21 +++++++++------------ drivers/pcmcia/at91_cf.c | 7 +++---- drivers/pcmcia/electra_cf.c | 4 ++-- drivers/pcmcia/rsrc_iodyn.c | 6 +++--- drivers/pcmcia/rsrc_nonstatic.c | 6 +++--- drivers/pnp/pnpacpi/rsparser.c | 10 +++++----- drivers/pnp/pnpbios/rsparser.c | 12 ++++++------ drivers/rtc/rtc-at32ap700x.c | 2 +- drivers/rtc/rtc-cmos.c | 6 +++--- drivers/rtc/rtc-ds1286.c | 2 +- drivers/rtc/rtc-ds1511.c | 2 +- drivers/rtc/rtc-ds1742.c | 2 +- drivers/rtc/rtc-m48t35.c | 2 +- drivers/rtc/rtc-m48t59.c | 2 +- drivers/rtc/rtc-mrst.c | 5 ++--- drivers/rtc/rtc-puv3.c | 5 ++--- drivers/rtc/rtc-s3c.c | 5 ++--- drivers/staging/generic_serial/ser_a2232.c | 3 ++- drivers/staging/gma500/psb_gtt.c | 8 ++++---- drivers/tty/serial/bfin_5xx.c | 5 ++--- drivers/tty/serial/imx.c | 5 ++--- drivers/tty/serial/m32r_sio.c | 2 +- drivers/tty/serial/omap-serial.c | 6 +++--- drivers/tty/serial/pxa.c | 2 +- drivers/tty/serial/sunsu.c | 2 +- drivers/tty/serial/vt8500_serial.c | 3 +-- drivers/uio/uio_pdrv.c | 2 +- drivers/uio/uio_pdrv_genirq.c | 2 +- drivers/usb/gadget/atmel_usba_udc.c | 2 +- drivers/usb/gadget/fsl_udc_core.c | 6 +++--- drivers/usb/host/ehci-ath79.c | 2 +- drivers/usb/host/ehci-cns3xxx.c | 2 +- drivers/usb/host/ehci-fsl.c | 2 +- drivers/usb/host/ehci-grlib.c | 2 +- drivers/usb/host/ehci-ixp4xx.c | 2 +- drivers/usb/host/ehci-octeon.c | 2 +- drivers/usb/host/ehci-pmcmsp.c | 10 +++++----- drivers/usb/host/ehci-ppc-of.c | 2 +- drivers/usb/host/ehci-w90x900.c | 2 +- drivers/usb/host/ehci-xilinx-of.c | 2 +- drivers/usb/host/fhci-hcd.c | 2 +- drivers/usb/host/ohci-ath79.c | 4 ++-- drivers/usb/host/ohci-cns3xxx.c | 2 +- drivers/usb/host/ohci-da8xx.c | 2 +- drivers/usb/host/ohci-octeon.c | 2 +- drivers/usb/host/ohci-ppc-of.c | 2 +- drivers/usb/host/ohci-ppc-soc.c | 2 +- drivers/usb/host/ohci-sa1111.c | 2 +- drivers/usb/host/ohci-sm501.c | 11 +++++------ drivers/usb/host/ohci-tmio.c | 6 +++--- drivers/usb/host/oxu210hp-hcd.c | 2 +- drivers/usb/host/uhci-grlib.c | 2 +- drivers/usb/host/whci/init.c | 2 +- drivers/uwb/whc-rc.c | 2 +- drivers/video/atmel_lcdfb.c | 4 ++-- drivers/video/aty/atyfb_base.c | 7 ++++--- drivers/video/au1100fb.c | 2 +- drivers/video/cobalt_lcdfb.c | 2 +- drivers/video/controlfb.c | 4 ++-- drivers/video/mb862xx/mb862xxfbdrv.c | 4 ++-- drivers/video/msm/mdp.c | 3 +-- drivers/video/msm/msm_fb.c | 7 +++---- drivers/video/nuc900fb.c | 2 +- drivers/video/platinumfb.c | 5 ++--- drivers/video/pxa168fb.c | 2 +- include/linux/dio.h | 2 +- include/linux/pnp.h | 2 +- include/linux/zorro.h | 2 +- kernel/kexec.c | 2 +- sound/aoa/soundbus/i2sbus/core.c | 9 ++++----- sound/atmel/abdac.c | 2 +- sound/atmel/ac97c.c | 2 +- sound/ppc/pmac.c | 9 +++------ sound/soc/fsl/fsl_ssi.c | 2 +- sound/soc/fsl/mpc5200_dma.c | 2 +- 168 files changed, 308 insertions(+), 333 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/arm/common/scoop.c b/arch/arm/common/scoop.c index c11af1e4bad3..a07b0e763a80 100644 --- a/arch/arm/common/scoop.c +++ b/arch/arm/common/scoop.c @@ -193,7 +193,7 @@ static int __devinit scoop_probe(struct platform_device *pdev) spin_lock_init(&devptr->scoop_lock); inf = pdev->dev.platform_data; - devptr->base = ioremap(mem->start, mem->end - mem->start + 1); + devptr->base = ioremap(mem->start, resource_size(mem)); if (!devptr->base) { ret = -ENOMEM; diff --git a/arch/arm/mach-at91/at91sam9261_devices.c b/arch/arm/mach-at91/at91sam9261_devices.c index 3eb4538fceeb..14dd666850b0 100644 --- a/arch/arm/mach-at91/at91sam9261_devices.c +++ b/arch/arm/mach-at91/at91sam9261_devices.c @@ -525,7 +525,7 @@ void __init at91_add_device_lcdc(struct atmel_lcdfb_info *data) if (ARRAY_SIZE(lcdc_resources) > 2) { void __iomem *fb; struct resource *fb_res = &lcdc_resources[2]; - size_t fb_len = fb_res->end - fb_res->start + 1; + size_t fb_len = resource_size(fb_res); fb = ioremap(fb_res->start, fb_len); if (fb) { diff --git a/arch/arm/mach-mv78xx0/pcie.c b/arch/arm/mach-mv78xx0/pcie.c index a560439dcc3c..f27c7d2fa9f7 100644 --- a/arch/arm/mach-mv78xx0/pcie.c +++ b/arch/arm/mach-mv78xx0/pcie.c @@ -129,12 +129,12 @@ static void __init mv78xx0_pcie_preinit(void) struct pcie_port *pp = pcie_port + i; mv78xx0_setup_pcie_io_win(win++, pp->res[0].start, - pp->res[0].end - pp->res[0].start + 1, - pp->maj, pp->min); + resource_size(&pp->res[0]), + pp->maj, pp->min); mv78xx0_setup_pcie_mem_win(win++, pp->res[1].start, - pp->res[1].end - pp->res[1].start + 1, - pp->maj, pp->min); + resource_size(&pp->res[1]), + pp->maj, pp->min); } } diff --git a/arch/arm/mach-u300/core.c b/arch/arm/mach-u300/core.c index 513d6abec1f5..399c89f14dfb 100644 --- a/arch/arm/mach-u300/core.c +++ b/arch/arm/mach-u300/core.c @@ -1791,7 +1791,7 @@ static void __init u300_assign_physmem(void) 0 == res->start) { res->start = curr_start; res->end += curr_start; - curr_start += (res->end - res->start + 1); + curr_start += resource_size(res); printk(KERN_INFO "core.c: Mapping RAM " \ "%#x-%#x to device %s:%s\n", diff --git a/arch/arm/plat-mxc/pwm.c b/arch/arm/plat-mxc/pwm.c index 7a61ef8f471a..761c3c940a68 100644 --- a/arch/arm/plat-mxc/pwm.c +++ b/arch/arm/plat-mxc/pwm.c @@ -214,14 +214,14 @@ static int __devinit mxc_pwm_probe(struct platform_device *pdev) goto err_free_clk; } - r = request_mem_region(r->start, r->end - r->start + 1, pdev->name); + r = request_mem_region(r->start, resource_size(r), pdev->name); if (r == NULL) { dev_err(&pdev->dev, "failed to request memory resource\n"); ret = -EBUSY; goto err_free_clk; } - pwm->mmio_base = ioremap(r->start, r->end - r->start + 1); + pwm->mmio_base = ioremap(r->start, resource_size(r)); if (pwm->mmio_base == NULL) { dev_err(&pdev->dev, "failed to ioremap() registers\n"); ret = -ENODEV; @@ -236,7 +236,7 @@ static int __devinit mxc_pwm_probe(struct platform_device *pdev) return 0; err_free_mem: - release_mem_region(r->start, r->end - r->start + 1); + release_mem_region(r->start, resource_size(r)); err_free_clk: clk_put(pwm->clk); err_free: @@ -260,7 +260,7 @@ static int __devexit mxc_pwm_remove(struct platform_device *pdev) iounmap(pwm->mmio_base); r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(r->start, r->end - r->start + 1); + release_mem_region(r->start, resource_size(r)); clk_put(pwm->clk); diff --git a/arch/arm/plat-s5p/sysmmu.c b/arch/arm/plat-s5p/sysmmu.c index 54f5eddc921d..e1cbc728c775 100644 --- a/arch/arm/plat-s5p/sysmmu.c +++ b/arch/arm/plat-s5p/sysmmu.c @@ -232,8 +232,8 @@ static int s5p_sysmmu_probe(struct platform_device *pdev) goto err_res; } - mem = request_mem_region(res->start, - ((res->end) - (res->start)) + 1, pdev->name); + mem = request_mem_region(res->start, resource_size(res), + pdev->name); if (!mem) { dev_err(dev, "Failed to request the memory region of %s.\n", sysmmu_ips_name[i]); @@ -241,7 +241,7 @@ static int s5p_sysmmu_probe(struct platform_device *pdev) goto err_res; } - sysmmusfrs[i] = ioremap(res->start, res->end - res->start + 1); + sysmmusfrs[i] = ioremap(res->start, resource_size(res)); if (!sysmmusfrs[i]) { dev_err(dev, "Failed to ioremap() for %s.\n", sysmmu_ips_name[i]); diff --git a/arch/arm/plat-samsung/pm-check.c b/arch/arm/plat-samsung/pm-check.c index 6b733fafe7cd..3cbd62666b1e 100644 --- a/arch/arm/plat-samsung/pm-check.c +++ b/arch/arm/plat-samsung/pm-check.c @@ -72,7 +72,7 @@ static void s3c_pm_run_sysram(run_fn_t fn, u32 *arg) static u32 *s3c_pm_countram(struct resource *res, u32 *val) { - u32 size = (u32)(res->end - res->start)+1; + u32 size = (u32)resource_size(res); size += CHECK_CHUNKSIZE-1; size /= CHECK_CHUNKSIZE; diff --git a/arch/avr32/kernel/setup.c b/arch/avr32/kernel/setup.c index bb0974cce4ac..b4247f478065 100644 --- a/arch/avr32/kernel/setup.c +++ b/arch/avr32/kernel/setup.c @@ -444,7 +444,7 @@ static unsigned long __init find_bootmap_pfn(const struct resource *mem) { unsigned long bootmap_pages, bootmap_len; - unsigned long node_pages = PFN_UP(mem->end - mem->start + 1); + unsigned long node_pages = PFN_UP(resource_size(mem)); unsigned long bootmap_start; bootmap_pages = bootmem_bootmap_pages(node_pages); @@ -541,10 +541,10 @@ static void __init setup_bootmem(void) */ if (res->start >= PFN_PHYS(first_pfn) && res->end < PFN_PHYS(max_pfn)) - reserve_bootmem_node( - NODE_DATA(node), res->start, - res->end - res->start + 1, - BOOTMEM_DEFAULT); + reserve_bootmem_node(NODE_DATA(node), + res->start, + resource_size(res), + BOOTMEM_DEFAULT); } node_set_online(node); diff --git a/arch/avr32/mach-at32ap/extint.c b/arch/avr32/mach-at32ap/extint.c index fbc2aeaebddb..cfb298d66305 100644 --- a/arch/avr32/mach-at32ap/extint.c +++ b/arch/avr32/mach-at32ap/extint.c @@ -204,7 +204,7 @@ static int __init eic_probe(struct platform_device *pdev) } eic->first_irq = EIM_IRQ_BASE + 32 * pdev->id; - eic->regs = ioremap(regs->start, regs->end - regs->start + 1); + eic->regs = ioremap(regs->start, resource_size(regs)); if (!eic->regs) { dev_dbg(&pdev->dev, "failed to map regs\n"); goto err_ioremap; diff --git a/arch/avr32/mach-at32ap/hsmc.c b/arch/avr32/mach-at32ap/hsmc.c index f7672d3e86b8..f66245e6e63e 100644 --- a/arch/avr32/mach-at32ap/hsmc.c +++ b/arch/avr32/mach-at32ap/hsmc.c @@ -245,7 +245,7 @@ static int hsmc_probe(struct platform_device *pdev) hsmc->pclk = pclk; hsmc->mck = mck; - hsmc->regs = ioremap(regs->start, regs->end - regs->start + 1); + hsmc->regs = ioremap(regs->start, resource_size(regs)); if (!hsmc->regs) goto out_disable_clocks; diff --git a/arch/avr32/mach-at32ap/intc.c b/arch/avr32/mach-at32ap/intc.c index 3e3646186c9f..6c700431e34a 100644 --- a/arch/avr32/mach-at32ap/intc.c +++ b/arch/avr32/mach-at32ap/intc.c @@ -107,7 +107,7 @@ void __init init_IRQ(void) clk_enable(pclk); - intc0.regs = ioremap(regs->start, regs->end - regs->start + 1); + intc0.regs = ioremap(regs->start, resource_size(regs)); if (!intc0.regs) { printk(KERN_EMERG "intc: failed to map registers (0x%08lx)\n", (unsigned long)regs->start); diff --git a/arch/avr32/mach-at32ap/pio.c b/arch/avr32/mach-at32ap/pio.c index 2e0aa853a4bc..9b39dea6682f 100644 --- a/arch/avr32/mach-at32ap/pio.c +++ b/arch/avr32/mach-at32ap/pio.c @@ -461,7 +461,7 @@ void __init at32_init_pio(struct platform_device *pdev) clk_enable(pio->clk); pio->pdev = pdev; - pio->regs = ioremap(regs->start, regs->end - regs->start + 1); + pio->regs = ioremap(regs->start, resource_size(regs)); /* start with irqs disabled and acked */ pio_writel(pio, IDR, ~0UL); diff --git a/arch/microblaze/pci/pci-common.c b/arch/microblaze/pci/pci-common.c index 53599067d2f9..dd3fae06ece4 100644 --- a/arch/microblaze/pci/pci-common.c +++ b/arch/microblaze/pci/pci-common.c @@ -89,7 +89,7 @@ void pcibios_free_controller(struct pci_controller *phb) static resource_size_t pcibios_io_size(const struct pci_controller *hose) { - return hose->io_resource.end - hose->io_resource.start + 1; + return resource_size(&hose->io_resource); } int pcibios_vaddr_is_ioport(void __iomem *address) diff --git a/arch/mips/pci/pci-rc32434.c b/arch/mips/pci/pci-rc32434.c index f31218e17d3c..764362ce5e40 100644 --- a/arch/mips/pci/pci-rc32434.c +++ b/arch/mips/pci/pci-rc32434.c @@ -215,7 +215,7 @@ static int __init rc32434_pci_init(void) rc32434_pcibridge_init(); io_map_base = ioremap(rc32434_res_pci_io1.start, - rc32434_res_pci_io1.end - rc32434_res_pci_io1.start + 1); + resource_size(&rcrc32434_res_pci_io1)); if (!io_map_base) return -ENOMEM; diff --git a/arch/mips/pci/pci-vr41xx.c b/arch/mips/pci/pci-vr41xx.c index 56525711f8b7..444b8d8004ad 100644 --- a/arch/mips/pci/pci-vr41xx.c +++ b/arch/mips/pci/pci-vr41xx.c @@ -305,7 +305,7 @@ static int __init vr41xx_pciu_init(void) struct resource *res = vr41xx_pci_controller.io_resource; master = setup->master_io; io_map_base = ioremap(master->bus_base_address, - res->end - res->start + 1); + resource_size(res)); if (!io_map_base) return -EBUSY; diff --git a/arch/mips/powertv/asic/asic_devices.c b/arch/mips/powertv/asic/asic_devices.c index e56fa61b3991..bce1872249ba 100644 --- a/arch/mips/powertv/asic/asic_devices.c +++ b/arch/mips/powertv/asic/asic_devices.c @@ -394,23 +394,21 @@ void __init platform_alloc_bootmem(void) /* Loop through looking for resources that want a particular address */ for (i = 0; gp_resources[i].flags != 0; i++) { - int size = gp_resources[i].end - gp_resources[i].start + 1; + int size = resource_size(&gp_resources[i]); if ((gp_resources[i].start != 0) && ((gp_resources[i].flags & IORESOURCE_MEM) != 0)) { reserve_bootmem(dma_to_phys(gp_resources[i].start), size, 0); - total += gp_resources[i].end - - gp_resources[i].start + 1; + total += resource_size(&gp_resources[i]); pr_info("reserve resource %s at %08x (%u bytes)\n", gp_resources[i].name, gp_resources[i].start, - gp_resources[i].end - - gp_resources[i].start + 1); + resource_size(&gp_resources[i])); } } /* Loop through assigning addresses for those that are left */ for (i = 0; gp_resources[i].flags != 0; i++) { - int size = gp_resources[i].end - gp_resources[i].start + 1; + int size = resource_size(&gp_resources[i]); if ((gp_resources[i].start == 0) && ((gp_resources[i].flags & IORESOURCE_MEM) != 0)) { void *mem = alloc_bootmem_pages(size); diff --git a/arch/powerpc/include/asm/macio.h b/arch/powerpc/include/asm/macio.h index 7ab82c825a03..27af7f8bbb8d 100644 --- a/arch/powerpc/include/asm/macio.h +++ b/arch/powerpc/include/asm/macio.h @@ -76,7 +76,7 @@ static inline unsigned long macio_resource_len(struct macio_dev *dev, int resour struct resource *res = &dev->resource[resource_no]; if (res->start == 0 || res->end == 0 || res->end < res->start) return 0; - return res->end - res->start + 1; + return resource_size(res); } extern int macio_enable_devres(struct macio_dev *dev); diff --git a/arch/powerpc/kernel/machine_kexec.c b/arch/powerpc/kernel/machine_kexec.c index 7ee50f0547cb..6658a1589955 100644 --- a/arch/powerpc/kernel/machine_kexec.c +++ b/arch/powerpc/kernel/machine_kexec.c @@ -126,7 +126,7 @@ void __init reserve_crashkernel(void) /* We might have got these values via the command line or the * device tree, either way sanitise them now. */ - crash_size = crashk_res.end - crashk_res.start + 1; + crash_size = resource_size(&crashk_res); #ifndef CONFIG_RELOCATABLE if (crashk_res.start != KDUMP_KERNELBASE) @@ -222,7 +222,7 @@ static void __init export_crashk_values(struct device_node *node) if (crashk_res.start != 0) { prom_add_property(node, &crashk_base_prop); - crashk_size = crashk_res.end - crashk_res.start + 1; + crashk_size = resource_size(&crashk_res); prom_add_property(node, &crashk_size_prop); } } diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 893af2a9cd03..3764e37205fb 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -107,7 +107,7 @@ static resource_size_t pcibios_io_size(const struct pci_controller *hose) #ifdef CONFIG_PPC64 return hose->pci_io_size; #else - return hose->io_resource.end - hose->io_resource.start + 1; + return resource_size(&hose->io_resource); #endif } diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pci.c b/arch/powerpc/platforms/52xx/mpc52xx_pci.c index da110bd88346..5f5e69309080 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_pci.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_pci.c @@ -264,7 +264,7 @@ mpc52xx_pci_setup(struct pci_controller *hose, (unsigned long long)res->flags); out_be32(&pci_regs->iw0btar, MPC52xx_PCI_IWBTAR_TRANSLATION(res->start, res->start, - res->end - res->start + 1)); + resource_size(res))); iwcr0 = MPC52xx_PCI_IWCR_ENABLE | MPC52xx_PCI_IWCR_MEM; if (res->flags & IORESOURCE_PREFETCH) iwcr0 |= MPC52xx_PCI_IWCR_READ_MULTI; @@ -278,7 +278,7 @@ mpc52xx_pci_setup(struct pci_controller *hose, res->start, res->end, res->flags); out_be32(&pci_regs->iw1btar, MPC52xx_PCI_IWBTAR_TRANSLATION(res->start, res->start, - res->end - res->start + 1)); + resource_size(res))); iwcr1 = MPC52xx_PCI_IWCR_ENABLE | MPC52xx_PCI_IWCR_MEM; if (res->flags & IORESOURCE_PREFETCH) iwcr1 |= MPC52xx_PCI_IWCR_READ_MULTI; @@ -300,7 +300,7 @@ mpc52xx_pci_setup(struct pci_controller *hose, out_be32(&pci_regs->iw2btar, MPC52xx_PCI_IWBTAR_TRANSLATION(hose->io_base_phys, res->start, - res->end - res->start + 1)); + resource_size(res))); iwcr2 = MPC52xx_PCI_IWCR_ENABLE | MPC52xx_PCI_IWCR_IO; /* Set all the IWCR fields at once; they're in the same reg */ @@ -402,7 +402,7 @@ mpc52xx_add_bridge(struct device_node *node) hose->ops = &mpc52xx_pci_ops; - pci_regs = ioremap(rsrc.start, rsrc.end - rsrc.start + 1); + pci_regs = ioremap(rsrc.start, resource_size(&rsrc)); if (!pci_regs) return -ENOMEM; diff --git a/arch/powerpc/platforms/83xx/km83xx.c b/arch/powerpc/platforms/83xx/km83xx.c index a2b9b9ef1240..f8fa2fc3129f 100644 --- a/arch/powerpc/platforms/83xx/km83xx.c +++ b/arch/powerpc/platforms/83xx/km83xx.c @@ -101,7 +101,7 @@ static void __init mpc83xx_km_setup_arch(void) __func__); return; } - base = ioremap(res.start, res.end - res.start + 1); + base = ioremap(res.start, resource_size(&res)); /* * IMMR + 0x14A8[4:5] = 11 (clk delay for UCC 2) diff --git a/arch/powerpc/platforms/83xx/mpc832x_mds.c b/arch/powerpc/platforms/83xx/mpc832x_mds.c index ec0b401bc9cf..93e60f1f21a9 100644 --- a/arch/powerpc/platforms/83xx/mpc832x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc832x_mds.c @@ -68,7 +68,7 @@ static void __init mpc832x_sys_setup_arch(void) struct resource res; of_address_to_resource(np, 0, &res); - bcsr_regs = ioremap(res.start, res.end - res.start +1); + bcsr_regs = ioremap(res.start, resource_size(&res)); of_node_put(np); } diff --git a/arch/powerpc/platforms/83xx/mpc834x_mds.c b/arch/powerpc/platforms/83xx/mpc834x_mds.c index d0a634b056ca..c1b1dc50b32a 100644 --- a/arch/powerpc/platforms/83xx/mpc834x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc834x_mds.c @@ -53,7 +53,7 @@ static int mpc834xemds_usb_cfg(void) struct resource res; of_address_to_resource(np, 0, &res); - bcsr_regs = ioremap(res.start, res.end - res.start + 1); + bcsr_regs = ioremap(res.start, resource_size(&res)); of_node_put(np); } if (!bcsr_regs) diff --git a/arch/powerpc/platforms/83xx/mpc836x_mds.c b/arch/powerpc/platforms/83xx/mpc836x_mds.c index 09e9d6fb7411..81c052b1353e 100644 --- a/arch/powerpc/platforms/83xx/mpc836x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc836x_mds.c @@ -76,7 +76,7 @@ static void __init mpc836x_mds_setup_arch(void) struct resource res; of_address_to_resource(np, 0, &res); - bcsr_regs = ioremap(res.start, res.end - res.start +1); + bcsr_regs = ioremap(res.start, resource_size(&res)); of_node_put(np); } diff --git a/arch/powerpc/platforms/83xx/usb.c b/arch/powerpc/platforms/83xx/usb.c index 2c64164722d0..1ad748bb39b4 100644 --- a/arch/powerpc/platforms/83xx/usb.c +++ b/arch/powerpc/platforms/83xx/usb.c @@ -171,7 +171,7 @@ int mpc831x_usb_cfg(void) of_node_put(np); return ret; } - usb_regs = ioremap(res.start, res.end - res.start + 1); + usb_regs = ioremap(res.start, resource_size(&res)); /* Using on-chip PHY */ if (prop && (!strcmp(prop, "utmi_wide") || diff --git a/arch/powerpc/platforms/85xx/sbc8560.c b/arch/powerpc/platforms/85xx/sbc8560.c index d2dfd465fbf6..09ced7221750 100644 --- a/arch/powerpc/platforms/85xx/sbc8560.c +++ b/arch/powerpc/platforms/85xx/sbc8560.c @@ -285,7 +285,7 @@ static int __init sbc8560_bdrstcr_init(void) printk(KERN_INFO "sbc8560: Found BRSTCR at i/o 0x%x\n", res.start); - brstcr = ioremap(res.start, res.end - res.start); + brstcr = ioremap(res.start, resource_size(&res)); if(!brstcr) printk(KERN_WARNING "sbc8560: ioremap of brstcr failed.\n"); diff --git a/arch/powerpc/platforms/85xx/xes_mpc85xx.c b/arch/powerpc/platforms/85xx/xes_mpc85xx.c index 0125604d096e..a9dc5e795123 100644 --- a/arch/powerpc/platforms/85xx/xes_mpc85xx.c +++ b/arch/powerpc/platforms/85xx/xes_mpc85xx.c @@ -123,7 +123,7 @@ static void xes_mpc85xx_fixups(void) continue; } - l2_base = ioremap(r[0].start, r[0].end - r[0].start + 1); + l2_base = ioremap(r[0].start, resource_size(&r[0])); xes_mpc85xx_configure_l2(l2_base); } diff --git a/arch/powerpc/platforms/cell/celleb_scc_epci.c b/arch/powerpc/platforms/cell/celleb_scc_epci.c index 05b0db3ef638..844c0facb4f7 100644 --- a/arch/powerpc/platforms/cell/celleb_scc_epci.c +++ b/arch/powerpc/platforms/cell/celleb_scc_epci.c @@ -393,19 +393,19 @@ static int __init celleb_setup_epci(struct device_node *node, if (of_address_to_resource(node, 0, &r)) goto error; - hose->cfg_addr = ioremap(r.start, (r.end - r.start + 1)); + hose->cfg_addr = ioremap(r.start, resource_size(&r)); if (!hose->cfg_addr) goto error; pr_debug("EPCI: cfg_addr map 0x%016llx->0x%016lx + 0x%016llx\n", - r.start, (unsigned long)hose->cfg_addr, (r.end - r.start + 1)); + r.start, (unsigned long)hose->cfg_addr, resource_size(&r)); if (of_address_to_resource(node, 2, &r)) goto error; - hose->cfg_data = ioremap(r.start, (r.end - r.start + 1)); + hose->cfg_data = ioremap(r.start, resource_size(&r)); if (!hose->cfg_data) goto error; pr_debug("EPCI: cfg_data map 0x%016llx->0x%016lx + 0x%016llx\n", - r.start, (unsigned long)hose->cfg_data, (r.end - r.start + 1)); + r.start, (unsigned long)hose->cfg_data, resource_size(&r)); hose->ops = &celleb_epci_ops; celleb_epci_init(hose); diff --git a/arch/powerpc/platforms/cell/celleb_scc_pciex.c b/arch/powerpc/platforms/cell/celleb_scc_pciex.c index a881bbee8de0..ae790ac4a589 100644 --- a/arch/powerpc/platforms/cell/celleb_scc_pciex.c +++ b/arch/powerpc/platforms/cell/celleb_scc_pciex.c @@ -494,7 +494,7 @@ static __init int celleb_setup_pciex(struct device_node *node, pr_err("PCIEXC:Failed to get config resource.\n"); return 1; } - phb->cfg_addr = ioremap(r.start, r.end - r.start + 1); + phb->cfg_addr = ioremap(r.start, resource_size(&r)); if (!phb->cfg_addr) { pr_err("PCIEXC:Failed to remap SMMIO region.\n"); return 1; diff --git a/arch/powerpc/platforms/cell/spu_manage.c b/arch/powerpc/platforms/cell/spu_manage.c index f465d474ad9b..4e5c91489c02 100644 --- a/arch/powerpc/platforms/cell/spu_manage.c +++ b/arch/powerpc/platforms/cell/spu_manage.c @@ -222,7 +222,7 @@ static int spu_map_resource(struct spu *spu, int nr, return ret; if (phys) *phys = resource.start; - len = resource.end - resource.start + 1; + len = resource_size(&resource); *virt = ioremap(resource.start, len); if (!*virt) return -EINVAL; diff --git a/arch/powerpc/platforms/chrp/pci.c b/arch/powerpc/platforms/chrp/pci.c index 8f67a394b2d0..3f65443f1714 100644 --- a/arch/powerpc/platforms/chrp/pci.c +++ b/arch/powerpc/platforms/chrp/pci.c @@ -142,7 +142,7 @@ hydra_init(void) return 0; } of_node_put(np); - Hydra = ioremap(r.start, r.end-r.start); + Hydra = ioremap(r.start, resource_size(&r)); printk("Hydra Mac I/O at %llx\n", (unsigned long long)r.start); printk("Hydra Feature_Control was %x", in_le32(&Hydra->Feature_Control)); diff --git a/arch/powerpc/platforms/pasemi/dma_lib.c b/arch/powerpc/platforms/pasemi/dma_lib.c index 321a9b3a2d00..756123bf06ac 100644 --- a/arch/powerpc/platforms/pasemi/dma_lib.c +++ b/arch/powerpc/platforms/pasemi/dma_lib.c @@ -576,7 +576,7 @@ int pasemi_dma_init(void) res.start = 0xfd800000; res.end = res.start + 0x1000; } - dma_status = __ioremap(res.start, res.end-res.start, 0); + dma_status = __ioremap(res.start, resource_size(&res), 0); pci_dev_put(iob_pdev); for (i = 0; i < MAX_TXCH; i++) diff --git a/arch/powerpc/platforms/powermac/nvram.c b/arch/powerpc/platforms/powermac/nvram.c index b1cdcf94aa8e..695443bfdb08 100644 --- a/arch/powerpc/platforms/powermac/nvram.c +++ b/arch/powerpc/platforms/powermac/nvram.c @@ -580,10 +580,10 @@ int __init pmac_nvram_init(void) /* Try to obtain an address */ if (of_address_to_resource(dp, 0, &r1) == 0) { nvram_naddrs = 1; - s1 = (r1.end - r1.start) + 1; + s1 = resource_size(&r1); if (of_address_to_resource(dp, 1, &r2) == 0) { nvram_naddrs = 2; - s2 = (r2.end - r2.start) + 1; + s2 = resource_size(&r2); } } diff --git a/arch/powerpc/platforms/powermac/pci.c b/arch/powerpc/platforms/powermac/pci.c index f33e08d573ce..4d4eba324837 100644 --- a/arch/powerpc/platforms/powermac/pci.c +++ b/arch/powerpc/platforms/powermac/pci.c @@ -838,8 +838,7 @@ static void __init setup_u3_ht(struct pci_controller* hose) * into cfg_addr */ hose->cfg_data = ioremap(cfg_res.start, 0x02000000); - hose->cfg_addr = ioremap(self_res.start, - self_res.end - self_res.start + 1); + hose->cfg_addr = ioremap(self_res.start, resource_size(&self_res)); /* * /ht node doesn't expose a "ranges" property, we read the register @@ -1323,8 +1322,7 @@ static void fixup_u4_pcie(struct pci_dev* dev) */ if (r->start >= 0xf0000000 && r->start < 0xf3000000) continue; - if (!region || (r->end - r->start) > - (region->end - region->start)) + if (!region || resource_size(r) > resource_size(region)) region = r; } /* Nothing found, bail */ diff --git a/arch/powerpc/platforms/powermac/time.c b/arch/powerpc/platforms/powermac/time.c index 48211ca134c3..11c9fce43b5b 100644 --- a/arch/powerpc/platforms/powermac/time.c +++ b/arch/powerpc/platforms/powermac/time.c @@ -274,7 +274,7 @@ int __init via_calibrate_decr(void) return 0; } of_node_put(vias); - via = ioremap(rsrc.start, rsrc.end - rsrc.start + 1); + via = ioremap(rsrc.start, resource_size(&rsrc)); if (via == NULL) { printk(KERN_ERR "Failed to map VIA for timer calibration !\n"); return 0; diff --git a/arch/powerpc/sysdev/axonram.c b/arch/powerpc/sysdev/axonram.c index bd0d54060b94..265f0f09395a 100644 --- a/arch/powerpc/sysdev/axonram.c +++ b/arch/powerpc/sysdev/axonram.c @@ -203,7 +203,7 @@ static int axon_ram_probe(struct platform_device *device) goto failed; } - bank->size = resource.end - resource.start + 1; + bank->size = resource_size(&resource); if (bank->size == 0) { dev_err(&device->dev, "No DDR2 memory found for %s%d\n", diff --git a/arch/powerpc/sysdev/cpm1.c b/arch/powerpc/sysdev/cpm1.c index 350787c83e22..5d7d59a43c4c 100644 --- a/arch/powerpc/sysdev/cpm1.c +++ b/arch/powerpc/sysdev/cpm1.c @@ -148,7 +148,7 @@ unsigned int cpm_pic_init(void) if (ret) goto end; - cpic_reg = ioremap(res.start, res.end - res.start + 1); + cpic_reg = ioremap(res.start, resource_size(&res)); if (cpic_reg == NULL) goto end; diff --git a/arch/powerpc/sysdev/cpm_common.c b/arch/powerpc/sysdev/cpm_common.c index 2b69aa0315b3..d55d0ad0deab 100644 --- a/arch/powerpc/sysdev/cpm_common.c +++ b/arch/powerpc/sysdev/cpm_common.c @@ -115,7 +115,7 @@ int cpm_muram_init(void) max = r.end; rh_attach_region(&cpm_muram_info, r.start - muram_pbase, - r.end - r.start + 1); + resource_size(&r)); } muram_vbase = ioremap(muram_pbase, max - muram_pbase + 1); diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c index 8e9e06a7ca59..4f2680f431b5 100644 --- a/arch/powerpc/sysdev/dart_iommu.c +++ b/arch/powerpc/sysdev/dart_iommu.c @@ -239,7 +239,7 @@ static int __init dart_init(struct device_node *dart_node) DARTMAP_RPNMASK); /* Map in DART registers */ - dart = ioremap(r.start, r.end - r.start + 1); + dart = ioremap(r.start, resource_size(&r)); if (dart == NULL) panic("DART: Cannot map registers!"); diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c index 92e78333c47c..419a77239bd7 100644 --- a/arch/powerpc/sysdev/fsl_msi.c +++ b/arch/powerpc/sysdev/fsl_msi.c @@ -349,7 +349,7 @@ static int __devinit fsl_of_msi_probe(struct platform_device *dev) goto error_out; } - msi->msi_regs = ioremap(res.start, res.end - res.start + 1); + msi->msi_regs = ioremap(res.start, resource_size(&res)); if (!msi->msi_regs) { dev_err(&dev->dev, "ioremap problem failed\n"); goto error_out; diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c index 68ca9290df94..ba5cb3fa7074 100644 --- a/arch/powerpc/sysdev/fsl_pci.c +++ b/arch/powerpc/sysdev/fsl_pci.c @@ -64,7 +64,7 @@ static int __init setup_one_atmu(struct ccsr_pci __iomem *pci, { resource_size_t pci_addr = res->start - offset; resource_size_t phys_addr = res->start; - resource_size_t size = res->end - res->start + 1; + resource_size_t size = resource_size(res); u32 flags = 0x80044000; /* enable & mem R/W */ unsigned int i; @@ -108,7 +108,7 @@ static void __init setup_pci_atmu(struct pci_controller *hose, char *name = hose->dn->full_name; pr_debug("PCI memory map start 0x%016llx, size 0x%016llx\n", - (u64)rsrc->start, (u64)rsrc->end - (u64)rsrc->start + 1); + (u64)rsrc->start, (u64)resource_size(rsrc)); if (of_device_is_compatible(hose->dn, "fsl,qoriq-pcie-v2.2")) { win_idx = 2; @@ -116,7 +116,7 @@ static void __init setup_pci_atmu(struct pci_controller *hose, end_idx = 3; } - pci = ioremap(rsrc->start, rsrc->end - rsrc->start + 1); + pci = ioremap(rsrc->start, resource_size(rsrc)); if (!pci) { dev_err(hose->parent, "Unable to map ATMU registers\n"); return; @@ -153,9 +153,9 @@ static void __init setup_pci_atmu(struct pci_controller *hose, } else { pr_debug("PCI IO resource start 0x%016llx, size 0x%016llx, " "phy base 0x%016llx.\n", - (u64)hose->io_resource.start, - (u64)hose->io_resource.end - (u64)hose->io_resource.start + 1, - (u64)hose->io_base_phys); + (u64)hose->io_resource.start, + (u64)resource_size(&hose->io_resource), + (u64)hose->io_base_phys); out_be32(&pci->pow[j].potar, (hose->io_resource.start >> 12)); out_be32(&pci->pow[j].potear, 0); out_be32(&pci->pow[j].powbar, (hose->io_base_phys >> 12)); diff --git a/arch/powerpc/sysdev/fsl_rio.c b/arch/powerpc/sysdev/fsl_rio.c index 5b206a2fe17c..95853386a664 100644 --- a/arch/powerpc/sysdev/fsl_rio.c +++ b/arch/powerpc/sysdev/fsl_rio.c @@ -1523,7 +1523,7 @@ int fsl_rio_setup(struct platform_device *dev) port->priv = priv; port->phys_efptr = 0x100; - priv->regs_win = ioremap(regs.start, regs.end - regs.start + 1); + priv->regs_win = ioremap(regs.start, resource_size(®s)); rio_regs_win = priv->regs_win; /* Probe the master port phy type */ diff --git a/arch/powerpc/sysdev/ipic.c b/arch/powerpc/sysdev/ipic.c index 7367d17364cb..95da897f05a7 100644 --- a/arch/powerpc/sysdev/ipic.c +++ b/arch/powerpc/sysdev/ipic.c @@ -736,7 +736,7 @@ struct ipic * __init ipic_init(struct device_node *node, unsigned int flags) return NULL; } - ipic->regs = ioremap(res.start, res.end - res.start + 1); + ipic->regs = ioremap(res.start, resource_size(&res)); ipic->irqhost->host_data = ipic; diff --git a/arch/powerpc/sysdev/mmio_nvram.c b/arch/powerpc/sysdev/mmio_nvram.c index ddc877a3a23a..69f5814ae6d4 100644 --- a/arch/powerpc/sysdev/mmio_nvram.c +++ b/arch/powerpc/sysdev/mmio_nvram.c @@ -129,7 +129,7 @@ int __init mmio_nvram_init(void) goto out; } nvram_addr = r.start; - mmio_nvram_len = r.end - r.start + 1; + mmio_nvram_len = resource_size(&r); if ( (!mmio_nvram_len) || (!nvram_addr) ) { printk(KERN_WARNING "nvram: address or length is 0\n"); ret = -EIO; diff --git a/arch/powerpc/sysdev/mpc8xx_pic.c b/arch/powerpc/sysdev/mpc8xx_pic.c index 20924f2246f0..22e48e2d71f1 100644 --- a/arch/powerpc/sysdev/mpc8xx_pic.c +++ b/arch/powerpc/sysdev/mpc8xx_pic.c @@ -166,7 +166,7 @@ int mpc8xx_pic_init(void) if (ret) goto out; - siu_reg = ioremap(res.start, res.end - res.start + 1); + siu_reg = ioremap(res.start, resource_size(&res)); if (siu_reg == NULL) { ret = -EINVAL; goto out; diff --git a/arch/powerpc/sysdev/mv64x60_udbg.c b/arch/powerpc/sysdev/mv64x60_udbg.c index 2792dc8b038c..50a81387e9b1 100644 --- a/arch/powerpc/sysdev/mv64x60_udbg.c +++ b/arch/powerpc/sysdev/mv64x60_udbg.c @@ -125,11 +125,11 @@ static void mv64x60_udbg_init(void) of_node_put(np); - mpsc_base = ioremap(r[0].start, r[0].end - r[0].start + 1); + mpsc_base = ioremap(r[0].start, resource_size(&r[0])); if (!mpsc_base) return; - mpsc_intr_cause = ioremap(r[1].start, r[1].end - r[1].start + 1); + mpsc_intr_cause = ioremap(r[1].start, resource_size(&r[1])); if (!mpsc_intr_cause) { iounmap(mpsc_base); return; diff --git a/arch/powerpc/sysdev/ppc4xx_pci.c b/arch/powerpc/sysdev/ppc4xx_pci.c index 156aa7d36258..deda60a7f996 100644 --- a/arch/powerpc/sysdev/ppc4xx_pci.c +++ b/arch/powerpc/sysdev/ppc4xx_pci.c @@ -265,7 +265,7 @@ static void __init ppc4xx_configure_pci_PMMs(struct pci_controller *hose, if (ppc4xx_setup_one_pci_PMM(hose, reg, res->start, res->start - hose->pci_mem_offset, - res->end + 1 - res->start, + resource_size(res), res->flags, j) == 0) { j++; @@ -290,7 +290,7 @@ static void __init ppc4xx_configure_pci_PTMs(struct pci_controller *hose, void __iomem *reg, const struct resource *res) { - resource_size_t size = res->end - res->start + 1; + resource_size_t size = resource_size(res); u32 sa; /* Calculate window size */ @@ -349,7 +349,7 @@ static void __init ppc4xx_probe_pci_bridge(struct device_node *np) bus_range = of_get_property(np, "bus-range", NULL); /* Map registers */ - reg = ioremap(rsrc_reg.start, rsrc_reg.end + 1 - rsrc_reg.start); + reg = ioremap(rsrc_reg.start, resource_size(&rsrc_reg)); if (reg == NULL) { printk(KERN_ERR "%s: Can't map registers !", np->full_name); goto fail; @@ -465,7 +465,7 @@ static void __init ppc4xx_configure_pcix_POMs(struct pci_controller *hose, if (ppc4xx_setup_one_pcix_POM(hose, reg, res->start, res->start - hose->pci_mem_offset, - res->end + 1 - res->start, + resource_size(res), res->flags, j) == 0) { j++; @@ -492,7 +492,7 @@ static void __init ppc4xx_configure_pcix_PIMs(struct pci_controller *hose, int big_pim, int enable_msi_hole) { - resource_size_t size = res->end - res->start + 1; + resource_size_t size = resource_size(res); u32 sa; /* RAM is always at 0 */ @@ -555,7 +555,7 @@ static void __init ppc4xx_probe_pcix_bridge(struct device_node *np) bus_range = of_get_property(np, "bus-range", NULL); /* Map registers */ - reg = ioremap(rsrc_reg.start, rsrc_reg.end + 1 - rsrc_reg.start); + reg = ioremap(rsrc_reg.start, resource_size(&rsrc_reg)); if (reg == NULL) { printk(KERN_ERR "%s: Can't map registers !", np->full_name); goto fail; @@ -1604,7 +1604,7 @@ static void __init ppc4xx_configure_pciex_POMs(struct ppc4xx_pciex_port *port, if (ppc4xx_setup_one_pciex_POM(port, hose, mbase, res->start, res->start - hose->pci_mem_offset, - res->end + 1 - res->start, + resource_size(res), res->flags, j) == 0) { j++; @@ -1639,7 +1639,7 @@ static void __init ppc4xx_configure_pciex_PIMs(struct ppc4xx_pciex_port *port, void __iomem *mbase, struct resource *res) { - resource_size_t size = res->end - res->start + 1; + resource_size_t size = resource_size(res); u64 sa; if (port->endpoint) { diff --git a/arch/powerpc/sysdev/qe_lib/qe_ic.c b/arch/powerpc/sysdev/qe_lib/qe_ic.c index b2acda07220d..18e75ca19fe6 100644 --- a/arch/powerpc/sysdev/qe_lib/qe_ic.c +++ b/arch/powerpc/sysdev/qe_lib/qe_ic.c @@ -347,7 +347,7 @@ void __init qe_ic_init(struct device_node *node, unsigned int flags, return; } - qe_ic->regs = ioremap(res.start, res.end - res.start + 1); + qe_ic->regs = ioremap(res.start, resource_size(&res)); qe_ic->irqhost->host_data = qe_ic; qe_ic->hc_irq = qe_ic_irq_chip; diff --git a/arch/powerpc/sysdev/qe_lib/qe_io.c b/arch/powerpc/sysdev/qe_lib/qe_io.c index 77e4934b88c5..fd1a6c3b1721 100644 --- a/arch/powerpc/sysdev/qe_lib/qe_io.c +++ b/arch/powerpc/sysdev/qe_lib/qe_io.c @@ -41,7 +41,7 @@ int par_io_init(struct device_node *np) ret = of_address_to_resource(np, 0, &res); if (ret) return ret; - par_io = ioremap(res.start, res.end - res.start + 1); + par_io = ioremap(res.start, resource_size(&res)); num_ports = of_get_property(np, "num-ports", NULL); if (num_ports) diff --git a/arch/powerpc/sysdev/xics/icp-native.c b/arch/powerpc/sysdev/xics/icp-native.c index 1f15ad436140..039a7820ba7f 100644 --- a/arch/powerpc/sysdev/xics/icp-native.c +++ b/arch/powerpc/sysdev/xics/icp-native.c @@ -247,7 +247,7 @@ static int __init icp_native_init_one_node(struct device_node *np, return -1; } - if (icp_native_map_one_cpu(*indx, r.start, r.end - r.start)) + if (icp_native_map_one_cpu(*indx, r.start, resource_size(&r))) return -1; (*indx)++; diff --git a/arch/sh/kernel/io_trapped.c b/arch/sh/kernel/io_trapped.c index 32c385ef1011..0f62f4672754 100644 --- a/arch/sh/kernel/io_trapped.c +++ b/arch/sh/kernel/io_trapped.c @@ -58,7 +58,7 @@ int register_trapped_io(struct trapped_io *tiop) for (k = 0; k < tiop->num_resources; k++) { res = tiop->resource + k; - len += roundup((res->end - res->start) + 1, PAGE_SIZE); + len += roundup(resource_size(res), PAGE_SIZE); flags |= res->flags; } @@ -85,7 +85,7 @@ int register_trapped_io(struct trapped_io *tiop) (unsigned long)(tiop->virt_base + len), res->flags & IORESOURCE_IO ? "io" : "mmio", (unsigned long)res->start); - len += roundup((res->end - res->start) + 1, PAGE_SIZE); + len += roundup(resource_size(res), PAGE_SIZE); } tiop->magic = IO_TRAPPED_MAGIC; @@ -128,7 +128,7 @@ void __iomem *match_trapped_io_handler(struct list_head *list, return tiop->virt_base + voffs; } - len = (res->end - res->start) + 1; + len = resource_size(res); voffs += roundup(len, PAGE_SIZE); } } @@ -173,7 +173,7 @@ static unsigned long lookup_address(struct trapped_io *tiop, for (k = 0; k < tiop->num_resources; k++) { res = tiop->resource + k; - len = roundup((res->end - res->start) + 1, PAGE_SIZE); + len = roundup(resource_size(res), PAGE_SIZE); if (address < (vaddr + len)) return res->start + (address - vaddr); vaddr += len; diff --git a/arch/sh/kernel/machine_kexec.c b/arch/sh/kernel/machine_kexec.c index e2a3af31ff99..c5a33f007f88 100644 --- a/arch/sh/kernel/machine_kexec.c +++ b/arch/sh/kernel/machine_kexec.c @@ -170,7 +170,7 @@ void __init reserve_crashkernel(void) if (crashk_res.end == crashk_res.start) goto disable; - crash_size = PAGE_ALIGN(crashk_res.end - crashk_res.start + 1); + crash_size = PAGE_ALIGN(resource_size(&crashk_res)); if (!crashk_res.start) { unsigned long max = memblock_end_of_DRAM() - memory_limit; crashk_res.start = __memblock_alloc_base(crash_size, PAGE_SIZE, max); diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c index 1c9c80a1a86a..6ffccd6e0156 100644 --- a/arch/sparc/kernel/ioport.c +++ b/arch/sparc/kernel/ioport.c @@ -228,7 +228,7 @@ _sparc_ioremap(struct resource *res, u32 bus, u32 pa, int sz) } pa &= PAGE_MASK; - sparc_mapiorange(bus, pa, res->start, res->end - res->start + 1); + sparc_mapiorange(bus, pa, res->start, resource_size(res)); return (void __iomem *)(unsigned long)(res->start + offset); } @@ -240,7 +240,7 @@ static void _sparc_free_io(struct resource *res) { unsigned long plen; - plen = res->end - res->start + 1; + plen = resource_size(res); BUG_ON((plen & (PAGE_SIZE-1)) != 0); sparc_unmapiorange(res->start, plen); release_resource(res); @@ -331,9 +331,9 @@ static void sbus_free_coherent(struct device *dev, size_t n, void *p, } n = PAGE_ALIGN(n); - if ((res->end-res->start)+1 != n) { + if (resource_size(res) != n) { printk("sbus_free_consistent: region 0x%lx asked 0x%zx\n", - (long)((res->end-res->start)+1), n); + (long)resource_size(res), n); return; } @@ -504,9 +504,9 @@ static void pci32_free_coherent(struct device *dev, size_t n, void *p, } n = PAGE_ALIGN(n); - if ((res->end-res->start)+1 != n) { + if (resource_size(res) != n) { printk("pci_free_consistent: region 0x%lx asked 0x%lx\n", - (long)((res->end-res->start)+1), (long)n); + (long)resource_size(res), (long)n); return; } diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index 713dc91020a6..2d1453dd93d5 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c @@ -820,11 +820,9 @@ static int __pci_mmap_make_offset_bus(struct pci_dev *pdev, struct vm_area_struc unsigned long space_size, user_offset, user_size; if (mmap_state == pci_mmap_io) { - space_size = (pbm->io_space.end - - pbm->io_space.start) + 1; + space_size = resource_size(&pbm->io_space); } else { - space_size = (pbm->mem_space.end - - pbm->mem_space.start) + 1; + space_size = resource_size(&pbm->mem_space); } /* Make sure the request is in range. */ diff --git a/arch/tile/kernel/setup.c b/arch/tile/kernel/setup.c index 6cdc9ba55fe0..5f85d8b34dbb 100644 --- a/arch/tile/kernel/setup.c +++ b/arch/tile/kernel/setup.c @@ -553,8 +553,7 @@ static void __init setup_bootmem_allocator(void) #ifdef CONFIG_KEXEC if (crashk_res.start != crashk_res.end) - reserve_bootmem(crashk_res.start, - crashk_res.end - crashk_res.start + 1, 0); + reserve_bootmem(crashk_res.start, resource_size(&crashk_res), 0); #endif } diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c index e8c33a302006..726494b58345 100644 --- a/arch/x86/kernel/pci-calgary_64.c +++ b/arch/x86/kernel/pci-calgary_64.c @@ -1553,7 +1553,7 @@ static void __init calgary_fixup_one_tce_space(struct pci_dev *dev) continue; /* cover the whole region */ - npages = (r->end - r->start) >> PAGE_SHIFT; + npages = resource_size(r) >> PAGE_SHIFT; npages++; iommu_range_reserve(tbl, r->start, npages); diff --git a/arch/x86/kernel/probe_roms.c b/arch/x86/kernel/probe_roms.c index ba0a4cce53be..63228035f9d7 100644 --- a/arch/x86/kernel/probe_roms.c +++ b/arch/x86/kernel/probe_roms.c @@ -234,7 +234,7 @@ void __init probe_roms(void) /* check for extension rom (ignore length byte!) */ rom = isa_bus_to_virt(extension_rom_resource.start); if (romsignature(rom)) { - length = extension_rom_resource.end - extension_rom_resource.start + 1; + length = resource_size(&extension_rom_resource); if (romchecksum(rom, length)) { request_resource(&iomem_resource, &extension_rom_resource); upper = extension_rom_resource.start; diff --git a/drivers/char/bsr.c b/drivers/char/bsr.c index cf39bc08ce08..0c688232aab3 100644 --- a/drivers/char/bsr.c +++ b/drivers/char/bsr.c @@ -212,7 +212,7 @@ static int bsr_add_node(struct device_node *bn) cur->bsr_minor = i + total_bsr_devs; cur->bsr_addr = res.start; - cur->bsr_len = res.end - res.start + 1; + cur->bsr_len = resource_size(&res); cur->bsr_bytes = bsr_bytes[i]; cur->bsr_stride = bsr_stride[i]; cur->bsr_dev = MKDEV(bsr_major, i + total_bsr_devs); diff --git a/drivers/char/xilinx_hwicap/xilinx_hwicap.c b/drivers/char/xilinx_hwicap/xilinx_hwicap.c index 39ccdeada791..e90e1c74fd4c 100644 --- a/drivers/char/xilinx_hwicap/xilinx_hwicap.c +++ b/drivers/char/xilinx_hwicap/xilinx_hwicap.c @@ -621,7 +621,7 @@ static int __devinit hwicap_setup(struct device *dev, int id, drvdata->mem_start = regs_res->start; drvdata->mem_end = regs_res->end; - drvdata->mem_size = regs_res->end - regs_res->start + 1; + drvdata->mem_size = resource_size(regs_res); if (!request_mem_region(drvdata->mem_start, drvdata->mem_size, DRIVER_NAME)) { diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 954e334e01bb..06f9f27dbe7c 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -1304,8 +1304,7 @@ static int mv_xor_shared_probe(struct platform_device *pdev) if (!res) return -ENODEV; - msp->xor_base = devm_ioremap(&pdev->dev, res->start, - res->end - res->start + 1); + msp->xor_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); if (!msp->xor_base) return -EBUSY; @@ -1314,7 +1313,7 @@ static int mv_xor_shared_probe(struct platform_device *pdev) return -ENODEV; msp->xor_high_base = devm_ioremap(&pdev->dev, res->start, - res->end - res->start + 1); + resource_size(res)); if (!msp->xor_high_base) return -EBUSY; diff --git a/drivers/edac/cell_edac.c b/drivers/edac/cell_edac.c index db1df59ae2b6..9a6a274e6925 100644 --- a/drivers/edac/cell_edac.c +++ b/drivers/edac/cell_edac.c @@ -140,7 +140,7 @@ static void __devinit cell_edac_init_csrows(struct mem_ctl_info *mci) if (of_node_to_nid(np) != priv->node) continue; csrow->first_page = r.start >> PAGE_SHIFT; - csrow->nr_pages = (r.end - r.start + 1) >> PAGE_SHIFT; + csrow->nr_pages = resource_size(&r) >> PAGE_SHIFT; csrow->last_page = csrow->first_page + csrow->nr_pages - 1; csrow->mtype = MEM_XDR; csrow->edac_mode = EDAC_SECDED; diff --git a/drivers/edac/mpc85xx_edac.c b/drivers/edac/mpc85xx_edac.c index 38ab8e2cd7f4..11e1a5dad96f 100644 --- a/drivers/edac/mpc85xx_edac.c +++ b/drivers/edac/mpc85xx_edac.c @@ -538,15 +538,15 @@ static int __devinit mpc85xx_l2_err_probe(struct platform_device *op) /* we only need the error registers */ r.start += 0xe00; - if (!devm_request_mem_region(&op->dev, r.start, - r.end - r.start + 1, pdata->name)) { + if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r), + pdata->name)) { printk(KERN_ERR "%s: Error while requesting mem region\n", __func__); res = -EBUSY; goto err; } - pdata->l2_vbase = devm_ioremap(&op->dev, r.start, r.end - r.start + 1); + pdata->l2_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r)); if (!pdata->l2_vbase) { printk(KERN_ERR "%s: Unable to setup L2 err regs\n", __func__); res = -ENOMEM; @@ -987,15 +987,15 @@ static int __devinit mpc85xx_mc_err_probe(struct platform_device *op) goto err; } - if (!devm_request_mem_region(&op->dev, r.start, - r.end - r.start + 1, pdata->name)) { + if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r), + pdata->name)) { printk(KERN_ERR "%s: Error while requesting mem region\n", __func__); res = -EBUSY; goto err; } - pdata->mc_vbase = devm_ioremap(&op->dev, r.start, r.end - r.start + 1); + pdata->mc_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r)); if (!pdata->mc_vbase) { printk(KERN_ERR "%s: Unable to setup MC err regs\n", __func__); res = -ENOMEM; diff --git a/drivers/gpio/gpio-u300.c b/drivers/gpio/gpio-u300.c index d92790140fe5..1a86fefd0f83 100644 --- a/drivers/gpio/gpio-u300.c +++ b/drivers/gpio/gpio-u300.c @@ -581,8 +581,8 @@ static int __init gpio_probe(struct platform_device *pdev) if (!memres) goto err_no_resource; - if (request_mem_region(memres->start, memres->end - memres->start, "GPIO Controller") - == NULL) { + if (!request_mem_region(memres->start, resource_size(memres), + "GPIO Controller")) { err = -ENODEV; goto err_no_ioregion; } @@ -640,7 +640,7 @@ static int __init gpio_probe(struct platform_device *pdev) free_irq(gpio_ports[i].irq, &gpio_ports[i]); iounmap(virtbase); err_no_ioremap: - release_mem_region(memres->start, memres->end - memres->start); + release_mem_region(memres->start, resource_size(memres)); err_no_ioregion: err_no_resource: clk_disable(clk); @@ -660,7 +660,7 @@ static int __exit gpio_remove(struct platform_device *pdev) for (i = 0 ; i < U300_GPIO_NUM_PORTS; i++) free_irq(gpio_ports[i].irq, &gpio_ports[i]); iounmap(virtbase); - release_mem_region(memres->start, memres->end - memres->start); + release_mem_region(memres->start, resource_size(memres)); clk_disable(clk); clk_put(clk); return 0; diff --git a/drivers/ide/palm_bk3710.c b/drivers/ide/palm_bk3710.c index 9e8f4e1b0cc9..712c7904d03e 100644 --- a/drivers/ide/palm_bk3710.c +++ b/drivers/ide/palm_bk3710.c @@ -342,7 +342,7 @@ static int __init palm_bk3710_probe(struct platform_device *pdev) return -ENODEV; } - mem_size = mem->end - mem->start + 1; + mem_size = resource_size(mem); if (request_mem_region(mem->start, mem_size, "palm_bk3710") == NULL) { printk(KERN_ERR "failed to request memory region\n"); return -EBUSY; diff --git a/drivers/ide/tx4939ide.c b/drivers/ide/tx4939ide.c index bed3e39aac96..71c231954972 100644 --- a/drivers/ide/tx4939ide.c +++ b/drivers/ide/tx4939ide.c @@ -551,10 +551,10 @@ static int __init tx4939ide_probe(struct platform_device *pdev) return -ENODEV; if (!devm_request_mem_region(&pdev->dev, res->start, - res->end - res->start + 1, "tx4938ide")) + resource_size(res), "tx4938ide")) return -EBUSY; mapbase = (unsigned long)devm_ioremap(&pdev->dev, res->start, - res->end - res->start + 1); + resource_size(res)); if (!mapbase) return -EBUSY; memset(&hw, 0, sizeof(hw)); diff --git a/drivers/input/serio/sa1111ps2.c b/drivers/input/serio/sa1111ps2.c index d55874e5d1c2..44fc8b4bcd81 100644 --- a/drivers/input/serio/sa1111ps2.c +++ b/drivers/input/serio/sa1111ps2.c @@ -300,8 +300,7 @@ static int __devinit ps2_probe(struct sa1111_dev *dev) out: sa1111_disable_device(ps2if->dev); - release_mem_region(dev->res.start, - dev->res.end - dev->res.start + 1); + release_mem_region(dev->res.start, resource_size(&dev->res)); free: sa1111_set_drvdata(dev, NULL); kfree(ps2if); @@ -317,8 +316,7 @@ static int __devexit ps2_remove(struct sa1111_dev *dev) struct ps2if *ps2if = sa1111_get_drvdata(dev); serio_unregister_port(ps2if->io); - release_mem_region(dev->res.start, - dev->res.end - dev->res.start + 1); + release_mem_region(dev->res.start, resource_size(&dev->res)); sa1111_set_drvdata(dev, NULL); kfree(ps2if); diff --git a/drivers/media/video/davinci/vpif.c b/drivers/media/video/davinci/vpif.c index 9f3bfc1eb240..af9680273ff9 100644 --- a/drivers/media/video/davinci/vpif.c +++ b/drivers/media/video/davinci/vpif.c @@ -422,7 +422,7 @@ static int __init vpif_probe(struct platform_device *pdev) if (!res) return -ENOENT; - res_len = res->end - res->start + 1; + res_len = resource_size(res); res = request_mem_region(res->start, res_len, res->name); if (!res) diff --git a/drivers/media/video/omap24xxcam.c b/drivers/media/video/omap24xxcam.c index f6626e87dbc5..69b60ba5dd7a 100644 --- a/drivers/media/video/omap24xxcam.c +++ b/drivers/media/video/omap24xxcam.c @@ -1768,14 +1768,13 @@ static int __devinit omap24xxcam_probe(struct platform_device *pdev) dev_err(cam->dev, "no mem resource?\n"); goto err; } - if (!request_mem_region(mem->start, (mem->end - mem->start) + 1, - pdev->name)) { + if (!request_mem_region(mem->start, resource_size(mem), pdev->name)) { dev_err(cam->dev, "cannot reserve camera register I/O region\n"); goto err; } cam->mmio_base_phys = mem->start; - cam->mmio_size = (mem->end - mem->start) + 1; + cam->mmio_size = resource_size(mem); /* map the region */ cam->mmio_base = (unsigned long) diff --git a/drivers/message/i2o/iop.c b/drivers/message/i2o/iop.c index 090d2a3a6548..a8c08f332da0 100644 --- a/drivers/message/i2o/iop.c +++ b/drivers/message/i2o/iop.c @@ -681,11 +681,11 @@ static int i2o_iop_systab_set(struct i2o_controller *c) if (root && allocate_resource(root, res, sb->desired_mem_size, sb->desired_mem_size, sb->desired_mem_size, 1 << 20, /* Unspecified, so use 1Mb and play safe */ NULL, NULL) >= 0) { c->mem_alloc = 1; - sb->current_mem_size = 1 + res->end - res->start; + sb->current_mem_size = resource_size(res); sb->current_mem_base = res->start; osm_info("%s: allocated %llu bytes of PCI memory at " "0x%016llX.\n", c->name, - (unsigned long long)(1 + res->end - res->start), + (unsigned long long)resource_size(res), (unsigned long long)res->start); } } @@ -703,11 +703,11 @@ static int i2o_iop_systab_set(struct i2o_controller *c) if (root && allocate_resource(root, res, sb->desired_io_size, sb->desired_io_size, sb->desired_io_size, 1 << 20, /* Unspecified, so use 1Mb and play safe */ NULL, NULL) >= 0) { c->io_alloc = 1; - sb->current_io_size = 1 + res->end - res->start; + sb->current_io_size = resource_size(res); sb->current_mem_base = res->start; osm_info("%s: allocated %llu bytes of PCI I/O at " "0x%016llX.\n", c->name, - (unsigned long long)(1 + res->end - res->start), + (unsigned long long)resource_size(res), (unsigned long long)res->start); } } diff --git a/drivers/mfd/tc6387xb.c b/drivers/mfd/tc6387xb.c index ad715bf49cac..71bc835324d8 100644 --- a/drivers/mfd/tc6387xb.c +++ b/drivers/mfd/tc6387xb.c @@ -177,7 +177,7 @@ static int __devinit tc6387xb_probe(struct platform_device *dev) if (ret) goto err_resource; - tc6387xb->scr = ioremap(rscr->start, rscr->end - rscr->start + 1); + tc6387xb->scr = ioremap(rscr->start, resource_size(rscr)); if (!tc6387xb->scr) { ret = -ENOMEM; goto err_ioremap; diff --git a/drivers/misc/atmel-ssc.c b/drivers/misc/atmel-ssc.c index 4afffe610f99..769a4e8e10dc 100644 --- a/drivers/misc/atmel-ssc.c +++ b/drivers/misc/atmel-ssc.c @@ -95,7 +95,7 @@ static int __init ssc_probe(struct platform_device *pdev) } ssc->pdev = pdev; - ssc->regs = ioremap(regs->start, regs->end - regs->start + 1); + ssc->regs = ioremap(regs->start, resource_size(regs)); if (!ssc->regs) { dev_dbg(&pdev->dev, "ioremap failed\n"); retval = -EINVAL; diff --git a/drivers/misc/atmel_pwm.c b/drivers/misc/atmel_pwm.c index 0f3fb4f03bdf..28f5aaa19d4a 100644 --- a/drivers/misc/atmel_pwm.c +++ b/drivers/misc/atmel_pwm.c @@ -329,7 +329,7 @@ static int __init pwm_probe(struct platform_device *pdev) p->pdev = pdev; p->mask = *mp; p->irq = irq; - p->base = ioremap(r->start, r->end - r->start + 1); + p->base = ioremap(r->start, resource_size(r)); if (!p->base) goto fail; p->clk = clk_get(&pdev->dev, "pwm_clk"); diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c index 66dcddb9c205..2a069f908b27 100644 --- a/drivers/mmc/host/dw_mmc.c +++ b/drivers/mmc/host/dw_mmc.c @@ -1595,7 +1595,7 @@ static int dw_mci_probe(struct platform_device *pdev) INIT_LIST_HEAD(&host->queue); ret = -ENOMEM; - host->regs = ioremap(regs->start, regs->end - regs->start + 1); + host->regs = ioremap(regs->start, resource_size(regs)); if (!host->regs) goto err_freehost; diff --git a/drivers/mtd/maps/bfin-async-flash.c b/drivers/mtd/maps/bfin-async-flash.c index d4297a97e100..67815eed2f00 100644 --- a/drivers/mtd/maps/bfin-async-flash.c +++ b/drivers/mtd/maps/bfin-async-flash.c @@ -142,7 +142,7 @@ static int __devinit bfin_flash_probe(struct platform_device *pdev) state->map.write = bfin_flash_write; state->map.copy_to = bfin_flash_copy_to; state->map.bankwidth = pdata->width; - state->map.size = memory->end - memory->start + 1; + state->map.size = resource_size(memory); state->map.virt = (void __iomem *)memory->start; state->map.phys = memory->start; state->map.map_priv_1 = (unsigned long)state; diff --git a/drivers/mtd/maps/ixp2000.c b/drivers/mtd/maps/ixp2000.c index c00b9175ba9e..1594a802631d 100644 --- a/drivers/mtd/maps/ixp2000.c +++ b/drivers/mtd/maps/ixp2000.c @@ -155,7 +155,7 @@ static int ixp2000_flash_probe(struct platform_device *dev) if (!plat) return -ENODEV; - window_size = dev->resource->end - dev->resource->start + 1; + window_size = resource_size(dev->resource); dev_info(&dev->dev, "Probe of IXP2000 flash(%d banks x %dMiB)\n", ixp_data->nr_banks, ((u32)window_size >> 20)); @@ -194,16 +194,17 @@ static int ixp2000_flash_probe(struct platform_device *dev) info->map.copy_to = ixp2000_flash_copy_to; info->res = request_mem_region(dev->resource->start, - dev->resource->end - dev->resource->start + 1, - dev_name(&dev->dev)); + resource_size(dev->resource), + dev_name(&dev->dev)); if (!info->res) { dev_err(&dev->dev, "Could not reserve memory region\n"); err = -ENOMEM; goto Error; } - info->map.map_priv_1 = (unsigned long) ioremap(dev->resource->start, - dev->resource->end - dev->resource->start + 1); + info->map.map_priv_1 = + (unsigned long)ioremap(dev->resource->start, + resource_size(dev->resource)); if (!info->map.map_priv_1) { dev_err(&dev->dev, "Failed to ioremap flash region\n"); err = -EIO; diff --git a/drivers/mtd/maps/pxa2xx-flash.c b/drivers/mtd/maps/pxa2xx-flash.c index f59d62f74d44..7ae137d4b998 100644 --- a/drivers/mtd/maps/pxa2xx-flash.c +++ b/drivers/mtd/maps/pxa2xx-flash.c @@ -70,7 +70,7 @@ static int __devinit pxa2xx_flash_probe(struct platform_device *pdev) info->map.name = (char *) flash->name; info->map.bankwidth = flash->width; info->map.phys = res->start; - info->map.size = res->end - res->start + 1; + info->map.size = resource_size(res); info->parts = flash->parts; info->nr_parts = flash->nr_parts; diff --git a/drivers/mtd/nand/atmel_nand.c b/drivers/mtd/nand/atmel_nand.c index b300705d41cb..d4ba1f218e9a 100644 --- a/drivers/mtd/nand/atmel_nand.c +++ b/drivers/mtd/nand/atmel_nand.c @@ -513,7 +513,7 @@ static int __init atmel_nand_probe(struct platform_device *pdev) host->io_phys = (dma_addr_t)mem->start; - host->io_base = ioremap(mem->start, mem->end - mem->start + 1); + host->io_base = ioremap(mem->start, resource_size(mem)); if (host->io_base == NULL) { printk(KERN_ERR "atmel_nand: ioremap failed\n"); res = -EIO; @@ -547,7 +547,7 @@ static int __init atmel_nand_probe(struct platform_device *pdev) if (no_ecc) nand_chip->ecc.mode = NAND_ECC_NONE; if (hard_ecc && regs) { - host->ecc = ioremap(regs->start, regs->end - regs->start + 1); + host->ecc = ioremap(regs->start, resource_size(regs)); if (host->ecc == NULL) { printk(KERN_ERR "atmel_nand: ioremap failed\n"); res = -EIO; diff --git a/drivers/mtd/nand/bcm_umi_nand.c b/drivers/mtd/nand/bcm_umi_nand.c index 9ec280738a9a..8c569e454dc5 100644 --- a/drivers/mtd/nand/bcm_umi_nand.c +++ b/drivers/mtd/nand/bcm_umi_nand.c @@ -380,7 +380,7 @@ static int __devinit bcm_umi_nand_probe(struct platform_device *pdev) return -ENXIO; /* map physical address */ - bcm_umi_io_base = ioremap(r->start, r->end - r->start + 1); + bcm_umi_io_base = ioremap(r->start, resource_size(r)); if (!bcm_umi_io_base) { printk(KERN_ERR "ioremap to access BCM UMI NAND chip failed\n"); diff --git a/drivers/mtd/nand/mpc5121_nfc.c b/drivers/mtd/nand/mpc5121_nfc.c index 2f7c930872f9..eb1fbac63eb6 100644 --- a/drivers/mtd/nand/mpc5121_nfc.c +++ b/drivers/mtd/nand/mpc5121_nfc.c @@ -713,7 +713,7 @@ static int __devinit mpc5121_nfc_probe(struct platform_device *op) } regs_paddr = res.start; - regs_size = res.end - res.start + 1; + regs_size = resource_size(&res); if (!devm_request_mem_region(dev, regs_paddr, regs_size, DRV_NAME)) { dev_err(dev, "Error requesting memory region!\n"); diff --git a/drivers/net/bcm63xx_enet.c b/drivers/net/bcm63xx_enet.c index f1573d492e90..85045cde3126 100644 --- a/drivers/net/bcm63xx_enet.c +++ b/drivers/net/bcm63xx_enet.c @@ -1646,7 +1646,7 @@ static int __devinit bcm_enet_probe(struct platform_device *pdev) if (ret) goto out; - iomem_size = res_mem->end - res_mem->start + 1; + iomem_size = resource_size(res_mem); if (!request_mem_region(res_mem->start, iomem_size, "bcm63xx_enet")) { ret = -EBUSY; goto out; @@ -1861,7 +1861,7 @@ static int __devexit bcm_enet_remove(struct platform_device *pdev) /* release device resources */ iounmap(priv->base); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(res->start, res->end - res->start + 1); + release_mem_region(res->start, resource_size(res)); /* disable hw block clocks */ if (priv->phy_clk) { @@ -1897,7 +1897,7 @@ static int __devinit bcm_enet_shared_probe(struct platform_device *pdev) if (!res) return -ENODEV; - iomem_size = res->end - res->start + 1; + iomem_size = resource_size(res); if (!request_mem_region(res->start, iomem_size, "bcm63xx_enet_dma")) return -EBUSY; @@ -1915,7 +1915,7 @@ static int __devexit bcm_enet_shared_remove(struct platform_device *pdev) iounmap(bcm_enet_shared_base); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(res->start, res->end - res->start + 1); + release_mem_region(res->start, resource_size(res)); return 0; } diff --git a/drivers/net/can/softing/softing_main.c b/drivers/net/can/softing/softing_main.c index 60a49e5a2a53..f76e88e74285 100644 --- a/drivers/net/can/softing/softing_main.c +++ b/drivers/net/can/softing/softing_main.c @@ -799,7 +799,7 @@ static __devinit int softing_pdev_probe(struct platform_device *pdev) if (!pres) goto platform_resource_failed; card->dpram_phys = pres->start; - card->dpram_size = pres->end - pres->start + 1; + card->dpram_size = resource_size(pres); card->dpram = ioremap_nocache(card->dpram_phys, card->dpram_size); if (!card->dpram) { dev_alert(&card->pdev->dev, "dpram ioremap failed\n"); diff --git a/drivers/net/davinci_emac.c b/drivers/net/davinci_emac.c index dcc4a170b0f3..c35ba5fba8f6 100644 --- a/drivers/net/davinci_emac.c +++ b/drivers/net/davinci_emac.c @@ -1821,7 +1821,7 @@ static int __devinit davinci_emac_probe(struct platform_device *pdev) } priv->emac_base_phys = res->start + pdata->ctrl_reg_offset; - size = res->end - res->start + 1; + size = resource_size(res); if (!request_mem_region(res->start, size, ndev->name)) { dev_err(&pdev->dev, "failed request_mem_region() for regs\n"); rc = -ENXIO; @@ -1926,7 +1926,7 @@ no_irq_res: cpdma_ctlr_destroy(priv->dma); no_dma: res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(res->start, res->end - res->start + 1); + release_mem_region(res->start, resource_size(res)); iounmap(priv->remap_addr); probe_quit: @@ -1960,7 +1960,7 @@ static int __devexit davinci_emac_remove(struct platform_device *pdev) cpdma_chan_destroy(priv->rxchan); cpdma_ctlr_destroy(priv->dma); - release_mem_region(res->start, res->end - res->start + 1); + release_mem_region(res->start, resource_size(res)); unregister_netdev(ndev); iounmap(priv->remap_addr); diff --git a/drivers/net/ethoc.c b/drivers/net/ethoc.c index a83dd312c3ac..15e4a71fbf1e 100644 --- a/drivers/net/ethoc.c +++ b/drivers/net/ethoc.c @@ -965,7 +965,7 @@ static int __devinit ethoc_probe(struct platform_device *pdev) priv = netdev_priv(netdev); priv->netdev = netdev; priv->dma_alloc = 0; - priv->io_region_size = mmio->end - mmio->start + 1; + priv->io_region_size = resource_size(mmio); priv->iobase = devm_ioremap_nocache(&pdev->dev, netdev->base_addr, resource_size(mmio)); diff --git a/drivers/net/fec_mpc52xx.c b/drivers/net/fec_mpc52xx.c index 9f81b1ac130e..fe57eee8a679 100644 --- a/drivers/net/fec_mpc52xx.c +++ b/drivers/net/fec_mpc52xx.c @@ -867,10 +867,11 @@ static int __devinit mpc52xx_fec_probe(struct platform_device *op) "Error while parsing device node resource\n" ); goto err_netdev; } - if ((mem.end - mem.start + 1) < sizeof(struct mpc52xx_fec)) { + if (resource_size(&mem) < sizeof(struct mpc52xx_fec)) { printk(KERN_ERR DRIVER_NAME - " - invalid resource size (%lx < %x), check mpc52xx_devices.c\n", - (unsigned long)(mem.end - mem.start + 1), sizeof(struct mpc52xx_fec)); + " - invalid resource size (%lx < %x), check mpc52xx_devices.c\n", + (unsigned long)resource_size(&mem), + sizeof(struct mpc52xx_fec)); rv = -EINVAL; goto err_netdev; } diff --git a/drivers/net/fs_enet/mii-bitbang.c b/drivers/net/fs_enet/mii-bitbang.c index ad2975440719..b09270b5d0a5 100644 --- a/drivers/net/fs_enet/mii-bitbang.c +++ b/drivers/net/fs_enet/mii-bitbang.c @@ -120,7 +120,7 @@ static int __devinit fs_mii_bitbang_init(struct mii_bus *bus, if (ret) return ret; - if (res.end - res.start < 13) + if (resource_size(&res) <= 13) return -ENODEV; /* This should really encode the pin number as well, but all @@ -139,7 +139,7 @@ static int __devinit fs_mii_bitbang_init(struct mii_bus *bus, return -ENODEV; mdc_pin = *data; - bitbang->dir = ioremap(res.start, res.end - res.start + 1); + bitbang->dir = ioremap(res.start, resource_size(&res)); if (!bitbang->dir) return -ENOMEM; diff --git a/drivers/net/fs_enet/mii-fec.c b/drivers/net/fs_enet/mii-fec.c index 6a2e150e75bb..e0e9d6c35d83 100644 --- a/drivers/net/fs_enet/mii-fec.c +++ b/drivers/net/fs_enet/mii-fec.c @@ -136,7 +136,7 @@ static int __devinit fs_enet_mdio_probe(struct platform_device *ofdev) snprintf(new_bus->id, MII_BUS_ID_SIZE, "%x", res.start); - fec->fecp = ioremap(res.start, res.end - res.start + 1); + fec->fecp = ioremap(res.start, resource_size(&res)); if (!fec->fecp) goto out_fec; diff --git a/drivers/net/gianfar_ptp.c b/drivers/net/gianfar_ptp.c index d8e175382d1d..1c97861596f0 100644 --- a/drivers/net/gianfar_ptp.c +++ b/drivers/net/gianfar_ptp.c @@ -491,7 +491,7 @@ static int gianfar_ptp_probe(struct platform_device *dev) spin_lock_init(&etsects->lock); etsects->regs = ioremap(etsects->rsrc->start, - 1 + etsects->rsrc->end - etsects->rsrc->start); + resource_size(etsects->rsrc)); if (!etsects->regs) { pr_err("ioremap ptp registers failed\n"); goto no_ioremap; diff --git a/drivers/net/ibm_newemac/core.c b/drivers/net/ibm_newemac/core.c index 079450fe5e96..725399ea0690 100644 --- a/drivers/net/ibm_newemac/core.c +++ b/drivers/net/ibm_newemac/core.c @@ -2770,7 +2770,7 @@ static int __devinit emac_probe(struct platform_device *ofdev) } // TODO : request_mem_region dev->emacp = ioremap(dev->rsrc_regs.start, - dev->rsrc_regs.end - dev->rsrc_regs.start + 1); + resource_size(&dev->rsrc_regs)); if (dev->emacp == NULL) { printk(KERN_ERR "%s: Can't map device registers!\n", np->full_name); diff --git a/drivers/net/macb.c b/drivers/net/macb.c index 6c6a02869dfc..27125cdd7e01 100644 --- a/drivers/net/macb.c +++ b/drivers/net/macb.c @@ -1169,7 +1169,7 @@ static int __init macb_probe(struct platform_device *pdev) clk_enable(bp->hclk); #endif - bp->regs = ioremap(regs->start, regs->end - regs->start + 1); + bp->regs = ioremap(regs->start, resource_size(regs)); if (!bp->regs) { dev_err(&pdev->dev, "failed to map registers, aborting.\n"); err = -ENOMEM; diff --git a/drivers/net/mv643xx_eth.c b/drivers/net/mv643xx_eth.c index a5d9b1c310b3..b75648257381 100644 --- a/drivers/net/mv643xx_eth.c +++ b/drivers/net/mv643xx_eth.c @@ -2593,7 +2593,7 @@ static int mv643xx_eth_shared_probe(struct platform_device *pdev) if (msp == NULL) goto out; - msp->base = ioremap(res->start, res->end - res->start + 1); + msp->base = ioremap(res->start, resource_size(res)); if (msp->base == NULL) goto out_free; diff --git a/drivers/net/pxa168_eth.c b/drivers/net/pxa168_eth.c index 89f7540d90f9..df1292eb9c2e 100644 --- a/drivers/net/pxa168_eth.c +++ b/drivers/net/pxa168_eth.c @@ -1502,7 +1502,7 @@ static int pxa168_eth_probe(struct platform_device *pdev) err = -ENODEV; goto err_netdev; } - pep->base = ioremap(res->start, res->end - res->start + 1); + pep->base = ioremap(res->start, resource_size(res)); if (pep->base == NULL) { err = -ENOMEM; goto err_netdev; diff --git a/drivers/net/sb1250-mac.c b/drivers/net/sb1250-mac.c index 68d50429ddf3..ea65f7ec360a 100644 --- a/drivers/net/sb1250-mac.c +++ b/drivers/net/sb1250-mac.c @@ -2597,7 +2597,7 @@ static int __devinit sbmac_probe(struct platform_device *pldev) res = platform_get_resource(pldev, IORESOURCE_MEM, 0); BUG_ON(!res); - sbm_base = ioremap_nocache(res->start, res->end - res->start + 1); + sbm_base = ioremap_nocache(res->start, resource_size(res)); if (!sbm_base) { printk(KERN_ERR "%s: unable to map device registers\n", dev_name(&pldev->dev)); diff --git a/drivers/parport/parport_ax88796.c b/drivers/parport/parport_ax88796.c index 2c5ac2bf5c56..844f6137970a 100644 --- a/drivers/parport/parport_ax88796.c +++ b/drivers/parport/parport_ax88796.c @@ -293,7 +293,7 @@ static int parport_ax88796_probe(struct platform_device *pdev) goto exit_mem; } - size = (res->end - res->start) + 1; + size = resource_size(res); spacing = size / 3; dd->io = request_mem_region(res->start, size, pdev->name); diff --git a/drivers/pci/hotplug/shpchp_sysfs.c b/drivers/pci/hotplug/shpchp_sysfs.c index 071b7dc0094b..efa30da1ae8f 100644 --- a/drivers/pci/hotplug/shpchp_sysfs.c +++ b/drivers/pci/hotplug/shpchp_sysfs.c @@ -50,29 +50,26 @@ static ssize_t show_ctrl (struct device *dev, struct device_attribute *attr, cha pci_bus_for_each_resource(bus, res, index) { if (res && (res->flags & IORESOURCE_MEM) && !(res->flags & IORESOURCE_PREFETCH)) { - out += sprintf(out, "start = %8.8llx, " - "length = %8.8llx\n", - (unsigned long long)res->start, - (unsigned long long)(res->end - res->start)); + out += sprintf(out, "start = %8.8llx, length = %8.8llx\n", + (unsigned long long)res->start, + (unsigned long long)resource_size(res)); } } out += sprintf(out, "Free resources: prefetchable memory\n"); pci_bus_for_each_resource(bus, res, index) { if (res && (res->flags & IORESOURCE_MEM) && (res->flags & IORESOURCE_PREFETCH)) { - out += sprintf(out, "start = %8.8llx, " - "length = %8.8llx\n", - (unsigned long long)res->start, - (unsigned long long)(res->end - res->start)); + out += sprintf(out, "start = %8.8llx, length = %8.8llx\n", + (unsigned long long)res->start, + (unsigned long long)resource_size(res)); } } out += sprintf(out, "Free resources: IO\n"); pci_bus_for_each_resource(bus, res, index) { if (res && (res->flags & IORESOURCE_IO)) { - out += sprintf(out, "start = %8.8llx, " - "length = %8.8llx\n", - (unsigned long long)res->start, - (unsigned long long)(res->end - res->start)); + out += sprintf(out, "start = %8.8llx, length = %8.8llx\n", + (unsigned long long)res->start, + (unsigned long long)resource_size(res)); } } out += sprintf(out, "Free resources: bus numbers\n"); diff --git a/drivers/pcmcia/at91_cf.c b/drivers/pcmcia/at91_cf.c index fb33fa42d249..4902206f53d9 100644 --- a/drivers/pcmcia/at91_cf.c +++ b/drivers/pcmcia/at91_cf.c @@ -283,8 +283,7 @@ static int __init at91_cf_probe(struct platform_device *pdev) } /* reserve chip-select regions */ - if (!request_mem_region(io->start, io->end + 1 - io->start, - driver_name)) { + if (!request_mem_region(io->start, resource_size(io), driver_name)) { status = -ENXIO; goto fail1; } @@ -308,7 +307,7 @@ static int __init at91_cf_probe(struct platform_device *pdev) return 0; fail2: - release_mem_region(io->start, io->end + 1 - io->start); + release_mem_region(io->start, resource_size(io)); fail1: if (cf->socket.io_offset) iounmap((void __iomem *) cf->socket.io_offset); @@ -339,7 +338,7 @@ static int __exit at91_cf_remove(struct platform_device *pdev) struct resource *io = cf->socket.io[0].res; pcmcia_unregister_socket(&cf->socket); - release_mem_region(io->start, io->end + 1 - io->start); + release_mem_region(io->start, resource_size(io)); iounmap((void __iomem *) cf->socket.io_offset); if (board->irq_pin) { free_irq(board->irq_pin, cf); diff --git a/drivers/pcmcia/electra_cf.c b/drivers/pcmcia/electra_cf.c index 6defd4a8168e..06ad3e5e7d3d 100644 --- a/drivers/pcmcia/electra_cf.c +++ b/drivers/pcmcia/electra_cf.c @@ -209,9 +209,9 @@ static int __devinit electra_cf_probe(struct platform_device *ofdev) cf->ofdev = ofdev; cf->mem_phys = mem.start; - cf->mem_size = PAGE_ALIGN(mem.end - mem.start); + cf->mem_size = PAGE_ALIGN(resource_size(&mem)); cf->mem_base = ioremap(cf->mem_phys, cf->mem_size); - cf->io_size = PAGE_ALIGN(io.end - io.start); + cf->io_size = PAGE_ALIGN(resource_size(&io)); area = __get_vm_area(cf->io_size, 0, PHB_IO_BASE, PHB_IO_END); if (area == NULL) diff --git a/drivers/pcmcia/rsrc_iodyn.c b/drivers/pcmcia/rsrc_iodyn.c index 523eb691c30b..f53c237bda2f 100644 --- a/drivers/pcmcia/rsrc_iodyn.c +++ b/drivers/pcmcia/rsrc_iodyn.c @@ -135,7 +135,7 @@ static int iodyn_find_io(struct pcmcia_socket *s, unsigned int attr, try = res->end + 1; if ((*base == 0) || (*base == try)) { if (adjust_resource(s->io[i].res, res->start, - res->end - res->start + num + 1)) + resource_size(res) + num)) continue; *base = try; s->io[i].InUse += num; @@ -147,8 +147,8 @@ static int iodyn_find_io(struct pcmcia_socket *s, unsigned int attr, try = res->start - num; if ((*base == 0) || (*base == try)) { if (adjust_resource(s->io[i].res, - res->start - num, - res->end - res->start + num + 1)) + res->start - num, + resource_size(res) + num)) continue; *base = try; s->io[i].InUse += num; diff --git a/drivers/pcmcia/rsrc_nonstatic.c b/drivers/pcmcia/rsrc_nonstatic.c index b187555d4388..9da9656242af 100644 --- a/drivers/pcmcia/rsrc_nonstatic.c +++ b/drivers/pcmcia/rsrc_nonstatic.c @@ -770,7 +770,7 @@ static int nonstatic_find_io(struct pcmcia_socket *s, unsigned int attr, res->end + num); if (!ret) { ret = adjust_resource(s->io[i].res, res->start, - res->end - res->start + num + 1); + resource_size(res) + num); if (ret) continue; *base = try; @@ -788,8 +788,8 @@ static int nonstatic_find_io(struct pcmcia_socket *s, unsigned int attr, res->end); if (!ret) { ret = adjust_resource(s->io[i].res, - res->start - num, - res->end - res->start + num + 1); + res->start - num, + resource_size(res) + num); if (ret) continue; *base = try; diff --git a/drivers/pnp/pnpacpi/rsparser.c b/drivers/pnp/pnpacpi/rsparser.c index 100e4d9372f1..1a6937d9118f 100644 --- a/drivers/pnp/pnpacpi/rsparser.c +++ b/drivers/pnp/pnpacpi/rsparser.c @@ -1018,7 +1018,7 @@ static void pnpacpi_encode_io(struct pnp_dev *dev, io->minimum = p->start; io->maximum = p->end; io->alignment = 0; /* Correct? */ - io->address_length = p->end - p->start + 1; + io->address_length = resource_size(p); } else { io->minimum = 0; io->address_length = 0; @@ -1036,7 +1036,7 @@ static void pnpacpi_encode_fixed_io(struct pnp_dev *dev, if (pnp_resource_enabled(p)) { fixed_io->address = p->start; - fixed_io->address_length = p->end - p->start + 1; + fixed_io->address_length = resource_size(p); } else { fixed_io->address = 0; fixed_io->address_length = 0; @@ -1059,7 +1059,7 @@ static void pnpacpi_encode_mem24(struct pnp_dev *dev, memory24->minimum = p->start; memory24->maximum = p->end; memory24->alignment = 0; - memory24->address_length = p->end - p->start + 1; + memory24->address_length = resource_size(p); } else { memory24->minimum = 0; memory24->address_length = 0; @@ -1083,7 +1083,7 @@ static void pnpacpi_encode_mem32(struct pnp_dev *dev, memory32->minimum = p->start; memory32->maximum = p->end; memory32->alignment = 0; - memory32->address_length = p->end - p->start + 1; + memory32->address_length = resource_size(p); } else { memory32->minimum = 0; memory32->alignment = 0; @@ -1106,7 +1106,7 @@ static void pnpacpi_encode_fixed_mem32(struct pnp_dev *dev, p->flags & IORESOURCE_MEM_WRITEABLE ? ACPI_READ_WRITE_MEMORY : ACPI_READ_ONLY_MEMORY; fixed_memory32->address = p->start; - fixed_memory32->address_length = p->end - p->start + 1; + fixed_memory32->address_length = resource_size(p); } else { fixed_memory32->address = 0; fixed_memory32->address_length = 0; diff --git a/drivers/pnp/pnpbios/rsparser.c b/drivers/pnp/pnpbios/rsparser.c index cb1f47bfee96..cca2f9f9f3e3 100644 --- a/drivers/pnp/pnpbios/rsparser.c +++ b/drivers/pnp/pnpbios/rsparser.c @@ -505,7 +505,7 @@ static void pnpbios_encode_mem(struct pnp_dev *dev, unsigned char *p, if (pnp_resource_enabled(res)) { base = res->start; - len = res->end - res->start + 1; + len = resource_size(res); } else { base = 0; len = 0; @@ -529,7 +529,7 @@ static void pnpbios_encode_mem32(struct pnp_dev *dev, unsigned char *p, if (pnp_resource_enabled(res)) { base = res->start; - len = res->end - res->start + 1; + len = resource_size(res); } else { base = 0; len = 0; @@ -559,7 +559,7 @@ static void pnpbios_encode_fixed_mem32(struct pnp_dev *dev, unsigned char *p, if (pnp_resource_enabled(res)) { base = res->start; - len = res->end - res->start + 1; + len = resource_size(res); } else { base = 0; len = 0; @@ -617,7 +617,7 @@ static void pnpbios_encode_port(struct pnp_dev *dev, unsigned char *p, if (pnp_resource_enabled(res)) { base = res->start; - len = res->end - res->start + 1; + len = resource_size(res); } else { base = 0; len = 0; @@ -636,11 +636,11 @@ static void pnpbios_encode_fixed_port(struct pnp_dev *dev, unsigned char *p, struct resource *res) { unsigned long base = res->start; - unsigned long len = res->end - res->start + 1; + unsigned long len = resource_size(res); if (pnp_resource_enabled(res)) { base = res->start; - len = res->end - res->start + 1; + len = resource_size(res); } else { base = 0; len = 0; diff --git a/drivers/rtc/rtc-at32ap700x.c b/drivers/rtc/rtc-at32ap700x.c index e725d51e773d..8dd08305aae1 100644 --- a/drivers/rtc/rtc-at32ap700x.c +++ b/drivers/rtc/rtc-at32ap700x.c @@ -223,7 +223,7 @@ static int __init at32_rtc_probe(struct platform_device *pdev) } rtc->irq = irq; - rtc->regs = ioremap(regs->start, regs->end - regs->start + 1); + rtc->regs = ioremap(regs->start, resource_size(regs)); if (!rtc->regs) { ret = -ENOMEM; dev_dbg(&pdev->dev, "could not map I/O memory\n"); diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index 911e75cdc125..05beb6c1ca79 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -606,7 +606,7 @@ cmos_do_probe(struct device *dev, struct resource *ports, int rtc_irq) * (needing ioremap etc), not i/o space resources like this ... */ ports = request_region(ports->start, - ports->end + 1 - ports->start, + resource_size(ports), driver_name); if (!ports) { dev_dbg(dev, "i/o registers already in use\n"); @@ -750,7 +750,7 @@ cleanup1: cmos_rtc.dev = NULL; rtc_device_unregister(cmos_rtc.rtc); cleanup0: - release_region(ports->start, ports->end + 1 - ports->start); + release_region(ports->start, resource_size(ports)); return retval; } @@ -779,7 +779,7 @@ static void __exit cmos_do_remove(struct device *dev) cmos->rtc = NULL; ports = cmos->iomem; - release_region(ports->start, ports->end + 1 - ports->start); + release_region(ports->start, resource_size(ports)); cmos->iomem = NULL; cmos->dev = NULL; diff --git a/drivers/rtc/rtc-ds1286.c b/drivers/rtc/rtc-ds1286.c index 47e681df31e2..68e6caf25496 100644 --- a/drivers/rtc/rtc-ds1286.c +++ b/drivers/rtc/rtc-ds1286.c @@ -343,7 +343,7 @@ static int __devinit ds1286_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; - priv->size = res->end - res->start + 1; + priv->size = resource_size(res); if (!request_mem_region(res->start, priv->size, pdev->name)) { ret = -EBUSY; goto out; diff --git a/drivers/rtc/rtc-ds1511.c b/drivers/rtc/rtc-ds1511.c index fbabc773dded..568ad30617e7 100644 --- a/drivers/rtc/rtc-ds1511.c +++ b/drivers/rtc/rtc-ds1511.c @@ -490,7 +490,7 @@ ds1511_rtc_probe(struct platform_device *pdev) pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) return -ENOMEM; - pdata->size = res->end - res->start + 1; + pdata->size = resource_size(res); if (!devm_request_mem_region(&pdev->dev, res->start, pdata->size, pdev->name)) return -EBUSY; diff --git a/drivers/rtc/rtc-ds1742.c b/drivers/rtc/rtc-ds1742.c index 042630c90dd3..d84a448dd754 100644 --- a/drivers/rtc/rtc-ds1742.c +++ b/drivers/rtc/rtc-ds1742.c @@ -173,7 +173,7 @@ static int __devinit ds1742_rtc_probe(struct platform_device *pdev) pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) return -ENOMEM; - pdata->size = res->end - res->start + 1; + pdata->size = resource_size(res); if (!devm_request_mem_region(&pdev->dev, res->start, pdata->size, pdev->name)) return -EBUSY; diff --git a/drivers/rtc/rtc-m48t35.c b/drivers/rtc/rtc-m48t35.c index 7410875e5838..8e2a24e33ed6 100644 --- a/drivers/rtc/rtc-m48t35.c +++ b/drivers/rtc/rtc-m48t35.c @@ -154,7 +154,7 @@ static int __devinit m48t35_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; - priv->size = res->end - res->start + 1; + priv->size = resource_size(res); /* * kludge: remove the #ifndef after ioc3 resource * conflicts are resolved diff --git a/drivers/rtc/rtc-m48t59.c b/drivers/rtc/rtc-m48t59.c index 3978f4caf724..28365388fb6c 100644 --- a/drivers/rtc/rtc-m48t59.c +++ b/drivers/rtc/rtc-m48t59.c @@ -433,7 +433,7 @@ static int __devinit m48t59_rtc_probe(struct platform_device *pdev) if (!m48t59->ioaddr) { /* ioaddr not mapped externally */ - m48t59->ioaddr = ioremap(res->start, res->end - res->start + 1); + m48t59->ioaddr = ioremap(res->start, resource_size(res)); if (!m48t59->ioaddr) goto out; } diff --git a/drivers/rtc/rtc-mrst.c b/drivers/rtc/rtc-mrst.c index 0cec5650d56a..d33544802a2e 100644 --- a/drivers/rtc/rtc-mrst.c +++ b/drivers/rtc/rtc-mrst.c @@ -332,9 +332,8 @@ vrtc_mrst_do_probe(struct device *dev, struct resource *iomem, int rtc_irq) if (!iomem) return -ENODEV; - iomem = request_mem_region(iomem->start, - iomem->end + 1 - iomem->start, - driver_name); + iomem = request_mem_region(iomem->start, resource_size(iomem), + driver_name); if (!iomem) { dev_dbg(dev, "i/o mem already in use.\n"); return -EBUSY; diff --git a/drivers/rtc/rtc-puv3.c b/drivers/rtc/rtc-puv3.c index 46f14b82f3ab..b3eba3cddd42 100644 --- a/drivers/rtc/rtc-puv3.c +++ b/drivers/rtc/rtc-puv3.c @@ -267,9 +267,8 @@ static int puv3_rtc_probe(struct platform_device *pdev) return -ENOENT; } - puv3_rtc_mem = request_mem_region(res->start, - res->end-res->start+1, - pdev->name); + puv3_rtc_mem = request_mem_region(res->start, resource_size(res), + pdev->name); if (puv3_rtc_mem == NULL) { dev_err(&pdev->dev, "failed to reserve memory region\n"); diff --git a/drivers/rtc/rtc-s3c.c b/drivers/rtc/rtc-s3c.c index 16512ecae31a..2a65e85e0f56 100644 --- a/drivers/rtc/rtc-s3c.c +++ b/drivers/rtc/rtc-s3c.c @@ -455,8 +455,7 @@ static int __devinit s3c_rtc_probe(struct platform_device *pdev) return -ENOENT; } - s3c_rtc_mem = request_mem_region(res->start, - res->end-res->start+1, + s3c_rtc_mem = request_mem_region(res->start, resource_size(res), pdev->name); if (s3c_rtc_mem == NULL) { @@ -465,7 +464,7 @@ static int __devinit s3c_rtc_probe(struct platform_device *pdev) goto err_nores; } - s3c_rtc_base = ioremap(res->start, res->end - res->start + 1); + s3c_rtc_base = ioremap(res->start, resource_size(res)); if (s3c_rtc_base == NULL) { dev_err(&pdev->dev, "failed ioremap()\n"); ret = -EINVAL; diff --git a/drivers/staging/generic_serial/ser_a2232.c b/drivers/staging/generic_serial/ser_a2232.c index 3f47c2ead8e5..0c08e1c25583 100644 --- a/drivers/staging/generic_serial/ser_a2232.c +++ b/drivers/staging/generic_serial/ser_a2232.c @@ -746,7 +746,8 @@ static int __init a2232board_init(void) zd_a2232[nr_a2232] = z; boardaddr = ZTWO_VADDR( z->resource.start ); - printk("Board is located at address 0x%x, size is 0x%x.\n", boardaddr, (unsigned int) ((z->resource.end+1) - (z->resource.start))); + printk("Board is located at address 0x%x, size is 0x%x\n", + boardaddr, (unsigned int)resource_size(&z->resource)); mem = (volatile struct a2232memory *) boardaddr; diff --git a/drivers/staging/gma500/psb_gtt.c b/drivers/staging/gma500/psb_gtt.c index 74c5a6569d08..280f9d445468 100644 --- a/drivers/staging/gma500/psb_gtt.c +++ b/drivers/staging/gma500/psb_gtt.c @@ -80,7 +80,7 @@ static int psb_gtt_insert(struct drm_device *dev, struct gtt_range *r) { struct drm_psb_private *dev_priv = dev->dev_private; u32 *gtt_slot, pte; - int numpages = (r->resource.end + 1 - r->resource.start) >> PAGE_SHIFT; + int numpages = resource_size(&r->resource) >> PAGE_SHIFT; struct page **pages; int i; @@ -121,7 +121,7 @@ static void psb_gtt_remove(struct drm_device *dev, struct gtt_range *r) { struct drm_psb_private *dev_priv = dev->dev_private; u32 *gtt_slot, pte; - int numpages = (r->resource.end + 1 - r->resource.start) >> PAGE_SHIFT; + int numpages = resource_size(&r->resource) >> PAGE_SHIFT; int i; WARN_ON(r->stolen); @@ -149,7 +149,7 @@ static int psb_gtt_attach_pages(struct gtt_range *gt) struct address_space *mapping; int i; struct page *p; - int pages = (gt->resource.end + 1 - gt->resource.start) >> PAGE_SHIFT; + int pages = resource_size(>->resource) >> PAGE_SHIFT; WARN_ON(gt->pages); @@ -191,7 +191,7 @@ err: static void psb_gtt_detach_pages(struct gtt_range *gt) { int i; - int pages = (gt->resource.end + 1 - gt->resource.start) >> PAGE_SHIFT; + int pages = resource_size(>->resource) >> PAGE_SHIFT; for (i = 0; i < pages; i++) { /* FIXME: do we need to force dirty */ diff --git a/drivers/tty/serial/bfin_5xx.c b/drivers/tty/serial/bfin_5xx.c index 9b1ff2b6bb37..ff6979181ac5 100644 --- a/drivers/tty/serial/bfin_5xx.c +++ b/drivers/tty/serial/bfin_5xx.c @@ -1304,8 +1304,7 @@ static int bfin_serial_probe(struct platform_device *pdev) goto out_error_free_peripherals; } - uart->port.membase = ioremap(res->start, - res->end - res->start); + uart->port.membase = ioremap(res->start, resource_size(res)); if (!uart->port.membase) { dev_err(&pdev->dev, "Cannot map uart IO\n"); ret = -ENXIO; @@ -1483,7 +1482,7 @@ static int bfin_earlyprintk_probe(struct platform_device *pdev) } bfin_earlyprintk_port.port.membase = ioremap(res->start, - res->end - res->start); + resource_size(res)); if (!bfin_earlyprintk_port.port.membase) { dev_err(&pdev->dev, "Cannot map uart IO\n"); ret = -ENXIO; diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index a54473123e0a..22fe801cce31 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -954,7 +954,7 @@ static void imx_release_port(struct uart_port *port) struct resource *mmres; mmres = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(mmres->start, mmres->end - mmres->start + 1); + release_mem_region(mmres->start, resource_size(mmres)); } /* @@ -970,8 +970,7 @@ static int imx_request_port(struct uart_port *port) if (!mmres) return -ENODEV; - ret = request_mem_region(mmres->start, mmres->end - mmres->start + 1, - "imx-uart"); + ret = request_mem_region(mmres->start, resource_size(mmres), "imx-uart"); return ret ? 0 : -EBUSY; } diff --git a/drivers/tty/serial/m32r_sio.c b/drivers/tty/serial/m32r_sio.c index 84db7321cce8..8e07517f8acd 100644 --- a/drivers/tty/serial/m32r_sio.c +++ b/drivers/tty/serial/m32r_sio.c @@ -892,7 +892,7 @@ static int m32r_sio_request_port(struct uart_port *port) * If we have a mapbase, then request that as well. */ if (ret == 0 && up->port.flags & UPF_IOREMAP) { - int size = res->end - res->start + 1; + int size = resource_size(res); up->port.membase = ioremap(up->port.mapbase, size); if (!up->port.membase) diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c index 47cadf474149..c37df8d0fa28 100644 --- a/drivers/tty/serial/omap-serial.c +++ b/drivers/tty/serial/omap-serial.c @@ -1241,8 +1241,8 @@ static int serial_omap_probe(struct platform_device *pdev) return -ENODEV; } - if (!request_mem_region(mem->start, (mem->end - mem->start) + 1, - pdev->dev.driver->name)) { + if (!request_mem_region(mem->start, resource_size(mem), + pdev->dev.driver->name)) { dev_err(&pdev->dev, "memory region already claimed\n"); return -EBUSY; } @@ -1308,7 +1308,7 @@ err: dev_err(&pdev->dev, "[UART%d]: failure [%s]: %d\n", pdev->id, __func__, ret); do_release_region: - release_mem_region(mem->start, (mem->end - mem->start) + 1); + release_mem_region(mem->start, resource_size(mem)); return ret; } diff --git a/drivers/tty/serial/pxa.c b/drivers/tty/serial/pxa.c index 4302e6e3768e..531931c1b250 100644 --- a/drivers/tty/serial/pxa.c +++ b/drivers/tty/serial/pxa.c @@ -803,7 +803,7 @@ static int serial_pxa_probe(struct platform_device *dev) break; } - sport->port.membase = ioremap(mmres->start, mmres->end - mmres->start + 1); + sport->port.membase = ioremap(mmres->start, resource_size(mmres)); if (!sport->port.membase) { ret = -ENOMEM; goto err_clk; diff --git a/drivers/tty/serial/sunsu.c b/drivers/tty/serial/sunsu.c index 92aa54550e84..ad0f8f5f6ea1 100644 --- a/drivers/tty/serial/sunsu.c +++ b/drivers/tty/serial/sunsu.c @@ -1435,7 +1435,7 @@ static int __devinit su_probe(struct platform_device *op) rp = &op->resource[0]; up->port.mapbase = rp->start; - up->reg_size = (rp->end - rp->start) + 1; + up->reg_size = resource_size(rp); up->port.membase = of_ioremap(rp, 0, up->reg_size, "su"); if (!up->port.membase) { if (type != SU_PORT_PORT) diff --git a/drivers/tty/serial/vt8500_serial.c b/drivers/tty/serial/vt8500_serial.c index 37fc4e3d487c..026cb9ea5cd1 100644 --- a/drivers/tty/serial/vt8500_serial.c +++ b/drivers/tty/serial/vt8500_serial.c @@ -573,8 +573,7 @@ static int __init vt8500_serial_probe(struct platform_device *pdev) snprintf(vt8500_port->name, sizeof(vt8500_port->name), "VT8500 UART%d", pdev->id); - vt8500_port->uart.membase = ioremap(mmres->start, - mmres->end - mmres->start + 1); + vt8500_port->uart.membase = ioremap(mmres->start, resource_size(mmres)); if (!vt8500_port->uart.membase) { ret = -ENOMEM; goto err; diff --git a/drivers/uio/uio_pdrv.c b/drivers/uio/uio_pdrv.c index 7d3e469b9904..bdc3db946122 100644 --- a/drivers/uio/uio_pdrv.c +++ b/drivers/uio/uio_pdrv.c @@ -58,7 +58,7 @@ static int uio_pdrv_probe(struct platform_device *pdev) uiomem->memtype = UIO_MEM_PHYS; uiomem->addr = r->start; - uiomem->size = r->end - r->start + 1; + uiomem->size = resource_size(r); ++uiomem; } diff --git a/drivers/uio/uio_pdrv_genirq.c b/drivers/uio/uio_pdrv_genirq.c index 0f424af7f109..31e799d9efe5 100644 --- a/drivers/uio/uio_pdrv_genirq.c +++ b/drivers/uio/uio_pdrv_genirq.c @@ -137,7 +137,7 @@ static int uio_pdrv_genirq_probe(struct platform_device *pdev) uiomem->memtype = UIO_MEM_PHYS; uiomem->addr = r->start; - uiomem->size = r->end - r->start + 1; + uiomem->size = resource_size(r); ++uiomem; } diff --git a/drivers/usb/gadget/atmel_usba_udc.c b/drivers/usb/gadget/atmel_usba_udc.c index db1a659702ba..f045c8968a6e 100644 --- a/drivers/usb/gadget/atmel_usba_udc.c +++ b/drivers/usb/gadget/atmel_usba_udc.c @@ -272,7 +272,7 @@ static void usba_init_debugfs(struct usba_udc *udc) regs_resource = platform_get_resource(udc->pdev, IORESOURCE_MEM, CTRL_IOMEM_ID); - regs->d_inode->i_size = regs_resource->end - regs_resource->start + 1; + regs->d_inode->i_size = resource_size(regs_resource); udc->debugfs_regs = regs; usba_ep_init_debugfs(udc, to_usba_ep(udc->gadget.ep0)); diff --git a/drivers/usb/gadget/fsl_udc_core.c b/drivers/usb/gadget/fsl_udc_core.c index 2cd9a60c7f3a..9c8e56fd0ffe 100644 --- a/drivers/usb/gadget/fsl_udc_core.c +++ b/drivers/usb/gadget/fsl_udc_core.c @@ -2445,7 +2445,7 @@ static int __init fsl_udc_probe(struct platform_device *pdev) } if (pdata->operating_mode == FSL_USB2_DR_DEVICE) { - if (!request_mem_region(res->start, res->end - res->start + 1, + if (!request_mem_region(res->start, resource_size(res), driver_name)) { ERR("request mem region for %s failed\n", pdev->name); ret = -EBUSY; @@ -2593,7 +2593,7 @@ err_iounmap_noclk: iounmap(dr_regs); err_release_mem_region: if (pdata->operating_mode == FSL_USB2_DR_DEVICE) - release_mem_region(res->start, res->end - res->start + 1); + release_mem_region(res->start, resource_size(res)); err_kfree: kfree(udc_controller); udc_controller = NULL; @@ -2628,7 +2628,7 @@ static int __exit fsl_udc_remove(struct platform_device *pdev) free_irq(udc_controller->irq, udc_controller); iounmap(dr_regs); if (pdata->operating_mode == FSL_USB2_DR_DEVICE) - release_mem_region(res->start, res->end - res->start + 1); + release_mem_region(res->start, resource_size(res)); device_unregister(&udc_controller->gadget.dev); /* free udc --wait for the release() finished */ diff --git a/drivers/usb/host/ehci-ath79.c b/drivers/usb/host/ehci-ath79.c index 98cc8a13169c..eab3d7059fbe 100644 --- a/drivers/usb/host/ehci-ath79.c +++ b/drivers/usb/host/ehci-ath79.c @@ -146,7 +146,7 @@ static int ehci_ath79_probe(struct platform_device *pdev) return -ENOMEM; hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { dev_dbg(&pdev->dev, "controller already in use\n"); diff --git a/drivers/usb/host/ehci-cns3xxx.c b/drivers/usb/host/ehci-cns3xxx.c index d41745c6f0c4..6536abdea6e6 100644 --- a/drivers/usb/host/ehci-cns3xxx.c +++ b/drivers/usb/host/ehci-cns3xxx.c @@ -107,7 +107,7 @@ static int cns3xxx_ehci_probe(struct platform_device *pdev) } hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, driver->description)) { diff --git a/drivers/usb/host/ehci-fsl.c b/drivers/usb/host/ehci-fsl.c index f380bf97e5af..34a3140d1e5f 100644 --- a/drivers/usb/host/ehci-fsl.c +++ b/drivers/usb/host/ehci-fsl.c @@ -100,7 +100,7 @@ static int usb_hcd_fsl_probe(const struct hc_driver *driver, goto err2; } hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, driver->description)) { dev_dbg(&pdev->dev, "controller already in use\n"); diff --git a/drivers/usb/host/ehci-grlib.c b/drivers/usb/host/ehci-grlib.c index 93b230dc51a2..fdfd8c5b639b 100644 --- a/drivers/usb/host/ehci-grlib.c +++ b/drivers/usb/host/ehci-grlib.c @@ -130,7 +130,7 @@ static int __devinit ehci_hcd_grlib_probe(struct platform_device *op) return -ENOMEM; hcd->rsrc_start = res.start; - hcd->rsrc_len = res.end - res.start + 1; + hcd->rsrc_len = resource_size(&res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { printk(KERN_ERR "%s: request_mem_region failed\n", __FILE__); diff --git a/drivers/usb/host/ehci-ixp4xx.c b/drivers/usb/host/ehci-ixp4xx.c index 50e600d26e28..c4460f3d009f 100644 --- a/drivers/usb/host/ehci-ixp4xx.c +++ b/drivers/usb/host/ehci-ixp4xx.c @@ -100,7 +100,7 @@ static int ixp4xx_ehci_probe(struct platform_device *pdev) goto fail_request_resource; } hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, driver->description)) { diff --git a/drivers/usb/host/ehci-octeon.c b/drivers/usb/host/ehci-octeon.c index ff55757ba7d8..c3ba3ed5f3a6 100644 --- a/drivers/usb/host/ehci-octeon.c +++ b/drivers/usb/host/ehci-octeon.c @@ -124,7 +124,7 @@ static int ehci_octeon_drv_probe(struct platform_device *pdev) return -ENOMEM; hcd->rsrc_start = res_mem->start; - hcd->rsrc_len = res_mem->end - res_mem->start + 1; + hcd->rsrc_len = resource_size(res_mem); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, OCTEON_EHCI_HCD_NAME)) { diff --git a/drivers/usb/host/ehci-pmcmsp.c b/drivers/usb/host/ehci-pmcmsp.c index cd69099cda19..e8d54de44acc 100644 --- a/drivers/usb/host/ehci-pmcmsp.c +++ b/drivers/usb/host/ehci-pmcmsp.c @@ -124,7 +124,7 @@ static int usb_hcd_msp_map_regs(struct mspusb_device *dev) res = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (res == NULL) return -ENOMEM; - res_len = res->end - res->start + 1; + res_len = resource_size(res); if (!request_mem_region(res->start, res_len, "mab regs")) return -EBUSY; @@ -140,7 +140,7 @@ static int usb_hcd_msp_map_regs(struct mspusb_device *dev) retval = -ENOMEM; goto err2; } - res_len = res->end - res->start + 1; + res_len = resource_size(res); if (!request_mem_region(res->start, res_len, "usbid regs")) { retval = -EBUSY; goto err2; @@ -154,13 +154,13 @@ static int usb_hcd_msp_map_regs(struct mspusb_device *dev) return 0; err3: res = platform_get_resource(pdev, IORESOURCE_MEM, 2); - res_len = res->end - res->start + 1; + res_len = resource_size(res); release_mem_region(res->start, res_len); err2: iounmap(dev->mab_regs); err1: res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - res_len = res->end - res->start + 1; + res_len = resource_size(res); release_mem_region(res->start, res_len); dev_err(&pdev->dev, "Failed to map non-EHCI regs.\n"); return retval; @@ -194,7 +194,7 @@ int usb_hcd_msp_probe(const struct hc_driver *driver, goto err1; } hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, dev->name)) { retval = -EBUSY; goto err1; diff --git a/drivers/usb/host/ehci-ppc-of.c b/drivers/usb/host/ehci-ppc-of.c index 8552db6c29c9..41d11fe14252 100644 --- a/drivers/usb/host/ehci-ppc-of.c +++ b/drivers/usb/host/ehci-ppc-of.c @@ -130,7 +130,7 @@ static int __devinit ehci_hcd_ppc_of_probe(struct platform_device *op) return -ENOMEM; hcd->rsrc_start = res.start; - hcd->rsrc_len = res.end - res.start + 1; + hcd->rsrc_len = resource_size(&res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { printk(KERN_ERR "%s: request_mem_region failed\n", __FILE__); diff --git a/drivers/usb/host/ehci-w90x900.c b/drivers/usb/host/ehci-w90x900.c index 52a027aaa370..d661cf7de140 100644 --- a/drivers/usb/host/ehci-w90x900.c +++ b/drivers/usb/host/ehci-w90x900.c @@ -41,7 +41,7 @@ static int __devinit usb_w90x900_probe(const struct hc_driver *driver, } hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { retval = -EBUSY; diff --git a/drivers/usb/host/ehci-xilinx-of.c b/drivers/usb/host/ehci-xilinx-of.c index a64d6d66d760..32793ce3d9e9 100644 --- a/drivers/usb/host/ehci-xilinx-of.c +++ b/drivers/usb/host/ehci-xilinx-of.c @@ -174,7 +174,7 @@ static int __devinit ehci_hcd_xilinx_of_probe(struct platform_device *op) return -ENOMEM; hcd->rsrc_start = res.start; - hcd->rsrc_len = res.end - res.start + 1; + hcd->rsrc_len = resource_size(&res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { printk(KERN_ERR "%s: request_mem_region failed\n", __FILE__); diff --git a/drivers/usb/host/fhci-hcd.c b/drivers/usb/host/fhci-hcd.c index 19223c7449e1..572ea53b0226 100644 --- a/drivers/usb/host/fhci-hcd.c +++ b/drivers/usb/host/fhci-hcd.c @@ -605,7 +605,7 @@ static int __devinit of_fhci_probe(struct platform_device *ofdev) goto err_regs; } - hcd->regs = ioremap(usb_regs.start, usb_regs.end - usb_regs.start + 1); + hcd->regs = ioremap(usb_regs.start, resource_size(&usb_regs)); if (!hcd->regs) { dev_err(dev, "could not ioremap regs\n"); ret = -ENOMEM; diff --git a/drivers/usb/host/ohci-ath79.c b/drivers/usb/host/ohci-ath79.c index ffea3e7cb0a8..c620c50f6770 100644 --- a/drivers/usb/host/ohci-ath79.c +++ b/drivers/usb/host/ohci-ath79.c @@ -93,8 +93,8 @@ static int ohci_ath79_probe(struct platform_device *pdev) ret = -ENODEV; goto err_put_hcd; } - hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_start = res->start; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { dev_dbg(&pdev->dev, "controller already in use\n"); diff --git a/drivers/usb/host/ohci-cns3xxx.c b/drivers/usb/host/ohci-cns3xxx.c index f05ef87e934c..5a00a1e1c6ca 100644 --- a/drivers/usb/host/ohci-cns3xxx.c +++ b/drivers/usb/host/ohci-cns3xxx.c @@ -100,7 +100,7 @@ static int cns3xxx_ohci_probe(struct platform_device *pdev) goto err1; } hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, driver->description)) { diff --git a/drivers/usb/host/ohci-da8xx.c b/drivers/usb/host/ohci-da8xx.c index d22fb4d577b7..6aca2c4453f7 100644 --- a/drivers/usb/host/ohci-da8xx.c +++ b/drivers/usb/host/ohci-da8xx.c @@ -322,7 +322,7 @@ static int usb_hcd_da8xx_probe(const struct hc_driver *driver, goto err2; } hcd->rsrc_start = mem->start; - hcd->rsrc_len = mem->end - mem->start + 1; + hcd->rsrc_len = resource_size(mem); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { dev_dbg(&pdev->dev, "request_mem_region failed\n"); diff --git a/drivers/usb/host/ohci-octeon.c b/drivers/usb/host/ohci-octeon.c index e4ddfaf8870f..d8b45647d1dc 100644 --- a/drivers/usb/host/ohci-octeon.c +++ b/drivers/usb/host/ohci-octeon.c @@ -135,7 +135,7 @@ static int ohci_octeon_drv_probe(struct platform_device *pdev) return -ENOMEM; hcd->rsrc_start = res_mem->start; - hcd->rsrc_len = res_mem->end - res_mem->start + 1; + hcd->rsrc_len = resource_size(res_mem); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, OCTEON_OHCI_HCD_NAME)) { diff --git a/drivers/usb/host/ohci-ppc-of.c b/drivers/usb/host/ohci-ppc-of.c index 1ca1821320f4..0c12f4e14dcd 100644 --- a/drivers/usb/host/ohci-ppc-of.c +++ b/drivers/usb/host/ohci-ppc-of.c @@ -110,7 +110,7 @@ static int __devinit ohci_hcd_ppc_of_probe(struct platform_device *op) return -ENOMEM; hcd->rsrc_start = res.start; - hcd->rsrc_len = res.end - res.start + 1; + hcd->rsrc_len = resource_size(&res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { printk(KERN_ERR "%s: request_mem_region failed\n", __FILE__); diff --git a/drivers/usb/host/ohci-ppc-soc.c b/drivers/usb/host/ohci-ppc-soc.c index 89e670e38c10..c0f595c44487 100644 --- a/drivers/usb/host/ohci-ppc-soc.c +++ b/drivers/usb/host/ohci-ppc-soc.c @@ -56,7 +56,7 @@ static int usb_hcd_ppc_soc_probe(const struct hc_driver *driver, if (!hcd) return -ENOMEM; hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { pr_debug("%s: request_mem_region failed\n", __FILE__); diff --git a/drivers/usb/host/ohci-sa1111.c b/drivers/usb/host/ohci-sa1111.c index d8eb3bdafabb..4204d9720d23 100644 --- a/drivers/usb/host/ohci-sa1111.c +++ b/drivers/usb/host/ohci-sa1111.c @@ -131,7 +131,7 @@ int usb_hcd_sa1111_probe (const struct hc_driver *driver, if (!hcd) return -ENOMEM; hcd->rsrc_start = dev->res.start; - hcd->rsrc_len = dev->res.end - dev->res.start + 1; + hcd->rsrc_len = resource_size(&dev->res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { dbg("request_mem_region failed"); diff --git a/drivers/usb/host/ohci-sm501.c b/drivers/usb/host/ohci-sm501.c index 041d30f30c10..78918ca0da23 100644 --- a/drivers/usb/host/ohci-sm501.c +++ b/drivers/usb/host/ohci-sm501.c @@ -103,8 +103,7 @@ static int ohci_hcd_sm501_drv_probe(struct platform_device *pdev) goto err0; } - if (!request_mem_region(mem->start, mem->end - mem->start + 1, - pdev->name)) { + if (!request_mem_region(mem->start, resource_size(mem), pdev->name)) { dev_err(dev, "request_mem_region failed\n"); retval = -EBUSY; goto err0; @@ -126,7 +125,7 @@ static int ohci_hcd_sm501_drv_probe(struct platform_device *pdev) if (!dma_declare_coherent_memory(dev, mem->start, mem->start - mem->parent->start, - (mem->end - mem->start) + 1, + resource_size(mem), DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE)) { dev_err(dev, "cannot declare coherent memory\n"); @@ -149,7 +148,7 @@ static int ohci_hcd_sm501_drv_probe(struct platform_device *pdev) } hcd->rsrc_start = res->start; - hcd->rsrc_len = res->end - res->start + 1; + hcd->rsrc_len = resource_size(res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, pdev->name)) { dev_err(dev, "request_mem_region failed\n"); @@ -185,7 +184,7 @@ err3: err2: dma_release_declared_memory(dev); err1: - release_mem_region(mem->start, mem->end - mem->start + 1); + release_mem_region(mem->start, resource_size(mem)); err0: return retval; } @@ -201,7 +200,7 @@ static int ohci_hcd_sm501_drv_remove(struct platform_device *pdev) dma_release_declared_memory(&pdev->dev); mem = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (mem) - release_mem_region(mem->start, mem->end - mem->start + 1); + release_mem_region(mem->start, resource_size(mem)); /* mask interrupts and disable power */ diff --git a/drivers/usb/host/ohci-tmio.c b/drivers/usb/host/ohci-tmio.c index 3558491dd87d..57ad1271fc9b 100644 --- a/drivers/usb/host/ohci-tmio.c +++ b/drivers/usb/host/ohci-tmio.c @@ -208,13 +208,13 @@ static int __devinit ohci_hcd_tmio_drv_probe(struct platform_device *dev) } hcd->rsrc_start = regs->start; - hcd->rsrc_len = regs->end - regs->start + 1; + hcd->rsrc_len = resource_size(regs); tmio = hcd_to_tmio(hcd); spin_lock_init(&tmio->lock); - tmio->ccr = ioremap(config->start, config->end - config->start + 1); + tmio->ccr = ioremap(config->start, resource_size(config)); if (!tmio->ccr) { ret = -ENOMEM; goto err_ioremap_ccr; @@ -228,7 +228,7 @@ static int __devinit ohci_hcd_tmio_drv_probe(struct platform_device *dev) if (!dma_declare_coherent_memory(&dev->dev, sram->start, sram->start, - sram->end - sram->start + 1, + resource_size(sram), DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE)) { ret = -EBUSY; goto err_dma_declare; diff --git a/drivers/usb/host/oxu210hp-hcd.c b/drivers/usb/host/oxu210hp-hcd.c index 5fbe997dc6df..dcd889803f0f 100644 --- a/drivers/usb/host/oxu210hp-hcd.c +++ b/drivers/usb/host/oxu210hp-hcd.c @@ -3828,7 +3828,7 @@ static int oxu_drv_probe(struct platform_device *pdev) return -ENODEV; } memstart = res->start; - memlen = res->end - res->start + 1; + memlen = resource_size(res); dev_dbg(&pdev->dev, "MEM resource %lx-%lx\n", memstart, memlen); if (!request_mem_region(memstart, memlen, oxu_hc_driver.description)) { diff --git a/drivers/usb/host/uhci-grlib.c b/drivers/usb/host/uhci-grlib.c index d01c1e227681..f7a62138e3e0 100644 --- a/drivers/usb/host/uhci-grlib.c +++ b/drivers/usb/host/uhci-grlib.c @@ -111,7 +111,7 @@ static int __devinit uhci_hcd_grlib_probe(struct platform_device *op) return -ENOMEM; hcd->rsrc_start = res.start; - hcd->rsrc_len = res.end - res.start + 1; + hcd->rsrc_len = resource_size(&res); if (!request_mem_region(hcd->rsrc_start, hcd->rsrc_len, hcd_name)) { printk(KERN_ERR "%s: request_mem_region failed\n", __FILE__); diff --git a/drivers/usb/host/whci/init.c b/drivers/usb/host/whci/init.c index f7582e8e2169..d3e13b640d4b 100644 --- a/drivers/usb/host/whci/init.c +++ b/drivers/usb/host/whci/init.c @@ -178,7 +178,7 @@ void whc_clean_up(struct whc *whc) if (whc->qset_pool) dma_pool_destroy(whc->qset_pool); - len = whc->umc->resource.end - whc->umc->resource.start + 1; + len = resource_size(&whc->umc->resource); if (whc->base) iounmap(whc->base); if (whc->base_phys) diff --git a/drivers/uwb/whc-rc.c b/drivers/uwb/whc-rc.c index 70a004aa19db..3ae3c702500d 100644 --- a/drivers/uwb/whc-rc.c +++ b/drivers/uwb/whc-rc.c @@ -222,7 +222,7 @@ int whcrc_setup_rc_umc(struct whcrc *whcrc) struct umc_dev *umc_dev = whcrc->umc_dev; whcrc->area = umc_dev->resource.start; - whcrc->rc_len = umc_dev->resource.end - umc_dev->resource.start + 1; + whcrc->rc_len = resource_size(&umc_dev->resource); result = -EBUSY; if (request_mem_region(whcrc->area, whcrc->rc_len, KBUILD_MODNAME) == NULL) { dev_err(dev, "can't request URC region (%zu bytes @ 0x%lx): %d\n", diff --git a/drivers/video/atmel_lcdfb.c b/drivers/video/atmel_lcdfb.c index 4484c721f0f9..817ab60f7537 100644 --- a/drivers/video/atmel_lcdfb.c +++ b/drivers/video/atmel_lcdfb.c @@ -906,7 +906,7 @@ static int __init atmel_lcdfb_probe(struct platform_device *pdev) if (map) { /* use a pre-allocated memory buffer */ info->fix.smem_start = map->start; - info->fix.smem_len = map->end - map->start + 1; + info->fix.smem_len = resource_size(map); if (!request_mem_region(info->fix.smem_start, info->fix.smem_len, pdev->name)) { ret = -EBUSY; @@ -932,7 +932,7 @@ static int __init atmel_lcdfb_probe(struct platform_device *pdev) /* LCDC registers */ info->fix.mmio_start = regs->start; - info->fix.mmio_len = regs->end - regs->start + 1; + info->fix.mmio_len = resource_size(regs); if (!request_mem_region(info->fix.mmio_start, info->fix.mmio_len, pdev->name)) { diff --git a/drivers/video/aty/atyfb_base.c b/drivers/video/aty/atyfb_base.c index ebb893c49e90..ad41f508b423 100644 --- a/drivers/video/aty/atyfb_base.c +++ b/drivers/video/aty/atyfb_base.c @@ -3460,9 +3460,10 @@ static int __devinit atyfb_setup_generic(struct pci_dev *pdev, raddr = addr + 0x7ff000UL; rrp = &pdev->resource[2]; - if ((rrp->flags & IORESOURCE_MEM) && request_mem_region(rrp->start, rrp->end - rrp->start + 1, "atyfb")) { + if ((rrp->flags & IORESOURCE_MEM) && + request_mem_region(rrp->start, resource_size(rrp), "atyfb")) { par->aux_start = rrp->start; - par->aux_size = rrp->end - rrp->start + 1; + par->aux_size = resource_size(rrp); raddr = rrp->start; PRINTKI("using auxiliary register aperture\n"); } @@ -3552,7 +3553,7 @@ static int __devinit atyfb_pci_probe(struct pci_dev *pdev, /* Reserve space */ res_start = rp->start; - res_size = rp->end - rp->start + 1; + res_size = resource_size(rp); if (!request_mem_region(res_start, res_size, "atyfb")) return -EBUSY; diff --git a/drivers/video/au1100fb.c b/drivers/video/au1100fb.c index 34b2fc472fe8..01a8fde67f20 100644 --- a/drivers/video/au1100fb.c +++ b/drivers/video/au1100fb.c @@ -486,7 +486,7 @@ static int __devinit au1100fb_drv_probe(struct platform_device *dev) } au1100fb_fix.mmio_start = regs_res->start; - au1100fb_fix.mmio_len = regs_res->end - regs_res->start + 1; + au1100fb_fix.mmio_len = resource_size(regs_res); if (!request_mem_region(au1100fb_fix.mmio_start, au1100fb_fix.mmio_len, DRIVER_NAME)) { diff --git a/drivers/video/cobalt_lcdfb.c b/drivers/video/cobalt_lcdfb.c index 42fe155aba0e..e02764319ff7 100644 --- a/drivers/video/cobalt_lcdfb.c +++ b/drivers/video/cobalt_lcdfb.c @@ -303,7 +303,7 @@ static int __devinit cobalt_lcdfb_probe(struct platform_device *dev) return -EBUSY; } - info->screen_size = res->end - res->start + 1; + info->screen_size = resource_size(res); info->screen_base = ioremap(res->start, info->screen_size); info->fbops = &cobalt_lcd_fbops; info->fix = cobalt_lcdfb_fix; diff --git a/drivers/video/controlfb.c b/drivers/video/controlfb.c index c225dcce89e7..9075bea55879 100644 --- a/drivers/video/controlfb.c +++ b/drivers/video/controlfb.c @@ -709,11 +709,11 @@ static int __init control_of_init(struct device_node *dp) /* Map in frame buffer and registers */ p->fb_orig_base = fb_res.start; - p->fb_orig_size = fb_res.end - fb_res.start + 1; + p->fb_orig_size = resource_size(&fb_res); /* use the big-endian aperture (??) */ p->frame_buffer_phys = fb_res.start + 0x800000; p->control_regs_phys = reg_res.start; - p->control_regs_size = reg_res.end - reg_res.start + 1; + p->control_regs_size = resource_size(®_res); if (!p->fb_orig_base || !request_mem_region(p->fb_orig_base,p->fb_orig_size,"controlfb")) { diff --git a/drivers/video/mb862xx/mb862xxfbdrv.c b/drivers/video/mb862xx/mb862xxfbdrv.c index f70bd63b0187..ee1de3e26dec 100644 --- a/drivers/video/mb862xx/mb862xxfbdrv.c +++ b/drivers/video/mb862xx/mb862xxfbdrv.c @@ -697,7 +697,7 @@ static int __devinit of_platform_mb862xx_probe(struct platform_device *ofdev) goto fbrel; } - res_size = 1 + res.end - res.start; + res_size = resource_size(&res); par->res = request_mem_region(res.start, res_size, DRV_NAME); if (par->res == NULL) { dev_err(dev, "Cannot claim framebuffer/mmio\n"); @@ -787,7 +787,7 @@ static int __devexit of_platform_mb862xx_remove(struct platform_device *ofdev) { struct fb_info *fbi = dev_get_drvdata(&ofdev->dev); struct mb862xxfb_par *par = fbi->par; - resource_size_t res_size = 1 + par->res->end - par->res->start; + resource_size_t res_size = resource_size(par->res); unsigned long reg; dev_dbg(fbi->dev, "%s release\n", fbi->fix.id); diff --git a/drivers/video/msm/mdp.c b/drivers/video/msm/mdp.c index c3636d55a3c5..243d16f09b8a 100644 --- a/drivers/video/msm/mdp.c +++ b/drivers/video/msm/mdp.c @@ -406,8 +406,7 @@ int mdp_probe(struct platform_device *pdev) goto error_get_irq; } - mdp->base = ioremap(resource->start, - resource->end - resource->start); + mdp->base = ioremap(resource->start, resource_size(resource)); if (mdp->base == 0) { printk(KERN_ERR "msmfb: cannot allocate mdp regs!\n"); ret = -ENOMEM; diff --git a/drivers/video/msm/msm_fb.c b/drivers/video/msm/msm_fb.c index ec351309e607..c6e3b4fcdd68 100644 --- a/drivers/video/msm/msm_fb.c +++ b/drivers/video/msm/msm_fb.c @@ -525,10 +525,9 @@ static int setup_fbmem(struct msmfb_info *msmfb, struct platform_device *pdev) return -ENOMEM; } fb->fix.smem_start = resource->start; - fb->fix.smem_len = resource->end - resource->start; - fbram = ioremap(resource->start, - resource->end - resource->start); - if (fbram == 0) { + fb->fix.smem_len = resource_size(resource); + fbram = ioremap(resource->start, resource_size(resource)); + if (fbram == NULL) { printk(KERN_ERR "msmfb: cannot allocate fbram!\n"); return -ENOMEM; } diff --git a/drivers/video/nuc900fb.c b/drivers/video/nuc900fb.c index f838d9e277f0..0fff59782e45 100644 --- a/drivers/video/nuc900fb.c +++ b/drivers/video/nuc900fb.c @@ -551,7 +551,7 @@ static int __devinit nuc900fb_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - size = (res->end - res->start) + 1; + size = resource_size(res); fbi->mem = request_mem_region(res->start, size, pdev->name); if (fbi->mem == NULL) { dev_err(&pdev->dev, "failed to alloc memory region\n"); diff --git a/drivers/video/platinumfb.c b/drivers/video/platinumfb.c index ef532d9d3c99..f27ae16ead2e 100644 --- a/drivers/video/platinumfb.c +++ b/drivers/video/platinumfb.c @@ -567,7 +567,7 @@ static int __devinit platinumfb_probe(struct platform_device* odev) * northbridge and that can fail. Only request framebuffer */ if (!request_mem_region(pinfo->rsrc_fb.start, - pinfo->rsrc_fb.end - pinfo->rsrc_fb.start + 1, + resource_size(&pinfo->rsrc_fb), "platinumfb framebuffer")) { printk(KERN_ERR "platinumfb: Can't request framebuffer !\n"); framebuffer_release(info); @@ -658,8 +658,7 @@ static int __devexit platinumfb_remove(struct platform_device* odev) iounmap(pinfo->cmap_regs); release_mem_region(pinfo->rsrc_fb.start, - pinfo->rsrc_fb.end - - pinfo->rsrc_fb.start + 1); + resource_size(&pinfo->rsrc_fb)); release_mem_region(pinfo->cmap_regs_phys, 0x1000); diff --git a/drivers/video/pxa168fb.c b/drivers/video/pxa168fb.c index bb95ec56d25d..18ead6f0184d 100644 --- a/drivers/video/pxa168fb.c +++ b/drivers/video/pxa168fb.c @@ -662,7 +662,7 @@ static int __devinit pxa168fb_probe(struct platform_device *pdev) info->fix.ypanstep = 0; info->fix.ywrapstep = 0; info->fix.mmio_start = res->start; - info->fix.mmio_len = res->end - res->start + 1; + info->fix.mmio_len = resource_size(res); info->fix.accel = FB_ACCEL_NONE; info->fbops = &pxa168fb_ops; info->pseudo_palette = fbi->pseudo_palette; diff --git a/include/linux/dio.h b/include/linux/dio.h index b2dd31ca1710..2cc0fd00463f 100644 --- a/include/linux/dio.h +++ b/include/linux/dio.h @@ -254,7 +254,7 @@ static inline struct dio_driver *dio_dev_driver(const struct dio_dev *d) #define dio_resource_start(d) ((d)->resource.start) #define dio_resource_end(d) ((d)->resource.end) -#define dio_resource_len(d) ((d)->resource.end-(d)->resource.start+1) +#define dio_resource_len(d) (resource_size(&(d)->resource)) #define dio_resource_flags(d) ((d)->resource.flags) #define dio_request_device(d, name) \ diff --git a/include/linux/pnp.h b/include/linux/pnp.h index 1bc1338b817b..195aafc6cd07 100644 --- a/include/linux/pnp.h +++ b/include/linux/pnp.h @@ -50,7 +50,7 @@ static inline resource_size_t pnp_resource_len(struct resource *res) { if (res->start == 0 && res->end == 0) return 0; - return res->end - res->start + 1; + return resource_size(res); } diff --git a/include/linux/zorro.h b/include/linux/zorro.h index 7bf9db525e9e..dff42025649b 100644 --- a/include/linux/zorro.h +++ b/include/linux/zorro.h @@ -187,7 +187,7 @@ extern struct zorro_dev *zorro_find_device(zorro_id id, #define zorro_resource_start(z) ((z)->resource.start) #define zorro_resource_end(z) ((z)->resource.end) -#define zorro_resource_len(z) ((z)->resource.end-(z)->resource.start+1) +#define zorro_resource_len(z) (resource_size(&(z)->resource)) #define zorro_resource_flags(z) ((z)->resource.flags) #define zorro_request_device(z, name) \ diff --git a/kernel/kexec.c b/kernel/kexec.c index 8d814cbc8109..296fbc84d659 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -1095,7 +1095,7 @@ size_t crash_get_memory_size(void) size_t size = 0; mutex_lock(&kexec_mutex); if (crashk_res.end != crashk_res.start) - size = crashk_res.end - crashk_res.start + 1; + size = resource_size(&crashk_res); mutex_unlock(&kexec_mutex); return size; } diff --git a/sound/aoa/soundbus/i2sbus/core.c b/sound/aoa/soundbus/i2sbus/core.c index 3ff8cc5f487a..010658335881 100644 --- a/sound/aoa/soundbus/i2sbus/core.c +++ b/sound/aoa/soundbus/i2sbus/core.c @@ -262,8 +262,7 @@ static int i2sbus_add_dev(struct macio_dev *macio, */ dev->allocated_resource[i] = request_mem_region(dev->resources[i].start, - dev->resources[i].end - - dev->resources[i].start + 1, + resource_size(&dev->resources[i]), dev->rnames[i]); if (!dev->allocated_resource[i]) { printk(KERN_ERR "i2sbus: failed to claim resource %d!\n", i); @@ -272,19 +271,19 @@ static int i2sbus_add_dev(struct macio_dev *macio, } r = &dev->resources[aoa_resource_i2smmio]; - rlen = r->end - r->start + 1; + rlen = resource_size(r); if (rlen < sizeof(struct i2s_interface_regs)) goto err; dev->intfregs = ioremap(r->start, rlen); r = &dev->resources[aoa_resource_txdbdma]; - rlen = r->end - r->start + 1; + rlen = resource_size(r); if (rlen < sizeof(struct dbdma_regs)) goto err; dev->out.dbdma = ioremap(r->start, rlen); r = &dev->resources[aoa_resource_rxdbdma]; - rlen = r->end - r->start + 1; + rlen = resource_size(r); if (rlen < sizeof(struct dbdma_regs)) goto err; dev->in.dbdma = ioremap(r->start, rlen); diff --git a/sound/atmel/abdac.c b/sound/atmel/abdac.c index 6e2409181895..30468b31cad8 100644 --- a/sound/atmel/abdac.c +++ b/sound/atmel/abdac.c @@ -448,7 +448,7 @@ static int __devinit atmel_abdac_probe(struct platform_device *pdev) goto out_free_card; } - dac->regs = ioremap(regs->start, regs->end - regs->start + 1); + dac->regs = ioremap(regs->start, resource_size(regs)); if (!dac->regs) { dev_dbg(&pdev->dev, "could not remap register memory\n"); goto out_free_card; diff --git a/sound/atmel/ac97c.c b/sound/atmel/ac97c.c index b310702c646e..41b901bde5c7 100644 --- a/sound/atmel/ac97c.c +++ b/sound/atmel/ac97c.c @@ -971,7 +971,7 @@ static int __devinit atmel_ac97c_probe(struct platform_device *pdev) chip->card = card; chip->pclk = pclk; chip->pdev = pdev; - chip->regs = ioremap(regs->start, regs->end - regs->start + 1); + chip->regs = ioremap(regs->start, resource_size(regs)); if (!chip->regs) { dev_dbg(&pdev->dev, "could not remap register memory\n"); diff --git a/sound/ppc/pmac.c b/sound/ppc/pmac.c index 3ecbd67f88c9..ab96cde7417b 100644 --- a/sound/ppc/pmac.c +++ b/sound/ppc/pmac.c @@ -881,8 +881,7 @@ static int snd_pmac_free(struct snd_pmac *chip) for (i = 0; i < 3; i++) { if (chip->requested & (1 << i)) release_mem_region(chip->rsrc[i].start, - chip->rsrc[i].end - - chip->rsrc[i].start + 1); + resource_size(&chip->rsrc[i])); } } @@ -1228,8 +1227,7 @@ int __devinit snd_pmac_new(struct snd_card *card, struct snd_pmac **chip_return) goto __error; } if (request_mem_region(chip->rsrc[i].start, - chip->rsrc[i].end - - chip->rsrc[i].start + 1, + resource_size(&chip->rsrc[i]), rnames[i]) == NULL) { printk(KERN_ERR "snd: can't request rsrc " " %d (%s: %pR)\n", @@ -1254,8 +1252,7 @@ int __devinit snd_pmac_new(struct snd_card *card, struct snd_pmac **chip_return) goto __error; } if (request_mem_region(chip->rsrc[i].start, - chip->rsrc[i].end - - chip->rsrc[i].start + 1, + resource_size(&chip->rsrc[i]), rnames[i]) == NULL) { printk(KERN_ERR "snd: can't request rsrc " " %d (%s: %pR)\n", diff --git a/sound/soc/fsl/fsl_ssi.c b/sound/soc/fsl/fsl_ssi.c index 313e0ccedd5b..6a882aa55530 100644 --- a/sound/soc/fsl/fsl_ssi.c +++ b/sound/soc/fsl/fsl_ssi.c @@ -678,7 +678,7 @@ static int __devinit fsl_ssi_probe(struct platform_device *pdev) kfree(ssi_private); return ret; } - ssi_private->ssi = ioremap(res.start, 1 + res.end - res.start); + ssi_private->ssi = ioremap(res.start, resource_size(&res)); ssi_private->ssi_phys = res.start; ssi_private->irq = irq_of_parse_and_map(np, 0); diff --git a/sound/soc/fsl/mpc5200_dma.c b/sound/soc/fsl/mpc5200_dma.c index fff695ccdd3e..86023142a4cb 100644 --- a/sound/soc/fsl/mpc5200_dma.c +++ b/sound/soc/fsl/mpc5200_dma.c @@ -384,7 +384,7 @@ static int mpc5200_hpcd_probe(struct of_device *op) dev_err(&op->dev, "Missing reg property\n"); return -ENODEV; } - regs = ioremap(res.start, 1 + res.end - res.start); + regs = ioremap(res.start, resource_size(&res)); if (!regs) { dev_err(&op->dev, "Could not map registers\n"); return -ENODEV; -- cgit v1.2.3 From 39c555460cbc6d35656878c89d4664fbd6c81551 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 11:52:27 +0200 Subject: x86/amd-iommu: Remove redundant device_flush_dte() calls Remove these function calls from places where the function has already been called by another function. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 7c3a95e54ec5..cc6f6da630e8 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -1798,7 +1798,6 @@ static int device_change_notifier(struct notifier_block *nb, goto out; } - device_flush_dte(dev); iommu_completion_wait(iommu); out: @@ -2605,7 +2604,6 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, if (!iommu) return; - device_flush_dte(dev); iommu_completion_wait(iommu); } -- cgit v1.2.3 From 8fa5f802abf3cd374b5f07418cea72c5d9d204cc Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 12:24:45 +0200 Subject: x86/amd-iommu: Introduce global dev_data_list This list keeps all allocated iommu_dev_data structs in a list together. This is needed for instances that have no associated device. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 56 ++++++++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 9 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 4c9982995414..35520e31c1b4 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -310,6 +310,7 @@ struct protection_domain { */ struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ + struct list_head dev_data_list; /* For global dev_data_list */ struct device *dev; /* Device this data belong to */ struct device *alias; /* The Alias Device */ struct protection_domain *domain; /* Domain the device is bound to */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index cc6f6da630e8..8b8489084649 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -45,6 +45,10 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); static LIST_HEAD(iommu_pd_list); static DEFINE_SPINLOCK(iommu_pd_list_lock); +/* List of all available dev_data structures */ +static LIST_HEAD(dev_data_list); +static DEFINE_SPINLOCK(dev_data_list_lock); + /* * Domain for untranslated devices - only allocated * if iommu=pt passed on kernel cmd line. @@ -68,6 +72,35 @@ static void update_domain(struct protection_domain *domain); * ****************************************************************************/ +static struct iommu_dev_data *alloc_dev_data(void) +{ + struct iommu_dev_data *dev_data; + unsigned long flags; + + dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return NULL; + + atomic_set(&dev_data->bind, 0); + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_add_tail(&dev_data->dev_data_list, &dev_data_list); + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + return dev_data; +} + +static void free_dev_data(struct iommu_dev_data *dev_data) +{ + unsigned long flags; + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_del(&dev_data->dev_data_list); + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + kfree(dev_data); +} + static inline u16 get_device_id(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -139,32 +172,28 @@ static int iommu_init_device(struct device *dev) { struct iommu_dev_data *dev_data; struct pci_dev *pdev; - u16 devid, alias; + u16 alias; if (dev->archdata.iommu) return 0; - dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); + dev_data = alloc_dev_data(); if (!dev_data) return -ENOMEM; dev_data->dev = dev; - devid = get_device_id(dev); - alias = amd_iommu_alias_table[devid]; + alias = amd_iommu_alias_table[get_device_id(dev)]; pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); if (pdev) dev_data->alias = &pdev->dev; else { - kfree(dev_data); + free_dev_data(dev_data); return -ENOTSUPP; } - atomic_set(&dev_data->bind, 0); - dev->archdata.iommu = dev_data; - return 0; } @@ -184,11 +213,16 @@ static void iommu_ignore_device(struct device *dev) static void iommu_uninit_device(struct device *dev) { - kfree(dev->archdata.iommu); + /* + * Nothing to do here - we keep dev_data around for unplugged devices + * and reuse it when the device is re-plugged - not doing so would + * introduce a ton of races. + */ } void __init amd_iommu_uninit_devices(void) { + struct iommu_dev_data *dev_data, *n; struct pci_dev *pdev = NULL; for_each_pci_dev(pdev) { @@ -198,6 +232,10 @@ void __init amd_iommu_uninit_devices(void) iommu_uninit_device(&pdev->dev); } + + /* Free all of our dev_data structures */ + list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list) + free_dev_data(dev_data); } int __init amd_iommu_init_devices(void) -- cgit v1.2.3 From f62dda66b5601396c28b563cc18b481af04a91b3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 12:55:35 +0200 Subject: x86/amd-iommu: Store devid in dev_data This allows to use dev_data independent of struct device later. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 39 +++++++++++++--------------------- 2 files changed, 16 insertions(+), 24 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 35520e31c1b4..2833862311b5 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -315,6 +315,7 @@ struct iommu_dev_data { struct device *alias; /* The Alias Device */ struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ + u16 devid; /* PCI Device ID */ }; /* diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 8b8489084649..4e8b176297b5 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -72,7 +72,7 @@ static void update_domain(struct protection_domain *domain); * ****************************************************************************/ -static struct iommu_dev_data *alloc_dev_data(void) +static struct iommu_dev_data *alloc_dev_data(u16 devid) { struct iommu_dev_data *dev_data; unsigned long flags; @@ -81,6 +81,7 @@ static struct iommu_dev_data *alloc_dev_data(void) if (!dev_data) return NULL; + dev_data->devid = devid; atomic_set(&dev_data->bind, 0); spin_lock_irqsave(&dev_data_list_lock, flags); @@ -177,13 +178,13 @@ static int iommu_init_device(struct device *dev) if (dev->archdata.iommu) return 0; - dev_data = alloc_dev_data(); + dev_data = alloc_dev_data(get_device_id(dev)); if (!dev_data) return -ENOMEM; dev_data->dev = dev; - alias = amd_iommu_alias_table[get_device_id(dev)]; + alias = amd_iommu_alias_table[dev_data->devid]; pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); if (pdev) dev_data->alias = &pdev->dev; @@ -714,16 +715,16 @@ static int device_flush_iotlb(struct device *dev, u64 address, size_t size) */ static int device_flush_dte(struct device *dev) { + struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct pci_dev *pdev; - u16 devid; int ret; - pdev = to_pci_dev(dev); - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; + pdev = to_pci_dev(dev); + dev_data = get_dev_data(dev); + iommu = amd_iommu_rlookup_table[dev_data->devid]; - ret = iommu_flush_dte(iommu, devid); + ret = iommu_flush_dte(iommu, dev_data->devid); if (ret) return ret; @@ -1570,11 +1571,9 @@ static void do_attach(struct device *dev, struct protection_domain *domain) struct amd_iommu *iommu; struct pci_dev *pdev; bool ats = false; - u16 devid; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; dev_data = get_dev_data(dev); + iommu = amd_iommu_rlookup_table[dev_data->devid]; pdev = to_pci_dev(dev); if (amd_iommu_iotlb_sup) @@ -1583,7 +1582,7 @@ static void do_attach(struct device *dev, struct protection_domain *domain) /* Update data structures */ dev_data->domain = domain; list_add(&dev_data->list, &domain->dev_list); - set_dte_entry(devid, domain, ats); + set_dte_entry(dev_data->devid, domain, ats); /* Do reference counting */ domain->dev_iommu[iommu->index] += 1; @@ -1597,11 +1596,9 @@ static void do_detach(struct device *dev) { struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - u16 devid; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; dev_data = get_dev_data(dev); + iommu = amd_iommu_rlookup_table[dev_data->devid]; /* decrease reference counters */ dev_data->domain->dev_iommu[iommu->index] -= 1; @@ -1610,7 +1607,7 @@ static void do_detach(struct device *dev) /* Update data structures */ dev_data->domain = NULL; list_del(&dev_data->list); - clear_dte_entry(devid); + clear_dte_entry(dev_data->devid); /* Flush the DTE entry */ device_flush_dte(dev); @@ -1760,9 +1757,7 @@ static struct protection_domain *domain_for_device(struct device *dev) struct protection_domain *dom; struct iommu_dev_data *dev_data, *alias_data; unsigned long flags; - u16 devid; - devid = get_device_id(dev); dev_data = get_dev_data(dev); alias_data = get_dev_data(dev_data->alias); if (!alias_data) @@ -1897,8 +1892,7 @@ static void update_device_table(struct protection_domain *domain) list_for_each_entry(dev_data, &domain->dev_list, list) { struct pci_dev *pdev = to_pci_dev(dev_data->dev); - u16 devid = get_device_id(dev_data->dev); - set_dte_entry(devid, domain, pci_ats_enabled(pdev)); + set_dte_entry(dev_data->devid, domain, pci_ats_enabled(pdev)); } } @@ -2652,16 +2646,13 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct iommu_dev_data *dev_data; struct amd_iommu *iommu; int ret; - u16 devid; if (!check_device(dev)) return -EINVAL; dev_data = dev->archdata.iommu; - devid = get_device_id(dev); - - iommu = amd_iommu_rlookup_table[devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; if (!iommu) return -EINVAL; -- cgit v1.2.3 From ea61cddb9dcb9da9bcabd40f1620c24abaf645c9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 12:56:30 +0200 Subject: x86/amd-iommu: Store ATS state in dev_data This allows the low-level functions to operate on dev_data exclusivly later. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 4 ++++ arch/x86/kernel/amd_iommu.c | 44 ++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 20 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 2833862311b5..9fc045ee2fcb 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -316,6 +316,10 @@ struct iommu_dev_data { struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ u16 devid; /* PCI Device ID */ + struct { + bool enabled; + int qdep; + } ats; /* ATS state */ }; /* diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 4e8b176297b5..9e07ef65a016 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -695,17 +695,16 @@ void iommu_flush_all_caches(struct amd_iommu *iommu) */ static int device_flush_iotlb(struct device *dev, u64 address, size_t size) { - struct pci_dev *pdev = to_pci_dev(dev); + struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct iommu_cmd cmd; - u16 devid; int qdep; - qdep = pci_ats_queue_depth(pdev); - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; + dev_data = get_dev_data(dev); + qdep = dev_data->ats.qdep; + iommu = amd_iommu_rlookup_table[dev_data->devid]; - build_inv_iotlb_pages(&cmd, devid, qdep, address, size); + build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size); return iommu_queue_command(iommu, &cmd); } @@ -728,7 +727,7 @@ static int device_flush_dte(struct device *dev) if (ret) return ret; - if (pci_ats_enabled(pdev)) + if (dev_data->ats.enabled) ret = device_flush_iotlb(dev, 0, ~0UL); return ret; @@ -760,9 +759,8 @@ static void __domain_flush_pages(struct protection_domain *domain, } list_for_each_entry(dev_data, &domain->dev_list, list) { - struct pci_dev *pdev = to_pci_dev(dev_data->dev); - if (!pci_ats_enabled(pdev)) + if (!dev_data->ats.enabled) continue; ret |= device_flush_iotlb(dev_data->dev, address, size); @@ -1576,8 +1574,7 @@ static void do_attach(struct device *dev, struct protection_domain *domain) iommu = amd_iommu_rlookup_table[dev_data->devid]; pdev = to_pci_dev(dev); - if (amd_iommu_iotlb_sup) - ats = pci_ats_enabled(pdev); + ats = dev_data->ats.enabled; /* Update data structures */ dev_data->domain = domain; @@ -1674,11 +1671,16 @@ static int attach_device(struct device *dev, struct protection_domain *domain) { struct pci_dev *pdev = to_pci_dev(dev); + struct iommu_dev_data *dev_data; unsigned long flags; int ret; - if (amd_iommu_iotlb_sup) - pci_enable_ats(pdev, PAGE_SHIFT); + dev_data = get_dev_data(dev); + + if (amd_iommu_iotlb_sup && pci_enable_ats(pdev, PAGE_SHIFT) == 0) { + dev_data->ats.enabled = true; + dev_data->ats.qdep = pci_ats_queue_depth(pdev); + } write_lock_irqsave(&amd_iommu_devtable_lock, flags); ret = __attach_device(dev, domain); @@ -1736,7 +1738,7 @@ static void __detach_device(struct device *dev) */ static void detach_device(struct device *dev) { - struct pci_dev *pdev = to_pci_dev(dev); + struct iommu_dev_data *dev_data; unsigned long flags; /* lock device table */ @@ -1744,8 +1746,12 @@ static void detach_device(struct device *dev) __detach_device(dev); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev)) - pci_disable_ats(pdev); + dev_data = get_dev_data(dev); + + if (dev_data->ats.enabled) { + pci_disable_ats(to_pci_dev(dev)); + dev_data->ats.enabled = false; + } } /* @@ -1890,10 +1896,8 @@ static void update_device_table(struct protection_domain *domain) { struct iommu_dev_data *dev_data; - list_for_each_entry(dev_data, &domain->dev_list, list) { - struct pci_dev *pdev = to_pci_dev(dev_data->dev); - set_dte_entry(dev_data->devid, domain, pci_ats_enabled(pdev)); - } + list_for_each_entry(dev_data, &domain->dev_list, list) + set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled); } static void update_domain(struct protection_domain *domain) -- cgit v1.2.3 From 6c542047937ad66db1d31e290fd2a3b290424ffa Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 17:07:31 +0200 Subject: x86/amd-iommu: Use only dev_data for dte and iotlb flushing routines This patch make the functions flushing the DTE and IOTLBs only take the dev_data structure instead of the struct device directly. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 9e07ef65a016..0a5b46504b5d 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -693,14 +693,13 @@ void iommu_flush_all_caches(struct amd_iommu *iommu) /* * Command send function for flushing on-device TLB */ -static int device_flush_iotlb(struct device *dev, u64 address, size_t size) +static int device_flush_iotlb(struct iommu_dev_data *dev_data, + u64 address, size_t size) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct iommu_cmd cmd; int qdep; - dev_data = get_dev_data(dev); qdep = dev_data->ats.qdep; iommu = amd_iommu_rlookup_table[dev_data->devid]; @@ -712,23 +711,19 @@ static int device_flush_iotlb(struct device *dev, u64 address, size_t size) /* * Command send function for invalidating a device table entry */ -static int device_flush_dte(struct device *dev) +static int device_flush_dte(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - struct pci_dev *pdev; int ret; - pdev = to_pci_dev(dev); - dev_data = get_dev_data(dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; ret = iommu_flush_dte(iommu, dev_data->devid); if (ret) return ret; if (dev_data->ats.enabled) - ret = device_flush_iotlb(dev, 0, ~0UL); + ret = device_flush_iotlb(dev_data, 0, ~0UL); return ret; } @@ -763,7 +758,7 @@ static void __domain_flush_pages(struct protection_domain *domain, if (!dev_data->ats.enabled) continue; - ret |= device_flush_iotlb(dev_data->dev, address, size); + ret |= device_flush_iotlb(dev_data, address, size); } WARN_ON(ret); @@ -815,7 +810,7 @@ static void domain_flush_devices(struct protection_domain *domain) spin_lock_irqsave(&domain->lock, flags); list_for_each_entry(dev_data, &domain->dev_list, list) - device_flush_dte(dev_data->dev); + device_flush_dte(dev_data); spin_unlock_irqrestore(&domain->lock, flags); } @@ -1586,7 +1581,7 @@ static void do_attach(struct device *dev, struct protection_domain *domain) domain->dev_cnt += 1; /* Flush the DTE entry */ - device_flush_dte(dev); + device_flush_dte(dev_data); } static void do_detach(struct device *dev) @@ -1607,7 +1602,7 @@ static void do_detach(struct device *dev) clear_dte_entry(dev_data->devid); /* Flush the DTE entry */ - device_flush_dte(dev); + device_flush_dte(dev_data); } /* -- cgit v1.2.3 From ec9e79ef062272a119994c4fdd5e434f3e1bf352 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 17:25:50 +0200 Subject: x86/amd-iommu: Use only dev_data in low-level domain attach/detach functions With this patch the low-level attach/detach functions only work on dev_data structures. This allows to remove the dev_data->dev pointer. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 - arch/x86/kernel/amd_iommu.c | 61 +++++++++++++--------------------- 2 files changed, 24 insertions(+), 38 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 9fc045ee2fcb..fffbe3c9e6c2 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -311,7 +311,6 @@ struct protection_domain { struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ struct list_head dev_data_list; /* For global dev_data_list */ - struct device *dev; /* Device this data belong to */ struct device *alias; /* The Alias Device */ struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 0a5b46504b5d..a15a17231909 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -182,8 +182,6 @@ static int iommu_init_device(struct device *dev) if (!dev_data) return -ENOMEM; - dev_data->dev = dev; - alias = amd_iommu_alias_table[dev_data->devid]; pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); if (pdev) @@ -1558,18 +1556,14 @@ static void clear_dte_entry(u16 devid) amd_iommu_apply_erratum_63(devid); } -static void do_attach(struct device *dev, struct protection_domain *domain) +static void do_attach(struct iommu_dev_data *dev_data, + struct protection_domain *domain) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - struct pci_dev *pdev; - bool ats = false; - - dev_data = get_dev_data(dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; - pdev = to_pci_dev(dev); + bool ats; - ats = dev_data->ats.enabled; + iommu = amd_iommu_rlookup_table[dev_data->devid]; + ats = dev_data->ats.enabled; /* Update data structures */ dev_data->domain = domain; @@ -1584,13 +1578,11 @@ static void do_attach(struct device *dev, struct protection_domain *domain) device_flush_dte(dev_data); } -static void do_detach(struct device *dev) +static void do_detach(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - dev_data = get_dev_data(dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; /* decrease reference counters */ dev_data->domain->dev_iommu[iommu->index] -= 1; @@ -1609,13 +1601,12 @@ static void do_detach(struct device *dev) * If a device is not yet associated with a domain, this function does * assigns it visible for the hardware */ -static int __attach_device(struct device *dev, +static int __attach_device(struct iommu_dev_data *dev_data, struct protection_domain *domain) { - struct iommu_dev_data *dev_data, *alias_data; + struct iommu_dev_data *alias_data; int ret; - dev_data = get_dev_data(dev); alias_data = get_dev_data(dev_data->alias); if (!alias_data) @@ -1635,16 +1626,15 @@ static int __attach_device(struct device *dev, goto out_unlock; /* Do real assignment */ - if (dev_data->alias != dev) { - alias_data = get_dev_data(dev_data->alias); + if (dev_data->devid != alias_data->devid) { if (alias_data->domain == NULL) - do_attach(dev_data->alias, domain); + do_attach(alias_data, domain); atomic_inc(&alias_data->bind); } if (dev_data->domain == NULL) - do_attach(dev, domain); + do_attach(dev_data, domain); atomic_inc(&dev_data->bind); @@ -1678,7 +1668,7 @@ static int attach_device(struct device *dev, } write_lock_irqsave(&amd_iommu_devtable_lock, flags); - ret = __attach_device(dev, domain); + ret = __attach_device(dev_data, domain); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); /* @@ -1694,9 +1684,8 @@ static int attach_device(struct device *dev, /* * Removes a device from a protection domain (unlocked) */ -static void __detach_device(struct device *dev) +static void __detach_device(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *dev_data = get_dev_data(dev); struct iommu_dev_data *alias_data; struct protection_domain *domain; unsigned long flags; @@ -1707,14 +1696,14 @@ static void __detach_device(struct device *dev) spin_lock_irqsave(&domain->lock, flags); - if (dev_data->alias != dev) { - alias_data = get_dev_data(dev_data->alias); + alias_data = get_dev_data(dev_data->alias); + if (dev_data->devid != alias_data->devid) { if (atomic_dec_and_test(&alias_data->bind)) - do_detach(dev_data->alias); + do_detach(alias_data); } if (atomic_dec_and_test(&dev_data->bind)) - do_detach(dev); + do_detach(dev_data); spin_unlock_irqrestore(&domain->lock, flags); @@ -1725,7 +1714,7 @@ static void __detach_device(struct device *dev) */ if (iommu_pass_through && (dev_data->domain == NULL && domain != pt_domain)) - __attach_device(dev, pt_domain); + __attach_device(dev_data, pt_domain); } /* @@ -1736,13 +1725,13 @@ static void detach_device(struct device *dev) struct iommu_dev_data *dev_data; unsigned long flags; + dev_data = get_dev_data(dev); + /* lock device table */ write_lock_irqsave(&amd_iommu_devtable_lock, flags); - __detach_device(dev); + __detach_device(dev_data); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - dev_data = get_dev_data(dev); - if (dev_data->ats.enabled) { pci_disable_ats(to_pci_dev(dev)); dev_data->ats.enabled = false; @@ -1768,7 +1757,7 @@ static struct protection_domain *domain_for_device(struct device *dev) dom = dev_data->domain; if (dom == NULL && alias_data->domain != NULL) { - __attach_device(dev, alias_data->domain); + __attach_device(dev_data, alias_data->domain); dom = alias_data->domain; } @@ -2527,9 +2516,7 @@ static void cleanup_domain(struct protection_domain *domain) write_lock_irqsave(&amd_iommu_devtable_lock, flags); list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) { - struct device *dev = dev_data->dev; - - __detach_device(dev); + __detach_device(dev_data); atomic_set(&dev_data->bind, 0); } -- cgit v1.2.3 From 2b02b091ab3d1685926bfc76486bf7a22d92979a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 17:48:39 +0200 Subject: x86/amd-iommu: Allow dev_data->alias to be NULL Let dev_data->alias be just NULL if the device has no alias. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 77 ++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 36 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index a15a17231909..2d914a49b19a 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -183,12 +183,14 @@ static int iommu_init_device(struct device *dev) return -ENOMEM; alias = amd_iommu_alias_table[dev_data->devid]; - pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); - if (pdev) - dev_data->alias = &pdev->dev; - else { - free_dev_data(dev_data); - return -ENOTSUPP; + if (alias != dev_data->devid) { + pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); + if (pdev) + dev_data->alias = &pdev->dev; + else { + free_dev_data(dev_data); + return -ENOTSUPP; + } } dev->archdata.iommu = dev_data; @@ -1604,29 +1606,31 @@ static void do_detach(struct iommu_dev_data *dev_data) static int __attach_device(struct iommu_dev_data *dev_data, struct protection_domain *domain) { - struct iommu_dev_data *alias_data; int ret; - alias_data = get_dev_data(dev_data->alias); - - if (!alias_data) - return -EINVAL; - /* lock domain */ spin_lock(&domain->lock); - /* Some sanity checks */ - ret = -EBUSY; - if (alias_data->domain != NULL && - alias_data->domain != domain) - goto out_unlock; + if (dev_data->alias != NULL) { + struct iommu_dev_data *alias_data; + + alias_data = get_dev_data(dev_data->alias); + + ret = -EINVAL; + if (!alias_data) + goto out_unlock; - if (dev_data->domain != NULL && - dev_data->domain != domain) - goto out_unlock; + /* Some sanity checks */ + ret = -EBUSY; + if (alias_data->domain != NULL && + alias_data->domain != domain) + goto out_unlock; - /* Do real assignment */ - if (dev_data->devid != alias_data->devid) { + if (dev_data->domain != NULL && + dev_data->domain != domain) + goto out_unlock; + + /* Do real assignment */ if (alias_data->domain == NULL) do_attach(alias_data, domain); @@ -1696,8 +1700,8 @@ static void __detach_device(struct iommu_dev_data *dev_data) spin_lock_irqsave(&domain->lock, flags); - alias_data = get_dev_data(dev_data->alias); - if (dev_data->devid != alias_data->devid) { + if (dev_data->alias) { + alias_data = get_dev_data(dev_data->alias); if (atomic_dec_and_test(&alias_data->bind)) do_detach(alias_data); } @@ -1744,24 +1748,25 @@ static void detach_device(struct device *dev) */ static struct protection_domain *domain_for_device(struct device *dev) { - struct protection_domain *dom; struct iommu_dev_data *dev_data, *alias_data; + struct protection_domain *dom = NULL; unsigned long flags; dev_data = get_dev_data(dev); - alias_data = get_dev_data(dev_data->alias); - if (!alias_data) - return NULL; - read_lock_irqsave(&amd_iommu_devtable_lock, flags); - dom = dev_data->domain; - if (dom == NULL && - alias_data->domain != NULL) { - __attach_device(dev_data, alias_data->domain); - dom = alias_data->domain; - } + if (dev_data->domain) + return dev_data->domain; - read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + if (dev_data->alias != NULL) { + alias_data = get_dev_data(dev_data->alias); + + read_lock_irqsave(&amd_iommu_devtable_lock, flags); + if (alias_data->domain != NULL) { + __attach_device(dev_data, alias_data->domain); + dom = alias_data->domain; + } + read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + } return dom; } -- cgit v1.2.3 From 3b03bb745eb9fcafc6ef80fcbd04f0f0512acdc9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 18:53:25 +0200 Subject: x86/amd-iommu: Search for existind dev_data before allocting a new one Search for existing dev_data first will allow to switch dev_data->alias to just store dev_data instead of struct device. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 2d914a49b19a..f64799878d45 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -102,6 +102,37 @@ static void free_dev_data(struct iommu_dev_data *dev_data) kfree(dev_data); } +static struct iommu_dev_data *search_dev_data(u16 devid) +{ + struct iommu_dev_data *dev_data; + unsigned long flags; + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_for_each_entry(dev_data, &dev_data_list, dev_data_list) { + if (dev_data->devid == devid) + goto out_unlock; + } + + dev_data = NULL; + +out_unlock: + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + return dev_data; +} + +static struct iommu_dev_data *find_dev_data(u16 devid) +{ + struct iommu_dev_data *dev_data; + + dev_data = search_dev_data(devid); + + if (dev_data == NULL) + dev_data = alloc_dev_data(devid); + + return dev_data; +} + static inline u16 get_device_id(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -178,7 +209,7 @@ static int iommu_init_device(struct device *dev) if (dev->archdata.iommu) return 0; - dev_data = alloc_dev_data(get_device_id(dev)); + dev_data = find_dev_data(get_device_id(dev)); if (!dev_data) return -ENOMEM; -- cgit v1.2.3 From 71f77580906b60eb43daff0efb99792e76a3d06d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 19:03:15 +0200 Subject: x86/amd-iommu: Store device alias as dev_data pointer This finally allows PCI-Device-IDs to be handled by the IOMMU driver that have no corresponding struct device present in the system. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 2 +- arch/x86/kernel/amd_iommu.c | 34 +++++++++++++++------------------- 2 files changed, 16 insertions(+), 20 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index fffbe3c9e6c2..5b9c5075e81a 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -311,7 +311,7 @@ struct protection_domain { struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ struct list_head dev_data_list; /* For global dev_data_list */ - struct device *alias; /* The Alias Device */ + struct iommu_dev_data *alias_data;/* The alias dev_data */ struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ u16 devid; /* PCI Device ID */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index f64799878d45..0ed1c940d76b 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -203,7 +203,6 @@ static bool check_device(struct device *dev) static int iommu_init_device(struct device *dev) { struct iommu_dev_data *dev_data; - struct pci_dev *pdev; u16 alias; if (dev->archdata.iommu) @@ -215,13 +214,16 @@ static int iommu_init_device(struct device *dev) alias = amd_iommu_alias_table[dev_data->devid]; if (alias != dev_data->devid) { - pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); - if (pdev) - dev_data->alias = &pdev->dev; - else { + struct iommu_dev_data *alias_data; + + alias_data = find_dev_data(alias); + if (alias_data == NULL) { + pr_err("AMD-Vi: Warning: Unhandled device %s\n", + dev_name(dev)); free_dev_data(dev_data); return -ENOTSUPP; } + dev_data->alias_data = alias_data; } dev->archdata.iommu = dev_data; @@ -1642,14 +1644,8 @@ static int __attach_device(struct iommu_dev_data *dev_data, /* lock domain */ spin_lock(&domain->lock); - if (dev_data->alias != NULL) { - struct iommu_dev_data *alias_data; - - alias_data = get_dev_data(dev_data->alias); - - ret = -EINVAL; - if (!alias_data) - goto out_unlock; + if (dev_data->alias_data != NULL) { + struct iommu_dev_data *alias_data = dev_data->alias_data; /* Some sanity checks */ ret = -EBUSY; @@ -1721,7 +1717,6 @@ static int attach_device(struct device *dev, */ static void __detach_device(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *alias_data; struct protection_domain *domain; unsigned long flags; @@ -1731,8 +1726,9 @@ static void __detach_device(struct iommu_dev_data *dev_data) spin_lock_irqsave(&domain->lock, flags); - if (dev_data->alias) { - alias_data = get_dev_data(dev_data->alias); + if (dev_data->alias_data != NULL) { + struct iommu_dev_data *alias_data = dev_data->alias_data; + if (atomic_dec_and_test(&alias_data->bind)) do_detach(alias_data); } @@ -1779,7 +1775,7 @@ static void detach_device(struct device *dev) */ static struct protection_domain *domain_for_device(struct device *dev) { - struct iommu_dev_data *dev_data, *alias_data; + struct iommu_dev_data *dev_data; struct protection_domain *dom = NULL; unsigned long flags; @@ -1788,8 +1784,8 @@ static struct protection_domain *domain_for_device(struct device *dev) if (dev_data->domain) return dev_data->domain; - if (dev_data->alias != NULL) { - alias_data = get_dev_data(dev_data->alias); + if (dev_data->alias_data != NULL) { + struct iommu_dev_data *alias_data = dev_data->alias_data; read_lock_irqsave(&amd_iommu_devtable_lock, flags); if (alias_data->domain != NULL) { -- cgit v1.2.3 From 395810627b6a43c8d0ec948884043946fa162308 Mon Sep 17 00:00:00 2001 From: Masami Hiramatsu Date: Wed, 8 Jun 2011 16:09:21 +0900 Subject: x86: Swap save_stack_trace_regs parameters Swap the 1st and 2nd parameters of save_stack_trace_regs() as same as the parameters of save_stack_trace_tsk(). Signed-off-by: Masami Hiramatsu Cc: yrl.pp-manager.tt@hitachi.com Cc: Frederic Weisbecker Cc: Peter Zijlstra Cc: Namhyung Kim Link: http://lkml.kernel.org/r/20110608070921.17777.31103.stgit@fedora15 Signed-off-by: Steven Rostedt --- arch/x86/kernel/stacktrace.c | 2 +- arch/x86/mm/kmemcheck/error.c | 2 +- include/linux/stacktrace.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/stacktrace.c b/arch/x86/kernel/stacktrace.c index 55d9bc03f696..fdd0c6430e5a 100644 --- a/arch/x86/kernel/stacktrace.c +++ b/arch/x86/kernel/stacktrace.c @@ -66,7 +66,7 @@ void save_stack_trace(struct stack_trace *trace) } EXPORT_SYMBOL_GPL(save_stack_trace); -void save_stack_trace_regs(struct stack_trace *trace, struct pt_regs *regs) +void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) { dump_trace(current, regs, NULL, 0, &save_stack_ops, trace); if (trace->nr_entries < trace->max_entries) diff --git a/arch/x86/mm/kmemcheck/error.c b/arch/x86/mm/kmemcheck/error.c index 704a37cedddb..dab41876cdd5 100644 --- a/arch/x86/mm/kmemcheck/error.c +++ b/arch/x86/mm/kmemcheck/error.c @@ -185,7 +185,7 @@ void kmemcheck_error_save(enum kmemcheck_shadow state, e->trace.entries = e->trace_entries; e->trace.max_entries = ARRAY_SIZE(e->trace_entries); e->trace.skip = 0; - save_stack_trace_regs(&e->trace, regs); + save_stack_trace_regs(regs, &e->trace); /* Round address down to nearest 16 bytes */ shadow_copy = kmemcheck_shadow_lookup(address diff --git a/include/linux/stacktrace.h b/include/linux/stacktrace.h index 25310f1d7f37..115b570e3bff 100644 --- a/include/linux/stacktrace.h +++ b/include/linux/stacktrace.h @@ -14,8 +14,8 @@ struct stack_trace { }; extern void save_stack_trace(struct stack_trace *trace); -extern void save_stack_trace_regs(struct stack_trace *trace, - struct pt_regs *regs); +extern void save_stack_trace_regs(struct pt_regs *regs, + struct stack_trace *trace); extern void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace); -- cgit v1.2.3 From 86b445676d13f520ef9ab7aebe933aa6684ce84c Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Tue, 22 Feb 2011 18:41:48 +0100 Subject: x86, microcode, AMD: Correct buf references Both the equivalence table and the microcode patch types are u32. Access them properly through the buf-ptr. Signed-off-by: Borislav Petkov --- arch/x86/kernel/microcode_amd.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/microcode_amd.c b/arch/x86/kernel/microcode_amd.c index c5610384ab16..d30d67cd33ad 100644 --- a/arch/x86/kernel/microcode_amd.c +++ b/arch/x86/kernel/microcode_amd.c @@ -157,7 +157,7 @@ static int apply_microcode_amd(int cpu) static unsigned int verify_ucode_size(int cpu, const u8 *buf, unsigned int size) { struct cpuinfo_x86 *c = &cpu_data(cpu); - unsigned int max_size, actual_size; + u32 max_size, actual_size; #define F1XH_MPB_MAX_SIZE 2048 #define F14H_MPB_MAX_SIZE 1824 @@ -175,7 +175,7 @@ static unsigned int verify_ucode_size(int cpu, const u8 *buf, unsigned int size) break; } - actual_size = buf[4] + (buf[5] << 8); + actual_size = *(u32 *)(buf + 4); if (actual_size > size || actual_size > max_size) { pr_err("section size mismatch\n"); @@ -191,7 +191,7 @@ get_next_ucode(int cpu, const u8 *buf, unsigned int size, unsigned int *mc_size) struct microcode_header_amd *mc = NULL; unsigned int actual_size = 0; - if (buf[0] != UCODE_UCODE_TYPE) { + if (*(u32 *)buf != UCODE_UCODE_TYPE) { pr_err("invalid type field in container file section header\n"); goto out; } -- cgit v1.2.3 From 880a317abccf60b989951bf1515964cd73245970 Mon Sep 17 00:00:00 2001 From: Tony Luck Date: Tue, 7 Jun 2011 21:51:56 -0400 Subject: x86, mce, severity: Fix two severities table signatures The "Spurious not enabled" entry is redundant: the "Not enabled" entry earlier in the table will cover this case. The "Action required; unknown MCACOD" entry shouldn't specify MCACOD in the .mask field. Current code will only match for mcacod==0 rather than all AR=1 entries. Signed-off-by: Tony Luck Signed-off-by: Hidetoshi Seto Link: http://lkml.kernel.org/r/4DEED5BC.8030703@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/mce-severity.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mcheck/mce-severity.c b/arch/x86/kernel/cpu/mcheck/mce-severity.c index 1e8d66c1336a..352d16a6c9b7 100644 --- a/arch/x86/kernel/cpu/mcheck/mce-severity.c +++ b/arch/x86/kernel/cpu/mcheck/mce-severity.c @@ -69,8 +69,6 @@ static struct severity { MCGMASK(MCG_STATUS_RIPV, 0, PANIC, "In kernel and no restart IP", KERNEL), BITCLR(MCI_STATUS_UC, KEEP, "Corrected error", NOSER), - MASK(MCI_STATUS_OVER|MCI_STATUS_UC|MCI_STATUS_EN, MCI_STATUS_UC, SOME, - "Spurious not enabled", SER), /* ignore OVER for UCNA */ MASK(MCI_UC_SAR, MCI_STATUS_UC, KEEP, @@ -82,7 +80,7 @@ static struct severity { /* AR add known MCACODs here */ MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_OVER|MCI_UC_SAR, PANIC, "Action required with lost events", SER), - MASK(MCI_STATUS_OVER|MCI_UC_SAR|MCACOD, MCI_UC_SAR, PANIC, + MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_SAR, PANIC, "Action required; unknown MCACOD", SER), /* known AO MCACODs: */ -- cgit v1.2.3 From 901d7691d3238ad68c80a567b88b1e5d614137fb Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 10:52:43 +0900 Subject: x86, mce, severity: Make formatting a bit more readable The table looks very complicated and hard to read for people other than skilled developers. So let's clean it up a bit. At first, change format to ease reading elements in the table. Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED5EB.6050400@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/mce-severity.c | 118 ++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 30 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mcheck/mce-severity.c b/arch/x86/kernel/cpu/mcheck/mce-severity.c index 352d16a6c9b7..eaf5a43ff082 100644 --- a/arch/x86/kernel/cpu/mcheck/mce-severity.c +++ b/arch/x86/kernel/cpu/mcheck/mce-severity.c @@ -58,44 +58,102 @@ static struct severity { #define MCI_UC_SAR (MCI_STATUS_UC|MCI_STATUS_S|MCI_STATUS_AR) #define MCACOD 0xffff - BITCLR(MCI_STATUS_VAL, NO, "Invalid"), - BITCLR(MCI_STATUS_EN, NO, "Not enabled"), - BITSET(MCI_STATUS_PCC, PANIC, "Processor context corrupt"), + BITCLR( + MCI_STATUS_VAL, + NO, "Invalid" + ), + BITCLR( + MCI_STATUS_EN, + NO, "Not enabled" + ), + BITSET( + MCI_STATUS_PCC, + PANIC, "Processor context corrupt" + ), /* When MCIP is not set something is very confused */ - MCGMASK(MCG_STATUS_MCIP, 0, PANIC, "MCIP not set in MCA handler"), + MCGMASK( + MCG_STATUS_MCIP, 0, + PANIC, "MCIP not set in MCA handler" + ), /* Neither return not error IP -- no chance to recover -> PANIC */ - MCGMASK(MCG_STATUS_RIPV|MCG_STATUS_EIPV, 0, PANIC, - "Neither restart nor error IP"), - MCGMASK(MCG_STATUS_RIPV, 0, PANIC, "In kernel and no restart IP", - KERNEL), - BITCLR(MCI_STATUS_UC, KEEP, "Corrected error", NOSER), + MCGMASK( + MCG_STATUS_RIPV|MCG_STATUS_EIPV, 0, + PANIC, "Neither restart nor error IP" + ), + MCGMASK( + MCG_STATUS_RIPV, 0, + PANIC, "In kernel and no restart IP", + KERNEL + ), + BITCLR( + MCI_STATUS_UC, + KEEP, "Corrected error", + NOSER + ), /* ignore OVER for UCNA */ - MASK(MCI_UC_SAR, MCI_STATUS_UC, KEEP, - "Uncorrected no action required", SER), - MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_UC|MCI_STATUS_AR, PANIC, - "Illegal combination (UCNA with AR=1)", SER), - MASK(MCI_STATUS_S, 0, KEEP, "Non signalled machine check", SER), + MASK( + MCI_UC_SAR, MCI_STATUS_UC, + KEEP, "Uncorrected no action required", + SER + ), + MASK( + MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_UC|MCI_STATUS_AR, + PANIC, "Illegal combination (UCNA with AR=1)", + SER + ), + MASK( + MCI_STATUS_S, 0, + KEEP, "Non signalled machine check", + SER + ), /* AR add known MCACODs here */ - MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_OVER|MCI_UC_SAR, PANIC, - "Action required with lost events", SER), - MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_SAR, PANIC, - "Action required; unknown MCACOD", SER), + MASK( + MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_OVER|MCI_UC_SAR, + PANIC, "Action required with lost events", + SER + ), + MASK( + MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_SAR, + PANIC, "Action required; unknown MCACOD", + SER + ), /* known AO MCACODs: */ - MASK(MCI_UC_SAR|MCI_STATUS_OVER|0xfff0, MCI_UC_S|0xc0, AO, - "Action optional: memory scrubbing error", SER), - MASK(MCI_UC_SAR|MCI_STATUS_OVER|MCACOD, MCI_UC_S|0x17a, AO, - "Action optional: last level cache writeback error", SER), - - MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S, SOME, - "Action optional unknown MCACOD", SER), - MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S|MCI_STATUS_OVER, SOME, - "Action optional with lost events", SER), - BITSET(MCI_STATUS_UC|MCI_STATUS_OVER, PANIC, "Overflowed uncorrected"), - BITSET(MCI_STATUS_UC, UC, "Uncorrected"), - BITSET(0, SOME, "No match") /* always matches. keep at end */ + MASK( + MCI_UC_SAR|MCI_STATUS_OVER|0xfff0, MCI_UC_S|0xc0, + AO, "Action optional: memory scrubbing error", + SER + ), + MASK( + MCI_UC_SAR|MCI_STATUS_OVER|MCACOD, MCI_UC_S|0x17a, + AO, "Action optional: last level cache writeback error", + SER + ), + + MASK( + MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S, + SOME, "Action optional unknown MCACOD", + SER + ), + MASK( + MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S|MCI_STATUS_OVER, + SOME, "Action optional with lost events", + SER + ), + BITSET( + MCI_STATUS_UC|MCI_STATUS_OVER, + PANIC, "Overflowed uncorrected" + ), + BITSET( + MCI_STATUS_UC, + UC, "Uncorrected" + ), + BITSET( + 0, + SOME, "No match" + ) /* always matches. keep at end */ }; /* -- cgit v1.2.3 From a17957cdec69acb9e26319618b95a810a936e637 Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 10:53:35 +0900 Subject: x86, mce, severity: Cleanup severity table The current format of an item in this table is: condition(param, ..., level, message [, condition2 ...]) So we have to check both an item's head and tail to find the conditions which match the item. Format them in a more straight forward manner: item(level, message, condition [, condition2 ...]) Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED61F.5010502@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/mce-severity.c | 127 ++++++++++++++---------------- 1 file changed, 58 insertions(+), 69 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mcheck/mce-severity.c b/arch/x86/kernel/cpu/mcheck/mce-severity.c index eaf5a43ff082..27e778eac8c6 100644 --- a/arch/x86/kernel/cpu/mcheck/mce-severity.c +++ b/arch/x86/kernel/cpu/mcheck/mce-severity.c @@ -43,116 +43,105 @@ static struct severity { unsigned char covered; char *msg; } severities[] = { -#define KERNEL .context = IN_KERNEL -#define USER .context = IN_USER -#define SER .ser = SER_REQUIRED -#define NOSER .ser = NO_SER -#define SEV(s) .sev = MCE_ ## s ## _SEVERITY -#define BITCLR(x, s, m, r...) { .mask = x, .result = 0, SEV(s), .msg = m, ## r } -#define BITSET(x, s, m, r...) { .mask = x, .result = x, SEV(s), .msg = m, ## r } -#define MCGMASK(x, res, s, m, r...) \ - { .mcgmask = x, .mcgres = res, SEV(s), .msg = m, ## r } -#define MASK(x, y, s, m, r...) \ - { .mask = x, .result = y, SEV(s), .msg = m, ## r } +#define MCESEV(s, m, c...) { .sev = MCE_ ## s ## _SEVERITY, .msg = m, ## c } +#define KERNEL .context = IN_KERNEL +#define USER .context = IN_USER +#define SER .ser = SER_REQUIRED +#define NOSER .ser = NO_SER +#define BITCLR(x) .mask = x, .result = 0 +#define BITSET(x) .mask = x, .result = x +#define MCGMASK(x, y) .mcgmask = x, .mcgres = y +#define MASK(x, y) .mask = x, .result = y #define MCI_UC_S (MCI_STATUS_UC|MCI_STATUS_S) #define MCI_UC_SAR (MCI_STATUS_UC|MCI_STATUS_S|MCI_STATUS_AR) #define MCACOD 0xffff - BITCLR( - MCI_STATUS_VAL, - NO, "Invalid" + MCESEV( + NO, "Invalid", + BITCLR(MCI_STATUS_VAL) ), - BITCLR( - MCI_STATUS_EN, - NO, "Not enabled" + MCESEV( + NO, "Not enabled", + BITCLR(MCI_STATUS_EN) ), - BITSET( - MCI_STATUS_PCC, - PANIC, "Processor context corrupt" + MCESEV( + PANIC, "Processor context corrupt", + BITSET(MCI_STATUS_PCC) ), /* When MCIP is not set something is very confused */ - MCGMASK( - MCG_STATUS_MCIP, 0, - PANIC, "MCIP not set in MCA handler" + MCESEV( + PANIC, "MCIP not set in MCA handler", + MCGMASK(MCG_STATUS_MCIP, 0) ), /* Neither return not error IP -- no chance to recover -> PANIC */ - MCGMASK( - MCG_STATUS_RIPV|MCG_STATUS_EIPV, 0, - PANIC, "Neither restart nor error IP" + MCESEV( + PANIC, "Neither restart nor error IP", + MCGMASK(MCG_STATUS_RIPV|MCG_STATUS_EIPV, 0) ), - MCGMASK( - MCG_STATUS_RIPV, 0, + MCESEV( PANIC, "In kernel and no restart IP", - KERNEL + KERNEL, MCGMASK(MCG_STATUS_RIPV, 0) ), - BITCLR( - MCI_STATUS_UC, + MCESEV( KEEP, "Corrected error", - NOSER + NOSER, BITCLR(MCI_STATUS_UC) ), /* ignore OVER for UCNA */ - MASK( - MCI_UC_SAR, MCI_STATUS_UC, + MCESEV( KEEP, "Uncorrected no action required", - SER + SER, MASK(MCI_UC_SAR, MCI_STATUS_UC) ), - MASK( - MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_UC|MCI_STATUS_AR, + MCESEV( PANIC, "Illegal combination (UCNA with AR=1)", - SER + SER, + MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_UC|MCI_STATUS_AR) ), - MASK( - MCI_STATUS_S, 0, + MCESEV( KEEP, "Non signalled machine check", - SER + SER, MASK(MCI_STATUS_S, 0) ), /* AR add known MCACODs here */ - MASK( - MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_OVER|MCI_UC_SAR, + MCESEV( PANIC, "Action required with lost events", - SER + SER, + MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_OVER|MCI_UC_SAR) ), - MASK( - MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_SAR, + MCESEV( PANIC, "Action required; unknown MCACOD", - SER + SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_SAR) ), /* known AO MCACODs: */ - MASK( - MCI_UC_SAR|MCI_STATUS_OVER|0xfff0, MCI_UC_S|0xc0, + MCESEV( AO, "Action optional: memory scrubbing error", - SER + SER, MASK(MCI_UC_SAR|MCI_STATUS_OVER|0xfff0, MCI_UC_S|0xc0) ), - MASK( - MCI_UC_SAR|MCI_STATUS_OVER|MCACOD, MCI_UC_S|0x17a, + MCESEV( AO, "Action optional: last level cache writeback error", - SER + SER, MASK(MCI_UC_SAR|MCI_STATUS_OVER|MCACOD, MCI_UC_S|0x17a) ), - - MASK( - MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S, + MCESEV( SOME, "Action optional unknown MCACOD", - SER + SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S) ), - MASK( - MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S|MCI_STATUS_OVER, + MCESEV( SOME, "Action optional with lost events", - SER + SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S|MCI_STATUS_OVER) ), - BITSET( - MCI_STATUS_UC|MCI_STATUS_OVER, - PANIC, "Overflowed uncorrected" + + MCESEV( + PANIC, "Overflowed uncorrected", + BITSET(MCI_STATUS_UC|MCI_STATUS_OVER) ), - BITSET( - MCI_STATUS_UC, - UC, "Uncorrected" + MCESEV( + UC, "Uncorrected", + BITSET(MCI_STATUS_UC) ), - BITSET( - 0, - SOME, "No match" + MCESEV( + SOME, "No match", + BITSET(0) ) /* always matches. keep at end */ }; -- cgit v1.2.3 From 7639bfc753f70321dbea83852e1cc47a45b681d7 Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 10:55:05 +0900 Subject: x86, mce, severity: Clean up trivial coding style problems More specifically: - sort bits in the macros - use BITCLR/BITSET - coordinate message pattern - use m for struct mce - cleanup for severities_debugfs_init() No functional change. Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED679.9090503@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/mce-severity.c | 39 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 20 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mcheck/mce-severity.c b/arch/x86/kernel/cpu/mcheck/mce-severity.c index 27e778eac8c6..7395d5f4272d 100644 --- a/arch/x86/kernel/cpu/mcheck/mce-severity.c +++ b/arch/x86/kernel/cpu/mcheck/mce-severity.c @@ -99,41 +99,40 @@ static struct severity { ), MCESEV( KEEP, "Non signalled machine check", - SER, MASK(MCI_STATUS_S, 0) + SER, BITCLR(MCI_STATUS_S) ), /* AR add known MCACODs here */ MCESEV( PANIC, "Action required with lost events", - SER, - MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_OVER|MCI_UC_SAR) + SER, BITSET(MCI_STATUS_OVER|MCI_UC_SAR) ), MCESEV( - PANIC, "Action required; unknown MCACOD", + PANIC, "Action required: unknown MCACOD", SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_SAR) ), /* known AO MCACODs: */ MCESEV( AO, "Action optional: memory scrubbing error", - SER, MASK(MCI_UC_SAR|MCI_STATUS_OVER|0xfff0, MCI_UC_S|0xc0) + SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR|0xfff0, MCI_UC_S|0x00c0) ), MCESEV( AO, "Action optional: last level cache writeback error", - SER, MASK(MCI_UC_SAR|MCI_STATUS_OVER|MCACOD, MCI_UC_S|0x17a) + SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR|MCACOD, MCI_UC_S|0x017a) ), MCESEV( - SOME, "Action optional unknown MCACOD", + SOME, "Action optional: unknown MCACOD", SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S) ), MCESEV( SOME, "Action optional with lost events", - SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S|MCI_STATUS_OVER) + SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_STATUS_OVER|MCI_UC_S) ), MCESEV( PANIC, "Overflowed uncorrected", - BITSET(MCI_STATUS_UC|MCI_STATUS_OVER) + BITSET(MCI_STATUS_OVER|MCI_STATUS_UC) ), MCESEV( UC, "Uncorrected", @@ -157,15 +156,15 @@ static int error_context(struct mce *m) return IN_KERNEL; } -int mce_severity(struct mce *a, int tolerant, char **msg) +int mce_severity(struct mce *m, int tolerant, char **msg) { - enum context ctx = error_context(a); + enum context ctx = error_context(m); struct severity *s; for (s = severities;; s++) { - if ((a->status & s->mask) != s->result) + if ((m->status & s->mask) != s->result) continue; - if ((a->mcgstatus & s->mcgmask) != s->mcgres) + if ((m->mcgstatus & s->mcgmask) != s->mcgres) continue; if (s->ser == SER_REQUIRED && !mce_ser) continue; @@ -242,15 +241,15 @@ static const struct file_operations severities_coverage_fops = { static int __init severities_debugfs_init(void) { - struct dentry *dmce = NULL, *fseverities_coverage = NULL; + struct dentry *dmce, *fsev; dmce = mce_get_debugfs_dir(); - if (dmce == NULL) + if (!dmce) goto err_out; - fseverities_coverage = debugfs_create_file("severities-coverage", - 0444, dmce, NULL, - &severities_coverage_fops); - if (fseverities_coverage == NULL) + + fsev = debugfs_create_file("severities-coverage", 0444, dmce, NULL, + &severities_coverage_fops); + if (!fsev) goto err_out; return 0; @@ -259,4 +258,4 @@ err_out: return -ENOMEM; } late_initcall(severities_debugfs_init); -#endif +#endif /* CONFIG_DEBUG_FS */ -- cgit v1.2.3 From b77e70bf3535e0bd5472e0681f41cce4ae0598bb Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 10:56:02 +0900 Subject: x86, mce: Replace MCE_SELF_VECTOR by irq_work The MCE handler uses a special vector for self IPI to invoke post-emergency processing in an interrupt context, e.g. call an NMI-unsafe function, wakeup loggers, schedule time-consuming work for recovery, etc. This mechanism is now generalized by the following commit: > e360adbe29241a0194e10e20595360dd7b98a2b3 > Author: Peter Zijlstra > Date: Thu Oct 14 14:01:34 2010 +0800 > > irq_work: Add generic hardirq context callbacks > > Provide a mechanism that allows running code in IRQ context. It is > most useful for NMI code that needs to interact with the rest of the > system -- like wakeup a task to drain buffers. : So change to use provided generic mechanism. Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED6B2.6080005@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/include/asm/entry_arch.h | 4 ---- arch/x86/include/asm/hw_irq.h | 1 - arch/x86/include/asm/irq_vectors.h | 5 ---- arch/x86/kernel/cpu/mcheck/mce.c | 47 +++++--------------------------------- arch/x86/kernel/entry_64.S | 5 ---- arch/x86/kernel/irqinit.c | 3 --- 6 files changed, 6 insertions(+), 59 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h index 1cd6d26a0a8d..0baa628e330c 100644 --- a/arch/x86/include/asm/entry_arch.h +++ b/arch/x86/include/asm/entry_arch.h @@ -53,8 +53,4 @@ BUILD_INTERRUPT(thermal_interrupt,THERMAL_APIC_VECTOR) BUILD_INTERRUPT(threshold_interrupt,THRESHOLD_APIC_VECTOR) #endif -#ifdef CONFIG_X86_MCE -BUILD_INTERRUPT(mce_self_interrupt,MCE_SELF_VECTOR) -#endif - #endif diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index bb9efe8706e2..13f5504c76c0 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -34,7 +34,6 @@ extern void irq_work_interrupt(void); extern void spurious_interrupt(void); extern void thermal_interrupt(void); extern void reschedule_interrupt(void); -extern void mce_self_interrupt(void); extern void invalidate_interrupt(void); extern void invalidate_interrupt0(void); diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h index 6e976ee3b3ef..6665026ea3ea 100644 --- a/arch/x86/include/asm/irq_vectors.h +++ b/arch/x86/include/asm/irq_vectors.h @@ -109,11 +109,6 @@ #define UV_BAU_MESSAGE 0xf5 -/* - * Self IPI vector for machine checks - */ -#define MCE_SELF_VECTOR 0xf4 - /* Xen vector callback to receive events in a HVM domain */ #define XEN_HVM_EVTCHN_CALLBACK 0xf3 diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index ff1ae9b6464d..e81d48b05618 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -38,12 +37,9 @@ #include #include #include +#include #include -#include -#include -#include -#include #include #include @@ -461,22 +457,13 @@ static inline void mce_get_rip(struct mce *m, struct pt_regs *regs) m->ip = mce_rdmsrl(rip_msr); } -#ifdef CONFIG_X86_LOCAL_APIC -/* - * Called after interrupts have been reenabled again - * when a MCE happened during an interrupts off region - * in the kernel. - */ -asmlinkage void smp_mce_self_interrupt(struct pt_regs *regs) +DEFINE_PER_CPU(struct irq_work, mce_irq_work); + +static void mce_irq_work_cb(struct irq_work *entry) { - ack_APIC_irq(); - exit_idle(); - irq_enter(); mce_notify_irq(); mce_schedule_work(); - irq_exit(); } -#endif static void mce_report_event(struct pt_regs *regs) { @@ -492,29 +479,7 @@ static void mce_report_event(struct pt_regs *regs) return; } -#ifdef CONFIG_X86_LOCAL_APIC - /* - * Without APIC do not notify. The event will be picked - * up eventually. - */ - if (!cpu_has_apic) - return; - - /* - * When interrupts are disabled we cannot use - * kernel services safely. Trigger an self interrupt - * through the APIC to instead do the notification - * after interrupts are reenabled again. - */ - apic->send_IPI_self(MCE_SELF_VECTOR); - - /* - * Wait for idle afterwards again so that we don't leave the - * APIC in a non idle state because the normal APIC writes - * cannot exclude us. - */ - apic_wait_icr_idle(); -#endif + irq_work_queue(&__get_cpu_var(mce_irq_work)); } DEFINE_PER_CPU(unsigned, mce_poll_count); @@ -1444,7 +1409,7 @@ void __cpuinit mcheck_cpu_init(struct cpuinfo_x86 *c) __mcheck_cpu_init_vendor(c); __mcheck_cpu_init_timer(); INIT_WORK(&__get_cpu_var(mce_work), mce_process_work); - + init_irq_work(&__get_cpu_var(mce_irq_work), &mce_irq_work_cb); } /* diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 8a445a0c989e..9fa65460d33a 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -991,11 +991,6 @@ apicinterrupt THRESHOLD_APIC_VECTOR \ apicinterrupt THERMAL_APIC_VECTOR \ thermal_interrupt smp_thermal_interrupt -#ifdef CONFIG_X86_MCE -apicinterrupt MCE_SELF_VECTOR \ - mce_self_interrupt smp_mce_self_interrupt -#endif - #ifdef CONFIG_SMP apicinterrupt CALL_FUNCTION_SINGLE_VECTOR \ call_function_single_interrupt smp_call_function_single_interrupt diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c index f470e4ef993e..f09d4bbe2d2d 100644 --- a/arch/x86/kernel/irqinit.c +++ b/arch/x86/kernel/irqinit.c @@ -272,9 +272,6 @@ static void __init apic_intr_init(void) #ifdef CONFIG_X86_MCE_THRESHOLD alloc_intr_gate(THRESHOLD_APIC_VECTOR, threshold_interrupt); #endif -#if defined(CONFIG_X86_MCE) && defined(CONFIG_X86_LOCAL_APIC) - alloc_intr_gate(MCE_SELF_VECTOR, mce_self_interrupt); -#endif #if defined(CONFIG_X86_64) || defined(CONFIG_X86_LOCAL_APIC) /* self generated IPI for local APIC timer */ -- cgit v1.2.3 From 2b90e77eaee8809073db5cf43ac9795cc2054dc0 Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 10:56:56 +0900 Subject: x86, mce: Replace MCM_ with MCI_MISC_ Follow other MCi register defines. Plus define MCI_MISC_ADDR_LSB() and MCI_MISC_ADDR_MODE(). Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED6E8.9090509@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/include/asm/mce.h | 17 +++++++++++------ arch/x86/kernel/cpu/mcheck/mce.c | 4 ++-- 2 files changed, 13 insertions(+), 8 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index 021979a6e23f..bbb328fd117b 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -8,6 +8,7 @@ * Machine Check support for x86 */ +/* MCG_CAP register defines */ #define MCG_BANKCNT_MASK 0xff /* Number of Banks */ #define MCG_CTL_P (1ULL<<8) /* MCG_CTL register available */ #define MCG_EXT_P (1ULL<<9) /* Extended registers available */ @@ -17,10 +18,12 @@ #define MCG_EXT_CNT(c) (((c) & MCG_EXT_CNT_MASK) >> MCG_EXT_CNT_SHIFT) #define MCG_SER_P (1ULL<<24) /* MCA recovery/new status bits */ +/* MCG_STATUS register defines */ #define MCG_STATUS_RIPV (1ULL<<0) /* restart ip valid */ #define MCG_STATUS_EIPV (1ULL<<1) /* ip points to correct instruction */ #define MCG_STATUS_MCIP (1ULL<<2) /* machine check in progress */ +/* MCi_STATUS register defines */ #define MCI_STATUS_VAL (1ULL<<63) /* valid error */ #define MCI_STATUS_OVER (1ULL<<62) /* previous errors lost */ #define MCI_STATUS_UC (1ULL<<61) /* uncorrected error */ @@ -31,12 +34,14 @@ #define MCI_STATUS_S (1ULL<<56) /* Signaled machine check */ #define MCI_STATUS_AR (1ULL<<55) /* Action required */ -/* MISC register defines */ -#define MCM_ADDR_SEGOFF 0 /* segment offset */ -#define MCM_ADDR_LINEAR 1 /* linear address */ -#define MCM_ADDR_PHYS 2 /* physical address */ -#define MCM_ADDR_MEM 3 /* memory address */ -#define MCM_ADDR_GENERIC 7 /* generic */ +/* MCi_MISC register defines */ +#define MCI_MISC_ADDR_LSB(m) ((m) & 0x3f) +#define MCI_MISC_ADDR_MODE(m) (((m) >> 6) & 7) +#define MCI_MISC_ADDR_SEGOFF 0 /* segment offset */ +#define MCI_MISC_ADDR_LINEAR 1 /* linear address */ +#define MCI_MISC_ADDR_PHYS 2 /* physical address */ +#define MCI_MISC_ADDR_MEM 3 /* memory address */ +#define MCI_MISC_ADDR_GENERIC 7 /* generic */ /* CTL2 register defines */ #define MCI_CTL2_CMCI_EN (1ULL << 30) diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index e81d48b05618..e80750802d7f 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -844,9 +844,9 @@ static int mce_usable_address(struct mce *m) { if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV)) return 0; - if ((m->misc & 0x3f) > PAGE_SHIFT) + if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT) return 0; - if (((m->misc >> 6) & 7) != MCM_ADDR_PHYS) + if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS) return 0; return 1; } -- cgit v1.2.3 From b8325c5b110d7ff460b79588e7e9afdcc73d5c3c Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 10:57:46 +0900 Subject: x86, mce: Introduce mce_gather_info() This patch introduces mce_gather_info() which is to be called at the beginning of error handling and gathers minimum error information from proper error registers (and saved registers). As the result of mce_get_rip() is integrated, unnecessary zeroing is removed. This also takes care of saving RIP which is required to make some decision about error severity for SRAR errors, instead of retrieving it later in the handler. Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED71A.1060906@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/mce.c | 50 ++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 23 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index e80750802d7f..a18287567865 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -368,6 +368,31 @@ static void mce_wrmsrl(u32 msr, u64 v) wrmsrl(msr, v); } +/* + * Collect all global (w.r.t. this processor) status about this machine + * check into our "mce" struct so that we can use it later to assess + * the severity of the problem as we read per-bank specific details. + */ +static inline void mce_gather_info(struct mce *m, struct pt_regs *regs) +{ + mce_setup(m); + + m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); + if (regs) { + /* + * Get the address of the instruction at the time of + * the machine check error. + */ + if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) { + m->ip = regs->ip; + m->cs = regs->cs; + } + /* Use accurate RIP reporting if available. */ + if (rip_msr) + m->ip = mce_rdmsrl(rip_msr); + } +} + /* * Simple lockless ring to communicate PFNs from the exception handler with the * process context work function. This is vastly simplified because there's @@ -439,24 +464,6 @@ static void mce_schedule_work(void) } } -/* - * Get the address of the instruction at the time of the machine check - * error. - */ -static inline void mce_get_rip(struct mce *m, struct pt_regs *regs) -{ - - if (regs && (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV))) { - m->ip = regs->ip; - m->cs = regs->cs; - } else { - m->ip = 0; - m->cs = 0; - } - if (rip_msr) - m->ip = mce_rdmsrl(rip_msr); -} - DEFINE_PER_CPU(struct irq_work, mce_irq_work); static void mce_irq_work_cb(struct irq_work *entry) @@ -506,9 +513,8 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b) percpu_inc(mce_poll_count); - mce_setup(&m); + mce_gather_info(&m, NULL); - m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); for (i = 0; i < banks; i++) { if (!mce_banks[i].ctl || !test_bit(i, *b)) continue; @@ -907,9 +913,8 @@ void do_machine_check(struct pt_regs *regs, long error_code) if (!banks) goto out; - mce_setup(&m); + mce_gather_info(&m, regs); - m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); final = &__get_cpu_var(mces_seen); *final = m; @@ -993,7 +998,6 @@ void do_machine_check(struct pt_regs *regs, long error_code) if (severity == MCE_AO_SEVERITY && mce_usable_address(&m)) mce_ring_add(m.addr >> PAGE_SHIFT); - mce_get_rip(&m, regs); mce_log(&m); if (severity > worst) { -- cgit v1.2.3 From 3a97fc34130326da87b20de5d0259c35406707ce Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 10:58:35 +0900 Subject: x86, mce: Check the result of ancient_init() Because "ancient CPUs" like p5 and winchip don't have X86_FEATURE_MCA (I suppose so), mcheck_cpu_init() on such CPUs will return at check of mce_available() after __mcheck_cpu_ancient_init(). It is hard to know this implicit behavior without knowing the CPUs well. So make it clear that we leave mcheck_cpu_init() when the CPU is initialized in __mcheck_cpu_ancient_init(). Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED74B.20502@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/mce.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index a18287567865..2cc98d517f36 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -1332,18 +1332,23 @@ static int __cpuinit __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) return 0; } -static void __cpuinit __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) +static int __cpuinit __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) { if (c->x86 != 5) - return; + return 0; + switch (c->x86_vendor) { case X86_VENDOR_INTEL: intel_p5_mcheck_init(c); + return 1; break; case X86_VENDOR_CENTAUR: winchip_mcheck_init(c); + return 1; break; } + + return 0; } static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c) @@ -1397,7 +1402,8 @@ void __cpuinit mcheck_cpu_init(struct cpuinfo_x86 *c) if (mce_disabled) return; - __mcheck_cpu_ancient_init(c); + if (__mcheck_cpu_ancient_init(c)) + return; if (!mce_available(c)) return; -- cgit v1.2.3 From f6783c4234e65bd6f85596d97745ccdbf041cb63 Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 10:59:19 +0900 Subject: x86, mce: Cleanup mce_create()/remove_device() Use temporary local variable sysdev to simplify the code. No change in logic. Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED777.7080205@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/mce.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 2cc98d517f36..f3f648c8e087 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -1925,28 +1925,28 @@ static cpumask_var_t mce_dev_initialized; /* Per cpu sysdev init. All of the cpus still share the same ctrl bank: */ static __cpuinit int mce_create_device(unsigned int cpu) { + struct sys_device *sysdev = &per_cpu(mce_dev, cpu); int err; int i, j; if (!mce_available(&boot_cpu_data)) return -EIO; - memset(&per_cpu(mce_dev, cpu).kobj, 0, sizeof(struct kobject)); - per_cpu(mce_dev, cpu).id = cpu; - per_cpu(mce_dev, cpu).cls = &mce_sysclass; + memset(&sysdev->kobj, 0, sizeof(struct kobject)); + sysdev->id = cpu; + sysdev->cls = &mce_sysclass; - err = sysdev_register(&per_cpu(mce_dev, cpu)); + err = sysdev_register(sysdev); if (err) return err; for (i = 0; mce_attrs[i]; i++) { - err = sysdev_create_file(&per_cpu(mce_dev, cpu), mce_attrs[i]); + err = sysdev_create_file(sysdev, mce_attrs[i]); if (err) goto error; } for (j = 0; j < banks; j++) { - err = sysdev_create_file(&per_cpu(mce_dev, cpu), - &mce_banks[j].attr); + err = sysdev_create_file(sysdev, &mce_banks[j].attr); if (err) goto error2; } @@ -1955,30 +1955,31 @@ static __cpuinit int mce_create_device(unsigned int cpu) return 0; error2: while (--j >= 0) - sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[j].attr); + sysdev_remove_file(sysdev, &mce_banks[j].attr); error: while (--i >= 0) - sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]); + sysdev_remove_file(sysdev, mce_attrs[i]); - sysdev_unregister(&per_cpu(mce_dev, cpu)); + sysdev_unregister(sysdev); return err; } static __cpuinit void mce_remove_device(unsigned int cpu) { + struct sys_device *sysdev = &per_cpu(mce_dev, cpu); int i; if (!cpumask_test_cpu(cpu, mce_dev_initialized)) return; for (i = 0; mce_attrs[i]; i++) - sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]); + sysdev_remove_file(sysdev, mce_attrs[i]); for (i = 0; i < banks; i++) - sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[i].attr); + sysdev_remove_file(sysdev, &mce_banks[i].attr); - sysdev_unregister(&per_cpu(mce_dev, cpu)); + sysdev_unregister(sysdev); cpumask_clear_cpu(cpu, mce_dev_initialized); } -- cgit v1.2.3 From 559faa6be143b8aa7a07b12f618d29fbc1c8eb0d Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 11:00:08 +0900 Subject: x86, mce: Cleanup mce_read() Use a temporary local variable m to simplify the code. No change in logic. Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED7A8.8020307@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/mce.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index f3f648c8e087..54360e82854d 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -1537,19 +1537,18 @@ static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize, do { for (i = prev; i < next; i++) { unsigned long start = jiffies; + struct mce *m = &mcelog.entry[i]; - while (!mcelog.entry[i].finished) { + while (!m->finished) { if (time_after_eq(jiffies, start + 2)) { - memset(mcelog.entry + i, 0, - sizeof(struct mce)); + memset(m, 0, sizeof(*m)); goto timeout; } cpu_relax(); } smp_rmb(); - err |= copy_to_user(buf, mcelog.entry + i, - sizeof(struct mce)); - buf += sizeof(struct mce); + err |= copy_to_user(buf, m, sizeof(*m)); + buf += sizeof(*m); timeout: ; } @@ -1569,13 +1568,13 @@ timeout: on_each_cpu(collect_tscs, cpu_tsc, 1); for (i = next; i < MCE_LOG_LEN; i++) { - if (mcelog.entry[i].finished && - mcelog.entry[i].tsc < cpu_tsc[mcelog.entry[i].cpu]) { - err |= copy_to_user(buf, mcelog.entry+i, - sizeof(struct mce)); + struct mce *m = &mcelog.entry[i]; + + if (m->finished && m->tsc < cpu_tsc[m->cpu]) { + err |= copy_to_user(buf, m, sizeof(*m)); smp_rmb(); - buf += sizeof(struct mce); - memset(&mcelog.entry[i], 0, sizeof(struct mce)); + buf += sizeof(*m); + memset(m, 0, sizeof(*m)); } } -- cgit v1.2.3 From 93b62c3cf59d44850cbe9f04d58da08930e3fb0b Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 11:00:45 +0900 Subject: x86, mce: Use mce_chrdev_ prefix to group functions There are many functions named mce_* so use a new prefix for the subset of functions dealing with the character device /dev/mcelog. This change doesn't impact the mce-inject module because the exported symbol mce_chrdev_ops already has the prefix, therefore it is left unchanged. Before: After: mce_wait mce_chrdev_wait mce_state_lock mce_chrdev_state_lock open_count mce_chrdev_open_count open_exclu mce_chrdev_open_exclu mce_open mce_chrdev_open mce_release mce_chrdev_release mce_read_mutex mce_chrdev_read_mutex mce_read mce_chrdev_read mce_poll mce_chrdev_poll mce_ioctl mce_chrdev_ioctl mce_log_device mce_chrdev_device Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED7CD.3040500@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/mce.c | 77 +++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 36 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 54360e82854d..9a735168e40f 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -45,12 +45,12 @@ #include "mce-internal.h" -static DEFINE_MUTEX(mce_read_mutex); +static DEFINE_MUTEX(mce_chrdev_read_mutex); #define rcu_dereference_check_mce(p) \ rcu_dereference_index_check((p), \ rcu_read_lock_sched_held() || \ - lockdep_is_held(&mce_read_mutex)) + lockdep_is_held(&mce_chrdev_read_mutex)) #define CREATE_TRACE_POINTS #include @@ -90,7 +90,8 @@ static unsigned long mce_need_notify; static char mce_helper[128]; static char *mce_helper_argv[2] = { mce_helper, NULL }; -static DECLARE_WAIT_QUEUE_HEAD(mce_wait); +static DECLARE_WAIT_QUEUE_HEAD(mce_chrdev_wait); + static DEFINE_PER_CPU(struct mce, mces_seen); static int cpu_missing; @@ -1159,7 +1160,8 @@ int mce_notify_irq(void) clear_thread_flag(TIF_MCE_NOTIFY); if (test_and_clear_bit(0, &mce_need_notify)) { - wake_up_interruptible(&mce_wait); + /* wake processes polling /dev/mcelog */ + wake_up_interruptible(&mce_chrdev_wait); /* * There is no risk of missing notifications because @@ -1423,40 +1425,41 @@ void __cpuinit mcheck_cpu_init(struct cpuinfo_x86 *c) } /* - * Character device to read and clear the MCE log. + * mce_chrdev: Character device /dev/mcelog to read and clear the MCE log. */ -static DEFINE_SPINLOCK(mce_state_lock); -static int open_count; /* #times opened */ -static int open_exclu; /* already open exclusive? */ +static DEFINE_SPINLOCK(mce_chrdev_state_lock); +static int mce_chrdev_open_count; /* #times opened */ +static int mce_chrdev_open_exclu; /* already open exclusive? */ -static int mce_open(struct inode *inode, struct file *file) +static int mce_chrdev_open(struct inode *inode, struct file *file) { - spin_lock(&mce_state_lock); + spin_lock(&mce_chrdev_state_lock); - if (open_exclu || (open_count && (file->f_flags & O_EXCL))) { - spin_unlock(&mce_state_lock); + if (mce_chrdev_open_exclu || + (mce_chrdev_open_count && (file->f_flags & O_EXCL))) { + spin_unlock(&mce_chrdev_state_lock); return -EBUSY; } if (file->f_flags & O_EXCL) - open_exclu = 1; - open_count++; + mce_chrdev_open_exclu = 1; + mce_chrdev_open_count++; - spin_unlock(&mce_state_lock); + spin_unlock(&mce_chrdev_state_lock); return nonseekable_open(inode, file); } -static int mce_release(struct inode *inode, struct file *file) +static int mce_chrdev_release(struct inode *inode, struct file *file) { - spin_lock(&mce_state_lock); + spin_lock(&mce_chrdev_state_lock); - open_count--; - open_exclu = 0; + mce_chrdev_open_count--; + mce_chrdev_open_exclu = 0; - spin_unlock(&mce_state_lock); + spin_unlock(&mce_chrdev_state_lock); return 0; } @@ -1505,8 +1508,8 @@ static int __mce_read_apei(char __user **ubuf, size_t usize) return 0; } -static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize, - loff_t *off) +static ssize_t mce_chrdev_read(struct file *filp, char __user *ubuf, + size_t usize, loff_t *off) { char __user *buf = ubuf; unsigned long *cpu_tsc; @@ -1517,7 +1520,7 @@ static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize, if (!cpu_tsc) return -ENOMEM; - mutex_lock(&mce_read_mutex); + mutex_lock(&mce_chrdev_read_mutex); if (!mce_apei_read_done) { err = __mce_read_apei(&buf, usize); @@ -1582,15 +1585,15 @@ timeout: err = -EFAULT; out: - mutex_unlock(&mce_read_mutex); + mutex_unlock(&mce_chrdev_read_mutex); kfree(cpu_tsc); return err ? err : buf - ubuf; } -static unsigned int mce_poll(struct file *file, poll_table *wait) +static unsigned int mce_chrdev_poll(struct file *file, poll_table *wait) { - poll_wait(file, &mce_wait, wait); + poll_wait(file, &mce_chrdev_wait, wait); if (rcu_access_index(mcelog.next)) return POLLIN | POLLRDNORM; if (!mce_apei_read_done && apei_check_mce()) @@ -1598,7 +1601,8 @@ static unsigned int mce_poll(struct file *file, poll_table *wait) return 0; } -static long mce_ioctl(struct file *f, unsigned int cmd, unsigned long arg) +static long mce_chrdev_ioctl(struct file *f, unsigned int cmd, + unsigned long arg) { int __user *p = (int __user *)arg; @@ -1626,16 +1630,16 @@ static long mce_ioctl(struct file *f, unsigned int cmd, unsigned long arg) /* Modified in mce-inject.c, so not static or const */ struct file_operations mce_chrdev_ops = { - .open = mce_open, - .release = mce_release, - .read = mce_read, - .poll = mce_poll, - .unlocked_ioctl = mce_ioctl, - .llseek = no_llseek, + .open = mce_chrdev_open, + .release = mce_chrdev_release, + .read = mce_chrdev_read, + .poll = mce_chrdev_poll, + .unlocked_ioctl = mce_chrdev_ioctl, + .llseek = no_llseek, }; EXPORT_SYMBOL_GPL(mce_chrdev_ops); -static struct miscdevice mce_log_device = { +static struct miscdevice mce_chrdev_device = { MISC_MCELOG_MINOR, "mcelog", &mce_chrdev_ops, @@ -2107,11 +2111,12 @@ static __init int mcheck_init_device(void) register_syscore_ops(&mce_syscore_ops); register_hotcpu_notifier(&mce_cpu_notifier); - misc_register(&mce_log_device); + + /* register character device /dev/mcelog */ + misc_register(&mce_chrdev_device); return err; } - device_initcall(mcheck_init_device); /* -- cgit v1.2.3 From c7cece89f1b00b56276303942f96ec67cf206e1e Mon Sep 17 00:00:00 2001 From: Hidetoshi Seto Date: Wed, 8 Jun 2011 11:02:03 +0900 Subject: x86, mce: Use mce_sysdev_ prefix to group functions There are many functions named mce_* so use a new prefix for the subset of functions related to sysfs support. And since f3c6ea1b06c71b43f751b36bd99345369fe911af introduces syscore_ops, use the prefix mce_syscore for some functions related to power management which were in sysdev_class before. Before: After: mce_device mce_sysdev mce_sysclass mce_sysdev_class mce_attrs mce_sysdev_attrs mce_dev_initialized mce_sysdev_initialized mce_create_device mce_sysdev_create mce_remove_device mce_sysdev_remove mce_suspend mce_syscore_suspend mce_shutdown mce_syscore_shutdown mce_resume mce_syscore_resume Signed-off-by: Hidetoshi Seto Acked-by: Tony Luck Link: http://lkml.kernel.org/r/4DEED81B.8020506@jp.fujitsu.com Signed-off-by: Borislav Petkov --- arch/x86/include/asm/mce.h | 2 +- arch/x86/kernel/cpu/mcheck/mce.c | 62 +++++++++++++++++++----------------- arch/x86/kernel/cpu/mcheck/mce_amd.c | 10 +++--- 3 files changed, 39 insertions(+), 35 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index bbb328fd117b..716b48af7863 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -149,7 +149,7 @@ static inline void enable_p5_mce(void) {} void mce_setup(struct mce *m); void mce_log(struct mce *m); -DECLARE_PER_CPU(struct sys_device, mce_dev); +DECLARE_PER_CPU(struct sys_device, mce_sysdev); /* * Maximum banks number. diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 9a735168e40f..08363b042122 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -1697,7 +1697,7 @@ int __init mcheck_init(void) } /* - * Sysfs support + * mce_syscore: PM support */ /* @@ -1717,12 +1717,12 @@ static int mce_disable_error_reporting(void) return 0; } -static int mce_suspend(void) +static int mce_syscore_suspend(void) { return mce_disable_error_reporting(); } -static void mce_shutdown(void) +static void mce_syscore_shutdown(void) { mce_disable_error_reporting(); } @@ -1732,18 +1732,22 @@ static void mce_shutdown(void) * Only one CPU is active at this time, the others get re-added later using * CPU hotplug: */ -static void mce_resume(void) +static void mce_syscore_resume(void) { __mcheck_cpu_init_generic(); __mcheck_cpu_init_vendor(__this_cpu_ptr(&cpu_info)); } static struct syscore_ops mce_syscore_ops = { - .suspend = mce_suspend, - .shutdown = mce_shutdown, - .resume = mce_resume, + .suspend = mce_syscore_suspend, + .shutdown = mce_syscore_shutdown, + .resume = mce_syscore_resume, }; +/* + * mce_sysdev: Sysfs support + */ + static void mce_cpu_restart(void *data) { del_timer_sync(&__get_cpu_var(mce_timer)); @@ -1779,11 +1783,11 @@ static void mce_enable_ce(void *all) __mcheck_cpu_init_timer(); } -static struct sysdev_class mce_sysclass = { +static struct sysdev_class mce_sysdev_class = { .name = "machinecheck", }; -DEFINE_PER_CPU(struct sys_device, mce_dev); +DEFINE_PER_CPU(struct sys_device, mce_sysdev); __cpuinitdata void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu); @@ -1912,7 +1916,7 @@ static struct sysdev_ext_attribute attr_cmci_disabled = { &mce_cmci_disabled }; -static struct sysdev_attribute *mce_attrs[] = { +static struct sysdev_attribute *mce_sysdev_attrs[] = { &attr_tolerant.attr, &attr_check_interval.attr, &attr_trigger, @@ -1923,12 +1927,12 @@ static struct sysdev_attribute *mce_attrs[] = { NULL }; -static cpumask_var_t mce_dev_initialized; +static cpumask_var_t mce_sysdev_initialized; /* Per cpu sysdev init. All of the cpus still share the same ctrl bank: */ -static __cpuinit int mce_create_device(unsigned int cpu) +static __cpuinit int mce_sysdev_create(unsigned int cpu) { - struct sys_device *sysdev = &per_cpu(mce_dev, cpu); + struct sys_device *sysdev = &per_cpu(mce_sysdev, cpu); int err; int i, j; @@ -1937,14 +1941,14 @@ static __cpuinit int mce_create_device(unsigned int cpu) memset(&sysdev->kobj, 0, sizeof(struct kobject)); sysdev->id = cpu; - sysdev->cls = &mce_sysclass; + sysdev->cls = &mce_sysdev_class; err = sysdev_register(sysdev); if (err) return err; - for (i = 0; mce_attrs[i]; i++) { - err = sysdev_create_file(sysdev, mce_attrs[i]); + for (i = 0; mce_sysdev_attrs[i]; i++) { + err = sysdev_create_file(sysdev, mce_sysdev_attrs[i]); if (err) goto error; } @@ -1953,7 +1957,7 @@ static __cpuinit int mce_create_device(unsigned int cpu) if (err) goto error2; } - cpumask_set_cpu(cpu, mce_dev_initialized); + cpumask_set_cpu(cpu, mce_sysdev_initialized); return 0; error2: @@ -1961,29 +1965,29 @@ error2: sysdev_remove_file(sysdev, &mce_banks[j].attr); error: while (--i >= 0) - sysdev_remove_file(sysdev, mce_attrs[i]); + sysdev_remove_file(sysdev, mce_sysdev_attrs[i]); sysdev_unregister(sysdev); return err; } -static __cpuinit void mce_remove_device(unsigned int cpu) +static __cpuinit void mce_sysdev_remove(unsigned int cpu) { - struct sys_device *sysdev = &per_cpu(mce_dev, cpu); + struct sys_device *sysdev = &per_cpu(mce_sysdev, cpu); int i; - if (!cpumask_test_cpu(cpu, mce_dev_initialized)) + if (!cpumask_test_cpu(cpu, mce_sysdev_initialized)) return; - for (i = 0; mce_attrs[i]; i++) - sysdev_remove_file(sysdev, mce_attrs[i]); + for (i = 0; mce_sysdev_attrs[i]; i++) + sysdev_remove_file(sysdev, mce_sysdev_attrs[i]); for (i = 0; i < banks; i++) sysdev_remove_file(sysdev, &mce_banks[i].attr); sysdev_unregister(sysdev); - cpumask_clear_cpu(cpu, mce_dev_initialized); + cpumask_clear_cpu(cpu, mce_sysdev_initialized); } /* Make sure there are no machine checks on offlined CPUs. */ @@ -2033,7 +2037,7 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) switch (action) { case CPU_ONLINE: case CPU_ONLINE_FROZEN: - mce_create_device(cpu); + mce_sysdev_create(cpu); if (threshold_cpu_callback) threshold_cpu_callback(action, cpu); break; @@ -2041,7 +2045,7 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) case CPU_DEAD_FROZEN: if (threshold_cpu_callback) threshold_cpu_callback(action, cpu); - mce_remove_device(cpu); + mce_sysdev_remove(cpu); break; case CPU_DOWN_PREPARE: case CPU_DOWN_PREPARE_FROZEN: @@ -2095,16 +2099,16 @@ static __init int mcheck_init_device(void) if (!mce_available(&boot_cpu_data)) return -EIO; - zalloc_cpumask_var(&mce_dev_initialized, GFP_KERNEL); + zalloc_cpumask_var(&mce_sysdev_initialized, GFP_KERNEL); mce_init_banks(); - err = sysdev_class_register(&mce_sysclass); + err = sysdev_class_register(&mce_sysdev_class); if (err) return err; for_each_online_cpu(i) { - err = mce_create_device(i); + err = mce_sysdev_create(i); if (err) return err; } diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c index bb0adad35143..f5474218cffe 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c @@ -548,7 +548,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) if (!b) goto out; - err = sysfs_create_link(&per_cpu(mce_dev, cpu).kobj, + err = sysfs_create_link(&per_cpu(mce_sysdev, cpu).kobj, b->kobj, name); if (err) goto out; @@ -571,7 +571,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) goto out; } - b->kobj = kobject_create_and_add(name, &per_cpu(mce_dev, cpu).kobj); + b->kobj = kobject_create_and_add(name, &per_cpu(mce_sysdev, cpu).kobj); if (!b->kobj) goto out_free; @@ -591,7 +591,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) if (i == cpu) continue; - err = sysfs_create_link(&per_cpu(mce_dev, i).kobj, + err = sysfs_create_link(&per_cpu(mce_sysdev, i).kobj, b->kobj, name); if (err) goto out; @@ -669,7 +669,7 @@ static void threshold_remove_bank(unsigned int cpu, int bank) #ifdef CONFIG_SMP /* sibling symlink */ if (shared_bank[bank] && b->blocks->cpu != cpu) { - sysfs_remove_link(&per_cpu(mce_dev, cpu).kobj, name); + sysfs_remove_link(&per_cpu(mce_sysdev, cpu).kobj, name); per_cpu(threshold_banks, cpu)[bank] = NULL; return; @@ -681,7 +681,7 @@ static void threshold_remove_bank(unsigned int cpu, int bank) if (i == cpu) continue; - sysfs_remove_link(&per_cpu(mce_dev, i).kobj, name); + sysfs_remove_link(&per_cpu(mce_sysdev, i).kobj, name); per_cpu(threshold_banks, i)[bank] = NULL; } -- cgit v1.2.3 From 40b7f3dfcc5ab211a0b8d916751bb22ac2290806 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Wed, 15 Jun 2011 15:34:57 +0200 Subject: x86, microcode, AMD: Fix section header size check The ucode size check has to take the section header size into account too when sanity checking the section length. Shorten and clarify define names, while at it. Caught-by: Ben Hutchings Link: http://lkml.kernel.org/r/1302752223.5282.674.camel@localhost Signed-off-by: Borislav Petkov --- arch/x86/kernel/microcode_amd.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/microcode_amd.c b/arch/x86/kernel/microcode_amd.c index d30d67cd33ad..591be0ee1934 100644 --- a/arch/x86/kernel/microcode_amd.c +++ b/arch/x86/kernel/microcode_amd.c @@ -66,8 +66,8 @@ struct microcode_amd { unsigned int mpb[0]; }; -#define UCODE_CONTAINER_SECTION_HDR 8 -#define UCODE_CONTAINER_HEADER_SIZE 12 +#define SECTION_HDR_SIZE 8 +#define CONTAINER_HDR_SZ 12 static struct equiv_cpu_entry *equiv_cpu_table; @@ -177,7 +177,7 @@ static unsigned int verify_ucode_size(int cpu, const u8 *buf, unsigned int size) actual_size = *(u32 *)(buf + 4); - if (actual_size > size || actual_size > max_size) { + if (actual_size + SECTION_HDR_SIZE > size || actual_size > max_size) { pr_err("section size mismatch\n"); return 0; } @@ -204,8 +204,8 @@ get_next_ucode(int cpu, const u8 *buf, unsigned int size, unsigned int *mc_size) if (!mc) goto out; - get_ucode_data(mc, buf + UCODE_CONTAINER_SECTION_HDR, actual_size); - *mc_size = actual_size + UCODE_CONTAINER_SECTION_HDR; + get_ucode_data(mc, buf + SECTION_HDR_SIZE, actual_size); + *mc_size = actual_size + SECTION_HDR_SIZE; out: return mc; @@ -229,9 +229,10 @@ static int install_equiv_cpu_table(const u8 *buf) return -ENOMEM; } - get_ucode_data(equiv_cpu_table, buf + UCODE_CONTAINER_HEADER_SIZE, size); + get_ucode_data(equiv_cpu_table, buf + CONTAINER_HDR_SZ, size); - return size + UCODE_CONTAINER_HEADER_SIZE; /* add header length */ + /* add header length */ + return size + CONTAINER_HDR_SZ; } static void free_equiv_cpu_table(void) -- cgit v1.2.3 From 29b68415e335ba9e0eb6057f9405aa4d9c23efe4 Mon Sep 17 00:00:00 2001 From: Ohad Ben-Cohen Date: Sun, 5 Jun 2011 18:22:18 +0300 Subject: x86: amd_iommu: move to drivers/iommu/ This should ease finding similarities with different platforms, with the intention of solving problems once in a generic framework which everyone can use. Compile-tested on x86_64. Signed-off-by: Ohad Ben-Cohen Signed-off-by: Joerg Roedel --- arch/x86/Kconfig | 28 - arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/amd_iommu.c | 2764 ------------------------------------------- drivers/iommu/Kconfig | 29 + drivers/iommu/Makefile | 1 + drivers/iommu/amd_iommu.c | 2764 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 2795 insertions(+), 2793 deletions(-) delete mode 100644 arch/x86/kernel/amd_iommu.c create mode 100644 drivers/iommu/amd_iommu.c (limited to 'arch/x86/kernel') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 460d57370016..1b6a2e212db4 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -680,34 +680,6 @@ config CALGARY_IOMMU_ENABLED_BY_DEFAULT Calgary anyway, pass 'iommu=calgary' on the kernel command line. If unsure, say Y. -config AMD_IOMMU - bool "AMD IOMMU support" - select SWIOTLB - select PCI_MSI - select PCI_IOV - select IOMMU_API - depends on X86_64 && PCI && ACPI - ---help--- - With this option you can enable support for AMD IOMMU hardware in - your system. An IOMMU is a hardware component which provides - remapping of DMA memory accesses from devices. With an AMD IOMMU you - can isolate the the DMA memory of different devices and protect the - system from misbehaving device drivers or hardware. - - You can find out if your system has an AMD IOMMU if you look into - your BIOS for an option to enable it or if you have an IVRS ACPI - table. - -config AMD_IOMMU_STATS - bool "Export AMD IOMMU statistics to debugfs" - depends on AMD_IOMMU - select DEBUG_FS - ---help--- - This option enables code in the AMD IOMMU driver to collect various - statistics about whats happening in the driver and exports that - information to userspace via debugfs. - If unsure, say N. - # need this always selected by IOMMU for the VIA workaround config SWIOTLB def_bool y if X86_64 diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 90b06d4daee2..aef02989c930 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -123,7 +123,7 @@ ifeq ($(CONFIG_X86_64),y) obj-$(CONFIG_GART_IOMMU) += amd_gart_64.o aperture_64.o obj-$(CONFIG_CALGARY_IOMMU) += pci-calgary_64.o tce_64.o - obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o amd_iommu.o + obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o obj-y += vsmp_64.o diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c deleted file mode 100644 index 7c3a95e54ec5..000000000000 --- a/arch/x86/kernel/amd_iommu.c +++ /dev/null @@ -1,2764 +0,0 @@ -/* - * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. - * Author: Joerg Roedel - * Leo Duran - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) - -#define LOOP_TIMEOUT 100000 - -static DEFINE_RWLOCK(amd_iommu_devtable_lock); - -/* A list of preallocated protection domains */ -static LIST_HEAD(iommu_pd_list); -static DEFINE_SPINLOCK(iommu_pd_list_lock); - -/* - * Domain for untranslated devices - only allocated - * if iommu=pt passed on kernel cmd line. - */ -static struct protection_domain *pt_domain; - -static struct iommu_ops amd_iommu_ops; - -/* - * general struct to manage commands send to an IOMMU - */ -struct iommu_cmd { - u32 data[4]; -}; - -static void update_domain(struct protection_domain *domain); - -/**************************************************************************** - * - * Helper functions - * - ****************************************************************************/ - -static inline u16 get_device_id(struct device *dev) -{ - struct pci_dev *pdev = to_pci_dev(dev); - - return calc_devid(pdev->bus->number, pdev->devfn); -} - -static struct iommu_dev_data *get_dev_data(struct device *dev) -{ - return dev->archdata.iommu; -} - -/* - * In this function the list of preallocated protection domains is traversed to - * find the domain for a specific device - */ -static struct dma_ops_domain *find_protection_domain(u16 devid) -{ - struct dma_ops_domain *entry, *ret = NULL; - unsigned long flags; - u16 alias = amd_iommu_alias_table[devid]; - - if (list_empty(&iommu_pd_list)) - return NULL; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); - - list_for_each_entry(entry, &iommu_pd_list, list) { - if (entry->target_dev == devid || - entry->target_dev == alias) { - ret = entry; - break; - } - } - - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - - return ret; -} - -/* - * This function checks if the driver got a valid device from the caller to - * avoid dereferencing invalid pointers. - */ -static bool check_device(struct device *dev) -{ - u16 devid; - - if (!dev || !dev->dma_mask) - return false; - - /* No device or no PCI device */ - if (dev->bus != &pci_bus_type) - return false; - - devid = get_device_id(dev); - - /* Out of our scope? */ - if (devid > amd_iommu_last_bdf) - return false; - - if (amd_iommu_rlookup_table[devid] == NULL) - return false; - - return true; -} - -static int iommu_init_device(struct device *dev) -{ - struct iommu_dev_data *dev_data; - struct pci_dev *pdev; - u16 devid, alias; - - if (dev->archdata.iommu) - return 0; - - dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); - if (!dev_data) - return -ENOMEM; - - dev_data->dev = dev; - - devid = get_device_id(dev); - alias = amd_iommu_alias_table[devid]; - pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); - if (pdev) - dev_data->alias = &pdev->dev; - else { - kfree(dev_data); - return -ENOTSUPP; - } - - atomic_set(&dev_data->bind, 0); - - dev->archdata.iommu = dev_data; - - - return 0; -} - -static void iommu_ignore_device(struct device *dev) -{ - u16 devid, alias; - - devid = get_device_id(dev); - alias = amd_iommu_alias_table[devid]; - - memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry)); - memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry)); - - amd_iommu_rlookup_table[devid] = NULL; - amd_iommu_rlookup_table[alias] = NULL; -} - -static void iommu_uninit_device(struct device *dev) -{ - kfree(dev->archdata.iommu); -} - -void __init amd_iommu_uninit_devices(void) -{ - struct pci_dev *pdev = NULL; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - iommu_uninit_device(&pdev->dev); - } -} - -int __init amd_iommu_init_devices(void) -{ - struct pci_dev *pdev = NULL; - int ret = 0; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - ret = iommu_init_device(&pdev->dev); - if (ret == -ENOTSUPP) - iommu_ignore_device(&pdev->dev); - else if (ret) - goto out_free; - } - - return 0; - -out_free: - - amd_iommu_uninit_devices(); - - return ret; -} -#ifdef CONFIG_AMD_IOMMU_STATS - -/* - * Initialization code for statistics collection - */ - -DECLARE_STATS_COUNTER(compl_wait); -DECLARE_STATS_COUNTER(cnt_map_single); -DECLARE_STATS_COUNTER(cnt_unmap_single); -DECLARE_STATS_COUNTER(cnt_map_sg); -DECLARE_STATS_COUNTER(cnt_unmap_sg); -DECLARE_STATS_COUNTER(cnt_alloc_coherent); -DECLARE_STATS_COUNTER(cnt_free_coherent); -DECLARE_STATS_COUNTER(cross_page); -DECLARE_STATS_COUNTER(domain_flush_single); -DECLARE_STATS_COUNTER(domain_flush_all); -DECLARE_STATS_COUNTER(alloced_io_mem); -DECLARE_STATS_COUNTER(total_map_requests); - -static struct dentry *stats_dir; -static struct dentry *de_fflush; - -static void amd_iommu_stats_add(struct __iommu_counter *cnt) -{ - if (stats_dir == NULL) - return; - - cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir, - &cnt->value); -} - -static void amd_iommu_stats_init(void) -{ - stats_dir = debugfs_create_dir("amd-iommu", NULL); - if (stats_dir == NULL) - return; - - de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir, - (u32 *)&amd_iommu_unmap_flush); - - amd_iommu_stats_add(&compl_wait); - amd_iommu_stats_add(&cnt_map_single); - amd_iommu_stats_add(&cnt_unmap_single); - amd_iommu_stats_add(&cnt_map_sg); - amd_iommu_stats_add(&cnt_unmap_sg); - amd_iommu_stats_add(&cnt_alloc_coherent); - amd_iommu_stats_add(&cnt_free_coherent); - amd_iommu_stats_add(&cross_page); - amd_iommu_stats_add(&domain_flush_single); - amd_iommu_stats_add(&domain_flush_all); - amd_iommu_stats_add(&alloced_io_mem); - amd_iommu_stats_add(&total_map_requests); -} - -#endif - -/**************************************************************************** - * - * Interrupt handling functions - * - ****************************************************************************/ - -static void dump_dte_entry(u16 devid) -{ - int i; - - for (i = 0; i < 8; ++i) - pr_err("AMD-Vi: DTE[%d]: %08x\n", i, - amd_iommu_dev_table[devid].data[i]); -} - -static void dump_command(unsigned long phys_addr) -{ - struct iommu_cmd *cmd = phys_to_virt(phys_addr); - int i; - - for (i = 0; i < 4; ++i) - pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]); -} - -static void iommu_print_event(struct amd_iommu *iommu, void *__evt) -{ - u32 *event = __evt; - int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK; - int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; - int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK; - int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; - u64 address = (u64)(((u64)event[3]) << 32) | event[2]; - - printk(KERN_ERR "AMD-Vi: Event logged ["); - - switch (type) { - case EVENT_TYPE_ILL_DEV: - printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x " - "address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), - address, flags); - dump_dte_entry(devid); - break; - case EVENT_TYPE_IO_FAULT: - printk("IO_PAGE_FAULT device=%02x:%02x.%x " - "domain=0x%04x address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), - domid, address, flags); - break; - case EVENT_TYPE_DEV_TAB_ERR: - printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " - "address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), - address, flags); - break; - case EVENT_TYPE_PAGE_TAB_ERR: - printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x " - "domain=0x%04x address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), - domid, address, flags); - break; - case EVENT_TYPE_ILL_CMD: - printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); - dump_command(address); - break; - case EVENT_TYPE_CMD_HARD_ERR: - printk("COMMAND_HARDWARE_ERROR address=0x%016llx " - "flags=0x%04x]\n", address, flags); - break; - case EVENT_TYPE_IOTLB_INV_TO: - printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x " - "address=0x%016llx]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), - address); - break; - case EVENT_TYPE_INV_DEV_REQ: - printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x " - "address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), - address, flags); - break; - default: - printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type); - } -} - -static void iommu_poll_events(struct amd_iommu *iommu) -{ - u32 head, tail; - unsigned long flags; - - spin_lock_irqsave(&iommu->lock, flags); - - head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); - tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); - - while (head != tail) { - iommu_print_event(iommu, iommu->evt_buf + head); - head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size; - } - - writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); - - spin_unlock_irqrestore(&iommu->lock, flags); -} - -irqreturn_t amd_iommu_int_thread(int irq, void *data) -{ - struct amd_iommu *iommu; - - for_each_iommu(iommu) - iommu_poll_events(iommu); - - return IRQ_HANDLED; -} - -irqreturn_t amd_iommu_int_handler(int irq, void *data) -{ - return IRQ_WAKE_THREAD; -} - -/**************************************************************************** - * - * IOMMU command queuing functions - * - ****************************************************************************/ - -static int wait_on_sem(volatile u64 *sem) -{ - int i = 0; - - while (*sem == 0 && i < LOOP_TIMEOUT) { - udelay(1); - i += 1; - } - - if (i == LOOP_TIMEOUT) { - pr_alert("AMD-Vi: Completion-Wait loop timed out\n"); - return -EIO; - } - - return 0; -} - -static void copy_cmd_to_buffer(struct amd_iommu *iommu, - struct iommu_cmd *cmd, - u32 tail) -{ - u8 *target; - - target = iommu->cmd_buf + tail; - tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size; - - /* Copy command to buffer */ - memcpy(target, cmd, sizeof(*cmd)); - - /* Tell the IOMMU about it */ - writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); -} - -static void build_completion_wait(struct iommu_cmd *cmd, u64 address) -{ - WARN_ON(address & 0x7ULL); - - memset(cmd, 0, sizeof(*cmd)); - cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK; - cmd->data[1] = upper_32_bits(__pa(address)); - cmd->data[2] = 1; - CMD_SET_TYPE(cmd, CMD_COMPL_WAIT); -} - -static void build_inv_dte(struct iommu_cmd *cmd, u16 devid) -{ - memset(cmd, 0, sizeof(*cmd)); - cmd->data[0] = devid; - CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY); -} - -static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, - size_t size, u16 domid, int pde) -{ - u64 pages; - int s; - - pages = iommu_num_pages(address, size, PAGE_SIZE); - s = 0; - - if (pages > 1) { - /* - * If we have to flush more than one page, flush all - * TLB entries for this domain - */ - address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; - s = 1; - } - - address &= PAGE_MASK; - - memset(cmd, 0, sizeof(*cmd)); - cmd->data[1] |= domid; - cmd->data[2] = lower_32_bits(address); - cmd->data[3] = upper_32_bits(address); - CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES); - if (s) /* size bit - we flush more than one 4kb page */ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; - if (pde) /* PDE bit - we wan't flush everything not only the PTEs */ - cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK; -} - -static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, - u64 address, size_t size) -{ - u64 pages; - int s; - - pages = iommu_num_pages(address, size, PAGE_SIZE); - s = 0; - - if (pages > 1) { - /* - * If we have to flush more than one page, flush all - * TLB entries for this domain - */ - address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; - s = 1; - } - - address &= PAGE_MASK; - - memset(cmd, 0, sizeof(*cmd)); - cmd->data[0] = devid; - cmd->data[0] |= (qdep & 0xff) << 24; - cmd->data[1] = devid; - cmd->data[2] = lower_32_bits(address); - cmd->data[3] = upper_32_bits(address); - CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES); - if (s) - cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; -} - -static void build_inv_all(struct iommu_cmd *cmd) -{ - memset(cmd, 0, sizeof(*cmd)); - CMD_SET_TYPE(cmd, CMD_INV_ALL); -} - -/* - * Writes the command to the IOMMUs command buffer and informs the - * hardware about the new command. - */ -static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) -{ - u32 left, tail, head, next_tail; - unsigned long flags; - - WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED); - -again: - spin_lock_irqsave(&iommu->lock, flags); - - head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); - tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); - next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size; - left = (head - next_tail) % iommu->cmd_buf_size; - - if (left <= 2) { - struct iommu_cmd sync_cmd; - volatile u64 sem = 0; - int ret; - - build_completion_wait(&sync_cmd, (u64)&sem); - copy_cmd_to_buffer(iommu, &sync_cmd, tail); - - spin_unlock_irqrestore(&iommu->lock, flags); - - if ((ret = wait_on_sem(&sem)) != 0) - return ret; - - goto again; - } - - copy_cmd_to_buffer(iommu, cmd, tail); - - /* We need to sync now to make sure all commands are processed */ - iommu->need_sync = true; - - spin_unlock_irqrestore(&iommu->lock, flags); - - return 0; -} - -/* - * This function queues a completion wait command into the command - * buffer of an IOMMU - */ -static int iommu_completion_wait(struct amd_iommu *iommu) -{ - struct iommu_cmd cmd; - volatile u64 sem = 0; - int ret; - - if (!iommu->need_sync) - return 0; - - build_completion_wait(&cmd, (u64)&sem); - - ret = iommu_queue_command(iommu, &cmd); - if (ret) - return ret; - - return wait_on_sem(&sem); -} - -static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid) -{ - struct iommu_cmd cmd; - - build_inv_dte(&cmd, devid); - - return iommu_queue_command(iommu, &cmd); -} - -static void iommu_flush_dte_all(struct amd_iommu *iommu) -{ - u32 devid; - - for (devid = 0; devid <= 0xffff; ++devid) - iommu_flush_dte(iommu, devid); - - iommu_completion_wait(iommu); -} - -/* - * This function uses heavy locking and may disable irqs for some time. But - * this is no issue because it is only called during resume. - */ -static void iommu_flush_tlb_all(struct amd_iommu *iommu) -{ - u32 dom_id; - - for (dom_id = 0; dom_id <= 0xffff; ++dom_id) { - struct iommu_cmd cmd; - build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, - dom_id, 1); - iommu_queue_command(iommu, &cmd); - } - - iommu_completion_wait(iommu); -} - -static void iommu_flush_all(struct amd_iommu *iommu) -{ - struct iommu_cmd cmd; - - build_inv_all(&cmd); - - iommu_queue_command(iommu, &cmd); - iommu_completion_wait(iommu); -} - -void iommu_flush_all_caches(struct amd_iommu *iommu) -{ - if (iommu_feature(iommu, FEATURE_IA)) { - iommu_flush_all(iommu); - } else { - iommu_flush_dte_all(iommu); - iommu_flush_tlb_all(iommu); - } -} - -/* - * Command send function for flushing on-device TLB - */ -static int device_flush_iotlb(struct device *dev, u64 address, size_t size) -{ - struct pci_dev *pdev = to_pci_dev(dev); - struct amd_iommu *iommu; - struct iommu_cmd cmd; - u16 devid; - int qdep; - - qdep = pci_ats_queue_depth(pdev); - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; - - build_inv_iotlb_pages(&cmd, devid, qdep, address, size); - - return iommu_queue_command(iommu, &cmd); -} - -/* - * Command send function for invalidating a device table entry - */ -static int device_flush_dte(struct device *dev) -{ - struct amd_iommu *iommu; - struct pci_dev *pdev; - u16 devid; - int ret; - - pdev = to_pci_dev(dev); - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; - - ret = iommu_flush_dte(iommu, devid); - if (ret) - return ret; - - if (pci_ats_enabled(pdev)) - ret = device_flush_iotlb(dev, 0, ~0UL); - - return ret; -} - -/* - * TLB invalidation function which is called from the mapping functions. - * It invalidates a single PTE if the range to flush is within a single - * page. Otherwise it flushes the whole TLB of the IOMMU. - */ -static void __domain_flush_pages(struct protection_domain *domain, - u64 address, size_t size, int pde) -{ - struct iommu_dev_data *dev_data; - struct iommu_cmd cmd; - int ret = 0, i; - - build_inv_iommu_pages(&cmd, address, size, domain->id, pde); - - for (i = 0; i < amd_iommus_present; ++i) { - if (!domain->dev_iommu[i]) - continue; - - /* - * Devices of this domain are behind this IOMMU - * We need a TLB flush - */ - ret |= iommu_queue_command(amd_iommus[i], &cmd); - } - - list_for_each_entry(dev_data, &domain->dev_list, list) { - struct pci_dev *pdev = to_pci_dev(dev_data->dev); - - if (!pci_ats_enabled(pdev)) - continue; - - ret |= device_flush_iotlb(dev_data->dev, address, size); - } - - WARN_ON(ret); -} - -static void domain_flush_pages(struct protection_domain *domain, - u64 address, size_t size) -{ - __domain_flush_pages(domain, address, size, 0); -} - -/* Flush the whole IO/TLB for a given protection domain */ -static void domain_flush_tlb(struct protection_domain *domain) -{ - __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0); -} - -/* Flush the whole IO/TLB for a given protection domain - including PDE */ -static void domain_flush_tlb_pde(struct protection_domain *domain) -{ - __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1); -} - -static void domain_flush_complete(struct protection_domain *domain) -{ - int i; - - for (i = 0; i < amd_iommus_present; ++i) { - if (!domain->dev_iommu[i]) - continue; - - /* - * Devices of this domain are behind this IOMMU - * We need to wait for completion of all commands. - */ - iommu_completion_wait(amd_iommus[i]); - } -} - - -/* - * This function flushes the DTEs for all devices in domain - */ -static void domain_flush_devices(struct protection_domain *domain) -{ - struct iommu_dev_data *dev_data; - unsigned long flags; - - spin_lock_irqsave(&domain->lock, flags); - - list_for_each_entry(dev_data, &domain->dev_list, list) - device_flush_dte(dev_data->dev); - - spin_unlock_irqrestore(&domain->lock, flags); -} - -/**************************************************************************** - * - * The functions below are used the create the page table mappings for - * unity mapped regions. - * - ****************************************************************************/ - -/* - * This function is used to add another level to an IO page table. Adding - * another level increases the size of the address space by 9 bits to a size up - * to 64 bits. - */ -static bool increase_address_space(struct protection_domain *domain, - gfp_t gfp) -{ - u64 *pte; - - if (domain->mode == PAGE_MODE_6_LEVEL) - /* address space already 64 bit large */ - return false; - - pte = (void *)get_zeroed_page(gfp); - if (!pte) - return false; - - *pte = PM_LEVEL_PDE(domain->mode, - virt_to_phys(domain->pt_root)); - domain->pt_root = pte; - domain->mode += 1; - domain->updated = true; - - return true; -} - -static u64 *alloc_pte(struct protection_domain *domain, - unsigned long address, - unsigned long page_size, - u64 **pte_page, - gfp_t gfp) -{ - int level, end_lvl; - u64 *pte, *page; - - BUG_ON(!is_power_of_2(page_size)); - - while (address > PM_LEVEL_SIZE(domain->mode)) - increase_address_space(domain, gfp); - - level = domain->mode - 1; - pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; - address = PAGE_SIZE_ALIGN(address, page_size); - end_lvl = PAGE_SIZE_LEVEL(page_size); - - while (level > end_lvl) { - if (!IOMMU_PTE_PRESENT(*pte)) { - page = (u64 *)get_zeroed_page(gfp); - if (!page) - return NULL; - *pte = PM_LEVEL_PDE(level, virt_to_phys(page)); - } - - /* No level skipping support yet */ - if (PM_PTE_LEVEL(*pte) != level) - return NULL; - - level -= 1; - - pte = IOMMU_PTE_PAGE(*pte); - - if (pte_page && level == end_lvl) - *pte_page = pte; - - pte = &pte[PM_LEVEL_INDEX(level, address)]; - } - - return pte; -} - -/* - * This function checks if there is a PTE for a given dma address. If - * there is one, it returns the pointer to it. - */ -static u64 *fetch_pte(struct protection_domain *domain, unsigned long address) -{ - int level; - u64 *pte; - - if (address > PM_LEVEL_SIZE(domain->mode)) - return NULL; - - level = domain->mode - 1; - pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; - - while (level > 0) { - - /* Not Present */ - if (!IOMMU_PTE_PRESENT(*pte)) - return NULL; - - /* Large PTE */ - if (PM_PTE_LEVEL(*pte) == 0x07) { - unsigned long pte_mask, __pte; - - /* - * If we have a series of large PTEs, make - * sure to return a pointer to the first one. - */ - pte_mask = PTE_PAGE_SIZE(*pte); - pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1); - __pte = ((unsigned long)pte) & pte_mask; - - return (u64 *)__pte; - } - - /* No level skipping support yet */ - if (PM_PTE_LEVEL(*pte) != level) - return NULL; - - level -= 1; - - /* Walk to the next level */ - pte = IOMMU_PTE_PAGE(*pte); - pte = &pte[PM_LEVEL_INDEX(level, address)]; - } - - return pte; -} - -/* - * Generic mapping functions. It maps a physical address into a DMA - * address space. It allocates the page table pages if necessary. - * In the future it can be extended to a generic mapping function - * supporting all features of AMD IOMMU page tables like level skipping - * and full 64 bit address spaces. - */ -static int iommu_map_page(struct protection_domain *dom, - unsigned long bus_addr, - unsigned long phys_addr, - int prot, - unsigned long page_size) -{ - u64 __pte, *pte; - int i, count; - - if (!(prot & IOMMU_PROT_MASK)) - return -EINVAL; - - bus_addr = PAGE_ALIGN(bus_addr); - phys_addr = PAGE_ALIGN(phys_addr); - count = PAGE_SIZE_PTE_COUNT(page_size); - pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL); - - for (i = 0; i < count; ++i) - if (IOMMU_PTE_PRESENT(pte[i])) - return -EBUSY; - - if (page_size > PAGE_SIZE) { - __pte = PAGE_SIZE_PTE(phys_addr, page_size); - __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC; - } else - __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC; - - if (prot & IOMMU_PROT_IR) - __pte |= IOMMU_PTE_IR; - if (prot & IOMMU_PROT_IW) - __pte |= IOMMU_PTE_IW; - - for (i = 0; i < count; ++i) - pte[i] = __pte; - - update_domain(dom); - - return 0; -} - -static unsigned long iommu_unmap_page(struct protection_domain *dom, - unsigned long bus_addr, - unsigned long page_size) -{ - unsigned long long unmap_size, unmapped; - u64 *pte; - - BUG_ON(!is_power_of_2(page_size)); - - unmapped = 0; - - while (unmapped < page_size) { - - pte = fetch_pte(dom, bus_addr); - - if (!pte) { - /* - * No PTE for this address - * move forward in 4kb steps - */ - unmap_size = PAGE_SIZE; - } else if (PM_PTE_LEVEL(*pte) == 0) { - /* 4kb PTE found for this address */ - unmap_size = PAGE_SIZE; - *pte = 0ULL; - } else { - int count, i; - - /* Large PTE found which maps this address */ - unmap_size = PTE_PAGE_SIZE(*pte); - count = PAGE_SIZE_PTE_COUNT(unmap_size); - for (i = 0; i < count; i++) - pte[i] = 0ULL; - } - - bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size; - unmapped += unmap_size; - } - - BUG_ON(!is_power_of_2(unmapped)); - - return unmapped; -} - -/* - * This function checks if a specific unity mapping entry is needed for - * this specific IOMMU. - */ -static int iommu_for_unity_map(struct amd_iommu *iommu, - struct unity_map_entry *entry) -{ - u16 bdf, i; - - for (i = entry->devid_start; i <= entry->devid_end; ++i) { - bdf = amd_iommu_alias_table[i]; - if (amd_iommu_rlookup_table[bdf] == iommu) - return 1; - } - - return 0; -} - -/* - * This function actually applies the mapping to the page table of the - * dma_ops domain. - */ -static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, - struct unity_map_entry *e) -{ - u64 addr; - int ret; - - for (addr = e->address_start; addr < e->address_end; - addr += PAGE_SIZE) { - ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot, - PAGE_SIZE); - if (ret) - return ret; - /* - * if unity mapping is in aperture range mark the page - * as allocated in the aperture - */ - if (addr < dma_dom->aperture_size) - __set_bit(addr >> PAGE_SHIFT, - dma_dom->aperture[0]->bitmap); - } - - return 0; -} - -/* - * Init the unity mappings for a specific IOMMU in the system - * - * Basically iterates over all unity mapping entries and applies them to - * the default domain DMA of that IOMMU if necessary. - */ -static int iommu_init_unity_mappings(struct amd_iommu *iommu) -{ - struct unity_map_entry *entry; - int ret; - - list_for_each_entry(entry, &amd_iommu_unity_map, list) { - if (!iommu_for_unity_map(iommu, entry)) - continue; - ret = dma_ops_unity_map(iommu->default_dom, entry); - if (ret) - return ret; - } - - return 0; -} - -/* - * Inits the unity mappings required for a specific device - */ -static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, - u16 devid) -{ - struct unity_map_entry *e; - int ret; - - list_for_each_entry(e, &amd_iommu_unity_map, list) { - if (!(devid >= e->devid_start && devid <= e->devid_end)) - continue; - ret = dma_ops_unity_map(dma_dom, e); - if (ret) - return ret; - } - - return 0; -} - -/**************************************************************************** - * - * The next functions belong to the address allocator for the dma_ops - * interface functions. They work like the allocators in the other IOMMU - * drivers. Its basically a bitmap which marks the allocated pages in - * the aperture. Maybe it could be enhanced in the future to a more - * efficient allocator. - * - ****************************************************************************/ - -/* - * The address allocator core functions. - * - * called with domain->lock held - */ - -/* - * Used to reserve address ranges in the aperture (e.g. for exclusion - * ranges. - */ -static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, - unsigned long start_page, - unsigned int pages) -{ - unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT; - - if (start_page + pages > last_page) - pages = last_page - start_page; - - for (i = start_page; i < start_page + pages; ++i) { - int index = i / APERTURE_RANGE_PAGES; - int page = i % APERTURE_RANGE_PAGES; - __set_bit(page, dom->aperture[index]->bitmap); - } -} - -/* - * This function is used to add a new aperture range to an existing - * aperture in case of dma_ops domain allocation or address allocation - * failure. - */ -static int alloc_new_range(struct dma_ops_domain *dma_dom, - bool populate, gfp_t gfp) -{ - int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT; - struct amd_iommu *iommu; - unsigned long i; - -#ifdef CONFIG_IOMMU_STRESS - populate = false; -#endif - - if (index >= APERTURE_MAX_RANGES) - return -ENOMEM; - - dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp); - if (!dma_dom->aperture[index]) - return -ENOMEM; - - dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp); - if (!dma_dom->aperture[index]->bitmap) - goto out_free; - - dma_dom->aperture[index]->offset = dma_dom->aperture_size; - - if (populate) { - unsigned long address = dma_dom->aperture_size; - int i, num_ptes = APERTURE_RANGE_PAGES / 512; - u64 *pte, *pte_page; - - for (i = 0; i < num_ptes; ++i) { - pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE, - &pte_page, gfp); - if (!pte) - goto out_free; - - dma_dom->aperture[index]->pte_pages[i] = pte_page; - - address += APERTURE_RANGE_SIZE / 64; - } - } - - dma_dom->aperture_size += APERTURE_RANGE_SIZE; - - /* Initialize the exclusion range if necessary */ - for_each_iommu(iommu) { - if (iommu->exclusion_start && - iommu->exclusion_start >= dma_dom->aperture[index]->offset - && iommu->exclusion_start < dma_dom->aperture_size) { - unsigned long startpage; - int pages = iommu_num_pages(iommu->exclusion_start, - iommu->exclusion_length, - PAGE_SIZE); - startpage = iommu->exclusion_start >> PAGE_SHIFT; - dma_ops_reserve_addresses(dma_dom, startpage, pages); - } - } - - /* - * Check for areas already mapped as present in the new aperture - * range and mark those pages as reserved in the allocator. Such - * mappings may already exist as a result of requested unity - * mappings for devices. - */ - for (i = dma_dom->aperture[index]->offset; - i < dma_dom->aperture_size; - i += PAGE_SIZE) { - u64 *pte = fetch_pte(&dma_dom->domain, i); - if (!pte || !IOMMU_PTE_PRESENT(*pte)) - continue; - - dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1); - } - - update_domain(&dma_dom->domain); - - return 0; - -out_free: - update_domain(&dma_dom->domain); - - free_page((unsigned long)dma_dom->aperture[index]->bitmap); - - kfree(dma_dom->aperture[index]); - dma_dom->aperture[index] = NULL; - - return -ENOMEM; -} - -static unsigned long dma_ops_area_alloc(struct device *dev, - struct dma_ops_domain *dom, - unsigned int pages, - unsigned long align_mask, - u64 dma_mask, - unsigned long start) -{ - unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE; - int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT; - int i = start >> APERTURE_RANGE_SHIFT; - unsigned long boundary_size; - unsigned long address = -1; - unsigned long limit; - - next_bit >>= PAGE_SHIFT; - - boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, - PAGE_SIZE) >> PAGE_SHIFT; - - for (;i < max_index; ++i) { - unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT; - - if (dom->aperture[i]->offset >= dma_mask) - break; - - limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset, - dma_mask >> PAGE_SHIFT); - - address = iommu_area_alloc(dom->aperture[i]->bitmap, - limit, next_bit, pages, 0, - boundary_size, align_mask); - if (address != -1) { - address = dom->aperture[i]->offset + - (address << PAGE_SHIFT); - dom->next_address = address + (pages << PAGE_SHIFT); - break; - } - - next_bit = 0; - } - - return address; -} - -static unsigned long dma_ops_alloc_addresses(struct device *dev, - struct dma_ops_domain *dom, - unsigned int pages, - unsigned long align_mask, - u64 dma_mask) -{ - unsigned long address; - -#ifdef CONFIG_IOMMU_STRESS - dom->next_address = 0; - dom->need_flush = true; -#endif - - address = dma_ops_area_alloc(dev, dom, pages, align_mask, - dma_mask, dom->next_address); - - if (address == -1) { - dom->next_address = 0; - address = dma_ops_area_alloc(dev, dom, pages, align_mask, - dma_mask, 0); - dom->need_flush = true; - } - - if (unlikely(address == -1)) - address = DMA_ERROR_CODE; - - WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size); - - return address; -} - -/* - * The address free function. - * - * called with domain->lock held - */ -static void dma_ops_free_addresses(struct dma_ops_domain *dom, - unsigned long address, - unsigned int pages) -{ - unsigned i = address >> APERTURE_RANGE_SHIFT; - struct aperture_range *range = dom->aperture[i]; - - BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL); - -#ifdef CONFIG_IOMMU_STRESS - if (i < 4) - return; -#endif - - if (address >= dom->next_address) - dom->need_flush = true; - - address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT; - - bitmap_clear(range->bitmap, address, pages); - -} - -/**************************************************************************** - * - * The next functions belong to the domain allocation. A domain is - * allocated for every IOMMU as the default domain. If device isolation - * is enabled, every device get its own domain. The most important thing - * about domains is the page table mapping the DMA address space they - * contain. - * - ****************************************************************************/ - -/* - * This function adds a protection domain to the global protection domain list - */ -static void add_domain_to_list(struct protection_domain *domain) -{ - unsigned long flags; - - spin_lock_irqsave(&amd_iommu_pd_lock, flags); - list_add(&domain->list, &amd_iommu_pd_list); - spin_unlock_irqrestore(&amd_iommu_pd_lock, flags); -} - -/* - * This function removes a protection domain to the global - * protection domain list - */ -static void del_domain_from_list(struct protection_domain *domain) -{ - unsigned long flags; - - spin_lock_irqsave(&amd_iommu_pd_lock, flags); - list_del(&domain->list); - spin_unlock_irqrestore(&amd_iommu_pd_lock, flags); -} - -static u16 domain_id_alloc(void) -{ - unsigned long flags; - int id; - - write_lock_irqsave(&amd_iommu_devtable_lock, flags); - id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID); - BUG_ON(id == 0); - if (id > 0 && id < MAX_DOMAIN_ID) - __set_bit(id, amd_iommu_pd_alloc_bitmap); - else - id = 0; - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - - return id; -} - -static void domain_id_free(int id) -{ - unsigned long flags; - - write_lock_irqsave(&amd_iommu_devtable_lock, flags); - if (id > 0 && id < MAX_DOMAIN_ID) - __clear_bit(id, amd_iommu_pd_alloc_bitmap); - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); -} - -static void free_pagetable(struct protection_domain *domain) -{ - int i, j; - u64 *p1, *p2, *p3; - - p1 = domain->pt_root; - - if (!p1) - return; - - for (i = 0; i < 512; ++i) { - if (!IOMMU_PTE_PRESENT(p1[i])) - continue; - - p2 = IOMMU_PTE_PAGE(p1[i]); - for (j = 0; j < 512; ++j) { - if (!IOMMU_PTE_PRESENT(p2[j])) - continue; - p3 = IOMMU_PTE_PAGE(p2[j]); - free_page((unsigned long)p3); - } - - free_page((unsigned long)p2); - } - - free_page((unsigned long)p1); - - domain->pt_root = NULL; -} - -/* - * Free a domain, only used if something went wrong in the - * allocation path and we need to free an already allocated page table - */ -static void dma_ops_domain_free(struct dma_ops_domain *dom) -{ - int i; - - if (!dom) - return; - - del_domain_from_list(&dom->domain); - - free_pagetable(&dom->domain); - - for (i = 0; i < APERTURE_MAX_RANGES; ++i) { - if (!dom->aperture[i]) - continue; - free_page((unsigned long)dom->aperture[i]->bitmap); - kfree(dom->aperture[i]); - } - - kfree(dom); -} - -/* - * Allocates a new protection domain usable for the dma_ops functions. - * It also initializes the page table and the address allocator data - * structures required for the dma_ops interface - */ -static struct dma_ops_domain *dma_ops_domain_alloc(void) -{ - struct dma_ops_domain *dma_dom; - - dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL); - if (!dma_dom) - return NULL; - - spin_lock_init(&dma_dom->domain.lock); - - dma_dom->domain.id = domain_id_alloc(); - if (dma_dom->domain.id == 0) - goto free_dma_dom; - INIT_LIST_HEAD(&dma_dom->domain.dev_list); - dma_dom->domain.mode = PAGE_MODE_2_LEVEL; - dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL); - dma_dom->domain.flags = PD_DMA_OPS_MASK; - dma_dom->domain.priv = dma_dom; - if (!dma_dom->domain.pt_root) - goto free_dma_dom; - - dma_dom->need_flush = false; - dma_dom->target_dev = 0xffff; - - add_domain_to_list(&dma_dom->domain); - - if (alloc_new_range(dma_dom, true, GFP_KERNEL)) - goto free_dma_dom; - - /* - * mark the first page as allocated so we never return 0 as - * a valid dma-address. So we can use 0 as error value - */ - dma_dom->aperture[0]->bitmap[0] = 1; - dma_dom->next_address = 0; - - - return dma_dom; - -free_dma_dom: - dma_ops_domain_free(dma_dom); - - return NULL; -} - -/* - * little helper function to check whether a given protection domain is a - * dma_ops domain - */ -static bool dma_ops_domain(struct protection_domain *domain) -{ - return domain->flags & PD_DMA_OPS_MASK; -} - -static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats) -{ - u64 pte_root = virt_to_phys(domain->pt_root); - u32 flags = 0; - - pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK) - << DEV_ENTRY_MODE_SHIFT; - pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV; - - if (ats) - flags |= DTE_FLAG_IOTLB; - - amd_iommu_dev_table[devid].data[3] |= flags; - amd_iommu_dev_table[devid].data[2] = domain->id; - amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root); - amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root); -} - -static void clear_dte_entry(u16 devid) -{ - /* remove entry from the device table seen by the hardware */ - amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV; - amd_iommu_dev_table[devid].data[1] = 0; - amd_iommu_dev_table[devid].data[2] = 0; - - amd_iommu_apply_erratum_63(devid); -} - -static void do_attach(struct device *dev, struct protection_domain *domain) -{ - struct iommu_dev_data *dev_data; - struct amd_iommu *iommu; - struct pci_dev *pdev; - bool ats = false; - u16 devid; - - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; - dev_data = get_dev_data(dev); - pdev = to_pci_dev(dev); - - if (amd_iommu_iotlb_sup) - ats = pci_ats_enabled(pdev); - - /* Update data structures */ - dev_data->domain = domain; - list_add(&dev_data->list, &domain->dev_list); - set_dte_entry(devid, domain, ats); - - /* Do reference counting */ - domain->dev_iommu[iommu->index] += 1; - domain->dev_cnt += 1; - - /* Flush the DTE entry */ - device_flush_dte(dev); -} - -static void do_detach(struct device *dev) -{ - struct iommu_dev_data *dev_data; - struct amd_iommu *iommu; - u16 devid; - - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; - dev_data = get_dev_data(dev); - - /* decrease reference counters */ - dev_data->domain->dev_iommu[iommu->index] -= 1; - dev_data->domain->dev_cnt -= 1; - - /* Update data structures */ - dev_data->domain = NULL; - list_del(&dev_data->list); - clear_dte_entry(devid); - - /* Flush the DTE entry */ - device_flush_dte(dev); -} - -/* - * If a device is not yet associated with a domain, this function does - * assigns it visible for the hardware - */ -static int __attach_device(struct device *dev, - struct protection_domain *domain) -{ - struct iommu_dev_data *dev_data, *alias_data; - int ret; - - dev_data = get_dev_data(dev); - alias_data = get_dev_data(dev_data->alias); - - if (!alias_data) - return -EINVAL; - - /* lock domain */ - spin_lock(&domain->lock); - - /* Some sanity checks */ - ret = -EBUSY; - if (alias_data->domain != NULL && - alias_data->domain != domain) - goto out_unlock; - - if (dev_data->domain != NULL && - dev_data->domain != domain) - goto out_unlock; - - /* Do real assignment */ - if (dev_data->alias != dev) { - alias_data = get_dev_data(dev_data->alias); - if (alias_data->domain == NULL) - do_attach(dev_data->alias, domain); - - atomic_inc(&alias_data->bind); - } - - if (dev_data->domain == NULL) - do_attach(dev, domain); - - atomic_inc(&dev_data->bind); - - ret = 0; - -out_unlock: - - /* ready */ - spin_unlock(&domain->lock); - - return ret; -} - -/* - * If a device is not yet associated with a domain, this function does - * assigns it visible for the hardware - */ -static int attach_device(struct device *dev, - struct protection_domain *domain) -{ - struct pci_dev *pdev = to_pci_dev(dev); - unsigned long flags; - int ret; - - if (amd_iommu_iotlb_sup) - pci_enable_ats(pdev, PAGE_SHIFT); - - write_lock_irqsave(&amd_iommu_devtable_lock, flags); - ret = __attach_device(dev, domain); - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - - /* - * We might boot into a crash-kernel here. The crashed kernel - * left the caches in the IOMMU dirty. So we have to flush - * here to evict all dirty stuff. - */ - domain_flush_tlb_pde(domain); - - return ret; -} - -/* - * Removes a device from a protection domain (unlocked) - */ -static void __detach_device(struct device *dev) -{ - struct iommu_dev_data *dev_data = get_dev_data(dev); - struct iommu_dev_data *alias_data; - struct protection_domain *domain; - unsigned long flags; - - BUG_ON(!dev_data->domain); - - domain = dev_data->domain; - - spin_lock_irqsave(&domain->lock, flags); - - if (dev_data->alias != dev) { - alias_data = get_dev_data(dev_data->alias); - if (atomic_dec_and_test(&alias_data->bind)) - do_detach(dev_data->alias); - } - - if (atomic_dec_and_test(&dev_data->bind)) - do_detach(dev); - - spin_unlock_irqrestore(&domain->lock, flags); - - /* - * If we run in passthrough mode the device must be assigned to the - * passthrough domain if it is detached from any other domain. - * Make sure we can deassign from the pt_domain itself. - */ - if (iommu_pass_through && - (dev_data->domain == NULL && domain != pt_domain)) - __attach_device(dev, pt_domain); -} - -/* - * Removes a device from a protection domain (with devtable_lock held) - */ -static void detach_device(struct device *dev) -{ - struct pci_dev *pdev = to_pci_dev(dev); - unsigned long flags; - - /* lock device table */ - write_lock_irqsave(&amd_iommu_devtable_lock, flags); - __detach_device(dev); - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - - if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev)) - pci_disable_ats(pdev); -} - -/* - * Find out the protection domain structure for a given PCI device. This - * will give us the pointer to the page table root for example. - */ -static struct protection_domain *domain_for_device(struct device *dev) -{ - struct protection_domain *dom; - struct iommu_dev_data *dev_data, *alias_data; - unsigned long flags; - u16 devid; - - devid = get_device_id(dev); - dev_data = get_dev_data(dev); - alias_data = get_dev_data(dev_data->alias); - if (!alias_data) - return NULL; - - read_lock_irqsave(&amd_iommu_devtable_lock, flags); - dom = dev_data->domain; - if (dom == NULL && - alias_data->domain != NULL) { - __attach_device(dev, alias_data->domain); - dom = alias_data->domain; - } - - read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - - return dom; -} - -static int device_change_notifier(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct device *dev = data; - u16 devid; - struct protection_domain *domain; - struct dma_ops_domain *dma_domain; - struct amd_iommu *iommu; - unsigned long flags; - - if (!check_device(dev)) - return 0; - - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; - - switch (action) { - case BUS_NOTIFY_UNBOUND_DRIVER: - - domain = domain_for_device(dev); - - if (!domain) - goto out; - if (iommu_pass_through) - break; - detach_device(dev); - break; - case BUS_NOTIFY_ADD_DEVICE: - - iommu_init_device(dev); - - domain = domain_for_device(dev); - - /* allocate a protection domain if a device is added */ - dma_domain = find_protection_domain(devid); - if (dma_domain) - goto out; - dma_domain = dma_ops_domain_alloc(); - if (!dma_domain) - goto out; - dma_domain->target_dev = devid; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); - list_add_tail(&dma_domain->list, &iommu_pd_list); - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - - break; - case BUS_NOTIFY_DEL_DEVICE: - - iommu_uninit_device(dev); - - default: - goto out; - } - - device_flush_dte(dev); - iommu_completion_wait(iommu); - -out: - return 0; -} - -static struct notifier_block device_nb = { - .notifier_call = device_change_notifier, -}; - -void amd_iommu_init_notifier(void) -{ - bus_register_notifier(&pci_bus_type, &device_nb); -} - -/***************************************************************************** - * - * The next functions belong to the dma_ops mapping/unmapping code. - * - *****************************************************************************/ - -/* - * In the dma_ops path we only have the struct device. This function - * finds the corresponding IOMMU, the protection domain and the - * requestor id for a given device. - * If the device is not yet associated with a domain this is also done - * in this function. - */ -static struct protection_domain *get_domain(struct device *dev) -{ - struct protection_domain *domain; - struct dma_ops_domain *dma_dom; - u16 devid = get_device_id(dev); - - if (!check_device(dev)) - return ERR_PTR(-EINVAL); - - domain = domain_for_device(dev); - if (domain != NULL && !dma_ops_domain(domain)) - return ERR_PTR(-EBUSY); - - if (domain != NULL) - return domain; - - /* Device not bount yet - bind it */ - dma_dom = find_protection_domain(devid); - if (!dma_dom) - dma_dom = amd_iommu_rlookup_table[devid]->default_dom; - attach_device(dev, &dma_dom->domain); - DUMP_printk("Using protection domain %d for device %s\n", - dma_dom->domain.id, dev_name(dev)); - - return &dma_dom->domain; -} - -static void update_device_table(struct protection_domain *domain) -{ - struct iommu_dev_data *dev_data; - - list_for_each_entry(dev_data, &domain->dev_list, list) { - struct pci_dev *pdev = to_pci_dev(dev_data->dev); - u16 devid = get_device_id(dev_data->dev); - set_dte_entry(devid, domain, pci_ats_enabled(pdev)); - } -} - -static void update_domain(struct protection_domain *domain) -{ - if (!domain->updated) - return; - - update_device_table(domain); - - domain_flush_devices(domain); - domain_flush_tlb_pde(domain); - - domain->updated = false; -} - -/* - * This function fetches the PTE for a given address in the aperture - */ -static u64* dma_ops_get_pte(struct dma_ops_domain *dom, - unsigned long address) -{ - struct aperture_range *aperture; - u64 *pte, *pte_page; - - aperture = dom->aperture[APERTURE_RANGE_INDEX(address)]; - if (!aperture) - return NULL; - - pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)]; - if (!pte) { - pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page, - GFP_ATOMIC); - aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page; - } else - pte += PM_LEVEL_INDEX(0, address); - - update_domain(&dom->domain); - - return pte; -} - -/* - * This is the generic map function. It maps one 4kb page at paddr to - * the given address in the DMA address space for the domain. - */ -static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom, - unsigned long address, - phys_addr_t paddr, - int direction) -{ - u64 *pte, __pte; - - WARN_ON(address > dom->aperture_size); - - paddr &= PAGE_MASK; - - pte = dma_ops_get_pte(dom, address); - if (!pte) - return DMA_ERROR_CODE; - - __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC; - - if (direction == DMA_TO_DEVICE) - __pte |= IOMMU_PTE_IR; - else if (direction == DMA_FROM_DEVICE) - __pte |= IOMMU_PTE_IW; - else if (direction == DMA_BIDIRECTIONAL) - __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW; - - WARN_ON(*pte); - - *pte = __pte; - - return (dma_addr_t)address; -} - -/* - * The generic unmapping function for on page in the DMA address space. - */ -static void dma_ops_domain_unmap(struct dma_ops_domain *dom, - unsigned long address) -{ - struct aperture_range *aperture; - u64 *pte; - - if (address >= dom->aperture_size) - return; - - aperture = dom->aperture[APERTURE_RANGE_INDEX(address)]; - if (!aperture) - return; - - pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)]; - if (!pte) - return; - - pte += PM_LEVEL_INDEX(0, address); - - WARN_ON(!*pte); - - *pte = 0ULL; -} - -/* - * This function contains common code for mapping of a physically - * contiguous memory region into DMA address space. It is used by all - * mapping functions provided with this IOMMU driver. - * Must be called with the domain lock held. - */ -static dma_addr_t __map_single(struct device *dev, - struct dma_ops_domain *dma_dom, - phys_addr_t paddr, - size_t size, - int dir, - bool align, - u64 dma_mask) -{ - dma_addr_t offset = paddr & ~PAGE_MASK; - dma_addr_t address, start, ret; - unsigned int pages; - unsigned long align_mask = 0; - int i; - - pages = iommu_num_pages(paddr, size, PAGE_SIZE); - paddr &= PAGE_MASK; - - INC_STATS_COUNTER(total_map_requests); - - if (pages > 1) - INC_STATS_COUNTER(cross_page); - - if (align) - align_mask = (1UL << get_order(size)) - 1; - -retry: - address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask, - dma_mask); - if (unlikely(address == DMA_ERROR_CODE)) { - /* - * setting next_address here will let the address - * allocator only scan the new allocated range in the - * first run. This is a small optimization. - */ - dma_dom->next_address = dma_dom->aperture_size; - - if (alloc_new_range(dma_dom, false, GFP_ATOMIC)) - goto out; - - /* - * aperture was successfully enlarged by 128 MB, try - * allocation again - */ - goto retry; - } - - start = address; - for (i = 0; i < pages; ++i) { - ret = dma_ops_domain_map(dma_dom, start, paddr, dir); - if (ret == DMA_ERROR_CODE) - goto out_unmap; - - paddr += PAGE_SIZE; - start += PAGE_SIZE; - } - address += offset; - - ADD_STATS_COUNTER(alloced_io_mem, size); - - if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) { - domain_flush_tlb(&dma_dom->domain); - dma_dom->need_flush = false; - } else if (unlikely(amd_iommu_np_cache)) - domain_flush_pages(&dma_dom->domain, address, size); - -out: - return address; - -out_unmap: - - for (--i; i >= 0; --i) { - start -= PAGE_SIZE; - dma_ops_domain_unmap(dma_dom, start); - } - - dma_ops_free_addresses(dma_dom, address, pages); - - return DMA_ERROR_CODE; -} - -/* - * Does the reverse of the __map_single function. Must be called with - * the domain lock held too - */ -static void __unmap_single(struct dma_ops_domain *dma_dom, - dma_addr_t dma_addr, - size_t size, - int dir) -{ - dma_addr_t flush_addr; - dma_addr_t i, start; - unsigned int pages; - - if ((dma_addr == DMA_ERROR_CODE) || - (dma_addr + size > dma_dom->aperture_size)) - return; - - flush_addr = dma_addr; - pages = iommu_num_pages(dma_addr, size, PAGE_SIZE); - dma_addr &= PAGE_MASK; - start = dma_addr; - - for (i = 0; i < pages; ++i) { - dma_ops_domain_unmap(dma_dom, start); - start += PAGE_SIZE; - } - - SUB_STATS_COUNTER(alloced_io_mem, size); - - dma_ops_free_addresses(dma_dom, dma_addr, pages); - - if (amd_iommu_unmap_flush || dma_dom->need_flush) { - domain_flush_pages(&dma_dom->domain, flush_addr, size); - dma_dom->need_flush = false; - } -} - -/* - * The exported map_single function for dma_ops. - */ -static dma_addr_t map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - struct dma_attrs *attrs) -{ - unsigned long flags; - struct protection_domain *domain; - dma_addr_t addr; - u64 dma_mask; - phys_addr_t paddr = page_to_phys(page) + offset; - - INC_STATS_COUNTER(cnt_map_single); - - domain = get_domain(dev); - if (PTR_ERR(domain) == -EINVAL) - return (dma_addr_t)paddr; - else if (IS_ERR(domain)) - return DMA_ERROR_CODE; - - dma_mask = *dev->dma_mask; - - spin_lock_irqsave(&domain->lock, flags); - - addr = __map_single(dev, domain->priv, paddr, size, dir, false, - dma_mask); - if (addr == DMA_ERROR_CODE) - goto out; - - domain_flush_complete(domain); - -out: - spin_unlock_irqrestore(&domain->lock, flags); - - return addr; -} - -/* - * The exported unmap_single function for dma_ops. - */ -static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) -{ - unsigned long flags; - struct protection_domain *domain; - - INC_STATS_COUNTER(cnt_unmap_single); - - domain = get_domain(dev); - if (IS_ERR(domain)) - return; - - spin_lock_irqsave(&domain->lock, flags); - - __unmap_single(domain->priv, dma_addr, size, dir); - - domain_flush_complete(domain); - - spin_unlock_irqrestore(&domain->lock, flags); -} - -/* - * This is a special map_sg function which is used if we should map a - * device which is not handled by an AMD IOMMU in the system. - */ -static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist, - int nelems, int dir) -{ - struct scatterlist *s; - int i; - - for_each_sg(sglist, s, nelems, i) { - s->dma_address = (dma_addr_t)sg_phys(s); - s->dma_length = s->length; - } - - return nelems; -} - -/* - * The exported map_sg function for dma_ops (handles scatter-gather - * lists). - */ -static int map_sg(struct device *dev, struct scatterlist *sglist, - int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) -{ - unsigned long flags; - struct protection_domain *domain; - int i; - struct scatterlist *s; - phys_addr_t paddr; - int mapped_elems = 0; - u64 dma_mask; - - INC_STATS_COUNTER(cnt_map_sg); - - domain = get_domain(dev); - if (PTR_ERR(domain) == -EINVAL) - return map_sg_no_iommu(dev, sglist, nelems, dir); - else if (IS_ERR(domain)) - return 0; - - dma_mask = *dev->dma_mask; - - spin_lock_irqsave(&domain->lock, flags); - - for_each_sg(sglist, s, nelems, i) { - paddr = sg_phys(s); - - s->dma_address = __map_single(dev, domain->priv, - paddr, s->length, dir, false, - dma_mask); - - if (s->dma_address) { - s->dma_length = s->length; - mapped_elems++; - } else - goto unmap; - } - - domain_flush_complete(domain); - -out: - spin_unlock_irqrestore(&domain->lock, flags); - - return mapped_elems; -unmap: - for_each_sg(sglist, s, mapped_elems, i) { - if (s->dma_address) - __unmap_single(domain->priv, s->dma_address, - s->dma_length, dir); - s->dma_address = s->dma_length = 0; - } - - mapped_elems = 0; - - goto out; -} - -/* - * The exported map_sg function for dma_ops (handles scatter-gather - * lists). - */ -static void unmap_sg(struct device *dev, struct scatterlist *sglist, - int nelems, enum dma_data_direction dir, - struct dma_attrs *attrs) -{ - unsigned long flags; - struct protection_domain *domain; - struct scatterlist *s; - int i; - - INC_STATS_COUNTER(cnt_unmap_sg); - - domain = get_domain(dev); - if (IS_ERR(domain)) - return; - - spin_lock_irqsave(&domain->lock, flags); - - for_each_sg(sglist, s, nelems, i) { - __unmap_single(domain->priv, s->dma_address, - s->dma_length, dir); - s->dma_address = s->dma_length = 0; - } - - domain_flush_complete(domain); - - spin_unlock_irqrestore(&domain->lock, flags); -} - -/* - * The exported alloc_coherent function for dma_ops. - */ -static void *alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_addr, gfp_t flag) -{ - unsigned long flags; - void *virt_addr; - struct protection_domain *domain; - phys_addr_t paddr; - u64 dma_mask = dev->coherent_dma_mask; - - INC_STATS_COUNTER(cnt_alloc_coherent); - - domain = get_domain(dev); - if (PTR_ERR(domain) == -EINVAL) { - virt_addr = (void *)__get_free_pages(flag, get_order(size)); - *dma_addr = __pa(virt_addr); - return virt_addr; - } else if (IS_ERR(domain)) - return NULL; - - dma_mask = dev->coherent_dma_mask; - flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); - flag |= __GFP_ZERO; - - virt_addr = (void *)__get_free_pages(flag, get_order(size)); - if (!virt_addr) - return NULL; - - paddr = virt_to_phys(virt_addr); - - if (!dma_mask) - dma_mask = *dev->dma_mask; - - spin_lock_irqsave(&domain->lock, flags); - - *dma_addr = __map_single(dev, domain->priv, paddr, - size, DMA_BIDIRECTIONAL, true, dma_mask); - - if (*dma_addr == DMA_ERROR_CODE) { - spin_unlock_irqrestore(&domain->lock, flags); - goto out_free; - } - - domain_flush_complete(domain); - - spin_unlock_irqrestore(&domain->lock, flags); - - return virt_addr; - -out_free: - - free_pages((unsigned long)virt_addr, get_order(size)); - - return NULL; -} - -/* - * The exported free_coherent function for dma_ops. - */ -static void free_coherent(struct device *dev, size_t size, - void *virt_addr, dma_addr_t dma_addr) -{ - unsigned long flags; - struct protection_domain *domain; - - INC_STATS_COUNTER(cnt_free_coherent); - - domain = get_domain(dev); - if (IS_ERR(domain)) - goto free_mem; - - spin_lock_irqsave(&domain->lock, flags); - - __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL); - - domain_flush_complete(domain); - - spin_unlock_irqrestore(&domain->lock, flags); - -free_mem: - free_pages((unsigned long)virt_addr, get_order(size)); -} - -/* - * This function is called by the DMA layer to find out if we can handle a - * particular device. It is part of the dma_ops. - */ -static int amd_iommu_dma_supported(struct device *dev, u64 mask) -{ - return check_device(dev); -} - -/* - * The function for pre-allocating protection domains. - * - * If the driver core informs the DMA layer if a driver grabs a device - * we don't need to preallocate the protection domains anymore. - * For now we have to. - */ -static void prealloc_protection_domains(void) -{ - struct pci_dev *dev = NULL; - struct dma_ops_domain *dma_dom; - u16 devid; - - for_each_pci_dev(dev) { - - /* Do we handle this device? */ - if (!check_device(&dev->dev)) - continue; - - /* Is there already any domain for it? */ - if (domain_for_device(&dev->dev)) - continue; - - devid = get_device_id(&dev->dev); - - dma_dom = dma_ops_domain_alloc(); - if (!dma_dom) - continue; - init_unity_mappings_for_device(dma_dom, devid); - dma_dom->target_dev = devid; - - attach_device(&dev->dev, &dma_dom->domain); - - list_add_tail(&dma_dom->list, &iommu_pd_list); - } -} - -static struct dma_map_ops amd_iommu_dma_ops = { - .alloc_coherent = alloc_coherent, - .free_coherent = free_coherent, - .map_page = map_page, - .unmap_page = unmap_page, - .map_sg = map_sg, - .unmap_sg = unmap_sg, - .dma_supported = amd_iommu_dma_supported, -}; - -static unsigned device_dma_ops_init(void) -{ - struct pci_dev *pdev = NULL; - unsigned unhandled = 0; - - for_each_pci_dev(pdev) { - if (!check_device(&pdev->dev)) { - unhandled += 1; - continue; - } - - pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops; - } - - return unhandled; -} - -/* - * The function which clues the AMD IOMMU driver into dma_ops. - */ - -void __init amd_iommu_init_api(void) -{ - register_iommu(&amd_iommu_ops); -} - -int __init amd_iommu_init_dma_ops(void) -{ - struct amd_iommu *iommu; - int ret, unhandled; - - /* - * first allocate a default protection domain for every IOMMU we - * found in the system. Devices not assigned to any other - * protection domain will be assigned to the default one. - */ - for_each_iommu(iommu) { - iommu->default_dom = dma_ops_domain_alloc(); - if (iommu->default_dom == NULL) - return -ENOMEM; - iommu->default_dom->domain.flags |= PD_DEFAULT_MASK; - ret = iommu_init_unity_mappings(iommu); - if (ret) - goto free_domains; - } - - /* - * Pre-allocate the protection domains for each device. - */ - prealloc_protection_domains(); - - iommu_detected = 1; - swiotlb = 0; - - /* Make the driver finally visible to the drivers */ - unhandled = device_dma_ops_init(); - if (unhandled && max_pfn > MAX_DMA32_PFN) { - /* There are unhandled devices - initialize swiotlb for them */ - swiotlb = 1; - } - - amd_iommu_stats_init(); - - return 0; - -free_domains: - - for_each_iommu(iommu) { - if (iommu->default_dom) - dma_ops_domain_free(iommu->default_dom); - } - - return ret; -} - -/***************************************************************************** - * - * The following functions belong to the exported interface of AMD IOMMU - * - * This interface allows access to lower level functions of the IOMMU - * like protection domain handling and assignement of devices to domains - * which is not possible with the dma_ops interface. - * - *****************************************************************************/ - -static void cleanup_domain(struct protection_domain *domain) -{ - struct iommu_dev_data *dev_data, *next; - unsigned long flags; - - write_lock_irqsave(&amd_iommu_devtable_lock, flags); - - list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) { - struct device *dev = dev_data->dev; - - __detach_device(dev); - atomic_set(&dev_data->bind, 0); - } - - write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); -} - -static void protection_domain_free(struct protection_domain *domain) -{ - if (!domain) - return; - - del_domain_from_list(domain); - - if (domain->id) - domain_id_free(domain->id); - - kfree(domain); -} - -static struct protection_domain *protection_domain_alloc(void) -{ - struct protection_domain *domain; - - domain = kzalloc(sizeof(*domain), GFP_KERNEL); - if (!domain) - return NULL; - - spin_lock_init(&domain->lock); - mutex_init(&domain->api_lock); - domain->id = domain_id_alloc(); - if (!domain->id) - goto out_err; - INIT_LIST_HEAD(&domain->dev_list); - - add_domain_to_list(domain); - - return domain; - -out_err: - kfree(domain); - - return NULL; -} - -static int amd_iommu_domain_init(struct iommu_domain *dom) -{ - struct protection_domain *domain; - - domain = protection_domain_alloc(); - if (!domain) - goto out_free; - - domain->mode = PAGE_MODE_3_LEVEL; - domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!domain->pt_root) - goto out_free; - - dom->priv = domain; - - return 0; - -out_free: - protection_domain_free(domain); - - return -ENOMEM; -} - -static void amd_iommu_domain_destroy(struct iommu_domain *dom) -{ - struct protection_domain *domain = dom->priv; - - if (!domain) - return; - - if (domain->dev_cnt > 0) - cleanup_domain(domain); - - BUG_ON(domain->dev_cnt != 0); - - free_pagetable(domain); - - protection_domain_free(domain); - - dom->priv = NULL; -} - -static void amd_iommu_detach_device(struct iommu_domain *dom, - struct device *dev) -{ - struct iommu_dev_data *dev_data = dev->archdata.iommu; - struct amd_iommu *iommu; - u16 devid; - - if (!check_device(dev)) - return; - - devid = get_device_id(dev); - - if (dev_data->domain != NULL) - detach_device(dev); - - iommu = amd_iommu_rlookup_table[devid]; - if (!iommu) - return; - - device_flush_dte(dev); - iommu_completion_wait(iommu); -} - -static int amd_iommu_attach_device(struct iommu_domain *dom, - struct device *dev) -{ - struct protection_domain *domain = dom->priv; - struct iommu_dev_data *dev_data; - struct amd_iommu *iommu; - int ret; - u16 devid; - - if (!check_device(dev)) - return -EINVAL; - - dev_data = dev->archdata.iommu; - - devid = get_device_id(dev); - - iommu = amd_iommu_rlookup_table[devid]; - if (!iommu) - return -EINVAL; - - if (dev_data->domain) - detach_device(dev); - - ret = attach_device(dev, domain); - - iommu_completion_wait(iommu); - - return ret; -} - -static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, - phys_addr_t paddr, int gfp_order, int iommu_prot) -{ - unsigned long page_size = 0x1000UL << gfp_order; - struct protection_domain *domain = dom->priv; - int prot = 0; - int ret; - - if (iommu_prot & IOMMU_READ) - prot |= IOMMU_PROT_IR; - if (iommu_prot & IOMMU_WRITE) - prot |= IOMMU_PROT_IW; - - mutex_lock(&domain->api_lock); - ret = iommu_map_page(domain, iova, paddr, prot, page_size); - mutex_unlock(&domain->api_lock); - - return ret; -} - -static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, - int gfp_order) -{ - struct protection_domain *domain = dom->priv; - unsigned long page_size, unmap_size; - - page_size = 0x1000UL << gfp_order; - - mutex_lock(&domain->api_lock); - unmap_size = iommu_unmap_page(domain, iova, page_size); - mutex_unlock(&domain->api_lock); - - domain_flush_tlb_pde(domain); - - return get_order(unmap_size); -} - -static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, - unsigned long iova) -{ - struct protection_domain *domain = dom->priv; - unsigned long offset_mask; - phys_addr_t paddr; - u64 *pte, __pte; - - pte = fetch_pte(domain, iova); - - if (!pte || !IOMMU_PTE_PRESENT(*pte)) - return 0; - - if (PM_PTE_LEVEL(*pte) == 0) - offset_mask = PAGE_SIZE - 1; - else - offset_mask = PTE_PAGE_SIZE(*pte) - 1; - - __pte = *pte & PM_ADDR_MASK; - paddr = (__pte & ~offset_mask) | (iova & offset_mask); - - return paddr; -} - -static int amd_iommu_domain_has_cap(struct iommu_domain *domain, - unsigned long cap) -{ - switch (cap) { - case IOMMU_CAP_CACHE_COHERENCY: - return 1; - } - - return 0; -} - -static struct iommu_ops amd_iommu_ops = { - .domain_init = amd_iommu_domain_init, - .domain_destroy = amd_iommu_domain_destroy, - .attach_dev = amd_iommu_attach_device, - .detach_dev = amd_iommu_detach_device, - .map = amd_iommu_map, - .unmap = amd_iommu_unmap, - .iova_to_phys = amd_iommu_iova_to_phys, - .domain_has_cap = amd_iommu_domain_has_cap, -}; - -/***************************************************************************** - * - * The next functions do a basic initialization of IOMMU for pass through - * mode - * - * In passthrough mode the IOMMU is initialized and enabled but not used for - * DMA-API translation. - * - *****************************************************************************/ - -int __init amd_iommu_init_passthrough(void) -{ - struct amd_iommu *iommu; - struct pci_dev *dev = NULL; - u16 devid; - - /* allocate passthrough domain */ - pt_domain = protection_domain_alloc(); - if (!pt_domain) - return -ENOMEM; - - pt_domain->mode |= PAGE_MODE_NONE; - - for_each_pci_dev(dev) { - if (!check_device(&dev->dev)) - continue; - - devid = get_device_id(&dev->dev); - - iommu = amd_iommu_rlookup_table[devid]; - if (!iommu) - continue; - - attach_device(&dev->dev, pt_domain); - } - - pr_info("AMD-Vi: Initialized for Passthrough Mode\n"); - - return 0; -} diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 21a80bfbdb52..9246c5bf25af 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -17,3 +17,32 @@ config MSM_IOMMU config IOMMU_PGTABLES_L2 def_bool y depends on MSM_IOMMU && MMU && SMP && CPU_DCACHE_DISABLE=n + +# AMD IOMMU support +config AMD_IOMMU + bool "AMD IOMMU support" + select SWIOTLB + select PCI_MSI + select PCI_IOV + select IOMMU_API + depends on X86_64 && PCI && ACPI + ---help--- + With this option you can enable support for AMD IOMMU hardware in + your system. An IOMMU is a hardware component which provides + remapping of DMA memory accesses from devices. With an AMD IOMMU you + can isolate the the DMA memory of different devices and protect the + system from misbehaving device drivers or hardware. + + You can find out if your system has an AMD IOMMU if you look into + your BIOS for an option to enable it or if you have an IVRS ACPI + table. + +config AMD_IOMMU_STATS + bool "Export AMD IOMMU statistics to debugfs" + depends on AMD_IOMMU + select DEBUG_FS + ---help--- + This option enables code in the AMD IOMMU driver to collect various + statistics about whats happening in the driver and exports that + information to userspace via debugfs. + If unsure, say N. diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 1a71c82b1af2..4237eaf84609 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o +obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c new file mode 100644 index 000000000000..7c3a95e54ec5 --- /dev/null +++ b/drivers/iommu/amd_iommu.c @@ -0,0 +1,2764 @@ +/* + * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * Leo Duran + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) + +#define LOOP_TIMEOUT 100000 + +static DEFINE_RWLOCK(amd_iommu_devtable_lock); + +/* A list of preallocated protection domains */ +static LIST_HEAD(iommu_pd_list); +static DEFINE_SPINLOCK(iommu_pd_list_lock); + +/* + * Domain for untranslated devices - only allocated + * if iommu=pt passed on kernel cmd line. + */ +static struct protection_domain *pt_domain; + +static struct iommu_ops amd_iommu_ops; + +/* + * general struct to manage commands send to an IOMMU + */ +struct iommu_cmd { + u32 data[4]; +}; + +static void update_domain(struct protection_domain *domain); + +/**************************************************************************** + * + * Helper functions + * + ****************************************************************************/ + +static inline u16 get_device_id(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + + return calc_devid(pdev->bus->number, pdev->devfn); +} + +static struct iommu_dev_data *get_dev_data(struct device *dev) +{ + return dev->archdata.iommu; +} + +/* + * In this function the list of preallocated protection domains is traversed to + * find the domain for a specific device + */ +static struct dma_ops_domain *find_protection_domain(u16 devid) +{ + struct dma_ops_domain *entry, *ret = NULL; + unsigned long flags; + u16 alias = amd_iommu_alias_table[devid]; + + if (list_empty(&iommu_pd_list)) + return NULL; + + spin_lock_irqsave(&iommu_pd_list_lock, flags); + + list_for_each_entry(entry, &iommu_pd_list, list) { + if (entry->target_dev == devid || + entry->target_dev == alias) { + ret = entry; + break; + } + } + + spin_unlock_irqrestore(&iommu_pd_list_lock, flags); + + return ret; +} + +/* + * This function checks if the driver got a valid device from the caller to + * avoid dereferencing invalid pointers. + */ +static bool check_device(struct device *dev) +{ + u16 devid; + + if (!dev || !dev->dma_mask) + return false; + + /* No device or no PCI device */ + if (dev->bus != &pci_bus_type) + return false; + + devid = get_device_id(dev); + + /* Out of our scope? */ + if (devid > amd_iommu_last_bdf) + return false; + + if (amd_iommu_rlookup_table[devid] == NULL) + return false; + + return true; +} + +static int iommu_init_device(struct device *dev) +{ + struct iommu_dev_data *dev_data; + struct pci_dev *pdev; + u16 devid, alias; + + if (dev->archdata.iommu) + return 0; + + dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return -ENOMEM; + + dev_data->dev = dev; + + devid = get_device_id(dev); + alias = amd_iommu_alias_table[devid]; + pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); + if (pdev) + dev_data->alias = &pdev->dev; + else { + kfree(dev_data); + return -ENOTSUPP; + } + + atomic_set(&dev_data->bind, 0); + + dev->archdata.iommu = dev_data; + + + return 0; +} + +static void iommu_ignore_device(struct device *dev) +{ + u16 devid, alias; + + devid = get_device_id(dev); + alias = amd_iommu_alias_table[devid]; + + memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry)); + memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry)); + + amd_iommu_rlookup_table[devid] = NULL; + amd_iommu_rlookup_table[alias] = NULL; +} + +static void iommu_uninit_device(struct device *dev) +{ + kfree(dev->archdata.iommu); +} + +void __init amd_iommu_uninit_devices(void) +{ + struct pci_dev *pdev = NULL; + + for_each_pci_dev(pdev) { + + if (!check_device(&pdev->dev)) + continue; + + iommu_uninit_device(&pdev->dev); + } +} + +int __init amd_iommu_init_devices(void) +{ + struct pci_dev *pdev = NULL; + int ret = 0; + + for_each_pci_dev(pdev) { + + if (!check_device(&pdev->dev)) + continue; + + ret = iommu_init_device(&pdev->dev); + if (ret == -ENOTSUPP) + iommu_ignore_device(&pdev->dev); + else if (ret) + goto out_free; + } + + return 0; + +out_free: + + amd_iommu_uninit_devices(); + + return ret; +} +#ifdef CONFIG_AMD_IOMMU_STATS + +/* + * Initialization code for statistics collection + */ + +DECLARE_STATS_COUNTER(compl_wait); +DECLARE_STATS_COUNTER(cnt_map_single); +DECLARE_STATS_COUNTER(cnt_unmap_single); +DECLARE_STATS_COUNTER(cnt_map_sg); +DECLARE_STATS_COUNTER(cnt_unmap_sg); +DECLARE_STATS_COUNTER(cnt_alloc_coherent); +DECLARE_STATS_COUNTER(cnt_free_coherent); +DECLARE_STATS_COUNTER(cross_page); +DECLARE_STATS_COUNTER(domain_flush_single); +DECLARE_STATS_COUNTER(domain_flush_all); +DECLARE_STATS_COUNTER(alloced_io_mem); +DECLARE_STATS_COUNTER(total_map_requests); + +static struct dentry *stats_dir; +static struct dentry *de_fflush; + +static void amd_iommu_stats_add(struct __iommu_counter *cnt) +{ + if (stats_dir == NULL) + return; + + cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir, + &cnt->value); +} + +static void amd_iommu_stats_init(void) +{ + stats_dir = debugfs_create_dir("amd-iommu", NULL); + if (stats_dir == NULL) + return; + + de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir, + (u32 *)&amd_iommu_unmap_flush); + + amd_iommu_stats_add(&compl_wait); + amd_iommu_stats_add(&cnt_map_single); + amd_iommu_stats_add(&cnt_unmap_single); + amd_iommu_stats_add(&cnt_map_sg); + amd_iommu_stats_add(&cnt_unmap_sg); + amd_iommu_stats_add(&cnt_alloc_coherent); + amd_iommu_stats_add(&cnt_free_coherent); + amd_iommu_stats_add(&cross_page); + amd_iommu_stats_add(&domain_flush_single); + amd_iommu_stats_add(&domain_flush_all); + amd_iommu_stats_add(&alloced_io_mem); + amd_iommu_stats_add(&total_map_requests); +} + +#endif + +/**************************************************************************** + * + * Interrupt handling functions + * + ****************************************************************************/ + +static void dump_dte_entry(u16 devid) +{ + int i; + + for (i = 0; i < 8; ++i) + pr_err("AMD-Vi: DTE[%d]: %08x\n", i, + amd_iommu_dev_table[devid].data[i]); +} + +static void dump_command(unsigned long phys_addr) +{ + struct iommu_cmd *cmd = phys_to_virt(phys_addr); + int i; + + for (i = 0; i < 4; ++i) + pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]); +} + +static void iommu_print_event(struct amd_iommu *iommu, void *__evt) +{ + u32 *event = __evt; + int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK; + int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; + int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK; + int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; + u64 address = (u64)(((u64)event[3]) << 32) | event[2]; + + printk(KERN_ERR "AMD-Vi: Event logged ["); + + switch (type) { + case EVENT_TYPE_ILL_DEV: + printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x " + "address=0x%016llx flags=0x%04x]\n", + PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + address, flags); + dump_dte_entry(devid); + break; + case EVENT_TYPE_IO_FAULT: + printk("IO_PAGE_FAULT device=%02x:%02x.%x " + "domain=0x%04x address=0x%016llx flags=0x%04x]\n", + PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + domid, address, flags); + break; + case EVENT_TYPE_DEV_TAB_ERR: + printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " + "address=0x%016llx flags=0x%04x]\n", + PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + address, flags); + break; + case EVENT_TYPE_PAGE_TAB_ERR: + printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x " + "domain=0x%04x address=0x%016llx flags=0x%04x]\n", + PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + domid, address, flags); + break; + case EVENT_TYPE_ILL_CMD: + printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); + dump_command(address); + break; + case EVENT_TYPE_CMD_HARD_ERR: + printk("COMMAND_HARDWARE_ERROR address=0x%016llx " + "flags=0x%04x]\n", address, flags); + break; + case EVENT_TYPE_IOTLB_INV_TO: + printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x " + "address=0x%016llx]\n", + PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + address); + break; + case EVENT_TYPE_INV_DEV_REQ: + printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x " + "address=0x%016llx flags=0x%04x]\n", + PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + address, flags); + break; + default: + printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type); + } +} + +static void iommu_poll_events(struct amd_iommu *iommu) +{ + u32 head, tail; + unsigned long flags; + + spin_lock_irqsave(&iommu->lock, flags); + + head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); + tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); + + while (head != tail) { + iommu_print_event(iommu, iommu->evt_buf + head); + head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size; + } + + writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); + + spin_unlock_irqrestore(&iommu->lock, flags); +} + +irqreturn_t amd_iommu_int_thread(int irq, void *data) +{ + struct amd_iommu *iommu; + + for_each_iommu(iommu) + iommu_poll_events(iommu); + + return IRQ_HANDLED; +} + +irqreturn_t amd_iommu_int_handler(int irq, void *data) +{ + return IRQ_WAKE_THREAD; +} + +/**************************************************************************** + * + * IOMMU command queuing functions + * + ****************************************************************************/ + +static int wait_on_sem(volatile u64 *sem) +{ + int i = 0; + + while (*sem == 0 && i < LOOP_TIMEOUT) { + udelay(1); + i += 1; + } + + if (i == LOOP_TIMEOUT) { + pr_alert("AMD-Vi: Completion-Wait loop timed out\n"); + return -EIO; + } + + return 0; +} + +static void copy_cmd_to_buffer(struct amd_iommu *iommu, + struct iommu_cmd *cmd, + u32 tail) +{ + u8 *target; + + target = iommu->cmd_buf + tail; + tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size; + + /* Copy command to buffer */ + memcpy(target, cmd, sizeof(*cmd)); + + /* Tell the IOMMU about it */ + writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); +} + +static void build_completion_wait(struct iommu_cmd *cmd, u64 address) +{ + WARN_ON(address & 0x7ULL); + + memset(cmd, 0, sizeof(*cmd)); + cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK; + cmd->data[1] = upper_32_bits(__pa(address)); + cmd->data[2] = 1; + CMD_SET_TYPE(cmd, CMD_COMPL_WAIT); +} + +static void build_inv_dte(struct iommu_cmd *cmd, u16 devid) +{ + memset(cmd, 0, sizeof(*cmd)); + cmd->data[0] = devid; + CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY); +} + +static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, + size_t size, u16 domid, int pde) +{ + u64 pages; + int s; + + pages = iommu_num_pages(address, size, PAGE_SIZE); + s = 0; + + if (pages > 1) { + /* + * If we have to flush more than one page, flush all + * TLB entries for this domain + */ + address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; + s = 1; + } + + address &= PAGE_MASK; + + memset(cmd, 0, sizeof(*cmd)); + cmd->data[1] |= domid; + cmd->data[2] = lower_32_bits(address); + cmd->data[3] = upper_32_bits(address); + CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES); + if (s) /* size bit - we flush more than one 4kb page */ + cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; + if (pde) /* PDE bit - we wan't flush everything not only the PTEs */ + cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK; +} + +static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, + u64 address, size_t size) +{ + u64 pages; + int s; + + pages = iommu_num_pages(address, size, PAGE_SIZE); + s = 0; + + if (pages > 1) { + /* + * If we have to flush more than one page, flush all + * TLB entries for this domain + */ + address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; + s = 1; + } + + address &= PAGE_MASK; + + memset(cmd, 0, sizeof(*cmd)); + cmd->data[0] = devid; + cmd->data[0] |= (qdep & 0xff) << 24; + cmd->data[1] = devid; + cmd->data[2] = lower_32_bits(address); + cmd->data[3] = upper_32_bits(address); + CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES); + if (s) + cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; +} + +static void build_inv_all(struct iommu_cmd *cmd) +{ + memset(cmd, 0, sizeof(*cmd)); + CMD_SET_TYPE(cmd, CMD_INV_ALL); +} + +/* + * Writes the command to the IOMMUs command buffer and informs the + * hardware about the new command. + */ +static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) +{ + u32 left, tail, head, next_tail; + unsigned long flags; + + WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED); + +again: + spin_lock_irqsave(&iommu->lock, flags); + + head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); + tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); + next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size; + left = (head - next_tail) % iommu->cmd_buf_size; + + if (left <= 2) { + struct iommu_cmd sync_cmd; + volatile u64 sem = 0; + int ret; + + build_completion_wait(&sync_cmd, (u64)&sem); + copy_cmd_to_buffer(iommu, &sync_cmd, tail); + + spin_unlock_irqrestore(&iommu->lock, flags); + + if ((ret = wait_on_sem(&sem)) != 0) + return ret; + + goto again; + } + + copy_cmd_to_buffer(iommu, cmd, tail); + + /* We need to sync now to make sure all commands are processed */ + iommu->need_sync = true; + + spin_unlock_irqrestore(&iommu->lock, flags); + + return 0; +} + +/* + * This function queues a completion wait command into the command + * buffer of an IOMMU + */ +static int iommu_completion_wait(struct amd_iommu *iommu) +{ + struct iommu_cmd cmd; + volatile u64 sem = 0; + int ret; + + if (!iommu->need_sync) + return 0; + + build_completion_wait(&cmd, (u64)&sem); + + ret = iommu_queue_command(iommu, &cmd); + if (ret) + return ret; + + return wait_on_sem(&sem); +} + +static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid) +{ + struct iommu_cmd cmd; + + build_inv_dte(&cmd, devid); + + return iommu_queue_command(iommu, &cmd); +} + +static void iommu_flush_dte_all(struct amd_iommu *iommu) +{ + u32 devid; + + for (devid = 0; devid <= 0xffff; ++devid) + iommu_flush_dte(iommu, devid); + + iommu_completion_wait(iommu); +} + +/* + * This function uses heavy locking and may disable irqs for some time. But + * this is no issue because it is only called during resume. + */ +static void iommu_flush_tlb_all(struct amd_iommu *iommu) +{ + u32 dom_id; + + for (dom_id = 0; dom_id <= 0xffff; ++dom_id) { + struct iommu_cmd cmd; + build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, + dom_id, 1); + iommu_queue_command(iommu, &cmd); + } + + iommu_completion_wait(iommu); +} + +static void iommu_flush_all(struct amd_iommu *iommu) +{ + struct iommu_cmd cmd; + + build_inv_all(&cmd); + + iommu_queue_command(iommu, &cmd); + iommu_completion_wait(iommu); +} + +void iommu_flush_all_caches(struct amd_iommu *iommu) +{ + if (iommu_feature(iommu, FEATURE_IA)) { + iommu_flush_all(iommu); + } else { + iommu_flush_dte_all(iommu); + iommu_flush_tlb_all(iommu); + } +} + +/* + * Command send function for flushing on-device TLB + */ +static int device_flush_iotlb(struct device *dev, u64 address, size_t size) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct amd_iommu *iommu; + struct iommu_cmd cmd; + u16 devid; + int qdep; + + qdep = pci_ats_queue_depth(pdev); + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; + + build_inv_iotlb_pages(&cmd, devid, qdep, address, size); + + return iommu_queue_command(iommu, &cmd); +} + +/* + * Command send function for invalidating a device table entry + */ +static int device_flush_dte(struct device *dev) +{ + struct amd_iommu *iommu; + struct pci_dev *pdev; + u16 devid; + int ret; + + pdev = to_pci_dev(dev); + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; + + ret = iommu_flush_dte(iommu, devid); + if (ret) + return ret; + + if (pci_ats_enabled(pdev)) + ret = device_flush_iotlb(dev, 0, ~0UL); + + return ret; +} + +/* + * TLB invalidation function which is called from the mapping functions. + * It invalidates a single PTE if the range to flush is within a single + * page. Otherwise it flushes the whole TLB of the IOMMU. + */ +static void __domain_flush_pages(struct protection_domain *domain, + u64 address, size_t size, int pde) +{ + struct iommu_dev_data *dev_data; + struct iommu_cmd cmd; + int ret = 0, i; + + build_inv_iommu_pages(&cmd, address, size, domain->id, pde); + + for (i = 0; i < amd_iommus_present; ++i) { + if (!domain->dev_iommu[i]) + continue; + + /* + * Devices of this domain are behind this IOMMU + * We need a TLB flush + */ + ret |= iommu_queue_command(amd_iommus[i], &cmd); + } + + list_for_each_entry(dev_data, &domain->dev_list, list) { + struct pci_dev *pdev = to_pci_dev(dev_data->dev); + + if (!pci_ats_enabled(pdev)) + continue; + + ret |= device_flush_iotlb(dev_data->dev, address, size); + } + + WARN_ON(ret); +} + +static void domain_flush_pages(struct protection_domain *domain, + u64 address, size_t size) +{ + __domain_flush_pages(domain, address, size, 0); +} + +/* Flush the whole IO/TLB for a given protection domain */ +static void domain_flush_tlb(struct protection_domain *domain) +{ + __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0); +} + +/* Flush the whole IO/TLB for a given protection domain - including PDE */ +static void domain_flush_tlb_pde(struct protection_domain *domain) +{ + __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1); +} + +static void domain_flush_complete(struct protection_domain *domain) +{ + int i; + + for (i = 0; i < amd_iommus_present; ++i) { + if (!domain->dev_iommu[i]) + continue; + + /* + * Devices of this domain are behind this IOMMU + * We need to wait for completion of all commands. + */ + iommu_completion_wait(amd_iommus[i]); + } +} + + +/* + * This function flushes the DTEs for all devices in domain + */ +static void domain_flush_devices(struct protection_domain *domain) +{ + struct iommu_dev_data *dev_data; + unsigned long flags; + + spin_lock_irqsave(&domain->lock, flags); + + list_for_each_entry(dev_data, &domain->dev_list, list) + device_flush_dte(dev_data->dev); + + spin_unlock_irqrestore(&domain->lock, flags); +} + +/**************************************************************************** + * + * The functions below are used the create the page table mappings for + * unity mapped regions. + * + ****************************************************************************/ + +/* + * This function is used to add another level to an IO page table. Adding + * another level increases the size of the address space by 9 bits to a size up + * to 64 bits. + */ +static bool increase_address_space(struct protection_domain *domain, + gfp_t gfp) +{ + u64 *pte; + + if (domain->mode == PAGE_MODE_6_LEVEL) + /* address space already 64 bit large */ + return false; + + pte = (void *)get_zeroed_page(gfp); + if (!pte) + return false; + + *pte = PM_LEVEL_PDE(domain->mode, + virt_to_phys(domain->pt_root)); + domain->pt_root = pte; + domain->mode += 1; + domain->updated = true; + + return true; +} + +static u64 *alloc_pte(struct protection_domain *domain, + unsigned long address, + unsigned long page_size, + u64 **pte_page, + gfp_t gfp) +{ + int level, end_lvl; + u64 *pte, *page; + + BUG_ON(!is_power_of_2(page_size)); + + while (address > PM_LEVEL_SIZE(domain->mode)) + increase_address_space(domain, gfp); + + level = domain->mode - 1; + pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; + address = PAGE_SIZE_ALIGN(address, page_size); + end_lvl = PAGE_SIZE_LEVEL(page_size); + + while (level > end_lvl) { + if (!IOMMU_PTE_PRESENT(*pte)) { + page = (u64 *)get_zeroed_page(gfp); + if (!page) + return NULL; + *pte = PM_LEVEL_PDE(level, virt_to_phys(page)); + } + + /* No level skipping support yet */ + if (PM_PTE_LEVEL(*pte) != level) + return NULL; + + level -= 1; + + pte = IOMMU_PTE_PAGE(*pte); + + if (pte_page && level == end_lvl) + *pte_page = pte; + + pte = &pte[PM_LEVEL_INDEX(level, address)]; + } + + return pte; +} + +/* + * This function checks if there is a PTE for a given dma address. If + * there is one, it returns the pointer to it. + */ +static u64 *fetch_pte(struct protection_domain *domain, unsigned long address) +{ + int level; + u64 *pte; + + if (address > PM_LEVEL_SIZE(domain->mode)) + return NULL; + + level = domain->mode - 1; + pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)]; + + while (level > 0) { + + /* Not Present */ + if (!IOMMU_PTE_PRESENT(*pte)) + return NULL; + + /* Large PTE */ + if (PM_PTE_LEVEL(*pte) == 0x07) { + unsigned long pte_mask, __pte; + + /* + * If we have a series of large PTEs, make + * sure to return a pointer to the first one. + */ + pte_mask = PTE_PAGE_SIZE(*pte); + pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1); + __pte = ((unsigned long)pte) & pte_mask; + + return (u64 *)__pte; + } + + /* No level skipping support yet */ + if (PM_PTE_LEVEL(*pte) != level) + return NULL; + + level -= 1; + + /* Walk to the next level */ + pte = IOMMU_PTE_PAGE(*pte); + pte = &pte[PM_LEVEL_INDEX(level, address)]; + } + + return pte; +} + +/* + * Generic mapping functions. It maps a physical address into a DMA + * address space. It allocates the page table pages if necessary. + * In the future it can be extended to a generic mapping function + * supporting all features of AMD IOMMU page tables like level skipping + * and full 64 bit address spaces. + */ +static int iommu_map_page(struct protection_domain *dom, + unsigned long bus_addr, + unsigned long phys_addr, + int prot, + unsigned long page_size) +{ + u64 __pte, *pte; + int i, count; + + if (!(prot & IOMMU_PROT_MASK)) + return -EINVAL; + + bus_addr = PAGE_ALIGN(bus_addr); + phys_addr = PAGE_ALIGN(phys_addr); + count = PAGE_SIZE_PTE_COUNT(page_size); + pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL); + + for (i = 0; i < count; ++i) + if (IOMMU_PTE_PRESENT(pte[i])) + return -EBUSY; + + if (page_size > PAGE_SIZE) { + __pte = PAGE_SIZE_PTE(phys_addr, page_size); + __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC; + } else + __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC; + + if (prot & IOMMU_PROT_IR) + __pte |= IOMMU_PTE_IR; + if (prot & IOMMU_PROT_IW) + __pte |= IOMMU_PTE_IW; + + for (i = 0; i < count; ++i) + pte[i] = __pte; + + update_domain(dom); + + return 0; +} + +static unsigned long iommu_unmap_page(struct protection_domain *dom, + unsigned long bus_addr, + unsigned long page_size) +{ + unsigned long long unmap_size, unmapped; + u64 *pte; + + BUG_ON(!is_power_of_2(page_size)); + + unmapped = 0; + + while (unmapped < page_size) { + + pte = fetch_pte(dom, bus_addr); + + if (!pte) { + /* + * No PTE for this address + * move forward in 4kb steps + */ + unmap_size = PAGE_SIZE; + } else if (PM_PTE_LEVEL(*pte) == 0) { + /* 4kb PTE found for this address */ + unmap_size = PAGE_SIZE; + *pte = 0ULL; + } else { + int count, i; + + /* Large PTE found which maps this address */ + unmap_size = PTE_PAGE_SIZE(*pte); + count = PAGE_SIZE_PTE_COUNT(unmap_size); + for (i = 0; i < count; i++) + pte[i] = 0ULL; + } + + bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size; + unmapped += unmap_size; + } + + BUG_ON(!is_power_of_2(unmapped)); + + return unmapped; +} + +/* + * This function checks if a specific unity mapping entry is needed for + * this specific IOMMU. + */ +static int iommu_for_unity_map(struct amd_iommu *iommu, + struct unity_map_entry *entry) +{ + u16 bdf, i; + + for (i = entry->devid_start; i <= entry->devid_end; ++i) { + bdf = amd_iommu_alias_table[i]; + if (amd_iommu_rlookup_table[bdf] == iommu) + return 1; + } + + return 0; +} + +/* + * This function actually applies the mapping to the page table of the + * dma_ops domain. + */ +static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, + struct unity_map_entry *e) +{ + u64 addr; + int ret; + + for (addr = e->address_start; addr < e->address_end; + addr += PAGE_SIZE) { + ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot, + PAGE_SIZE); + if (ret) + return ret; + /* + * if unity mapping is in aperture range mark the page + * as allocated in the aperture + */ + if (addr < dma_dom->aperture_size) + __set_bit(addr >> PAGE_SHIFT, + dma_dom->aperture[0]->bitmap); + } + + return 0; +} + +/* + * Init the unity mappings for a specific IOMMU in the system + * + * Basically iterates over all unity mapping entries and applies them to + * the default domain DMA of that IOMMU if necessary. + */ +static int iommu_init_unity_mappings(struct amd_iommu *iommu) +{ + struct unity_map_entry *entry; + int ret; + + list_for_each_entry(entry, &amd_iommu_unity_map, list) { + if (!iommu_for_unity_map(iommu, entry)) + continue; + ret = dma_ops_unity_map(iommu->default_dom, entry); + if (ret) + return ret; + } + + return 0; +} + +/* + * Inits the unity mappings required for a specific device + */ +static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, + u16 devid) +{ + struct unity_map_entry *e; + int ret; + + list_for_each_entry(e, &amd_iommu_unity_map, list) { + if (!(devid >= e->devid_start && devid <= e->devid_end)) + continue; + ret = dma_ops_unity_map(dma_dom, e); + if (ret) + return ret; + } + + return 0; +} + +/**************************************************************************** + * + * The next functions belong to the address allocator for the dma_ops + * interface functions. They work like the allocators in the other IOMMU + * drivers. Its basically a bitmap which marks the allocated pages in + * the aperture. Maybe it could be enhanced in the future to a more + * efficient allocator. + * + ****************************************************************************/ + +/* + * The address allocator core functions. + * + * called with domain->lock held + */ + +/* + * Used to reserve address ranges in the aperture (e.g. for exclusion + * ranges. + */ +static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, + unsigned long start_page, + unsigned int pages) +{ + unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT; + + if (start_page + pages > last_page) + pages = last_page - start_page; + + for (i = start_page; i < start_page + pages; ++i) { + int index = i / APERTURE_RANGE_PAGES; + int page = i % APERTURE_RANGE_PAGES; + __set_bit(page, dom->aperture[index]->bitmap); + } +} + +/* + * This function is used to add a new aperture range to an existing + * aperture in case of dma_ops domain allocation or address allocation + * failure. + */ +static int alloc_new_range(struct dma_ops_domain *dma_dom, + bool populate, gfp_t gfp) +{ + int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT; + struct amd_iommu *iommu; + unsigned long i; + +#ifdef CONFIG_IOMMU_STRESS + populate = false; +#endif + + if (index >= APERTURE_MAX_RANGES) + return -ENOMEM; + + dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp); + if (!dma_dom->aperture[index]) + return -ENOMEM; + + dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp); + if (!dma_dom->aperture[index]->bitmap) + goto out_free; + + dma_dom->aperture[index]->offset = dma_dom->aperture_size; + + if (populate) { + unsigned long address = dma_dom->aperture_size; + int i, num_ptes = APERTURE_RANGE_PAGES / 512; + u64 *pte, *pte_page; + + for (i = 0; i < num_ptes; ++i) { + pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE, + &pte_page, gfp); + if (!pte) + goto out_free; + + dma_dom->aperture[index]->pte_pages[i] = pte_page; + + address += APERTURE_RANGE_SIZE / 64; + } + } + + dma_dom->aperture_size += APERTURE_RANGE_SIZE; + + /* Initialize the exclusion range if necessary */ + for_each_iommu(iommu) { + if (iommu->exclusion_start && + iommu->exclusion_start >= dma_dom->aperture[index]->offset + && iommu->exclusion_start < dma_dom->aperture_size) { + unsigned long startpage; + int pages = iommu_num_pages(iommu->exclusion_start, + iommu->exclusion_length, + PAGE_SIZE); + startpage = iommu->exclusion_start >> PAGE_SHIFT; + dma_ops_reserve_addresses(dma_dom, startpage, pages); + } + } + + /* + * Check for areas already mapped as present in the new aperture + * range and mark those pages as reserved in the allocator. Such + * mappings may already exist as a result of requested unity + * mappings for devices. + */ + for (i = dma_dom->aperture[index]->offset; + i < dma_dom->aperture_size; + i += PAGE_SIZE) { + u64 *pte = fetch_pte(&dma_dom->domain, i); + if (!pte || !IOMMU_PTE_PRESENT(*pte)) + continue; + + dma_ops_reserve_addresses(dma_dom, i << PAGE_SHIFT, 1); + } + + update_domain(&dma_dom->domain); + + return 0; + +out_free: + update_domain(&dma_dom->domain); + + free_page((unsigned long)dma_dom->aperture[index]->bitmap); + + kfree(dma_dom->aperture[index]); + dma_dom->aperture[index] = NULL; + + return -ENOMEM; +} + +static unsigned long dma_ops_area_alloc(struct device *dev, + struct dma_ops_domain *dom, + unsigned int pages, + unsigned long align_mask, + u64 dma_mask, + unsigned long start) +{ + unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE; + int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT; + int i = start >> APERTURE_RANGE_SHIFT; + unsigned long boundary_size; + unsigned long address = -1; + unsigned long limit; + + next_bit >>= PAGE_SHIFT; + + boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, + PAGE_SIZE) >> PAGE_SHIFT; + + for (;i < max_index; ++i) { + unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT; + + if (dom->aperture[i]->offset >= dma_mask) + break; + + limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset, + dma_mask >> PAGE_SHIFT); + + address = iommu_area_alloc(dom->aperture[i]->bitmap, + limit, next_bit, pages, 0, + boundary_size, align_mask); + if (address != -1) { + address = dom->aperture[i]->offset + + (address << PAGE_SHIFT); + dom->next_address = address + (pages << PAGE_SHIFT); + break; + } + + next_bit = 0; + } + + return address; +} + +static unsigned long dma_ops_alloc_addresses(struct device *dev, + struct dma_ops_domain *dom, + unsigned int pages, + unsigned long align_mask, + u64 dma_mask) +{ + unsigned long address; + +#ifdef CONFIG_IOMMU_STRESS + dom->next_address = 0; + dom->need_flush = true; +#endif + + address = dma_ops_area_alloc(dev, dom, pages, align_mask, + dma_mask, dom->next_address); + + if (address == -1) { + dom->next_address = 0; + address = dma_ops_area_alloc(dev, dom, pages, align_mask, + dma_mask, 0); + dom->need_flush = true; + } + + if (unlikely(address == -1)) + address = DMA_ERROR_CODE; + + WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size); + + return address; +} + +/* + * The address free function. + * + * called with domain->lock held + */ +static void dma_ops_free_addresses(struct dma_ops_domain *dom, + unsigned long address, + unsigned int pages) +{ + unsigned i = address >> APERTURE_RANGE_SHIFT; + struct aperture_range *range = dom->aperture[i]; + + BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL); + +#ifdef CONFIG_IOMMU_STRESS + if (i < 4) + return; +#endif + + if (address >= dom->next_address) + dom->need_flush = true; + + address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT; + + bitmap_clear(range->bitmap, address, pages); + +} + +/**************************************************************************** + * + * The next functions belong to the domain allocation. A domain is + * allocated for every IOMMU as the default domain. If device isolation + * is enabled, every device get its own domain. The most important thing + * about domains is the page table mapping the DMA address space they + * contain. + * + ****************************************************************************/ + +/* + * This function adds a protection domain to the global protection domain list + */ +static void add_domain_to_list(struct protection_domain *domain) +{ + unsigned long flags; + + spin_lock_irqsave(&amd_iommu_pd_lock, flags); + list_add(&domain->list, &amd_iommu_pd_list); + spin_unlock_irqrestore(&amd_iommu_pd_lock, flags); +} + +/* + * This function removes a protection domain to the global + * protection domain list + */ +static void del_domain_from_list(struct protection_domain *domain) +{ + unsigned long flags; + + spin_lock_irqsave(&amd_iommu_pd_lock, flags); + list_del(&domain->list); + spin_unlock_irqrestore(&amd_iommu_pd_lock, flags); +} + +static u16 domain_id_alloc(void) +{ + unsigned long flags; + int id; + + write_lock_irqsave(&amd_iommu_devtable_lock, flags); + id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID); + BUG_ON(id == 0); + if (id > 0 && id < MAX_DOMAIN_ID) + __set_bit(id, amd_iommu_pd_alloc_bitmap); + else + id = 0; + write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + + return id; +} + +static void domain_id_free(int id) +{ + unsigned long flags; + + write_lock_irqsave(&amd_iommu_devtable_lock, flags); + if (id > 0 && id < MAX_DOMAIN_ID) + __clear_bit(id, amd_iommu_pd_alloc_bitmap); + write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); +} + +static void free_pagetable(struct protection_domain *domain) +{ + int i, j; + u64 *p1, *p2, *p3; + + p1 = domain->pt_root; + + if (!p1) + return; + + for (i = 0; i < 512; ++i) { + if (!IOMMU_PTE_PRESENT(p1[i])) + continue; + + p2 = IOMMU_PTE_PAGE(p1[i]); + for (j = 0; j < 512; ++j) { + if (!IOMMU_PTE_PRESENT(p2[j])) + continue; + p3 = IOMMU_PTE_PAGE(p2[j]); + free_page((unsigned long)p3); + } + + free_page((unsigned long)p2); + } + + free_page((unsigned long)p1); + + domain->pt_root = NULL; +} + +/* + * Free a domain, only used if something went wrong in the + * allocation path and we need to free an already allocated page table + */ +static void dma_ops_domain_free(struct dma_ops_domain *dom) +{ + int i; + + if (!dom) + return; + + del_domain_from_list(&dom->domain); + + free_pagetable(&dom->domain); + + for (i = 0; i < APERTURE_MAX_RANGES; ++i) { + if (!dom->aperture[i]) + continue; + free_page((unsigned long)dom->aperture[i]->bitmap); + kfree(dom->aperture[i]); + } + + kfree(dom); +} + +/* + * Allocates a new protection domain usable for the dma_ops functions. + * It also initializes the page table and the address allocator data + * structures required for the dma_ops interface + */ +static struct dma_ops_domain *dma_ops_domain_alloc(void) +{ + struct dma_ops_domain *dma_dom; + + dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL); + if (!dma_dom) + return NULL; + + spin_lock_init(&dma_dom->domain.lock); + + dma_dom->domain.id = domain_id_alloc(); + if (dma_dom->domain.id == 0) + goto free_dma_dom; + INIT_LIST_HEAD(&dma_dom->domain.dev_list); + dma_dom->domain.mode = PAGE_MODE_2_LEVEL; + dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL); + dma_dom->domain.flags = PD_DMA_OPS_MASK; + dma_dom->domain.priv = dma_dom; + if (!dma_dom->domain.pt_root) + goto free_dma_dom; + + dma_dom->need_flush = false; + dma_dom->target_dev = 0xffff; + + add_domain_to_list(&dma_dom->domain); + + if (alloc_new_range(dma_dom, true, GFP_KERNEL)) + goto free_dma_dom; + + /* + * mark the first page as allocated so we never return 0 as + * a valid dma-address. So we can use 0 as error value + */ + dma_dom->aperture[0]->bitmap[0] = 1; + dma_dom->next_address = 0; + + + return dma_dom; + +free_dma_dom: + dma_ops_domain_free(dma_dom); + + return NULL; +} + +/* + * little helper function to check whether a given protection domain is a + * dma_ops domain + */ +static bool dma_ops_domain(struct protection_domain *domain) +{ + return domain->flags & PD_DMA_OPS_MASK; +} + +static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats) +{ + u64 pte_root = virt_to_phys(domain->pt_root); + u32 flags = 0; + + pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK) + << DEV_ENTRY_MODE_SHIFT; + pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV; + + if (ats) + flags |= DTE_FLAG_IOTLB; + + amd_iommu_dev_table[devid].data[3] |= flags; + amd_iommu_dev_table[devid].data[2] = domain->id; + amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root); + amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root); +} + +static void clear_dte_entry(u16 devid) +{ + /* remove entry from the device table seen by the hardware */ + amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV; + amd_iommu_dev_table[devid].data[1] = 0; + amd_iommu_dev_table[devid].data[2] = 0; + + amd_iommu_apply_erratum_63(devid); +} + +static void do_attach(struct device *dev, struct protection_domain *domain) +{ + struct iommu_dev_data *dev_data; + struct amd_iommu *iommu; + struct pci_dev *pdev; + bool ats = false; + u16 devid; + + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; + dev_data = get_dev_data(dev); + pdev = to_pci_dev(dev); + + if (amd_iommu_iotlb_sup) + ats = pci_ats_enabled(pdev); + + /* Update data structures */ + dev_data->domain = domain; + list_add(&dev_data->list, &domain->dev_list); + set_dte_entry(devid, domain, ats); + + /* Do reference counting */ + domain->dev_iommu[iommu->index] += 1; + domain->dev_cnt += 1; + + /* Flush the DTE entry */ + device_flush_dte(dev); +} + +static void do_detach(struct device *dev) +{ + struct iommu_dev_data *dev_data; + struct amd_iommu *iommu; + u16 devid; + + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; + dev_data = get_dev_data(dev); + + /* decrease reference counters */ + dev_data->domain->dev_iommu[iommu->index] -= 1; + dev_data->domain->dev_cnt -= 1; + + /* Update data structures */ + dev_data->domain = NULL; + list_del(&dev_data->list); + clear_dte_entry(devid); + + /* Flush the DTE entry */ + device_flush_dte(dev); +} + +/* + * If a device is not yet associated with a domain, this function does + * assigns it visible for the hardware + */ +static int __attach_device(struct device *dev, + struct protection_domain *domain) +{ + struct iommu_dev_data *dev_data, *alias_data; + int ret; + + dev_data = get_dev_data(dev); + alias_data = get_dev_data(dev_data->alias); + + if (!alias_data) + return -EINVAL; + + /* lock domain */ + spin_lock(&domain->lock); + + /* Some sanity checks */ + ret = -EBUSY; + if (alias_data->domain != NULL && + alias_data->domain != domain) + goto out_unlock; + + if (dev_data->domain != NULL && + dev_data->domain != domain) + goto out_unlock; + + /* Do real assignment */ + if (dev_data->alias != dev) { + alias_data = get_dev_data(dev_data->alias); + if (alias_data->domain == NULL) + do_attach(dev_data->alias, domain); + + atomic_inc(&alias_data->bind); + } + + if (dev_data->domain == NULL) + do_attach(dev, domain); + + atomic_inc(&dev_data->bind); + + ret = 0; + +out_unlock: + + /* ready */ + spin_unlock(&domain->lock); + + return ret; +} + +/* + * If a device is not yet associated with a domain, this function does + * assigns it visible for the hardware + */ +static int attach_device(struct device *dev, + struct protection_domain *domain) +{ + struct pci_dev *pdev = to_pci_dev(dev); + unsigned long flags; + int ret; + + if (amd_iommu_iotlb_sup) + pci_enable_ats(pdev, PAGE_SHIFT); + + write_lock_irqsave(&amd_iommu_devtable_lock, flags); + ret = __attach_device(dev, domain); + write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + + /* + * We might boot into a crash-kernel here. The crashed kernel + * left the caches in the IOMMU dirty. So we have to flush + * here to evict all dirty stuff. + */ + domain_flush_tlb_pde(domain); + + return ret; +} + +/* + * Removes a device from a protection domain (unlocked) + */ +static void __detach_device(struct device *dev) +{ + struct iommu_dev_data *dev_data = get_dev_data(dev); + struct iommu_dev_data *alias_data; + struct protection_domain *domain; + unsigned long flags; + + BUG_ON(!dev_data->domain); + + domain = dev_data->domain; + + spin_lock_irqsave(&domain->lock, flags); + + if (dev_data->alias != dev) { + alias_data = get_dev_data(dev_data->alias); + if (atomic_dec_and_test(&alias_data->bind)) + do_detach(dev_data->alias); + } + + if (atomic_dec_and_test(&dev_data->bind)) + do_detach(dev); + + spin_unlock_irqrestore(&domain->lock, flags); + + /* + * If we run in passthrough mode the device must be assigned to the + * passthrough domain if it is detached from any other domain. + * Make sure we can deassign from the pt_domain itself. + */ + if (iommu_pass_through && + (dev_data->domain == NULL && domain != pt_domain)) + __attach_device(dev, pt_domain); +} + +/* + * Removes a device from a protection domain (with devtable_lock held) + */ +static void detach_device(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + unsigned long flags; + + /* lock device table */ + write_lock_irqsave(&amd_iommu_devtable_lock, flags); + __detach_device(dev); + write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + + if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev)) + pci_disable_ats(pdev); +} + +/* + * Find out the protection domain structure for a given PCI device. This + * will give us the pointer to the page table root for example. + */ +static struct protection_domain *domain_for_device(struct device *dev) +{ + struct protection_domain *dom; + struct iommu_dev_data *dev_data, *alias_data; + unsigned long flags; + u16 devid; + + devid = get_device_id(dev); + dev_data = get_dev_data(dev); + alias_data = get_dev_data(dev_data->alias); + if (!alias_data) + return NULL; + + read_lock_irqsave(&amd_iommu_devtable_lock, flags); + dom = dev_data->domain; + if (dom == NULL && + alias_data->domain != NULL) { + __attach_device(dev, alias_data->domain); + dom = alias_data->domain; + } + + read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + + return dom; +} + +static int device_change_notifier(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct device *dev = data; + u16 devid; + struct protection_domain *domain; + struct dma_ops_domain *dma_domain; + struct amd_iommu *iommu; + unsigned long flags; + + if (!check_device(dev)) + return 0; + + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; + + switch (action) { + case BUS_NOTIFY_UNBOUND_DRIVER: + + domain = domain_for_device(dev); + + if (!domain) + goto out; + if (iommu_pass_through) + break; + detach_device(dev); + break; + case BUS_NOTIFY_ADD_DEVICE: + + iommu_init_device(dev); + + domain = domain_for_device(dev); + + /* allocate a protection domain if a device is added */ + dma_domain = find_protection_domain(devid); + if (dma_domain) + goto out; + dma_domain = dma_ops_domain_alloc(); + if (!dma_domain) + goto out; + dma_domain->target_dev = devid; + + spin_lock_irqsave(&iommu_pd_list_lock, flags); + list_add_tail(&dma_domain->list, &iommu_pd_list); + spin_unlock_irqrestore(&iommu_pd_list_lock, flags); + + break; + case BUS_NOTIFY_DEL_DEVICE: + + iommu_uninit_device(dev); + + default: + goto out; + } + + device_flush_dte(dev); + iommu_completion_wait(iommu); + +out: + return 0; +} + +static struct notifier_block device_nb = { + .notifier_call = device_change_notifier, +}; + +void amd_iommu_init_notifier(void) +{ + bus_register_notifier(&pci_bus_type, &device_nb); +} + +/***************************************************************************** + * + * The next functions belong to the dma_ops mapping/unmapping code. + * + *****************************************************************************/ + +/* + * In the dma_ops path we only have the struct device. This function + * finds the corresponding IOMMU, the protection domain and the + * requestor id for a given device. + * If the device is not yet associated with a domain this is also done + * in this function. + */ +static struct protection_domain *get_domain(struct device *dev) +{ + struct protection_domain *domain; + struct dma_ops_domain *dma_dom; + u16 devid = get_device_id(dev); + + if (!check_device(dev)) + return ERR_PTR(-EINVAL); + + domain = domain_for_device(dev); + if (domain != NULL && !dma_ops_domain(domain)) + return ERR_PTR(-EBUSY); + + if (domain != NULL) + return domain; + + /* Device not bount yet - bind it */ + dma_dom = find_protection_domain(devid); + if (!dma_dom) + dma_dom = amd_iommu_rlookup_table[devid]->default_dom; + attach_device(dev, &dma_dom->domain); + DUMP_printk("Using protection domain %d for device %s\n", + dma_dom->domain.id, dev_name(dev)); + + return &dma_dom->domain; +} + +static void update_device_table(struct protection_domain *domain) +{ + struct iommu_dev_data *dev_data; + + list_for_each_entry(dev_data, &domain->dev_list, list) { + struct pci_dev *pdev = to_pci_dev(dev_data->dev); + u16 devid = get_device_id(dev_data->dev); + set_dte_entry(devid, domain, pci_ats_enabled(pdev)); + } +} + +static void update_domain(struct protection_domain *domain) +{ + if (!domain->updated) + return; + + update_device_table(domain); + + domain_flush_devices(domain); + domain_flush_tlb_pde(domain); + + domain->updated = false; +} + +/* + * This function fetches the PTE for a given address in the aperture + */ +static u64* dma_ops_get_pte(struct dma_ops_domain *dom, + unsigned long address) +{ + struct aperture_range *aperture; + u64 *pte, *pte_page; + + aperture = dom->aperture[APERTURE_RANGE_INDEX(address)]; + if (!aperture) + return NULL; + + pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)]; + if (!pte) { + pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page, + GFP_ATOMIC); + aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page; + } else + pte += PM_LEVEL_INDEX(0, address); + + update_domain(&dom->domain); + + return pte; +} + +/* + * This is the generic map function. It maps one 4kb page at paddr to + * the given address in the DMA address space for the domain. + */ +static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom, + unsigned long address, + phys_addr_t paddr, + int direction) +{ + u64 *pte, __pte; + + WARN_ON(address > dom->aperture_size); + + paddr &= PAGE_MASK; + + pte = dma_ops_get_pte(dom, address); + if (!pte) + return DMA_ERROR_CODE; + + __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC; + + if (direction == DMA_TO_DEVICE) + __pte |= IOMMU_PTE_IR; + else if (direction == DMA_FROM_DEVICE) + __pte |= IOMMU_PTE_IW; + else if (direction == DMA_BIDIRECTIONAL) + __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW; + + WARN_ON(*pte); + + *pte = __pte; + + return (dma_addr_t)address; +} + +/* + * The generic unmapping function for on page in the DMA address space. + */ +static void dma_ops_domain_unmap(struct dma_ops_domain *dom, + unsigned long address) +{ + struct aperture_range *aperture; + u64 *pte; + + if (address >= dom->aperture_size) + return; + + aperture = dom->aperture[APERTURE_RANGE_INDEX(address)]; + if (!aperture) + return; + + pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)]; + if (!pte) + return; + + pte += PM_LEVEL_INDEX(0, address); + + WARN_ON(!*pte); + + *pte = 0ULL; +} + +/* + * This function contains common code for mapping of a physically + * contiguous memory region into DMA address space. It is used by all + * mapping functions provided with this IOMMU driver. + * Must be called with the domain lock held. + */ +static dma_addr_t __map_single(struct device *dev, + struct dma_ops_domain *dma_dom, + phys_addr_t paddr, + size_t size, + int dir, + bool align, + u64 dma_mask) +{ + dma_addr_t offset = paddr & ~PAGE_MASK; + dma_addr_t address, start, ret; + unsigned int pages; + unsigned long align_mask = 0; + int i; + + pages = iommu_num_pages(paddr, size, PAGE_SIZE); + paddr &= PAGE_MASK; + + INC_STATS_COUNTER(total_map_requests); + + if (pages > 1) + INC_STATS_COUNTER(cross_page); + + if (align) + align_mask = (1UL << get_order(size)) - 1; + +retry: + address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask, + dma_mask); + if (unlikely(address == DMA_ERROR_CODE)) { + /* + * setting next_address here will let the address + * allocator only scan the new allocated range in the + * first run. This is a small optimization. + */ + dma_dom->next_address = dma_dom->aperture_size; + + if (alloc_new_range(dma_dom, false, GFP_ATOMIC)) + goto out; + + /* + * aperture was successfully enlarged by 128 MB, try + * allocation again + */ + goto retry; + } + + start = address; + for (i = 0; i < pages; ++i) { + ret = dma_ops_domain_map(dma_dom, start, paddr, dir); + if (ret == DMA_ERROR_CODE) + goto out_unmap; + + paddr += PAGE_SIZE; + start += PAGE_SIZE; + } + address += offset; + + ADD_STATS_COUNTER(alloced_io_mem, size); + + if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) { + domain_flush_tlb(&dma_dom->domain); + dma_dom->need_flush = false; + } else if (unlikely(amd_iommu_np_cache)) + domain_flush_pages(&dma_dom->domain, address, size); + +out: + return address; + +out_unmap: + + for (--i; i >= 0; --i) { + start -= PAGE_SIZE; + dma_ops_domain_unmap(dma_dom, start); + } + + dma_ops_free_addresses(dma_dom, address, pages); + + return DMA_ERROR_CODE; +} + +/* + * Does the reverse of the __map_single function. Must be called with + * the domain lock held too + */ +static void __unmap_single(struct dma_ops_domain *dma_dom, + dma_addr_t dma_addr, + size_t size, + int dir) +{ + dma_addr_t flush_addr; + dma_addr_t i, start; + unsigned int pages; + + if ((dma_addr == DMA_ERROR_CODE) || + (dma_addr + size > dma_dom->aperture_size)) + return; + + flush_addr = dma_addr; + pages = iommu_num_pages(dma_addr, size, PAGE_SIZE); + dma_addr &= PAGE_MASK; + start = dma_addr; + + for (i = 0; i < pages; ++i) { + dma_ops_domain_unmap(dma_dom, start); + start += PAGE_SIZE; + } + + SUB_STATS_COUNTER(alloced_io_mem, size); + + dma_ops_free_addresses(dma_dom, dma_addr, pages); + + if (amd_iommu_unmap_flush || dma_dom->need_flush) { + domain_flush_pages(&dma_dom->domain, flush_addr, size); + dma_dom->need_flush = false; + } +} + +/* + * The exported map_single function for dma_ops. + */ +static dma_addr_t map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, + struct dma_attrs *attrs) +{ + unsigned long flags; + struct protection_domain *domain; + dma_addr_t addr; + u64 dma_mask; + phys_addr_t paddr = page_to_phys(page) + offset; + + INC_STATS_COUNTER(cnt_map_single); + + domain = get_domain(dev); + if (PTR_ERR(domain) == -EINVAL) + return (dma_addr_t)paddr; + else if (IS_ERR(domain)) + return DMA_ERROR_CODE; + + dma_mask = *dev->dma_mask; + + spin_lock_irqsave(&domain->lock, flags); + + addr = __map_single(dev, domain->priv, paddr, size, dir, false, + dma_mask); + if (addr == DMA_ERROR_CODE) + goto out; + + domain_flush_complete(domain); + +out: + spin_unlock_irqrestore(&domain->lock, flags); + + return addr; +} + +/* + * The exported unmap_single function for dma_ops. + */ +static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + unsigned long flags; + struct protection_domain *domain; + + INC_STATS_COUNTER(cnt_unmap_single); + + domain = get_domain(dev); + if (IS_ERR(domain)) + return; + + spin_lock_irqsave(&domain->lock, flags); + + __unmap_single(domain->priv, dma_addr, size, dir); + + domain_flush_complete(domain); + + spin_unlock_irqrestore(&domain->lock, flags); +} + +/* + * This is a special map_sg function which is used if we should map a + * device which is not handled by an AMD IOMMU in the system. + */ +static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist, + int nelems, int dir) +{ + struct scatterlist *s; + int i; + + for_each_sg(sglist, s, nelems, i) { + s->dma_address = (dma_addr_t)sg_phys(s); + s->dma_length = s->length; + } + + return nelems; +} + +/* + * The exported map_sg function for dma_ops (handles scatter-gather + * lists). + */ +static int map_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction dir, + struct dma_attrs *attrs) +{ + unsigned long flags; + struct protection_domain *domain; + int i; + struct scatterlist *s; + phys_addr_t paddr; + int mapped_elems = 0; + u64 dma_mask; + + INC_STATS_COUNTER(cnt_map_sg); + + domain = get_domain(dev); + if (PTR_ERR(domain) == -EINVAL) + return map_sg_no_iommu(dev, sglist, nelems, dir); + else if (IS_ERR(domain)) + return 0; + + dma_mask = *dev->dma_mask; + + spin_lock_irqsave(&domain->lock, flags); + + for_each_sg(sglist, s, nelems, i) { + paddr = sg_phys(s); + + s->dma_address = __map_single(dev, domain->priv, + paddr, s->length, dir, false, + dma_mask); + + if (s->dma_address) { + s->dma_length = s->length; + mapped_elems++; + } else + goto unmap; + } + + domain_flush_complete(domain); + +out: + spin_unlock_irqrestore(&domain->lock, flags); + + return mapped_elems; +unmap: + for_each_sg(sglist, s, mapped_elems, i) { + if (s->dma_address) + __unmap_single(domain->priv, s->dma_address, + s->dma_length, dir); + s->dma_address = s->dma_length = 0; + } + + mapped_elems = 0; + + goto out; +} + +/* + * The exported map_sg function for dma_ops (handles scatter-gather + * lists). + */ +static void unmap_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction dir, + struct dma_attrs *attrs) +{ + unsigned long flags; + struct protection_domain *domain; + struct scatterlist *s; + int i; + + INC_STATS_COUNTER(cnt_unmap_sg); + + domain = get_domain(dev); + if (IS_ERR(domain)) + return; + + spin_lock_irqsave(&domain->lock, flags); + + for_each_sg(sglist, s, nelems, i) { + __unmap_single(domain->priv, s->dma_address, + s->dma_length, dir); + s->dma_address = s->dma_length = 0; + } + + domain_flush_complete(domain); + + spin_unlock_irqrestore(&domain->lock, flags); +} + +/* + * The exported alloc_coherent function for dma_ops. + */ +static void *alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_addr, gfp_t flag) +{ + unsigned long flags; + void *virt_addr; + struct protection_domain *domain; + phys_addr_t paddr; + u64 dma_mask = dev->coherent_dma_mask; + + INC_STATS_COUNTER(cnt_alloc_coherent); + + domain = get_domain(dev); + if (PTR_ERR(domain) == -EINVAL) { + virt_addr = (void *)__get_free_pages(flag, get_order(size)); + *dma_addr = __pa(virt_addr); + return virt_addr; + } else if (IS_ERR(domain)) + return NULL; + + dma_mask = dev->coherent_dma_mask; + flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); + flag |= __GFP_ZERO; + + virt_addr = (void *)__get_free_pages(flag, get_order(size)); + if (!virt_addr) + return NULL; + + paddr = virt_to_phys(virt_addr); + + if (!dma_mask) + dma_mask = *dev->dma_mask; + + spin_lock_irqsave(&domain->lock, flags); + + *dma_addr = __map_single(dev, domain->priv, paddr, + size, DMA_BIDIRECTIONAL, true, dma_mask); + + if (*dma_addr == DMA_ERROR_CODE) { + spin_unlock_irqrestore(&domain->lock, flags); + goto out_free; + } + + domain_flush_complete(domain); + + spin_unlock_irqrestore(&domain->lock, flags); + + return virt_addr; + +out_free: + + free_pages((unsigned long)virt_addr, get_order(size)); + + return NULL; +} + +/* + * The exported free_coherent function for dma_ops. + */ +static void free_coherent(struct device *dev, size_t size, + void *virt_addr, dma_addr_t dma_addr) +{ + unsigned long flags; + struct protection_domain *domain; + + INC_STATS_COUNTER(cnt_free_coherent); + + domain = get_domain(dev); + if (IS_ERR(domain)) + goto free_mem; + + spin_lock_irqsave(&domain->lock, flags); + + __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL); + + domain_flush_complete(domain); + + spin_unlock_irqrestore(&domain->lock, flags); + +free_mem: + free_pages((unsigned long)virt_addr, get_order(size)); +} + +/* + * This function is called by the DMA layer to find out if we can handle a + * particular device. It is part of the dma_ops. + */ +static int amd_iommu_dma_supported(struct device *dev, u64 mask) +{ + return check_device(dev); +} + +/* + * The function for pre-allocating protection domains. + * + * If the driver core informs the DMA layer if a driver grabs a device + * we don't need to preallocate the protection domains anymore. + * For now we have to. + */ +static void prealloc_protection_domains(void) +{ + struct pci_dev *dev = NULL; + struct dma_ops_domain *dma_dom; + u16 devid; + + for_each_pci_dev(dev) { + + /* Do we handle this device? */ + if (!check_device(&dev->dev)) + continue; + + /* Is there already any domain for it? */ + if (domain_for_device(&dev->dev)) + continue; + + devid = get_device_id(&dev->dev); + + dma_dom = dma_ops_domain_alloc(); + if (!dma_dom) + continue; + init_unity_mappings_for_device(dma_dom, devid); + dma_dom->target_dev = devid; + + attach_device(&dev->dev, &dma_dom->domain); + + list_add_tail(&dma_dom->list, &iommu_pd_list); + } +} + +static struct dma_map_ops amd_iommu_dma_ops = { + .alloc_coherent = alloc_coherent, + .free_coherent = free_coherent, + .map_page = map_page, + .unmap_page = unmap_page, + .map_sg = map_sg, + .unmap_sg = unmap_sg, + .dma_supported = amd_iommu_dma_supported, +}; + +static unsigned device_dma_ops_init(void) +{ + struct pci_dev *pdev = NULL; + unsigned unhandled = 0; + + for_each_pci_dev(pdev) { + if (!check_device(&pdev->dev)) { + unhandled += 1; + continue; + } + + pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops; + } + + return unhandled; +} + +/* + * The function which clues the AMD IOMMU driver into dma_ops. + */ + +void __init amd_iommu_init_api(void) +{ + register_iommu(&amd_iommu_ops); +} + +int __init amd_iommu_init_dma_ops(void) +{ + struct amd_iommu *iommu; + int ret, unhandled; + + /* + * first allocate a default protection domain for every IOMMU we + * found in the system. Devices not assigned to any other + * protection domain will be assigned to the default one. + */ + for_each_iommu(iommu) { + iommu->default_dom = dma_ops_domain_alloc(); + if (iommu->default_dom == NULL) + return -ENOMEM; + iommu->default_dom->domain.flags |= PD_DEFAULT_MASK; + ret = iommu_init_unity_mappings(iommu); + if (ret) + goto free_domains; + } + + /* + * Pre-allocate the protection domains for each device. + */ + prealloc_protection_domains(); + + iommu_detected = 1; + swiotlb = 0; + + /* Make the driver finally visible to the drivers */ + unhandled = device_dma_ops_init(); + if (unhandled && max_pfn > MAX_DMA32_PFN) { + /* There are unhandled devices - initialize swiotlb for them */ + swiotlb = 1; + } + + amd_iommu_stats_init(); + + return 0; + +free_domains: + + for_each_iommu(iommu) { + if (iommu->default_dom) + dma_ops_domain_free(iommu->default_dom); + } + + return ret; +} + +/***************************************************************************** + * + * The following functions belong to the exported interface of AMD IOMMU + * + * This interface allows access to lower level functions of the IOMMU + * like protection domain handling and assignement of devices to domains + * which is not possible with the dma_ops interface. + * + *****************************************************************************/ + +static void cleanup_domain(struct protection_domain *domain) +{ + struct iommu_dev_data *dev_data, *next; + unsigned long flags; + + write_lock_irqsave(&amd_iommu_devtable_lock, flags); + + list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) { + struct device *dev = dev_data->dev; + + __detach_device(dev); + atomic_set(&dev_data->bind, 0); + } + + write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); +} + +static void protection_domain_free(struct protection_domain *domain) +{ + if (!domain) + return; + + del_domain_from_list(domain); + + if (domain->id) + domain_id_free(domain->id); + + kfree(domain); +} + +static struct protection_domain *protection_domain_alloc(void) +{ + struct protection_domain *domain; + + domain = kzalloc(sizeof(*domain), GFP_KERNEL); + if (!domain) + return NULL; + + spin_lock_init(&domain->lock); + mutex_init(&domain->api_lock); + domain->id = domain_id_alloc(); + if (!domain->id) + goto out_err; + INIT_LIST_HEAD(&domain->dev_list); + + add_domain_to_list(domain); + + return domain; + +out_err: + kfree(domain); + + return NULL; +} + +static int amd_iommu_domain_init(struct iommu_domain *dom) +{ + struct protection_domain *domain; + + domain = protection_domain_alloc(); + if (!domain) + goto out_free; + + domain->mode = PAGE_MODE_3_LEVEL; + domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); + if (!domain->pt_root) + goto out_free; + + dom->priv = domain; + + return 0; + +out_free: + protection_domain_free(domain); + + return -ENOMEM; +} + +static void amd_iommu_domain_destroy(struct iommu_domain *dom) +{ + struct protection_domain *domain = dom->priv; + + if (!domain) + return; + + if (domain->dev_cnt > 0) + cleanup_domain(domain); + + BUG_ON(domain->dev_cnt != 0); + + free_pagetable(domain); + + protection_domain_free(domain); + + dom->priv = NULL; +} + +static void amd_iommu_detach_device(struct iommu_domain *dom, + struct device *dev) +{ + struct iommu_dev_data *dev_data = dev->archdata.iommu; + struct amd_iommu *iommu; + u16 devid; + + if (!check_device(dev)) + return; + + devid = get_device_id(dev); + + if (dev_data->domain != NULL) + detach_device(dev); + + iommu = amd_iommu_rlookup_table[devid]; + if (!iommu) + return; + + device_flush_dte(dev); + iommu_completion_wait(iommu); +} + +static int amd_iommu_attach_device(struct iommu_domain *dom, + struct device *dev) +{ + struct protection_domain *domain = dom->priv; + struct iommu_dev_data *dev_data; + struct amd_iommu *iommu; + int ret; + u16 devid; + + if (!check_device(dev)) + return -EINVAL; + + dev_data = dev->archdata.iommu; + + devid = get_device_id(dev); + + iommu = amd_iommu_rlookup_table[devid]; + if (!iommu) + return -EINVAL; + + if (dev_data->domain) + detach_device(dev); + + ret = attach_device(dev, domain); + + iommu_completion_wait(iommu); + + return ret; +} + +static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, + phys_addr_t paddr, int gfp_order, int iommu_prot) +{ + unsigned long page_size = 0x1000UL << gfp_order; + struct protection_domain *domain = dom->priv; + int prot = 0; + int ret; + + if (iommu_prot & IOMMU_READ) + prot |= IOMMU_PROT_IR; + if (iommu_prot & IOMMU_WRITE) + prot |= IOMMU_PROT_IW; + + mutex_lock(&domain->api_lock); + ret = iommu_map_page(domain, iova, paddr, prot, page_size); + mutex_unlock(&domain->api_lock); + + return ret; +} + +static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, + int gfp_order) +{ + struct protection_domain *domain = dom->priv; + unsigned long page_size, unmap_size; + + page_size = 0x1000UL << gfp_order; + + mutex_lock(&domain->api_lock); + unmap_size = iommu_unmap_page(domain, iova, page_size); + mutex_unlock(&domain->api_lock); + + domain_flush_tlb_pde(domain); + + return get_order(unmap_size); +} + +static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, + unsigned long iova) +{ + struct protection_domain *domain = dom->priv; + unsigned long offset_mask; + phys_addr_t paddr; + u64 *pte, __pte; + + pte = fetch_pte(domain, iova); + + if (!pte || !IOMMU_PTE_PRESENT(*pte)) + return 0; + + if (PM_PTE_LEVEL(*pte) == 0) + offset_mask = PAGE_SIZE - 1; + else + offset_mask = PTE_PAGE_SIZE(*pte) - 1; + + __pte = *pte & PM_ADDR_MASK; + paddr = (__pte & ~offset_mask) | (iova & offset_mask); + + return paddr; +} + +static int amd_iommu_domain_has_cap(struct iommu_domain *domain, + unsigned long cap) +{ + switch (cap) { + case IOMMU_CAP_CACHE_COHERENCY: + return 1; + } + + return 0; +} + +static struct iommu_ops amd_iommu_ops = { + .domain_init = amd_iommu_domain_init, + .domain_destroy = amd_iommu_domain_destroy, + .attach_dev = amd_iommu_attach_device, + .detach_dev = amd_iommu_detach_device, + .map = amd_iommu_map, + .unmap = amd_iommu_unmap, + .iova_to_phys = amd_iommu_iova_to_phys, + .domain_has_cap = amd_iommu_domain_has_cap, +}; + +/***************************************************************************** + * + * The next functions do a basic initialization of IOMMU for pass through + * mode + * + * In passthrough mode the IOMMU is initialized and enabled but not used for + * DMA-API translation. + * + *****************************************************************************/ + +int __init amd_iommu_init_passthrough(void) +{ + struct amd_iommu *iommu; + struct pci_dev *dev = NULL; + u16 devid; + + /* allocate passthrough domain */ + pt_domain = protection_domain_alloc(); + if (!pt_domain) + return -ENOMEM; + + pt_domain->mode |= PAGE_MODE_NONE; + + for_each_pci_dev(dev) { + if (!check_device(&dev->dev)) + continue; + + devid = get_device_id(&dev->dev); + + iommu = amd_iommu_rlookup_table[devid]; + if (!iommu) + continue; + + attach_device(&dev->dev, pt_domain); + } + + pr_info("AMD-Vi: Initialized for Passthrough Mode\n"); + + return 0; +} -- cgit v1.2.3 From 403f81d8ee532c976d50a5e1051f14ec78ae8db3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 14 Jun 2011 16:44:25 +0200 Subject: iommu/amd: Move missing parts to drivers/iommu A few parts of the driver were missing in drivers/iommu. Move them there to have the complete driver in that directory. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu.h | 35 - arch/x86/include/asm/amd_iommu_proto.h | 54 -- arch/x86/include/asm/amd_iommu_types.h | 580 ------------ arch/x86/kernel/Makefile | 1 - arch/x86/kernel/amd_iommu_init.c | 1572 ------------------------------- drivers/iommu/Makefile | 2 +- drivers/iommu/amd_iommu.c | 7 +- drivers/iommu/amd_iommu_init.c | 1574 ++++++++++++++++++++++++++++++++ drivers/iommu/amd_iommu_proto.h | 54 ++ drivers/iommu/amd_iommu_types.h | 580 ++++++++++++ include/linux/amd-iommu.h | 35 + 11 files changed, 2248 insertions(+), 2246 deletions(-) delete mode 100644 arch/x86/include/asm/amd_iommu.h delete mode 100644 arch/x86/include/asm/amd_iommu_proto.h delete mode 100644 arch/x86/include/asm/amd_iommu_types.h delete mode 100644 arch/x86/kernel/amd_iommu_init.c create mode 100644 drivers/iommu/amd_iommu_init.c create mode 100644 drivers/iommu/amd_iommu_proto.h create mode 100644 drivers/iommu/amd_iommu_types.h create mode 100644 include/linux/amd-iommu.h (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/amd_iommu.h b/arch/x86/include/asm/amd_iommu.h deleted file mode 100644 index a6863a2dec1f..000000000000 --- a/arch/x86/include/asm/amd_iommu.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. - * Author: Joerg Roedel - * Leo Duran - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _ASM_X86_AMD_IOMMU_H -#define _ASM_X86_AMD_IOMMU_H - -#include - -#ifdef CONFIG_AMD_IOMMU - -extern int amd_iommu_detect(void); - -#else - -static inline int amd_iommu_detect(void) { return -ENODEV; } - -#endif - -#endif /* _ASM_X86_AMD_IOMMU_H */ diff --git a/arch/x86/include/asm/amd_iommu_proto.h b/arch/x86/include/asm/amd_iommu_proto.h deleted file mode 100644 index 55d95eb789b3..000000000000 --- a/arch/x86/include/asm/amd_iommu_proto.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (C) 2009-2010 Advanced Micro Devices, Inc. - * Author: Joerg Roedel - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _ASM_X86_AMD_IOMMU_PROTO_H -#define _ASM_X86_AMD_IOMMU_PROTO_H - -#include - -extern int amd_iommu_init_dma_ops(void); -extern int amd_iommu_init_passthrough(void); -extern irqreturn_t amd_iommu_int_thread(int irq, void *data); -extern irqreturn_t amd_iommu_int_handler(int irq, void *data); -extern void amd_iommu_apply_erratum_63(u16 devid); -extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu); -extern int amd_iommu_init_devices(void); -extern void amd_iommu_uninit_devices(void); -extern void amd_iommu_init_notifier(void); -extern void amd_iommu_init_api(void); -#ifndef CONFIG_AMD_IOMMU_STATS - -static inline void amd_iommu_stats_init(void) { } - -#endif /* !CONFIG_AMD_IOMMU_STATS */ - -static inline bool is_rd890_iommu(struct pci_dev *pdev) -{ - return (pdev->vendor == PCI_VENDOR_ID_ATI) && - (pdev->device == PCI_DEVICE_ID_RD890_IOMMU); -} - -static inline bool iommu_feature(struct amd_iommu *iommu, u64 f) -{ - if (!(iommu->cap & (1 << IOMMU_CAP_EFR))) - return false; - - return !!(iommu->features & f); -} - -#endif /* _ASM_X86_AMD_IOMMU_PROTO_H */ diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h deleted file mode 100644 index 4c9982995414..000000000000 --- a/arch/x86/include/asm/amd_iommu_types.h +++ /dev/null @@ -1,580 +0,0 @@ -/* - * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. - * Author: Joerg Roedel - * Leo Duran - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef _ASM_X86_AMD_IOMMU_TYPES_H -#define _ASM_X86_AMD_IOMMU_TYPES_H - -#include -#include -#include -#include - -/* - * Maximum number of IOMMUs supported - */ -#define MAX_IOMMUS 32 - -/* - * some size calculation constants - */ -#define DEV_TABLE_ENTRY_SIZE 32 -#define ALIAS_TABLE_ENTRY_SIZE 2 -#define RLOOKUP_TABLE_ENTRY_SIZE (sizeof(void *)) - -/* Length of the MMIO region for the AMD IOMMU */ -#define MMIO_REGION_LENGTH 0x4000 - -/* Capability offsets used by the driver */ -#define MMIO_CAP_HDR_OFFSET 0x00 -#define MMIO_RANGE_OFFSET 0x0c -#define MMIO_MISC_OFFSET 0x10 - -/* Masks, shifts and macros to parse the device range capability */ -#define MMIO_RANGE_LD_MASK 0xff000000 -#define MMIO_RANGE_FD_MASK 0x00ff0000 -#define MMIO_RANGE_BUS_MASK 0x0000ff00 -#define MMIO_RANGE_LD_SHIFT 24 -#define MMIO_RANGE_FD_SHIFT 16 -#define MMIO_RANGE_BUS_SHIFT 8 -#define MMIO_GET_LD(x) (((x) & MMIO_RANGE_LD_MASK) >> MMIO_RANGE_LD_SHIFT) -#define MMIO_GET_FD(x) (((x) & MMIO_RANGE_FD_MASK) >> MMIO_RANGE_FD_SHIFT) -#define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT) -#define MMIO_MSI_NUM(x) ((x) & 0x1f) - -/* Flag masks for the AMD IOMMU exclusion range */ -#define MMIO_EXCL_ENABLE_MASK 0x01ULL -#define MMIO_EXCL_ALLOW_MASK 0x02ULL - -/* Used offsets into the MMIO space */ -#define MMIO_DEV_TABLE_OFFSET 0x0000 -#define MMIO_CMD_BUF_OFFSET 0x0008 -#define MMIO_EVT_BUF_OFFSET 0x0010 -#define MMIO_CONTROL_OFFSET 0x0018 -#define MMIO_EXCL_BASE_OFFSET 0x0020 -#define MMIO_EXCL_LIMIT_OFFSET 0x0028 -#define MMIO_EXT_FEATURES 0x0030 -#define MMIO_CMD_HEAD_OFFSET 0x2000 -#define MMIO_CMD_TAIL_OFFSET 0x2008 -#define MMIO_EVT_HEAD_OFFSET 0x2010 -#define MMIO_EVT_TAIL_OFFSET 0x2018 -#define MMIO_STATUS_OFFSET 0x2020 - - -/* Extended Feature Bits */ -#define FEATURE_PREFETCH (1ULL<<0) -#define FEATURE_PPR (1ULL<<1) -#define FEATURE_X2APIC (1ULL<<2) -#define FEATURE_NX (1ULL<<3) -#define FEATURE_GT (1ULL<<4) -#define FEATURE_IA (1ULL<<6) -#define FEATURE_GA (1ULL<<7) -#define FEATURE_HE (1ULL<<8) -#define FEATURE_PC (1ULL<<9) - -/* MMIO status bits */ -#define MMIO_STATUS_COM_WAIT_INT_MASK 0x04 - -/* event logging constants */ -#define EVENT_ENTRY_SIZE 0x10 -#define EVENT_TYPE_SHIFT 28 -#define EVENT_TYPE_MASK 0xf -#define EVENT_TYPE_ILL_DEV 0x1 -#define EVENT_TYPE_IO_FAULT 0x2 -#define EVENT_TYPE_DEV_TAB_ERR 0x3 -#define EVENT_TYPE_PAGE_TAB_ERR 0x4 -#define EVENT_TYPE_ILL_CMD 0x5 -#define EVENT_TYPE_CMD_HARD_ERR 0x6 -#define EVENT_TYPE_IOTLB_INV_TO 0x7 -#define EVENT_TYPE_INV_DEV_REQ 0x8 -#define EVENT_DEVID_MASK 0xffff -#define EVENT_DEVID_SHIFT 0 -#define EVENT_DOMID_MASK 0xffff -#define EVENT_DOMID_SHIFT 0 -#define EVENT_FLAGS_MASK 0xfff -#define EVENT_FLAGS_SHIFT 0x10 - -/* feature control bits */ -#define CONTROL_IOMMU_EN 0x00ULL -#define CONTROL_HT_TUN_EN 0x01ULL -#define CONTROL_EVT_LOG_EN 0x02ULL -#define CONTROL_EVT_INT_EN 0x03ULL -#define CONTROL_COMWAIT_EN 0x04ULL -#define CONTROL_PASSPW_EN 0x08ULL -#define CONTROL_RESPASSPW_EN 0x09ULL -#define CONTROL_COHERENT_EN 0x0aULL -#define CONTROL_ISOC_EN 0x0bULL -#define CONTROL_CMDBUF_EN 0x0cULL -#define CONTROL_PPFLOG_EN 0x0dULL -#define CONTROL_PPFINT_EN 0x0eULL - -/* command specific defines */ -#define CMD_COMPL_WAIT 0x01 -#define CMD_INV_DEV_ENTRY 0x02 -#define CMD_INV_IOMMU_PAGES 0x03 -#define CMD_INV_IOTLB_PAGES 0x04 -#define CMD_INV_ALL 0x08 - -#define CMD_COMPL_WAIT_STORE_MASK 0x01 -#define CMD_COMPL_WAIT_INT_MASK 0x02 -#define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01 -#define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02 - -#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL - -/* macros and definitions for device table entries */ -#define DEV_ENTRY_VALID 0x00 -#define DEV_ENTRY_TRANSLATION 0x01 -#define DEV_ENTRY_IR 0x3d -#define DEV_ENTRY_IW 0x3e -#define DEV_ENTRY_NO_PAGE_FAULT 0x62 -#define DEV_ENTRY_EX 0x67 -#define DEV_ENTRY_SYSMGT1 0x68 -#define DEV_ENTRY_SYSMGT2 0x69 -#define DEV_ENTRY_INIT_PASS 0xb8 -#define DEV_ENTRY_EINT_PASS 0xb9 -#define DEV_ENTRY_NMI_PASS 0xba -#define DEV_ENTRY_LINT0_PASS 0xbe -#define DEV_ENTRY_LINT1_PASS 0xbf -#define DEV_ENTRY_MODE_MASK 0x07 -#define DEV_ENTRY_MODE_SHIFT 0x09 - -/* constants to configure the command buffer */ -#define CMD_BUFFER_SIZE 8192 -#define CMD_BUFFER_UNINITIALIZED 1 -#define CMD_BUFFER_ENTRIES 512 -#define MMIO_CMD_SIZE_SHIFT 56 -#define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT) - -/* constants for event buffer handling */ -#define EVT_BUFFER_SIZE 8192 /* 512 entries */ -#define EVT_LEN_MASK (0x9ULL << 56) - -#define PAGE_MODE_NONE 0x00 -#define PAGE_MODE_1_LEVEL 0x01 -#define PAGE_MODE_2_LEVEL 0x02 -#define PAGE_MODE_3_LEVEL 0x03 -#define PAGE_MODE_4_LEVEL 0x04 -#define PAGE_MODE_5_LEVEL 0x05 -#define PAGE_MODE_6_LEVEL 0x06 - -#define PM_LEVEL_SHIFT(x) (12 + ((x) * 9)) -#define PM_LEVEL_SIZE(x) (((x) < 6) ? \ - ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \ - (0xffffffffffffffffULL)) -#define PM_LEVEL_INDEX(x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL) -#define PM_LEVEL_ENC(x) (((x) << 9) & 0xe00ULL) -#define PM_LEVEL_PDE(x, a) ((a) | PM_LEVEL_ENC((x)) | \ - IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) -#define PM_PTE_LEVEL(pte) (((pte) >> 9) & 0x7ULL) - -#define PM_MAP_4k 0 -#define PM_ADDR_MASK 0x000ffffffffff000ULL -#define PM_MAP_MASK(lvl) (PM_ADDR_MASK & \ - (~((1ULL << (12 + ((lvl) * 9))) - 1))) -#define PM_ALIGNED(lvl, addr) ((PM_MAP_MASK(lvl) & (addr)) == (addr)) - -/* - * Returns the page table level to use for a given page size - * Pagesize is expected to be a power-of-two - */ -#define PAGE_SIZE_LEVEL(pagesize) \ - ((__ffs(pagesize) - 12) / 9) -/* - * Returns the number of ptes to use for a given page size - * Pagesize is expected to be a power-of-two - */ -#define PAGE_SIZE_PTE_COUNT(pagesize) \ - (1ULL << ((__ffs(pagesize) - 12) % 9)) - -/* - * Aligns a given io-virtual address to a given page size - * Pagesize is expected to be a power-of-two - */ -#define PAGE_SIZE_ALIGN(address, pagesize) \ - ((address) & ~((pagesize) - 1)) -/* - * Creates an IOMMU PTE for an address an a given pagesize - * The PTE has no permission bits set - * Pagesize is expected to be a power-of-two larger than 4096 - */ -#define PAGE_SIZE_PTE(address, pagesize) \ - (((address) | ((pagesize) - 1)) & \ - (~(pagesize >> 1)) & PM_ADDR_MASK) - -/* - * Takes a PTE value with mode=0x07 and returns the page size it maps - */ -#define PTE_PAGE_SIZE(pte) \ - (1ULL << (1 + ffz(((pte) | 0xfffULL)))) - -#define IOMMU_PTE_P (1ULL << 0) -#define IOMMU_PTE_TV (1ULL << 1) -#define IOMMU_PTE_U (1ULL << 59) -#define IOMMU_PTE_FC (1ULL << 60) -#define IOMMU_PTE_IR (1ULL << 61) -#define IOMMU_PTE_IW (1ULL << 62) - -#define DTE_FLAG_IOTLB 0x01 - -#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL) -#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P) -#define IOMMU_PTE_PAGE(pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK)) -#define IOMMU_PTE_MODE(pte) (((pte) >> 9) & 0x07) - -#define IOMMU_PROT_MASK 0x03 -#define IOMMU_PROT_IR 0x01 -#define IOMMU_PROT_IW 0x02 - -/* IOMMU capabilities */ -#define IOMMU_CAP_IOTLB 24 -#define IOMMU_CAP_NPCACHE 26 -#define IOMMU_CAP_EFR 27 - -#define MAX_DOMAIN_ID 65536 - -/* FIXME: move this macro to */ -#define PCI_BUS(x) (((x) >> 8) & 0xff) - -/* Protection domain flags */ -#define PD_DMA_OPS_MASK (1UL << 0) /* domain used for dma_ops */ -#define PD_DEFAULT_MASK (1UL << 1) /* domain is a default dma_ops - domain for an IOMMU */ -#define PD_PASSTHROUGH_MASK (1UL << 2) /* domain has no page - translation */ - -extern bool amd_iommu_dump; -#define DUMP_printk(format, arg...) \ - do { \ - if (amd_iommu_dump) \ - printk(KERN_INFO "AMD-Vi: " format, ## arg); \ - } while(0); - -/* global flag if IOMMUs cache non-present entries */ -extern bool amd_iommu_np_cache; -/* Only true if all IOMMUs support device IOTLBs */ -extern bool amd_iommu_iotlb_sup; - -/* - * Make iterating over all IOMMUs easier - */ -#define for_each_iommu(iommu) \ - list_for_each_entry((iommu), &amd_iommu_list, list) -#define for_each_iommu_safe(iommu, next) \ - list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list) - -#define APERTURE_RANGE_SHIFT 27 /* 128 MB */ -#define APERTURE_RANGE_SIZE (1ULL << APERTURE_RANGE_SHIFT) -#define APERTURE_RANGE_PAGES (APERTURE_RANGE_SIZE >> PAGE_SHIFT) -#define APERTURE_MAX_RANGES 32 /* allows 4GB of DMA address space */ -#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT) -#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL) - -/* - * This structure contains generic data for IOMMU protection domains - * independent of their use. - */ -struct protection_domain { - struct list_head list; /* for list of all protection domains */ - struct list_head dev_list; /* List of all devices in this domain */ - spinlock_t lock; /* mostly used to lock the page table*/ - struct mutex api_lock; /* protect page tables in the iommu-api path */ - u16 id; /* the domain id written to the device table */ - int mode; /* paging mode (0-6 levels) */ - u64 *pt_root; /* page table root pointer */ - unsigned long flags; /* flags to find out type of domain */ - bool updated; /* complete domain flush required */ - unsigned dev_cnt; /* devices assigned to this domain */ - unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ - void *priv; /* private data */ - -}; - -/* - * This struct contains device specific data for the IOMMU - */ -struct iommu_dev_data { - struct list_head list; /* For domain->dev_list */ - struct device *dev; /* Device this data belong to */ - struct device *alias; /* The Alias Device */ - struct protection_domain *domain; /* Domain the device is bound to */ - atomic_t bind; /* Domain attach reverent count */ -}; - -/* - * For dynamic growth the aperture size is split into ranges of 128MB of - * DMA address space each. This struct represents one such range. - */ -struct aperture_range { - - /* address allocation bitmap */ - unsigned long *bitmap; - - /* - * Array of PTE pages for the aperture. In this array we save all the - * leaf pages of the domain page table used for the aperture. This way - * we don't need to walk the page table to find a specific PTE. We can - * just calculate its address in constant time. - */ - u64 *pte_pages[64]; - - unsigned long offset; -}; - -/* - * Data container for a dma_ops specific protection domain - */ -struct dma_ops_domain { - struct list_head list; - - /* generic protection domain information */ - struct protection_domain domain; - - /* size of the aperture for the mappings */ - unsigned long aperture_size; - - /* address we start to search for free addresses */ - unsigned long next_address; - - /* address space relevant data */ - struct aperture_range *aperture[APERTURE_MAX_RANGES]; - - /* This will be set to true when TLB needs to be flushed */ - bool need_flush; - - /* - * if this is a preallocated domain, keep the device for which it was - * preallocated in this variable - */ - u16 target_dev; -}; - -/* - * Structure where we save information about one hardware AMD IOMMU in the - * system. - */ -struct amd_iommu { - struct list_head list; - - /* Index within the IOMMU array */ - int index; - - /* locks the accesses to the hardware */ - spinlock_t lock; - - /* Pointer to PCI device of this IOMMU */ - struct pci_dev *dev; - - /* physical address of MMIO space */ - u64 mmio_phys; - /* virtual address of MMIO space */ - u8 *mmio_base; - - /* capabilities of that IOMMU read from ACPI */ - u32 cap; - - /* flags read from acpi table */ - u8 acpi_flags; - - /* Extended features */ - u64 features; - - /* - * Capability pointer. There could be more than one IOMMU per PCI - * device function if there are more than one AMD IOMMU capability - * pointers. - */ - u16 cap_ptr; - - /* pci domain of this IOMMU */ - u16 pci_seg; - - /* first device this IOMMU handles. read from PCI */ - u16 first_device; - /* last device this IOMMU handles. read from PCI */ - u16 last_device; - - /* start of exclusion range of that IOMMU */ - u64 exclusion_start; - /* length of exclusion range of that IOMMU */ - u64 exclusion_length; - - /* command buffer virtual address */ - u8 *cmd_buf; - /* size of command buffer */ - u32 cmd_buf_size; - - /* size of event buffer */ - u32 evt_buf_size; - /* event buffer virtual address */ - u8 *evt_buf; - /* MSI number for event interrupt */ - u16 evt_msi_num; - - /* true if interrupts for this IOMMU are already enabled */ - bool int_enabled; - - /* if one, we need to send a completion wait command */ - bool need_sync; - - /* default dma_ops domain for that IOMMU */ - struct dma_ops_domain *default_dom; - - /* - * We can't rely on the BIOS to restore all values on reinit, so we - * need to stash them - */ - - /* The iommu BAR */ - u32 stored_addr_lo; - u32 stored_addr_hi; - - /* - * Each iommu has 6 l1s, each of which is documented as having 0x12 - * registers - */ - u32 stored_l1[6][0x12]; - - /* The l2 indirect registers */ - u32 stored_l2[0x83]; -}; - -/* - * List with all IOMMUs in the system. This list is not locked because it is - * only written and read at driver initialization or suspend time - */ -extern struct list_head amd_iommu_list; - -/* - * Array with pointers to each IOMMU struct - * The indices are referenced in the protection domains - */ -extern struct amd_iommu *amd_iommus[MAX_IOMMUS]; - -/* Number of IOMMUs present in the system */ -extern int amd_iommus_present; - -/* - * Declarations for the global list of all protection domains - */ -extern spinlock_t amd_iommu_pd_lock; -extern struct list_head amd_iommu_pd_list; - -/* - * Structure defining one entry in the device table - */ -struct dev_table_entry { - u32 data[8]; -}; - -/* - * One entry for unity mappings parsed out of the ACPI table. - */ -struct unity_map_entry { - struct list_head list; - - /* starting device id this entry is used for (including) */ - u16 devid_start; - /* end device id this entry is used for (including) */ - u16 devid_end; - - /* start address to unity map (including) */ - u64 address_start; - /* end address to unity map (including) */ - u64 address_end; - - /* required protection */ - int prot; -}; - -/* - * List of all unity mappings. It is not locked because as runtime it is only - * read. It is created at ACPI table parsing time. - */ -extern struct list_head amd_iommu_unity_map; - -/* - * Data structures for device handling - */ - -/* - * Device table used by hardware. Read and write accesses by software are - * locked with the amd_iommu_pd_table lock. - */ -extern struct dev_table_entry *amd_iommu_dev_table; - -/* - * Alias table to find requestor ids to device ids. Not locked because only - * read on runtime. - */ -extern u16 *amd_iommu_alias_table; - -/* - * Reverse lookup table to find the IOMMU which translates a specific device. - */ -extern struct amd_iommu **amd_iommu_rlookup_table; - -/* size of the dma_ops aperture as power of 2 */ -extern unsigned amd_iommu_aperture_order; - -/* largest PCI device id we expect translation requests for */ -extern u16 amd_iommu_last_bdf; - -/* allocation bitmap for domain ids */ -extern unsigned long *amd_iommu_pd_alloc_bitmap; - -/* - * If true, the addresses will be flushed on unmap time, not when - * they are reused - */ -extern bool amd_iommu_unmap_flush; - -/* takes bus and device/function and returns the device id - * FIXME: should that be in generic PCI code? */ -static inline u16 calc_devid(u8 bus, u8 devfn) -{ - return (((u16)bus) << 8) | devfn; -} - -#ifdef CONFIG_AMD_IOMMU_STATS - -struct __iommu_counter { - char *name; - struct dentry *dent; - u64 value; -}; - -#define DECLARE_STATS_COUNTER(nm) \ - static struct __iommu_counter nm = { \ - .name = #nm, \ - } - -#define INC_STATS_COUNTER(name) name.value += 1 -#define ADD_STATS_COUNTER(name, x) name.value += (x) -#define SUB_STATS_COUNTER(name, x) name.value -= (x) - -#else /* CONFIG_AMD_IOMMU_STATS */ - -#define DECLARE_STATS_COUNTER(name) -#define INC_STATS_COUNTER(name) -#define ADD_STATS_COUNTER(name, x) -#define SUB_STATS_COUNTER(name, x) - -#endif /* CONFIG_AMD_IOMMU_STATS */ - -#endif /* _ASM_X86_AMD_IOMMU_TYPES_H */ diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index aef02989c930..11817ff85399 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -123,7 +123,6 @@ ifeq ($(CONFIG_X86_64),y) obj-$(CONFIG_GART_IOMMU) += amd_gart_64.o aperture_64.o obj-$(CONFIG_CALGARY_IOMMU) += pci-calgary_64.o tce_64.o - obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o obj-y += vsmp_64.o diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c deleted file mode 100644 index bfc8453bd98d..000000000000 --- a/arch/x86/kernel/amd_iommu_init.c +++ /dev/null @@ -1,1572 +0,0 @@ -/* - * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. - * Author: Joerg Roedel - * Leo Duran - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -/* - * definitions for the ACPI scanning code - */ -#define IVRS_HEADER_LENGTH 48 - -#define ACPI_IVHD_TYPE 0x10 -#define ACPI_IVMD_TYPE_ALL 0x20 -#define ACPI_IVMD_TYPE 0x21 -#define ACPI_IVMD_TYPE_RANGE 0x22 - -#define IVHD_DEV_ALL 0x01 -#define IVHD_DEV_SELECT 0x02 -#define IVHD_DEV_SELECT_RANGE_START 0x03 -#define IVHD_DEV_RANGE_END 0x04 -#define IVHD_DEV_ALIAS 0x42 -#define IVHD_DEV_ALIAS_RANGE 0x43 -#define IVHD_DEV_EXT_SELECT 0x46 -#define IVHD_DEV_EXT_SELECT_RANGE 0x47 - -#define IVHD_FLAG_HT_TUN_EN_MASK 0x01 -#define IVHD_FLAG_PASSPW_EN_MASK 0x02 -#define IVHD_FLAG_RESPASSPW_EN_MASK 0x04 -#define IVHD_FLAG_ISOC_EN_MASK 0x08 - -#define IVMD_FLAG_EXCL_RANGE 0x08 -#define IVMD_FLAG_UNITY_MAP 0x01 - -#define ACPI_DEVFLAG_INITPASS 0x01 -#define ACPI_DEVFLAG_EXTINT 0x02 -#define ACPI_DEVFLAG_NMI 0x04 -#define ACPI_DEVFLAG_SYSMGT1 0x10 -#define ACPI_DEVFLAG_SYSMGT2 0x20 -#define ACPI_DEVFLAG_LINT0 0x40 -#define ACPI_DEVFLAG_LINT1 0x80 -#define ACPI_DEVFLAG_ATSDIS 0x10000000 - -/* - * ACPI table definitions - * - * These data structures are laid over the table to parse the important values - * out of it. - */ - -/* - * structure describing one IOMMU in the ACPI table. Typically followed by one - * or more ivhd_entrys. - */ -struct ivhd_header { - u8 type; - u8 flags; - u16 length; - u16 devid; - u16 cap_ptr; - u64 mmio_phys; - u16 pci_seg; - u16 info; - u32 reserved; -} __attribute__((packed)); - -/* - * A device entry describing which devices a specific IOMMU translates and - * which requestor ids they use. - */ -struct ivhd_entry { - u8 type; - u16 devid; - u8 flags; - u32 ext; -} __attribute__((packed)); - -/* - * An AMD IOMMU memory definition structure. It defines things like exclusion - * ranges for devices and regions that should be unity mapped. - */ -struct ivmd_header { - u8 type; - u8 flags; - u16 length; - u16 devid; - u16 aux; - u64 resv; - u64 range_start; - u64 range_length; -} __attribute__((packed)); - -bool amd_iommu_dump; - -static int __initdata amd_iommu_detected; -static bool __initdata amd_iommu_disabled; - -u16 amd_iommu_last_bdf; /* largest PCI device id we have - to handle */ -LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings - we find in ACPI */ -bool amd_iommu_unmap_flush; /* if true, flush on every unmap */ - -LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the - system */ - -/* Array to assign indices to IOMMUs*/ -struct amd_iommu *amd_iommus[MAX_IOMMUS]; -int amd_iommus_present; - -/* IOMMUs have a non-present cache? */ -bool amd_iommu_np_cache __read_mostly; -bool amd_iommu_iotlb_sup __read_mostly = true; - -/* - * The ACPI table parsing functions set this variable on an error - */ -static int __initdata amd_iommu_init_err; - -/* - * List of protection domains - used during resume - */ -LIST_HEAD(amd_iommu_pd_list); -spinlock_t amd_iommu_pd_lock; - -/* - * Pointer to the device table which is shared by all AMD IOMMUs - * it is indexed by the PCI device id or the HT unit id and contains - * information about the domain the device belongs to as well as the - * page table root pointer. - */ -struct dev_table_entry *amd_iommu_dev_table; - -/* - * The alias table is a driver specific data structure which contains the - * mappings of the PCI device ids to the actual requestor ids on the IOMMU. - * More than one device can share the same requestor id. - */ -u16 *amd_iommu_alias_table; - -/* - * The rlookup table is used to find the IOMMU which is responsible - * for a specific device. It is also indexed by the PCI device id. - */ -struct amd_iommu **amd_iommu_rlookup_table; - -/* - * AMD IOMMU allows up to 2^16 differend protection domains. This is a bitmap - * to know which ones are already in use. - */ -unsigned long *amd_iommu_pd_alloc_bitmap; - -static u32 dev_table_size; /* size of the device table */ -static u32 alias_table_size; /* size of the alias table */ -static u32 rlookup_table_size; /* size if the rlookup table */ - -/* - * This function flushes all internal caches of - * the IOMMU used by this driver. - */ -extern void iommu_flush_all_caches(struct amd_iommu *iommu); - -static inline void update_last_devid(u16 devid) -{ - if (devid > amd_iommu_last_bdf) - amd_iommu_last_bdf = devid; -} - -static inline unsigned long tbl_size(int entry_size) -{ - unsigned shift = PAGE_SHIFT + - get_order(((int)amd_iommu_last_bdf + 1) * entry_size); - - return 1UL << shift; -} - -/* Access to l1 and l2 indexed register spaces */ - -static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address) -{ - u32 val; - - pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); - pci_read_config_dword(iommu->dev, 0xfc, &val); - return val; -} - -static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val) -{ - pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31)); - pci_write_config_dword(iommu->dev, 0xfc, val); - pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); -} - -static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address) -{ - u32 val; - - pci_write_config_dword(iommu->dev, 0xf0, address); - pci_read_config_dword(iommu->dev, 0xf4, &val); - return val; -} - -static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val) -{ - pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8)); - pci_write_config_dword(iommu->dev, 0xf4, val); -} - -/**************************************************************************** - * - * AMD IOMMU MMIO register space handling functions - * - * These functions are used to program the IOMMU device registers in - * MMIO space required for that driver. - * - ****************************************************************************/ - -/* - * This function set the exclusion range in the IOMMU. DMA accesses to the - * exclusion range are passed through untranslated - */ -static void iommu_set_exclusion_range(struct amd_iommu *iommu) -{ - u64 start = iommu->exclusion_start & PAGE_MASK; - u64 limit = (start + iommu->exclusion_length) & PAGE_MASK; - u64 entry; - - if (!iommu->exclusion_start) - return; - - entry = start | MMIO_EXCL_ENABLE_MASK; - memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, - &entry, sizeof(entry)); - - entry = limit; - memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, - &entry, sizeof(entry)); -} - -/* Programs the physical address of the device table into the IOMMU hardware */ -static void __init iommu_set_device_table(struct amd_iommu *iommu) -{ - u64 entry; - - BUG_ON(iommu->mmio_base == NULL); - - entry = virt_to_phys(amd_iommu_dev_table); - entry |= (dev_table_size >> 12) - 1; - memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET, - &entry, sizeof(entry)); -} - -/* Generic functions to enable/disable certain features of the IOMMU. */ -static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) -{ - u32 ctrl; - - ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); - ctrl |= (1 << bit); - writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); -} - -static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit) -{ - u32 ctrl; - - ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); - ctrl &= ~(1 << bit); - writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); -} - -/* Function to enable the hardware */ -static void iommu_enable(struct amd_iommu *iommu) -{ - static const char * const feat_str[] = { - "PreF", "PPR", "X2APIC", "NX", "GT", "[5]", - "IA", "GA", "HE", "PC", NULL - }; - int i; - - printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx", - dev_name(&iommu->dev->dev), iommu->cap_ptr); - - if (iommu->cap & (1 << IOMMU_CAP_EFR)) { - printk(KERN_CONT " extended features: "); - for (i = 0; feat_str[i]; ++i) - if (iommu_feature(iommu, (1ULL << i))) - printk(KERN_CONT " %s", feat_str[i]); - } - printk(KERN_CONT "\n"); - - iommu_feature_enable(iommu, CONTROL_IOMMU_EN); -} - -static void iommu_disable(struct amd_iommu *iommu) -{ - /* Disable command buffer */ - iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); - - /* Disable event logging and event interrupts */ - iommu_feature_disable(iommu, CONTROL_EVT_INT_EN); - iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); - - /* Disable IOMMU hardware itself */ - iommu_feature_disable(iommu, CONTROL_IOMMU_EN); -} - -/* - * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in - * the system has one. - */ -static u8 * __init iommu_map_mmio_space(u64 address) -{ - u8 *ret; - - if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu")) { - pr_err("AMD-Vi: Can not reserve memory region %llx for mmio\n", - address); - pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n"); - return NULL; - } - - ret = ioremap_nocache(address, MMIO_REGION_LENGTH); - if (ret != NULL) - return ret; - - release_mem_region(address, MMIO_REGION_LENGTH); - - return NULL; -} - -static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu) -{ - if (iommu->mmio_base) - iounmap(iommu->mmio_base); - release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH); -} - -/**************************************************************************** - * - * The functions below belong to the first pass of AMD IOMMU ACPI table - * parsing. In this pass we try to find out the highest device id this - * code has to handle. Upon this information the size of the shared data - * structures is determined later. - * - ****************************************************************************/ - -/* - * This function calculates the length of a given IVHD entry - */ -static inline int ivhd_entry_length(u8 *ivhd) -{ - return 0x04 << (*ivhd >> 6); -} - -/* - * This function reads the last device id the IOMMU has to handle from the PCI - * capability header for this IOMMU - */ -static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr) -{ - u32 cap; - - cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET); - update_last_devid(calc_devid(MMIO_GET_BUS(cap), MMIO_GET_LD(cap))); - - return 0; -} - -/* - * After reading the highest device id from the IOMMU PCI capability header - * this function looks if there is a higher device id defined in the ACPI table - */ -static int __init find_last_devid_from_ivhd(struct ivhd_header *h) -{ - u8 *p = (void *)h, *end = (void *)h; - struct ivhd_entry *dev; - - p += sizeof(*h); - end += h->length; - - find_last_devid_on_pci(PCI_BUS(h->devid), - PCI_SLOT(h->devid), - PCI_FUNC(h->devid), - h->cap_ptr); - - while (p < end) { - dev = (struct ivhd_entry *)p; - switch (dev->type) { - case IVHD_DEV_SELECT: - case IVHD_DEV_RANGE_END: - case IVHD_DEV_ALIAS: - case IVHD_DEV_EXT_SELECT: - /* all the above subfield types refer to device ids */ - update_last_devid(dev->devid); - break; - default: - break; - } - p += ivhd_entry_length(p); - } - - WARN_ON(p != end); - - return 0; -} - -/* - * Iterate over all IVHD entries in the ACPI table and find the highest device - * id which we need to handle. This is the first of three functions which parse - * the ACPI table. So we check the checksum here. - */ -static int __init find_last_devid_acpi(struct acpi_table_header *table) -{ - int i; - u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table; - struct ivhd_header *h; - - /* - * Validate checksum here so we don't need to do it when - * we actually parse the table - */ - for (i = 0; i < table->length; ++i) - checksum += p[i]; - if (checksum != 0) { - /* ACPI table corrupt */ - amd_iommu_init_err = -ENODEV; - return 0; - } - - p += IVRS_HEADER_LENGTH; - - end += table->length; - while (p < end) { - h = (struct ivhd_header *)p; - switch (h->type) { - case ACPI_IVHD_TYPE: - find_last_devid_from_ivhd(h); - break; - default: - break; - } - p += h->length; - } - WARN_ON(p != end); - - return 0; -} - -/**************************************************************************** - * - * The following functions belong the the code path which parses the ACPI table - * the second time. In this ACPI parsing iteration we allocate IOMMU specific - * data structures, initialize the device/alias/rlookup table and also - * basically initialize the hardware. - * - ****************************************************************************/ - -/* - * Allocates the command buffer. This buffer is per AMD IOMMU. We can - * write commands to that buffer later and the IOMMU will execute them - * asynchronously - */ -static u8 * __init alloc_command_buffer(struct amd_iommu *iommu) -{ - u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(CMD_BUFFER_SIZE)); - - if (cmd_buf == NULL) - return NULL; - - iommu->cmd_buf_size = CMD_BUFFER_SIZE | CMD_BUFFER_UNINITIALIZED; - - return cmd_buf; -} - -/* - * This function resets the command buffer if the IOMMU stopped fetching - * commands from it. - */ -void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu) -{ - iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); - - writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); - writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); - - iommu_feature_enable(iommu, CONTROL_CMDBUF_EN); -} - -/* - * This function writes the command buffer address to the hardware and - * enables it. - */ -static void iommu_enable_command_buffer(struct amd_iommu *iommu) -{ - u64 entry; - - BUG_ON(iommu->cmd_buf == NULL); - - entry = (u64)virt_to_phys(iommu->cmd_buf); - entry |= MMIO_CMD_SIZE_512; - - memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET, - &entry, sizeof(entry)); - - amd_iommu_reset_cmd_buffer(iommu); - iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED); -} - -static void __init free_command_buffer(struct amd_iommu *iommu) -{ - free_pages((unsigned long)iommu->cmd_buf, - get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED))); -} - -/* allocates the memory where the IOMMU will log its events to */ -static u8 * __init alloc_event_buffer(struct amd_iommu *iommu) -{ - iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(EVT_BUFFER_SIZE)); - - if (iommu->evt_buf == NULL) - return NULL; - - iommu->evt_buf_size = EVT_BUFFER_SIZE; - - return iommu->evt_buf; -} - -static void iommu_enable_event_buffer(struct amd_iommu *iommu) -{ - u64 entry; - - BUG_ON(iommu->evt_buf == NULL); - - entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK; - - memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET, - &entry, sizeof(entry)); - - /* set head and tail to zero manually */ - writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); - writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); - - iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN); -} - -static void __init free_event_buffer(struct amd_iommu *iommu) -{ - free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); -} - -/* sets a specific bit in the device table entry. */ -static void set_dev_entry_bit(u16 devid, u8 bit) -{ - int i = (bit >> 5) & 0x07; - int _bit = bit & 0x1f; - - amd_iommu_dev_table[devid].data[i] |= (1 << _bit); -} - -static int get_dev_entry_bit(u16 devid, u8 bit) -{ - int i = (bit >> 5) & 0x07; - int _bit = bit & 0x1f; - - return (amd_iommu_dev_table[devid].data[i] & (1 << _bit)) >> _bit; -} - - -void amd_iommu_apply_erratum_63(u16 devid) -{ - int sysmgt; - - sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) | - (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1); - - if (sysmgt == 0x01) - set_dev_entry_bit(devid, DEV_ENTRY_IW); -} - -/* Writes the specific IOMMU for a device into the rlookup table */ -static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid) -{ - amd_iommu_rlookup_table[devid] = iommu; -} - -/* - * This function takes the device specific flags read from the ACPI - * table and sets up the device table entry with that information - */ -static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, - u16 devid, u32 flags, u32 ext_flags) -{ - if (flags & ACPI_DEVFLAG_INITPASS) - set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS); - if (flags & ACPI_DEVFLAG_EXTINT) - set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS); - if (flags & ACPI_DEVFLAG_NMI) - set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS); - if (flags & ACPI_DEVFLAG_SYSMGT1) - set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1); - if (flags & ACPI_DEVFLAG_SYSMGT2) - set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2); - if (flags & ACPI_DEVFLAG_LINT0) - set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS); - if (flags & ACPI_DEVFLAG_LINT1) - set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS); - - amd_iommu_apply_erratum_63(devid); - - set_iommu_for_device(iommu, devid); -} - -/* - * Reads the device exclusion range from ACPI and initialize IOMMU with - * it - */ -static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m) -{ - struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; - - if (!(m->flags & IVMD_FLAG_EXCL_RANGE)) - return; - - if (iommu) { - /* - * We only can configure exclusion ranges per IOMMU, not - * per device. But we can enable the exclusion range per - * device. This is done here - */ - set_dev_entry_bit(m->devid, DEV_ENTRY_EX); - iommu->exclusion_start = m->range_start; - iommu->exclusion_length = m->range_length; - } -} - -/* - * This function reads some important data from the IOMMU PCI space and - * initializes the driver data structure with it. It reads the hardware - * capabilities and the first/last device entries - */ -static void __init init_iommu_from_pci(struct amd_iommu *iommu) -{ - int cap_ptr = iommu->cap_ptr; - u32 range, misc, low, high; - int i, j; - - pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, - &iommu->cap); - pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET, - &range); - pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET, - &misc); - - iommu->first_device = calc_devid(MMIO_GET_BUS(range), - MMIO_GET_FD(range)); - iommu->last_device = calc_devid(MMIO_GET_BUS(range), - MMIO_GET_LD(range)); - iommu->evt_msi_num = MMIO_MSI_NUM(misc); - - if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB))) - amd_iommu_iotlb_sup = false; - - /* read extended feature bits */ - low = readl(iommu->mmio_base + MMIO_EXT_FEATURES); - high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4); - - iommu->features = ((u64)high << 32) | low; - - if (!is_rd890_iommu(iommu->dev)) - return; - - /* - * Some rd890 systems may not be fully reconfigured by the BIOS, so - * it's necessary for us to store this information so it can be - * reprogrammed on resume - */ - - pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4, - &iommu->stored_addr_lo); - pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8, - &iommu->stored_addr_hi); - - /* Low bit locks writes to configuration space */ - iommu->stored_addr_lo &= ~1; - - for (i = 0; i < 6; i++) - for (j = 0; j < 0x12; j++) - iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j); - - for (i = 0; i < 0x83; i++) - iommu->stored_l2[i] = iommu_read_l2(iommu, i); -} - -/* - * Takes a pointer to an AMD IOMMU entry in the ACPI table and - * initializes the hardware and our data structures with it. - */ -static void __init init_iommu_from_acpi(struct amd_iommu *iommu, - struct ivhd_header *h) -{ - u8 *p = (u8 *)h; - u8 *end = p, flags = 0; - u16 devid = 0, devid_start = 0, devid_to = 0; - u32 dev_i, ext_flags = 0; - bool alias = false; - struct ivhd_entry *e; - - /* - * First save the recommended feature enable bits from ACPI - */ - iommu->acpi_flags = h->flags; - - /* - * Done. Now parse the device entries - */ - p += sizeof(struct ivhd_header); - end += h->length; - - - while (p < end) { - e = (struct ivhd_entry *)p; - switch (e->type) { - case IVHD_DEV_ALL: - - DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x" - " last device %02x:%02x.%x flags: %02x\n", - PCI_BUS(iommu->first_device), - PCI_SLOT(iommu->first_device), - PCI_FUNC(iommu->first_device), - PCI_BUS(iommu->last_device), - PCI_SLOT(iommu->last_device), - PCI_FUNC(iommu->last_device), - e->flags); - - for (dev_i = iommu->first_device; - dev_i <= iommu->last_device; ++dev_i) - set_dev_entry_from_acpi(iommu, dev_i, - e->flags, 0); - break; - case IVHD_DEV_SELECT: - - DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x " - "flags: %02x\n", - PCI_BUS(e->devid), - PCI_SLOT(e->devid), - PCI_FUNC(e->devid), - e->flags); - - devid = e->devid; - set_dev_entry_from_acpi(iommu, devid, e->flags, 0); - break; - case IVHD_DEV_SELECT_RANGE_START: - - DUMP_printk(" DEV_SELECT_RANGE_START\t " - "devid: %02x:%02x.%x flags: %02x\n", - PCI_BUS(e->devid), - PCI_SLOT(e->devid), - PCI_FUNC(e->devid), - e->flags); - - devid_start = e->devid; - flags = e->flags; - ext_flags = 0; - alias = false; - break; - case IVHD_DEV_ALIAS: - - DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x " - "flags: %02x devid_to: %02x:%02x.%x\n", - PCI_BUS(e->devid), - PCI_SLOT(e->devid), - PCI_FUNC(e->devid), - e->flags, - PCI_BUS(e->ext >> 8), - PCI_SLOT(e->ext >> 8), - PCI_FUNC(e->ext >> 8)); - - devid = e->devid; - devid_to = e->ext >> 8; - set_dev_entry_from_acpi(iommu, devid , e->flags, 0); - set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0); - amd_iommu_alias_table[devid] = devid_to; - break; - case IVHD_DEV_ALIAS_RANGE: - - DUMP_printk(" DEV_ALIAS_RANGE\t\t " - "devid: %02x:%02x.%x flags: %02x " - "devid_to: %02x:%02x.%x\n", - PCI_BUS(e->devid), - PCI_SLOT(e->devid), - PCI_FUNC(e->devid), - e->flags, - PCI_BUS(e->ext >> 8), - PCI_SLOT(e->ext >> 8), - PCI_FUNC(e->ext >> 8)); - - devid_start = e->devid; - flags = e->flags; - devid_to = e->ext >> 8; - ext_flags = 0; - alias = true; - break; - case IVHD_DEV_EXT_SELECT: - - DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x " - "flags: %02x ext: %08x\n", - PCI_BUS(e->devid), - PCI_SLOT(e->devid), - PCI_FUNC(e->devid), - e->flags, e->ext); - - devid = e->devid; - set_dev_entry_from_acpi(iommu, devid, e->flags, - e->ext); - break; - case IVHD_DEV_EXT_SELECT_RANGE: - - DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: " - "%02x:%02x.%x flags: %02x ext: %08x\n", - PCI_BUS(e->devid), - PCI_SLOT(e->devid), - PCI_FUNC(e->devid), - e->flags, e->ext); - - devid_start = e->devid; - flags = e->flags; - ext_flags = e->ext; - alias = false; - break; - case IVHD_DEV_RANGE_END: - - DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n", - PCI_BUS(e->devid), - PCI_SLOT(e->devid), - PCI_FUNC(e->devid)); - - devid = e->devid; - for (dev_i = devid_start; dev_i <= devid; ++dev_i) { - if (alias) { - amd_iommu_alias_table[dev_i] = devid_to; - set_dev_entry_from_acpi(iommu, - devid_to, flags, ext_flags); - } - set_dev_entry_from_acpi(iommu, dev_i, - flags, ext_flags); - } - break; - default: - break; - } - - p += ivhd_entry_length(p); - } -} - -/* Initializes the device->iommu mapping for the driver */ -static int __init init_iommu_devices(struct amd_iommu *iommu) -{ - u32 i; - - for (i = iommu->first_device; i <= iommu->last_device; ++i) - set_iommu_for_device(iommu, i); - - return 0; -} - -static void __init free_iommu_one(struct amd_iommu *iommu) -{ - free_command_buffer(iommu); - free_event_buffer(iommu); - iommu_unmap_mmio_space(iommu); -} - -static void __init free_iommu_all(void) -{ - struct amd_iommu *iommu, *next; - - for_each_iommu_safe(iommu, next) { - list_del(&iommu->list); - free_iommu_one(iommu); - kfree(iommu); - } -} - -/* - * This function clues the initialization function for one IOMMU - * together and also allocates the command buffer and programs the - * hardware. It does NOT enable the IOMMU. This is done afterwards. - */ -static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) -{ - spin_lock_init(&iommu->lock); - - /* Add IOMMU to internal data structures */ - list_add_tail(&iommu->list, &amd_iommu_list); - iommu->index = amd_iommus_present++; - - if (unlikely(iommu->index >= MAX_IOMMUS)) { - WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n"); - return -ENOSYS; - } - - /* Index is fine - add IOMMU to the array */ - amd_iommus[iommu->index] = iommu; - - /* - * Copy data from ACPI table entry to the iommu struct - */ - iommu->dev = pci_get_bus_and_slot(PCI_BUS(h->devid), h->devid & 0xff); - if (!iommu->dev) - return 1; - - iommu->cap_ptr = h->cap_ptr; - iommu->pci_seg = h->pci_seg; - iommu->mmio_phys = h->mmio_phys; - iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys); - if (!iommu->mmio_base) - return -ENOMEM; - - iommu->cmd_buf = alloc_command_buffer(iommu); - if (!iommu->cmd_buf) - return -ENOMEM; - - iommu->evt_buf = alloc_event_buffer(iommu); - if (!iommu->evt_buf) - return -ENOMEM; - - iommu->int_enabled = false; - - init_iommu_from_pci(iommu); - init_iommu_from_acpi(iommu, h); - init_iommu_devices(iommu); - - if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) - amd_iommu_np_cache = true; - - return pci_enable_device(iommu->dev); -} - -/* - * Iterates over all IOMMU entries in the ACPI table, allocates the - * IOMMU structure and initializes it with init_iommu_one() - */ -static int __init init_iommu_all(struct acpi_table_header *table) -{ - u8 *p = (u8 *)table, *end = (u8 *)table; - struct ivhd_header *h; - struct amd_iommu *iommu; - int ret; - - end += table->length; - p += IVRS_HEADER_LENGTH; - - while (p < end) { - h = (struct ivhd_header *)p; - switch (*p) { - case ACPI_IVHD_TYPE: - - DUMP_printk("device: %02x:%02x.%01x cap: %04x " - "seg: %d flags: %01x info %04x\n", - PCI_BUS(h->devid), PCI_SLOT(h->devid), - PCI_FUNC(h->devid), h->cap_ptr, - h->pci_seg, h->flags, h->info); - DUMP_printk(" mmio-addr: %016llx\n", - h->mmio_phys); - - iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL); - if (iommu == NULL) { - amd_iommu_init_err = -ENOMEM; - return 0; - } - - ret = init_iommu_one(iommu, h); - if (ret) { - amd_iommu_init_err = ret; - return 0; - } - break; - default: - break; - } - p += h->length; - - } - WARN_ON(p != end); - - return 0; -} - -/**************************************************************************** - * - * The following functions initialize the MSI interrupts for all IOMMUs - * in the system. Its a bit challenging because there could be multiple - * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per - * pci_dev. - * - ****************************************************************************/ - -static int iommu_setup_msi(struct amd_iommu *iommu) -{ - int r; - - if (pci_enable_msi(iommu->dev)) - return 1; - - r = request_threaded_irq(iommu->dev->irq, - amd_iommu_int_handler, - amd_iommu_int_thread, - 0, "AMD-Vi", - iommu->dev); - - if (r) { - pci_disable_msi(iommu->dev); - return 1; - } - - iommu->int_enabled = true; - iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); - - return 0; -} - -static int iommu_init_msi(struct amd_iommu *iommu) -{ - if (iommu->int_enabled) - return 0; - - if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI)) - return iommu_setup_msi(iommu); - - return 1; -} - -/**************************************************************************** - * - * The next functions belong to the third pass of parsing the ACPI - * table. In this last pass the memory mapping requirements are - * gathered (like exclusion and unity mapping reanges). - * - ****************************************************************************/ - -static void __init free_unity_maps(void) -{ - struct unity_map_entry *entry, *next; - - list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) { - list_del(&entry->list); - kfree(entry); - } -} - -/* called when we find an exclusion range definition in ACPI */ -static int __init init_exclusion_range(struct ivmd_header *m) -{ - int i; - - switch (m->type) { - case ACPI_IVMD_TYPE: - set_device_exclusion_range(m->devid, m); - break; - case ACPI_IVMD_TYPE_ALL: - for (i = 0; i <= amd_iommu_last_bdf; ++i) - set_device_exclusion_range(i, m); - break; - case ACPI_IVMD_TYPE_RANGE: - for (i = m->devid; i <= m->aux; ++i) - set_device_exclusion_range(i, m); - break; - default: - break; - } - - return 0; -} - -/* called for unity map ACPI definition */ -static int __init init_unity_map_range(struct ivmd_header *m) -{ - struct unity_map_entry *e = 0; - char *s; - - e = kzalloc(sizeof(*e), GFP_KERNEL); - if (e == NULL) - return -ENOMEM; - - switch (m->type) { - default: - kfree(e); - return 0; - case ACPI_IVMD_TYPE: - s = "IVMD_TYPEi\t\t\t"; - e->devid_start = e->devid_end = m->devid; - break; - case ACPI_IVMD_TYPE_ALL: - s = "IVMD_TYPE_ALL\t\t"; - e->devid_start = 0; - e->devid_end = amd_iommu_last_bdf; - break; - case ACPI_IVMD_TYPE_RANGE: - s = "IVMD_TYPE_RANGE\t\t"; - e->devid_start = m->devid; - e->devid_end = m->aux; - break; - } - e->address_start = PAGE_ALIGN(m->range_start); - e->address_end = e->address_start + PAGE_ALIGN(m->range_length); - e->prot = m->flags >> 1; - - DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x" - " range_start: %016llx range_end: %016llx flags: %x\n", s, - PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start), - PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end), - PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end), - e->address_start, e->address_end, m->flags); - - list_add_tail(&e->list, &amd_iommu_unity_map); - - return 0; -} - -/* iterates over all memory definitions we find in the ACPI table */ -static int __init init_memory_definitions(struct acpi_table_header *table) -{ - u8 *p = (u8 *)table, *end = (u8 *)table; - struct ivmd_header *m; - - end += table->length; - p += IVRS_HEADER_LENGTH; - - while (p < end) { - m = (struct ivmd_header *)p; - if (m->flags & IVMD_FLAG_EXCL_RANGE) - init_exclusion_range(m); - else if (m->flags & IVMD_FLAG_UNITY_MAP) - init_unity_map_range(m); - - p += m->length; - } - - return 0; -} - -/* - * Init the device table to not allow DMA access for devices and - * suppress all page faults - */ -static void init_device_table(void) -{ - u32 devid; - - for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { - set_dev_entry_bit(devid, DEV_ENTRY_VALID); - set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION); - } -} - -static void iommu_init_flags(struct amd_iommu *iommu) -{ - iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ? - iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) : - iommu_feature_disable(iommu, CONTROL_HT_TUN_EN); - - iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ? - iommu_feature_enable(iommu, CONTROL_PASSPW_EN) : - iommu_feature_disable(iommu, CONTROL_PASSPW_EN); - - iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ? - iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) : - iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN); - - iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ? - iommu_feature_enable(iommu, CONTROL_ISOC_EN) : - iommu_feature_disable(iommu, CONTROL_ISOC_EN); - - /* - * make IOMMU memory accesses cache coherent - */ - iommu_feature_enable(iommu, CONTROL_COHERENT_EN); -} - -static void iommu_apply_resume_quirks(struct amd_iommu *iommu) -{ - int i, j; - u32 ioc_feature_control; - struct pci_dev *pdev = NULL; - - /* RD890 BIOSes may not have completely reconfigured the iommu */ - if (!is_rd890_iommu(iommu->dev)) - return; - - /* - * First, we need to ensure that the iommu is enabled. This is - * controlled by a register in the northbridge - */ - pdev = pci_get_bus_and_slot(iommu->dev->bus->number, PCI_DEVFN(0, 0)); - - if (!pdev) - return; - - /* Select Northbridge indirect register 0x75 and enable writing */ - pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7)); - pci_read_config_dword(pdev, 0x64, &ioc_feature_control); - - /* Enable the iommu */ - if (!(ioc_feature_control & 0x1)) - pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1); - - pci_dev_put(pdev); - - /* Restore the iommu BAR */ - pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, - iommu->stored_addr_lo); - pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8, - iommu->stored_addr_hi); - - /* Restore the l1 indirect regs for each of the 6 l1s */ - for (i = 0; i < 6; i++) - for (j = 0; j < 0x12; j++) - iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]); - - /* Restore the l2 indirect regs */ - for (i = 0; i < 0x83; i++) - iommu_write_l2(iommu, i, iommu->stored_l2[i]); - - /* Lock PCI setup registers */ - pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, - iommu->stored_addr_lo | 1); -} - -/* - * This function finally enables all IOMMUs found in the system after - * they have been initialized - */ -static void enable_iommus(void) -{ - struct amd_iommu *iommu; - - for_each_iommu(iommu) { - iommu_disable(iommu); - iommu_init_flags(iommu); - iommu_set_device_table(iommu); - iommu_enable_command_buffer(iommu); - iommu_enable_event_buffer(iommu); - iommu_set_exclusion_range(iommu); - iommu_init_msi(iommu); - iommu_enable(iommu); - iommu_flush_all_caches(iommu); - } -} - -static void disable_iommus(void) -{ - struct amd_iommu *iommu; - - for_each_iommu(iommu) - iommu_disable(iommu); -} - -/* - * Suspend/Resume support - * disable suspend until real resume implemented - */ - -static void amd_iommu_resume(void) -{ - struct amd_iommu *iommu; - - for_each_iommu(iommu) - iommu_apply_resume_quirks(iommu); - - /* re-load the hardware */ - enable_iommus(); - - /* - * we have to flush after the IOMMUs are enabled because a - * disabled IOMMU will never execute the commands we send - */ - for_each_iommu(iommu) - iommu_flush_all_caches(iommu); -} - -static int amd_iommu_suspend(void) -{ - /* disable IOMMUs to go out of the way for BIOS */ - disable_iommus(); - - return 0; -} - -static struct syscore_ops amd_iommu_syscore_ops = { - .suspend = amd_iommu_suspend, - .resume = amd_iommu_resume, -}; - -/* - * This is the core init function for AMD IOMMU hardware in the system. - * This function is called from the generic x86 DMA layer initialization - * code. - * - * This function basically parses the ACPI table for AMD IOMMU (IVRS) - * three times: - * - * 1 pass) Find the highest PCI device id the driver has to handle. - * Upon this information the size of the data structures is - * determined that needs to be allocated. - * - * 2 pass) Initialize the data structures just allocated with the - * information in the ACPI table about available AMD IOMMUs - * in the system. It also maps the PCI devices in the - * system to specific IOMMUs - * - * 3 pass) After the basic data structures are allocated and - * initialized we update them with information about memory - * remapping requirements parsed out of the ACPI table in - * this last pass. - * - * After that the hardware is initialized and ready to go. In the last - * step we do some Linux specific things like registering the driver in - * the dma_ops interface and initializing the suspend/resume support - * functions. Finally it prints some information about AMD IOMMUs and - * the driver state and enables the hardware. - */ -static int __init amd_iommu_init(void) -{ - int i, ret = 0; - - /* - * First parse ACPI tables to find the largest Bus/Dev/Func - * we need to handle. Upon this information the shared data - * structures for the IOMMUs in the system will be allocated - */ - if (acpi_table_parse("IVRS", find_last_devid_acpi) != 0) - return -ENODEV; - - ret = amd_iommu_init_err; - if (ret) - goto out; - - dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); - alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); - rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); - - ret = -ENOMEM; - - /* Device table - directly used by all IOMMUs */ - amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(dev_table_size)); - if (amd_iommu_dev_table == NULL) - goto out; - - /* - * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the - * IOMMU see for that device - */ - amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL, - get_order(alias_table_size)); - if (amd_iommu_alias_table == NULL) - goto free; - - /* IOMMU rlookup table - find the IOMMU for a specific device */ - amd_iommu_rlookup_table = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(rlookup_table_size)); - if (amd_iommu_rlookup_table == NULL) - goto free; - - amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(MAX_DOMAIN_ID/8)); - if (amd_iommu_pd_alloc_bitmap == NULL) - goto free; - - /* init the device table */ - init_device_table(); - - /* - * let all alias entries point to itself - */ - for (i = 0; i <= amd_iommu_last_bdf; ++i) - amd_iommu_alias_table[i] = i; - - /* - * never allocate domain 0 because its used as the non-allocated and - * error value placeholder - */ - amd_iommu_pd_alloc_bitmap[0] = 1; - - spin_lock_init(&amd_iommu_pd_lock); - - /* - * now the data structures are allocated and basically initialized - * start the real acpi table scan - */ - ret = -ENODEV; - if (acpi_table_parse("IVRS", init_iommu_all) != 0) - goto free; - - if (amd_iommu_init_err) { - ret = amd_iommu_init_err; - goto free; - } - - if (acpi_table_parse("IVRS", init_memory_definitions) != 0) - goto free; - - if (amd_iommu_init_err) { - ret = amd_iommu_init_err; - goto free; - } - - ret = amd_iommu_init_devices(); - if (ret) - goto free; - - enable_iommus(); - - if (iommu_pass_through) - ret = amd_iommu_init_passthrough(); - else - ret = amd_iommu_init_dma_ops(); - - if (ret) - goto free_disable; - - amd_iommu_init_api(); - - amd_iommu_init_notifier(); - - register_syscore_ops(&amd_iommu_syscore_ops); - - if (iommu_pass_through) - goto out; - - if (amd_iommu_unmap_flush) - printk(KERN_INFO "AMD-Vi: IO/TLB flush on unmap enabled\n"); - else - printk(KERN_INFO "AMD-Vi: Lazy IO/TLB flushing enabled\n"); - - x86_platform.iommu_shutdown = disable_iommus; -out: - return ret; - -free_disable: - disable_iommus(); - -free: - amd_iommu_uninit_devices(); - - free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, - get_order(MAX_DOMAIN_ID/8)); - - free_pages((unsigned long)amd_iommu_rlookup_table, - get_order(rlookup_table_size)); - - free_pages((unsigned long)amd_iommu_alias_table, - get_order(alias_table_size)); - - free_pages((unsigned long)amd_iommu_dev_table, - get_order(dev_table_size)); - - free_iommu_all(); - - free_unity_maps(); - -#ifdef CONFIG_GART_IOMMU - /* - * We failed to initialize the AMD IOMMU - try fallback to GART - * if possible. - */ - gart_iommu_init(); - -#endif - - goto out; -} - -/**************************************************************************** - * - * Early detect code. This code runs at IOMMU detection time in the DMA - * layer. It just looks if there is an IVRS ACPI table to detect AMD - * IOMMUs - * - ****************************************************************************/ -static int __init early_amd_iommu_detect(struct acpi_table_header *table) -{ - return 0; -} - -int __init amd_iommu_detect(void) -{ - if (no_iommu || (iommu_detected && !gart_iommu_aperture)) - return -ENODEV; - - if (amd_iommu_disabled) - return -ENODEV; - - if (acpi_table_parse("IVRS", early_amd_iommu_detect) == 0) { - iommu_detected = 1; - amd_iommu_detected = 1; - x86_init.iommu.iommu_init = amd_iommu_init; - - /* Make sure ACS will be enabled */ - pci_request_acs(); - return 1; - } - return -ENODEV; -} - -/**************************************************************************** - * - * Parsing functions for the AMD IOMMU specific kernel command line - * options. - * - ****************************************************************************/ - -static int __init parse_amd_iommu_dump(char *str) -{ - amd_iommu_dump = true; - - return 1; -} - -static int __init parse_amd_iommu_options(char *str) -{ - for (; *str; ++str) { - if (strncmp(str, "fullflush", 9) == 0) - amd_iommu_unmap_flush = true; - if (strncmp(str, "off", 3) == 0) - amd_iommu_disabled = true; - } - - return 1; -} - -__setup("amd_iommu_dump", parse_amd_iommu_dump); -__setup("amd_iommu=", parse_amd_iommu_options); - -IOMMU_INIT_FINISH(amd_iommu_detect, - gart_iommu_hole_init, - 0, - 0); diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 49e9c0f46bd5..4d4d77df7cac 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,5 +1,5 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o -obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o +obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_DMAR) += dmar.o iova.o intel-iommu.o obj-$(CONFIG_INTR_REMAP) += dmar.o intr_remapping.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 7c3a95e54ec5..5aa12eaabd21 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -27,13 +27,14 @@ #include #include #include +#include #include #include #include #include -#include -#include -#include + +#include "amd_iommu_proto.h" +#include "amd_iommu_types.h" #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c new file mode 100644 index 000000000000..82d2410f4205 --- /dev/null +++ b/drivers/iommu/amd_iommu_init.c @@ -0,0 +1,1574 @@ +/* + * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * Leo Duran + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "amd_iommu_proto.h" +#include "amd_iommu_types.h" + +/* + * definitions for the ACPI scanning code + */ +#define IVRS_HEADER_LENGTH 48 + +#define ACPI_IVHD_TYPE 0x10 +#define ACPI_IVMD_TYPE_ALL 0x20 +#define ACPI_IVMD_TYPE 0x21 +#define ACPI_IVMD_TYPE_RANGE 0x22 + +#define IVHD_DEV_ALL 0x01 +#define IVHD_DEV_SELECT 0x02 +#define IVHD_DEV_SELECT_RANGE_START 0x03 +#define IVHD_DEV_RANGE_END 0x04 +#define IVHD_DEV_ALIAS 0x42 +#define IVHD_DEV_ALIAS_RANGE 0x43 +#define IVHD_DEV_EXT_SELECT 0x46 +#define IVHD_DEV_EXT_SELECT_RANGE 0x47 + +#define IVHD_FLAG_HT_TUN_EN_MASK 0x01 +#define IVHD_FLAG_PASSPW_EN_MASK 0x02 +#define IVHD_FLAG_RESPASSPW_EN_MASK 0x04 +#define IVHD_FLAG_ISOC_EN_MASK 0x08 + +#define IVMD_FLAG_EXCL_RANGE 0x08 +#define IVMD_FLAG_UNITY_MAP 0x01 + +#define ACPI_DEVFLAG_INITPASS 0x01 +#define ACPI_DEVFLAG_EXTINT 0x02 +#define ACPI_DEVFLAG_NMI 0x04 +#define ACPI_DEVFLAG_SYSMGT1 0x10 +#define ACPI_DEVFLAG_SYSMGT2 0x20 +#define ACPI_DEVFLAG_LINT0 0x40 +#define ACPI_DEVFLAG_LINT1 0x80 +#define ACPI_DEVFLAG_ATSDIS 0x10000000 + +/* + * ACPI table definitions + * + * These data structures are laid over the table to parse the important values + * out of it. + */ + +/* + * structure describing one IOMMU in the ACPI table. Typically followed by one + * or more ivhd_entrys. + */ +struct ivhd_header { + u8 type; + u8 flags; + u16 length; + u16 devid; + u16 cap_ptr; + u64 mmio_phys; + u16 pci_seg; + u16 info; + u32 reserved; +} __attribute__((packed)); + +/* + * A device entry describing which devices a specific IOMMU translates and + * which requestor ids they use. + */ +struct ivhd_entry { + u8 type; + u16 devid; + u8 flags; + u32 ext; +} __attribute__((packed)); + +/* + * An AMD IOMMU memory definition structure. It defines things like exclusion + * ranges for devices and regions that should be unity mapped. + */ +struct ivmd_header { + u8 type; + u8 flags; + u16 length; + u16 devid; + u16 aux; + u64 resv; + u64 range_start; + u64 range_length; +} __attribute__((packed)); + +bool amd_iommu_dump; + +static int __initdata amd_iommu_detected; +static bool __initdata amd_iommu_disabled; + +u16 amd_iommu_last_bdf; /* largest PCI device id we have + to handle */ +LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings + we find in ACPI */ +bool amd_iommu_unmap_flush; /* if true, flush on every unmap */ + +LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the + system */ + +/* Array to assign indices to IOMMUs*/ +struct amd_iommu *amd_iommus[MAX_IOMMUS]; +int amd_iommus_present; + +/* IOMMUs have a non-present cache? */ +bool amd_iommu_np_cache __read_mostly; +bool amd_iommu_iotlb_sup __read_mostly = true; + +/* + * The ACPI table parsing functions set this variable on an error + */ +static int __initdata amd_iommu_init_err; + +/* + * List of protection domains - used during resume + */ +LIST_HEAD(amd_iommu_pd_list); +spinlock_t amd_iommu_pd_lock; + +/* + * Pointer to the device table which is shared by all AMD IOMMUs + * it is indexed by the PCI device id or the HT unit id and contains + * information about the domain the device belongs to as well as the + * page table root pointer. + */ +struct dev_table_entry *amd_iommu_dev_table; + +/* + * The alias table is a driver specific data structure which contains the + * mappings of the PCI device ids to the actual requestor ids on the IOMMU. + * More than one device can share the same requestor id. + */ +u16 *amd_iommu_alias_table; + +/* + * The rlookup table is used to find the IOMMU which is responsible + * for a specific device. It is also indexed by the PCI device id. + */ +struct amd_iommu **amd_iommu_rlookup_table; + +/* + * AMD IOMMU allows up to 2^16 differend protection domains. This is a bitmap + * to know which ones are already in use. + */ +unsigned long *amd_iommu_pd_alloc_bitmap; + +static u32 dev_table_size; /* size of the device table */ +static u32 alias_table_size; /* size of the alias table */ +static u32 rlookup_table_size; /* size if the rlookup table */ + +/* + * This function flushes all internal caches of + * the IOMMU used by this driver. + */ +extern void iommu_flush_all_caches(struct amd_iommu *iommu); + +static inline void update_last_devid(u16 devid) +{ + if (devid > amd_iommu_last_bdf) + amd_iommu_last_bdf = devid; +} + +static inline unsigned long tbl_size(int entry_size) +{ + unsigned shift = PAGE_SHIFT + + get_order(((int)amd_iommu_last_bdf + 1) * entry_size); + + return 1UL << shift; +} + +/* Access to l1 and l2 indexed register spaces */ + +static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address) +{ + u32 val; + + pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); + pci_read_config_dword(iommu->dev, 0xfc, &val); + return val; +} + +static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val) +{ + pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31)); + pci_write_config_dword(iommu->dev, 0xfc, val); + pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); +} + +static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address) +{ + u32 val; + + pci_write_config_dword(iommu->dev, 0xf0, address); + pci_read_config_dword(iommu->dev, 0xf4, &val); + return val; +} + +static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val) +{ + pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8)); + pci_write_config_dword(iommu->dev, 0xf4, val); +} + +/**************************************************************************** + * + * AMD IOMMU MMIO register space handling functions + * + * These functions are used to program the IOMMU device registers in + * MMIO space required for that driver. + * + ****************************************************************************/ + +/* + * This function set the exclusion range in the IOMMU. DMA accesses to the + * exclusion range are passed through untranslated + */ +static void iommu_set_exclusion_range(struct amd_iommu *iommu) +{ + u64 start = iommu->exclusion_start & PAGE_MASK; + u64 limit = (start + iommu->exclusion_length) & PAGE_MASK; + u64 entry; + + if (!iommu->exclusion_start) + return; + + entry = start | MMIO_EXCL_ENABLE_MASK; + memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, + &entry, sizeof(entry)); + + entry = limit; + memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, + &entry, sizeof(entry)); +} + +/* Programs the physical address of the device table into the IOMMU hardware */ +static void __init iommu_set_device_table(struct amd_iommu *iommu) +{ + u64 entry; + + BUG_ON(iommu->mmio_base == NULL); + + entry = virt_to_phys(amd_iommu_dev_table); + entry |= (dev_table_size >> 12) - 1; + memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET, + &entry, sizeof(entry)); +} + +/* Generic functions to enable/disable certain features of the IOMMU. */ +static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) +{ + u32 ctrl; + + ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); + ctrl |= (1 << bit); + writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); +} + +static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit) +{ + u32 ctrl; + + ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET); + ctrl &= ~(1 << bit); + writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); +} + +/* Function to enable the hardware */ +static void iommu_enable(struct amd_iommu *iommu) +{ + static const char * const feat_str[] = { + "PreF", "PPR", "X2APIC", "NX", "GT", "[5]", + "IA", "GA", "HE", "PC", NULL + }; + int i; + + printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx", + dev_name(&iommu->dev->dev), iommu->cap_ptr); + + if (iommu->cap & (1 << IOMMU_CAP_EFR)) { + printk(KERN_CONT " extended features: "); + for (i = 0; feat_str[i]; ++i) + if (iommu_feature(iommu, (1ULL << i))) + printk(KERN_CONT " %s", feat_str[i]); + } + printk(KERN_CONT "\n"); + + iommu_feature_enable(iommu, CONTROL_IOMMU_EN); +} + +static void iommu_disable(struct amd_iommu *iommu) +{ + /* Disable command buffer */ + iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); + + /* Disable event logging and event interrupts */ + iommu_feature_disable(iommu, CONTROL_EVT_INT_EN); + iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); + + /* Disable IOMMU hardware itself */ + iommu_feature_disable(iommu, CONTROL_IOMMU_EN); +} + +/* + * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in + * the system has one. + */ +static u8 * __init iommu_map_mmio_space(u64 address) +{ + u8 *ret; + + if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu")) { + pr_err("AMD-Vi: Can not reserve memory region %llx for mmio\n", + address); + pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n"); + return NULL; + } + + ret = ioremap_nocache(address, MMIO_REGION_LENGTH); + if (ret != NULL) + return ret; + + release_mem_region(address, MMIO_REGION_LENGTH); + + return NULL; +} + +static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu) +{ + if (iommu->mmio_base) + iounmap(iommu->mmio_base); + release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH); +} + +/**************************************************************************** + * + * The functions below belong to the first pass of AMD IOMMU ACPI table + * parsing. In this pass we try to find out the highest device id this + * code has to handle. Upon this information the size of the shared data + * structures is determined later. + * + ****************************************************************************/ + +/* + * This function calculates the length of a given IVHD entry + */ +static inline int ivhd_entry_length(u8 *ivhd) +{ + return 0x04 << (*ivhd >> 6); +} + +/* + * This function reads the last device id the IOMMU has to handle from the PCI + * capability header for this IOMMU + */ +static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr) +{ + u32 cap; + + cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET); + update_last_devid(calc_devid(MMIO_GET_BUS(cap), MMIO_GET_LD(cap))); + + return 0; +} + +/* + * After reading the highest device id from the IOMMU PCI capability header + * this function looks if there is a higher device id defined in the ACPI table + */ +static int __init find_last_devid_from_ivhd(struct ivhd_header *h) +{ + u8 *p = (void *)h, *end = (void *)h; + struct ivhd_entry *dev; + + p += sizeof(*h); + end += h->length; + + find_last_devid_on_pci(PCI_BUS(h->devid), + PCI_SLOT(h->devid), + PCI_FUNC(h->devid), + h->cap_ptr); + + while (p < end) { + dev = (struct ivhd_entry *)p; + switch (dev->type) { + case IVHD_DEV_SELECT: + case IVHD_DEV_RANGE_END: + case IVHD_DEV_ALIAS: + case IVHD_DEV_EXT_SELECT: + /* all the above subfield types refer to device ids */ + update_last_devid(dev->devid); + break; + default: + break; + } + p += ivhd_entry_length(p); + } + + WARN_ON(p != end); + + return 0; +} + +/* + * Iterate over all IVHD entries in the ACPI table and find the highest device + * id which we need to handle. This is the first of three functions which parse + * the ACPI table. So we check the checksum here. + */ +static int __init find_last_devid_acpi(struct acpi_table_header *table) +{ + int i; + u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table; + struct ivhd_header *h; + + /* + * Validate checksum here so we don't need to do it when + * we actually parse the table + */ + for (i = 0; i < table->length; ++i) + checksum += p[i]; + if (checksum != 0) { + /* ACPI table corrupt */ + amd_iommu_init_err = -ENODEV; + return 0; + } + + p += IVRS_HEADER_LENGTH; + + end += table->length; + while (p < end) { + h = (struct ivhd_header *)p; + switch (h->type) { + case ACPI_IVHD_TYPE: + find_last_devid_from_ivhd(h); + break; + default: + break; + } + p += h->length; + } + WARN_ON(p != end); + + return 0; +} + +/**************************************************************************** + * + * The following functions belong the the code path which parses the ACPI table + * the second time. In this ACPI parsing iteration we allocate IOMMU specific + * data structures, initialize the device/alias/rlookup table and also + * basically initialize the hardware. + * + ****************************************************************************/ + +/* + * Allocates the command buffer. This buffer is per AMD IOMMU. We can + * write commands to that buffer later and the IOMMU will execute them + * asynchronously + */ +static u8 * __init alloc_command_buffer(struct amd_iommu *iommu) +{ + u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(CMD_BUFFER_SIZE)); + + if (cmd_buf == NULL) + return NULL; + + iommu->cmd_buf_size = CMD_BUFFER_SIZE | CMD_BUFFER_UNINITIALIZED; + + return cmd_buf; +} + +/* + * This function resets the command buffer if the IOMMU stopped fetching + * commands from it. + */ +void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu) +{ + iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); + + writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); + writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); + + iommu_feature_enable(iommu, CONTROL_CMDBUF_EN); +} + +/* + * This function writes the command buffer address to the hardware and + * enables it. + */ +static void iommu_enable_command_buffer(struct amd_iommu *iommu) +{ + u64 entry; + + BUG_ON(iommu->cmd_buf == NULL); + + entry = (u64)virt_to_phys(iommu->cmd_buf); + entry |= MMIO_CMD_SIZE_512; + + memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET, + &entry, sizeof(entry)); + + amd_iommu_reset_cmd_buffer(iommu); + iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED); +} + +static void __init free_command_buffer(struct amd_iommu *iommu) +{ + free_pages((unsigned long)iommu->cmd_buf, + get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED))); +} + +/* allocates the memory where the IOMMU will log its events to */ +static u8 * __init alloc_event_buffer(struct amd_iommu *iommu) +{ + iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(EVT_BUFFER_SIZE)); + + if (iommu->evt_buf == NULL) + return NULL; + + iommu->evt_buf_size = EVT_BUFFER_SIZE; + + return iommu->evt_buf; +} + +static void iommu_enable_event_buffer(struct amd_iommu *iommu) +{ + u64 entry; + + BUG_ON(iommu->evt_buf == NULL); + + entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK; + + memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET, + &entry, sizeof(entry)); + + /* set head and tail to zero manually */ + writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); + writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); + + iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN); +} + +static void __init free_event_buffer(struct amd_iommu *iommu) +{ + free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); +} + +/* sets a specific bit in the device table entry. */ +static void set_dev_entry_bit(u16 devid, u8 bit) +{ + int i = (bit >> 5) & 0x07; + int _bit = bit & 0x1f; + + amd_iommu_dev_table[devid].data[i] |= (1 << _bit); +} + +static int get_dev_entry_bit(u16 devid, u8 bit) +{ + int i = (bit >> 5) & 0x07; + int _bit = bit & 0x1f; + + return (amd_iommu_dev_table[devid].data[i] & (1 << _bit)) >> _bit; +} + + +void amd_iommu_apply_erratum_63(u16 devid) +{ + int sysmgt; + + sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) | + (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1); + + if (sysmgt == 0x01) + set_dev_entry_bit(devid, DEV_ENTRY_IW); +} + +/* Writes the specific IOMMU for a device into the rlookup table */ +static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid) +{ + amd_iommu_rlookup_table[devid] = iommu; +} + +/* + * This function takes the device specific flags read from the ACPI + * table and sets up the device table entry with that information + */ +static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, + u16 devid, u32 flags, u32 ext_flags) +{ + if (flags & ACPI_DEVFLAG_INITPASS) + set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS); + if (flags & ACPI_DEVFLAG_EXTINT) + set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS); + if (flags & ACPI_DEVFLAG_NMI) + set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS); + if (flags & ACPI_DEVFLAG_SYSMGT1) + set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1); + if (flags & ACPI_DEVFLAG_SYSMGT2) + set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2); + if (flags & ACPI_DEVFLAG_LINT0) + set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS); + if (flags & ACPI_DEVFLAG_LINT1) + set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS); + + amd_iommu_apply_erratum_63(devid); + + set_iommu_for_device(iommu, devid); +} + +/* + * Reads the device exclusion range from ACPI and initialize IOMMU with + * it + */ +static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m) +{ + struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; + + if (!(m->flags & IVMD_FLAG_EXCL_RANGE)) + return; + + if (iommu) { + /* + * We only can configure exclusion ranges per IOMMU, not + * per device. But we can enable the exclusion range per + * device. This is done here + */ + set_dev_entry_bit(m->devid, DEV_ENTRY_EX); + iommu->exclusion_start = m->range_start; + iommu->exclusion_length = m->range_length; + } +} + +/* + * This function reads some important data from the IOMMU PCI space and + * initializes the driver data structure with it. It reads the hardware + * capabilities and the first/last device entries + */ +static void __init init_iommu_from_pci(struct amd_iommu *iommu) +{ + int cap_ptr = iommu->cap_ptr; + u32 range, misc, low, high; + int i, j; + + pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, + &iommu->cap); + pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET, + &range); + pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET, + &misc); + + iommu->first_device = calc_devid(MMIO_GET_BUS(range), + MMIO_GET_FD(range)); + iommu->last_device = calc_devid(MMIO_GET_BUS(range), + MMIO_GET_LD(range)); + iommu->evt_msi_num = MMIO_MSI_NUM(misc); + + if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB))) + amd_iommu_iotlb_sup = false; + + /* read extended feature bits */ + low = readl(iommu->mmio_base + MMIO_EXT_FEATURES); + high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4); + + iommu->features = ((u64)high << 32) | low; + + if (!is_rd890_iommu(iommu->dev)) + return; + + /* + * Some rd890 systems may not be fully reconfigured by the BIOS, so + * it's necessary for us to store this information so it can be + * reprogrammed on resume + */ + + pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4, + &iommu->stored_addr_lo); + pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8, + &iommu->stored_addr_hi); + + /* Low bit locks writes to configuration space */ + iommu->stored_addr_lo &= ~1; + + for (i = 0; i < 6; i++) + for (j = 0; j < 0x12; j++) + iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j); + + for (i = 0; i < 0x83; i++) + iommu->stored_l2[i] = iommu_read_l2(iommu, i); +} + +/* + * Takes a pointer to an AMD IOMMU entry in the ACPI table and + * initializes the hardware and our data structures with it. + */ +static void __init init_iommu_from_acpi(struct amd_iommu *iommu, + struct ivhd_header *h) +{ + u8 *p = (u8 *)h; + u8 *end = p, flags = 0; + u16 devid = 0, devid_start = 0, devid_to = 0; + u32 dev_i, ext_flags = 0; + bool alias = false; + struct ivhd_entry *e; + + /* + * First save the recommended feature enable bits from ACPI + */ + iommu->acpi_flags = h->flags; + + /* + * Done. Now parse the device entries + */ + p += sizeof(struct ivhd_header); + end += h->length; + + + while (p < end) { + e = (struct ivhd_entry *)p; + switch (e->type) { + case IVHD_DEV_ALL: + + DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x" + " last device %02x:%02x.%x flags: %02x\n", + PCI_BUS(iommu->first_device), + PCI_SLOT(iommu->first_device), + PCI_FUNC(iommu->first_device), + PCI_BUS(iommu->last_device), + PCI_SLOT(iommu->last_device), + PCI_FUNC(iommu->last_device), + e->flags); + + for (dev_i = iommu->first_device; + dev_i <= iommu->last_device; ++dev_i) + set_dev_entry_from_acpi(iommu, dev_i, + e->flags, 0); + break; + case IVHD_DEV_SELECT: + + DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x " + "flags: %02x\n", + PCI_BUS(e->devid), + PCI_SLOT(e->devid), + PCI_FUNC(e->devid), + e->flags); + + devid = e->devid; + set_dev_entry_from_acpi(iommu, devid, e->flags, 0); + break; + case IVHD_DEV_SELECT_RANGE_START: + + DUMP_printk(" DEV_SELECT_RANGE_START\t " + "devid: %02x:%02x.%x flags: %02x\n", + PCI_BUS(e->devid), + PCI_SLOT(e->devid), + PCI_FUNC(e->devid), + e->flags); + + devid_start = e->devid; + flags = e->flags; + ext_flags = 0; + alias = false; + break; + case IVHD_DEV_ALIAS: + + DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x " + "flags: %02x devid_to: %02x:%02x.%x\n", + PCI_BUS(e->devid), + PCI_SLOT(e->devid), + PCI_FUNC(e->devid), + e->flags, + PCI_BUS(e->ext >> 8), + PCI_SLOT(e->ext >> 8), + PCI_FUNC(e->ext >> 8)); + + devid = e->devid; + devid_to = e->ext >> 8; + set_dev_entry_from_acpi(iommu, devid , e->flags, 0); + set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0); + amd_iommu_alias_table[devid] = devid_to; + break; + case IVHD_DEV_ALIAS_RANGE: + + DUMP_printk(" DEV_ALIAS_RANGE\t\t " + "devid: %02x:%02x.%x flags: %02x " + "devid_to: %02x:%02x.%x\n", + PCI_BUS(e->devid), + PCI_SLOT(e->devid), + PCI_FUNC(e->devid), + e->flags, + PCI_BUS(e->ext >> 8), + PCI_SLOT(e->ext >> 8), + PCI_FUNC(e->ext >> 8)); + + devid_start = e->devid; + flags = e->flags; + devid_to = e->ext >> 8; + ext_flags = 0; + alias = true; + break; + case IVHD_DEV_EXT_SELECT: + + DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x " + "flags: %02x ext: %08x\n", + PCI_BUS(e->devid), + PCI_SLOT(e->devid), + PCI_FUNC(e->devid), + e->flags, e->ext); + + devid = e->devid; + set_dev_entry_from_acpi(iommu, devid, e->flags, + e->ext); + break; + case IVHD_DEV_EXT_SELECT_RANGE: + + DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: " + "%02x:%02x.%x flags: %02x ext: %08x\n", + PCI_BUS(e->devid), + PCI_SLOT(e->devid), + PCI_FUNC(e->devid), + e->flags, e->ext); + + devid_start = e->devid; + flags = e->flags; + ext_flags = e->ext; + alias = false; + break; + case IVHD_DEV_RANGE_END: + + DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n", + PCI_BUS(e->devid), + PCI_SLOT(e->devid), + PCI_FUNC(e->devid)); + + devid = e->devid; + for (dev_i = devid_start; dev_i <= devid; ++dev_i) { + if (alias) { + amd_iommu_alias_table[dev_i] = devid_to; + set_dev_entry_from_acpi(iommu, + devid_to, flags, ext_flags); + } + set_dev_entry_from_acpi(iommu, dev_i, + flags, ext_flags); + } + break; + default: + break; + } + + p += ivhd_entry_length(p); + } +} + +/* Initializes the device->iommu mapping for the driver */ +static int __init init_iommu_devices(struct amd_iommu *iommu) +{ + u32 i; + + for (i = iommu->first_device; i <= iommu->last_device; ++i) + set_iommu_for_device(iommu, i); + + return 0; +} + +static void __init free_iommu_one(struct amd_iommu *iommu) +{ + free_command_buffer(iommu); + free_event_buffer(iommu); + iommu_unmap_mmio_space(iommu); +} + +static void __init free_iommu_all(void) +{ + struct amd_iommu *iommu, *next; + + for_each_iommu_safe(iommu, next) { + list_del(&iommu->list); + free_iommu_one(iommu); + kfree(iommu); + } +} + +/* + * This function clues the initialization function for one IOMMU + * together and also allocates the command buffer and programs the + * hardware. It does NOT enable the IOMMU. This is done afterwards. + */ +static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) +{ + spin_lock_init(&iommu->lock); + + /* Add IOMMU to internal data structures */ + list_add_tail(&iommu->list, &amd_iommu_list); + iommu->index = amd_iommus_present++; + + if (unlikely(iommu->index >= MAX_IOMMUS)) { + WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n"); + return -ENOSYS; + } + + /* Index is fine - add IOMMU to the array */ + amd_iommus[iommu->index] = iommu; + + /* + * Copy data from ACPI table entry to the iommu struct + */ + iommu->dev = pci_get_bus_and_slot(PCI_BUS(h->devid), h->devid & 0xff); + if (!iommu->dev) + return 1; + + iommu->cap_ptr = h->cap_ptr; + iommu->pci_seg = h->pci_seg; + iommu->mmio_phys = h->mmio_phys; + iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys); + if (!iommu->mmio_base) + return -ENOMEM; + + iommu->cmd_buf = alloc_command_buffer(iommu); + if (!iommu->cmd_buf) + return -ENOMEM; + + iommu->evt_buf = alloc_event_buffer(iommu); + if (!iommu->evt_buf) + return -ENOMEM; + + iommu->int_enabled = false; + + init_iommu_from_pci(iommu); + init_iommu_from_acpi(iommu, h); + init_iommu_devices(iommu); + + if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) + amd_iommu_np_cache = true; + + return pci_enable_device(iommu->dev); +} + +/* + * Iterates over all IOMMU entries in the ACPI table, allocates the + * IOMMU structure and initializes it with init_iommu_one() + */ +static int __init init_iommu_all(struct acpi_table_header *table) +{ + u8 *p = (u8 *)table, *end = (u8 *)table; + struct ivhd_header *h; + struct amd_iommu *iommu; + int ret; + + end += table->length; + p += IVRS_HEADER_LENGTH; + + while (p < end) { + h = (struct ivhd_header *)p; + switch (*p) { + case ACPI_IVHD_TYPE: + + DUMP_printk("device: %02x:%02x.%01x cap: %04x " + "seg: %d flags: %01x info %04x\n", + PCI_BUS(h->devid), PCI_SLOT(h->devid), + PCI_FUNC(h->devid), h->cap_ptr, + h->pci_seg, h->flags, h->info); + DUMP_printk(" mmio-addr: %016llx\n", + h->mmio_phys); + + iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL); + if (iommu == NULL) { + amd_iommu_init_err = -ENOMEM; + return 0; + } + + ret = init_iommu_one(iommu, h); + if (ret) { + amd_iommu_init_err = ret; + return 0; + } + break; + default: + break; + } + p += h->length; + + } + WARN_ON(p != end); + + return 0; +} + +/**************************************************************************** + * + * The following functions initialize the MSI interrupts for all IOMMUs + * in the system. Its a bit challenging because there could be multiple + * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per + * pci_dev. + * + ****************************************************************************/ + +static int iommu_setup_msi(struct amd_iommu *iommu) +{ + int r; + + if (pci_enable_msi(iommu->dev)) + return 1; + + r = request_threaded_irq(iommu->dev->irq, + amd_iommu_int_handler, + amd_iommu_int_thread, + 0, "AMD-Vi", + iommu->dev); + + if (r) { + pci_disable_msi(iommu->dev); + return 1; + } + + iommu->int_enabled = true; + iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); + + return 0; +} + +static int iommu_init_msi(struct amd_iommu *iommu) +{ + if (iommu->int_enabled) + return 0; + + if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI)) + return iommu_setup_msi(iommu); + + return 1; +} + +/**************************************************************************** + * + * The next functions belong to the third pass of parsing the ACPI + * table. In this last pass the memory mapping requirements are + * gathered (like exclusion and unity mapping reanges). + * + ****************************************************************************/ + +static void __init free_unity_maps(void) +{ + struct unity_map_entry *entry, *next; + + list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) { + list_del(&entry->list); + kfree(entry); + } +} + +/* called when we find an exclusion range definition in ACPI */ +static int __init init_exclusion_range(struct ivmd_header *m) +{ + int i; + + switch (m->type) { + case ACPI_IVMD_TYPE: + set_device_exclusion_range(m->devid, m); + break; + case ACPI_IVMD_TYPE_ALL: + for (i = 0; i <= amd_iommu_last_bdf; ++i) + set_device_exclusion_range(i, m); + break; + case ACPI_IVMD_TYPE_RANGE: + for (i = m->devid; i <= m->aux; ++i) + set_device_exclusion_range(i, m); + break; + default: + break; + } + + return 0; +} + +/* called for unity map ACPI definition */ +static int __init init_unity_map_range(struct ivmd_header *m) +{ + struct unity_map_entry *e = 0; + char *s; + + e = kzalloc(sizeof(*e), GFP_KERNEL); + if (e == NULL) + return -ENOMEM; + + switch (m->type) { + default: + kfree(e); + return 0; + case ACPI_IVMD_TYPE: + s = "IVMD_TYPEi\t\t\t"; + e->devid_start = e->devid_end = m->devid; + break; + case ACPI_IVMD_TYPE_ALL: + s = "IVMD_TYPE_ALL\t\t"; + e->devid_start = 0; + e->devid_end = amd_iommu_last_bdf; + break; + case ACPI_IVMD_TYPE_RANGE: + s = "IVMD_TYPE_RANGE\t\t"; + e->devid_start = m->devid; + e->devid_end = m->aux; + break; + } + e->address_start = PAGE_ALIGN(m->range_start); + e->address_end = e->address_start + PAGE_ALIGN(m->range_length); + e->prot = m->flags >> 1; + + DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x" + " range_start: %016llx range_end: %016llx flags: %x\n", s, + PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start), + PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end), + PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end), + e->address_start, e->address_end, m->flags); + + list_add_tail(&e->list, &amd_iommu_unity_map); + + return 0; +} + +/* iterates over all memory definitions we find in the ACPI table */ +static int __init init_memory_definitions(struct acpi_table_header *table) +{ + u8 *p = (u8 *)table, *end = (u8 *)table; + struct ivmd_header *m; + + end += table->length; + p += IVRS_HEADER_LENGTH; + + while (p < end) { + m = (struct ivmd_header *)p; + if (m->flags & IVMD_FLAG_EXCL_RANGE) + init_exclusion_range(m); + else if (m->flags & IVMD_FLAG_UNITY_MAP) + init_unity_map_range(m); + + p += m->length; + } + + return 0; +} + +/* + * Init the device table to not allow DMA access for devices and + * suppress all page faults + */ +static void init_device_table(void) +{ + u32 devid; + + for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { + set_dev_entry_bit(devid, DEV_ENTRY_VALID); + set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION); + } +} + +static void iommu_init_flags(struct amd_iommu *iommu) +{ + iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ? + iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) : + iommu_feature_disable(iommu, CONTROL_HT_TUN_EN); + + iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ? + iommu_feature_enable(iommu, CONTROL_PASSPW_EN) : + iommu_feature_disable(iommu, CONTROL_PASSPW_EN); + + iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ? + iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) : + iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN); + + iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ? + iommu_feature_enable(iommu, CONTROL_ISOC_EN) : + iommu_feature_disable(iommu, CONTROL_ISOC_EN); + + /* + * make IOMMU memory accesses cache coherent + */ + iommu_feature_enable(iommu, CONTROL_COHERENT_EN); +} + +static void iommu_apply_resume_quirks(struct amd_iommu *iommu) +{ + int i, j; + u32 ioc_feature_control; + struct pci_dev *pdev = NULL; + + /* RD890 BIOSes may not have completely reconfigured the iommu */ + if (!is_rd890_iommu(iommu->dev)) + return; + + /* + * First, we need to ensure that the iommu is enabled. This is + * controlled by a register in the northbridge + */ + pdev = pci_get_bus_and_slot(iommu->dev->bus->number, PCI_DEVFN(0, 0)); + + if (!pdev) + return; + + /* Select Northbridge indirect register 0x75 and enable writing */ + pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7)); + pci_read_config_dword(pdev, 0x64, &ioc_feature_control); + + /* Enable the iommu */ + if (!(ioc_feature_control & 0x1)) + pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1); + + pci_dev_put(pdev); + + /* Restore the iommu BAR */ + pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, + iommu->stored_addr_lo); + pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8, + iommu->stored_addr_hi); + + /* Restore the l1 indirect regs for each of the 6 l1s */ + for (i = 0; i < 6; i++) + for (j = 0; j < 0x12; j++) + iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]); + + /* Restore the l2 indirect regs */ + for (i = 0; i < 0x83; i++) + iommu_write_l2(iommu, i, iommu->stored_l2[i]); + + /* Lock PCI setup registers */ + pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, + iommu->stored_addr_lo | 1); +} + +/* + * This function finally enables all IOMMUs found in the system after + * they have been initialized + */ +static void enable_iommus(void) +{ + struct amd_iommu *iommu; + + for_each_iommu(iommu) { + iommu_disable(iommu); + iommu_init_flags(iommu); + iommu_set_device_table(iommu); + iommu_enable_command_buffer(iommu); + iommu_enable_event_buffer(iommu); + iommu_set_exclusion_range(iommu); + iommu_init_msi(iommu); + iommu_enable(iommu); + iommu_flush_all_caches(iommu); + } +} + +static void disable_iommus(void) +{ + struct amd_iommu *iommu; + + for_each_iommu(iommu) + iommu_disable(iommu); +} + +/* + * Suspend/Resume support + * disable suspend until real resume implemented + */ + +static void amd_iommu_resume(void) +{ + struct amd_iommu *iommu; + + for_each_iommu(iommu) + iommu_apply_resume_quirks(iommu); + + /* re-load the hardware */ + enable_iommus(); + + /* + * we have to flush after the IOMMUs are enabled because a + * disabled IOMMU will never execute the commands we send + */ + for_each_iommu(iommu) + iommu_flush_all_caches(iommu); +} + +static int amd_iommu_suspend(void) +{ + /* disable IOMMUs to go out of the way for BIOS */ + disable_iommus(); + + return 0; +} + +static struct syscore_ops amd_iommu_syscore_ops = { + .suspend = amd_iommu_suspend, + .resume = amd_iommu_resume, +}; + +/* + * This is the core init function for AMD IOMMU hardware in the system. + * This function is called from the generic x86 DMA layer initialization + * code. + * + * This function basically parses the ACPI table for AMD IOMMU (IVRS) + * three times: + * + * 1 pass) Find the highest PCI device id the driver has to handle. + * Upon this information the size of the data structures is + * determined that needs to be allocated. + * + * 2 pass) Initialize the data structures just allocated with the + * information in the ACPI table about available AMD IOMMUs + * in the system. It also maps the PCI devices in the + * system to specific IOMMUs + * + * 3 pass) After the basic data structures are allocated and + * initialized we update them with information about memory + * remapping requirements parsed out of the ACPI table in + * this last pass. + * + * After that the hardware is initialized and ready to go. In the last + * step we do some Linux specific things like registering the driver in + * the dma_ops interface and initializing the suspend/resume support + * functions. Finally it prints some information about AMD IOMMUs and + * the driver state and enables the hardware. + */ +static int __init amd_iommu_init(void) +{ + int i, ret = 0; + + /* + * First parse ACPI tables to find the largest Bus/Dev/Func + * we need to handle. Upon this information the shared data + * structures for the IOMMUs in the system will be allocated + */ + if (acpi_table_parse("IVRS", find_last_devid_acpi) != 0) + return -ENODEV; + + ret = amd_iommu_init_err; + if (ret) + goto out; + + dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); + alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); + rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); + + ret = -ENOMEM; + + /* Device table - directly used by all IOMMUs */ + amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(dev_table_size)); + if (amd_iommu_dev_table == NULL) + goto out; + + /* + * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the + * IOMMU see for that device + */ + amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL, + get_order(alias_table_size)); + if (amd_iommu_alias_table == NULL) + goto free; + + /* IOMMU rlookup table - find the IOMMU for a specific device */ + amd_iommu_rlookup_table = (void *)__get_free_pages( + GFP_KERNEL | __GFP_ZERO, + get_order(rlookup_table_size)); + if (amd_iommu_rlookup_table == NULL) + goto free; + + amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages( + GFP_KERNEL | __GFP_ZERO, + get_order(MAX_DOMAIN_ID/8)); + if (amd_iommu_pd_alloc_bitmap == NULL) + goto free; + + /* init the device table */ + init_device_table(); + + /* + * let all alias entries point to itself + */ + for (i = 0; i <= amd_iommu_last_bdf; ++i) + amd_iommu_alias_table[i] = i; + + /* + * never allocate domain 0 because its used as the non-allocated and + * error value placeholder + */ + amd_iommu_pd_alloc_bitmap[0] = 1; + + spin_lock_init(&amd_iommu_pd_lock); + + /* + * now the data structures are allocated and basically initialized + * start the real acpi table scan + */ + ret = -ENODEV; + if (acpi_table_parse("IVRS", init_iommu_all) != 0) + goto free; + + if (amd_iommu_init_err) { + ret = amd_iommu_init_err; + goto free; + } + + if (acpi_table_parse("IVRS", init_memory_definitions) != 0) + goto free; + + if (amd_iommu_init_err) { + ret = amd_iommu_init_err; + goto free; + } + + ret = amd_iommu_init_devices(); + if (ret) + goto free; + + enable_iommus(); + + if (iommu_pass_through) + ret = amd_iommu_init_passthrough(); + else + ret = amd_iommu_init_dma_ops(); + + if (ret) + goto free_disable; + + amd_iommu_init_api(); + + amd_iommu_init_notifier(); + + register_syscore_ops(&amd_iommu_syscore_ops); + + if (iommu_pass_through) + goto out; + + if (amd_iommu_unmap_flush) + printk(KERN_INFO "AMD-Vi: IO/TLB flush on unmap enabled\n"); + else + printk(KERN_INFO "AMD-Vi: Lazy IO/TLB flushing enabled\n"); + + x86_platform.iommu_shutdown = disable_iommus; +out: + return ret; + +free_disable: + disable_iommus(); + +free: + amd_iommu_uninit_devices(); + + free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, + get_order(MAX_DOMAIN_ID/8)); + + free_pages((unsigned long)amd_iommu_rlookup_table, + get_order(rlookup_table_size)); + + free_pages((unsigned long)amd_iommu_alias_table, + get_order(alias_table_size)); + + free_pages((unsigned long)amd_iommu_dev_table, + get_order(dev_table_size)); + + free_iommu_all(); + + free_unity_maps(); + +#ifdef CONFIG_GART_IOMMU + /* + * We failed to initialize the AMD IOMMU - try fallback to GART + * if possible. + */ + gart_iommu_init(); + +#endif + + goto out; +} + +/**************************************************************************** + * + * Early detect code. This code runs at IOMMU detection time in the DMA + * layer. It just looks if there is an IVRS ACPI table to detect AMD + * IOMMUs + * + ****************************************************************************/ +static int __init early_amd_iommu_detect(struct acpi_table_header *table) +{ + return 0; +} + +int __init amd_iommu_detect(void) +{ + if (no_iommu || (iommu_detected && !gart_iommu_aperture)) + return -ENODEV; + + if (amd_iommu_disabled) + return -ENODEV; + + if (acpi_table_parse("IVRS", early_amd_iommu_detect) == 0) { + iommu_detected = 1; + amd_iommu_detected = 1; + x86_init.iommu.iommu_init = amd_iommu_init; + + /* Make sure ACS will be enabled */ + pci_request_acs(); + return 1; + } + return -ENODEV; +} + +/**************************************************************************** + * + * Parsing functions for the AMD IOMMU specific kernel command line + * options. + * + ****************************************************************************/ + +static int __init parse_amd_iommu_dump(char *str) +{ + amd_iommu_dump = true; + + return 1; +} + +static int __init parse_amd_iommu_options(char *str) +{ + for (; *str; ++str) { + if (strncmp(str, "fullflush", 9) == 0) + amd_iommu_unmap_flush = true; + if (strncmp(str, "off", 3) == 0) + amd_iommu_disabled = true; + } + + return 1; +} + +__setup("amd_iommu_dump", parse_amd_iommu_dump); +__setup("amd_iommu=", parse_amd_iommu_options); + +IOMMU_INIT_FINISH(amd_iommu_detect, + gart_iommu_hole_init, + 0, + 0); diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h new file mode 100644 index 000000000000..7ffaa64410b0 --- /dev/null +++ b/drivers/iommu/amd_iommu_proto.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2009-2010 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef _ASM_X86_AMD_IOMMU_PROTO_H +#define _ASM_X86_AMD_IOMMU_PROTO_H + +#include "amd_iommu_types.h" + +extern int amd_iommu_init_dma_ops(void); +extern int amd_iommu_init_passthrough(void); +extern irqreturn_t amd_iommu_int_thread(int irq, void *data); +extern irqreturn_t amd_iommu_int_handler(int irq, void *data); +extern void amd_iommu_apply_erratum_63(u16 devid); +extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu); +extern int amd_iommu_init_devices(void); +extern void amd_iommu_uninit_devices(void); +extern void amd_iommu_init_notifier(void); +extern void amd_iommu_init_api(void); +#ifndef CONFIG_AMD_IOMMU_STATS + +static inline void amd_iommu_stats_init(void) { } + +#endif /* !CONFIG_AMD_IOMMU_STATS */ + +static inline bool is_rd890_iommu(struct pci_dev *pdev) +{ + return (pdev->vendor == PCI_VENDOR_ID_ATI) && + (pdev->device == PCI_DEVICE_ID_RD890_IOMMU); +} + +static inline bool iommu_feature(struct amd_iommu *iommu, u64 f) +{ + if (!(iommu->cap & (1 << IOMMU_CAP_EFR))) + return false; + + return !!(iommu->features & f); +} + +#endif /* _ASM_X86_AMD_IOMMU_PROTO_H */ diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h new file mode 100644 index 000000000000..4c9982995414 --- /dev/null +++ b/drivers/iommu/amd_iommu_types.h @@ -0,0 +1,580 @@ +/* + * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * Leo Duran + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef _ASM_X86_AMD_IOMMU_TYPES_H +#define _ASM_X86_AMD_IOMMU_TYPES_H + +#include +#include +#include +#include + +/* + * Maximum number of IOMMUs supported + */ +#define MAX_IOMMUS 32 + +/* + * some size calculation constants + */ +#define DEV_TABLE_ENTRY_SIZE 32 +#define ALIAS_TABLE_ENTRY_SIZE 2 +#define RLOOKUP_TABLE_ENTRY_SIZE (sizeof(void *)) + +/* Length of the MMIO region for the AMD IOMMU */ +#define MMIO_REGION_LENGTH 0x4000 + +/* Capability offsets used by the driver */ +#define MMIO_CAP_HDR_OFFSET 0x00 +#define MMIO_RANGE_OFFSET 0x0c +#define MMIO_MISC_OFFSET 0x10 + +/* Masks, shifts and macros to parse the device range capability */ +#define MMIO_RANGE_LD_MASK 0xff000000 +#define MMIO_RANGE_FD_MASK 0x00ff0000 +#define MMIO_RANGE_BUS_MASK 0x0000ff00 +#define MMIO_RANGE_LD_SHIFT 24 +#define MMIO_RANGE_FD_SHIFT 16 +#define MMIO_RANGE_BUS_SHIFT 8 +#define MMIO_GET_LD(x) (((x) & MMIO_RANGE_LD_MASK) >> MMIO_RANGE_LD_SHIFT) +#define MMIO_GET_FD(x) (((x) & MMIO_RANGE_FD_MASK) >> MMIO_RANGE_FD_SHIFT) +#define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT) +#define MMIO_MSI_NUM(x) ((x) & 0x1f) + +/* Flag masks for the AMD IOMMU exclusion range */ +#define MMIO_EXCL_ENABLE_MASK 0x01ULL +#define MMIO_EXCL_ALLOW_MASK 0x02ULL + +/* Used offsets into the MMIO space */ +#define MMIO_DEV_TABLE_OFFSET 0x0000 +#define MMIO_CMD_BUF_OFFSET 0x0008 +#define MMIO_EVT_BUF_OFFSET 0x0010 +#define MMIO_CONTROL_OFFSET 0x0018 +#define MMIO_EXCL_BASE_OFFSET 0x0020 +#define MMIO_EXCL_LIMIT_OFFSET 0x0028 +#define MMIO_EXT_FEATURES 0x0030 +#define MMIO_CMD_HEAD_OFFSET 0x2000 +#define MMIO_CMD_TAIL_OFFSET 0x2008 +#define MMIO_EVT_HEAD_OFFSET 0x2010 +#define MMIO_EVT_TAIL_OFFSET 0x2018 +#define MMIO_STATUS_OFFSET 0x2020 + + +/* Extended Feature Bits */ +#define FEATURE_PREFETCH (1ULL<<0) +#define FEATURE_PPR (1ULL<<1) +#define FEATURE_X2APIC (1ULL<<2) +#define FEATURE_NX (1ULL<<3) +#define FEATURE_GT (1ULL<<4) +#define FEATURE_IA (1ULL<<6) +#define FEATURE_GA (1ULL<<7) +#define FEATURE_HE (1ULL<<8) +#define FEATURE_PC (1ULL<<9) + +/* MMIO status bits */ +#define MMIO_STATUS_COM_WAIT_INT_MASK 0x04 + +/* event logging constants */ +#define EVENT_ENTRY_SIZE 0x10 +#define EVENT_TYPE_SHIFT 28 +#define EVENT_TYPE_MASK 0xf +#define EVENT_TYPE_ILL_DEV 0x1 +#define EVENT_TYPE_IO_FAULT 0x2 +#define EVENT_TYPE_DEV_TAB_ERR 0x3 +#define EVENT_TYPE_PAGE_TAB_ERR 0x4 +#define EVENT_TYPE_ILL_CMD 0x5 +#define EVENT_TYPE_CMD_HARD_ERR 0x6 +#define EVENT_TYPE_IOTLB_INV_TO 0x7 +#define EVENT_TYPE_INV_DEV_REQ 0x8 +#define EVENT_DEVID_MASK 0xffff +#define EVENT_DEVID_SHIFT 0 +#define EVENT_DOMID_MASK 0xffff +#define EVENT_DOMID_SHIFT 0 +#define EVENT_FLAGS_MASK 0xfff +#define EVENT_FLAGS_SHIFT 0x10 + +/* feature control bits */ +#define CONTROL_IOMMU_EN 0x00ULL +#define CONTROL_HT_TUN_EN 0x01ULL +#define CONTROL_EVT_LOG_EN 0x02ULL +#define CONTROL_EVT_INT_EN 0x03ULL +#define CONTROL_COMWAIT_EN 0x04ULL +#define CONTROL_PASSPW_EN 0x08ULL +#define CONTROL_RESPASSPW_EN 0x09ULL +#define CONTROL_COHERENT_EN 0x0aULL +#define CONTROL_ISOC_EN 0x0bULL +#define CONTROL_CMDBUF_EN 0x0cULL +#define CONTROL_PPFLOG_EN 0x0dULL +#define CONTROL_PPFINT_EN 0x0eULL + +/* command specific defines */ +#define CMD_COMPL_WAIT 0x01 +#define CMD_INV_DEV_ENTRY 0x02 +#define CMD_INV_IOMMU_PAGES 0x03 +#define CMD_INV_IOTLB_PAGES 0x04 +#define CMD_INV_ALL 0x08 + +#define CMD_COMPL_WAIT_STORE_MASK 0x01 +#define CMD_COMPL_WAIT_INT_MASK 0x02 +#define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01 +#define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02 + +#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL + +/* macros and definitions for device table entries */ +#define DEV_ENTRY_VALID 0x00 +#define DEV_ENTRY_TRANSLATION 0x01 +#define DEV_ENTRY_IR 0x3d +#define DEV_ENTRY_IW 0x3e +#define DEV_ENTRY_NO_PAGE_FAULT 0x62 +#define DEV_ENTRY_EX 0x67 +#define DEV_ENTRY_SYSMGT1 0x68 +#define DEV_ENTRY_SYSMGT2 0x69 +#define DEV_ENTRY_INIT_PASS 0xb8 +#define DEV_ENTRY_EINT_PASS 0xb9 +#define DEV_ENTRY_NMI_PASS 0xba +#define DEV_ENTRY_LINT0_PASS 0xbe +#define DEV_ENTRY_LINT1_PASS 0xbf +#define DEV_ENTRY_MODE_MASK 0x07 +#define DEV_ENTRY_MODE_SHIFT 0x09 + +/* constants to configure the command buffer */ +#define CMD_BUFFER_SIZE 8192 +#define CMD_BUFFER_UNINITIALIZED 1 +#define CMD_BUFFER_ENTRIES 512 +#define MMIO_CMD_SIZE_SHIFT 56 +#define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT) + +/* constants for event buffer handling */ +#define EVT_BUFFER_SIZE 8192 /* 512 entries */ +#define EVT_LEN_MASK (0x9ULL << 56) + +#define PAGE_MODE_NONE 0x00 +#define PAGE_MODE_1_LEVEL 0x01 +#define PAGE_MODE_2_LEVEL 0x02 +#define PAGE_MODE_3_LEVEL 0x03 +#define PAGE_MODE_4_LEVEL 0x04 +#define PAGE_MODE_5_LEVEL 0x05 +#define PAGE_MODE_6_LEVEL 0x06 + +#define PM_LEVEL_SHIFT(x) (12 + ((x) * 9)) +#define PM_LEVEL_SIZE(x) (((x) < 6) ? \ + ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \ + (0xffffffffffffffffULL)) +#define PM_LEVEL_INDEX(x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL) +#define PM_LEVEL_ENC(x) (((x) << 9) & 0xe00ULL) +#define PM_LEVEL_PDE(x, a) ((a) | PM_LEVEL_ENC((x)) | \ + IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) +#define PM_PTE_LEVEL(pte) (((pte) >> 9) & 0x7ULL) + +#define PM_MAP_4k 0 +#define PM_ADDR_MASK 0x000ffffffffff000ULL +#define PM_MAP_MASK(lvl) (PM_ADDR_MASK & \ + (~((1ULL << (12 + ((lvl) * 9))) - 1))) +#define PM_ALIGNED(lvl, addr) ((PM_MAP_MASK(lvl) & (addr)) == (addr)) + +/* + * Returns the page table level to use for a given page size + * Pagesize is expected to be a power-of-two + */ +#define PAGE_SIZE_LEVEL(pagesize) \ + ((__ffs(pagesize) - 12) / 9) +/* + * Returns the number of ptes to use for a given page size + * Pagesize is expected to be a power-of-two + */ +#define PAGE_SIZE_PTE_COUNT(pagesize) \ + (1ULL << ((__ffs(pagesize) - 12) % 9)) + +/* + * Aligns a given io-virtual address to a given page size + * Pagesize is expected to be a power-of-two + */ +#define PAGE_SIZE_ALIGN(address, pagesize) \ + ((address) & ~((pagesize) - 1)) +/* + * Creates an IOMMU PTE for an address an a given pagesize + * The PTE has no permission bits set + * Pagesize is expected to be a power-of-two larger than 4096 + */ +#define PAGE_SIZE_PTE(address, pagesize) \ + (((address) | ((pagesize) - 1)) & \ + (~(pagesize >> 1)) & PM_ADDR_MASK) + +/* + * Takes a PTE value with mode=0x07 and returns the page size it maps + */ +#define PTE_PAGE_SIZE(pte) \ + (1ULL << (1 + ffz(((pte) | 0xfffULL)))) + +#define IOMMU_PTE_P (1ULL << 0) +#define IOMMU_PTE_TV (1ULL << 1) +#define IOMMU_PTE_U (1ULL << 59) +#define IOMMU_PTE_FC (1ULL << 60) +#define IOMMU_PTE_IR (1ULL << 61) +#define IOMMU_PTE_IW (1ULL << 62) + +#define DTE_FLAG_IOTLB 0x01 + +#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL) +#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P) +#define IOMMU_PTE_PAGE(pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK)) +#define IOMMU_PTE_MODE(pte) (((pte) >> 9) & 0x07) + +#define IOMMU_PROT_MASK 0x03 +#define IOMMU_PROT_IR 0x01 +#define IOMMU_PROT_IW 0x02 + +/* IOMMU capabilities */ +#define IOMMU_CAP_IOTLB 24 +#define IOMMU_CAP_NPCACHE 26 +#define IOMMU_CAP_EFR 27 + +#define MAX_DOMAIN_ID 65536 + +/* FIXME: move this macro to */ +#define PCI_BUS(x) (((x) >> 8) & 0xff) + +/* Protection domain flags */ +#define PD_DMA_OPS_MASK (1UL << 0) /* domain used for dma_ops */ +#define PD_DEFAULT_MASK (1UL << 1) /* domain is a default dma_ops + domain for an IOMMU */ +#define PD_PASSTHROUGH_MASK (1UL << 2) /* domain has no page + translation */ + +extern bool amd_iommu_dump; +#define DUMP_printk(format, arg...) \ + do { \ + if (amd_iommu_dump) \ + printk(KERN_INFO "AMD-Vi: " format, ## arg); \ + } while(0); + +/* global flag if IOMMUs cache non-present entries */ +extern bool amd_iommu_np_cache; +/* Only true if all IOMMUs support device IOTLBs */ +extern bool amd_iommu_iotlb_sup; + +/* + * Make iterating over all IOMMUs easier + */ +#define for_each_iommu(iommu) \ + list_for_each_entry((iommu), &amd_iommu_list, list) +#define for_each_iommu_safe(iommu, next) \ + list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list) + +#define APERTURE_RANGE_SHIFT 27 /* 128 MB */ +#define APERTURE_RANGE_SIZE (1ULL << APERTURE_RANGE_SHIFT) +#define APERTURE_RANGE_PAGES (APERTURE_RANGE_SIZE >> PAGE_SHIFT) +#define APERTURE_MAX_RANGES 32 /* allows 4GB of DMA address space */ +#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT) +#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL) + +/* + * This structure contains generic data for IOMMU protection domains + * independent of their use. + */ +struct protection_domain { + struct list_head list; /* for list of all protection domains */ + struct list_head dev_list; /* List of all devices in this domain */ + spinlock_t lock; /* mostly used to lock the page table*/ + struct mutex api_lock; /* protect page tables in the iommu-api path */ + u16 id; /* the domain id written to the device table */ + int mode; /* paging mode (0-6 levels) */ + u64 *pt_root; /* page table root pointer */ + unsigned long flags; /* flags to find out type of domain */ + bool updated; /* complete domain flush required */ + unsigned dev_cnt; /* devices assigned to this domain */ + unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ + void *priv; /* private data */ + +}; + +/* + * This struct contains device specific data for the IOMMU + */ +struct iommu_dev_data { + struct list_head list; /* For domain->dev_list */ + struct device *dev; /* Device this data belong to */ + struct device *alias; /* The Alias Device */ + struct protection_domain *domain; /* Domain the device is bound to */ + atomic_t bind; /* Domain attach reverent count */ +}; + +/* + * For dynamic growth the aperture size is split into ranges of 128MB of + * DMA address space each. This struct represents one such range. + */ +struct aperture_range { + + /* address allocation bitmap */ + unsigned long *bitmap; + + /* + * Array of PTE pages for the aperture. In this array we save all the + * leaf pages of the domain page table used for the aperture. This way + * we don't need to walk the page table to find a specific PTE. We can + * just calculate its address in constant time. + */ + u64 *pte_pages[64]; + + unsigned long offset; +}; + +/* + * Data container for a dma_ops specific protection domain + */ +struct dma_ops_domain { + struct list_head list; + + /* generic protection domain information */ + struct protection_domain domain; + + /* size of the aperture for the mappings */ + unsigned long aperture_size; + + /* address we start to search for free addresses */ + unsigned long next_address; + + /* address space relevant data */ + struct aperture_range *aperture[APERTURE_MAX_RANGES]; + + /* This will be set to true when TLB needs to be flushed */ + bool need_flush; + + /* + * if this is a preallocated domain, keep the device for which it was + * preallocated in this variable + */ + u16 target_dev; +}; + +/* + * Structure where we save information about one hardware AMD IOMMU in the + * system. + */ +struct amd_iommu { + struct list_head list; + + /* Index within the IOMMU array */ + int index; + + /* locks the accesses to the hardware */ + spinlock_t lock; + + /* Pointer to PCI device of this IOMMU */ + struct pci_dev *dev; + + /* physical address of MMIO space */ + u64 mmio_phys; + /* virtual address of MMIO space */ + u8 *mmio_base; + + /* capabilities of that IOMMU read from ACPI */ + u32 cap; + + /* flags read from acpi table */ + u8 acpi_flags; + + /* Extended features */ + u64 features; + + /* + * Capability pointer. There could be more than one IOMMU per PCI + * device function if there are more than one AMD IOMMU capability + * pointers. + */ + u16 cap_ptr; + + /* pci domain of this IOMMU */ + u16 pci_seg; + + /* first device this IOMMU handles. read from PCI */ + u16 first_device; + /* last device this IOMMU handles. read from PCI */ + u16 last_device; + + /* start of exclusion range of that IOMMU */ + u64 exclusion_start; + /* length of exclusion range of that IOMMU */ + u64 exclusion_length; + + /* command buffer virtual address */ + u8 *cmd_buf; + /* size of command buffer */ + u32 cmd_buf_size; + + /* size of event buffer */ + u32 evt_buf_size; + /* event buffer virtual address */ + u8 *evt_buf; + /* MSI number for event interrupt */ + u16 evt_msi_num; + + /* true if interrupts for this IOMMU are already enabled */ + bool int_enabled; + + /* if one, we need to send a completion wait command */ + bool need_sync; + + /* default dma_ops domain for that IOMMU */ + struct dma_ops_domain *default_dom; + + /* + * We can't rely on the BIOS to restore all values on reinit, so we + * need to stash them + */ + + /* The iommu BAR */ + u32 stored_addr_lo; + u32 stored_addr_hi; + + /* + * Each iommu has 6 l1s, each of which is documented as having 0x12 + * registers + */ + u32 stored_l1[6][0x12]; + + /* The l2 indirect registers */ + u32 stored_l2[0x83]; +}; + +/* + * List with all IOMMUs in the system. This list is not locked because it is + * only written and read at driver initialization or suspend time + */ +extern struct list_head amd_iommu_list; + +/* + * Array with pointers to each IOMMU struct + * The indices are referenced in the protection domains + */ +extern struct amd_iommu *amd_iommus[MAX_IOMMUS]; + +/* Number of IOMMUs present in the system */ +extern int amd_iommus_present; + +/* + * Declarations for the global list of all protection domains + */ +extern spinlock_t amd_iommu_pd_lock; +extern struct list_head amd_iommu_pd_list; + +/* + * Structure defining one entry in the device table + */ +struct dev_table_entry { + u32 data[8]; +}; + +/* + * One entry for unity mappings parsed out of the ACPI table. + */ +struct unity_map_entry { + struct list_head list; + + /* starting device id this entry is used for (including) */ + u16 devid_start; + /* end device id this entry is used for (including) */ + u16 devid_end; + + /* start address to unity map (including) */ + u64 address_start; + /* end address to unity map (including) */ + u64 address_end; + + /* required protection */ + int prot; +}; + +/* + * List of all unity mappings. It is not locked because as runtime it is only + * read. It is created at ACPI table parsing time. + */ +extern struct list_head amd_iommu_unity_map; + +/* + * Data structures for device handling + */ + +/* + * Device table used by hardware. Read and write accesses by software are + * locked with the amd_iommu_pd_table lock. + */ +extern struct dev_table_entry *amd_iommu_dev_table; + +/* + * Alias table to find requestor ids to device ids. Not locked because only + * read on runtime. + */ +extern u16 *amd_iommu_alias_table; + +/* + * Reverse lookup table to find the IOMMU which translates a specific device. + */ +extern struct amd_iommu **amd_iommu_rlookup_table; + +/* size of the dma_ops aperture as power of 2 */ +extern unsigned amd_iommu_aperture_order; + +/* largest PCI device id we expect translation requests for */ +extern u16 amd_iommu_last_bdf; + +/* allocation bitmap for domain ids */ +extern unsigned long *amd_iommu_pd_alloc_bitmap; + +/* + * If true, the addresses will be flushed on unmap time, not when + * they are reused + */ +extern bool amd_iommu_unmap_flush; + +/* takes bus and device/function and returns the device id + * FIXME: should that be in generic PCI code? */ +static inline u16 calc_devid(u8 bus, u8 devfn) +{ + return (((u16)bus) << 8) | devfn; +} + +#ifdef CONFIG_AMD_IOMMU_STATS + +struct __iommu_counter { + char *name; + struct dentry *dent; + u64 value; +}; + +#define DECLARE_STATS_COUNTER(nm) \ + static struct __iommu_counter nm = { \ + .name = #nm, \ + } + +#define INC_STATS_COUNTER(name) name.value += 1 +#define ADD_STATS_COUNTER(name, x) name.value += (x) +#define SUB_STATS_COUNTER(name, x) name.value -= (x) + +#else /* CONFIG_AMD_IOMMU_STATS */ + +#define DECLARE_STATS_COUNTER(name) +#define INC_STATS_COUNTER(name) +#define ADD_STATS_COUNTER(name, x) +#define SUB_STATS_COUNTER(name, x) + +#endif /* CONFIG_AMD_IOMMU_STATS */ + +#endif /* _ASM_X86_AMD_IOMMU_TYPES_H */ diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h new file mode 100644 index 000000000000..a6863a2dec1f --- /dev/null +++ b/include/linux/amd-iommu.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * Leo Duran + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef _ASM_X86_AMD_IOMMU_H +#define _ASM_X86_AMD_IOMMU_H + +#include + +#ifdef CONFIG_AMD_IOMMU + +extern int amd_iommu_detect(void); + +#else + +static inline int amd_iommu_detect(void) { return -ENODEV; } + +#endif + +#endif /* _ASM_X86_AMD_IOMMU_H */ -- cgit v1.2.3 From b7f080cfe223b3b7424872639d153695615a9255 Mon Sep 17 00:00:00 2001 From: Alexey Dobriyan Date: Thu, 16 Jun 2011 11:01:34 +0000 Subject: net: remove mm.h inclusion from netdevice.h Remove linux/mm.h inclusion from netdevice.h -- it's unused (I've checked manually). To prevent mm.h inclusion via other channels also extract "enum dma_data_direction" definition into separate header. This tiny piece is what gluing netdevice.h with mm.h via "netdevice.h => dmaengine.h => dma-mapping.h => scatterlist.h => mm.h". Removal of mm.h from scatterlist.h was tried and was found not feasible on most archs, so the link was cutoff earlier. Hope people are OK with tiny include file. Note, that mm_types.h is still dragged in, but it is a separate story. Signed-off-by: Alexey Dobriyan Signed-off-by: David S. Miller --- arch/arm/mach-davinci/board-mityomapl138.c | 1 + arch/arm/mach-davinci/dm646x.c | 1 + arch/arm/mach-davinci/pm.c | 1 + arch/arm/mach-imx/dma-v1.c | 1 + arch/arm/mach-imx/mach-mx31_3ds.c | 1 + arch/arm/mach-iop13xx/setup.c | 1 + arch/arm/mach-mxs/devices/platform-auart.c | 1 + arch/arm/mach-mxs/devices/platform-dma.c | 1 + arch/arm/mach-mxs/devices/platform-fec.c | 1 + arch/arm/plat-mxc/devices/platform-fec.c | 1 + arch/arm/plat-mxc/devices/platform-fsl-usb2-udc.c | 1 + arch/arm/plat-mxc/devices/platform-imx-fb.c | 1 + arch/arm/plat-mxc/devices/platform-ipu-core.c | 1 + arch/arm/plat-mxc/devices/platform-mxc-ehci.c | 1 + arch/arm/plat-mxc/devices/platform-mxc-mmc.c | 1 + arch/arm/plat-nomadik/include/plat/ste_dma40.h | 1 + arch/x86/kernel/tboot.c | 1 + crypto/async_tx/raid6test.c | 1 + drivers/dma/coh901318.c | 1 + drivers/dma/dmaengine.c | 1 + drivers/dma/dmatest.c | 1 + drivers/dma/ipu/ipu_idmac.c | 1 + drivers/dma/ste_dma40.c | 1 + drivers/media/dvb/mantis/mantis_ca.c | 1 + drivers/media/dvb/mantis/mantis_evm.c | 1 + drivers/media/dvb/mantis/mantis_hif.c | 1 + drivers/media/dvb/mantis/mantis_ioc.c | 1 + drivers/media/dvb/mantis/mantis_pcmcia.c | 1 + drivers/media/dvb/mantis/mantis_uart.c | 1 + drivers/media/dvb/mantis/mantis_vp1034.c | 1 + drivers/mmc/host/tmio_mmc_dma.c | 1 + drivers/mtd/nand/atmel_nand.c | 1 + drivers/net/arm/ks8695net.c | 1 + drivers/net/bnx2x/bnx2x.h | 1 + drivers/net/can/janz-ican3.c | 1 + drivers/net/can/softing/softing_fw.c | 1 + drivers/net/can/softing/softing_main.c | 1 + drivers/net/ethoc.c | 1 + drivers/net/fec_mpc52xx.c | 1 + drivers/net/greth.c | 1 + drivers/net/irda/pxaficp_ir.c | 1 + drivers/net/ks8851_mll.c | 1 + drivers/net/sgiseeq.c | 1 + drivers/net/stmmac/dwmac1000_core.c | 1 + drivers/net/stmmac/dwmac1000_dma.c | 1 + drivers/net/stmmac/dwmac100_core.c | 1 + drivers/net/stmmac/dwmac100_dma.c | 1 + drivers/net/stmmac/stmmac_ethtool.c | 1 + drivers/net/stmmac/stmmac_mdio.c | 1 + drivers/net/usb/cdc-phonet.c | 1 + drivers/net/vxge/vxge-config.h | 1 + drivers/net/wireless/ath/ath5k/base.c | 1 + drivers/net/wireless/ath/ath9k/beacon.c | 1 + drivers/net/wireless/ath/ath9k/init.c | 1 + drivers/net/wireless/ath/ath9k/recv.c | 1 + drivers/net/wireless/ath/ath9k/xmit.c | 1 + drivers/staging/pohmelfs/crypto.c | 1 + drivers/tty/serial/ifx6x60.c | 1 + drivers/usb/gadget/f_phonet.c | 1 + include/crypto/if_alg.h | 1 + include/linux/dma-direction.h | 13 +++++++++++++ include/linux/dma-mapping.h | 10 +--------- include/linux/dmaengine.h | 4 +++- include/linux/netdevice.h | 1 - net/sched/sch_netem.c | 1 + security/apparmor/lib.c | 1 + 66 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 include/linux/dma-direction.h (limited to 'arch/x86/kernel') diff --git a/arch/arm/mach-davinci/board-mityomapl138.c b/arch/arm/mach-davinci/board-mityomapl138.c index 606a6f27ed6c..5f5d78308873 100644 --- a/arch/arm/mach-davinci/board-mityomapl138.c +++ b/arch/arm/mach-davinci/board-mityomapl138.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include diff --git a/arch/arm/mach-davinci/dm646x.c b/arch/arm/mach-davinci/dm646x.c index 1e0f809644bb..e00d61e2efbe 100644 --- a/arch/arm/mach-davinci/dm646x.c +++ b/arch/arm/mach-davinci/dm646x.c @@ -8,6 +8,7 @@ * is licensed "as is" without any warranty of any kind, whether express * or implied. */ +#include #include #include #include diff --git a/arch/arm/mach-davinci/pm.c b/arch/arm/mach-davinci/pm.c index 1bd73a04be20..04c49f7543ef 100644 --- a/arch/arm/mach-davinci/pm.c +++ b/arch/arm/mach-davinci/pm.c @@ -17,6 +17,7 @@ #include #include +#include #include #include diff --git a/arch/arm/mach-imx/dma-v1.c b/arch/arm/mach-imx/dma-v1.c index 236f1495efad..f8aa5be0eb15 100644 --- a/arch/arm/mach-imx/dma-v1.c +++ b/arch/arm/mach-imx/dma-v1.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/arm/mach-imx/mach-mx31_3ds.c b/arch/arm/mach-imx/mach-mx31_3ds.c index 9b982449cb52..b5ecc26d08a4 100644 --- a/arch/arm/mach-imx/mach-mx31_3ds.c +++ b/arch/arm/mach-imx/mach-mx31_3ds.c @@ -13,6 +13,7 @@ */ #include +#include #include #include #include diff --git a/arch/arm/mach-iop13xx/setup.c b/arch/arm/mach-iop13xx/setup.c index 5c147fb66a01..a5b989728b9e 100644 --- a/arch/arm/mach-iop13xx/setup.c +++ b/arch/arm/mach-iop13xx/setup.c @@ -17,6 +17,7 @@ * */ +#include #include #include #ifdef CONFIG_MTD_PHYSMAP diff --git a/arch/arm/mach-mxs/devices/platform-auart.c b/arch/arm/mach-mxs/devices/platform-auart.c index 796606cce0ce..27608f5d2ac8 100644 --- a/arch/arm/mach-mxs/devices/platform-auart.c +++ b/arch/arm/mach-mxs/devices/platform-auart.c @@ -6,6 +6,7 @@ * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. */ +#include #include #include #include diff --git a/arch/arm/mach-mxs/devices/platform-dma.c b/arch/arm/mach-mxs/devices/platform-dma.c index 295c4424d5d9..6a0202b1016c 100644 --- a/arch/arm/mach-mxs/devices/platform-dma.c +++ b/arch/arm/mach-mxs/devices/platform-dma.c @@ -6,6 +6,7 @@ * Free Software Foundation. */ #include +#include #include #include diff --git a/arch/arm/mach-mxs/devices/platform-fec.c b/arch/arm/mach-mxs/devices/platform-fec.c index 9859cf283335..ae96a4fd8f14 100644 --- a/arch/arm/mach-mxs/devices/platform-fec.c +++ b/arch/arm/mach-mxs/devices/platform-fec.c @@ -6,6 +6,7 @@ * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. */ +#include #include #include #include diff --git a/arch/arm/plat-mxc/devices/platform-fec.c b/arch/arm/plat-mxc/devices/platform-fec.c index ccc789e21daa..4fc6ffc2a13e 100644 --- a/arch/arm/plat-mxc/devices/platform-fec.c +++ b/arch/arm/plat-mxc/devices/platform-fec.c @@ -6,6 +6,7 @@ * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. */ +#include #include #include #include diff --git a/arch/arm/plat-mxc/devices/platform-fsl-usb2-udc.c b/arch/arm/plat-mxc/devices/platform-fsl-usb2-udc.c index 59c33f6e401c..23ce08e6ffd2 100644 --- a/arch/arm/plat-mxc/devices/platform-fsl-usb2-udc.c +++ b/arch/arm/plat-mxc/devices/platform-fsl-usb2-udc.c @@ -6,6 +6,7 @@ * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. */ +#include #include #include diff --git a/arch/arm/plat-mxc/devices/platform-imx-fb.c b/arch/arm/plat-mxc/devices/platform-imx-fb.c index 79a1cb18a5b0..2b0b5e0aa998 100644 --- a/arch/arm/plat-mxc/devices/platform-imx-fb.c +++ b/arch/arm/plat-mxc/devices/platform-imx-fb.c @@ -6,6 +6,7 @@ * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. */ +#include #include #include diff --git a/arch/arm/plat-mxc/devices/platform-ipu-core.c b/arch/arm/plat-mxc/devices/platform-ipu-core.c index edf65034aea5..79d340ae0af1 100644 --- a/arch/arm/plat-mxc/devices/platform-ipu-core.c +++ b/arch/arm/plat-mxc/devices/platform-ipu-core.c @@ -6,6 +6,7 @@ * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. */ +#include #include #include diff --git a/arch/arm/plat-mxc/devices/platform-mxc-ehci.c b/arch/arm/plat-mxc/devices/platform-mxc-ehci.c index cc488f4b6204..e1763e03e7cb 100644 --- a/arch/arm/plat-mxc/devices/platform-mxc-ehci.c +++ b/arch/arm/plat-mxc/devices/platform-mxc-ehci.c @@ -6,6 +6,7 @@ * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. */ +#include #include #include diff --git a/arch/arm/plat-mxc/devices/platform-mxc-mmc.c b/arch/arm/plat-mxc/devices/platform-mxc-mmc.c index 90d762f6f93b..540d3a7d92df 100644 --- a/arch/arm/plat-mxc/devices/platform-mxc-mmc.c +++ b/arch/arm/plat-mxc/devices/platform-mxc-mmc.c @@ -6,6 +6,7 @@ * the terms of the GNU General Public License version 2 as published by the * Free Software Foundation. */ +#include #include #include diff --git a/arch/arm/plat-nomadik/include/plat/ste_dma40.h b/arch/arm/plat-nomadik/include/plat/ste_dma40.h index c44886062f8e..685c78716d95 100644 --- a/arch/arm/plat-nomadik/include/plat/ste_dma40.h +++ b/arch/arm/plat-nomadik/include/plat/ste_dma40.h @@ -10,6 +10,7 @@ #define STE_DMA40_H #include +#include #include #include diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index 30ac65df7d4e..e07a2fc876b9 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c index c1321935ebcc..c88ff9e3fd30 100644 --- a/crypto/async_tx/raid6test.c +++ b/crypto/async_tx/raid6test.c @@ -21,6 +21,7 @@ */ #include #include +#include #include #undef pr diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c index af8c0b5ed70f..a92d95eac86b 100644 --- a/drivers/dma/coh901318.c +++ b/drivers/dma/coh901318.c @@ -11,6 +11,7 @@ #include #include /* printk() */ #include /* everything... */ +#include #include /* kmalloc() */ #include #include diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 8bcb15fb959d..48694c34d96b 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -45,6 +45,7 @@ * See Documentation/dmaengine.txt for more details */ +#include #include #include #include diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index b4f5c32b6a47..765f5ff22304 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -8,6 +8,7 @@ * published by the Free Software Foundation. */ #include +#include #include #include #include diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c index c1a125e7d1df..fd7d2b308cf2 100644 --- a/drivers/dma/ipu/ipu_idmac.c +++ b/drivers/dma/ipu/ipu_idmac.c @@ -9,6 +9,7 @@ * published by the Free Software Foundation. */ +#include #include #include #include diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 8f222d4db7de..29d1addbe0cf 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -6,6 +6,7 @@ * License terms: GNU General Public License (GPL) version 2 */ +#include #include #include #include diff --git a/drivers/media/dvb/mantis/mantis_ca.c b/drivers/media/dvb/mantis/mantis_ca.c index 330216febd78..3d7046909009 100644 --- a/drivers/media/dvb/mantis/mantis_ca.c +++ b/drivers/media/dvb/mantis/mantis_ca.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "dmxdev.h" #include "dvbdev.h" diff --git a/drivers/media/dvb/mantis/mantis_evm.c b/drivers/media/dvb/mantis/mantis_evm.c index 9f73c2cfc9ea..36f2256ebb0e 100644 --- a/drivers/media/dvb/mantis/mantis_evm.c +++ b/drivers/media/dvb/mantis/mantis_evm.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "dmxdev.h" #include "dvbdev.h" diff --git a/drivers/media/dvb/mantis/mantis_hif.c b/drivers/media/dvb/mantis/mantis_hif.c index 5772ebb3a69e..672cf4d2462d 100644 --- a/drivers/media/dvb/mantis/mantis_hif.c +++ b/drivers/media/dvb/mantis/mantis_hif.c @@ -23,6 +23,7 @@ #include #include +#include #include "dmxdev.h" #include "dvbdev.h" diff --git a/drivers/media/dvb/mantis/mantis_ioc.c b/drivers/media/dvb/mantis/mantis_ioc.c index 479086dbb9a8..24fcdc63d6d5 100644 --- a/drivers/media/dvb/mantis/mantis_ioc.c +++ b/drivers/media/dvb/mantis/mantis_ioc.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "dmxdev.h" #include "dvbdev.h" diff --git a/drivers/media/dvb/mantis/mantis_pcmcia.c b/drivers/media/dvb/mantis/mantis_pcmcia.c index 5cb545b913f6..2f188c089666 100644 --- a/drivers/media/dvb/mantis/mantis_pcmcia.c +++ b/drivers/media/dvb/mantis/mantis_pcmcia.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "dmxdev.h" #include "dvbdev.h" diff --git a/drivers/media/dvb/mantis/mantis_uart.c b/drivers/media/dvb/mantis/mantis_uart.c index f807c8ba26e4..18340dafa426 100644 --- a/drivers/media/dvb/mantis/mantis_uart.c +++ b/drivers/media/dvb/mantis/mantis_uart.c @@ -20,6 +20,7 @@ #include #include +#include #include #include diff --git a/drivers/media/dvb/mantis/mantis_vp1034.c b/drivers/media/dvb/mantis/mantis_vp1034.c index 26bc0cbe84d4..430ae84ce528 100644 --- a/drivers/media/dvb/mantis/mantis_vp1034.c +++ b/drivers/media/dvb/mantis/mantis_vp1034.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "dmxdev.h" #include "dvbdev.h" diff --git a/drivers/mmc/host/tmio_mmc_dma.c b/drivers/mmc/host/tmio_mmc_dma.c index 25f1ad6cbe09..bf12513d6297 100644 --- a/drivers/mmc/host/tmio_mmc_dma.c +++ b/drivers/mmc/host/tmio_mmc_dma.c @@ -11,6 +11,7 @@ */ #include +#include #include #include #include diff --git a/drivers/mtd/nand/atmel_nand.c b/drivers/mtd/nand/atmel_nand.c index b300705d41cb..1b90fd56bef1 100644 --- a/drivers/mtd/nand/atmel_nand.c +++ b/drivers/mtd/nand/atmel_nand.c @@ -22,6 +22,7 @@ * */ +#include #include #include #include diff --git a/drivers/net/arm/ks8695net.c b/drivers/net/arm/ks8695net.c index bb62b3f51837..c827a6097d02 100644 --- a/drivers/net/arm/ks8695net.c +++ b/drivers/net/arm/ks8695net.c @@ -16,6 +16,7 @@ * Vincent Sanders */ +#include #include #include #include diff --git a/drivers/net/bnx2x/bnx2x.h b/drivers/net/bnx2x/bnx2x.h index 69fc7280be30..eff78a439eeb 100644 --- a/drivers/net/bnx2x/bnx2x.h +++ b/drivers/net/bnx2x/bnx2x.h @@ -14,6 +14,7 @@ #ifndef BNX2X_H #define BNX2X_H #include +#include #include /* compilation time flags */ diff --git a/drivers/net/can/janz-ican3.c b/drivers/net/can/janz-ican3.c index f1942cab35f6..32778d56d330 100644 --- a/drivers/net/can/janz-ican3.c +++ b/drivers/net/can/janz-ican3.c @@ -22,6 +22,7 @@ #include #include +#include /* the DPM has 64k of memory, organized into 256x 256 byte pages */ #define DPM_NUM_PAGES 256 diff --git a/drivers/net/can/softing/softing_fw.c b/drivers/net/can/softing/softing_fw.c index b520784fb197..310596175676 100644 --- a/drivers/net/can/softing/softing_fw.c +++ b/drivers/net/can/softing/softing_fw.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "softing.h" diff --git a/drivers/net/can/softing/softing_main.c b/drivers/net/can/softing/softing_main.c index 60a49e5a2a53..954b2959b6b2 100644 --- a/drivers/net/can/softing/softing_main.c +++ b/drivers/net/can/softing/softing_main.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "softing.h" diff --git a/drivers/net/ethoc.c b/drivers/net/ethoc.c index 0e8cc75bb5f4..0da6295d9da6 100644 --- a/drivers/net/ethoc.c +++ b/drivers/net/ethoc.c @@ -11,6 +11,7 @@ * Written by Thierry Reding */ +#include #include #include #include diff --git a/drivers/net/fec_mpc52xx.c b/drivers/net/fec_mpc52xx.c index cecc3b1eb486..381bdea97d5f 100644 --- a/drivers/net/fec_mpc52xx.c +++ b/drivers/net/fec_mpc52xx.c @@ -14,6 +14,7 @@ * */ +#include #include #include diff --git a/drivers/net/greth.c b/drivers/net/greth.c index 69b86d7fac85..82c3767ec5f8 100644 --- a/drivers/net/greth.c +++ b/drivers/net/greth.c @@ -22,6 +22,7 @@ * Marko Isomaki */ +#include #include #include #include diff --git a/drivers/net/irda/pxaficp_ir.c b/drivers/net/irda/pxaficp_ir.c index b1d1ce3dd8b5..d0851dfa0378 100644 --- a/drivers/net/irda/pxaficp_ir.c +++ b/drivers/net/irda/pxaficp_ir.c @@ -12,6 +12,7 @@ * Infra-red driver (SIR/FIR) for the PXA2xx embedded microprocessor * */ +#include #include #include #include diff --git a/drivers/net/ks8851_mll.c b/drivers/net/ks8851_mll.c index aefbdd896d6a..a82378231fc5 100644 --- a/drivers/net/ks8851_mll.c +++ b/drivers/net/ks8851_mll.c @@ -35,6 +35,7 @@ #include #include #include +#include #define DRV_NAME "ks8851_mll" diff --git a/drivers/net/sgiseeq.c b/drivers/net/sgiseeq.c index 54415c7b84a2..52fb7ed9f365 100644 --- a/drivers/net/sgiseeq.c +++ b/drivers/net/sgiseeq.c @@ -6,6 +6,7 @@ #undef DEBUG +#include #include #include #include diff --git a/drivers/net/stmmac/dwmac1000_core.c b/drivers/net/stmmac/dwmac1000_core.c index f20455cbfbbc..0f63b3c83c19 100644 --- a/drivers/net/stmmac/dwmac1000_core.c +++ b/drivers/net/stmmac/dwmac1000_core.c @@ -28,6 +28,7 @@ #include #include +#include #include "dwmac1000.h" static void dwmac1000_core_init(void __iomem *ioaddr) diff --git a/drivers/net/stmmac/dwmac1000_dma.c b/drivers/net/stmmac/dwmac1000_dma.c index 2c47712d45d0..3dbeea619085 100644 --- a/drivers/net/stmmac/dwmac1000_dma.c +++ b/drivers/net/stmmac/dwmac1000_dma.c @@ -26,6 +26,7 @@ Author: Giuseppe Cavallaro *******************************************************************************/ +#include #include "dwmac1000.h" #include "dwmac_dma.h" diff --git a/drivers/net/stmmac/dwmac100_core.c b/drivers/net/stmmac/dwmac100_core.c index c724fc36a24f..743a58017637 100644 --- a/drivers/net/stmmac/dwmac100_core.c +++ b/drivers/net/stmmac/dwmac100_core.c @@ -29,6 +29,7 @@ *******************************************************************************/ #include +#include #include "dwmac100.h" static void dwmac100_core_init(void __iomem *ioaddr) diff --git a/drivers/net/stmmac/dwmac100_dma.c b/drivers/net/stmmac/dwmac100_dma.c index e3e224b7d9e2..627f656b0f3c 100644 --- a/drivers/net/stmmac/dwmac100_dma.c +++ b/drivers/net/stmmac/dwmac100_dma.c @@ -28,6 +28,7 @@ Author: Giuseppe Cavallaro *******************************************************************************/ +#include #include "dwmac100.h" #include "dwmac_dma.h" diff --git a/drivers/net/stmmac/stmmac_ethtool.c b/drivers/net/stmmac/stmmac_ethtool.c index 720c5a1531bc..7ed8fb6c2117 100644 --- a/drivers/net/stmmac/stmmac_ethtool.c +++ b/drivers/net/stmmac/stmmac_ethtool.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "stmmac.h" #include "dwmac_dma.h" diff --git a/drivers/net/stmmac/stmmac_mdio.c b/drivers/net/stmmac/stmmac_mdio.c index 234b4068a1fc..29a6bb6b8058 100644 --- a/drivers/net/stmmac/stmmac_mdio.c +++ b/drivers/net/stmmac/stmmac_mdio.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "stmmac.h" diff --git a/drivers/net/usb/cdc-phonet.c b/drivers/net/usb/cdc-phonet.c index f967913e11bc..a60d0069cc45 100644 --- a/drivers/net/usb/cdc-phonet.c +++ b/drivers/net/usb/cdc-phonet.c @@ -21,6 +21,7 @@ */ #include +#include #include #include #include diff --git a/drivers/net/vxge/vxge-config.h b/drivers/net/vxge/vxge-config.h index 6219006d9d2e..dd362584f5ca 100644 --- a/drivers/net/vxge/vxge-config.h +++ b/drivers/net/vxge/vxge-config.h @@ -16,6 +16,7 @@ #include #include #include +#include #ifndef VXGE_CACHE_LINE_SIZE #define VXGE_CACHE_LINE_SIZE 128 diff --git a/drivers/net/wireless/ath/ath5k/base.c b/drivers/net/wireless/ath/ath5k/base.c index b6c5d3715b96..779a1d270243 100644 --- a/drivers/net/wireless/ath/ath5k/base.c +++ b/drivers/net/wireless/ath/ath5k/base.c @@ -42,6 +42,7 @@ #include #include +#include #include #include #include diff --git a/drivers/net/wireless/ath/ath9k/beacon.c b/drivers/net/wireless/ath/ath9k/beacon.c index 0174cdb65a83..4c790b5f7937 100644 --- a/drivers/net/wireless/ath/ath9k/beacon.c +++ b/drivers/net/wireless/ath/ath9k/beacon.c @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include "ath9k.h" #define FUDGE 2 diff --git a/drivers/net/wireless/ath/ath9k/init.c b/drivers/net/wireless/ath/ath9k/init.c index d4b166cfdf60..1202bb0a5534 100644 --- a/drivers/net/wireless/ath/ath9k/init.c +++ b/drivers/net/wireless/ath/ath9k/init.c @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include diff --git a/drivers/net/wireless/ath/ath9k/recv.c b/drivers/net/wireless/ath/ath9k/recv.c index 07e35e59c9e3..c4c5d9f455bb 100644 --- a/drivers/net/wireless/ath/ath9k/recv.c +++ b/drivers/net/wireless/ath/ath9k/recv.c @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include "ath9k.h" #include "ar9003_mac.h" diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c index ec012b4317af..7e79bbaf2ba7 100644 --- a/drivers/net/wireless/ath/ath9k/xmit.c +++ b/drivers/net/wireless/ath/ath9k/xmit.c @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include "ath9k.h" #include "ar9003_mac.h" diff --git a/drivers/staging/pohmelfs/crypto.c b/drivers/staging/pohmelfs/crypto.c index 5cca24fcf6ca..ad92771dce57 100644 --- a/drivers/staging/pohmelfs/crypto.c +++ b/drivers/staging/pohmelfs/crypto.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "netfs.h" diff --git a/drivers/tty/serial/ifx6x60.c b/drivers/tty/serial/ifx6x60.c index 5315525220fb..426434e5eb7c 100644 --- a/drivers/tty/serial/ifx6x60.c +++ b/drivers/tty/serial/ifx6x60.c @@ -36,6 +36,7 @@ * you need to use this driver for another platform. * *****************************************************************************/ +#include #include #include #include diff --git a/drivers/usb/gadget/f_phonet.c b/drivers/usb/gadget/f_phonet.c index 5e1495097ec3..f22fc685ddfd 100644 --- a/drivers/usb/gadget/f_phonet.c +++ b/drivers/usb/gadget/f_phonet.c @@ -20,6 +20,7 @@ * 02110-1301 USA */ +#include #include #include #include diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index c5813c87de06..d61c11170213 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/include/linux/dma-direction.h b/include/linux/dma-direction.h new file mode 100644 index 000000000000..95b6a82f5951 --- /dev/null +++ b/include/linux/dma-direction.h @@ -0,0 +1,13 @@ +#ifndef _LINUX_DMA_DIRECTION_H +#define _LINUX_DMA_DIRECTION_H +/* + * These definitions mirror those in pci.h, so they can be used + * interchangeably with their PCI_ counterparts. + */ +enum dma_data_direction { + DMA_BIDIRECTIONAL = 0, + DMA_TO_DEVICE = 1, + DMA_FROM_DEVICE = 2, + DMA_NONE = 3, +}; +#endif diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index ba8319ae5fcc..1a167c48d84d 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -4,17 +4,9 @@ #include #include #include +#include #include -/* These definitions mirror those in pci.h, so they can be used - * interchangeably with their PCI_ counterparts */ -enum dma_data_direction { - DMA_BIDIRECTIONAL = 0, - DMA_TO_DEVICE = 1, - DMA_FROM_DEVICE = 2, - DMA_NONE = 3, -}; - struct dma_map_ops { void* (*alloc_coherent)(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp); diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index eee7addec282..8fbf40e0713c 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -23,7 +23,9 @@ #include #include -#include +#include + +struct scatterlist; /** * typedef dma_cookie_t - an opaque DMA cookie diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 22a8ceca0ed0..011eb89e9582 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 69c35f6cd13f..eb3b9a86c6ed 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -13,6 +13,7 @@ * Catalin(ux aka Dino) BOIE */ +#include #include #include #include diff --git a/security/apparmor/lib.c b/security/apparmor/lib.c index 506d2baf6147..b82e383beb77 100644 --- a/security/apparmor/lib.c +++ b/security/apparmor/lib.c @@ -12,6 +12,7 @@ * License. */ +#include #include #include #include -- cgit v1.2.3 From 6d3321e8e2b3bf6a5892e2ef673c7bf536e3f904 Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Thu, 23 Jun 2011 11:19:26 -0700 Subject: x86, mtrr: lock stop machine during MTRR rendezvous sequence MTRR rendezvous sequence using stop_one_cpu_nowait() can potentially happen in parallel with another system wide rendezvous using stop_machine(). This can lead to deadlock (The order in which works are queued can be different on different cpu's. Some cpu's will be running the first rendezvous handler and others will be running the second rendezvous handler. Each set waiting for the other set to join for the system wide rendezvous, leading to a deadlock). MTRR rendezvous sequence is not implemented using stop_machine() as this gets called both from the process context aswell as the cpu online paths (where the cpu has not come online and the interrupts are disabled etc). stop_machine() works with only online cpus. For now, take the stop_machine mutex in the MTRR rendezvous sequence that gets called from an online cpu (here we are in the process context and can potentially sleep while taking the mutex). And the MTRR rendezvous that gets triggered during cpu online doesn't need to take this stop_machine lock (as the stop_machine() already ensures that there is no cpu hotplug going on in parallel by doing get_online_cpus()) TBD: Pursue a cleaner solution of extending the stop_machine() infrastructure to handle the case where the calling cpu is still not online and use this for MTRR rendezvous sequence. fixes: https://bugzilla.novell.com/show_bug.cgi?id=672008 Reported-by: Vadim Kotelnikov Signed-off-by: Suresh Siddha Link: http://lkml.kernel.org/r/20110623182056.807230326@sbsiddha-MOBL3.sc.intel.com Cc: stable@kernel.org # 2.6.35+, backport a week or two after this gets more testing in mainline Signed-off-by: H. Peter Anvin --- arch/x86/kernel/cpu/mtrr/main.c | 23 +++++++++++++++++++++++ include/linux/stop_machine.h | 2 ++ kernel/stop_machine.c | 2 +- 3 files changed, 26 insertions(+), 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c index 929739a653d1..3d17bc7f06e6 100644 --- a/arch/x86/kernel/cpu/mtrr/main.c +++ b/arch/x86/kernel/cpu/mtrr/main.c @@ -248,6 +248,25 @@ set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type typ unsigned long flags; int cpu; +#ifdef CONFIG_SMP + /* + * If this cpu is not yet active, we are in the cpu online path. There + * can be no stop_machine() in parallel, as stop machine ensures this + * by using get_online_cpus(). We can skip taking the stop_cpus_mutex, + * as we don't need it and also we can't afford to block while waiting + * for the mutex. + * + * If this cpu is active, we need to prevent stop_machine() happening + * in parallel by taking the stop cpus mutex. + * + * Also, this is called in the context of cpu online path or in the + * context where cpu hotplug is prevented. So checking the active status + * of the raw_smp_processor_id() is safe. + */ + if (cpu_active(raw_smp_processor_id())) + mutex_lock(&stop_cpus_mutex); +#endif + preempt_disable(); data.smp_reg = reg; @@ -330,6 +349,10 @@ set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type typ local_irq_restore(flags); preempt_enable(); +#ifdef CONFIG_SMP + if (cpu_active(raw_smp_processor_id())) + mutex_unlock(&stop_cpus_mutex); +#endif } /** diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h index 092dc9b1ce7d..14d3524d1274 100644 --- a/include/linux/stop_machine.h +++ b/include/linux/stop_machine.h @@ -27,6 +27,8 @@ struct cpu_stop_work { struct cpu_stop_done *done; }; +extern struct mutex stop_cpus_mutex; + int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg); void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, struct cpu_stop_work *work_buf); diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index e3516b29076c..0cae1cc323dc 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -132,8 +132,8 @@ void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), work_buf); } +DEFINE_MUTEX(stop_cpus_mutex); /* static data for stop_cpus */ -static DEFINE_MUTEX(stop_cpus_mutex); static DEFINE_PER_CPU(struct cpu_stop_work, stop_cpus_work); int __stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg) -- cgit v1.2.3 From 06c3df49521c1b112b777cc4946e5de057c814ba Mon Sep 17 00:00:00 2001 From: Jamie Iles Date: Mon, 6 Jun 2011 12:43:07 +0100 Subject: clocksource: apb: Share APB timer code with other platforms The APB timers are an IP block from Synopsys (DesignWare APB timers) and are also found in other systems including ARM SoC's. This patch adds functions for creating clock_event_devices and clocksources from APB timers but does not do the resource allocation. This is handled in a higher layer to allow the timers to be created from multiple methods such as platform_devices. Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" Cc: Jacob Pan Signed-off-by: Jamie Iles Signed-off-by: John Stultz --- arch/x86/Kconfig | 1 + arch/x86/include/asm/apb_timer.h | 22 +- arch/x86/kernel/apb_timer.c | 409 +++++++------------------------------ drivers/clocksource/Kconfig | 3 + drivers/clocksource/Makefile | 1 + drivers/clocksource/dw_apb_timer.c | 401 ++++++++++++++++++++++++++++++++++++ include/linux/dw_apb_timer.h | 56 +++++ 7 files changed, 533 insertions(+), 360 deletions(-) create mode 100644 drivers/clocksource/dw_apb_timer.c create mode 100644 include/linux/dw_apb_timer.h (limited to 'arch/x86/kernel') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index da349723d411..10ce62f17ea1 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -617,6 +617,7 @@ config HPET_EMULATE_RTC config APB_TIMER def_bool y if MRST prompt "Langwell APB Timer Support" if X86_MRST + select DW_APB_TIMER help APB timer is the replacement for 8254, HPET on X86 MID platforms. The APBT provides a stable time base on SMP diff --git a/arch/x86/include/asm/apb_timer.h b/arch/x86/include/asm/apb_timer.h index 2fefa501d3ba..230000f124a7 100644 --- a/arch/x86/include/asm/apb_timer.h +++ b/arch/x86/include/asm/apb_timer.h @@ -18,24 +18,6 @@ #ifdef CONFIG_APB_TIMER -/* Langwell DW APB timer registers */ -#define APBTMR_N_LOAD_COUNT 0x00 -#define APBTMR_N_CURRENT_VALUE 0x04 -#define APBTMR_N_CONTROL 0x08 -#define APBTMR_N_EOI 0x0c -#define APBTMR_N_INT_STATUS 0x10 - -#define APBTMRS_INT_STATUS 0xa0 -#define APBTMRS_EOI 0xa4 -#define APBTMRS_RAW_INT_STATUS 0xa8 -#define APBTMRS_COMP_VERSION 0xac -#define APBTMRS_REG_SIZE 0x14 - -/* register bits */ -#define APBTMR_CONTROL_ENABLE (1<<0) -#define APBTMR_CONTROL_MODE_PERIODIC (1<<1) /*1: periodic 0:free running */ -#define APBTMR_CONTROL_INT (1<<2) - /* default memory mapped register base */ #define LNW_SCU_ADDR 0xFF100000 #define LNW_EXT_TIMER_OFFSET 0x1B800 @@ -43,8 +25,8 @@ #define LNW_EXT_TIMER_PGOFFSET 0x800 /* APBT clock speed range from PCLK to fabric base, 25-100MHz */ -#define APBT_MAX_FREQ 50 -#define APBT_MIN_FREQ 1 +#define APBT_MAX_FREQ 50000000 +#define APBT_MIN_FREQ 1000000 #define APBT_MMAP_SIZE 1024 #define APBT_DEV_USED 1 diff --git a/arch/x86/kernel/apb_timer.c b/arch/x86/kernel/apb_timer.c index 289e92862fd9..033d3ecc5744 100644 --- a/arch/x86/kernel/apb_timer.c +++ b/arch/x86/kernel/apb_timer.c @@ -27,15 +27,12 @@ * timer, but by default APB timer has higher rating than local APIC timers. */ -#include -#include #include +#include #include #include -#include #include #include -#include #include #include #include @@ -45,75 +42,46 @@ #include #include -#define APBT_MASK CLOCKSOURCE_MASK(32) -#define APBT_SHIFT 22 #define APBT_CLOCKEVENT_RATING 110 #define APBT_CLOCKSOURCE_RATING 250 -#define APBT_MIN_DELTA_USEC 200 -#define EVT_TO_APBT_DEV(evt) container_of(evt, struct apbt_dev, evt) #define APBT_CLOCKEVENT0_NUM (0) -#define APBT_CLOCKEVENT1_NUM (1) #define APBT_CLOCKSOURCE_NUM (2) -static unsigned long apbt_address; +static phys_addr_t apbt_address; static int apb_timer_block_enabled; static void __iomem *apbt_virt_address; -static int phy_cs_timer_id; /* * Common DW APB timer info */ -static uint64_t apbt_freq; - -static void apbt_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt); -static int apbt_next_event(unsigned long delta, - struct clock_event_device *evt); -static cycle_t apbt_read_clocksource(struct clocksource *cs); -static void apbt_restart_clocksource(struct clocksource *cs); +static unsigned long apbt_freq; struct apbt_dev { - struct clock_event_device evt; - unsigned int num; - int cpu; - unsigned int irq; - unsigned int tick; - unsigned int count; - unsigned int flags; - char name[10]; + struct dw_apb_clock_event_device *timer; + unsigned int num; + int cpu; + unsigned int irq; + char name[10]; }; -static DEFINE_PER_CPU(struct apbt_dev, cpu_apbt_dev); +static struct dw_apb_clocksource *clocksource_apbt; -#ifdef CONFIG_SMP -static unsigned int apbt_num_timers_used; -static struct apbt_dev *apbt_devs; -#endif - -static inline unsigned long apbt_readl_reg(unsigned long a) +static inline void __iomem *adev_virt_addr(struct apbt_dev *adev) { - return readl(apbt_virt_address + a); + return apbt_virt_address + adev->num * APBTMRS_REG_SIZE; } -static inline void apbt_writel_reg(unsigned long d, unsigned long a) -{ - writel(d, apbt_virt_address + a); -} - -static inline unsigned long apbt_readl(int n, unsigned long a) -{ - return readl(apbt_virt_address + a + n * APBTMRS_REG_SIZE); -} +static DEFINE_PER_CPU(struct apbt_dev, cpu_apbt_dev); -static inline void apbt_writel(int n, unsigned long d, unsigned long a) -{ - writel(d, apbt_virt_address + a + n * APBTMRS_REG_SIZE); -} +#ifdef CONFIG_SMP +static unsigned int apbt_num_timers_used; +#endif static inline void apbt_set_mapping(void) { struct sfi_timer_table_entry *mtmr; + int phy_cs_timer_id = 0; if (apbt_virt_address) { pr_debug("APBT base already mapped\n"); @@ -125,21 +93,18 @@ static inline void apbt_set_mapping(void) APBT_CLOCKEVENT0_NUM); return; } - apbt_address = (unsigned long)mtmr->phys_addr; + apbt_address = (phys_addr_t)mtmr->phys_addr; if (!apbt_address) { printk(KERN_WARNING "No timer base from SFI, use default\n"); apbt_address = APBT_DEFAULT_BASE; } apbt_virt_address = ioremap_nocache(apbt_address, APBT_MMAP_SIZE); - if (apbt_virt_address) { - pr_debug("Mapped APBT physical addr %p at virtual addr %p\n",\ - (void *)apbt_address, (void *)apbt_virt_address); - } else { - pr_debug("Failed mapping APBT phy address at %p\n",\ - (void *)apbt_address); + if (!apbt_virt_address) { + pr_debug("Failed mapping APBT phy address at %lu\n",\ + (unsigned long)apbt_address); goto panic_noapbt; } - apbt_freq = mtmr->freq_hz / USEC_PER_SEC; + apbt_freq = mtmr->freq_hz; sfi_free_mtmr(mtmr); /* Now figure out the physical timer id for clocksource device */ @@ -148,9 +113,14 @@ static inline void apbt_set_mapping(void) goto panic_noapbt; /* Now figure out the physical timer id */ - phy_cs_timer_id = (unsigned int)(mtmr->phys_addr & 0xff) - / APBTMRS_REG_SIZE; - pr_debug("Use timer %d for clocksource\n", phy_cs_timer_id); + pr_debug("Use timer %d for clocksource\n", + (int)(mtmr->phys_addr & 0xff) / APBTMRS_REG_SIZE); + phy_cs_timer_id = (unsigned int)(mtmr->phys_addr & 0xff) / + APBTMRS_REG_SIZE; + + clocksource_apbt = dw_apb_clocksource_init(APBT_CLOCKSOURCE_RATING, + "apbt0", apbt_virt_address + phy_cs_timer_id * + APBTMRS_REG_SIZE, apbt_freq); return; panic_noapbt: @@ -172,82 +142,6 @@ static inline int is_apbt_capable(void) return apbt_virt_address ? 1 : 0; } -static struct clocksource clocksource_apbt = { - .name = "apbt", - .rating = APBT_CLOCKSOURCE_RATING, - .read = apbt_read_clocksource, - .mask = APBT_MASK, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .resume = apbt_restart_clocksource, -}; - -/* boot APB clock event device */ -static struct clock_event_device apbt_clockevent = { - .name = "apbt0", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = apbt_set_mode, - .set_next_event = apbt_next_event, - .shift = APBT_SHIFT, - .irq = 0, - .rating = APBT_CLOCKEVENT_RATING, -}; - -/* - * start count down from 0xffff_ffff. this is done by toggling the enable bit - * then load initial load count to ~0. - */ -static void apbt_start_counter(int n) -{ - unsigned long ctrl = apbt_readl(n, APBTMR_N_CONTROL); - - ctrl &= ~APBTMR_CONTROL_ENABLE; - apbt_writel(n, ctrl, APBTMR_N_CONTROL); - apbt_writel(n, ~0, APBTMR_N_LOAD_COUNT); - /* enable, mask interrupt */ - ctrl &= ~APBTMR_CONTROL_MODE_PERIODIC; - ctrl |= (APBTMR_CONTROL_ENABLE | APBTMR_CONTROL_INT); - apbt_writel(n, ctrl, APBTMR_N_CONTROL); - /* read it once to get cached counter value initialized */ - apbt_read_clocksource(&clocksource_apbt); -} - -static irqreturn_t apbt_interrupt_handler(int irq, void *data) -{ - struct apbt_dev *dev = (struct apbt_dev *)data; - struct clock_event_device *aevt = &dev->evt; - - if (!aevt->event_handler) { - printk(KERN_INFO "Spurious APBT timer interrupt on %d\n", - dev->num); - return IRQ_NONE; - } - aevt->event_handler(aevt); - return IRQ_HANDLED; -} - -static void apbt_restart_clocksource(struct clocksource *cs) -{ - apbt_start_counter(phy_cs_timer_id); -} - -static void apbt_enable_int(int n) -{ - unsigned long ctrl = apbt_readl(n, APBTMR_N_CONTROL); - /* clear pending intr */ - apbt_readl(n, APBTMR_N_EOI); - ctrl &= ~APBTMR_CONTROL_INT; - apbt_writel(n, ctrl, APBTMR_N_CONTROL); -} - -static void apbt_disable_int(int n) -{ - unsigned long ctrl = apbt_readl(n, APBTMR_N_CONTROL); - - ctrl |= APBTMR_CONTROL_INT; - apbt_writel(n, ctrl, APBTMR_N_CONTROL); -} - - static int __init apbt_clockevent_register(void) { struct sfi_timer_table_entry *mtmr; @@ -260,45 +154,21 @@ static int __init apbt_clockevent_register(void) return -ENODEV; } - /* - * We need to calculate the scaled math multiplication factor for - * nanosecond to apbt tick conversion. - * mult = (nsec/cycle)*2^APBT_SHIFT - */ - apbt_clockevent.mult = div_sc((unsigned long) mtmr->freq_hz - , NSEC_PER_SEC, APBT_SHIFT); - - /* Calculate the min / max delta */ - apbt_clockevent.max_delta_ns = clockevent_delta2ns(0x7FFFFFFF, - &apbt_clockevent); - apbt_clockevent.min_delta_ns = clockevent_delta2ns( - APBT_MIN_DELTA_USEC*apbt_freq, - &apbt_clockevent); - /* - * Start apbt with the boot cpu mask and make it - * global if not used for per cpu timer. - */ - apbt_clockevent.cpumask = cpumask_of(smp_processor_id()); adev->num = smp_processor_id(); - memcpy(&adev->evt, &apbt_clockevent, sizeof(struct clock_event_device)); + adev->timer = dw_apb_clockevent_init(smp_processor_id(), "apbt0", + mrst_timer_options == MRST_TIMER_LAPIC_APBT ? + APBT_CLOCKEVENT_RATING - 100 : APBT_CLOCKEVENT_RATING, + adev_virt_addr(adev), 0, apbt_freq); + /* Firmware does EOI handling for us. */ + adev->timer->eoi = NULL; if (mrst_timer_options == MRST_TIMER_LAPIC_APBT) { - adev->evt.rating = APBT_CLOCKEVENT_RATING - 100; - global_clock_event = &adev->evt; + global_clock_event = &adev->timer->ced; printk(KERN_DEBUG "%s clockevent registered as global\n", global_clock_event->name); } - if (request_irq(apbt_clockevent.irq, apbt_interrupt_handler, - IRQF_TIMER | IRQF_DISABLED | IRQF_NOBALANCING, - apbt_clockevent.name, adev)) { - printk(KERN_ERR "Failed request IRQ for APBT%d\n", - apbt_clockevent.irq); - } - - clockevents_register_device(&adev->evt); - /* Start APBT 0 interrupts */ - apbt_enable_int(APBT_CLOCKEVENT0_NUM); + dw_apb_clockevent_register(adev->timer); sfi_free_mtmr(mtmr); return 0; @@ -316,52 +186,34 @@ static void apbt_setup_irq(struct apbt_dev *adev) irq_set_affinity(adev->irq, cpumask_of(adev->cpu)); /* APB timer irqs are set up as mp_irqs, timer is edge type */ __irq_set_handler(adev->irq, handle_edge_irq, 0, "edge"); - - if (system_state == SYSTEM_BOOTING) { - if (request_irq(adev->irq, apbt_interrupt_handler, - IRQF_TIMER | IRQF_DISABLED | - IRQF_NOBALANCING, - adev->name, adev)) { - printk(KERN_ERR "Failed request IRQ for APBT%d\n", - adev->num); - } - } else - enable_irq(adev->irq); } /* Should be called with per cpu */ void apbt_setup_secondary_clock(void) { struct apbt_dev *adev; - struct clock_event_device *aevt; int cpu; /* Don't register boot CPU clockevent */ cpu = smp_processor_id(); if (!cpu) return; - /* - * We need to calculate the scaled math multiplication factor for - * nanosecond to apbt tick conversion. - * mult = (nsec/cycle)*2^APBT_SHIFT - */ - printk(KERN_INFO "Init per CPU clockevent %d\n", cpu); - adev = &per_cpu(cpu_apbt_dev, cpu); - aevt = &adev->evt; - memcpy(aevt, &apbt_clockevent, sizeof(*aevt)); - aevt->cpumask = cpumask_of(cpu); - aevt->name = adev->name; - aevt->mode = CLOCK_EVT_MODE_UNUSED; + adev = &__get_cpu_var(cpu_apbt_dev); + if (!adev->timer) { + adev->timer = dw_apb_clockevent_init(cpu, adev->name, + APBT_CLOCKEVENT_RATING, adev_virt_addr(adev), + adev->irq, apbt_freq); + adev->timer->eoi = NULL; + } else { + dw_apb_clockevent_resume(adev->timer); + } - printk(KERN_INFO "Registering CPU %d clockevent device %s, mask %08x\n", - cpu, aevt->name, *(u32 *)aevt->cpumask); + printk(KERN_INFO "Registering CPU %d clockevent device %s, cpu %08x\n", + cpu, adev->name, adev->cpu); apbt_setup_irq(adev); - - clockevents_register_device(aevt); - - apbt_enable_int(cpu); + dw_apb_clockevent_register(adev->timer); return; } @@ -384,13 +236,12 @@ static int apbt_cpuhp_notify(struct notifier_block *n, switch (action & 0xf) { case CPU_DEAD: - disable_irq(adev->irq); - apbt_disable_int(cpu); + dw_apb_clockevent_pause(adev->timer); if (system_state == SYSTEM_RUNNING) { pr_debug("skipping APBT CPU %lu offline\n", cpu); } else if (adev) { pr_debug("APBT clockevent for cpu %lu offline\n", cpu); - free_irq(adev->irq, adev); + dw_apb_clockevent_stop(adev->timer); } break; default: @@ -415,116 +266,16 @@ void apbt_setup_secondary_clock(void) {} #endif /* CONFIG_SMP */ -static void apbt_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - unsigned long ctrl; - uint64_t delta; - int timer_num; - struct apbt_dev *adev = EVT_TO_APBT_DEV(evt); - - BUG_ON(!apbt_virt_address); - - timer_num = adev->num; - pr_debug("%s CPU %d timer %d mode=%d\n", - __func__, first_cpu(*evt->cpumask), timer_num, mode); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - delta = ((uint64_t)(NSEC_PER_SEC/HZ)) * apbt_clockevent.mult; - delta >>= apbt_clockevent.shift; - ctrl = apbt_readl(timer_num, APBTMR_N_CONTROL); - ctrl |= APBTMR_CONTROL_MODE_PERIODIC; - apbt_writel(timer_num, ctrl, APBTMR_N_CONTROL); - /* - * DW APB p. 46, have to disable timer before load counter, - * may cause sync problem. - */ - ctrl &= ~APBTMR_CONTROL_ENABLE; - apbt_writel(timer_num, ctrl, APBTMR_N_CONTROL); - udelay(1); - pr_debug("Setting clock period %d for HZ %d\n", (int)delta, HZ); - apbt_writel(timer_num, delta, APBTMR_N_LOAD_COUNT); - ctrl |= APBTMR_CONTROL_ENABLE; - apbt_writel(timer_num, ctrl, APBTMR_N_CONTROL); - break; - /* APB timer does not have one-shot mode, use free running mode */ - case CLOCK_EVT_MODE_ONESHOT: - ctrl = apbt_readl(timer_num, APBTMR_N_CONTROL); - /* - * set free running mode, this mode will let timer reload max - * timeout which will give time (3min on 25MHz clock) to rearm - * the next event, therefore emulate the one-shot mode. - */ - ctrl &= ~APBTMR_CONTROL_ENABLE; - ctrl &= ~APBTMR_CONTROL_MODE_PERIODIC; - - apbt_writel(timer_num, ctrl, APBTMR_N_CONTROL); - /* write again to set free running mode */ - apbt_writel(timer_num, ctrl, APBTMR_N_CONTROL); - - /* - * DW APB p. 46, load counter with all 1s before starting free - * running mode. - */ - apbt_writel(timer_num, ~0, APBTMR_N_LOAD_COUNT); - ctrl &= ~APBTMR_CONTROL_INT; - ctrl |= APBTMR_CONTROL_ENABLE; - apbt_writel(timer_num, ctrl, APBTMR_N_CONTROL); - break; - - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - apbt_disable_int(timer_num); - ctrl = apbt_readl(timer_num, APBTMR_N_CONTROL); - ctrl &= ~APBTMR_CONTROL_ENABLE; - apbt_writel(timer_num, ctrl, APBTMR_N_CONTROL); - break; - - case CLOCK_EVT_MODE_RESUME: - apbt_enable_int(timer_num); - break; - } -} - -static int apbt_next_event(unsigned long delta, - struct clock_event_device *evt) -{ - unsigned long ctrl; - int timer_num; - - struct apbt_dev *adev = EVT_TO_APBT_DEV(evt); - - timer_num = adev->num; - /* Disable timer */ - ctrl = apbt_readl(timer_num, APBTMR_N_CONTROL); - ctrl &= ~APBTMR_CONTROL_ENABLE; - apbt_writel(timer_num, ctrl, APBTMR_N_CONTROL); - /* write new count */ - apbt_writel(timer_num, delta, APBTMR_N_LOAD_COUNT); - ctrl |= APBTMR_CONTROL_ENABLE; - apbt_writel(timer_num, ctrl, APBTMR_N_CONTROL); - return 0; -} - -static cycle_t apbt_read_clocksource(struct clocksource *cs) -{ - unsigned long current_count; - - current_count = apbt_readl(phy_cs_timer_id, APBTMR_N_CURRENT_VALUE); - return (cycle_t)~current_count; -} - static int apbt_clocksource_register(void) { u64 start, now; cycle_t t1; /* Start the counter, use timer 2 as source, timer 0/1 for event */ - apbt_start_counter(phy_cs_timer_id); + dw_apb_clocksource_start(clocksource_apbt); /* Verify whether apbt counter works */ - t1 = apbt_read_clocksource(&clocksource_apbt); + t1 = dw_apb_clocksource_read(clocksource_apbt); rdtscll(start); /* @@ -539,10 +290,10 @@ static int apbt_clocksource_register(void) } while ((now - start) < 200000UL); /* APBT is the only always on clocksource, it has to work! */ - if (t1 == apbt_read_clocksource(&clocksource_apbt)) + if (t1 == dw_apb_clocksource_read(clocksource_apbt)) panic("APBT counter not counting. APBT disabled\n"); - clocksource_register_khz(&clocksource_apbt, (u32)apbt_freq*1000); + dw_apb_clocksource_register(clocksource_apbt); return 0; } @@ -566,10 +317,7 @@ void __init apbt_time_init(void) if (apb_timer_block_enabled) return; apbt_set_mapping(); - if (apbt_virt_address) { - pr_debug("Found APBT version 0x%lx\n",\ - apbt_readl_reg(APBTMRS_COMP_VERSION)); - } else + if (!apbt_virt_address) goto out_noapbt; /* * Read the frequency and check for a sane value, for ESL model @@ -577,7 +325,7 @@ void __init apbt_time_init(void) */ if (apbt_freq < APBT_MIN_FREQ || apbt_freq > APBT_MAX_FREQ) { - pr_debug("APBT has invalid freq 0x%llx\n", apbt_freq); + pr_debug("APBT has invalid freq 0x%lx\n", apbt_freq); goto out_noapbt; } if (apbt_clocksource_register()) { @@ -603,30 +351,20 @@ void __init apbt_time_init(void) } else { percpu_timer = 0; apbt_num_timers_used = 1; - adev = &per_cpu(cpu_apbt_dev, 0); - adev->flags &= ~APBT_DEV_USED; } pr_debug("%s: %d APB timers used\n", __func__, apbt_num_timers_used); /* here we set up per CPU timer data structure */ - apbt_devs = kzalloc(sizeof(struct apbt_dev) * apbt_num_timers_used, - GFP_KERNEL); - if (!apbt_devs) { - printk(KERN_ERR "Failed to allocate APB timer devices\n"); - return; - } for (i = 0; i < apbt_num_timers_used; i++) { adev = &per_cpu(cpu_apbt_dev, i); adev->num = i; adev->cpu = i; p_mtmr = sfi_get_mtmr(i); - if (p_mtmr) { - adev->tick = p_mtmr->freq_hz; + if (p_mtmr) adev->irq = p_mtmr->irq; - } else + else printk(KERN_ERR "Failed to get timer for cpu %d\n", i); - adev->count = 0; - sprintf(adev->name, "apbt%d", i); + snprintf(adev->name, sizeof(adev->name) - 1, "apbt%d", i); } #endif @@ -638,17 +376,8 @@ out_noapbt: panic("failed to enable APB timer\n"); } -static inline void apbt_disable(int n) -{ - if (is_apbt_capable()) { - unsigned long ctrl = apbt_readl(n, APBTMR_N_CONTROL); - ctrl &= ~APBTMR_CONTROL_ENABLE; - apbt_writel(n, ctrl, APBTMR_N_CONTROL); - } -} - /* called before apb_timer_enable, use early map */ -unsigned long apbt_quick_calibrate() +unsigned long apbt_quick_calibrate(void) { int i, scale; u64 old, new; @@ -657,31 +386,31 @@ unsigned long apbt_quick_calibrate() u32 loop, shift; apbt_set_mapping(); - apbt_start_counter(phy_cs_timer_id); + dw_apb_clocksource_start(clocksource_apbt); /* check if the timer can count down, otherwise return */ - old = apbt_read_clocksource(&clocksource_apbt); + old = dw_apb_clocksource_read(clocksource_apbt); i = 10000; while (--i) { - if (old != apbt_read_clocksource(&clocksource_apbt)) + if (old != dw_apb_clocksource_read(clocksource_apbt)) break; } if (!i) goto failed; /* count 16 ms */ - loop = (apbt_freq * 1000) << 4; + loop = (apbt_freq / 1000) << 4; /* restart the timer to ensure it won't get to 0 in the calibration */ - apbt_start_counter(phy_cs_timer_id); + dw_apb_clocksource_start(clocksource_apbt); - old = apbt_read_clocksource(&clocksource_apbt); + old = dw_apb_clocksource_read(clocksource_apbt); old += loop; t1 = __native_read_tsc(); do { - new = apbt_read_clocksource(&clocksource_apbt); + new = dw_apb_clocksource_read(clocksource_apbt); } while (new < old); t2 = __native_read_tsc(); @@ -693,7 +422,7 @@ unsigned long apbt_quick_calibrate() return 0; } scale = (int)div_u64((t2 - t1), loop >> shift); - khz = (scale * apbt_freq * 1000) >> shift; + khz = (scale * (apbt_freq / 1000)) >> shift; printk(KERN_INFO "TSC freq calculated by APB timer is %lu khz\n", khz); return khz; failed: diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 96c921910469..141417fcce30 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -3,3 +3,6 @@ config CLKSRC_I8253 config CLKSRC_MMIO bool + +config DW_APB_TIMER + bool diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index b995942a5060..4c3d8fda136a 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -8,3 +8,4 @@ obj-$(CONFIG_SH_TIMER_MTU2) += sh_mtu2.o obj-$(CONFIG_SH_TIMER_TMU) += sh_tmu.o obj-$(CONFIG_CLKSRC_I8253) += i8253.o obj-$(CONFIG_CLKSRC_MMIO) += mmio.o +obj-$(CONFIG_DW_APB_TIMER) += dw_apb_timer.o diff --git a/drivers/clocksource/dw_apb_timer.c b/drivers/clocksource/dw_apb_timer.c new file mode 100644 index 000000000000..580f870541a3 --- /dev/null +++ b/drivers/clocksource/dw_apb_timer.c @@ -0,0 +1,401 @@ +/* + * (C) Copyright 2009 Intel Corporation + * Author: Jacob Pan (jacob.jun.pan@intel.com) + * + * Shared with ARM platforms, Jamie Iles, Picochip 2011 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Support for the Synopsys DesignWare APB Timers. + */ +#include +#include +#include +#include +#include +#include +#include + +#define APBT_MIN_PERIOD 4 +#define APBT_MIN_DELTA_USEC 200 + +#define APBTMR_N_LOAD_COUNT 0x00 +#define APBTMR_N_CURRENT_VALUE 0x04 +#define APBTMR_N_CONTROL 0x08 +#define APBTMR_N_EOI 0x0c +#define APBTMR_N_INT_STATUS 0x10 + +#define APBTMRS_INT_STATUS 0xa0 +#define APBTMRS_EOI 0xa4 +#define APBTMRS_RAW_INT_STATUS 0xa8 +#define APBTMRS_COMP_VERSION 0xac + +#define APBTMR_CONTROL_ENABLE (1 << 0) +/* 1: periodic, 0:free running. */ +#define APBTMR_CONTROL_MODE_PERIODIC (1 << 1) +#define APBTMR_CONTROL_INT (1 << 2) + +static inline struct dw_apb_clock_event_device * +ced_to_dw_apb_ced(struct clock_event_device *evt) +{ + return container_of(evt, struct dw_apb_clock_event_device, ced); +} + +static inline struct dw_apb_clocksource * +clocksource_to_dw_apb_clocksource(struct clocksource *cs) +{ + return container_of(cs, struct dw_apb_clocksource, cs); +} + +static unsigned long apbt_readl(struct dw_apb_timer *timer, unsigned long offs) +{ + return readl(timer->base + offs); +} + +static void apbt_writel(struct dw_apb_timer *timer, unsigned long val, + unsigned long offs) +{ + writel(val, timer->base + offs); +} + +static void apbt_disable_int(struct dw_apb_timer *timer) +{ + unsigned long ctrl = apbt_readl(timer, APBTMR_N_CONTROL); + + ctrl |= APBTMR_CONTROL_INT; + apbt_writel(timer, ctrl, APBTMR_N_CONTROL); +} + +/** + * dw_apb_clockevent_pause() - stop the clock_event_device from running + * + * @dw_ced: The APB clock to stop generating events. + */ +void dw_apb_clockevent_pause(struct dw_apb_clock_event_device *dw_ced) +{ + disable_irq(dw_ced->timer.irq); + apbt_disable_int(&dw_ced->timer); +} + +static void apbt_eoi(struct dw_apb_timer *timer) +{ + apbt_readl(timer, APBTMR_N_EOI); +} + +static irqreturn_t dw_apb_clockevent_irq(int irq, void *data) +{ + struct clock_event_device *evt = data; + struct dw_apb_clock_event_device *dw_ced = ced_to_dw_apb_ced(evt); + + if (!evt->event_handler) { + pr_info("Spurious APBT timer interrupt %d", irq); + return IRQ_NONE; + } + + if (dw_ced->eoi) + dw_ced->eoi(&dw_ced->timer); + + evt->event_handler(evt); + return IRQ_HANDLED; +} + +static void apbt_enable_int(struct dw_apb_timer *timer) +{ + unsigned long ctrl = apbt_readl(timer, APBTMR_N_CONTROL); + /* clear pending intr */ + apbt_readl(timer, APBTMR_N_EOI); + ctrl &= ~APBTMR_CONTROL_INT; + apbt_writel(timer, ctrl, APBTMR_N_CONTROL); +} + +static void apbt_set_mode(enum clock_event_mode mode, + struct clock_event_device *evt) +{ + unsigned long ctrl; + unsigned long period; + struct dw_apb_clock_event_device *dw_ced = ced_to_dw_apb_ced(evt); + + pr_debug("%s CPU %d mode=%d\n", __func__, first_cpu(*evt->cpumask), + mode); + + switch (mode) { + case CLOCK_EVT_MODE_PERIODIC: + period = DIV_ROUND_UP(dw_ced->timer.freq, HZ); + ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); + ctrl |= APBTMR_CONTROL_MODE_PERIODIC; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + /* + * DW APB p. 46, have to disable timer before load counter, + * may cause sync problem. + */ + ctrl &= ~APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + udelay(1); + pr_debug("Setting clock period %lu for HZ %d\n", period, HZ); + apbt_writel(&dw_ced->timer, period, APBTMR_N_LOAD_COUNT); + ctrl |= APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + break; + + case CLOCK_EVT_MODE_ONESHOT: + ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); + /* + * set free running mode, this mode will let timer reload max + * timeout which will give time (3min on 25MHz clock) to rearm + * the next event, therefore emulate the one-shot mode. + */ + ctrl &= ~APBTMR_CONTROL_ENABLE; + ctrl &= ~APBTMR_CONTROL_MODE_PERIODIC; + + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + /* write again to set free running mode */ + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + + /* + * DW APB p. 46, load counter with all 1s before starting free + * running mode. + */ + apbt_writel(&dw_ced->timer, ~0, APBTMR_N_LOAD_COUNT); + ctrl &= ~APBTMR_CONTROL_INT; + ctrl |= APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + break; + + case CLOCK_EVT_MODE_UNUSED: + case CLOCK_EVT_MODE_SHUTDOWN: + ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); + ctrl &= ~APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + break; + + case CLOCK_EVT_MODE_RESUME: + apbt_enable_int(&dw_ced->timer); + break; + } +} + +static int apbt_next_event(unsigned long delta, + struct clock_event_device *evt) +{ + unsigned long ctrl; + struct dw_apb_clock_event_device *dw_ced = ced_to_dw_apb_ced(evt); + + /* Disable timer */ + ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); + ctrl &= ~APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + /* write new count */ + apbt_writel(&dw_ced->timer, delta, APBTMR_N_LOAD_COUNT); + ctrl |= APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + + return 0; +} + +/** + * dw_apb_clockevent_init() - use an APB timer as a clock_event_device + * + * @cpu: The CPU the events will be targeted at. + * @name: The name used for the timer and the IRQ for it. + * @rating: The rating to give the timer. + * @base: I/O base for the timer registers. + * @irq: The interrupt number to use for the timer. + * @freq: The frequency that the timer counts at. + * + * This creates a clock_event_device for using with the generic clock layer + * but does not start and register it. This should be done with + * dw_apb_clockevent_register() as the next step. If this is the first time + * it has been called for a timer then the IRQ will be requested, if not it + * just be enabled to allow CPU hotplug to avoid repeatedly requesting and + * releasing the IRQ. + */ +struct dw_apb_clock_event_device * +dw_apb_clockevent_init(int cpu, const char *name, unsigned rating, + void __iomem *base, int irq, unsigned long freq) +{ + struct dw_apb_clock_event_device *dw_ced = + kzalloc(sizeof(*dw_ced), GFP_KERNEL); + int err; + + if (!dw_ced) + return NULL; + + dw_ced->timer.base = base; + dw_ced->timer.irq = irq; + dw_ced->timer.freq = freq; + + clockevents_calc_mult_shift(&dw_ced->ced, freq, APBT_MIN_PERIOD); + dw_ced->ced.max_delta_ns = clockevent_delta2ns(0x7fffffff, + &dw_ced->ced); + dw_ced->ced.min_delta_ns = clockevent_delta2ns(5000, &dw_ced->ced); + dw_ced->ced.cpumask = cpumask_of(cpu); + dw_ced->ced.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; + dw_ced->ced.set_mode = apbt_set_mode; + dw_ced->ced.set_next_event = apbt_next_event; + dw_ced->ced.irq = dw_ced->timer.irq; + dw_ced->ced.rating = rating; + dw_ced->ced.name = name; + + dw_ced->irqaction.name = dw_ced->ced.name; + dw_ced->irqaction.handler = dw_apb_clockevent_irq; + dw_ced->irqaction.dev_id = &dw_ced->ced; + dw_ced->irqaction.irq = irq; + dw_ced->irqaction.flags = IRQF_TIMER | IRQF_IRQPOLL | + IRQF_NOBALANCING | + IRQF_DISABLED; + + dw_ced->eoi = apbt_eoi; + err = setup_irq(irq, &dw_ced->irqaction); + if (err) { + pr_err("failed to request timer irq\n"); + kfree(dw_ced); + dw_ced = NULL; + } + + return dw_ced; +} + +/** + * dw_apb_clockevent_resume() - resume a clock that has been paused. + * + * @dw_ced: The APB clock to resume. + */ +void dw_apb_clockevent_resume(struct dw_apb_clock_event_device *dw_ced) +{ + enable_irq(dw_ced->timer.irq); +} + +/** + * dw_apb_clockevent_stop() - stop the clock_event_device and release the IRQ. + * + * @dw_ced: The APB clock to stop generating the events. + */ +void dw_apb_clockevent_stop(struct dw_apb_clock_event_device *dw_ced) +{ + free_irq(dw_ced->timer.irq, &dw_ced->ced); +} + +/** + * dw_apb_clockevent_register() - register the clock with the generic layer + * + * @dw_ced: The APB clock to register as a clock_event_device. + */ +void dw_apb_clockevent_register(struct dw_apb_clock_event_device *dw_ced) +{ + apbt_writel(&dw_ced->timer, 0, APBTMR_N_CONTROL); + clockevents_register_device(&dw_ced->ced); + apbt_enable_int(&dw_ced->timer); +} + +/** + * dw_apb_clocksource_start() - start the clocksource counting. + * + * @dw_cs: The clocksource to start. + * + * This is used to start the clocksource before registration and can be used + * to enable calibration of timers. + */ +void dw_apb_clocksource_start(struct dw_apb_clocksource *dw_cs) +{ + /* + * start count down from 0xffff_ffff. this is done by toggling the + * enable bit then load initial load count to ~0. + */ + unsigned long ctrl = apbt_readl(&dw_cs->timer, APBTMR_N_CONTROL); + + ctrl &= ~APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_cs->timer, ctrl, APBTMR_N_CONTROL); + apbt_writel(&dw_cs->timer, ~0, APBTMR_N_LOAD_COUNT); + /* enable, mask interrupt */ + ctrl &= ~APBTMR_CONTROL_MODE_PERIODIC; + ctrl |= (APBTMR_CONTROL_ENABLE | APBTMR_CONTROL_INT); + apbt_writel(&dw_cs->timer, ctrl, APBTMR_N_CONTROL); + /* read it once to get cached counter value initialized */ + dw_apb_clocksource_read(dw_cs); +} + +static cycle_t __apbt_read_clocksource(struct clocksource *cs) +{ + unsigned long current_count; + struct dw_apb_clocksource *dw_cs = + clocksource_to_dw_apb_clocksource(cs); + + current_count = apbt_readl(&dw_cs->timer, APBTMR_N_CURRENT_VALUE); + + return (cycle_t)~current_count; +} + +static void apbt_restart_clocksource(struct clocksource *cs) +{ + struct dw_apb_clocksource *dw_cs = + clocksource_to_dw_apb_clocksource(cs); + + dw_apb_clocksource_start(dw_cs); +} + +/** + * dw_apb_clocksource_init() - use an APB timer as a clocksource. + * + * @rating: The rating to give the clocksource. + * @name: The name for the clocksource. + * @base: The I/O base for the timer registers. + * @freq: The frequency that the timer counts at. + * + * This creates a clocksource using an APB timer but does not yet register it + * with the clocksource system. This should be done with + * dw_apb_clocksource_register() as the next step. + */ +struct dw_apb_clocksource * +dw_apb_clocksource_init(unsigned rating, char *name, void __iomem *base, + unsigned long freq) +{ + struct dw_apb_clocksource *dw_cs = kzalloc(sizeof(*dw_cs), GFP_KERNEL); + + if (!dw_cs) + return NULL; + + dw_cs->timer.base = base; + dw_cs->timer.freq = freq; + dw_cs->cs.name = name; + dw_cs->cs.rating = rating; + dw_cs->cs.read = __apbt_read_clocksource; + dw_cs->cs.mask = CLOCKSOURCE_MASK(32); + dw_cs->cs.flags = CLOCK_SOURCE_IS_CONTINUOUS; + dw_cs->cs.resume = apbt_restart_clocksource; + + return dw_cs; +} + +/** + * dw_apb_clocksource_register() - register the APB clocksource. + * + * @dw_cs: The clocksource to register. + */ +void dw_apb_clocksource_register(struct dw_apb_clocksource *dw_cs) +{ + clocksource_register_hz(&dw_cs->cs, dw_cs->timer.freq); +} + +/** + * dw_apb_clocksource_read() - read the current value of a clocksource. + * + * @dw_cs: The clocksource to read. + */ +cycle_t dw_apb_clocksource_read(struct dw_apb_clocksource *dw_cs) +{ + return (cycle_t)~apbt_readl(&dw_cs->timer, APBTMR_N_CURRENT_VALUE); +} + +/** + * dw_apb_clocksource_unregister() - unregister and free a clocksource. + * + * @dw_cs: The clocksource to unregister/free. + */ +void dw_apb_clocksource_unregister(struct dw_apb_clocksource *dw_cs) +{ + clocksource_unregister(&dw_cs->cs); + + kfree(dw_cs); +} diff --git a/include/linux/dw_apb_timer.h b/include/linux/dw_apb_timer.h new file mode 100644 index 000000000000..49638ea3b776 --- /dev/null +++ b/include/linux/dw_apb_timer.h @@ -0,0 +1,56 @@ +/* + * (C) Copyright 2009 Intel Corporation + * Author: Jacob Pan (jacob.jun.pan@intel.com) + * + * Shared with ARM platforms, Jamie Iles, Picochip 2011 + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Support for the Synopsys DesignWare APB Timers. + */ +#ifndef __DW_APB_TIMER_H__ +#define __DW_APB_TIMER_H__ + +#include +#include +#include + +#define APBTMRS_REG_SIZE 0x14 + +struct dw_apb_timer { + void __iomem *base; + unsigned long freq; + int irq; +}; + +struct dw_apb_clock_event_device { + struct clock_event_device ced; + struct dw_apb_timer timer; + struct irqaction irqaction; + void (*eoi)(struct dw_apb_timer *); +}; + +struct dw_apb_clocksource { + struct dw_apb_timer timer; + struct clocksource cs; +}; + +void dw_apb_clockevent_register(struct dw_apb_clock_event_device *dw_ced); +void dw_apb_clockevent_pause(struct dw_apb_clock_event_device *dw_ced); +void dw_apb_clockevent_resume(struct dw_apb_clock_event_device *dw_ced); +void dw_apb_clockevent_stop(struct dw_apb_clock_event_device *dw_ced); + +struct dw_apb_clock_event_device * +dw_apb_clockevent_init(int cpu, const char *name, unsigned rating, + void __iomem *base, int irq, unsigned long freq); +struct dw_apb_clocksource * +dw_apb_clocksource_init(unsigned rating, char *name, void __iomem *base, + unsigned long freq); +void dw_apb_clocksource_register(struct dw_apb_clocksource *dw_cs); +void dw_apb_clocksource_start(struct dw_apb_clocksource *dw_cs); +cycle_t dw_apb_clocksource_read(struct dw_apb_clocksource *dw_cs); +void dw_apb_clocksource_unregister(struct dw_apb_clocksource *dw_cs); + +#endif /* __DW_APB_TIMER_H__ */ -- cgit v1.2.3 From 192d8857427dd23707d5f0b86ca990c3af6f2d74 Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Thu, 23 Jun 2011 11:19:29 -0700 Subject: x86, mtrr: use stop_machine APIs for doing MTRR rendezvous MTRR rendezvous sequence is not implemened using stop_machine() before, as this gets called both from the process context aswell as the cpu online paths (where the cpu has not come online and the interrupts are disabled etc). Now that we have a new stop_machine_from_inactive_cpu() API, use it for rendezvous during mtrr init of a logical processor that is coming online. For the rest (runtime MTRR modification, system boot, resume paths), use stop_machine() to implement the rendezvous sequence. This will consolidate and cleanup the code. Signed-off-by: Suresh Siddha Link: http://lkml.kernel.org/r/20110623182057.076997177@sbsiddha-MOBL3.sc.intel.com Signed-off-by: H. Peter Anvin --- arch/x86/kernel/cpu/mtrr/main.c | 192 +++++++++------------------------------- include/linux/stop_machine.h | 2 - kernel/stop_machine.c | 2 +- 3 files changed, 42 insertions(+), 154 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c index 3d17bc7f06e6..707b6377adf1 100644 --- a/arch/x86/kernel/cpu/mtrr/main.c +++ b/arch/x86/kernel/cpu/mtrr/main.c @@ -137,55 +137,43 @@ static void __init init_table(void) } struct set_mtrr_data { - atomic_t count; - atomic_t gate; unsigned long smp_base; unsigned long smp_size; unsigned int smp_reg; mtrr_type smp_type; }; -static DEFINE_PER_CPU(struct cpu_stop_work, mtrr_work); - /** - * mtrr_work_handler - Synchronisation handler. Executed by "other" CPUs. + * mtrr_rendezvous_handler - Work done in the synchronization handler. Executed + * by all the CPUs. * @info: pointer to mtrr configuration data * * Returns nothing. */ -static int mtrr_work_handler(void *info) +static int mtrr_rendezvous_handler(void *info) { #ifdef CONFIG_SMP struct set_mtrr_data *data = info; - unsigned long flags; - - atomic_dec(&data->count); - while (!atomic_read(&data->gate)) - cpu_relax(); - - local_irq_save(flags); - - atomic_dec(&data->count); - while (atomic_read(&data->gate)) - cpu_relax(); - /* The master has cleared me to execute */ + /* + * We use this same function to initialize the mtrrs during boot, + * resume, runtime cpu online and on an explicit request to set a + * specific MTRR. + * + * During boot or suspend, the state of the boot cpu's mtrrs has been + * saved, and we want to replicate that across all the cpus that come + * online (either at the end of boot or resume or during a runtime cpu + * online). If we're doing that, @reg is set to something special and on + * all the cpu's we do mtrr_if->set_all() (On the logical cpu that + * started the boot/resume sequence, this might be a duplicate + * set_all()). + */ if (data->smp_reg != ~0U) { mtrr_if->set(data->smp_reg, data->smp_base, data->smp_size, data->smp_type); - } else if (mtrr_aps_delayed_init) { - /* - * Initialize the MTRRs inaddition to the synchronisation. - */ + } else if (mtrr_aps_delayed_init || !cpu_online(smp_processor_id())) { mtrr_if->set_all(); } - - atomic_dec(&data->count); - while (!atomic_read(&data->gate)) - cpu_relax(); - - atomic_dec(&data->count); - local_irq_restore(flags); #endif return 0; } @@ -223,20 +211,11 @@ static inline int types_compatible(mtrr_type type1, mtrr_type type2) * 14. Wait for buddies to catch up * 15. Enable interrupts. * - * What does that mean for us? Well, first we set data.count to the number - * of CPUs. As each CPU announces that it started the rendezvous handler by - * decrementing the count, We reset data.count and set the data.gate flag - * allowing all the cpu's to proceed with the work. As each cpu disables - * interrupts, it'll decrement data.count once. We wait until it hits 0 and - * proceed. We clear the data.gate flag and reset data.count. Meanwhile, they - * are waiting for that flag to be cleared. Once it's cleared, each - * CPU goes through the transition of updating MTRRs. - * The CPU vendors may each do it differently, - * so we call mtrr_if->set() callback and let them take care of it. - * When they're done, they again decrement data->count and wait for data.gate - * to be set. - * When we finish, we wait for data.count to hit 0 and toggle the data.gate flag - * Everyone then enables interrupts and we all continue on. + * What does that mean for us? Well, stop_machine() will ensure that + * the rendezvous handler is started on each CPU. And in lockstep they + * do the state transition of disabling interrupts, updating MTRR's + * (the CPU vendors may each do it differently, so we call mtrr_if->set() + * callback and let them take care of it.) and enabling interrupts. * * Note that the mechanism is the same for UP systems, too; all the SMP stuff * becomes nops. @@ -244,115 +223,26 @@ static inline int types_compatible(mtrr_type type1, mtrr_type type2) static void set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type) { - struct set_mtrr_data data; - unsigned long flags; - int cpu; - -#ifdef CONFIG_SMP - /* - * If this cpu is not yet active, we are in the cpu online path. There - * can be no stop_machine() in parallel, as stop machine ensures this - * by using get_online_cpus(). We can skip taking the stop_cpus_mutex, - * as we don't need it and also we can't afford to block while waiting - * for the mutex. - * - * If this cpu is active, we need to prevent stop_machine() happening - * in parallel by taking the stop cpus mutex. - * - * Also, this is called in the context of cpu online path or in the - * context where cpu hotplug is prevented. So checking the active status - * of the raw_smp_processor_id() is safe. - */ - if (cpu_active(raw_smp_processor_id())) - mutex_lock(&stop_cpus_mutex); -#endif - - preempt_disable(); - - data.smp_reg = reg; - data.smp_base = base; - data.smp_size = size; - data.smp_type = type; - atomic_set(&data.count, num_booting_cpus() - 1); - - /* Make sure data.count is visible before unleashing other CPUs */ - smp_wmb(); - atomic_set(&data.gate, 0); - - /* Start the ball rolling on other CPUs */ - for_each_online_cpu(cpu) { - struct cpu_stop_work *work = &per_cpu(mtrr_work, cpu); - - if (cpu == smp_processor_id()) - continue; + struct set_mtrr_data data = { .smp_reg = reg, + .smp_base = base, + .smp_size = size, + .smp_type = type + }; - stop_one_cpu_nowait(cpu, mtrr_work_handler, &data, work); - } - - - while (atomic_read(&data.count)) - cpu_relax(); - - /* Ok, reset count and toggle gate */ - atomic_set(&data.count, num_booting_cpus() - 1); - smp_wmb(); - atomic_set(&data.gate, 1); - - local_irq_save(flags); - - while (atomic_read(&data.count)) - cpu_relax(); - - /* Ok, reset count and toggle gate */ - atomic_set(&data.count, num_booting_cpus() - 1); - smp_wmb(); - atomic_set(&data.gate, 0); - - /* Do our MTRR business */ - - /* - * HACK! - * - * We use this same function to initialize the mtrrs during boot, - * resume, runtime cpu online and on an explicit request to set a - * specific MTRR. - * - * During boot or suspend, the state of the boot cpu's mtrrs has been - * saved, and we want to replicate that across all the cpus that come - * online (either at the end of boot or resume or during a runtime cpu - * online). If we're doing that, @reg is set to something special and on - * this cpu we still do mtrr_if->set_all(). During boot/resume, this - * is unnecessary if at this point we are still on the cpu that started - * the boot/resume sequence. But there is no guarantee that we are still - * on the same cpu. So we do mtrr_if->set_all() on this cpu aswell to be - * sure that we are in sync with everyone else. - */ - if (reg != ~0U) - mtrr_if->set(reg, base, size, type); - else - mtrr_if->set_all(); - - /* Wait for the others */ - while (atomic_read(&data.count)) - cpu_relax(); - - atomic_set(&data.count, num_booting_cpus() - 1); - smp_wmb(); - atomic_set(&data.gate, 1); - - /* - * Wait here for everyone to have seen the gate change - * So we're the last ones to touch 'data' - */ - while (atomic_read(&data.count)) - cpu_relax(); + stop_machine(mtrr_rendezvous_handler, &data, cpu_online_mask); +} - local_irq_restore(flags); - preempt_enable(); -#ifdef CONFIG_SMP - if (cpu_active(raw_smp_processor_id())) - mutex_unlock(&stop_cpus_mutex); -#endif +static void set_mtrr_from_inactive_cpu(unsigned int reg, unsigned long base, + unsigned long size, mtrr_type type) +{ + struct set_mtrr_data data = { .smp_reg = reg, + .smp_base = base, + .smp_size = size, + .smp_type = type + }; + + stop_machine_from_inactive_cpu(mtrr_rendezvous_handler, &data, + cpu_callout_mask); } /** @@ -806,7 +696,7 @@ void mtrr_ap_init(void) * 2. cpu hotadd time. We let mtrr_add/del_page hold cpuhotplug * lock to prevent mtrr entry changes */ - set_mtrr(~0U, 0, 0, 0); + set_mtrr_from_inactive_cpu(~0U, 0, 0, 0); } /** diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h index e0f2da25d751..4a9d0c7edc65 100644 --- a/include/linux/stop_machine.h +++ b/include/linux/stop_machine.h @@ -27,8 +27,6 @@ struct cpu_stop_work { struct cpu_stop_done *done; }; -extern struct mutex stop_cpus_mutex; - int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg); void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, struct cpu_stop_work *work_buf); diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index e8f05b14cd43..c1124752e1d3 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -132,8 +132,8 @@ void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, cpu_stop_queue_work(&per_cpu(cpu_stopper, cpu), work_buf); } -DEFINE_MUTEX(stop_cpus_mutex); /* static data for stop_cpus */ +static DEFINE_MUTEX(stop_cpus_mutex); static DEFINE_PER_CPU(struct cpu_stop_work, stop_cpus_work); static void queue_stop_cpus_work(const struct cpumask *cpumask, -- cgit v1.2.3 From 3e7cf5b00dd5b577b4ee9b2a66e40fb670ef210b Mon Sep 17 00:00:00 2001 From: Suresh Siddha Date: Thu, 30 Jun 2011 17:19:32 -0700 Subject: x86-32, fpu: Fix DNA exception during check_fpu() Before check_fpu() is called, we have cr0.TS bit set and hence the floating point code to check the FDIV bug was generating a DNA exception. Use kernel_fpu_begin()/kernel_fpu_end() around the floating point code to avoid this unnecessary device not available exception during boot. Signed-off-by: Suresh Siddha Link: http://lkml.kernel.org/r/1309479572.2665.1372.camel@sbsiddha-MOBL3.sc.intel.com Signed-off-by: H. Peter Anvin --- arch/x86/kernel/cpu/bugs.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 525514cf33c3..46674fbb62ba 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -62,6 +62,8 @@ static void __init check_fpu(void) return; } + kernel_fpu_begin(); + /* * trap_init() enabled FXSR and company _before_ testing for FP * problems here. @@ -80,6 +82,8 @@ static void __init check_fpu(void) : "=m" (*&fdiv_bug) : "m" (*&x), "m" (*&y)); + kernel_fpu_end(); + boot_cpu_data.fdiv_bug = fdiv_bug; if (boot_cpu_data.fdiv_bug) printk(KERN_WARNING "Hmm, FPU with FDIV bug.\n"); -- cgit v1.2.3 From 0a779c571314b7951ff12edac80aa0cbe7c12b6e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 9 Jun 2011 13:08:26 +0000 Subject: x86: Use common i8253 clockevent Signed-off-by: Thomas Gleixner Cc: Russell King Cc: Ralf Baechle Cc: John Stultz Link: http://lkml.kernel.org/r/20110609130622.026152527@linutronix.de --- arch/x86/Kconfig | 2 +- arch/x86/kernel/i8253.c | 93 ++----------------------------------------------- 2 files changed, 4 insertions(+), 91 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 2dd95b55567b..26a14b4bdd14 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -71,7 +71,7 @@ config X86 select IRQ_FORCED_THREADING select USE_GENERIC_SMP_HELPERS if SMP select HAVE_BPF_JIT if (X86_64 && NET) - select I8253_LOCK + select CLKEVT_I8253 config INSTRUCTION_DECODER def_bool (KPROBES || PERF_EVENTS) diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c index 5783e6de2079..f2b96de3c7c1 100644 --- a/arch/x86/kernel/i8253.c +++ b/arch/x86/kernel/i8253.c @@ -3,15 +3,9 @@ * */ #include -#include -#include -#include #include #include -#include #include -#include -#include #include #include @@ -23,91 +17,10 @@ */ struct clock_event_device *global_clock_event; -/* - * Initialize the PIT timer. - * - * This is also called after resume to bring the PIT into operation again. - */ -static void init_pit_timer(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - raw_spin_lock(&i8253_lock); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - /* binary, mode 2, LSB/MSB, ch 0 */ - outb_pit(0x34, PIT_MODE); - outb_pit(LATCH & 0xff , PIT_CH0); /* LSB */ - outb_pit(LATCH >> 8 , PIT_CH0); /* MSB */ - break; - - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - if (evt->mode == CLOCK_EVT_MODE_PERIODIC || - evt->mode == CLOCK_EVT_MODE_ONESHOT) { - outb_pit(0x30, PIT_MODE); - outb_pit(0, PIT_CH0); - outb_pit(0, PIT_CH0); - } - break; - - case CLOCK_EVT_MODE_ONESHOT: - /* One shot setup */ - outb_pit(0x38, PIT_MODE); - break; - - case CLOCK_EVT_MODE_RESUME: - /* Nothing to do here */ - break; - } - raw_spin_unlock(&i8253_lock); -} - -/* - * Program the next event in oneshot mode - * - * Delta is given in PIT ticks - */ -static int pit_next_event(unsigned long delta, struct clock_event_device *evt) -{ - raw_spin_lock(&i8253_lock); - outb_pit(delta & 0xff , PIT_CH0); /* LSB */ - outb_pit(delta >> 8 , PIT_CH0); /* MSB */ - raw_spin_unlock(&i8253_lock); - - return 0; -} - -/* - * On UP the PIT can serve all of the possible timer functions. On SMP systems - * it can be solely used for the global tick. - * - * The profiling and update capabilities are switched off once the local apic is - * registered. This mechanism replaces the previous #ifdef LOCAL_APIC - - * !using_apic_timer decisions in do_timer_interrupt_hook() - */ -static struct clock_event_device pit_ce = { - .name = "pit", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = init_pit_timer, - .set_next_event = pit_next_event, - .irq = 0, -}; - -/* - * Initialize the conversion factor and the min/max deltas of the clock event - * structure and register the clock event source with the framework. - */ void __init setup_pit_timer(void) { - /* - * Start pit with the boot cpu mask and make it global after the - * IO_APIC has been initialized. - */ - pit_ce.cpumask = cpumask_of(smp_processor_id()); - - clockevents_config_and_register(&pit_ce, CLOCK_TICK_RATE, 0xF, 0x7FFF); - global_clock_event = &pit_ce; + clockevent_i8253_init(true); + global_clock_event = &i8253_clockevent; } #ifndef CONFIG_X86_64 @@ -121,7 +34,7 @@ static int __init init_pit_clocksource(void) * - when local APIC timer is active (PIT is switched off) */ if (num_possible_cpus() > 1 || is_hpet_enabled() || - pit_ce.mode != CLOCK_EVT_MODE_PERIODIC) + i8253_clockevent.mode != CLOCK_EVT_MODE_PERIODIC) return 0; return clocksource_i8253_init(); -- cgit v1.2.3 From 01898e3e29ea8242d81923da11ce88ba71290a48 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 9 Jun 2011 13:08:30 +0000 Subject: i8253: Cleanup outb/inb magic Remove the hysterical outb/inb_pit defines and use outb_p/inb_p in the code. Signed-off-by: Thomas Gleixner Cc: Russell King Cc: Ralf Baechle Cc: John Stultz Link: http://lkml.kernel.org/r/20110609130622.348437125@linutronix.de --- arch/x86/kernel/apm_32.c | 6 +++--- include/linux/i8253.h | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index a30740e6890e..0371c484bb8a 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c @@ -1220,11 +1220,11 @@ static void reinit_timer(void) raw_spin_lock_irqsave(&i8253_lock, flags); /* set the clock to HZ */ - outb_pit(0x34, PIT_MODE); /* binary, mode 2, LSB/MSB, ch 0 */ + outb_p(0x34, PIT_MODE); /* binary, mode 2, LSB/MSB, ch 0 */ udelay(10); - outb_pit(LATCH & 0xff, PIT_CH0); /* LSB */ + outb_p(LATCH & 0xff, PIT_CH0); /* LSB */ udelay(10); - outb_pit(LATCH >> 8, PIT_CH0); /* MSB */ + outb_p(LATCH >> 8, PIT_CH0); /* MSB */ udelay(10); raw_spin_unlock_irqrestore(&i8253_lock, flags); #endif diff --git a/include/linux/i8253.h b/include/linux/i8253.h index 487d50c7db39..e6bb36a97519 100644 --- a/include/linux/i8253.h +++ b/include/linux/i8253.h @@ -20,9 +20,6 @@ #define PIT_LATCH ((PIT_TICK_RATE + HZ/2) / HZ) -#define inb_pit inb_p -#define outb_pit outb_p - extern raw_spinlock_t i8253_lock; extern struct clock_event_device i8253_clockevent; extern void clockevent_i8253_init(bool oneshot); -- cgit v1.2.3 From 1880c4ae182afb5650c5678949ecfe7ff66a724e Mon Sep 17 00:00:00 2001 From: Cyrill Gorcunov Date: Thu, 23 Jun 2011 16:49:18 +0400 Subject: perf, x86: Add hw_watchdog_set_attr() in a sake of nmi-watchdog on P4 Due to restriction and specifics of Netburst PMU we need a separated event for NMI watchdog. In particular every Netburst event consumes not just a counter and a config register, but also an additional ESCR register. Since ESCR registers are grouped upon counters (i.e. if ESCR is occupied for some event there is no room for another event to enter until its released) we need to pick up the "least" used ESCR (or the most available one) for nmi-watchdog purposes -- so MSR_P4_CRU_ESCR2/3 was chosen. With this patch nmi-watchdog and perf top should be able to run simultaneously. Signed-off-by: Cyrill Gorcunov CC: Lin Ming CC: Arnaldo Carvalho de Melo CC: Frederic Weisbecker Tested-and-reviewed-by: Don Zickus Tested-and-reviewed-by: Stephane Eranian Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/20110623124918.GC13050@sun Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/perf_event.c | 7 +++++++ arch/x86/kernel/cpu/perf_event_p4.c | 26 ++++++++++++++++++++++++++ kernel/watchdog.c | 6 +++++- 3 files changed, 38 insertions(+), 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 3a0338b4b179..8a57f9aa8e36 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -233,6 +233,7 @@ struct x86_pmu { void (*enable_all)(int added); void (*enable)(struct perf_event *); void (*disable)(struct perf_event *); + void (*hw_watchdog_set_attr)(struct perf_event_attr *attr); int (*hw_config)(struct perf_event *event); int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign); unsigned eventsel; @@ -315,6 +316,12 @@ static u64 __read_mostly hw_cache_extra_regs [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX]; +void hw_nmi_watchdog_set_attr(struct perf_event_attr *wd_attr) +{ + if (x86_pmu.hw_watchdog_set_attr) + x86_pmu.hw_watchdog_set_attr(wd_attr); +} + /* * Propagate event elapsed time into the generic event. * Can only be executed on the CPU where the event is active. diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c index ead584fb6a7d..f76fddf63381 100644 --- a/arch/x86/kernel/cpu/perf_event_p4.c +++ b/arch/x86/kernel/cpu/perf_event_p4.c @@ -705,6 +705,31 @@ static int p4_validate_raw_event(struct perf_event *event) return 0; } +static void p4_hw_watchdog_set_attr(struct perf_event_attr *wd_attr) +{ + /* + * Watchdog ticks are special on Netburst, we use + * that named "non-sleeping" ticks as recommended + * by Intel SDM Vol3b. + */ + WARN_ON_ONCE(wd_attr->type != PERF_TYPE_HARDWARE || + wd_attr->config != PERF_COUNT_HW_CPU_CYCLES); + + wd_attr->type = PERF_TYPE_RAW; + wd_attr->config = + p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_EXECUTION_EVENT) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS0) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS1) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS2) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS3) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS0) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS1) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS2) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS3)) | + p4_config_pack_cccr(P4_CCCR_THRESHOLD(15) | P4_CCCR_COMPLEMENT | + P4_CCCR_COMPARE); +} + static int p4_hw_config(struct perf_event *event) { int cpu = get_cpu(); @@ -1179,6 +1204,7 @@ static __initconst const struct x86_pmu p4_pmu = { .cntval_bits = ARCH_P4_CNTRVAL_BITS, .cntval_mask = ARCH_P4_CNTRVAL_MASK, .max_period = (1ULL << (ARCH_P4_CNTRVAL_BITS - 1)) - 1, + .hw_watchdog_set_attr = p4_hw_watchdog_set_attr, .hw_config = p4_hw_config, .schedule_events = p4_pmu_schedule_events, /* diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 3d0c56ad4792..752b75ba662b 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -200,6 +200,8 @@ static int is_softlockup(unsigned long touch_ts) } #ifdef CONFIG_HARDLOCKUP_DETECTOR +void __weak hw_nmi_watchdog_set_attr(struct perf_event_attr *wd_attr) { } + static struct perf_event_attr wd_hw_attr = { .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES, @@ -368,9 +370,11 @@ static int watchdog_nmi_enable(int cpu) if (event != NULL) goto out_enable; - /* Try to register using hardware perf events */ wd_attr = &wd_hw_attr; wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); + hw_nmi_watchdog_set_attr(wd_attr); + + /* Try to register using hardware perf events */ event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback); if (!IS_ERR(event)) { printk(KERN_INFO "NMI watchdog enabled, takes one hw-pmu counter.\n"); -- cgit v1.2.3 From a8b0ca17b80e92faab46ee7179ba9e99ccb61233 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 27 Jun 2011 14:41:57 +0200 Subject: perf: Remove the nmi parameter from the swevent and overflow interface The nmi parameter indicated if we could do wakeups from the current context, if not, we would set some state and self-IPI and let the resulting interrupt do the wakeup. For the various event classes: - hardware: nmi=0; PMI is in fact an NMI or we run irq_work_run from the PMI-tail (ARM etc.) - tracepoint: nmi=0; since tracepoint could be from NMI context. - software: nmi=[0,1]; some, like the schedule thing cannot perform wakeups, and hence need 0. As one can see, there is very little nmi=1 usage, and the down-side of not using it is that on some platforms some software events can have a jiffy delay in wakeup (when arch_irq_work_raise isn't implemented). The up-side however is that we can remove the nmi parameter and save a bunch of conditionals in fast paths. Signed-off-by: Peter Zijlstra Cc: Michael Cree Cc: Will Deacon Cc: Deng-Cheng Zhu Cc: Anton Blanchard Cc: Eric B Munson Cc: Heiko Carstens Cc: Paul Mundt Cc: David S. Miller Cc: Frederic Weisbecker Cc: Jason Wessel Cc: Don Zickus Link: http://lkml.kernel.org/n/tip-agjev8eu666tvknpb3iaj0fg@git.kernel.org Signed-off-by: Ingo Molnar --- arch/alpha/kernel/perf_event.c | 2 +- arch/arm/kernel/perf_event_v6.c | 2 +- arch/arm/kernel/perf_event_v7.c | 2 +- arch/arm/kernel/perf_event_xscale.c | 4 +- arch/arm/kernel/ptrace.c | 2 +- arch/arm/kernel/swp_emulate.c | 2 +- arch/arm/mm/fault.c | 6 +-- arch/mips/kernel/perf_event.c | 2 +- arch/mips/kernel/traps.c | 8 ++-- arch/mips/kernel/unaligned.c | 5 +-- arch/mips/math-emu/cp1emu.c | 3 +- arch/mips/mm/fault.c | 8 ++-- arch/powerpc/include/asm/emulated_ops.h | 4 +- arch/powerpc/kernel/perf_event.c | 6 +-- arch/powerpc/kernel/perf_event_fsl_emb.c | 6 +-- arch/powerpc/kernel/ptrace.c | 2 +- arch/powerpc/mm/fault.c | 6 +-- arch/s390/mm/fault.c | 6 +-- arch/sh/kernel/ptrace_32.c | 2 +- arch/sh/kernel/traps_32.c | 2 +- arch/sh/kernel/traps_64.c | 8 ++-- arch/sh/math-emu/math.c | 2 +- arch/sh/mm/fault_32.c | 6 +-- arch/sh/mm/tlbflush_64.c | 6 +-- arch/sparc/kernel/perf_event.c | 2 +- arch/sparc/kernel/unaligned_32.c | 4 +- arch/sparc/kernel/unaligned_64.c | 12 +++--- arch/sparc/kernel/visemul.c | 2 +- arch/sparc/math-emu/math_32.c | 2 +- arch/sparc/math-emu/math_64.c | 2 +- arch/sparc/mm/fault_32.c | 8 ++-- arch/sparc/mm/fault_64.c | 8 ++-- arch/x86/kernel/cpu/perf_event.c | 2 +- arch/x86/kernel/cpu/perf_event_intel.c | 2 +- arch/x86/kernel/cpu/perf_event_intel_ds.c | 4 +- arch/x86/kernel/cpu/perf_event_p4.c | 2 +- arch/x86/kernel/kgdb.c | 2 +- arch/x86/kernel/ptrace.c | 2 +- arch/x86/mm/fault.c | 6 +-- include/linux/perf_event.h | 18 ++++----- kernel/events/core.c | 63 ++++++++++++++----------------- kernel/events/internal.h | 1 - kernel/events/ring_buffer.c | 10 ++--- kernel/sched.c | 2 +- kernel/watchdog.c | 2 +- samples/hw_breakpoint/data_breakpoint.c | 2 +- 46 files changed, 119 insertions(+), 141 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/alpha/kernel/perf_event.c b/arch/alpha/kernel/perf_event.c index 90561c45e7d8..8e47709160f8 100644 --- a/arch/alpha/kernel/perf_event.c +++ b/arch/alpha/kernel/perf_event.c @@ -847,7 +847,7 @@ static void alpha_perf_event_irq_handler(unsigned long la_ptr, data.period = event->hw.last_period; if (alpha_perf_event_set_period(event, hwc, idx)) { - if (perf_event_overflow(event, 1, &data, regs)) { + if (perf_event_overflow(event, &data, regs)) { /* Interrupts coming too quickly; "throttle" the * counter, i.e., disable it for a little while. */ diff --git a/arch/arm/kernel/perf_event_v6.c b/arch/arm/kernel/perf_event_v6.c index f1e8dd94afe8..38dc4da1d612 100644 --- a/arch/arm/kernel/perf_event_v6.c +++ b/arch/arm/kernel/perf_event_v6.c @@ -479,7 +479,7 @@ armv6pmu_handle_irq(int irq_num, if (!armpmu_event_set_period(event, hwc, idx)) continue; - if (perf_event_overflow(event, 0, &data, regs)) + if (perf_event_overflow(event, &data, regs)) armpmu->disable(hwc, idx); } diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c index 4960686afb58..6e5f8752303b 100644 --- a/arch/arm/kernel/perf_event_v7.c +++ b/arch/arm/kernel/perf_event_v7.c @@ -787,7 +787,7 @@ static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) if (!armpmu_event_set_period(event, hwc, idx)) continue; - if (perf_event_overflow(event, 0, &data, regs)) + if (perf_event_overflow(event, &data, regs)) armpmu->disable(hwc, idx); } diff --git a/arch/arm/kernel/perf_event_xscale.c b/arch/arm/kernel/perf_event_xscale.c index 39affbe4fdb2..99b6b85c7e49 100644 --- a/arch/arm/kernel/perf_event_xscale.c +++ b/arch/arm/kernel/perf_event_xscale.c @@ -251,7 +251,7 @@ xscale1pmu_handle_irq(int irq_num, void *dev) if (!armpmu_event_set_period(event, hwc, idx)) continue; - if (perf_event_overflow(event, 0, &data, regs)) + if (perf_event_overflow(event, &data, regs)) armpmu->disable(hwc, idx); } @@ -583,7 +583,7 @@ xscale2pmu_handle_irq(int irq_num, void *dev) if (!armpmu_event_set_period(event, hwc, idx)) continue; - if (perf_event_overflow(event, 0, &data, regs)) + if (perf_event_overflow(event, &data, regs)) armpmu->disable(hwc, idx); } diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index 97260060bf26..0c9b1054f790 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -396,7 +396,7 @@ static long ptrace_hbp_idx_to_num(int idx) /* * Handle hitting a HW-breakpoint. */ -static void ptrace_hbptriggered(struct perf_event *bp, int unused, +static void ptrace_hbptriggered(struct perf_event *bp, struct perf_sample_data *data, struct pt_regs *regs) { diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c index 40ee7e5045e4..5f452f8fde05 100644 --- a/arch/arm/kernel/swp_emulate.c +++ b/arch/arm/kernel/swp_emulate.c @@ -183,7 +183,7 @@ static int swp_handler(struct pt_regs *regs, unsigned int instr) unsigned int address, destreg, data, type; unsigned int res = 0; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, regs->ARM_pc); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->ARM_pc); if (current->pid != previous_pid) { pr_debug("\"%s\" (%ld) uses deprecated SWP{B} instruction\n", diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index bc0e1d88fd3b..9ea4f7ddd665 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -318,11 +318,11 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) fault = __do_page_fault(mm, addr, fsr, tsk); up_read(&mm->mmap_sem); - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, addr); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); if (fault & VM_FAULT_MAJOR) - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, regs, addr); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, addr); else if (fault & VM_FAULT_MINOR) - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, regs, addr); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, addr); /* * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR diff --git a/arch/mips/kernel/perf_event.c b/arch/mips/kernel/perf_event.c index a8244854d3dc..d0deaab9ace2 100644 --- a/arch/mips/kernel/perf_event.c +++ b/arch/mips/kernel/perf_event.c @@ -527,7 +527,7 @@ handle_associated_event(struct cpu_hw_events *cpuc, if (!mipspmu_event_set_period(event, hwc, idx)) return; - if (perf_event_overflow(event, 0, data, regs)) + if (perf_event_overflow(event, data, regs)) mipspmu->disable_event(idx); } diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index e9b3af27d844..b7517e3abc85 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -578,12 +578,12 @@ static int simulate_llsc(struct pt_regs *regs, unsigned int opcode) { if ((opcode & OPCODE) == LL) { perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); + 1, regs, 0); return simulate_ll(regs, opcode); } if ((opcode & OPCODE) == SC) { perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); + 1, regs, 0); return simulate_sc(regs, opcode); } @@ -602,7 +602,7 @@ static int simulate_rdhwr(struct pt_regs *regs, unsigned int opcode) int rd = (opcode & RD) >> 11; int rt = (opcode & RT) >> 16; perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); + 1, regs, 0); switch (rd) { case 0: /* CPU number */ regs->regs[rt] = smp_processor_id(); @@ -640,7 +640,7 @@ static int simulate_sync(struct pt_regs *regs, unsigned int opcode) { if ((opcode & OPCODE) == SPEC0 && (opcode & FUNC) == SYNC) { perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); + 1, regs, 0); return 0; } diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index cfea1adfa153..eb319b580353 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -111,8 +111,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, unsigned long value; unsigned int res; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); /* * This load never faults. @@ -517,7 +516,7 @@ asmlinkage void do_ade(struct pt_regs *regs) mm_segment_t seg; perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, - 1, 0, regs, regs->cp0_badvaddr); + 1, regs, regs->cp0_badvaddr); /* * Did we catch a fault trying to load an instruction? * Or are we running in MIPS16 mode? diff --git a/arch/mips/math-emu/cp1emu.c b/arch/mips/math-emu/cp1emu.c index d32cb0503110..dbf2f93a5091 100644 --- a/arch/mips/math-emu/cp1emu.c +++ b/arch/mips/math-emu/cp1emu.c @@ -272,8 +272,7 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx, } emul: - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, xcp, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, xcp, 0); MIPS_FPU_EMU_INC_STATS(emulated); switch (MIPSInst_OPCODE(ir)) { case ldc1_op:{ diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 137ee76a0045..937cf3368164 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -145,7 +145,7 @@ good_area: * the fault. */ fault = handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0); - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; @@ -154,12 +154,10 @@ good_area: BUG(); } if (fault & VM_FAULT_MAJOR) { - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, - 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); tsk->maj_flt++; } else { - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, - 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); tsk->min_flt++; } diff --git a/arch/powerpc/include/asm/emulated_ops.h b/arch/powerpc/include/asm/emulated_ops.h index 45921672b97a..2cc41c715d2b 100644 --- a/arch/powerpc/include/asm/emulated_ops.h +++ b/arch/powerpc/include/asm/emulated_ops.h @@ -78,14 +78,14 @@ extern void ppc_warn_emulated_print(const char *type); #define PPC_WARN_EMULATED(type, regs) \ do { \ perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, \ - 1, 0, regs, 0); \ + 1, regs, 0); \ __PPC_WARN_EMULATED(type); \ } while (0) #define PPC_WARN_ALIGNMENT(type, regs) \ do { \ perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, \ - 1, 0, regs, regs->dar); \ + 1, regs, regs->dar); \ __PPC_WARN_EMULATED(type); \ } while (0) diff --git a/arch/powerpc/kernel/perf_event.c b/arch/powerpc/kernel/perf_event.c index 822f63008ae1..14967de98876 100644 --- a/arch/powerpc/kernel/perf_event.c +++ b/arch/powerpc/kernel/perf_event.c @@ -1207,7 +1207,7 @@ struct pmu power_pmu = { * here so there is no possibility of being interrupted. */ static void record_and_restart(struct perf_event *event, unsigned long val, - struct pt_regs *regs, int nmi) + struct pt_regs *regs) { u64 period = event->hw.sample_period; s64 prev, delta, left; @@ -1258,7 +1258,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, if (event->attr.sample_type & PERF_SAMPLE_ADDR) perf_get_data_addr(regs, &data.addr); - if (perf_event_overflow(event, nmi, &data, regs)) + if (perf_event_overflow(event, &data, regs)) power_pmu_stop(event, 0); } } @@ -1346,7 +1346,7 @@ static void perf_event_interrupt(struct pt_regs *regs) if ((int)val < 0) { /* event has overflowed */ found = 1; - record_and_restart(event, val, regs, nmi); + record_and_restart(event, val, regs); } } diff --git a/arch/powerpc/kernel/perf_event_fsl_emb.c b/arch/powerpc/kernel/perf_event_fsl_emb.c index b0dc8f7069cd..0a6d2a9d569c 100644 --- a/arch/powerpc/kernel/perf_event_fsl_emb.c +++ b/arch/powerpc/kernel/perf_event_fsl_emb.c @@ -568,7 +568,7 @@ static struct pmu fsl_emb_pmu = { * here so there is no possibility of being interrupted. */ static void record_and_restart(struct perf_event *event, unsigned long val, - struct pt_regs *regs, int nmi) + struct pt_regs *regs) { u64 period = event->hw.sample_period; s64 prev, delta, left; @@ -616,7 +616,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, perf_sample_data_init(&data, 0); data.period = event->hw.last_period; - if (perf_event_overflow(event, nmi, &data, regs)) + if (perf_event_overflow(event, &data, regs)) fsl_emb_pmu_stop(event, 0); } } @@ -644,7 +644,7 @@ static void perf_event_interrupt(struct pt_regs *regs) if (event) { /* event has overflowed */ found = 1; - record_and_restart(event, val, regs, nmi); + record_and_restart(event, val, regs); } else { /* * Disabled counter is negative, diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index cb22024f2b42..3177617af2ef 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -882,7 +882,7 @@ void user_disable_single_step(struct task_struct *task) } #ifdef CONFIG_HAVE_HW_BREAKPOINT -void ptrace_triggered(struct perf_event *bp, int nmi, +void ptrace_triggered(struct perf_event *bp, struct perf_sample_data *data, struct pt_regs *regs) { struct perf_event_attr attr; diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 54f4fb994e99..dbc48254c6cc 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -173,7 +173,7 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address, die("Weird page fault", regs, SIGSEGV); } - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); /* When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in the @@ -319,7 +319,7 @@ good_area: } if (ret & VM_FAULT_MAJOR) { current->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); #ifdef CONFIG_PPC_SMLPAR if (firmware_has_feature(FW_FEATURE_CMO)) { @@ -330,7 +330,7 @@ good_area: #endif } else { current->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } up_read(&mm->mmap_sem); diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index fe103e891e7a..095f782a5512 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -299,7 +299,7 @@ static inline int do_exception(struct pt_regs *regs, int access, goto out; address = trans_exc_code & __FAIL_ADDR_MASK; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); flags = FAULT_FLAG_ALLOW_RETRY; if (access == VM_WRITE || (trans_exc_code & store_indication) == 0x400) flags |= FAULT_FLAG_WRITE; @@ -345,11 +345,11 @@ retry: if (flags & FAULT_FLAG_ALLOW_RETRY) { if (fault & VM_FAULT_MAJOR) { tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); } else { tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } if (fault & VM_FAULT_RETRY) { diff --git a/arch/sh/kernel/ptrace_32.c b/arch/sh/kernel/ptrace_32.c index 3d7b209b2178..8051976100a6 100644 --- a/arch/sh/kernel/ptrace_32.c +++ b/arch/sh/kernel/ptrace_32.c @@ -63,7 +63,7 @@ static inline int put_stack_long(struct task_struct *task, int offset, return 0; } -void ptrace_triggered(struct perf_event *bp, int nmi, +void ptrace_triggered(struct perf_event *bp, struct perf_sample_data *data, struct pt_regs *regs) { struct perf_event_attr attr; diff --git a/arch/sh/kernel/traps_32.c b/arch/sh/kernel/traps_32.c index b51a17104b5f..d9006f8ffc14 100644 --- a/arch/sh/kernel/traps_32.c +++ b/arch/sh/kernel/traps_32.c @@ -393,7 +393,7 @@ int handle_unaligned_access(insn_size_t instruction, struct pt_regs *regs, */ if (!expected) { unaligned_fixups_notify(current, instruction, regs); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, address); } diff --git a/arch/sh/kernel/traps_64.c b/arch/sh/kernel/traps_64.c index 6713ca97e553..67110be83fd7 100644 --- a/arch/sh/kernel/traps_64.c +++ b/arch/sh/kernel/traps_64.c @@ -434,7 +434,7 @@ static int misaligned_load(struct pt_regs *regs, return error; } - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, address); destreg = (opcode >> 4) & 0x3f; if (user_mode(regs)) { @@ -512,7 +512,7 @@ static int misaligned_store(struct pt_regs *regs, return error; } - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, address); srcreg = (opcode >> 4) & 0x3f; if (user_mode(regs)) { @@ -588,7 +588,7 @@ static int misaligned_fpu_load(struct pt_regs *regs, return error; } - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, address); destreg = (opcode >> 4) & 0x3f; if (user_mode(regs)) { @@ -665,7 +665,7 @@ static int misaligned_fpu_store(struct pt_regs *regs, return error; } - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, address); srcreg = (opcode >> 4) & 0x3f; if (user_mode(regs)) { diff --git a/arch/sh/math-emu/math.c b/arch/sh/math-emu/math.c index f76a5090d5d1..977195210653 100644 --- a/arch/sh/math-emu/math.c +++ b/arch/sh/math-emu/math.c @@ -620,7 +620,7 @@ int do_fpu_inst(unsigned short inst, struct pt_regs *regs) struct task_struct *tsk = current; struct sh_fpu_soft_struct *fpu = &(tsk->thread.xstate->softfpu); - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); if (!(task_thread_info(tsk)->status & TS_USEDFPU)) { /* initialize once. */ diff --git a/arch/sh/mm/fault_32.c b/arch/sh/mm/fault_32.c index d4c34d757f0d..7bebd044f2a1 100644 --- a/arch/sh/mm/fault_32.c +++ b/arch/sh/mm/fault_32.c @@ -160,7 +160,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, if ((regs->sr & SR_IMASK) != SR_IMASK) local_irq_enable(); - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); /* * If we're in an interrupt, have no user context or are running @@ -210,11 +210,11 @@ good_area: } if (fault & VM_FAULT_MAJOR) { tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); } else { tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } diff --git a/arch/sh/mm/tlbflush_64.c b/arch/sh/mm/tlbflush_64.c index 7f5810f5dfdc..e3430e093d43 100644 --- a/arch/sh/mm/tlbflush_64.c +++ b/arch/sh/mm/tlbflush_64.c @@ -116,7 +116,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess, /* Not an IO address, so reenable interrupts */ local_irq_enable(); - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); /* * If we're in an interrupt or have no user @@ -200,11 +200,11 @@ good_area: if (fault & VM_FAULT_MAJOR) { tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); } else { tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c index 2cb0e1c001e2..0b32f2e9e08d 100644 --- a/arch/sparc/kernel/perf_event.c +++ b/arch/sparc/kernel/perf_event.c @@ -1277,7 +1277,7 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, if (!sparc_perf_event_set_period(event, hwc, idx)) continue; - if (perf_event_overflow(event, 1, &data, regs)) + if (perf_event_overflow(event, &data, regs)) sparc_pmu_stop(event, 0); } diff --git a/arch/sparc/kernel/unaligned_32.c b/arch/sparc/kernel/unaligned_32.c index 4491f4cb2695..7efbb2f9e77f 100644 --- a/arch/sparc/kernel/unaligned_32.c +++ b/arch/sparc/kernel/unaligned_32.c @@ -247,7 +247,7 @@ asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) unsigned long addr = compute_effective_address(regs, insn); int err; - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, addr); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); switch (dir) { case load: err = do_int_load(fetch_reg_addr(((insn>>25)&0x1f), @@ -338,7 +338,7 @@ asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn) } addr = compute_effective_address(regs, insn); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, addr); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); switch(dir) { case load: err = do_int_load(fetch_reg_addr(((insn>>25)&0x1f), diff --git a/arch/sparc/kernel/unaligned_64.c b/arch/sparc/kernel/unaligned_64.c index b2b019ea8caa..35cff1673aa4 100644 --- a/arch/sparc/kernel/unaligned_64.c +++ b/arch/sparc/kernel/unaligned_64.c @@ -317,7 +317,7 @@ asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) addr = compute_effective_address(regs, insn, ((insn >> 25) & 0x1f)); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, addr); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); switch (asi) { case ASI_NL: case ASI_AIUPL: @@ -384,7 +384,7 @@ int handle_popc(u32 insn, struct pt_regs *regs) int ret, i, rd = ((insn >> 25) & 0x1f); int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); if (insn & 0x2000) { maybe_flush_windows(0, 0, rd, from_kernel); value = sign_extend_imm13(insn); @@ -431,7 +431,7 @@ int handle_ldf_stq(u32 insn, struct pt_regs *regs) int asi = decode_asi(insn, regs); int flag = (freg < 32) ? FPRS_DL : FPRS_DU; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); save_and_clear_fpu(); current_thread_info()->xfsr[0] &= ~0x1c000; @@ -554,7 +554,7 @@ void handle_ld_nf(u32 insn, struct pt_regs *regs) int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; unsigned long *reg; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); maybe_flush_windows(0, 0, rd, from_kernel); reg = fetch_reg_addr(rd, regs); @@ -586,7 +586,7 @@ void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr if (tstate & TSTATE_PRIV) die_if_kernel("lddfmna from kernel", regs); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, sfar); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar); if (test_thread_flag(TIF_32BIT)) pc = (u32)pc; if (get_user(insn, (u32 __user *) pc) != -EFAULT) { @@ -647,7 +647,7 @@ void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr if (tstate & TSTATE_PRIV) die_if_kernel("stdfmna from kernel", regs); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, sfar); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar); if (test_thread_flag(TIF_32BIT)) pc = (u32)pc; if (get_user(insn, (u32 __user *) pc) != -EFAULT) { diff --git a/arch/sparc/kernel/visemul.c b/arch/sparc/kernel/visemul.c index 36357717d691..32b626c9d815 100644 --- a/arch/sparc/kernel/visemul.c +++ b/arch/sparc/kernel/visemul.c @@ -802,7 +802,7 @@ int vis_emul(struct pt_regs *regs, unsigned int insn) BUG_ON(regs->tstate & TSTATE_PRIV); - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); if (test_thread_flag(TIF_32BIT)) pc = (u32)pc; diff --git a/arch/sparc/math-emu/math_32.c b/arch/sparc/math-emu/math_32.c index a3fccde894ec..aa4d55b0bdf0 100644 --- a/arch/sparc/math-emu/math_32.c +++ b/arch/sparc/math-emu/math_32.c @@ -164,7 +164,7 @@ int do_mathemu(struct pt_regs *regs, struct task_struct *fpt) int retcode = 0; /* assume all succeed */ unsigned long insn; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); #ifdef DEBUG_MATHEMU printk("In do_mathemu()... pc is %08lx\n", regs->pc); diff --git a/arch/sparc/math-emu/math_64.c b/arch/sparc/math-emu/math_64.c index 56d2c44747b8..e575bd2fe381 100644 --- a/arch/sparc/math-emu/math_64.c +++ b/arch/sparc/math-emu/math_64.c @@ -184,7 +184,7 @@ int do_mathemu(struct pt_regs *regs, struct fpustate *f) if (tstate & TSTATE_PRIV) die_if_kernel("unfinished/unimplemented FPop from kernel", regs); - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); if (test_thread_flag(TIF_32BIT)) pc = (u32)pc; if (get_user(insn, (u32 __user *) pc) != -EFAULT) { diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index 7543ddbdadb2..aa1c1b1ce5cc 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -251,7 +251,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, if (in_atomic() || !mm) goto no_context; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); down_read(&mm->mmap_sem); @@ -301,12 +301,10 @@ good_area: } if (fault & VM_FAULT_MAJOR) { current->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, - regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); } else { current->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, - regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } up_read(&mm->mmap_sem); return; diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index f92ce56a8b22..504c0622f729 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -325,7 +325,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) if (in_atomic() || !mm) goto intr_or_no_mm; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); if (!down_read_trylock(&mm->mmap_sem)) { if ((regs->tstate & TSTATE_PRIV) && @@ -433,12 +433,10 @@ good_area: } if (fault & VM_FAULT_MAJOR) { current->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, - regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); } else { current->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, - regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } up_read(&mm->mmap_sem); diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 8a57f9aa8e36..5b86ec51534c 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -1339,7 +1339,7 @@ static int x86_pmu_handle_irq(struct pt_regs *regs) if (!x86_perf_event_set_period(event)) continue; - if (perf_event_overflow(event, 1, &data, regs)) + if (perf_event_overflow(event, &data, regs)) x86_pmu_stop(event, 0); } diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index 41178c826c48..d38b0020f775 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -1003,7 +1003,7 @@ again: data.period = event->hw.last_period; - if (perf_event_overflow(event, 1, &data, regs)) + if (perf_event_overflow(event, &data, regs)) x86_pmu_stop(event, 0); } diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c index bab491b8ee25..0941f93f2940 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_ds.c +++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c @@ -340,7 +340,7 @@ static int intel_pmu_drain_bts_buffer(void) */ perf_prepare_sample(&header, &data, event, ®s); - if (perf_output_begin(&handle, event, header.size * (top - at), 1, 1)) + if (perf_output_begin(&handle, event, header.size * (top - at), 1)) return 1; for (; at < top; at++) { @@ -616,7 +616,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event, else regs.flags &= ~PERF_EFLAGS_EXACT; - if (perf_event_overflow(event, 1, &data, ®s)) + if (perf_event_overflow(event, &data, ®s)) x86_pmu_stop(event, 0); } diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c index f76fddf63381..d6e6a67b9608 100644 --- a/arch/x86/kernel/cpu/perf_event_p4.c +++ b/arch/x86/kernel/cpu/perf_event_p4.c @@ -970,7 +970,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) if (!x86_perf_event_set_period(event)) continue; - if (perf_event_overflow(event, 1, &data, regs)) + if (perf_event_overflow(event, &data, regs)) x86_pmu_stop(event, 0); } diff --git a/arch/x86/kernel/kgdb.c b/arch/x86/kernel/kgdb.c index 5f9ecff328b5..98da6a7b5e82 100644 --- a/arch/x86/kernel/kgdb.c +++ b/arch/x86/kernel/kgdb.c @@ -608,7 +608,7 @@ int kgdb_arch_init(void) return register_die_notifier(&kgdb_notifier); } -static void kgdb_hw_overflow_handler(struct perf_event *event, int nmi, +static void kgdb_hw_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) { struct task_struct *tsk = current; diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index 807c2a2b80f1..11db2e9b860a 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c @@ -528,7 +528,7 @@ static int genregs_set(struct task_struct *target, return ret; } -static void ptrace_triggered(struct perf_event *bp, int nmi, +static void ptrace_triggered(struct perf_event *bp, struct perf_sample_data *data, struct pt_regs *regs) { diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 2dbf6bf4c7e5..4d09df054e39 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1059,7 +1059,7 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code) if (unlikely(error_code & PF_RSVD)) pgtable_bad(regs, error_code, address); - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); /* * If we're in an interrupt, have no user context or are running @@ -1161,11 +1161,11 @@ good_area: if (flags & FAULT_FLAG_ALLOW_RETRY) { if (fault & VM_FAULT_MAJOR) { tsk->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address); } else { tsk->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } if (fault & VM_FAULT_RETRY) { diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 2f7b5d42ab41..0946a8bc098d 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -682,7 +682,7 @@ enum perf_event_active_state { struct file; struct perf_sample_data; -typedef void (*perf_overflow_handler_t)(struct perf_event *, int, +typedef void (*perf_overflow_handler_t)(struct perf_event *, struct perf_sample_data *, struct pt_regs *regs); @@ -925,7 +925,6 @@ struct perf_output_handle { unsigned long size; void *addr; int page; - int nmi; int sample; }; @@ -993,7 +992,7 @@ extern void perf_prepare_sample(struct perf_event_header *header, struct perf_event *event, struct pt_regs *regs); -extern int perf_event_overflow(struct perf_event *event, int nmi, +extern int perf_event_overflow(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs); @@ -1012,7 +1011,7 @@ static inline int is_software_event(struct perf_event *event) extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; -extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64); +extern void __perf_sw_event(u32, u64, struct pt_regs *, u64); #ifndef perf_arch_fetch_caller_regs static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { } @@ -1034,7 +1033,7 @@ static inline void perf_fetch_caller_regs(struct pt_regs *regs) } static __always_inline void -perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr) +perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { struct pt_regs hot_regs; @@ -1043,7 +1042,7 @@ perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr) perf_fetch_caller_regs(&hot_regs); regs = &hot_regs; } - __perf_sw_event(event_id, nr, nmi, regs, addr); + __perf_sw_event(event_id, nr, regs, addr); } } @@ -1057,7 +1056,7 @@ static inline void perf_event_task_sched_in(struct task_struct *task) static inline void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next) { - perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0); + perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0); __perf_event_task_sched_out(task, next); } @@ -1119,7 +1118,7 @@ extern void perf_bp_event(struct perf_event *event, void *data); extern int perf_output_begin(struct perf_output_handle *handle, struct perf_event *event, unsigned int size, - int nmi, int sample); + int sample); extern void perf_output_end(struct perf_output_handle *handle); extern void perf_output_copy(struct perf_output_handle *handle, const void *buf, unsigned int len); @@ -1143,8 +1142,7 @@ static inline int perf_event_task_disable(void) { return -EINVAL; } static inline int perf_event_task_enable(void) { return -EINVAL; } static inline void -perf_sw_event(u32 event_id, u64 nr, int nmi, - struct pt_regs *regs, u64 addr) { } +perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { } static inline void perf_bp_event(struct perf_event *event, void *data) { } diff --git a/kernel/events/core.c b/kernel/events/core.c index 270e32f9fc06..dbd1ca75bd3c 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3972,7 +3972,7 @@ void perf_prepare_sample(struct perf_event_header *header, } } -static void perf_event_output(struct perf_event *event, int nmi, +static void perf_event_output(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) { @@ -3984,7 +3984,7 @@ static void perf_event_output(struct perf_event *event, int nmi, perf_prepare_sample(&header, data, event, regs); - if (perf_output_begin(&handle, event, header.size, nmi, 1)) + if (perf_output_begin(&handle, event, header.size, 1)) goto exit; perf_output_sample(&handle, &header, data, event); @@ -4024,7 +4024,7 @@ perf_event_read_event(struct perf_event *event, int ret; perf_event_header__init_id(&read_event.header, &sample, event); - ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0); + ret = perf_output_begin(&handle, event, read_event.header.size, 0); if (ret) return; @@ -4067,7 +4067,7 @@ static void perf_event_task_output(struct perf_event *event, perf_event_header__init_id(&task_event->event_id.header, &sample, event); ret = perf_output_begin(&handle, event, - task_event->event_id.header.size, 0, 0); + task_event->event_id.header.size, 0); if (ret) goto out; @@ -4204,7 +4204,7 @@ static void perf_event_comm_output(struct perf_event *event, perf_event_header__init_id(&comm_event->event_id.header, &sample, event); ret = perf_output_begin(&handle, event, - comm_event->event_id.header.size, 0, 0); + comm_event->event_id.header.size, 0); if (ret) goto out; @@ -4351,7 +4351,7 @@ static void perf_event_mmap_output(struct perf_event *event, perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); ret = perf_output_begin(&handle, event, - mmap_event->event_id.header.size, 0, 0); + mmap_event->event_id.header.size, 0); if (ret) goto out; @@ -4546,7 +4546,7 @@ static void perf_log_throttle(struct perf_event *event, int enable) perf_event_header__init_id(&throttle_event.header, &sample, event); ret = perf_output_begin(&handle, event, - throttle_event.header.size, 1, 0); + throttle_event.header.size, 0); if (ret) return; @@ -4559,7 +4559,7 @@ static void perf_log_throttle(struct perf_event *event, int enable) * Generic event overflow handling, sampling. */ -static int __perf_event_overflow(struct perf_event *event, int nmi, +static int __perf_event_overflow(struct perf_event *event, int throttle, struct perf_sample_data *data, struct pt_regs *regs) { @@ -4602,34 +4602,28 @@ static int __perf_event_overflow(struct perf_event *event, int nmi, if (events && atomic_dec_and_test(&event->event_limit)) { ret = 1; event->pending_kill = POLL_HUP; - if (nmi) { - event->pending_disable = 1; - irq_work_queue(&event->pending); - } else - perf_event_disable(event); + event->pending_disable = 1; + irq_work_queue(&event->pending); } if (event->overflow_handler) - event->overflow_handler(event, nmi, data, regs); + event->overflow_handler(event, data, regs); else - perf_event_output(event, nmi, data, regs); + perf_event_output(event, data, regs); if (event->fasync && event->pending_kill) { - if (nmi) { - event->pending_wakeup = 1; - irq_work_queue(&event->pending); - } else - perf_event_wakeup(event); + event->pending_wakeup = 1; + irq_work_queue(&event->pending); } return ret; } -int perf_event_overflow(struct perf_event *event, int nmi, +int perf_event_overflow(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) { - return __perf_event_overflow(event, nmi, 1, data, regs); + return __perf_event_overflow(event, 1, data, regs); } /* @@ -4678,7 +4672,7 @@ again: } static void perf_swevent_overflow(struct perf_event *event, u64 overflow, - int nmi, struct perf_sample_data *data, + struct perf_sample_data *data, struct pt_regs *regs) { struct hw_perf_event *hwc = &event->hw; @@ -4692,7 +4686,7 @@ static void perf_swevent_overflow(struct perf_event *event, u64 overflow, return; for (; overflow; overflow--) { - if (__perf_event_overflow(event, nmi, throttle, + if (__perf_event_overflow(event, throttle, data, regs)) { /* * We inhibit the overflow from happening when @@ -4705,7 +4699,7 @@ static void perf_swevent_overflow(struct perf_event *event, u64 overflow, } static void perf_swevent_event(struct perf_event *event, u64 nr, - int nmi, struct perf_sample_data *data, + struct perf_sample_data *data, struct pt_regs *regs) { struct hw_perf_event *hwc = &event->hw; @@ -4719,12 +4713,12 @@ static void perf_swevent_event(struct perf_event *event, u64 nr, return; if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) - return perf_swevent_overflow(event, 1, nmi, data, regs); + return perf_swevent_overflow(event, 1, data, regs); if (local64_add_negative(nr, &hwc->period_left)) return; - perf_swevent_overflow(event, 0, nmi, data, regs); + perf_swevent_overflow(event, 0, data, regs); } static int perf_exclude_event(struct perf_event *event, @@ -4812,7 +4806,7 @@ find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) } static void do_perf_sw_event(enum perf_type_id type, u32 event_id, - u64 nr, int nmi, + u64 nr, struct perf_sample_data *data, struct pt_regs *regs) { @@ -4828,7 +4822,7 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id, hlist_for_each_entry_rcu(event, node, head, hlist_entry) { if (perf_swevent_match(event, type, event_id, data, regs)) - perf_swevent_event(event, nr, nmi, data, regs); + perf_swevent_event(event, nr, data, regs); } end: rcu_read_unlock(); @@ -4849,8 +4843,7 @@ inline void perf_swevent_put_recursion_context(int rctx) put_recursion_context(swhash->recursion, rctx); } -void __perf_sw_event(u32 event_id, u64 nr, int nmi, - struct pt_regs *regs, u64 addr) +void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { struct perf_sample_data data; int rctx; @@ -4862,7 +4855,7 @@ void __perf_sw_event(u32 event_id, u64 nr, int nmi, perf_sample_data_init(&data, addr); - do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs); + do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); perf_swevent_put_recursion_context(rctx); preempt_enable_notrace(); @@ -5110,7 +5103,7 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, hlist_for_each_entry_rcu(event, node, head, hlist_entry) { if (perf_tp_event_match(event, &data, regs)) - perf_swevent_event(event, count, 1, &data, regs); + perf_swevent_event(event, count, &data, regs); } perf_swevent_put_recursion_context(rctx); @@ -5203,7 +5196,7 @@ void perf_bp_event(struct perf_event *bp, void *data) perf_sample_data_init(&sample, bp->attr.bp_addr); if (!bp->hw.state && !perf_exclude_event(bp, regs)) - perf_swevent_event(bp, 1, 1, &sample, regs); + perf_swevent_event(bp, 1, &sample, regs); } #endif @@ -5232,7 +5225,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) if (regs && !perf_exclude_event(event, regs)) { if (!(event->attr.exclude_idle && current->pid == 0)) - if (perf_event_overflow(event, 0, &data, regs)) + if (perf_event_overflow(event, &data, regs)) ret = HRTIMER_NORESTART; } diff --git a/kernel/events/internal.h b/kernel/events/internal.h index 114f27f3a624..09097dd8116c 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h @@ -27,7 +27,6 @@ struct ring_buffer { void *data_pages[0]; }; - extern void rb_free(struct ring_buffer *rb); extern struct ring_buffer * rb_alloc(int nr_pages, long watermark, int cpu, int flags); diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index fc2701c99207..8b3b73630fa4 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -38,11 +38,8 @@ static void perf_output_wakeup(struct perf_output_handle *handle) { atomic_set(&handle->rb->poll, POLL_IN); - if (handle->nmi) { - handle->event->pending_wakeup = 1; - irq_work_queue(&handle->event->pending); - } else - perf_event_wakeup(handle->event); + handle->event->pending_wakeup = 1; + irq_work_queue(&handle->event->pending); } /* @@ -102,7 +99,7 @@ out: int perf_output_begin(struct perf_output_handle *handle, struct perf_event *event, unsigned int size, - int nmi, int sample) + int sample) { struct ring_buffer *rb; unsigned long tail, offset, head; @@ -127,7 +124,6 @@ int perf_output_begin(struct perf_output_handle *handle, handle->rb = rb; handle->event = event; - handle->nmi = nmi; handle->sample = sample; if (!rb->nr_pages) diff --git a/kernel/sched.c b/kernel/sched.c index 3f2e502d609b..d08d110b8976 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -2220,7 +2220,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) if (task_cpu(p) != new_cpu) { p->se.nr_migrations++; - perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0); + perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0); } __set_task_cpu(p, new_cpu); diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 752b75ba662b..a6708e677a0a 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -211,7 +211,7 @@ static struct perf_event_attr wd_hw_attr = { }; /* Callback function for perf event subsystem */ -static void watchdog_overflow_callback(struct perf_event *event, int nmi, +static void watchdog_overflow_callback(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) { diff --git a/samples/hw_breakpoint/data_breakpoint.c b/samples/hw_breakpoint/data_breakpoint.c index 063653955f9f..7b164d3200ff 100644 --- a/samples/hw_breakpoint/data_breakpoint.c +++ b/samples/hw_breakpoint/data_breakpoint.c @@ -41,7 +41,7 @@ module_param_string(ksym, ksym_name, KSYM_NAME_LEN, S_IRUGO); MODULE_PARM_DESC(ksym, "Kernel symbol to monitor; this module will report any" " write operations on the kernel symbol"); -static void sample_hbp_handler(struct perf_event *bp, int nmi, +static void sample_hbp_handler(struct perf_event *bp, struct perf_sample_data *data, struct pt_regs *regs) { -- cgit v1.2.3 From a7ac67ea021b4603095d2aa458bc41641238f22c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 27 Jun 2011 16:47:16 +0200 Subject: perf: Remove the perf_output_begin(.sample) argument Since only samples call perf_output_sample() its much saner (and more correct) to put the sample logic in there than in the perf_output_begin()/perf_output_end() pair. Saves a useless argument, reduces conditionals and shrinks struct perf_output_handle, win! Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/n/tip-2crpvsx3cqu67q3zqjbnlpsc@git.kernel.org Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/perf_event_intel_ds.c | 2 +- include/linux/perf_event.h | 4 +--- kernel/events/core.c | 26 ++++++++++++++++++++------ kernel/events/ring_buffer.c | 19 +------------------ 4 files changed, 23 insertions(+), 28 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c index 0941f93f2940..1b1ef3addcfd 100644 --- a/arch/x86/kernel/cpu/perf_event_intel_ds.c +++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c @@ -340,7 +340,7 @@ static int intel_pmu_drain_bts_buffer(void) */ perf_prepare_sample(&header, &data, event, ®s); - if (perf_output_begin(&handle, event, header.size * (top - at), 1)) + if (perf_output_begin(&handle, event, header.size * (top - at))) return 1; for (; at < top; at++) { diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 0946a8bc098d..771b0b2845e4 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -925,7 +925,6 @@ struct perf_output_handle { unsigned long size; void *addr; int page; - int sample; }; #ifdef CONFIG_PERF_EVENTS @@ -1117,8 +1116,7 @@ extern void perf_bp_event(struct perf_event *event, void *data); #endif extern int perf_output_begin(struct perf_output_handle *handle, - struct perf_event *event, unsigned int size, - int sample); + struct perf_event *event, unsigned int size); extern void perf_output_end(struct perf_output_handle *handle); extern void perf_output_copy(struct perf_output_handle *handle, const void *buf, unsigned int len); diff --git a/kernel/events/core.c b/kernel/events/core.c index dbd1ca75bd3c..81de28dcca8c 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3928,6 +3928,20 @@ void perf_output_sample(struct perf_output_handle *handle, perf_output_put(handle, raw); } } + + if (!event->attr.watermark) { + int wakeup_events = event->attr.wakeup_events; + + if (wakeup_events) { + struct ring_buffer *rb = handle->rb; + int events = local_inc_return(&rb->events); + + if (events >= wakeup_events) { + local_sub(wakeup_events, &rb->events); + local_inc(&rb->wakeup); + } + } + } } void perf_prepare_sample(struct perf_event_header *header, @@ -3984,7 +3998,7 @@ static void perf_event_output(struct perf_event *event, perf_prepare_sample(&header, data, event, regs); - if (perf_output_begin(&handle, event, header.size, 1)) + if (perf_output_begin(&handle, event, header.size)) goto exit; perf_output_sample(&handle, &header, data, event); @@ -4024,7 +4038,7 @@ perf_event_read_event(struct perf_event *event, int ret; perf_event_header__init_id(&read_event.header, &sample, event); - ret = perf_output_begin(&handle, event, read_event.header.size, 0); + ret = perf_output_begin(&handle, event, read_event.header.size); if (ret) return; @@ -4067,7 +4081,7 @@ static void perf_event_task_output(struct perf_event *event, perf_event_header__init_id(&task_event->event_id.header, &sample, event); ret = perf_output_begin(&handle, event, - task_event->event_id.header.size, 0); + task_event->event_id.header.size); if (ret) goto out; @@ -4204,7 +4218,7 @@ static void perf_event_comm_output(struct perf_event *event, perf_event_header__init_id(&comm_event->event_id.header, &sample, event); ret = perf_output_begin(&handle, event, - comm_event->event_id.header.size, 0); + comm_event->event_id.header.size); if (ret) goto out; @@ -4351,7 +4365,7 @@ static void perf_event_mmap_output(struct perf_event *event, perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); ret = perf_output_begin(&handle, event, - mmap_event->event_id.header.size, 0); + mmap_event->event_id.header.size); if (ret) goto out; @@ -4546,7 +4560,7 @@ static void perf_log_throttle(struct perf_event *event, int enable) perf_event_header__init_id(&throttle_event.header, &sample, event); ret = perf_output_begin(&handle, event, - throttle_event.header.size, 0); + throttle_event.header.size); if (ret) return; diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 8b3b73630fa4..a2a29205cc0f 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -98,8 +98,7 @@ out: } int perf_output_begin(struct perf_output_handle *handle, - struct perf_event *event, unsigned int size, - int sample) + struct perf_event *event, unsigned int size) { struct ring_buffer *rb; unsigned long tail, offset, head; @@ -124,7 +123,6 @@ int perf_output_begin(struct perf_output_handle *handle, handle->rb = rb; handle->event = event; - handle->sample = sample; if (!rb->nr_pages) goto out; @@ -192,21 +190,6 @@ void perf_output_copy(struct perf_output_handle *handle, void perf_output_end(struct perf_output_handle *handle) { - struct perf_event *event = handle->event; - struct ring_buffer *rb = handle->rb; - - if (handle->sample && !event->attr.watermark) { - int wakeup_events = event->attr.wakeup_events; - - if (wakeup_events) { - int events = local_inc_return(&rb->events); - if (events >= wakeup_events) { - local_sub(wakeup_events, &rb->events); - local_inc(&rb->wakeup); - } - } - } - perf_output_put_handle(handle); rcu_read_unlock(); } -- cgit v1.2.3 From efc9f05df2dd171280dcb736a4d973ffefd5508e Mon Sep 17 00:00:00 2001 From: Stephane Eranian Date: Mon, 6 Jun 2011 16:57:03 +0200 Subject: perf_events: Update Intel extra regs shared constraints management This patch improves the code managing the extra shared registers used for offcore_response events on Intel Nehalem/Westmere. The idea is to use static allocation instead of dynamic allocation. This simplifies greatly the get and put constraint routines for those events. The patch also renames per_core to shared_regs because the same data structure gets used whether or not HT is on. When HT is off, those events still need to coordination because they use a extra MSR that has to be shared within an event group. Signed-off-by: Stephane Eranian Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/20110606145703.GA7258@quad Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/perf_event.c | 78 +++++++--- arch/x86/kernel/cpu/perf_event_intel.c | 260 ++++++++++++++++----------------- include/linux/perf_event.h | 14 +- 3 files changed, 200 insertions(+), 152 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 5b86ec51534c..019fda7489e7 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -44,6 +44,29 @@ do { \ } while (0) #endif +/* + * | NHM/WSM | SNB | + * register ------------------------------- + * | HT | no HT | HT | no HT | + *----------------------------------------- + * offcore | core | core | cpu | core | + * lbr_sel | core | core | cpu | core | + * ld_lat | cpu | core | cpu | core | + *----------------------------------------- + * + * Given that there is a small number of shared regs, + * we can pre-allocate their slot in the per-cpu + * per-core reg tables. + */ +enum extra_reg_type { + EXTRA_REG_NONE = -1, /* not used */ + + EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */ + EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */ + + EXTRA_REG_MAX /* number of entries needed */ +}; + /* * best effort, GUP based copy_from_user() that assumes IRQ or NMI context */ @@ -132,11 +155,10 @@ struct cpu_hw_events { struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES]; /* - * Intel percore register state. - * Coordinate shared resources between HT threads. + * manage shared (per-core, per-cpu) registers + * used on Intel NHM/WSM/SNB */ - int percore_used; /* Used by this CPU? */ - struct intel_percore *per_core; + struct intel_shared_regs *shared_regs; /* * AMD specific bits @@ -186,27 +208,46 @@ struct cpu_hw_events { #define for_each_event_constraint(e, c) \ for ((e) = (c); (e)->weight; (e)++) +/* + * Per register state. + */ +struct er_account { + raw_spinlock_t lock; /* per-core: protect structure */ + u64 config; /* extra MSR config */ + u64 reg; /* extra MSR number */ + atomic_t ref; /* reference count */ +}; + /* * Extra registers for specific events. + * * Some events need large masks and require external MSRs. - * Define a mapping to these extra registers. + * Those extra MSRs end up being shared for all events on + * a PMU and sometimes between PMU of sibling HT threads. + * In either case, the kernel needs to handle conflicting + * accesses to those extra, shared, regs. The data structure + * to manage those registers is stored in cpu_hw_event. */ struct extra_reg { unsigned int event; unsigned int msr; u64 config_mask; u64 valid_mask; + int idx; /* per_xxx->regs[] reg index */ }; -#define EVENT_EXTRA_REG(e, ms, m, vm) { \ +#define EVENT_EXTRA_REG(e, ms, m, vm, i) { \ .event = (e), \ .msr = (ms), \ .config_mask = (m), \ .valid_mask = (vm), \ + .idx = EXTRA_REG_##i \ } -#define INTEL_EVENT_EXTRA_REG(event, msr, vm) \ - EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm) -#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0) + +#define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \ + EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx) + +#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0) union perf_capabilities { struct { @@ -253,7 +294,6 @@ struct x86_pmu { void (*put_event_constraints)(struct cpu_hw_events *cpuc, struct perf_event *event); struct event_constraint *event_constraints; - struct event_constraint *percore_constraints; void (*quirks)(void); int perfctr_second_write; @@ -400,10 +440,10 @@ static inline unsigned int x86_pmu_event_addr(int index) */ static int x86_pmu_extra_regs(u64 config, struct perf_event *event) { + struct hw_perf_event_extra *reg; struct extra_reg *er; - event->hw.extra_reg = 0; - event->hw.extra_config = 0; + reg = &event->hw.extra_reg; if (!x86_pmu.extra_regs) return 0; @@ -413,8 +453,10 @@ static int x86_pmu_extra_regs(u64 config, struct perf_event *event) continue; if (event->attr.config1 & ~er->valid_mask) return -EINVAL; - event->hw.extra_reg = er->msr; - event->hw.extra_config = event->attr.config1; + + reg->idx = er->idx; + reg->config = event->attr.config1; + reg->reg = er->msr; break; } return 0; @@ -713,6 +755,9 @@ static int __x86_pmu_event_init(struct perf_event *event) event->hw.last_cpu = -1; event->hw.last_tag = ~0ULL; + /* mark unused */ + event->hw.extra_reg.idx = EXTRA_REG_NONE; + return x86_pmu.hw_config(event); } @@ -754,8 +799,8 @@ static void x86_pmu_disable(struct pmu *pmu) static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc, u64 enable_mask) { - if (hwc->extra_reg) - wrmsrl(hwc->extra_reg, hwc->extra_config); + if (hwc->extra_reg.reg) + wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config); wrmsrl(hwc->config_base, hwc->config | enable_mask); } @@ -1692,7 +1737,6 @@ static int validate_group(struct perf_event *event) fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO); if (!fake_cpuc) goto out; - /* * the event is not yet connected with its * siblings therefore we must first collect diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index d38b0020f775..6ad95baff856 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -1,25 +1,15 @@ #ifdef CONFIG_CPU_SUP_INTEL -#define MAX_EXTRA_REGS 2 - -/* - * Per register state. - */ -struct er_account { - int ref; /* reference count */ - unsigned int extra_reg; /* extra MSR number */ - u64 extra_config; /* extra MSR config */ -}; - /* - * Per core state - * This used to coordinate shared registers for HT threads. + * Per core/cpu state + * + * Used to coordinate shared registers between HT threads or + * among events on a single PMU. */ -struct intel_percore { - raw_spinlock_t lock; /* protect structure */ - struct er_account regs[MAX_EXTRA_REGS]; - int refcnt; /* number of threads */ - unsigned core_id; +struct intel_shared_regs { + struct er_account regs[EXTRA_REG_MAX]; + int refcnt; /* per-core: #HT threads */ + unsigned core_id; /* per-core: core id */ }; /* @@ -88,16 +78,10 @@ static struct event_constraint intel_nehalem_event_constraints[] __read_mostly = static struct extra_reg intel_nehalem_extra_regs[] __read_mostly = { - INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff), + INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), EVENT_EXTRA_END }; -static struct event_constraint intel_nehalem_percore_constraints[] __read_mostly = -{ - INTEL_EVENT_CONSTRAINT(0xb7, 0), - EVENT_CONSTRAINT_END -}; - static struct event_constraint intel_westmere_event_constraints[] __read_mostly = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ @@ -125,18 +109,11 @@ static struct event_constraint intel_snb_event_constraints[] __read_mostly = static struct extra_reg intel_westmere_extra_regs[] __read_mostly = { - INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff), - INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff), + INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), + INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1), EVENT_EXTRA_END }; -static struct event_constraint intel_westmere_percore_constraints[] __read_mostly = -{ - INTEL_EVENT_CONSTRAINT(0xb7, 0), - INTEL_EVENT_CONSTRAINT(0xbb, 0), - EVENT_CONSTRAINT_END -}; - static struct event_constraint intel_gen_event_constraints[] __read_mostly = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ @@ -1037,65 +1014,89 @@ intel_bts_constraints(struct perf_event *event) return NULL; } +/* + * manage allocation of shared extra msr for certain events + * + * sharing can be: + * per-cpu: to be shared between the various events on a single PMU + * per-core: per-cpu + shared by HT threads + */ static struct event_constraint * -intel_percore_constraints(struct cpu_hw_events *cpuc, struct perf_event *event) +__intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc, + struct hw_perf_event_extra *reg) { - struct hw_perf_event *hwc = &event->hw; - unsigned int e = hwc->config & ARCH_PERFMON_EVENTSEL_EVENT; - struct event_constraint *c; - struct intel_percore *pc; + struct event_constraint *c = &emptyconstraint; struct er_account *era; - int i; - int free_slot; - int found; - if (!x86_pmu.percore_constraints || hwc->extra_alloc) - return NULL; + /* already allocated shared msr */ + if (reg->alloc || !cpuc->shared_regs) + return &unconstrained; - for (c = x86_pmu.percore_constraints; c->cmask; c++) { - if (e != c->code) - continue; + era = &cpuc->shared_regs->regs[reg->idx]; + + raw_spin_lock(&era->lock); + + if (!atomic_read(&era->ref) || era->config == reg->config) { + + /* lock in msr value */ + era->config = reg->config; + era->reg = reg->reg; + + /* one more user */ + atomic_inc(&era->ref); + + /* no need to reallocate during incremental event scheduling */ + reg->alloc = 1; /* - * Allocate resource per core. + * All events using extra_reg are unconstrained. + * Avoids calling x86_get_event_constraints() + * + * Must revisit if extra_reg controlling events + * ever have constraints. Worst case we go through + * the regular event constraint table. */ - pc = cpuc->per_core; - if (!pc) - break; - c = &emptyconstraint; - raw_spin_lock(&pc->lock); - free_slot = -1; - found = 0; - for (i = 0; i < MAX_EXTRA_REGS; i++) { - era = &pc->regs[i]; - if (era->ref > 0 && hwc->extra_reg == era->extra_reg) { - /* Allow sharing same config */ - if (hwc->extra_config == era->extra_config) { - era->ref++; - cpuc->percore_used = 1; - hwc->extra_alloc = 1; - c = NULL; - } - /* else conflict */ - found = 1; - break; - } else if (era->ref == 0 && free_slot == -1) - free_slot = i; - } - if (!found && free_slot != -1) { - era = &pc->regs[free_slot]; - era->ref = 1; - era->extra_reg = hwc->extra_reg; - era->extra_config = hwc->extra_config; - cpuc->percore_used = 1; - hwc->extra_alloc = 1; - c = NULL; - } - raw_spin_unlock(&pc->lock); - return c; + c = &unconstrained; } + raw_spin_unlock(&era->lock); - return NULL; + return c; +} + +static void +__intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc, + struct hw_perf_event_extra *reg) +{ + struct er_account *era; + + /* + * only put constraint if extra reg was actually + * allocated. Also takes care of event which do + * not use an extra shared reg + */ + if (!reg->alloc) + return; + + era = &cpuc->shared_regs->regs[reg->idx]; + + /* one fewer user */ + atomic_dec(&era->ref); + + /* allocate again next time */ + reg->alloc = 0; +} + +static struct event_constraint * +intel_shared_regs_constraints(struct cpu_hw_events *cpuc, + struct perf_event *event) +{ + struct event_constraint *c = NULL; + struct hw_perf_event_extra *xreg; + + xreg = &event->hw.extra_reg; + if (xreg->idx != EXTRA_REG_NONE) + c = __intel_shared_reg_get_constraints(cpuc, xreg); + return c; } static struct event_constraint * @@ -1111,49 +1112,28 @@ intel_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event if (c) return c; - c = intel_percore_constraints(cpuc, event); + c = intel_shared_regs_constraints(cpuc, event); if (c) return c; return x86_get_event_constraints(cpuc, event); } -static void intel_put_event_constraints(struct cpu_hw_events *cpuc, +static void +intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event) { - struct extra_reg *er; - struct intel_percore *pc; - struct er_account *era; - struct hw_perf_event *hwc = &event->hw; - int i, allref; - - if (!cpuc->percore_used) - return; + struct hw_perf_event_extra *reg; - for (er = x86_pmu.extra_regs; er->msr; er++) { - if (er->event != (hwc->config & er->config_mask)) - continue; + reg = &event->hw.extra_reg; + if (reg->idx != EXTRA_REG_NONE) + __intel_shared_reg_put_constraints(cpuc, reg); +} - pc = cpuc->per_core; - raw_spin_lock(&pc->lock); - for (i = 0; i < MAX_EXTRA_REGS; i++) { - era = &pc->regs[i]; - if (era->ref > 0 && - era->extra_config == hwc->extra_config && - era->extra_reg == er->msr) { - era->ref--; - hwc->extra_alloc = 0; - break; - } - } - allref = 0; - for (i = 0; i < MAX_EXTRA_REGS; i++) - allref += pc->regs[i].ref; - if (allref == 0) - cpuc->percore_used = 0; - raw_spin_unlock(&pc->lock); - break; - } +static void intel_put_event_constraints(struct cpu_hw_events *cpuc, + struct perf_event *event) +{ + intel_put_shared_regs_event_constraints(cpuc, event); } static int intel_pmu_hw_config(struct perf_event *event) @@ -1231,20 +1211,36 @@ static __initconst const struct x86_pmu core_pmu = { .event_constraints = intel_core_event_constraints, }; +static struct intel_shared_regs *allocate_shared_regs(int cpu) +{ + struct intel_shared_regs *regs; + int i; + + regs = kzalloc_node(sizeof(struct intel_shared_regs), + GFP_KERNEL, cpu_to_node(cpu)); + if (regs) { + /* + * initialize the locks to keep lockdep happy + */ + for (i = 0; i < EXTRA_REG_MAX; i++) + raw_spin_lock_init(®s->regs[i].lock); + + regs->core_id = -1; + } + return regs; +} + static int intel_pmu_cpu_prepare(int cpu) { struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); - if (!cpu_has_ht_siblings()) + if (!x86_pmu.extra_regs) return NOTIFY_OK; - cpuc->per_core = kzalloc_node(sizeof(struct intel_percore), - GFP_KERNEL, cpu_to_node(cpu)); - if (!cpuc->per_core) + cpuc->shared_regs = allocate_shared_regs(cpu); + if (!cpuc->shared_regs) return NOTIFY_BAD; - raw_spin_lock_init(&cpuc->per_core->lock); - cpuc->per_core->core_id = -1; return NOTIFY_OK; } @@ -1260,32 +1256,34 @@ static void intel_pmu_cpu_starting(int cpu) */ intel_pmu_lbr_reset(); - if (!cpu_has_ht_siblings()) + if (!cpuc->shared_regs) return; for_each_cpu(i, topology_thread_cpumask(cpu)) { - struct intel_percore *pc = per_cpu(cpu_hw_events, i).per_core; + struct intel_shared_regs *pc; + pc = per_cpu(cpu_hw_events, i).shared_regs; if (pc && pc->core_id == core_id) { - kfree(cpuc->per_core); - cpuc->per_core = pc; + kfree(cpuc->shared_regs); + cpuc->shared_regs = pc; break; } } - cpuc->per_core->core_id = core_id; - cpuc->per_core->refcnt++; + cpuc->shared_regs->core_id = core_id; + cpuc->shared_regs->refcnt++; } static void intel_pmu_cpu_dying(int cpu) { struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); - struct intel_percore *pc = cpuc->per_core; + struct intel_shared_regs *pc; + pc = cpuc->shared_regs; if (pc) { if (pc->core_id == -1 || --pc->refcnt == 0) kfree(pc); - cpuc->per_core = NULL; + cpuc->shared_regs = NULL; } fini_debug_store_on_cpu(cpu); @@ -1436,7 +1434,6 @@ static __init int intel_pmu_init(void) x86_pmu.event_constraints = intel_nehalem_event_constraints; x86_pmu.pebs_constraints = intel_nehalem_pebs_event_constraints; - x86_pmu.percore_constraints = intel_nehalem_percore_constraints; x86_pmu.enable_all = intel_pmu_nhm_enable_all; x86_pmu.extra_regs = intel_nehalem_extra_regs; @@ -1481,7 +1478,6 @@ static __init int intel_pmu_init(void) intel_pmu_lbr_init_nhm(); x86_pmu.event_constraints = intel_westmere_event_constraints; - x86_pmu.percore_constraints = intel_westmere_percore_constraints; x86_pmu.enable_all = intel_pmu_nhm_enable_all; x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints; x86_pmu.extra_regs = intel_westmere_extra_regs; diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 771b0b2845e4..069315eefb22 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -536,6 +536,16 @@ struct perf_branch_stack { struct task_struct; +/* + * extra PMU register associated with an event + */ +struct hw_perf_event_extra { + u64 config; /* register value */ + unsigned int reg; /* register address or index */ + int alloc; /* extra register already allocated */ + int idx; /* index in shared_regs->regs[] */ +}; + /** * struct hw_perf_event - performance event hardware details: */ @@ -549,9 +559,7 @@ struct hw_perf_event { unsigned long event_base; int idx; int last_cpu; - unsigned int extra_reg; - u64 extra_config; - int extra_alloc; + struct hw_perf_event_extra extra_reg; }; struct { /* software */ struct hrtimer hrtimer; -- cgit v1.2.3 From cd8a38d33e2528998998bae70a45ad27e442f114 Mon Sep 17 00:00:00 2001 From: Stephane Eranian Date: Mon, 6 Jun 2011 16:57:08 +0200 Subject: perf_events: Fix validation of events using an extra reg The validate_group() function needs to validate events with extra shared regs. Within an event group, only events with the same value for the extra reg can co-exist. This was not checked by validate_group() because it was missing the shared_regs logic. This patch changes the allocation of the fake cpuc used for validation to also point to a fake shared_regs structure such that group events be properly testing. It modifies __intel_shared_reg_get_constraints() to use spin_lock_irqsave() to avoid lockdep issues. Signed-off-by: Stephane Eranian Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/20110606145708.GA7279@quad Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/perf_event.c | 59 ++++++++++++++++++++++++++-------- arch/x86/kernel/cpu/perf_event_intel.c | 16 ++++++--- 2 files changed, 57 insertions(+), 18 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 019fda7489e7..9a0f55c99b6e 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -1689,6 +1689,40 @@ static int x86_pmu_commit_txn(struct pmu *pmu) perf_pmu_enable(pmu); return 0; } +/* + * a fake_cpuc is used to validate event groups. Due to + * the extra reg logic, we need to also allocate a fake + * per_core and per_cpu structure. Otherwise, group events + * using extra reg may conflict without the kernel being + * able to catch this when the last event gets added to + * the group. + */ +static void free_fake_cpuc(struct cpu_hw_events *cpuc) +{ + kfree(cpuc->shared_regs); + kfree(cpuc); +} + +static struct cpu_hw_events *allocate_fake_cpuc(void) +{ + struct cpu_hw_events *cpuc; + int cpu = raw_smp_processor_id(); + + cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); + if (!cpuc) + return ERR_PTR(-ENOMEM); + + /* only needed, if we have extra_regs */ + if (x86_pmu.extra_regs) { + cpuc->shared_regs = allocate_shared_regs(cpu); + if (!cpuc->shared_regs) + goto error; + } + return cpuc; +error: + free_fake_cpuc(cpuc); + return ERR_PTR(-ENOMEM); +} /* * validate that we can schedule this event @@ -1699,9 +1733,9 @@ static int validate_event(struct perf_event *event) struct event_constraint *c; int ret = 0; - fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO); - if (!fake_cpuc) - return -ENOMEM; + fake_cpuc = allocate_fake_cpuc(); + if (IS_ERR(fake_cpuc)) + return PTR_ERR(fake_cpuc); c = x86_pmu.get_event_constraints(fake_cpuc, event); @@ -1711,7 +1745,7 @@ static int validate_event(struct perf_event *event) if (x86_pmu.put_event_constraints) x86_pmu.put_event_constraints(fake_cpuc, event); - kfree(fake_cpuc); + free_fake_cpuc(fake_cpuc); return ret; } @@ -1731,35 +1765,32 @@ static int validate_group(struct perf_event *event) { struct perf_event *leader = event->group_leader; struct cpu_hw_events *fake_cpuc; - int ret, n; + int ret = -ENOSPC, n; - ret = -ENOMEM; - fake_cpuc = kmalloc(sizeof(*fake_cpuc), GFP_KERNEL | __GFP_ZERO); - if (!fake_cpuc) - goto out; + fake_cpuc = allocate_fake_cpuc(); + if (IS_ERR(fake_cpuc)) + return PTR_ERR(fake_cpuc); /* * the event is not yet connected with its * siblings therefore we must first collect * existing siblings, then add the new event * before we can simulate the scheduling */ - ret = -ENOSPC; n = collect_events(fake_cpuc, leader, true); if (n < 0) - goto out_free; + goto out; fake_cpuc->n_events = n; n = collect_events(fake_cpuc, event, false); if (n < 0) - goto out_free; + goto out; fake_cpuc->n_events = n; ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); -out_free: - kfree(fake_cpuc); out: + free_fake_cpuc(fake_cpuc); return ret; } diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index 6ad95baff856..ac02b83e8614 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -1027,14 +1027,18 @@ __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc, { struct event_constraint *c = &emptyconstraint; struct er_account *era; + unsigned long flags; /* already allocated shared msr */ - if (reg->alloc || !cpuc->shared_regs) + if (reg->alloc) return &unconstrained; era = &cpuc->shared_regs->regs[reg->idx]; - - raw_spin_lock(&era->lock); + /* + * we use spin_lock_irqsave() to avoid lockdep issues when + * passing a fake cpuc + */ + raw_spin_lock_irqsave(&era->lock, flags); if (!atomic_read(&era->ref) || era->config == reg->config) { @@ -1058,7 +1062,7 @@ __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc, */ c = &unconstrained; } - raw_spin_unlock(&era->lock); + raw_spin_unlock_irqrestore(&era->lock, flags); return c; } @@ -1524,4 +1528,8 @@ static int intel_pmu_init(void) return 0; } +static struct intel_shared_regs *allocate_shared_regs(int cpu) +{ + return NULL; +} #endif /* CONFIG_CPU_SUP_INTEL */ -- cgit v1.2.3 From ee89cbc2d48150c7c0e9f2aaac00afde99af098c Mon Sep 17 00:00:00 2001 From: Stephane Eranian Date: Mon, 6 Jun 2011 16:57:12 +0200 Subject: perf_events: Add Intel Sandy Bridge offcore_response low-level support This patch adds Intel Sandy Bridge offcore_response support by providing the low-level constraint table for those events. On Sandy Bridge, there are two offcore_response events. Each uses its own dedictated extra register. But those registers are NOT shared between sibling CPUs when HT is on unlike Nehalem/Westmere. They are always private to each CPU. But they still need to be controlled within an event group. All events within an event group must use the same value for the extra MSR. That's not controlled by the second patch in this series. Furthermore on Sandy Bridge, the offcore_response events have NO counter constraints contrary to what the official documentation indicates, so drop the events from the contraint table. Signed-off-by: Stephane Eranian Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/20110606145712.GA7304@quad Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/perf_event.c | 1 + arch/x86/kernel/cpu/perf_event_intel.c | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 9a0f55c99b6e..583f3113436d 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -327,6 +327,7 @@ struct x86_pmu { * Extra registers for events */ struct extra_reg *extra_regs; + bool regs_no_ht_sharing; }; static struct x86_pmu x86_pmu __read_mostly; diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index ac02b83e8614..a674ae45a472 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -100,8 +100,6 @@ static struct event_constraint intel_snb_event_constraints[] __read_mostly = FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ /* FIXED_EVENT_CONSTRAINT(0x013c, 2), CPU_CLK_UNHALTED.REF */ INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */ - INTEL_EVENT_CONSTRAINT(0xb7, 0x1), /* OFF_CORE_RESPONSE_0 */ - INTEL_EVENT_CONSTRAINT(0xbb, 0x8), /* OFF_CORE_RESPONSE_1 */ INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */ INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */ EVENT_CONSTRAINT_END @@ -122,6 +120,12 @@ static struct event_constraint intel_gen_event_constraints[] __read_mostly = EVENT_CONSTRAINT_END }; +static struct extra_reg intel_snb_extra_regs[] __read_mostly = { + INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0), + INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1), + EVENT_EXTRA_END +}; + static u64 intel_pmu_event_map(int hw_event) { return intel_perfmon_event_map[hw_event]; @@ -1260,7 +1264,7 @@ static void intel_pmu_cpu_starting(int cpu) */ intel_pmu_lbr_reset(); - if (!cpuc->shared_regs) + if (!cpuc->shared_regs || x86_pmu.regs_no_ht_sharing) return; for_each_cpu(i, topology_thread_cpumask(cpu)) { @@ -1502,6 +1506,9 @@ static __init int intel_pmu_init(void) x86_pmu.event_constraints = intel_snb_event_constraints; x86_pmu.pebs_constraints = intel_snb_pebs_events; + x86_pmu.extra_regs = intel_snb_extra_regs; + /* all extra regs are per-cpu when HT is on */ + x86_pmu.regs_no_ht_sharing = true; /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; -- cgit v1.2.3 From b79e8941fb9af07d810da91b4e29da2bba331b6e Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 23 May 2011 11:08:15 +0200 Subject: perf, intel: Try alternative OFFCORE encodings Since the OFFCORE registers are fully symmetric, try the other one when the specified one is already in use. Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/1306141897.18455.8.camel@twins Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/perf_event.c | 5 +++- arch/x86/kernel/cpu/perf_event_intel.c | 44 ++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 8 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index 583f3113436d..c53d433c3dde 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -327,9 +327,12 @@ struct x86_pmu { * Extra registers for events */ struct extra_reg *extra_regs; - bool regs_no_ht_sharing; + unsigned int er_flags; }; +#define ERF_NO_HT_SHARING 1 +#define ERF_HAS_RSP_1 2 + static struct x86_pmu x86_pmu __read_mostly; static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index a674ae45a472..5c448622468c 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -1018,6 +1018,29 @@ intel_bts_constraints(struct perf_event *event) return NULL; } +static bool intel_try_alt_er(struct perf_event *event, int orig_idx) +{ + if (!(x86_pmu.er_flags & ERF_HAS_RSP_1)) + return false; + + if (event->hw.extra_reg.idx == EXTRA_REG_RSP_0) { + event->hw.config &= ~INTEL_ARCH_EVENT_MASK; + event->hw.config |= 0x01bb; + event->hw.extra_reg.idx = EXTRA_REG_RSP_1; + event->hw.extra_reg.reg = MSR_OFFCORE_RSP_1; + } else if (event->hw.extra_reg.idx == EXTRA_REG_RSP_1) { + event->hw.config &= ~INTEL_ARCH_EVENT_MASK; + event->hw.config |= 0x01b7; + event->hw.extra_reg.idx = EXTRA_REG_RSP_0; + event->hw.extra_reg.reg = MSR_OFFCORE_RSP_0; + } + + if (event->hw.extra_reg.idx == orig_idx) + return false; + + return true; +} + /* * manage allocation of shared extra msr for certain events * @@ -1027,16 +1050,19 @@ intel_bts_constraints(struct perf_event *event) */ static struct event_constraint * __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc, - struct hw_perf_event_extra *reg) + struct perf_event *event) { struct event_constraint *c = &emptyconstraint; + struct hw_perf_event_extra *reg = &event->hw.extra_reg; struct er_account *era; unsigned long flags; + int orig_idx = reg->idx; /* already allocated shared msr */ if (reg->alloc) return &unconstrained; +again: era = &cpuc->shared_regs->regs[reg->idx]; /* * we use spin_lock_irqsave() to avoid lockdep issues when @@ -1065,6 +1091,9 @@ __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc, * the regular event constraint table. */ c = &unconstrained; + } else if (intel_try_alt_er(event, orig_idx)) { + raw_spin_unlock(&era->lock); + goto again; } raw_spin_unlock_irqrestore(&era->lock, flags); @@ -1099,11 +1128,10 @@ intel_shared_regs_constraints(struct cpu_hw_events *cpuc, struct perf_event *event) { struct event_constraint *c = NULL; - struct hw_perf_event_extra *xreg; - xreg = &event->hw.extra_reg; - if (xreg->idx != EXTRA_REG_NONE) - c = __intel_shared_reg_get_constraints(cpuc, xreg); + if (event->hw.extra_reg.idx != EXTRA_REG_NONE) + c = __intel_shared_reg_get_constraints(cpuc, event); + return c; } @@ -1264,7 +1292,7 @@ static void intel_pmu_cpu_starting(int cpu) */ intel_pmu_lbr_reset(); - if (!cpuc->shared_regs || x86_pmu.regs_no_ht_sharing) + if (!cpuc->shared_regs || (x86_pmu.er_flags & ERF_NO_HT_SHARING)) return; for_each_cpu(i, topology_thread_cpumask(cpu)) { @@ -1489,6 +1517,7 @@ static __init int intel_pmu_init(void) x86_pmu.enable_all = intel_pmu_nhm_enable_all; x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints; x86_pmu.extra_regs = intel_westmere_extra_regs; + x86_pmu.er_flags |= ERF_HAS_RSP_1; /* UOPS_ISSUED.STALLED_CYCLES */ intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; @@ -1508,7 +1537,8 @@ static __init int intel_pmu_init(void) x86_pmu.pebs_constraints = intel_snb_pebs_events; x86_pmu.extra_regs = intel_snb_extra_regs; /* all extra regs are per-cpu when HT is on */ - x86_pmu.regs_no_ht_sharing = true; + x86_pmu.er_flags |= ERF_HAS_RSP_1; + x86_pmu.er_flags |= ERF_NO_HT_SHARING; /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e; -- cgit v1.2.3 From 89d6c0b5bdbb1927775584dcf532d98b3efe1477 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 22 Apr 2011 23:37:06 +0200 Subject: perf, arch: Add generic NODE cache events Add a NODE level to the generic cache events which is used to measure local vs remote memory accesses. Like all other cache events, an ACCESS is HIT+MISS, if there is no way to distinguish between reads and writes do reads only etc.. The below needs filling out for !x86 (which I filled out with unsupported events). I'm fairly sure ARM can leave it like that since it doesn't strike me as an architecture that even has NUMA support. SH might have something since it does appear to have some NUMA bits. Sparc64, PowerPC and MIPS certainly want a good look there since they clearly are NUMA capable. Signed-off-by: Peter Zijlstra Cc: David Miller Cc: Anton Blanchard Cc: David Daney Cc: Deng-Cheng Zhu Cc: Paul Mundt Cc: Will Deacon Cc: Robert Richter Cc: Stephane Eranian Link: http://lkml.kernel.org/r/1303508226.4865.8.camel@laptop Signed-off-by: Ingo Molnar --- arch/arm/kernel/perf_event_v6.c | 28 ++++++++++++++++ arch/arm/kernel/perf_event_v7.c | 28 ++++++++++++++++ arch/arm/kernel/perf_event_xscale.c | 14 ++++++++ arch/mips/kernel/perf_event_mipsxx.c | 28 ++++++++++++++++ arch/powerpc/kernel/e500-pmu.c | 5 +++ arch/powerpc/kernel/mpc7450-pmu.c | 5 +++ arch/powerpc/kernel/power4-pmu.c | 5 +++ arch/powerpc/kernel/power5+-pmu.c | 5 +++ arch/powerpc/kernel/power5-pmu.c | 5 +++ arch/powerpc/kernel/power6-pmu.c | 5 +++ arch/powerpc/kernel/power7-pmu.c | 5 +++ arch/powerpc/kernel/ppc970-pmu.c | 5 +++ arch/sh/kernel/cpu/sh4/perf_event.c | 15 +++++++++ arch/sh/kernel/cpu/sh4a/perf_event.c | 15 +++++++++ arch/sparc/kernel/perf_event.c | 42 ++++++++++++++++++++++++ arch/x86/kernel/cpu/perf_event_amd.c | 14 ++++++++ arch/x86/kernel/cpu/perf_event_intel.c | 59 +++++++++++++++++++++++++++++++++- arch/x86/kernel/cpu/perf_event_p4.c | 14 ++++++++ include/linux/perf_event.h | 3 +- 19 files changed, 298 insertions(+), 2 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/arm/kernel/perf_event_v6.c b/arch/arm/kernel/perf_event_v6.c index 38dc4da1d612..dd7f3b9f4cb3 100644 --- a/arch/arm/kernel/perf_event_v6.c +++ b/arch/arm/kernel/perf_event_v6.c @@ -173,6 +173,20 @@ static const unsigned armv6_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, }, }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, }; enum armv6mpcore_perf_types { @@ -310,6 +324,20 @@ static const unsigned armv6mpcore_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, }, }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, }; static inline unsigned long diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c index 6e5f8752303b..e20ca9cafef5 100644 --- a/arch/arm/kernel/perf_event_v7.c +++ b/arch/arm/kernel/perf_event_v7.c @@ -255,6 +255,20 @@ static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, }, }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, }; /* @@ -371,6 +385,20 @@ static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, }, }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, }; /* diff --git a/arch/arm/kernel/perf_event_xscale.c b/arch/arm/kernel/perf_event_xscale.c index 99b6b85c7e49..3c4397491d08 100644 --- a/arch/arm/kernel/perf_event_xscale.c +++ b/arch/arm/kernel/perf_event_xscale.c @@ -144,6 +144,20 @@ static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, }, }, + [C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, }; #define XSCALE_PMU_ENABLE 0x001 diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index 75266ff4cc33..e5ad09a9baf7 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -377,6 +377,20 @@ static const struct mips_perf_event mipsxxcore_cache_map [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, }, }, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, +}, }; /* 74K core has completely different cache event map. */ @@ -480,6 +494,20 @@ static const struct mips_perf_event mipsxx74Kcore_cache_map [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, }, }, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + }, +}, }; #ifdef CONFIG_MIPS_MT_SMP diff --git a/arch/powerpc/kernel/e500-pmu.c b/arch/powerpc/kernel/e500-pmu.c index b150b510510f..cb2e2949c8d1 100644 --- a/arch/powerpc/kernel/e500-pmu.c +++ b/arch/powerpc/kernel/e500-pmu.c @@ -75,6 +75,11 @@ static int e500_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { [C(OP_WRITE)] = { -1, -1 }, [C(OP_PREFETCH)] = { -1, -1 }, }, + [C(NODE)] = { /* RESULT_ACCESS RESULT_MISS */ + [C(OP_READ)] = { -1, -1 }, + [C(OP_WRITE)] = { -1, -1 }, + [C(OP_PREFETCH)] = { -1, -1 }, + }, }; static int num_events = 128; diff --git a/arch/powerpc/kernel/mpc7450-pmu.c b/arch/powerpc/kernel/mpc7450-pmu.c index 2cc5e0301d0b..845a58478890 100644 --- a/arch/powerpc/kernel/mpc7450-pmu.c +++ b/arch/powerpc/kernel/mpc7450-pmu.c @@ -388,6 +388,11 @@ static int mpc7450_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { [C(OP_WRITE)] = { -1, -1 }, [C(OP_PREFETCH)] = { -1, -1 }, }, + [C(NODE)] = { /* RESULT_ACCESS RESULT_MISS */ + [C(OP_READ)] = { -1, -1 }, + [C(OP_WRITE)] = { -1, -1 }, + [C(OP_PREFETCH)] = { -1, -1 }, + }, }; struct power_pmu mpc7450_pmu = { diff --git a/arch/powerpc/kernel/power4-pmu.c b/arch/powerpc/kernel/power4-pmu.c index ead8b3c2649e..e9dbc2d35c9c 100644 --- a/arch/powerpc/kernel/power4-pmu.c +++ b/arch/powerpc/kernel/power4-pmu.c @@ -587,6 +587,11 @@ static int power4_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { [C(OP_WRITE)] = { -1, -1 }, [C(OP_PREFETCH)] = { -1, -1 }, }, + [C(NODE)] = { /* RESULT_ACCESS RESULT_MISS */ + [C(OP_READ)] = { -1, -1 }, + [C(OP_WRITE)] = { -1, -1 }, + [C(OP_PREFETCH)] = { -1, -1 }, + }, }; static struct power_pmu power4_pmu = { diff --git a/arch/powerpc/kernel/power5+-pmu.c b/arch/powerpc/kernel/power5+-pmu.c index eca0ac595cb6..f58a2bd41b59 100644 --- a/arch/powerpc/kernel/power5+-pmu.c +++ b/arch/powerpc/kernel/power5+-pmu.c @@ -653,6 +653,11 @@ static int power5p_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { [C(OP_WRITE)] = { -1, -1 }, [C(OP_PREFETCH)] = { -1, -1 }, }, + [C(NODE)] = { /* RESULT_ACCESS RESULT_MISS */ + [C(OP_READ)] = { -1, -1 }, + [C(OP_WRITE)] = { -1, -1 }, + [C(OP_PREFETCH)] = { -1, -1 }, + }, }; static struct power_pmu power5p_pmu = { diff --git a/arch/powerpc/kernel/power5-pmu.c b/arch/powerpc/kernel/power5-pmu.c index d5ff0f64a5e6..b1acab684142 100644 --- a/arch/powerpc/kernel/power5-pmu.c +++ b/arch/powerpc/kernel/power5-pmu.c @@ -595,6 +595,11 @@ static int power5_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { [C(OP_WRITE)] = { -1, -1 }, [C(OP_PREFETCH)] = { -1, -1 }, }, + [C(NODE)] = { /* RESULT_ACCESS RESULT_MISS */ + [C(OP_READ)] = { -1, -1 }, + [C(OP_WRITE)] = { -1, -1 }, + [C(OP_PREFETCH)] = { -1, -1 }, + }, }; static struct power_pmu power5_pmu = { diff --git a/arch/powerpc/kernel/power6-pmu.c b/arch/powerpc/kernel/power6-pmu.c index 31603927e376..b24a3a23d073 100644 --- a/arch/powerpc/kernel/power6-pmu.c +++ b/arch/powerpc/kernel/power6-pmu.c @@ -516,6 +516,11 @@ static int power6_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { [C(OP_WRITE)] = { -1, -1 }, [C(OP_PREFETCH)] = { -1, -1 }, }, + [C(NODE)] = { /* RESULT_ACCESS RESULT_MISS */ + [C(OP_READ)] = { -1, -1 }, + [C(OP_WRITE)] = { -1, -1 }, + [C(OP_PREFETCH)] = { -1, -1 }, + }, }; static struct power_pmu power6_pmu = { diff --git a/arch/powerpc/kernel/power7-pmu.c b/arch/powerpc/kernel/power7-pmu.c index 593740fcb799..6d9dccb2ea59 100644 --- a/arch/powerpc/kernel/power7-pmu.c +++ b/arch/powerpc/kernel/power7-pmu.c @@ -342,6 +342,11 @@ static int power7_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { [C(OP_WRITE)] = { -1, -1 }, [C(OP_PREFETCH)] = { -1, -1 }, }, + [C(NODE)] = { /* RESULT_ACCESS RESULT_MISS */ + [C(OP_READ)] = { -1, -1 }, + [C(OP_WRITE)] = { -1, -1 }, + [C(OP_PREFETCH)] = { -1, -1 }, + }, }; static struct power_pmu power7_pmu = { diff --git a/arch/powerpc/kernel/ppc970-pmu.c b/arch/powerpc/kernel/ppc970-pmu.c index 9a6e093858fe..b121de9658eb 100644 --- a/arch/powerpc/kernel/ppc970-pmu.c +++ b/arch/powerpc/kernel/ppc970-pmu.c @@ -467,6 +467,11 @@ static int ppc970_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { [C(OP_WRITE)] = { -1, -1 }, [C(OP_PREFETCH)] = { -1, -1 }, }, + [C(NODE)] = { /* RESULT_ACCESS RESULT_MISS */ + [C(OP_READ)] = { -1, -1 }, + [C(OP_WRITE)] = { -1, -1 }, + [C(OP_PREFETCH)] = { -1, -1 }, + }, }; static struct power_pmu ppc970_pmu = { diff --git a/arch/sh/kernel/cpu/sh4/perf_event.c b/arch/sh/kernel/cpu/sh4/perf_event.c index 748955df018d..fa4f724b295a 100644 --- a/arch/sh/kernel/cpu/sh4/perf_event.c +++ b/arch/sh/kernel/cpu/sh4/perf_event.c @@ -180,6 +180,21 @@ static const int sh7750_cache_events [ C(RESULT_MISS) ] = -1, }, }, + + [ C(NODE) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, }; static int sh7750_event_map(int event) diff --git a/arch/sh/kernel/cpu/sh4a/perf_event.c b/arch/sh/kernel/cpu/sh4a/perf_event.c index 17e6bebfede0..84a2c396ceee 100644 --- a/arch/sh/kernel/cpu/sh4a/perf_event.c +++ b/arch/sh/kernel/cpu/sh4a/perf_event.c @@ -205,6 +205,21 @@ static const int sh4a_cache_events [ C(RESULT_MISS) ] = -1, }, }, + + [ C(NODE) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, }; static int sh4a_event_map(int event) diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c index 0b32f2e9e08d..62a034318b18 100644 --- a/arch/sparc/kernel/perf_event.c +++ b/arch/sparc/kernel/perf_event.c @@ -246,6 +246,20 @@ static const cache_map_t ultra3_cache_map = { [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, }, }, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, }; static const struct sparc_pmu ultra3_pmu = { @@ -361,6 +375,20 @@ static const cache_map_t niagara1_cache_map = { [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, }, }, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, }; static const struct sparc_pmu niagara1_pmu = { @@ -473,6 +501,20 @@ static const cache_map_t niagara2_cache_map = { [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, }, }, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, }; static const struct sparc_pmu niagara2_pmu = { diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c index fe29c1d2219e..941caa2e449b 100644 --- a/arch/x86/kernel/cpu/perf_event_amd.c +++ b/arch/x86/kernel/cpu/perf_event_amd.c @@ -89,6 +89,20 @@ static __initconst const u64 amd_hw_cache_event_ids [ C(RESULT_MISS) ] = -1, }, }, + [ C(NODE) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0xb8e9, /* CPU Request to Memory, l+r */ + [ C(RESULT_MISS) ] = 0x98e9, /* CPU Request to Memory, r */ + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, }; /* diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index 5c448622468c..bf6f92f7c19d 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -226,6 +226,21 @@ static __initconst const u64 snb_hw_cache_event_ids [ C(RESULT_MISS) ] = -1, }, }, + [ C(NODE) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, + }; static __initconst const u64 westmere_hw_cache_event_ids @@ -327,6 +342,20 @@ static __initconst const u64 westmere_hw_cache_event_ids [ C(RESULT_MISS) ] = -1, }, }, + [ C(NODE) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x01b7, + [ C(RESULT_MISS) ] = 0x01b7, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x01b7, + [ C(RESULT_MISS) ] = 0x01b7, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = 0x01b7, + [ C(RESULT_MISS) ] = 0x01b7, + }, + }, }; /* @@ -379,7 +408,21 @@ static __initconst const u64 nehalem_hw_cache_extra_regs [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_L3_ACCESS, [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_L3_MISS, }, - } + }, + [ C(NODE) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_ALL_DRAM, + [ C(RESULT_MISS) ] = NHM_DMND_READ|NHM_REMOTE_DRAM, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_ALL_DRAM, + [ C(RESULT_MISS) ] = NHM_DMND_WRITE|NHM_REMOTE_DRAM, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_ALL_DRAM, + [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_REMOTE_DRAM, + }, + }, }; static __initconst const u64 nehalem_hw_cache_event_ids @@ -481,6 +524,20 @@ static __initconst const u64 nehalem_hw_cache_event_ids [ C(RESULT_MISS) ] = -1, }, }, + [ C(NODE) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = 0x01b7, + [ C(RESULT_MISS) ] = 0x01b7, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = 0x01b7, + [ C(RESULT_MISS) ] = 0x01b7, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = 0x01b7, + [ C(RESULT_MISS) ] = 0x01b7, + }, + }, }; static __initconst const u64 core2_hw_cache_event_ids diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c index d6e6a67b9608..fb901c5080f7 100644 --- a/arch/x86/kernel/cpu/perf_event_p4.c +++ b/arch/x86/kernel/cpu/perf_event_p4.c @@ -554,6 +554,20 @@ static __initconst const u64 p4_hw_cache_event_ids [ C(RESULT_MISS) ] = -1, }, }, + [ C(NODE) ] = { + [ C(OP_READ) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = -1, + [ C(RESULT_MISS) ] = -1, + }, + }, }; static u64 p4_general_events[PERF_COUNT_HW_MAX] = { diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 069315eefb22..a5f54b973bdb 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -61,7 +61,7 @@ enum perf_hw_id { /* * Generalized hardware cache events: * - * { L1-D, L1-I, LLC, ITLB, DTLB, BPU } x + * { L1-D, L1-I, LLC, ITLB, DTLB, BPU, NODE } x * { read, write, prefetch } x * { accesses, misses } */ @@ -72,6 +72,7 @@ enum perf_hw_cache_id { PERF_COUNT_HW_CACHE_DTLB = 3, PERF_COUNT_HW_CACHE_ITLB = 4, PERF_COUNT_HW_CACHE_BPU = 5, + PERF_COUNT_HW_CACHE_NODE = 6, PERF_COUNT_HW_CACHE_MAX, /* non-ABI */ }; -- cgit v1.2.3 From 4dc0da86967d5463708631d02a70cfed5b104884 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 29 Jun 2011 18:42:35 +0300 Subject: perf: Add context field to perf_event The perf_event overflow handler does not receive any caller-derived argument, so many callers need to resort to looking up the perf_event in their local data structure. This is ugly and doesn't scale if a single callback services many perf_events. Fix by adding a context parameter to perf_event_create_kernel_counter() (and derived hardware breakpoints APIs) and storing it in the perf_event. The field can be accessed from the callback as event->overflow_handler_context. All callers are updated. Signed-off-by: Avi Kivity Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/1309362157-6596-2-git-send-email-avi@redhat.com Signed-off-by: Ingo Molnar --- arch/arm/kernel/ptrace.c | 3 ++- arch/powerpc/kernel/ptrace.c | 2 +- arch/sh/kernel/ptrace_32.c | 3 ++- arch/x86/kernel/kgdb.c | 2 +- arch/x86/kernel/ptrace.c | 3 ++- drivers/oprofile/oprofile_perf.c | 2 +- include/linux/hw_breakpoint.h | 10 ++++++++-- include/linux/perf_event.h | 4 +++- kernel/events/core.c | 21 +++++++++++++++------ kernel/events/hw_breakpoint.c | 10 +++++++--- kernel/watchdog.c | 2 +- samples/hw_breakpoint/data_breakpoint.c | 2 +- 12 files changed, 44 insertions(+), 20 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index 0c9b1054f790..5c199610719f 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -479,7 +479,8 @@ static struct perf_event *ptrace_hbp_create(struct task_struct *tsk, int type) attr.bp_type = type; attr.disabled = 1; - return register_user_hw_breakpoint(&attr, ptrace_hbptriggered, tsk); + return register_user_hw_breakpoint(&attr, ptrace_hbptriggered, NULL, + tsk); } static int ptrace_gethbpregs(struct task_struct *tsk, long num, diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index 3177617af2ef..05b7dd217f60 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -973,7 +973,7 @@ int ptrace_set_debugreg(struct task_struct *task, unsigned long addr, &attr.bp_type); thread->ptrace_bps[0] = bp = register_user_hw_breakpoint(&attr, - ptrace_triggered, task); + ptrace_triggered, NULL, task); if (IS_ERR(bp)) { thread->ptrace_bps[0] = NULL; ptrace_put_breakpoints(task); diff --git a/arch/sh/kernel/ptrace_32.c b/arch/sh/kernel/ptrace_32.c index 8051976100a6..92b3c276339a 100644 --- a/arch/sh/kernel/ptrace_32.c +++ b/arch/sh/kernel/ptrace_32.c @@ -91,7 +91,8 @@ static int set_single_step(struct task_struct *tsk, unsigned long addr) attr.bp_len = HW_BREAKPOINT_LEN_2; attr.bp_type = HW_BREAKPOINT_R; - bp = register_user_hw_breakpoint(&attr, ptrace_triggered, tsk); + bp = register_user_hw_breakpoint(&attr, ptrace_triggered, + NULL, tsk); if (IS_ERR(bp)) return PTR_ERR(bp); diff --git a/arch/x86/kernel/kgdb.c b/arch/x86/kernel/kgdb.c index 98da6a7b5e82..00354d4919a9 100644 --- a/arch/x86/kernel/kgdb.c +++ b/arch/x86/kernel/kgdb.c @@ -638,7 +638,7 @@ void kgdb_arch_late(void) for (i = 0; i < HBP_NUM; i++) { if (breakinfo[i].pev) continue; - breakinfo[i].pev = register_wide_hw_breakpoint(&attr, NULL); + breakinfo[i].pev = register_wide_hw_breakpoint(&attr, NULL, NULL); if (IS_ERR((void * __force)breakinfo[i].pev)) { printk(KERN_ERR "kgdb: Could not allocate hw" "breakpoints\nDisabling the kernel debugger\n"); diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index 11db2e9b860a..82528799c5de 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c @@ -715,7 +715,8 @@ static int ptrace_set_breakpoint_addr(struct task_struct *tsk, int nr, attr.bp_type = HW_BREAKPOINT_W; attr.disabled = 1; - bp = register_user_hw_breakpoint(&attr, ptrace_triggered, tsk); + bp = register_user_hw_breakpoint(&attr, ptrace_triggered, + NULL, tsk); /* * CHECKME: the previous code returned -EIO if the addr wasn't diff --git a/drivers/oprofile/oprofile_perf.c b/drivers/oprofile/oprofile_perf.c index 9046f7b2ed79..59acf9ef78a4 100644 --- a/drivers/oprofile/oprofile_perf.c +++ b/drivers/oprofile/oprofile_perf.c @@ -79,7 +79,7 @@ static int op_create_counter(int cpu, int event) pevent = perf_event_create_kernel_counter(&counter_config[event].attr, cpu, NULL, - op_overflow_handler); + op_overflow_handler, NULL); if (IS_ERR(pevent)) return PTR_ERR(pevent); diff --git a/include/linux/hw_breakpoint.h b/include/linux/hw_breakpoint.h index d1e55fed2c7d..6ae9c631a1be 100644 --- a/include/linux/hw_breakpoint.h +++ b/include/linux/hw_breakpoint.h @@ -73,6 +73,7 @@ static inline unsigned long hw_breakpoint_len(struct perf_event *bp) extern struct perf_event * register_user_hw_breakpoint(struct perf_event_attr *attr, perf_overflow_handler_t triggered, + void *context, struct task_struct *tsk); /* FIXME: only change from the attr, and don't unregister */ @@ -85,11 +86,13 @@ modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr); extern struct perf_event * register_wide_hw_breakpoint_cpu(struct perf_event_attr *attr, perf_overflow_handler_t triggered, + void *context, int cpu); extern struct perf_event * __percpu * register_wide_hw_breakpoint(struct perf_event_attr *attr, - perf_overflow_handler_t triggered); + perf_overflow_handler_t triggered, + void *context); extern int register_perf_hw_breakpoint(struct perf_event *bp); extern int __register_perf_hw_breakpoint(struct perf_event *bp); @@ -115,6 +118,7 @@ static inline int __init init_hw_breakpoint(void) { return 0; } static inline struct perf_event * register_user_hw_breakpoint(struct perf_event_attr *attr, perf_overflow_handler_t triggered, + void *context, struct task_struct *tsk) { return NULL; } static inline int modify_user_hw_breakpoint(struct perf_event *bp, @@ -122,10 +126,12 @@ modify_user_hw_breakpoint(struct perf_event *bp, static inline struct perf_event * register_wide_hw_breakpoint_cpu(struct perf_event_attr *attr, perf_overflow_handler_t triggered, + void *context, int cpu) { return NULL; } static inline struct perf_event * __percpu * register_wide_hw_breakpoint(struct perf_event_attr *attr, - perf_overflow_handler_t triggered) { return NULL; } + perf_overflow_handler_t triggered, + void *context) { return NULL; } static inline int register_perf_hw_breakpoint(struct perf_event *bp) { return -ENOSYS; } static inline int diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index a5f54b973bdb..2a08cacb1628 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -839,6 +839,7 @@ struct perf_event { u64 id; perf_overflow_handler_t overflow_handler; + void *overflow_handler_context; #ifdef CONFIG_EVENT_TRACING struct ftrace_event_call *tp_event; @@ -960,7 +961,8 @@ extern struct perf_event * perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, struct task_struct *task, - perf_overflow_handler_t callback); + perf_overflow_handler_t callback, + void *context); extern u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running); diff --git a/kernel/events/core.c b/kernel/events/core.c index 81de28dcca8c..ba8e0f4a20e6 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -5745,7 +5745,8 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, struct task_struct *task, struct perf_event *group_leader, struct perf_event *parent_event, - perf_overflow_handler_t overflow_handler) + perf_overflow_handler_t overflow_handler, + void *context) { struct pmu *pmu; struct perf_event *event; @@ -5803,10 +5804,13 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu, #endif } - if (!overflow_handler && parent_event) + if (!overflow_handler && parent_event) { overflow_handler = parent_event->overflow_handler; + context = parent_event->overflow_handler_context; + } event->overflow_handler = overflow_handler; + event->overflow_handler_context = context; if (attr->disabled) event->state = PERF_EVENT_STATE_OFF; @@ -6073,7 +6077,8 @@ SYSCALL_DEFINE5(perf_event_open, } } - event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, NULL); + event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, + NULL, NULL); if (IS_ERR(event)) { err = PTR_ERR(event); goto err_task; @@ -6258,7 +6263,8 @@ err_fd: struct perf_event * perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, struct task_struct *task, - perf_overflow_handler_t overflow_handler) + perf_overflow_handler_t overflow_handler, + void *context) { struct perf_event_context *ctx; struct perf_event *event; @@ -6268,7 +6274,8 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, * Get the target context (task or percpu): */ - event = perf_event_alloc(attr, cpu, task, NULL, NULL, overflow_handler); + event = perf_event_alloc(attr, cpu, task, NULL, NULL, + overflow_handler, context); if (IS_ERR(event)) { err = PTR_ERR(event); goto err; @@ -6552,7 +6559,7 @@ inherit_event(struct perf_event *parent_event, parent_event->cpu, child, group_leader, parent_event, - NULL); + NULL, NULL); if (IS_ERR(child_event)) return child_event; get_ctx(child_ctx); @@ -6579,6 +6586,8 @@ inherit_event(struct perf_event *parent_event, child_event->ctx = child_ctx; child_event->overflow_handler = parent_event->overflow_handler; + child_event->overflow_handler_context + = parent_event->overflow_handler_context; /* * Precalculate sample_data sizes diff --git a/kernel/events/hw_breakpoint.c b/kernel/events/hw_breakpoint.c index 086adf25a55e..b7971d6f38bf 100644 --- a/kernel/events/hw_breakpoint.c +++ b/kernel/events/hw_breakpoint.c @@ -431,9 +431,11 @@ int register_perf_hw_breakpoint(struct perf_event *bp) struct perf_event * register_user_hw_breakpoint(struct perf_event_attr *attr, perf_overflow_handler_t triggered, + void *context, struct task_struct *tsk) { - return perf_event_create_kernel_counter(attr, -1, tsk, triggered); + return perf_event_create_kernel_counter(attr, -1, tsk, triggered, + context); } EXPORT_SYMBOL_GPL(register_user_hw_breakpoint); @@ -502,7 +504,8 @@ EXPORT_SYMBOL_GPL(unregister_hw_breakpoint); */ struct perf_event * __percpu * register_wide_hw_breakpoint(struct perf_event_attr *attr, - perf_overflow_handler_t triggered) + perf_overflow_handler_t triggered, + void *context) { struct perf_event * __percpu *cpu_events, **pevent, *bp; long err; @@ -515,7 +518,8 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr, get_online_cpus(); for_each_online_cpu(cpu) { pevent = per_cpu_ptr(cpu_events, cpu); - bp = perf_event_create_kernel_counter(attr, cpu, NULL, triggered); + bp = perf_event_create_kernel_counter(attr, cpu, NULL, + triggered, context); *pevent = bp; diff --git a/kernel/watchdog.c b/kernel/watchdog.c index a6708e677a0a..a933e3a0398b 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -375,7 +375,7 @@ static int watchdog_nmi_enable(int cpu) hw_nmi_watchdog_set_attr(wd_attr); /* Try to register using hardware perf events */ - event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback); + event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL); if (!IS_ERR(event)) { printk(KERN_INFO "NMI watchdog enabled, takes one hw-pmu counter.\n"); goto out_save; diff --git a/samples/hw_breakpoint/data_breakpoint.c b/samples/hw_breakpoint/data_breakpoint.c index 7b164d3200ff..ef7f32291852 100644 --- a/samples/hw_breakpoint/data_breakpoint.c +++ b/samples/hw_breakpoint/data_breakpoint.c @@ -60,7 +60,7 @@ static int __init hw_break_module_init(void) attr.bp_len = HW_BREAKPOINT_LEN_4; attr.bp_type = HW_BREAKPOINT_W | HW_BREAKPOINT_R; - sample_hbp = register_wide_hw_breakpoint(&attr, sample_hbp_handler); + sample_hbp = register_wide_hw_breakpoint(&attr, sample_hbp_handler, NULL); if (IS_ERR((void __force *)sample_hbp)) { ret = PTR_ERR((void __force *)sample_hbp); goto fail; -- cgit v1.2.3 From 0af3ac1fdb9d5c297b4b07c9e0172531d42b6716 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 29 Jun 2011 18:42:36 +0300 Subject: x86, perf: Add constraints for architectural PMU The v1 PMU does not have any fixed counters. Using the v2 constraints, which do have fixed counters, causes an additional choice to be present in the weight calculation, but not when actually scheduling the event, leading to an event being not scheduled at all. Signed-off-by: Avi Kivity Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/1309362157-6596-3-git-send-email-avi@redhat.com Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/perf_event_intel.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index bf6f92f7c19d..45fbb8f7f549 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -112,6 +112,11 @@ static struct extra_reg intel_westmere_extra_regs[] __read_mostly = EVENT_EXTRA_END }; +static struct event_constraint intel_v1_event_constraints[] __read_mostly = +{ + EVENT_CONSTRAINT_END +}; + static struct event_constraint intel_gen_event_constraints[] __read_mostly = { FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ @@ -1606,11 +1611,19 @@ static __init int intel_pmu_init(void) break; default: - /* - * default constraints for v2 and up - */ - x86_pmu.event_constraints = intel_gen_event_constraints; - pr_cont("generic architected perfmon, "); + switch (x86_pmu.version) { + case 1: + x86_pmu.event_constraints = intel_v1_event_constraints; + pr_cont("generic architected perfmon v1, "); + break; + default: + /* + * default constraints for v2 and up + */ + x86_pmu.event_constraints = intel_gen_event_constraints; + pr_cont("generic architected perfmon, "); + break; + } } return 0; } -- cgit v1.2.3 From 50c31e4a2497ea17747b587e8f96b278f07f5483 Mon Sep 17 00:00:00 2001 From: Sergei Shtylyov Date: Fri, 1 Jul 2011 22:42:08 +0400 Subject: x86, mtrr: Use pci_dev->revision This code uses PCI_CLASS_REVISION instead of PCI_REVISION_ID, so it wasn't converted by commit 44c10138fd4 ("PCI: Change all drivers to use pci_device->revision") before being moved to arch/x86/... Do it now at last -- and save one level of indentation... Signed-off-by: Sergei Shtylyov Cc: Suresh Siddha Link: http://lkml.kernel.org/r/201107012242.08347.sshtylyov@ru.mvista.com Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/mtrr/main.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c index 707b6377adf1..08119a37e53c 100644 --- a/arch/x86/kernel/cpu/mtrr/main.c +++ b/arch/x86/kernel/cpu/mtrr/main.c @@ -79,7 +79,6 @@ void set_mtrr_ops(const struct mtrr_ops *ops) static int have_wrcomb(void) { struct pci_dev *dev; - u8 rev; dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL); if (dev != NULL) { @@ -89,13 +88,11 @@ static int have_wrcomb(void) * chipsets to be tagged */ if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS && - dev->device == PCI_DEVICE_ID_SERVERWORKS_LE) { - pci_read_config_byte(dev, PCI_CLASS_REVISION, &rev); - if (rev <= 5) { - pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n"); - pci_dev_put(dev); - return 0; - } + dev->device == PCI_DEVICE_ID_SERVERWORKS_LE && + dev->revision <= 5) { + pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n"); + pci_dev_put(dev); + return 0; } /* * Intel 450NX errata # 23. Non ascending cacheline evictions to -- cgit v1.2.3 From 47ce11a2b6519f9c7843223ea8e561eb71ea5896 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 30 Jun 2011 19:04:56 +0200 Subject: x86: Fetch stack from regs when possible in dump_trace() When regs are passed to dump_stack(), we fetch the frame pointer from the regs but the stack pointer is taken from the current frame. Thus the frame and stack pointers may not come from the same context. For example this can result in the unwinder to think the context is in irq, due to the current value of the stack, but the frame pointer coming from the regs points to a frame from another place. It then tries to fix up the irq link but ends up dereferencing a random frame pointer that doesn't belong to the irq stack: [ 9131.706906] ------------[ cut here ]------------ [ 9131.707003] WARNING: at arch/x86/kernel/dumpstack_64.c:129 dump_trace+0x2aa/0x330() [ 9131.707003] Hardware name: AMD690VM-FMH [ 9131.707003] Perf: bad frame pointer = 0000000000000005 in callchain [ 9131.707003] Modules linked in: [ 9131.707003] Pid: 1050, comm: perf Not tainted 3.0.0-rc3+ #181 [ 9131.707003] Call Trace: [ 9131.707003] [] warn_slowpath_common+0x7a/0xb0 [ 9131.707003] [] warn_slowpath_fmt+0x41/0x50 [ 9131.707003] [] ? bad_to_user+0x6d/0x10be [ 9131.707003] [] dump_trace+0x2aa/0x330 [ 9131.707003] [] ? native_sched_clock+0x13/0x50 [ 9131.707003] [] perf_callchain_kernel+0x54/0x70 [ 9131.707003] [] perf_prepare_sample+0x19f/0x2a0 [ 9131.707003] [] __perf_event_overflow+0x16c/0x290 [ 9131.707003] [] ? __perf_event_overflow+0x130/0x290 [ 9131.707003] [] ? native_sched_clock+0x13/0x50 [ 9131.707003] [] ? sched_clock+0x9/0x10 [ 9131.707003] [] ? T.375+0x15/0x90 [ 9131.707003] [] ? trace_hardirqs_on_caller+0x64/0x180 [ 9131.707003] [] ? trace_hardirqs_off+0xd/0x10 [ 9131.707003] [] perf_event_overflow+0x14/0x20 [ 9131.707003] [] perf_swevent_hrtimer+0x11c/0x130 [ 9131.707003] [] ? error_exit+0x51/0xb0 [ 9131.707003] [] __run_hrtimer+0x83/0x1e0 [ 9131.707003] [] ? perf_event_overflow+0x20/0x20 [ 9131.707003] [] hrtimer_interrupt+0x106/0x250 [ 9131.707003] [] ? trace_hardirqs_off_thunk+0x3a/0x3c [ 9131.707003] [] smp_apic_timer_interrupt+0x53/0x90 [ 9131.707003] [] apic_timer_interrupt+0x13/0x20 [ 9131.707003] [] ? error_exit+0x51/0xb0 [ 9131.707003] [] ? error_exit+0x4c/0xb0 [ 9131.707003] ---[ end trace b2560d4876709347 ]--- Fix this by simply taking the stack pointer from regs->sp when regs are provided. Signed-off-by: Frederic Weisbecker Cc: Ingo Molnar Cc: Thomas Gleixner Cc: H. Peter Anvin Cc: Peter Zijlstra Cc: Arnaldo Carvalho de Melo --- arch/x86/kernel/dumpstack_64.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index e71c98d3c0d2..788295cbe4a7 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -155,9 +155,12 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs, task = current; if (!stack) { - stack = &dummy; - if (task && task != current) + if (regs) + stack = (unsigned long *)regs->sp; + else if (task && task != current) stack = (unsigned long *)task->thread.sp; + else + stack = &dummy; } if (!bp) -- cgit v1.2.3 From 1871853f7abc3c727c4346539c5062cbeaf016a4 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 1 Jul 2011 01:51:22 +0200 Subject: x86,64: Simplify save_regs() The save_regs function that saves the regs on low level irq entry is complicated because of the fact it changes its stack in the middle and also because it manipulates data allocated in the caller frame and accesses there are directly calculated from callee rsp value with the return address in the middle of the way. This complicates the static stack offsets calculation and require more dynamic ones. It also needs a save/restore of the function's return address. To simplify and optimize this, turn save_regs() into a macro. Signed-off-by: Frederic Weisbecker Cc: Ingo Molnar Cc: Thomas Gleixner Cc: H. Peter Anvin Cc: Peter Zijlstra Cc: Arnaldo Carvalho de Melo Cc: Jan Beulich --- arch/x86/kernel/entry_64.S | 44 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 8a445a0c989e..b6b2e85454cf 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -297,27 +297,22 @@ ENDPROC(native_usergs_sysret64) .endm /* save partial stack frame */ - .pushsection .kprobes.text, "ax" -ENTRY(save_args) - XCPT_FRAME + .macro SAVE_ARGS_IRQ cld - /* - * start from rbp in pt_regs and jump over - * return address. - */ - movq_cfi rdi, RDI+8-RBP - movq_cfi rsi, RSI+8-RBP - movq_cfi rdx, RDX+8-RBP - movq_cfi rcx, RCX+8-RBP - movq_cfi rax, RAX+8-RBP - movq_cfi r8, R8+8-RBP - movq_cfi r9, R9+8-RBP - movq_cfi r10, R10+8-RBP - movq_cfi r11, R11+8-RBP - - leaq -RBP+8(%rsp),%rdi /* arg1 for handler */ - movq_cfi rbp, 8 /* push %rbp */ - leaq 8(%rsp), %rbp /* mov %rsp, %ebp */ + /* start from rbp in pt_regs and jump over */ + movq_cfi rdi, RDI-RBP + movq_cfi rsi, RSI-RBP + movq_cfi rdx, RDX-RBP + movq_cfi rcx, RCX-RBP + movq_cfi rax, RAX-RBP + movq_cfi r8, R8-RBP + movq_cfi r9, R9-RBP + movq_cfi r10, R10-RBP + movq_cfi r11, R11-RBP + + leaq -RBP(%rsp),%rdi /* arg1 for handler */ + movq_cfi rbp, 0 /* push %rbp */ + movq %rsp, %rbp testl $3, CS(%rdi) je 1f SWAPGS @@ -329,19 +324,14 @@ ENTRY(save_args) */ 1: incl PER_CPU_VAR(irq_count) jne 2f - popq_cfi %rax /* move return address... */ mov PER_CPU_VAR(irq_stack_ptr),%rsp EMPTY_FRAME 0 pushq_cfi %rbp /* backlink for unwinder */ - pushq_cfi %rax /* ... to the new stack */ /* * We entered an interrupt context - irqs are off: */ 2: TRACE_IRQS_OFF - ret - CFI_ENDPROC -END(save_args) - .popsection + .endm ENTRY(save_rest) PARTIAL_FRAME 1 REST_SKIP+8 @@ -791,7 +781,7 @@ END(interrupt) /* reserve pt_regs for scratch regs and rbp */ subq $ORIG_RAX-RBP, %rsp CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP - call save_args + SAVE_ARGS_IRQ PARTIAL_FRAME 0 call \func .endm -- cgit v1.2.3 From 3b99a3ef55b292180473a221f3d6bc24455f0632 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 1 Jul 2011 02:25:17 +0200 Subject: x86,64: Separate arg1 from rbp handling in SAVE_REGS_IRQ Just for clarity in the code. Have a first block that handles the frame pointer and a separate one that handles pt_regs pointer and its use. Signed-off-by: Frederic Weisbecker Cc: Ingo Molnar Cc: Thomas Gleixner Cc: H. Peter Anvin Cc: Peter Zijlstra Cc: Arnaldo Carvalho de Melo Cc: Jan Beulich --- arch/x86/kernel/entry_64.S | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index b6b2e85454cf..20dc8e6b6d80 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -310,9 +310,10 @@ ENDPROC(native_usergs_sysret64) movq_cfi r10, R10-RBP movq_cfi r11, R11-RBP - leaq -RBP(%rsp),%rdi /* arg1 for handler */ movq_cfi rbp, 0 /* push %rbp */ movq %rsp, %rbp + + leaq -RBP(%rsp),%rdi /* arg1 for handler */ testl $3, CS(%rdi) je 1f SWAPGS -- cgit v1.2.3 From 48ffee7d9e6df51b4957bed64115b7beed671374 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Sat, 2 Jul 2011 15:03:44 +0200 Subject: x86: Remove useless unwinder backlink from irq regs saving The unwinder backlink in interrupt entry is very useless. It's actually not part of the stack frame chain and thus is never used. Signed-off-by: Frederic Weisbecker Cc: Ingo Molnar Cc: Thomas Gleixner Cc: H. Peter Anvin Cc: Peter Zijlstra Cc: Arnaldo Carvalho de Melo Cc: Jan Beulich --- arch/x86/kernel/entry_64.S | 1 - 1 file changed, 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 20dc8e6b6d80..6131432c5afa 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -327,7 +327,6 @@ ENDPROC(native_usergs_sysret64) jne 2f mov PER_CPU_VAR(irq_stack_ptr),%rsp EMPTY_FRAME 0 - pushq_cfi %rbp /* backlink for unwinder */ /* * We entered an interrupt context - irqs are off: */ -- cgit v1.2.3 From a2bbe75089d5eb9a3a46d50dd5c215e213790288 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Sat, 2 Jul 2011 16:52:45 +0200 Subject: x86: Don't use frame pointer to save old stack on irq entry rbp is used in SAVE_ARGS_IRQ to save the old stack pointer in order to restore it later in ret_from_intr. It is convenient because we save its value in the irq regs and it's easily restored using the leave instruction. However this is a kind of abuse of the frame pointer which role is to help unwinding the kernel by chaining frames together, each node following the return address to the previous frame. But although we are breaking the frame by changing the stack pointer, there is no preceding return address before the new frame. Hence using the frame pointer to link the two stacks breaks the stack unwinders that find a random value instead of a return address here. There is no workaround that can work in every case. We are using the fixup_bp_irq_link() function to dereference that abused frame pointer in the case of non nesting interrupt (which means stack changed). But that doesn't fix the case of interrupts that don't change the stack (but we still have the unconditional frame link), which is the case of hardirq interrupting softirq. We have no way to detect this transition so the frame irq link is considered as a real frame pointer and the return address is dereferenced but it is still a spurious one. There are two possible results of this: either the spurious return address, a random stack value, luckily belongs to the kernel text and then the unwinding can continue and we just have a weird entry in the stack trace. Or it doesn't belong to the kernel text and unwinding stops there. This is the reason why stacktraces (including perf callchains) on irqs that interrupted softirqs don't work very well. To solve this, we don't save the old stack pointer on rbp anymore but we save it to a scratch register that we push on the new stack and that we pop back later on irq return. This preserves the whole frame chain without spurious return addresses in the middle and drops the need for the horrid fixup_bp_irq_link() workaround. And finally irqs that interrupt softirq are sanely unwinded. Before: 99.81% perf [kernel.kallsyms] [k] perf_pending_event | --- perf_pending_event irq_work_run smp_irq_work_interrupt irq_work_interrupt | |--41.60%-- __read | | | |--99.90%-- create_worker | | bench_sched_messaging | | cmd_bench | | run_builtin | | main | | __libc_start_main | --0.10%-- [...] After: 1.64% swapper [kernel.kallsyms] [k] perf_pending_event | --- perf_pending_event irq_work_run smp_irq_work_interrupt irq_work_interrupt | |--95.00%-- arch_irq_work_raise | irq_work_queue | __perf_event_overflow | perf_swevent_overflow | perf_swevent_event | perf_tp_event | perf_trace_softirq | __do_softirq | call_softirq | do_softirq | irq_exit | | | |--73.68%-- smp_apic_timer_interrupt | | apic_timer_interrupt | | | | | |--96.43%-- amd_e400_idle | | | cpu_idle | | | start_secondary Signed-off-by: Frederic Weisbecker Cc: Ingo Molnar Cc: Thomas Gleixner Cc: H. Peter Anvin Cc: Peter Zijlstra Cc: Arnaldo Carvalho de Melo Cc: Jan Beulich --- arch/x86/kernel/dumpstack_64.c | 30 ------------------------------ arch/x86/kernel/entry_64.S | 27 +++++++++++++++------------ 2 files changed, 15 insertions(+), 42 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 788295cbe4a7..19853ad8afc5 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -104,34 +104,6 @@ in_irq_stack(unsigned long *stack, unsigned long *irq_stack, return (stack >= irq_stack && stack < irq_stack_end); } -/* - * We are returning from the irq stack and go to the previous one. - * If the previous stack is also in the irq stack, then bp in the first - * frame of the irq stack points to the previous, interrupted one. - * Otherwise we have another level of indirection: We first save - * the bp of the previous stack, then we switch the stack to the irq one - * and save a new bp that links to the previous one. - * (See save_args()) - */ -static inline unsigned long -fixup_bp_irq_link(unsigned long bp, unsigned long *stack, - unsigned long *irq_stack, unsigned long *irq_stack_end) -{ -#ifdef CONFIG_FRAME_POINTER - struct stack_frame *frame = (struct stack_frame *)bp; - unsigned long next; - - if (!in_irq_stack(stack, irq_stack, irq_stack_end)) { - if (!probe_kernel_address(&frame->next_frame, next)) - return next; - else - WARN_ONCE(1, "Perf: bad frame pointer = %p in " - "callchain\n", &frame->next_frame); - } -#endif - return bp; -} - /* * x86-64 can have up to three kernel stacks: * process stack @@ -208,8 +180,6 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs, * pointer (index -1 to end) in the IRQ stack: */ stack = (unsigned long *) (irq_stack_end[-1]); - bp = fixup_bp_irq_link(bp, stack, irq_stack, - irq_stack_end); irq_stack_end = NULL; ops->stack(data, "EOI"); continue; diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 6131432c5afa..d656f68371a4 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -310,8 +310,11 @@ ENDPROC(native_usergs_sysret64) movq_cfi r10, R10-RBP movq_cfi r11, R11-RBP - movq_cfi rbp, 0 /* push %rbp */ - movq %rsp, %rbp + /* Save rbp so that we can unwind from get_irq_regs() */ + movq_cfi rbp, 0 + + /* Save previous stack value */ + movq %rsp, %rsi leaq -RBP(%rsp),%rdi /* arg1 for handler */ testl $3, CS(%rdi) @@ -327,10 +330,11 @@ ENDPROC(native_usergs_sysret64) jne 2f mov PER_CPU_VAR(irq_stack_ptr),%rsp EMPTY_FRAME 0 - /* - * We entered an interrupt context - irqs are off: - */ -2: TRACE_IRQS_OFF + +2: /* Store previous stack value */ + pushq %rsi + /* We entered an interrupt context - irqs are off: */ + TRACE_IRQS_OFF .endm ENTRY(save_rest) @@ -804,15 +808,14 @@ ret_from_intr: DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF decl PER_CPU_VAR(irq_count) - leaveq - CFI_RESTORE rbp + /* Restore saved previous stack */ + popq %rsi + leaq 16(%rsi), %rsp + CFI_DEF_CFA_REGISTER rsp - CFI_ADJUST_CFA_OFFSET -8 + CFI_ADJUST_CFA_OFFSET -16 - /* we did not save rbx, restore only from ARGOFFSET */ - addq $8, %rsp - CFI_ADJUST_CFA_OFFSET -8 exit_intr: GET_THREAD_INFO(%rcx) testl $3,CS-ARGOFFSET(%rsp) -- cgit v1.2.3 From b49c78d4827be8d7e67e5b94adac6b30a4a9ad14 Mon Sep 17 00:00:00 2001 From: Peter Chubb Date: Wed, 6 Jul 2011 10:56:30 +1000 Subject: x86, reboot: Acer Aspire One A110 reboot quirk Since git commit 660e34cebf0a11d54f2d5dd8838607452355f321 x86: reorder reboot method preferences, my Acer Aspire One hangs on reboot. It appears that its ACPI method for rebooting is broken. The attached patch adds a quirk so that the machine will reboot via the BIOS. [ hpa: verified that the ACPI control on this machine is just plain broken. ] Signed-off-by: Peter Chubb Link: http://lkml.kernel.org/r/w439iki5vl.wl%25peter@chubb.wattle.id.au Signed-off-by: H. Peter Anvin --- arch/x86/kernel/reboot.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index 0c016f727695..4f0d46fefa7f 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -294,6 +294,14 @@ static struct dmi_system_id __initdata reboot_dmi_table[] = { DMI_MATCH(DMI_BOARD_NAME, "VersaLogic Menlow board"), }, }, + { /* Handle reboot issue on Acer Aspire one */ + .callback = set_bios_reboot, + .ident = "Acer Aspire One A110", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "AOA110"), + }, + }, { } }; -- cgit v1.2.3 From 7a3136666bc0f0419f7aaa7b1fabb4b0e0a7fb76 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 6 Jul 2011 18:10:34 -0700 Subject: x86, suspend: Restore MISC_ENABLE MSR in realmode wakeup Some BIOSes will reset the Intel MISC_ENABLE MSR (specifically the XD_DISABLE bit) when resuming from S3, which can interact poorly with ebba638ae723d8a8fc2f7abce5ec18b688b791d7. In 32bit PAE mode, this can lead to a fault when EFER is restored by the kernel wakeup routines, due to it setting the NX bit for a CPU that (thanks to the BIOS reset) now incorrectly thinks it lacks the NX feature. (64bit is not affected because it uses a common CPU bring-up that specifically handles the XD_DISABLE bit.) The need for MISC_ENABLE being restored so early is specific to the S3 resume path. Normally, MISC_ENABLE is saved in save_processor_state(), but this happens after the resume header is created, so just reproduce the logic here. (acpi_suspend_lowlevel() creates the header, calls do_suspend_lowlevel, which calls save_processor_state(), so the saved processor context isn't available during resume header creation.) [ hpa: Consider for stable if OK in mainline ] Signed-off-by: Kees Cook Link: http://lkml.kernel.org/r/20110707011034.GA8523@outflux.net Signed-off-by: H. Peter Anvin Cc: Rafael J. Wysocki Cc: 2.6.38+ --- arch/x86/kernel/acpi/realmode/wakeup.S | 14 ++++++++++++++ arch/x86/kernel/acpi/realmode/wakeup.h | 6 ++++++ arch/x86/kernel/acpi/sleep.c | 6 ++++++ 3 files changed, 26 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/acpi/realmode/wakeup.S b/arch/x86/kernel/acpi/realmode/wakeup.S index ead21b663117..b4fd836e4053 100644 --- a/arch/x86/kernel/acpi/realmode/wakeup.S +++ b/arch/x86/kernel/acpi/realmode/wakeup.S @@ -28,6 +28,8 @@ pmode_cr3: .long 0 /* Saved %cr3 */ pmode_cr4: .long 0 /* Saved %cr4 */ pmode_efer: .quad 0 /* Saved EFER */ pmode_gdt: .quad 0 +pmode_misc_en: .quad 0 /* Saved MISC_ENABLE MSR */ +pmode_behavior: .long 0 /* Wakeup behavior flags */ realmode_flags: .long 0 real_magic: .long 0 trampoline_segment: .word 0 @@ -91,6 +93,18 @@ wakeup_code: /* Call the C code */ calll main + /* Restore MISC_ENABLE before entering protected mode, in case + BIOS decided to clear XD_DISABLE during S3. */ + movl pmode_behavior, %eax + btl $WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE, %eax + jnc 1f + + movl pmode_misc_en, %eax + movl pmode_misc_en + 4, %edx + movl $MSR_IA32_MISC_ENABLE, %ecx + wrmsr +1: + /* Do any other stuff... */ #ifndef CONFIG_64BIT diff --git a/arch/x86/kernel/acpi/realmode/wakeup.h b/arch/x86/kernel/acpi/realmode/wakeup.h index e1828c07e79c..97a29e1430e3 100644 --- a/arch/x86/kernel/acpi/realmode/wakeup.h +++ b/arch/x86/kernel/acpi/realmode/wakeup.h @@ -21,6 +21,9 @@ struct wakeup_header { u32 pmode_efer_low; /* Protected mode EFER */ u32 pmode_efer_high; u64 pmode_gdt; + u32 pmode_misc_en_low; /* Protected mode MISC_ENABLE */ + u32 pmode_misc_en_high; + u32 pmode_behavior; /* Wakeup routine behavior flags */ u32 realmode_flags; u32 real_magic; u16 trampoline_segment; /* segment with trampoline code, 64-bit only */ @@ -39,4 +42,7 @@ extern struct wakeup_header wakeup_header; #define WAKEUP_HEADER_SIGNATURE 0x51ee1111 #define WAKEUP_END_SIGNATURE 0x65a22c82 +/* Wakeup behavior bits */ +#define WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE 0 + #endif /* ARCH_X86_KERNEL_ACPI_RM_WAKEUP_H */ diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c index 18a857ba7a25..103b6ab368d3 100644 --- a/arch/x86/kernel/acpi/sleep.c +++ b/arch/x86/kernel/acpi/sleep.c @@ -77,6 +77,12 @@ int acpi_suspend_lowlevel(void) header->pmode_cr0 = read_cr0(); header->pmode_cr4 = read_cr4_safe(); + header->pmode_behavior = 0; + if (!rdmsr_safe(MSR_IA32_MISC_ENABLE, + &header->pmode_misc_en_low, + &header->pmode_misc_en_high)) + header->pmode_behavior |= + (1 << WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE); header->realmode_flags = acpi_realmode_flags; header->real_magic = 0x12345678; -- cgit v1.2.3 From ded1f6ab43a03be74a53649cf388b5424d447584 Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Fri, 8 Jul 2011 08:36:34 +0000 Subject: x86: print APIC data a little later during boot To view IOAPIC data you could boot with "apic=debug". When booting in such a way then the kernel will dump the IO-APIC's registers, for example: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 000 1 0 0 0 0 0 0 00 01 000 0 0 0 0 0 0 0 31 02 000 0 0 0 0 0 0 0 30 03 000 0 0 0 0 0 0 0 33 04 000 0 0 0 0 0 0 0 34 05 000 0 0 0 0 0 0 0 35 06 000 0 0 0 0 0 0 0 36 07 000 0 0 0 0 0 0 0 37 08 000 0 0 0 0 0 0 0 38 09 000 0 1 0 0 0 0 0 39 0a 000 0 0 0 0 0 0 0 3A 0b 000 0 0 0 0 0 0 0 3B 0c 000 0 0 0 0 0 0 0 3C 0d 000 0 0 0 0 0 0 0 3D 0e 000 0 0 0 0 0 0 0 3E 0f 000 0 0 0 0 0 0 0 3F 10 000 1 0 0 0 0 0 0 00 11 000 1 0 0 0 0 0 0 00 12 000 1 0 0 0 0 0 0 00 13 000 1 0 0 0 0 0 0 00 14 000 1 0 0 0 0 0 0 00 15 000 1 0 0 0 0 0 0 00 16 000 1 0 0 0 0 0 0 00 17 000 1 0 0 0 0 0 0 00 Delaying the call to print_ICs() gives better results: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 000 1 0 0 0 0 0 0 00 01 000 0 0 0 0 0 0 0 31 02 000 0 0 0 0 0 0 0 30 03 000 1 0 0 0 0 0 0 33 04 000 1 0 0 0 0 0 0 34 05 000 1 0 0 0 0 0 0 35 06 000 1 0 0 0 0 0 0 36 07 000 1 0 0 0 0 0 0 37 08 000 0 0 0 0 0 0 0 38 09 000 0 1 0 0 0 0 0 39 0a 000 1 0 0 0 0 0 0 3A 0b 000 1 0 0 0 0 0 0 3B 0c 000 0 0 0 0 0 0 0 3C 0d 000 1 0 0 0 0 0 0 3D 0e 000 1 0 0 0 0 0 0 3E 0f 000 1 0 0 0 0 0 0 3F 10 000 1 1 0 1 0 0 0 29 11 000 1 0 0 0 0 0 0 00 12 000 1 0 0 0 0 0 0 00 13 000 1 0 0 0 0 0 0 00 14 000 0 1 0 1 0 0 0 51 15 000 1 0 0 0 0 0 0 00 16 000 0 1 0 1 0 0 0 61 17 000 0 1 0 1 0 0 0 59 Notice that the entries beyond interrupt input signal 0x0f also get populated and arent just the hw-initialization default of all zeroes. Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110708083555.2598.42216.sendpatchset@nchumbalkar.americas.hpqcorp.net Signed-off-by: Ingo Molnar --- arch/x86/kernel/apic/io_apic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index e5293394b548..98c8d7e11c40 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1792,7 +1792,7 @@ __apicdebuginit(int) print_ICs(void) return 0; } -fs_initcall(print_ICs); +late_initcall(print_ICs); /* Where if anywhere is the i8259 connect in external int mode */ -- cgit v1.2.3 From 14cb6dcf0a023f5977461c94d8d5a163c937979b Mon Sep 17 00:00:00 2001 From: Vivek Goyal Date: Fri, 8 Jul 2011 13:19:26 -0400 Subject: x86, boot: Wait for boot cpu to show up if nr_cpus limit is about to hit nr_cpus allows one to specify number of possible cpus in the system. Current assumption seems to be that first cpu to show up is boot cpu and this assumption will be broken in kdump scenario where we can be booting on a non boot cpu with nr_cpus=1. It might happen that first cpu we parse is not the cpu we boot on and later we ignore boot cpu. Though code later seems to recognize this anomaly and forcibly sets boot cpu in physical cpu map with following warning. if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) { printk(KERN_WARNING "weird, boot CPU (#%d) not listed by the BIOS.\n", hard_smp_processor_id()); physid_set(hard_smp_processor_id(), phys_cpu_present_map); } This patch waits for boot cpu to show up and starts ignoring the cpus once we have hit (nr_cpus - 1) number of cpus. So effectively we are reserving one slot out of nr_cpus for boot cpu explicitly. Signed-off-by: Vivek Goyal Acked-by: Yinghai Lu Link: http://lkml.kernel.org/r/20110708171926.GF2930@redhat.com Signed-off-by: H. Peter Anvin --- arch/x86/kernel/apic/apic.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index b9338b8cf420..68219a919dfd 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1943,10 +1943,28 @@ void disconnect_bsp_APIC(int virt_wire_setup) void __cpuinit generic_processor_info(int apicid, int version) { - int cpu; + int cpu, max = nr_cpu_ids; + bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid, + phys_cpu_present_map); + + /* + * If boot cpu has not been detected yet, then only allow upto + * nr_cpu_ids - 1 processors and keep one slot free for boot cpu + */ + if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 && + apicid != boot_cpu_physical_apicid) { + int thiscpu = max + disabled_cpus - 1; + + pr_warning( + "ACPI: NR_CPUS/possible_cpus limit of %i almost" + " reached. Keeping one slot for boot cpu." + " Processor %d/0x%x ignored.\n", max, thiscpu, apicid); + + disabled_cpus++; + return; + } if (num_processors >= nr_cpu_ids) { - int max = nr_cpu_ids; int thiscpu = max + disabled_cpus; pr_warning( -- cgit v1.2.3 From 24a42bae6852d27ae569757f5415c91538e6a255 Mon Sep 17 00:00:00 2001 From: Anupam Chanda Date: Fri, 8 Jul 2011 11:42:50 -0700 Subject: x86, hyper: Change hypervisor detection order Detect Xen before HyperV because in Viridian compatibility mode Xen presents itself as HyperV. Move Xen to the top since it seems more likely that Xen would emulate VMware than vice versa. Signed-off-by: Anupam Chanda Link: http://lkml.kernel.org/r/1310150570-26810-1-git-send-email-achanda@nicira.com Acked-by: Stefano Stabellini Acked-by: Yaozu (Eddie) Dong Reviewed-by: H. Peter Anvin Signed-off-by: H. Peter Anvin --- arch/x86/kernel/cpu/hypervisor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c index 8095f8611f8a..755f64fb0743 100644 --- a/arch/x86/kernel/cpu/hypervisor.c +++ b/arch/x86/kernel/cpu/hypervisor.c @@ -32,11 +32,11 @@ */ static const __initconst struct hypervisor_x86 * const hypervisors[] = { - &x86_hyper_vmware, - &x86_hyper_ms_hyperv, #ifdef CONFIG_XEN_PVHVM &x86_hyper_xen_hvm, #endif + &x86_hyper_vmware, + &x86_hyper_ms_hyperv, }; const struct hypervisor_x86 *x86_hyper; -- cgit v1.2.3 From bd6a46e087571897f0b2736917500b97d18dac13 Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Fri, 8 Jul 2011 18:46:36 +0000 Subject: x86, ioapic: Format clean up for IOAPIC output When IOAPIC data is displayed in "dmesg" with the help of the boot parameter "apic=debug" certain values are not formatted correctly wrt their size. In the "dmesg" snippet below, note that the output for "max redirection entries", and "IO APIC version" which are each defined to be just 8-bits long are displayed as 2 bytes in length. Similarly, "Dst" under the "IRQ redirection table" should only be 8-bits long. IO APIC #0...... ... ... .... register #01: 00170020 ....... : max redirection entries: 0017 ....... : PRQ implemented: 0 ....... : IO APIC version: 0020 ... ... .... IRQ redirection table: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 000 1 0 0 0 0 0 0 00 01 000 0 0 0 0 0 0 0 31 02 000 0 0 0 0 0 0 0 30 03 000 1 0 0 0 0 0 0 33 ... ... Do some formatting clean up, so you will see output like below: IO APIC #0...... ... ... .... register #01: 00170020 ....... : max redirection entries: 17 ....... : PRQ implemented: 0 ....... : IO APIC version: 20 ... ... .... IRQ redirection table: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 00 1 0 0 0 0 0 0 00 01 00 0 0 0 0 0 0 0 31 02 00 0 0 0 0 0 0 0 30 03 00 1 0 0 0 0 0 0 33 ... ... Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110708184557.2734.61830.sendpatchset@nchumbalkar.americas.cpqcorp.net Signed-off-by: Ingo Molnar --- arch/x86/kernel/apic/io_apic.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 98c8d7e11c40..ed4abaa301d0 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1522,10 +1522,12 @@ __apicdebuginit(void) print_IO_APIC(void) printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS); printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)®_01); - printk(KERN_DEBUG "....... : max redirection entries: %04X\n", reg_01.bits.entries); + printk(KERN_DEBUG "....... : max redirection entries: %02X\n", + reg_01.bits.entries); printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ); - printk(KERN_DEBUG "....... : IO APIC version: %04X\n", reg_01.bits.version); + printk(KERN_DEBUG "....... : IO APIC version: %02X\n", + reg_01.bits.version); /* * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02, @@ -1558,7 +1560,7 @@ __apicdebuginit(void) print_IO_APIC(void) entry = ioapic_read_entry(apic, i); - printk(KERN_DEBUG " %02x %03X ", + printk(KERN_DEBUG " %02x %02X ", i, entry.dest ); -- cgit v1.2.3 From 7fece83235a59b15d75d6c8ef2225c24abd4505b Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Fri, 8 Jul 2011 18:46:42 +0000 Subject: x86, ioapic: Also print Dest field The code in setup_ioapic_irq() determines the Destination Field, so why not also include it in the debug printk output that gets displayed when the boot parameter "apic=debug" is used. Before the change, "dmesg" will show: IOAPIC[0]: Set routing entry (8-1 -> 0x31 -> IRQ 1 Mode:0 Active:0) IOAPIC[0]: Set routing entry (8-2 -> 0x30 -> IRQ 0 Mode:0 Active:0) IOAPIC[0]: Set routing entry (8-3 -> 0x33 -> IRQ 3 Mode:0 Active:0) ... After the change, you will see: IOAPIC[0]: Set routing entry (8-1 -> 0x31 -> IRQ 1 Mode:0 Active:0 Dest:0) IOAPIC[0]: Set routing entry (8-2 -> 0x30 -> IRQ 0 Mode:0 Active:0 Dest:0) IOAPIC[0]: Set routing entry (8-3 -> 0x33 -> IRQ 3 Mode:0 Active:0 Dest:0) ... Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110708184603.2734.91071.sendpatchset@nchumbalkar.americas.cpqcorp.net Signed-off-by: Ingo Molnar --- arch/x86/kernel/apic/io_apic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index ed4abaa301d0..5cba200609c0 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1337,9 +1337,9 @@ static void setup_ioapic_irq(int apic_id, int pin, unsigned int irq, apic_printk(APIC_VERBOSE,KERN_DEBUG "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> " - "IRQ %d Mode:%i Active:%i)\n", + "IRQ %d Mode:%i Active:%i Dest:%d)\n", apic_id, mpc_ioapic_id(apic_id), pin, cfg->vector, - irq, trigger, polarity); + irq, trigger, polarity, dest); if (setup_ioapic_entry(mpc_ioapic_id(apic_id), irq, &entry, -- cgit v1.2.3 From 25970852280c9d5fb2de899769880d3e97332baa Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Tue, 12 Jul 2011 05:59:07 +0000 Subject: x86, x2apic: Preserve high 32-bits of IA32_APIC_BASE MSR If there's no special reason to zero-out the "high" 32-bits of the IA32_APIC_BASE MSR, let's preserve it. The x2APIC Specification doesn't explicitly state any such requirement. (Sec 2.2 in: http://www.intel.com/Assets/PDF/manual/318148.pdf). Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110712055831.2498.78521.sendpatchset@nchumbalkar.americas.cpqcorp.net Reviewed-by: Cyrill Gorcunov Reviewed-by: Suresh Siddha Signed-off-by: H. Peter Anvin --- arch/x86/kernel/apic/apic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index b9338b8cf420..f7b0c7a1dfaa 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1429,7 +1429,7 @@ void enable_x2apic(void) rdmsr(MSR_IA32_APICBASE, msr, msr2); if (!(msr & X2APIC_ENABLE)) { printk_once(KERN_INFO "Enabling x2apic\n"); - wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, 0); + wrmsr(MSR_IA32_APICBASE, msr | X2APIC_ENABLE, msr2); } } #endif /* CONFIG_X86_X2APIC */ -- cgit v1.2.3 From 3040db92ee1b6c5b6b6d73f8cdcad54c0da11563 Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Tue, 12 Jul 2011 21:17:41 +0000 Subject: x86, ioapic: Print IRTE when IR is enabled When "apic=debug" is used as a boot parameter, Linux prints the IOAPIC routing entries in "dmesg". Below is output from IOAPIC whose apic_id is 8: # dmesg | grep "routing entry" IOAPIC[8]: Set routing entry (8-1 -> 0x31 -> IRQ 1 Mode:0 Active:0 Dest:0) IOAPIC[8]: Set routing entry (8-2 -> 0x30 -> IRQ 0 Mode:0 Active:0 Dest:0) IOAPIC[8]: Set routing entry (8-3 -> 0x33 -> IRQ 3 Mode:0 Active:0 Dest:0) ... Similarly, when IR (interrupt remapping) is enabled, and the IRTE (interrupt remapping table entry) is set up we should display it. After the fix: # dmesg | grep IRTE IOAPIC[8]: Set IRTE entry (P:1 FPD:0 Dst_Mode:0 Redir_hint:1 Trig_Mode:0 Dlvry_Mode:0 Avail:0 Vector:31 Dest:00000000 SID:00F1 SQ:0 SVT:1) IOAPIC[8]: Set IRTE entry (P:1 FPD:0 Dst_Mode:0 Redir_hint:1 Trig_Mode:0 Dlvry_Mode:0 Avail:0 Vector:30 Dest:00000000 SID:00F1 SQ:0 SVT:1) IOAPIC[8]: Set IRTE entry (P:1 FPD:0 Dst_Mode:0 Redir_hint:1 Trig_Mode:0 Dlvry_Mode:0 Avail:0 Vector:33 Dest:00000000 SID:00F1 SQ:0 SVT:1) ... The IRTE is defined in Sec 9.5 of the Intel VT-d Specification. Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110712211704.2939.71291.sendpatchset@nchumbalkar.americas.cpqcorp.net Signed-off-by: H. Peter Anvin --- arch/x86/kernel/apic/io_apic.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 5cba200609c0..1ed5cbb9a14a 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1295,6 +1295,16 @@ static int setup_ioapic_entry(int apic_id, int irq, * irq handler will do the explicit EOI to the io-apic. */ ir_entry->vector = pin; + + apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: " + "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d " + "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X " + "Avail:%X Vector:%02X Dest:%08X " + "SID:%04X SQ:%X SVT:%X)\n", + apic_id, irte.present, irte.fpd, irte.dst_mode, + irte.redir_hint, irte.trigger_mode, irte.dlvry_mode, + irte.avail, irte.vector, irte.dest_id, + irte.sid, irte.sq, irte.svt); } else { entry->delivery_mode = apic->irq_delivery_mode; entry->dest_mode = apic->irq_dest_mode; -- cgit v1.2.3 From 42f0efc5aae2bd7e3bc420c0902c7024ef77391f Mon Sep 17 00:00:00 2001 From: Naga Chumbalkar Date: Tue, 12 Jul 2011 21:17:35 +0000 Subject: x86, ioapic: Print IR_IO_APIC_route_entry when IR is enabled When IR (interrupt remapping) is enabled print_IO_APIC() displays output according to legacy RTE (redirection table entry) definitons: NR Dst Mask Trig IRR Pol Stat Dmod Deli Vect: 00 00 1 0 0 0 0 0 0 00 01 00 0 0 0 0 0 0 0 01 02 00 0 0 0 0 0 0 0 02 03 00 1 0 0 0 0 0 0 03 04 00 1 0 0 0 0 0 0 04 05 00 1 0 0 0 0 0 0 05 06 00 1 0 0 0 0 0 0 06 ... The above output is as per Sec 3.2.4 of the IOAPIC datasheet: 82093AA I/O Advanced Programmable Interrupt Controller (IOAPIC): http://download.intel.com/design/chipsets/datashts/29056601.pdf Instead the output should display the fields as discussed in Sec 5.5.1 of the VT-d specification: (Intel Virtualization Technology for Directed I/O: http://download.intel.com/technology/computing/vptech/Intel(r)_VT_for_Direct_IO.pdf) After the fix: NR Indx Fmt Mask Trig IRR Pol Stat Indx2 Zero Vect: 00 0000 0 1 0 0 0 0 0 0 00 01 000F 1 0 0 0 0 0 0 0 01 02 0001 1 0 0 0 0 0 0 0 02 03 0002 1 1 0 0 0 0 0 0 03 04 0011 1 1 0 0 0 0 0 0 04 05 0004 1 1 0 0 0 0 0 0 05 06 0005 1 1 0 0 0 0 0 0 06 ... Signed-off-by: Naga Chumbalkar Link: http://lkml.kernel.org/r/20110712211658.2939.93123.sendpatchset@nchumbalkar.americas.cpqcorp.net Signed-off-by: H. Peter Anvin --- arch/x86/kernel/apic/io_apic.c | 69 ++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 20 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 1ed5cbb9a14a..8eb863e27ea6 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -1562,31 +1562,60 @@ __apicdebuginit(void) print_IO_APIC(void) printk(KERN_DEBUG ".... IRQ redirection table:\n"); - printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol" - " Stat Dmod Deli Vect:\n"); + if (intr_remapping_enabled) { + printk(KERN_DEBUG " NR Indx Fmt Mask Trig IRR" + " Pol Stat Indx2 Zero Vect:\n"); + } else { + printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol" + " Stat Dmod Deli Vect:\n"); + } for (i = 0; i <= reg_01.bits.entries; i++) { - struct IO_APIC_route_entry entry; - - entry = ioapic_read_entry(apic, i); - - printk(KERN_DEBUG " %02x %02X ", - i, - entry.dest - ); + if (intr_remapping_enabled) { + struct IO_APIC_route_entry entry; + struct IR_IO_APIC_route_entry *ir_entry; + + entry = ioapic_read_entry(apic, i); + ir_entry = (struct IR_IO_APIC_route_entry *) &entry; + printk(KERN_DEBUG " %02x %04X ", + i, + ir_entry->index + ); + printk("%1d %1d %1d %1d %1d " + "%1d %1d %X %02X\n", + ir_entry->format, + ir_entry->mask, + ir_entry->trigger, + ir_entry->irr, + ir_entry->polarity, + ir_entry->delivery_status, + ir_entry->index2, + ir_entry->zero, + ir_entry->vector + ); + } else { + struct IO_APIC_route_entry entry; - printk("%1d %1d %1d %1d %1d %1d %1d %02X\n", - entry.mask, - entry.trigger, - entry.irr, - entry.polarity, - entry.delivery_status, - entry.dest_mode, - entry.delivery_mode, - entry.vector - ); + entry = ioapic_read_entry(apic, i); + printk(KERN_DEBUG " %02x %02X ", + i, + entry.dest + ); + printk("%1d %1d %1d %1d %1d " + "%1d %1d %02X\n", + entry.mask, + entry.trigger, + entry.irr, + entry.polarity, + entry.delivery_status, + entry.dest_mode, + entry.delivery_mode, + entry.vector + ); + } } } + printk(KERN_DEBUG "IRQ to pin mappings:\n"); for_each_active_irq(irq) { struct irq_pin_list *entry; -- cgit v1.2.3 From 3628c3f5c818cfc6e588d1ccb31f19aa12345c02 Mon Sep 17 00:00:00 2001 From: Maxime Ripard Date: Tue, 28 Jun 2011 15:57:31 +0200 Subject: x86. reboot: Make Dell Latitude E6320 use reboot=pci The Dell Latitude E6320 doesn't reboot unless reboot=pci is set. Force it thanks to DMI. Signed-off-by: Maxime Ripard Link: http://lkml.kernel.org/r/1309269451-4966-1-git-send-email-maxime.ripard@free-electrons.com Cc: Matthew Garrett Signed-off-by: H. Peter Anvin --- arch/x86/kernel/reboot.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index 4f0d46fefa7f..14eed214b584 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -419,6 +419,14 @@ static struct dmi_system_id __initdata pci_reboot_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "iMac9,1"), }, }, + { /* Handle problems with rebooting on the Latitude E6320. */ + .callback = set_pci_reboot, + .ident = "Dell Latitude E6320", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Latitude E6320"), + }, + }, { } }; -- cgit v1.2.3 From c9712944b2a12373cb6ff8059afcfb7e826a6c54 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 13 Jul 2011 09:24:09 -0400 Subject: x86-64: Improve vsyscall emulation CS and RIP handling Three fixes here: - Send SIGSEGV if called from compat code or with a funny CS. - Don't BUG on impossible addresses. - Add a missing local_irq_disable. This patch also removes an unused variable. Signed-off-by: Andy Lutomirski Link: http://lkml.kernel.org/r/6fb2b13ab39b743d1e4f466eef13425854912f7f.1310563276.git.luto@mit.edu Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/vsyscall.h | 12 -------- arch/x86/kernel/vsyscall_64.c | 61 +++++++++++++++++++++++++++-------------- 2 files changed, 41 insertions(+), 32 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/vsyscall.h b/arch/x86/include/asm/vsyscall.h index bb710cb0cdc1..d55597351f6a 100644 --- a/arch/x86/include/asm/vsyscall.h +++ b/arch/x86/include/asm/vsyscall.h @@ -31,18 +31,6 @@ extern struct timezone sys_tz; extern void map_vsyscall(void); -/* Emulation */ - -static inline bool is_vsyscall_entry(unsigned long addr) -{ - return (addr & ~0xC00UL) == VSYSCALL_START; -} - -static inline int vsyscall_entry_nr(unsigned long addr) -{ - return (addr & 0xC00UL) >> 10; -} - #endif /* __KERNEL__ */ #endif /* _ASM_X86_VSYSCALL_H */ diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index 10cd8ac3395a..a262400c3479 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -97,33 +98,63 @@ static void warn_bad_vsyscall(const char *level, struct pt_regs *regs, tsk = current; - printk("%s%s[%d] %s ip:%lx sp:%lx ax:%lx si:%lx di:%lx\n", + printk("%s%s[%d] %s ip:%lx cs:%lx sp:%lx ax:%lx si:%lx di:%lx\n", level, tsk->comm, task_pid_nr(tsk), - message, regs->ip - 2, regs->sp, regs->ax, regs->si, regs->di); + message, regs->ip - 2, regs->cs, + regs->sp, regs->ax, regs->si, regs->di); +} + +static int addr_to_vsyscall_nr(unsigned long addr) +{ + int nr; + + if ((addr & ~0xC00UL) != VSYSCALL_START) + return -EINVAL; + + nr = (addr & 0xC00UL) >> 10; + if (nr >= 3) + return -EINVAL; + + return nr; } void dotraplinkage do_emulate_vsyscall(struct pt_regs *regs, long error_code) { - const char *vsyscall_name; struct task_struct *tsk; unsigned long caller; int vsyscall_nr; long ret; - /* Kernel code must never get here. */ - BUG_ON(!user_mode(regs)); - local_irq_enable(); + /* + * Real 64-bit user mode code has cs == __USER_CS. Anything else + * is bogus. + */ + if (regs->cs != __USER_CS) { + /* + * If we trapped from kernel mode, we might as well OOPS now + * instead of returning to some random address and OOPSing + * then. + */ + BUG_ON(!user_mode(regs)); + + /* Compat mode and non-compat 32-bit CS should both segfault. */ + warn_bad_vsyscall(KERN_WARNING, regs, + "illegal int 0xcc from 32-bit mode"); + goto sigsegv; + } + /* * x86-ism here: regs->ip points to the instruction after the int 0xcc, * and int 0xcc is two bytes long. */ - if (!is_vsyscall_entry(regs->ip - 2)) { - warn_bad_vsyscall(KERN_WARNING, regs, "illegal int 0xcc (exploit attempt?)"); + vsyscall_nr = addr_to_vsyscall_nr(regs->ip - 2); + if (vsyscall_nr < 0) { + warn_bad_vsyscall(KERN_WARNING, regs, + "illegal int 0xcc (exploit attempt?)"); goto sigsegv; } - vsyscall_nr = vsyscall_entry_nr(regs->ip - 2); if (get_user(caller, (unsigned long __user *)regs->sp) != 0) { warn_bad_vsyscall(KERN_WARNING, regs, "int 0xcc with bad stack (exploit attempt?)"); @@ -136,31 +167,20 @@ void dotraplinkage do_emulate_vsyscall(struct pt_regs *regs, long error_code) switch (vsyscall_nr) { case 0: - vsyscall_name = "gettimeofday"; ret = sys_gettimeofday( (struct timeval __user *)regs->di, (struct timezone __user *)regs->si); break; case 1: - vsyscall_name = "time"; ret = sys_time((time_t __user *)regs->di); break; case 2: - vsyscall_name = "getcpu"; ret = sys_getcpu((unsigned __user *)regs->di, (unsigned __user *)regs->si, 0); break; - - default: - /* - * If we get here, then vsyscall_nr indicates that int 0xcc - * happened at an address in the vsyscall page that doesn't - * contain int 0xcc. That can't happen. - */ - BUG(); } if (ret == -EFAULT) { @@ -188,6 +208,7 @@ void dotraplinkage do_emulate_vsyscall(struct pt_regs *regs, long error_code) sigsegv: regs->ip -= 2; /* The faulting instruction should be the int 0xcc. */ force_sig(SIGSEGV, current); + local_irq_disable(); } /* -- cgit v1.2.3 From 59e97e4d6fbcd5b74a94cb48bcbfc6f8478a5e93 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 13 Jul 2011 09:24:10 -0400 Subject: x86: Make alternative instruction pointers relative This save a few bytes on x86-64 and means that future patches can apply alternatives to unrelocated code. Signed-off-by: Andy Lutomirski Link: http://lkml.kernel.org/r/ff64a6b9a1a3860ca4a7b8b6dc7b4754f9491cd7.1310563276.git.luto@mit.edu Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/alternative-asm.h | 4 ++-- arch/x86/include/asm/alternative.h | 8 ++++---- arch/x86/include/asm/cpufeature.h | 8 ++++---- arch/x86/kernel/alternative.c | 21 +++++++++++++-------- arch/x86/lib/copy_page_64.S | 9 +++------ arch/x86/lib/memmove_64.S | 11 +++++------ 6 files changed, 31 insertions(+), 30 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/alternative-asm.h b/arch/x86/include/asm/alternative-asm.h index 94d420b360d1..4554cc6fb96a 100644 --- a/arch/x86/include/asm/alternative-asm.h +++ b/arch/x86/include/asm/alternative-asm.h @@ -17,8 +17,8 @@ .macro altinstruction_entry orig alt feature orig_len alt_len .align 8 - .quad \orig - .quad \alt + .long \orig - . + .long \alt - . .word \feature .byte \orig_len .byte \alt_len diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index bf535f947e8c..23fb6d79f209 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -43,8 +43,8 @@ #endif struct alt_instr { - u8 *instr; /* original instruction */ - u8 *replacement; + s32 instr_offset; /* original instruction */ + s32 repl_offset; /* offset to replacement instruction */ u16 cpuid; /* cpuid bit set for replacement */ u8 instrlen; /* length of original instruction */ u8 replacementlen; /* length of new instruction, <= instrlen */ @@ -84,8 +84,8 @@ static inline int alternatives_text_reserved(void *start, void *end) "661:\n\t" oldinstr "\n662:\n" \ ".section .altinstructions,\"a\"\n" \ _ASM_ALIGN "\n" \ - _ASM_PTR "661b\n" /* label */ \ - _ASM_PTR "663f\n" /* new instruction */ \ + " .long 661b - .\n" /* label */ \ + " .long 663f - .\n" /* new instruction */ \ " .word " __stringify(feature) "\n" /* feature bit */ \ " .byte 662b-661b\n" /* sourcelen */ \ " .byte 664f-663f\n" /* replacementlen */ \ diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 71cc3800712c..9929b35929ff 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -331,8 +331,8 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) "2:\n" ".section .altinstructions,\"a\"\n" _ASM_ALIGN "\n" - _ASM_PTR "1b\n" - _ASM_PTR "0\n" /* no replacement */ + " .long 1b - .\n" + " .long 0\n" /* no replacement */ " .word %P0\n" /* feature bit */ " .byte 2b - 1b\n" /* source len */ " .byte 0\n" /* replacement len */ @@ -349,8 +349,8 @@ static __always_inline __pure bool __static_cpu_has(u16 bit) "2:\n" ".section .altinstructions,\"a\"\n" _ASM_ALIGN "\n" - _ASM_PTR "1b\n" - _ASM_PTR "3f\n" + " .long 1b - .\n" + " .long 3f - .\n" " .word %P1\n" /* feature bit */ " .byte 2b - 1b\n" /* source len */ " .byte 4f - 3f\n" /* replacement len */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index a81f2d52f869..ddb207bb5f91 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -263,6 +263,7 @@ void __init_or_module apply_alternatives(struct alt_instr *start, struct alt_instr *end) { struct alt_instr *a; + u8 *instr, *replacement; u8 insnbuf[MAX_PATCH_LEN]; DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); @@ -276,25 +277,29 @@ void __init_or_module apply_alternatives(struct alt_instr *start, * order. */ for (a = start; a < end; a++) { - u8 *instr = a->instr; + instr = (u8 *)&a->instr_offset + a->instr_offset; + replacement = (u8 *)&a->repl_offset + a->repl_offset; BUG_ON(a->replacementlen > a->instrlen); BUG_ON(a->instrlen > sizeof(insnbuf)); BUG_ON(a->cpuid >= NCAPINTS*32); if (!boot_cpu_has(a->cpuid)) continue; + + memcpy(insnbuf, replacement, a->replacementlen); + + /* 0xe8 is a relative jump; fix the offset. */ + if (*insnbuf == 0xe8 && a->replacementlen == 5) + *(s32 *)(insnbuf + 1) += replacement - instr; + + add_nops(insnbuf + a->replacementlen, + a->instrlen - a->replacementlen); + #ifdef CONFIG_X86_64 /* vsyscall code is not mapped yet. resolve it manually. */ if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) { instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0)); - DPRINTK("%s: vsyscall fixup: %p => %p\n", - __func__, a->instr, instr); } #endif - memcpy(insnbuf, a->replacement, a->replacementlen); - if (*insnbuf == 0xe8 && a->replacementlen == 5) - *(s32 *)(insnbuf + 1) += a->replacement - a->instr; - add_nops(insnbuf + a->replacementlen, - a->instrlen - a->replacementlen); text_poke_early(instr, insnbuf, a->instrlen); } } diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S index 6fec2d1cebe1..01c805ba5359 100644 --- a/arch/x86/lib/copy_page_64.S +++ b/arch/x86/lib/copy_page_64.S @@ -2,6 +2,7 @@ #include #include +#include ALIGN copy_page_c: @@ -110,10 +111,6 @@ ENDPROC(copy_page) 2: .previous .section .altinstructions,"a" - .align 8 - .quad copy_page - .quad 1b - .word X86_FEATURE_REP_GOOD - .byte .Lcopy_page_end - copy_page - .byte 2b - 1b + altinstruction_entry copy_page, 1b, X86_FEATURE_REP_GOOD, \ + .Lcopy_page_end-copy_page, 2b-1b .previous diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S index d0ec9c2936d7..ee164610ec46 100644 --- a/arch/x86/lib/memmove_64.S +++ b/arch/x86/lib/memmove_64.S @@ -9,6 +9,7 @@ #include #include #include +#include #undef memmove @@ -214,11 +215,9 @@ ENTRY(memmove) .previous .section .altinstructions,"a" - .align 8 - .quad .Lmemmove_begin_forward - .quad .Lmemmove_begin_forward_efs - .word X86_FEATURE_ERMS - .byte .Lmemmove_end_forward-.Lmemmove_begin_forward - .byte .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs + altinstruction_entry .Lmemmove_begin_forward, \ + .Lmemmove_begin_forward_efs,X86_FEATURE_ERMS, \ + .Lmemmove_end_forward-.Lmemmove_begin_forward, \ + .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs .previous ENDPROC(memmove) -- cgit v1.2.3 From 433bd805e5fd2c731b3a9025b034f066272d336e Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Wed, 13 Jul 2011 09:24:13 -0400 Subject: clocksource: Replace vread with generic arch data The vread field was bloating struct clocksource everywhere except x86_64, and I want to change the way this works on x86_64, so let's split it out into per-arch data. Cc: x86@kernel.org Cc: Clemens Ladisch Cc: linux-ia64@vger.kernel.org Cc: Tony Luck Cc: Fenghua Yu Cc: John Stultz Cc: Thomas Gleixner Signed-off-by: Andy Lutomirski Link: http://lkml.kernel.org/r/3ae5ec76a168eaaae63f08a2a1060b91aa0b7759.1310563276.git.luto@mit.edu Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/clocksource.h | 16 ++++++++++++++++ arch/x86/kernel/hpet.c | 2 +- arch/x86/kernel/tsc.c | 2 +- arch/x86/kernel/vsyscall_64.c | 2 +- include/asm-generic/clocksource.h | 4 ++++ include/linux/clocksource.h | 10 ++++++++-- 6 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 arch/x86/include/asm/clocksource.h create mode 100644 include/asm-generic/clocksource.h (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/clocksource.h b/arch/x86/include/asm/clocksource.h new file mode 100644 index 000000000000..a5df33f614c9 --- /dev/null +++ b/arch/x86/include/asm/clocksource.h @@ -0,0 +1,16 @@ +/* x86-specific clocksource additions */ + +#ifndef _ASM_X86_CLOCKSOURCE_H +#define _ASM_X86_CLOCKSOURCE_H + +#ifdef CONFIG_X86_64 + +#define __ARCH_HAS_CLOCKSOURCE_DATA + +struct arch_clocksource_data { + cycle_t (*vread)(void); +}; + +#endif /* CONFIG_X86_64 */ + +#endif /* _ASM_X86_CLOCKSOURCE_H */ diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index e9f5605e4748..0e07257bb389 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -753,7 +753,7 @@ static struct clocksource clocksource_hpet = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, .resume = hpet_resume_counter, #ifdef CONFIG_X86_64 - .vread = vread_hpet, + .archdata = { .vread = vread_hpet }, #endif }; diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 6cc6922262af..e7a74b889ab3 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -777,7 +777,7 @@ static struct clocksource clocksource_tsc = { .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_MUST_VERIFY, #ifdef CONFIG_X86_64 - .vread = vread_tsc, + .archdata = { .vread = vread_tsc }, #endif }; diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index a262400c3479..12d488fd95d9 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -74,7 +74,7 @@ void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags); /* copy vsyscall data */ - vsyscall_gtod_data.clock.vread = clock->vread; + vsyscall_gtod_data.clock.vread = clock->archdata.vread; vsyscall_gtod_data.clock.cycle_last = clock->cycle_last; vsyscall_gtod_data.clock.mask = clock->mask; vsyscall_gtod_data.clock.mult = mult; diff --git a/include/asm-generic/clocksource.h b/include/asm-generic/clocksource.h new file mode 100644 index 000000000000..0a462d3fb05e --- /dev/null +++ b/include/asm-generic/clocksource.h @@ -0,0 +1,4 @@ +/* + * Architectures should override this file to add private userspace + * clock magic if needed. + */ diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index d4646b48dc4a..0fb83c224471 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -22,6 +22,8 @@ typedef u64 cycle_t; struct clocksource; +#include + /** * struct cyclecounter - hardware abstraction for a free running counter * Provides completely state-free accessors to the underlying hardware. @@ -153,7 +155,7 @@ extern u64 timecounter_cyc2time(struct timecounter *tc, * @shift: cycle to nanosecond divisor (power of two) * @max_idle_ns: max idle time permitted by the clocksource (nsecs) * @flags: flags describing special properties - * @vread: vsyscall based read + * @archdata: arch-specific data * @suspend: suspend function for the clocksource, if necessary * @resume: resume function for the clocksource, if necessary */ @@ -175,10 +177,14 @@ struct clocksource { #else #define CLKSRC_FSYS_MMIO_SET(mmio, addr) do { } while (0) #endif + +#ifdef __ARCH_HAS_CLOCKSOURCE_DATA + struct arch_clocksource_data archdata; +#endif + const char *name; struct list_head list; int rating; - cycle_t (*vread)(void); int (*enable)(struct clocksource *cs); void (*disable)(struct clocksource *cs); unsigned long flags; -- cgit v1.2.3 From 3c404b578fab699c4708279938078d9404b255a4 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Mon, 11 Jul 2011 15:28:15 -0400 Subject: KVM guest: Add a pv_ops stub for steal time This patch adds a function pointer in one of the many paravirt_ops structs, to allow guests to register a steal time function. Besides a steal time function, we also declare two jump_labels. They will be used to allow the steal time code to be easily bypassed when not in use. Signed-off-by: Glauber Costa Acked-by: Rik van Riel Tested-by: Eric B Munson CC: Jeremy Fitzhardinge CC: Peter Zijlstra CC: Anthony Liguori Signed-off-by: Avi Kivity --- arch/x86/include/asm/paravirt.h | 9 +++++++++ arch/x86/include/asm/paravirt_types.h | 1 + arch/x86/kernel/paravirt.c | 9 +++++++++ 3 files changed, 19 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index ebbc4d8ab170..a7d2db9a74fb 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -230,6 +230,15 @@ static inline unsigned long long paravirt_sched_clock(void) return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); } +struct jump_label_key; +extern struct jump_label_key paravirt_steal_enabled; +extern struct jump_label_key paravirt_steal_rq_enabled; + +static inline u64 paravirt_steal_clock(int cpu) +{ + return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu); +} + static inline unsigned long long paravirt_read_pmc(int counter) { return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index 82885099c869..2c7652163111 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -89,6 +89,7 @@ struct pv_lazy_ops { struct pv_time_ops { unsigned long long (*sched_clock)(void); + unsigned long long (*steal_clock)(int cpu); unsigned long (*get_tsc_khz)(void); }; diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 869e1aeeb71b..613a7931ecc1 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -202,6 +202,14 @@ static void native_flush_tlb_single(unsigned long addr) __native_flush_tlb_single(addr); } +struct jump_label_key paravirt_steal_enabled; +struct jump_label_key paravirt_steal_rq_enabled; + +static u64 native_steal_clock(int cpu) +{ + return 0; +} + /* These are in entry.S */ extern void native_iret(void); extern void native_irq_enable_sysexit(void); @@ -307,6 +315,7 @@ struct pv_init_ops pv_init_ops = { struct pv_time_ops pv_time_ops = { .sched_clock = native_sched_clock, + .steal_clock = native_steal_clock, }; struct pv_irq_ops pv_irq_ops = { -- cgit v1.2.3 From abe48b108247e9b90b4c6739662a2e5c765ed114 Mon Sep 17 00:00:00 2001 From: Len Brown Date: Thu, 14 Jul 2011 00:53:24 -0400 Subject: x86, intel, power: Initialize MSR_IA32_ENERGY_PERF_BIAS Since 2.6.36 (23016bf0d25), Linux prints the existence of "epb" in /proc/cpuinfo, Since 2.6.38 (d5532ee7b40), the x86_energy_perf_policy(8) utility has been available in-tree to update MSR_IA32_ENERGY_PERF_BIAS. However, the typical BIOS fails to initialize the MSR, presumably because this is handled by high-volume shrink-wrap operating systems... Linux distros, on the other hand, do not yet invoke x86_energy_perf_policy(8). As a result, WSM-EP, SNB, and later hardware from Intel will run in its default hardware power-on state (performance), which assumes that users care for performance at all costs and not for energy efficiency. While that is fine for performance benchmarks, the hardware's intended default operating point is "normal" mode... Initialize the MSR to the "normal" by default during kernel boot. x86_energy_perf_policy(8) is available to change the default after boot, should the user have a different preference. Signed-off-by: Len Brown Link: http://lkml.kernel.org/r/alpine.LFD.2.02.1107140051020.18606@x980 Acked-by: Rafael J. Wysocki Signed-off-by: H. Peter Anvin Cc: --- arch/x86/include/asm/msr-index.h | 3 +++ arch/x86/kernel/cpu/intel.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 485b4f1f079b..23a9d898baad 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -259,6 +259,9 @@ #define MSR_IA32_TEMPERATURE_TARGET 0x000001a2 #define MSR_IA32_ENERGY_PERF_BIAS 0x000001b0 +#define ENERGY_PERF_BIAS_PERFORMANCE 0 +#define ENERGY_PERF_BIAS_NORMAL 6 +#define ENERGY_PERF_BIAS_POWERSWAVE 15 #define MSR_IA32_PACKAGE_THERM_STATUS 0x000001b1 diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 1edf5ba4fb2b..da0d779ecd9e 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -456,6 +456,24 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c) if (cpu_has(c, X86_FEATURE_VMX)) detect_vmx_virtcap(c); + + /* + * Initialize MSR_IA32_ENERGY_PERF_BIAS if BIOS did not. + * x86_energy_perf_policy(8) is available to change it at run-time + */ + if (cpu_has(c, X86_FEATURE_EPB)) { + u64 epb; + + rdmsrl(MSR_IA32_ENERGY_PERF_BIAS, epb); + if ((epb & 0xF) == 0) { + printk_once(KERN_WARNING, "x86: updated energy_perf_bias" + " to 'normal' from 'performance'\n" + "You can view and update epb via utility," + " such as x86_energy_perf_policy(8)\n"); + epb = (epb & ~0xF) | ENERGY_PERF_BIAS_NORMAL; + wrmsrl(MSR_IA32_ENERGY_PERF_BIAS, epb); + } + } } #ifdef CONFIG_X86_32 -- cgit v1.2.3 From f91298709790b9a483752ca3c967845537df2af3 Mon Sep 17 00:00:00 2001 From: Cyrill Gorcunov Date: Sat, 9 Jul 2011 00:17:12 +0400 Subject: perf, x86: P4 PMU - Introduce event alias feature Instead of hw_nmi_watchdog_set_attr() weak function and appropriate x86_pmu::hw_watchdog_set_attr() call we introduce even alias mechanism which allow us to drop this routines completely and isolate quirks of Netburst architecture inside P4 PMU code only. The main idea remains the same though -- to allow nmi-watchdog and perf top run simultaneously. Note the aliasing mechanism applies to generic PERF_COUNT_HW_CPU_CYCLES event only because arbitrary event (say passed as RAW initially) might have some additional bits set inside ESCR register changing the behaviour of event and we can't guarantee anymore that alias event will give the same result. P.S. Thanks a huge to Don and Steven for for testing and early review. Acked-by: Don Zickus Tested-by: Steven Rostedt Signed-off-by: Cyrill Gorcunov CC: Ingo Molnar CC: Peter Zijlstra CC: Stephane Eranian CC: Lin Ming CC: Arnaldo Carvalho de Melo CC: Frederic Weisbecker Link: http://lkml.kernel.org/r/20110708201712.GS23657@sun Signed-off-by: Steven Rostedt --- arch/x86/include/asm/perf_event_p4.h | 33 +++++++++ arch/x86/kernel/cpu/perf_event.c | 7 -- arch/x86/kernel/cpu/perf_event_p4.c | 135 +++++++++++++++++++++++++++-------- kernel/watchdog.c | 2 - 4 files changed, 139 insertions(+), 38 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/perf_event_p4.h b/arch/x86/include/asm/perf_event_p4.h index 56fd9e3abbda..4d86c86178f2 100644 --- a/arch/x86/include/asm/perf_event_p4.h +++ b/arch/x86/include/asm/perf_event_p4.h @@ -101,6 +101,14 @@ #define P4_CONFIG_HT_SHIFT 63 #define P4_CONFIG_HT (1ULL << P4_CONFIG_HT_SHIFT) +/* + * If an event has alias it should be marked + * with a special bit. (Don't forget to check + * P4_PEBS_CONFIG_MASK and related bits on + * modification.) + */ +#define P4_CONFIG_ALIASABLE (1 << 9) + /* * The bits we allow to pass for RAW events */ @@ -123,6 +131,31 @@ (p4_config_pack_escr(P4_CONFIG_MASK_ESCR)) | \ (p4_config_pack_cccr(P4_CONFIG_MASK_CCCR)) +/* + * In case of event aliasing we need to preserve some + * caller bits otherwise the mapping won't be complete. + */ +#define P4_CONFIG_EVENT_ALIAS_MASK \ + (p4_config_pack_escr(P4_CONFIG_MASK_ESCR) | \ + p4_config_pack_cccr(P4_CCCR_EDGE | \ + P4_CCCR_THRESHOLD_MASK | \ + P4_CCCR_COMPLEMENT | \ + P4_CCCR_COMPARE)) + +#define P4_CONFIG_EVENT_ALIAS_IMMUTABLE_BITS \ + ((P4_CONFIG_HT) | \ + p4_config_pack_escr(P4_ESCR_T0_OS | \ + P4_ESCR_T0_USR | \ + P4_ESCR_T1_OS | \ + P4_ESCR_T1_USR) | \ + p4_config_pack_cccr(P4_CCCR_OVF | \ + P4_CCCR_CASCADE | \ + P4_CCCR_FORCE_OVF | \ + P4_CCCR_THREAD_ANY | \ + P4_CCCR_OVF_PMI_T0 | \ + P4_CCCR_OVF_PMI_T1 | \ + P4_CONFIG_ALIASABLE)) + static inline bool p4_is_event_cascaded(u64 config) { u32 cccr = p4_config_unpack_cccr(config); diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index c53d433c3dde..b7a010fce8c3 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -274,7 +274,6 @@ struct x86_pmu { void (*enable_all)(int added); void (*enable)(struct perf_event *); void (*disable)(struct perf_event *); - void (*hw_watchdog_set_attr)(struct perf_event_attr *attr); int (*hw_config)(struct perf_event *event); int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign); unsigned eventsel; @@ -360,12 +359,6 @@ static u64 __read_mostly hw_cache_extra_regs [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX]; -void hw_nmi_watchdog_set_attr(struct perf_event_attr *wd_attr) -{ - if (x86_pmu.hw_watchdog_set_attr) - x86_pmu.hw_watchdog_set_attr(wd_attr); -} - /* * Propagate event elapsed time into the generic event. * Can only be executed on the CPU where the event is active. diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c index fb901c5080f7..8b7a0c306784 100644 --- a/arch/x86/kernel/cpu/perf_event_p4.c +++ b/arch/x86/kernel/cpu/perf_event_p4.c @@ -570,11 +570,92 @@ static __initconst const u64 p4_hw_cache_event_ids }, }; +/* + * Because of Netburst being quite restricted in now + * many same events can run simultaneously, we use + * event aliases, ie different events which have the + * same functionallity but use non-intersected resources + * (ESCR/CCCR/couter registers). This allow us to run + * two or more semi-same events together. It is done + * transparently to a user space. + * + * Never set any cusom internal bits such as P4_CONFIG_HT, + * P4_CONFIG_ALIASABLE or bits for P4_PEBS_METRIC, they are + * either up-to-dated automatically either not appliable + * at all. + * + * And be really carefull choosing aliases! + */ +struct p4_event_alias { + u64 orig; + u64 alter; +} p4_event_aliases[] = { + { + /* + * Non-halted cycles can be substituted with + * non-sleeping cycles (see Intel SDM Vol3b for + * details). + */ + .orig = + p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_GLOBAL_POWER_EVENTS) | + P4_ESCR_EMASK_BIT(P4_EVENT_GLOBAL_POWER_EVENTS, RUNNING)), + .alter = + p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_EXECUTION_EVENT) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS0)| + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS1)| + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS2)| + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS3)| + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS0) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS1) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS2) | + P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS3))| + p4_config_pack_cccr(P4_CCCR_THRESHOLD(15) | P4_CCCR_COMPLEMENT | + P4_CCCR_COMPARE), + }, +}; + +static u64 p4_get_alias_event(u64 config) +{ + u64 config_match; + int i; + + /* + * Probably we're lucky and don't have to do + * matching over all config bits. + */ + if (!(config & P4_CONFIG_ALIASABLE)) + return 0; + + config_match = config & P4_CONFIG_EVENT_ALIAS_MASK; + + /* + * If an event was previously swapped to the alter config + * we should swap it back otherwise contnention on registers + * will return back. + */ + for (i = 0; i < ARRAY_SIZE(p4_event_aliases); i++) { + if (config_match == p4_event_aliases[i].orig) { + config_match = p4_event_aliases[i].alter; + break; + } else if (config_match == p4_event_aliases[i].alter) { + config_match = p4_event_aliases[i].orig; + break; + } + } + + if (i >= ARRAY_SIZE(p4_event_aliases)) + return 0; + + return config_match | + (config & P4_CONFIG_EVENT_ALIAS_IMMUTABLE_BITS); +} + static u64 p4_general_events[PERF_COUNT_HW_MAX] = { /* non-halted CPU clocks */ [PERF_COUNT_HW_CPU_CYCLES] = p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_GLOBAL_POWER_EVENTS) | - P4_ESCR_EMASK_BIT(P4_EVENT_GLOBAL_POWER_EVENTS, RUNNING)), + P4_ESCR_EMASK_BIT(P4_EVENT_GLOBAL_POWER_EVENTS, RUNNING)) | + P4_CONFIG_ALIASABLE, /* * retired instructions @@ -719,31 +800,6 @@ static int p4_validate_raw_event(struct perf_event *event) return 0; } -static void p4_hw_watchdog_set_attr(struct perf_event_attr *wd_attr) -{ - /* - * Watchdog ticks are special on Netburst, we use - * that named "non-sleeping" ticks as recommended - * by Intel SDM Vol3b. - */ - WARN_ON_ONCE(wd_attr->type != PERF_TYPE_HARDWARE || - wd_attr->config != PERF_COUNT_HW_CPU_CYCLES); - - wd_attr->type = PERF_TYPE_RAW; - wd_attr->config = - p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_EXECUTION_EVENT) | - P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS0) | - P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS1) | - P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS2) | - P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS3) | - P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS0) | - P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS1) | - P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS2) | - P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS3)) | - p4_config_pack_cccr(P4_CCCR_THRESHOLD(15) | P4_CCCR_COMPLEMENT | - P4_CCCR_COMPARE); -} - static int p4_hw_config(struct perf_event *event) { int cpu = get_cpu(); @@ -1159,6 +1215,8 @@ static int p4_pmu_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign struct p4_event_bind *bind; unsigned int i, thread, num; int cntr_idx, escr_idx; + u64 config_alias; + int pass; bitmap_zero(used_mask, X86_PMC_IDX_MAX); bitmap_zero(escr_mask, P4_ESCR_MSR_TABLE_SIZE); @@ -1167,6 +1225,17 @@ static int p4_pmu_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign hwc = &cpuc->event_list[i]->hw; thread = p4_ht_thread(cpu); + pass = 0; + +again: + /* + * Aliases are swappable so we may hit circular + * lock if both original config and alias need + * resources (MSR registers) which already busy. + */ + if (pass > 2) + goto done; + bind = p4_config_get_bind(hwc->config); escr_idx = p4_get_escr_idx(bind->escr_msr[thread]); if (unlikely(escr_idx == -1)) @@ -1180,8 +1249,17 @@ static int p4_pmu_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign } cntr_idx = p4_next_cntr(thread, used_mask, bind); - if (cntr_idx == -1 || test_bit(escr_idx, escr_mask)) - goto done; + if (cntr_idx == -1 || test_bit(escr_idx, escr_mask)) { + /* + * Probably an event alias is still available. + */ + config_alias = p4_get_alias_event(hwc->config); + if (!config_alias) + goto done; + hwc->config = config_alias; + pass++; + goto again; + } p4_pmu_swap_config_ts(hwc, cpu); if (assign) @@ -1218,7 +1296,6 @@ static __initconst const struct x86_pmu p4_pmu = { .cntval_bits = ARCH_P4_CNTRVAL_BITS, .cntval_mask = ARCH_P4_CNTRVAL_MASK, .max_period = (1ULL << (ARCH_P4_CNTRVAL_BITS - 1)) - 1, - .hw_watchdog_set_attr = p4_hw_watchdog_set_attr, .hw_config = p4_hw_config, .schedule_events = p4_pmu_schedule_events, /* diff --git a/kernel/watchdog.c b/kernel/watchdog.c index a933e3a0398b..36491cd5b7d4 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -200,7 +200,6 @@ static int is_softlockup(unsigned long touch_ts) } #ifdef CONFIG_HARDLOCKUP_DETECTOR -void __weak hw_nmi_watchdog_set_attr(struct perf_event_attr *wd_attr) { } static struct perf_event_attr wd_hw_attr = { .type = PERF_TYPE_HARDWARE, @@ -372,7 +371,6 @@ static int watchdog_nmi_enable(int cpu) wd_attr = &wd_hw_attr; wd_attr->sample_period = hw_nmi_get_sample_period(watchdog_thresh); - hw_nmi_watchdog_set_attr(wd_attr); /* Try to register using hardware perf events */ event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL); -- cgit v1.2.3 From 98d0ac38ca7b1b7a552c9a2359174ff84decb600 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Thu, 14 Jul 2011 06:47:22 -0400 Subject: x86-64: Move vread_tsc and vread_hpet into the vDSO The vsyscall page now consists entirely of trap instructions. Cc: John Stultz Signed-off-by: Andy Lutomirski Link: http://lkml.kernel.org/r/637648f303f2ef93af93bae25186e9a1bea093f5.1310639973.git.luto@mit.edu Signed-off-by: H. Peter Anvin --- arch/x86/include/asm/clocksource.h | 6 ++++- arch/x86/include/asm/tsc.h | 4 --- arch/x86/include/asm/vgtod.h | 2 +- arch/x86/include/asm/vsyscall.h | 4 --- arch/x86/kernel/Makefile | 7 +---- arch/x86/kernel/alternative.c | 8 ------ arch/x86/kernel/hpet.c | 9 +------ arch/x86/kernel/tsc.c | 2 +- arch/x86/kernel/vmlinux.lds.S | 3 --- arch/x86/kernel/vread_tsc_64.c | 36 -------------------------- arch/x86/kernel/vsyscall_64.c | 2 +- arch/x86/vdso/vclock_gettime.c | 53 +++++++++++++++++++++++++++++++++----- 12 files changed, 57 insertions(+), 79 deletions(-) delete mode 100644 arch/x86/kernel/vread_tsc_64.c (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/clocksource.h b/arch/x86/include/asm/clocksource.h index a5df33f614c9..3882c65dc19b 100644 --- a/arch/x86/include/asm/clocksource.h +++ b/arch/x86/include/asm/clocksource.h @@ -7,8 +7,12 @@ #define __ARCH_HAS_CLOCKSOURCE_DATA +#define VCLOCK_NONE 0 /* No vDSO clock available. */ +#define VCLOCK_TSC 1 /* vDSO should use vread_tsc. */ +#define VCLOCK_HPET 2 /* vDSO should use vread_hpet. */ + struct arch_clocksource_data { - cycle_t (*vread)(void); + int vclock_mode; }; #endif /* CONFIG_X86_64 */ diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h index 9db5583b6d38..83e2efd181e2 100644 --- a/arch/x86/include/asm/tsc.h +++ b/arch/x86/include/asm/tsc.h @@ -51,10 +51,6 @@ extern int unsynchronized_tsc(void); extern int check_tsc_unstable(void); extern unsigned long native_calibrate_tsc(void); -#ifdef CONFIG_X86_64 -extern cycles_t vread_tsc(void); -#endif - /* * Boot-time check whether the TSCs are synchronized across * all CPUs/cores: diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h index aa5add855a91..815285bcaceb 100644 --- a/arch/x86/include/asm/vgtod.h +++ b/arch/x86/include/asm/vgtod.h @@ -13,7 +13,7 @@ struct vsyscall_gtod_data { struct timezone sys_tz; struct { /* extract of a clocksource struct */ - cycle_t (*vread)(void); + int vclock_mode; cycle_t cycle_last; cycle_t mask; u32 mult; diff --git a/arch/x86/include/asm/vsyscall.h b/arch/x86/include/asm/vsyscall.h index d55597351f6a..60107072c28b 100644 --- a/arch/x86/include/asm/vsyscall.h +++ b/arch/x86/include/asm/vsyscall.h @@ -16,10 +16,6 @@ enum vsyscall_num { #ifdef __KERNEL__ #include -/* Definitions for CONFIG_GENERIC_TIME definitions */ -#define __vsyscall_fn \ - __attribute__ ((unused, __section__(".vsyscall_fn"))) notrace - #define VGETCPU_RDTSCP 1 #define VGETCPU_LSL 2 diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index cc0469a65120..2deef3d2435a 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -24,17 +24,12 @@ endif nostackp := $(call cc-option, -fno-stack-protector) CFLAGS_vsyscall_64.o := $(PROFILING) -g0 $(nostackp) CFLAGS_hpet.o := $(nostackp) -CFLAGS_vread_tsc_64.o := $(nostackp) CFLAGS_paravirt.o := $(nostackp) GCOV_PROFILE_vsyscall_64.o := n GCOV_PROFILE_hpet.o := n GCOV_PROFILE_tsc.o := n -GCOV_PROFILE_vread_tsc_64.o := n GCOV_PROFILE_paravirt.o := n -# vread_tsc_64 is hot and should be fully optimized: -CFLAGS_REMOVE_vread_tsc_64.o = -pg -fno-optimize-sibling-calls - obj-y := process_$(BITS).o signal.o entry_$(BITS).o obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o ldt.o dumpstack.o @@ -43,7 +38,7 @@ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-y += probe_roms.o obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o obj-$(CONFIG_X86_64) += sys_x86_64.o x8664_ksyms_64.o -obj-$(CONFIG_X86_64) += syscall_64.o vsyscall_64.o vread_tsc_64.o +obj-$(CONFIG_X86_64) += syscall_64.o vsyscall_64.o obj-$(CONFIG_X86_64) += vsyscall_emu_64.o obj-y += bootflag.o e820.o obj-y += pci-dma.o quirks.o topology.o kdebugfs.o diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index ddb207bb5f91..c63822816249 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -250,7 +249,6 @@ static void __init_or_module add_nops(void *insns, unsigned int len) extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; extern s32 __smp_locks[], __smp_locks_end[]; -extern char __vsyscall_0; void *text_poke_early(void *addr, const void *opcode, size_t len); /* Replace instructions with better alternatives for this CPU type. @@ -294,12 +292,6 @@ void __init_or_module apply_alternatives(struct alt_instr *start, add_nops(insnbuf + a->replacementlen, a->instrlen - a->replacementlen); -#ifdef CONFIG_X86_64 - /* vsyscall code is not mapped yet. resolve it manually. */ - if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) { - instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0)); - } -#endif text_poke_early(instr, insnbuf, a->instrlen); } } diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 0e07257bb389..d10cc009845f 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -738,13 +738,6 @@ static cycle_t read_hpet(struct clocksource *cs) return (cycle_t)hpet_readl(HPET_COUNTER); } -#ifdef CONFIG_X86_64 -static cycle_t __vsyscall_fn vread_hpet(void) -{ - return readl((const void __iomem *)fix_to_virt(VSYSCALL_HPET) + 0xf0); -} -#endif - static struct clocksource clocksource_hpet = { .name = "hpet", .rating = 250, @@ -753,7 +746,7 @@ static struct clocksource clocksource_hpet = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, .resume = hpet_resume_counter, #ifdef CONFIG_X86_64 - .archdata = { .vread = vread_hpet }, + .archdata = { .vclock_mode = VCLOCK_HPET }, #endif }; diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index e7a74b889ab3..56c633a5db72 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -777,7 +777,7 @@ static struct clocksource clocksource_tsc = { .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_MUST_VERIFY, #ifdef CONFIG_X86_64 - .archdata = { .vread = vread_tsc }, + .archdata = { .vclock_mode = VCLOCK_TSC }, #endif }; diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 80174719910c..4aa9c54a9b76 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -169,9 +169,6 @@ SECTIONS .vsyscall : AT(VLOAD(.vsyscall)) { *(.vsyscall_0) - . = ALIGN(L1_CACHE_BYTES); - *(.vsyscall_fn) - . = 1024; *(.vsyscall_1) diff --git a/arch/x86/kernel/vread_tsc_64.c b/arch/x86/kernel/vread_tsc_64.c deleted file mode 100644 index a81aa9e9894c..000000000000 --- a/arch/x86/kernel/vread_tsc_64.c +++ /dev/null @@ -1,36 +0,0 @@ -/* This code runs in userspace. */ - -#define DISABLE_BRANCH_PROFILING -#include - -notrace cycle_t __vsyscall_fn vread_tsc(void) -{ - cycle_t ret; - u64 last; - - /* - * Empirically, a fence (of type that depends on the CPU) - * before rdtsc is enough to ensure that rdtsc is ordered - * with respect to loads. The various CPU manuals are unclear - * as to whether rdtsc can be reordered with later loads, - * but no one has ever seen it happen. - */ - rdtsc_barrier(); - ret = (cycle_t)vget_cycles(); - - last = VVAR(vsyscall_gtod_data).clock.cycle_last; - - if (likely(ret >= last)) - return ret; - - /* - * GCC likes to generate cmov here, but this branch is extremely - * predictable (it's just a funciton of time and the likely is - * very likely) and there's a data dependence, so force GCC - * to generate a branch instead. I don't barrier() because - * we don't actually need a barrier, and if this function - * ever gets inlined it will generate worse code. - */ - asm volatile (""); - return last; -} diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index 12d488fd95d9..dda7dff9cef7 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -74,7 +74,7 @@ void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags); /* copy vsyscall data */ - vsyscall_gtod_data.clock.vread = clock->archdata.vread; + vsyscall_gtod_data.clock.vclock_mode = clock->archdata.vclock_mode; vsyscall_gtod_data.clock.cycle_last = clock->cycle_last; vsyscall_gtod_data.clock.mask = clock->mask; vsyscall_gtod_data.clock.mult = mult; diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c index cf54813ac527..8792d6e0a2c3 100644 --- a/arch/x86/vdso/vclock_gettime.c +++ b/arch/x86/vdso/vclock_gettime.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,43 @@ #define gtod (&VVAR(vsyscall_gtod_data)) +notrace static cycle_t vread_tsc(void) +{ + cycle_t ret; + u64 last; + + /* + * Empirically, a fence (of type that depends on the CPU) + * before rdtsc is enough to ensure that rdtsc is ordered + * with respect to loads. The various CPU manuals are unclear + * as to whether rdtsc can be reordered with later loads, + * but no one has ever seen it happen. + */ + rdtsc_barrier(); + ret = (cycle_t)vget_cycles(); + + last = VVAR(vsyscall_gtod_data).clock.cycle_last; + + if (likely(ret >= last)) + return ret; + + /* + * GCC likes to generate cmov here, but this branch is extremely + * predictable (it's just a funciton of time and the likely is + * very likely) and there's a data dependence, so force GCC + * to generate a branch instead. I don't barrier() because + * we don't actually need a barrier, and if this function + * ever gets inlined it will generate worse code. + */ + asm volatile (""); + return last; +} + +static notrace cycle_t vread_hpet(void) +{ + return readl((const void __iomem *)fix_to_virt(VSYSCALL_HPET) + 0xf0); +} + notrace static long vdso_fallback_gettime(long clock, struct timespec *ts) { long ret; @@ -36,9 +74,12 @@ notrace static long vdso_fallback_gettime(long clock, struct timespec *ts) notrace static inline long vgetns(void) { long v; - cycles_t (*vread)(void); - vread = gtod->clock.vread; - v = (vread() - gtod->clock.cycle_last) & gtod->clock.mask; + cycles_t cycles; + if (gtod->clock.vclock_mode == VCLOCK_TSC) + cycles = vread_tsc(); + else + cycles = vread_hpet(); + v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask; return (v * gtod->clock.mult) >> gtod->clock.shift; } @@ -118,11 +159,11 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) { switch (clock) { case CLOCK_REALTIME: - if (likely(gtod->clock.vread)) + if (likely(gtod->clock.vclock_mode != VCLOCK_NONE)) return do_realtime(ts); break; case CLOCK_MONOTONIC: - if (likely(gtod->clock.vread)) + if (likely(gtod->clock.vclock_mode != VCLOCK_NONE)) return do_monotonic(ts); break; case CLOCK_REALTIME_COARSE: @@ -139,7 +180,7 @@ int clock_gettime(clockid_t, struct timespec *) notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) { long ret; - if (likely(gtod->clock.vread)) { + if (likely(gtod->clock.vclock_mode != VCLOCK_NONE)) { if (likely(tv != NULL)) { BUILD_BUG_ON(offsetof(struct timeval, tv_usec) != offsetof(struct timespec, tv_nsec) || -- cgit v1.2.3 From 3982294b0342474ff91472b34c6afb701785f524 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sun, 10 Jul 2011 21:27:27 +0200 Subject: x86, signals: Convert the X86_32 code to use set_current_blocked() sys_sigsuspend() and sys_sigreturn() change ->blocked directly. This is not correct, see the changelog in e6fa16ab "signal: sigprocmask() should do retarget_shared_pending()" Change them to use set_current_blocked(). Signed-off-by: Oleg Nesterov Link: http://lkml.kernel.org/r/20110710192727.GA31759@redhat.com Signed-off-by: H. Peter Anvin --- arch/x86/kernel/signal.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index 40a24932a8a1..bf9345da380b 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -485,17 +485,18 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info, asmlinkage int sys_sigsuspend(int history0, int history1, old_sigset_t mask) { - mask &= _BLOCKABLE; - spin_lock_irq(¤t->sighand->siglock); + sigset_t blocked; + current->saved_sigmask = current->blocked; - siginitset(¤t->blocked, mask); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + + mask &= _BLOCKABLE; + siginitset(&blocked, mask); + set_current_blocked(&blocked); current->state = TASK_INTERRUPTIBLE; schedule(); - set_restore_sigmask(); + set_restore_sigmask(); return -ERESTARTNOHAND; } @@ -572,10 +573,7 @@ unsigned long sys_sigreturn(struct pt_regs *regs) goto badframe; sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + set_current_blocked(&set); if (restore_sigcontext(regs, &frame->sc, &ax)) goto badframe; -- cgit v1.2.3 From 9b429620740945363b746414e8b9a84b8119914c Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sun, 10 Jul 2011 20:22:03 +0200 Subject: x86, do_signal: Simplify the TS_RESTORE_SIGMASK logic 1. do_signal() looks at TS_RESTORE_SIGMASK and calculates the mask which should be stored in the signal frame, then it passes "oldset" to the callees, down to setup_rt_frame(). This is ugly, setup_rt_frame() can do this itself and nobody else needs this sigset_t. Move this code into setup_rt_frame. 2. do_signal() also clears TS_RESTORE_SIGMASK if handle_signal() succeeds. We can move this to setup_rt_frame() as well, this avoids the unnecessary checks and makes the logic more clear. 3. use set_current_blocked() instead of sigprocmask(SIG_SETMASK), sigprocmask() should be avoided. Signed-off-by: Oleg Nesterov Link: http://lkml.kernel.org/r/20110710182203.GA27979@redhat.com Signed-off-by: H. Peter Anvin --- arch/x86/kernel/signal.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index bf9345da380b..8c55f97728cd 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -651,11 +651,15 @@ int ia32_setup_frame(int sig, struct k_sigaction *ka, static int setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info, - sigset_t *set, struct pt_regs *regs) + struct pt_regs *regs) { int usig = signr_convert(sig); + sigset_t *set = ¤t->blocked; int ret; + if (current_thread_info()->status & TS_RESTORE_SIGMASK) + set = ¤t->saved_sigmask; + /* Set up the stack frame */ if (is_ia32) { if (ka->sa.sa_flags & SA_SIGINFO) @@ -670,12 +674,13 @@ setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info, return -EFAULT; } + current_thread_info()->status &= ~TS_RESTORE_SIGMASK; return ret; } static int handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka, - sigset_t *oldset, struct pt_regs *regs) + struct pt_regs *regs) { sigset_t blocked; int ret; @@ -710,7 +715,7 @@ handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka, likely(test_and_clear_thread_flag(TIF_FORCED_TF))) regs->flags &= ~X86_EFLAGS_TF; - ret = setup_rt_frame(sig, ka, info, oldset, regs); + ret = setup_rt_frame(sig, ka, info, regs); if (ret) return ret; @@ -765,7 +770,6 @@ static void do_signal(struct pt_regs *regs) struct k_sigaction ka; siginfo_t info; int signr; - sigset_t *oldset; /* * We want the common case to go fast, which is why we may in certain @@ -777,23 +781,10 @@ static void do_signal(struct pt_regs *regs) if (!user_mode(regs)) return; - if (current_thread_info()->status & TS_RESTORE_SIGMASK) - oldset = ¤t->saved_sigmask; - else - oldset = ¤t->blocked; - signr = get_signal_to_deliver(&info, &ka, regs, NULL); if (signr > 0) { /* Whee! Actually deliver the signal. */ - if (handle_signal(signr, &info, &ka, oldset, regs) == 0) { - /* - * A signal was successfully delivered; the saved - * sigmask will have been stored in the signal frame, - * and will be restored by sigreturn, so we can simply - * clear the TS_RESTORE_SIGMASK flag. - */ - current_thread_info()->status &= ~TS_RESTORE_SIGMASK; - } + handle_signal(signr, &info, &ka, regs); return; } @@ -821,7 +812,7 @@ static void do_signal(struct pt_regs *regs) */ if (current_thread_info()->status & TS_RESTORE_SIGMASK) { current_thread_info()->status &= ~TS_RESTORE_SIGMASK; - sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); + set_current_blocked(¤t->saved_sigmask); } } -- cgit v1.2.3 From 73d382deccac186d103496bf10388bc2432a8384 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sun, 10 Jul 2011 18:44:24 +0200 Subject: x86: Kill handle_signal()->set_fs() handle_signal()->set_fs() has a nice comment which explains what set_fs() is, but it doesn't explain why it is needed and why it depends on CONFIG_X86_64. Afaics, the history of this confusion is: 1. I guess today nobody can explain why it was needed in arch/i386/kernel/signal.c, perhaps it was always wrong. This predates 2.4.0 kernel. 2. then it was copy-and-past'ed to the new x86_64 arch. 3. then it was removed from i386 (but not from x86_64) by b93b6ca3 "i386: remove unnecessary code". 4. then it was reintroduced under CONFIG_X86_64 when x86 unified i386 and x86_64, because the patch above didn't touch x86_64. Remove it. ->addr_limit should be correct. Even if it was possible that it is wrong, it is too late to fix it after setup_rt_frame(). Linus commented in: http://lkml.kernel.org/r/alpine.LFD.0.999.0707170902570.19166@woody.linux-foundation.org ... about the equivalent bit from i386: Heh. I think it's entirely historical. Please realize that the whole reason that function is called "set_fs()" is that it literally used to set the %fs segment register, not "->addr_limit". So I think the "set_fs(USER_DS)" is there _only_ to match the other regs->xds = __USER_DS; regs->xes = __USER_DS; regs->xss = __USER_DS; regs->xcs = __USER_CS; things, and never mattered. And now it matters even less, and has been copied to all other architectures where it is just totally insane. Signed-off-by: Oleg Nesterov Link: http://lkml.kernel.org/r/20110710164424.GA20261@redhat.com Signed-off-by: H. Peter Anvin --- arch/x86/kernel/signal.c | 9 --------- 1 file changed, 9 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index 8c55f97728cd..54ddaeb221c1 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -720,15 +720,6 @@ handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka, if (ret) return ret; -#ifdef CONFIG_X86_64 - /* - * This has nothing to do with segment registers, - * despite the name. This magic affects uaccess.h - * macros' behavior. Reset it to the normal setting. - */ - set_fs(USER_DS); -#endif - /* * Clear the direction flag as per the ABI for function entry. */ -- cgit v1.2.3 From 17edf2d79f1ea6dfdb4c444801d928953b9f98d6 Mon Sep 17 00:00:00 2001 From: Len Brown Date: Fri, 15 Jul 2011 17:37:15 -0400 Subject: x86, intel, power: Correct the MSR_IA32_ENERGY_PERF_BIAS message Fix the printk_once() so that it actually prints (didn't print before due to a stray comma.) [ hpa: changed to an incremental patch and adjusted the description accordingly. ] Signed-off-by: Len Brown Link: http://lkml.kernel.org/r/alpine.LFD.2.02.1107151732480.18606@x980 Cc: Signed-off-by: H. Peter Anvin --- arch/x86/kernel/cpu/intel.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index da0d779ecd9e..ed6086eedf1d 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -465,11 +465,11 @@ static void __cpuinit init_intel(struct cpuinfo_x86 *c) u64 epb; rdmsrl(MSR_IA32_ENERGY_PERF_BIAS, epb); - if ((epb & 0xF) == 0) { - printk_once(KERN_WARNING, "x86: updated energy_perf_bias" - " to 'normal' from 'performance'\n" - "You can view and update epb via utility," - " such as x86_energy_perf_policy(8)\n"); + if ((epb & 0xF) == ENERGY_PERF_BIAS_PERFORMANCE) { + printk_once(KERN_WARNING "ENERGY_PERF_BIAS:" + " Set to 'normal', was 'performance'\n" + "ENERGY_PERF_BIAS: View and update with" + " x86_energy_perf_policy(8)\n"); epb = (epb & ~0xF) | ENERGY_PERF_BIAS_NORMAL; wrmsrl(MSR_IA32_ENERGY_PERF_BIAS, epb); } -- cgit v1.2.3 From a6c23905ff0d6bbddf590ef0838489ee0f6c74ac Mon Sep 17 00:00:00 2001 From: Greg Dietsche Date: Thu, 30 Jun 2011 20:10:53 -0500 Subject: x86, smpboot: Mark the names[] array in __inquire_remote_apic() as const This array is read-only. Make it explicit by marking as const. Signed-off-by: Greg Dietsche Link: http://lkml.kernel.org/r/1309482653-23648-1-git-send-email-Gregory.Dietsche@cuw.edu Signed-off-by: Ingo Molnar --- arch/x86/kernel/smpboot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index a3c430bdfb60..e02653a2e4f7 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -425,7 +425,7 @@ static void impress_friends(void) void __inquire_remote_apic(int apicid) { unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 }; - char *names[] = { "ID", "VERSION", "SPIV" }; + const char * const names[] = { "ID", "VERSION", "SPIV" }; int timeout; u32 status; -- cgit v1.2.3 From 38175051f8e79c5e9f65daab7200fd8d1fa4a912 Mon Sep 17 00:00:00 2001 From: Sergei Shtylyov Date: Mon, 11 Jul 2011 19:01:38 +0400 Subject: x86, quirks: Use pci_dev->revision This code uses PCI_CLASS_REVISION instead of PCI_REVISION_ID, so it wasn't converted by commit 44c10138fd4bbc ("PCI: Change all drivers to use pci_device->revision") before being moved to arch/x86/... Signed-off-by: Sergei Shtylyov Cc: Jesse Barnes Cc: Dave Jones Link: http://lkml.kernel.org/r/201107111901.39281.sshtylyov@ru.mvista.com Signed-off-by: Ingo Molnar --- arch/x86/kernel/quirks.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c index 8bbe8c56916d..b78643d0f9a5 100644 --- a/arch/x86/kernel/quirks.c +++ b/arch/x86/kernel/quirks.c @@ -10,7 +10,7 @@ static void __devinit quirk_intel_irqbalance(struct pci_dev *dev) { - u8 config, rev; + u8 config; u16 word; /* BIOS may enable hardware IRQ balancing for @@ -18,8 +18,7 @@ static void __devinit quirk_intel_irqbalance(struct pci_dev *dev) * based platforms. * Disable SW irqbalance/affinity on those platforms. */ - pci_read_config_byte(dev, PCI_CLASS_REVISION, &rev); - if (rev > 0x9) + if (dev->revision > 0x9) return; /* enable access to config space*/ -- cgit v1.2.3 From 050438ed5a05b25cdf287f5691e56a58c2606997 Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Thu, 14 Jul 2011 09:34:37 +0800 Subject: kexec, x86: Fix incorrect jump back address if not preserving context In kexec jump support, jump back address passed to the kexeced kernel via function calling ABI, that is, the function call return address is the jump back entry. Furthermore, jump back entry == 0 should be used to signal that the jump back or preserve context is not enabled in the original kernel. But in the current implementation the stack position used for function call return address is not cleared context preservation is disabled. The patch fixes this bug. Reported-and-tested-by: Yin Kangkai Signed-off-by: Huang Ying Cc: Eric W. Biederman Cc: Vivek Goyal Cc: Link: http://lkml.kernel.org/r/1310607277-25029-1-git-send-email-ying.huang@intel.com Signed-off-by: Ingo Molnar --- arch/x86/kernel/relocate_kernel_32.S | 2 ++ arch/x86/kernel/relocate_kernel_64.S | 2 ++ 2 files changed, 4 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/relocate_kernel_32.S b/arch/x86/kernel/relocate_kernel_32.S index 41235531b11c..36818f8ec2be 100644 --- a/arch/x86/kernel/relocate_kernel_32.S +++ b/arch/x86/kernel/relocate_kernel_32.S @@ -97,6 +97,8 @@ relocate_kernel: ret identity_mapped: + /* set return address to 0 if not preserving context */ + pushl $0 /* store the start address on the stack */ pushl %edx diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 4de8f5b3d476..7a6f3b3be3cf 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -100,6 +100,8 @@ relocate_kernel: ret identity_mapped: + /* set return address to 0 if not preserving context */ + pushq $0 /* store the start address on the stack */ pushq %rdx -- cgit v1.2.3 From 497888cf69bf607ac1fe061a6437e0a670b0022f Mon Sep 17 00:00:00 2001 From: Phil Carmody Date: Thu, 14 Jul 2011 15:07:13 +0300 Subject: treewide: fix potentially dangerous trailing ';' in #defined values/expressions All these are instances of #define NAME value; or #define NAME(params_opt) value; These of course fail to build when used in contexts like if(foo $OP NAME) while(bar $OP NAME) and may silently generate the wrong code in contexts such as foo = NAME + 1; /* foo = value; + 1; */ bar = NAME - 1; /* bar = value; - 1; */ baz = NAME & quux; /* baz = value; & quux; */ Reported on comp.lang.c, Message-ID: Initial analysis of the dangers provided by Keith Thompson in that thread. There are many more instances of more complicated macros having unnecessary trailing semicolons, but this pile seems to be all of the cases of simple values suffering from the problem. (Thus things that are likely to be found in one of the contexts above, more complicated ones aren't.) Signed-off-by: Phil Carmody Signed-off-by: Jiri Kosina --- Documentation/DocBook/v4l/io.xml | 2 +- arch/alpha/include/asm/floppy.h | 2 +- arch/h8300/kernel/setup.c | 2 +- arch/ia64/include/asm/sn/tioce.h | 2 +- arch/mips/include/asm/floppy.h | 2 +- arch/parisc/include/asm/dma-mapping.h | 2 +- arch/parisc/math-emu/decode_exc.c | 2 +- arch/powerpc/include/asm/elf.h | 4 ++-- arch/powerpc/include/asm/smu.h | 2 +- arch/sparc/include/asm/elf_64.h | 2 +- arch/um/sys-i386/signal.c | 2 +- arch/x86/kernel/i387.c | 2 +- drivers/acpi/ac.c | 2 +- drivers/acpi/battery.c | 2 +- drivers/acpi/sbs.c | 2 +- drivers/gpu/drm/sis/sis_drv.h | 4 ++-- drivers/hwmon/gl520sm.c | 2 +- drivers/isdn/i4l/isdn_bsdcomp.c | 2 +- drivers/net/bsd_comp.c | 2 +- drivers/net/natsemi.c | 2 +- drivers/net/r8169.c | 2 +- drivers/net/s2io.h | 4 ++-- drivers/net/wireless/iwlegacy/iwl-commands.h | 4 ++-- drivers/net/wireless/iwlwifi/iwl-commands.h | 4 ++-- drivers/net/wireless/rtlwifi/rtl8192ce/reg.h | 4 ++-- drivers/scsi/device_handler/scsi_dh_rdac.c | 2 +- drivers/scsi/lpfc/lpfc_hw.h | 8 ++++---- drivers/usb/otg/twl4030-usb.c | 2 +- drivers/video/i810/i810.h | 2 +- include/linux/ceph/libceph.h | 2 +- include/linux/mfd/tps65910.h | 2 +- include/scsi/scsi.h | 2 +- include/sound/soundfont.h | 2 +- mm/slub.c | 2 +- net/mac80211/mesh_hwmp.c | 20 ++++++++++---------- 35 files changed, 53 insertions(+), 53 deletions(-) (limited to 'arch/x86/kernel') diff --git a/Documentation/DocBook/v4l/io.xml b/Documentation/DocBook/v4l/io.xml index 227e7ac45a06..c57d1ec6291c 100644 --- a/Documentation/DocBook/v4l/io.xml +++ b/Documentation/DocBook/v4l/io.xml @@ -210,7 +210,7 @@ for (i = 0; i < reqbuf.count; i++) &v4l2-requestbuffers; reqbuf; /* Our current format uses 3 planes per buffer */ -#define FMT_NUM_PLANES = 3; +#define FMT_NUM_PLANES = 3 struct { void *start[FMT_NUM_PLANES]; diff --git a/arch/alpha/include/asm/floppy.h b/arch/alpha/include/asm/floppy.h index 0be50413b2b5..46cefbd50e73 100644 --- a/arch/alpha/include/asm/floppy.h +++ b/arch/alpha/include/asm/floppy.h @@ -27,7 +27,7 @@ #define fd_cacheflush(addr,size) /* nothing */ #define fd_request_irq() request_irq(FLOPPY_IRQ, floppy_interrupt,\ IRQF_DISABLED, "floppy", NULL) -#define fd_free_irq() free_irq(FLOPPY_IRQ, NULL); +#define fd_free_irq() free_irq(FLOPPY_IRQ, NULL) #ifdef CONFIG_PCI diff --git a/arch/h8300/kernel/setup.c b/arch/h8300/kernel/setup.c index 7fda657110eb..68d651081bd3 100644 --- a/arch/h8300/kernel/setup.c +++ b/arch/h8300/kernel/setup.c @@ -46,7 +46,7 @@ #include #endif -#define STUBSIZE 0xc000; +#define STUBSIZE 0xc000 unsigned long rom_length; unsigned long memory_start; diff --git a/arch/ia64/include/asm/sn/tioce.h b/arch/ia64/include/asm/sn/tioce.h index 893468e1b41b..6eae8ada90f0 100644 --- a/arch/ia64/include/asm/sn/tioce.h +++ b/arch/ia64/include/asm/sn/tioce.h @@ -467,7 +467,7 @@ typedef volatile struct tioce { #define CE_LSI_GB_CFG1_RXL0S_THS_SHFT 0 #define CE_LSI_GB_CFG1_RXL0S_THS_MASK (0xffULL << 0) #define CE_LSI_GB_CFG1_RXL0S_SMP_SHFT 8 -#define CE_LSI_GB_CFG1_RXL0S_SMP_MASK (0xfULL << 8); +#define CE_LSI_GB_CFG1_RXL0S_SMP_MASK (0xfULL << 8) #define CE_LSI_GB_CFG1_RXL0S_ADJ_SHFT 12 #define CE_LSI_GB_CFG1_RXL0S_ADJ_MASK (0x7ULL << 12) #define CE_LSI_GB_CFG1_RXL0S_FLT_SHFT 15 diff --git a/arch/mips/include/asm/floppy.h b/arch/mips/include/asm/floppy.h index c5c7c0e6064c..4456c9c47e21 100644 --- a/arch/mips/include/asm/floppy.h +++ b/arch/mips/include/asm/floppy.h @@ -29,7 +29,7 @@ static inline void fd_cacheflush(char * addr, long size) #define FLOPPY0_TYPE fd_drive_type(0) #define FLOPPY1_TYPE fd_drive_type(1) -#define FDC1 fd_getfdaddr1(); +#define FDC1 fd_getfdaddr1() #define N_FDC 1 /* do you *really* want a second controller? */ #define N_DRIVE 8 diff --git a/arch/parisc/include/asm/dma-mapping.h b/arch/parisc/include/asm/dma-mapping.h index 4ef73b09b168..890531e32fe8 100644 --- a/arch/parisc/include/asm/dma-mapping.h +++ b/arch/parisc/include/asm/dma-mapping.h @@ -210,7 +210,7 @@ parisc_walk_tree(struct device *dev) return dev->platform_data; } -#define GET_IOC(dev) (HBA_DATA(parisc_walk_tree(dev))->iommu); +#define GET_IOC(dev) (HBA_DATA(parisc_walk_tree(dev))->iommu) #ifdef CONFIG_IOMMU_CCIO diff --git a/arch/parisc/math-emu/decode_exc.c b/arch/parisc/math-emu/decode_exc.c index 27a7492ddb0d..04e550e76ae8 100644 --- a/arch/parisc/math-emu/decode_exc.c +++ b/arch/parisc/math-emu/decode_exc.c @@ -56,7 +56,7 @@ /* General definitions */ #define DOESTRAP 1 #define NOTRAP 0 -#define SIGNALCODE(signal, code) ((signal) << 24 | (code)); +#define SIGNALCODE(signal, code) ((signal) << 24 | (code)) #define copropbit 1<<31-2 /* bit position 2 */ #define opclass 9 /* bits 21 & 22 */ #define fmt 11 /* bits 19 & 20 */ diff --git a/arch/powerpc/include/asm/elf.h b/arch/powerpc/include/asm/elf.h index 2b917c69ed15..3bf9cca35147 100644 --- a/arch/powerpc/include/asm/elf.h +++ b/arch/powerpc/include/asm/elf.h @@ -267,7 +267,7 @@ extern int ucache_bsize; struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp); -#define VDSO_AUX_ENT(a,b) NEW_AUX_ENT(a,b); +#define VDSO_AUX_ENT(a,b) NEW_AUX_ENT(a,b) /* 1GB for 64bit, 8MB for 32bit */ #define STACK_RND_MASK (is_32bit_task() ? \ @@ -298,7 +298,7 @@ do { \ NEW_AUX_ENT(AT_DCACHEBSIZE, dcache_bsize); \ NEW_AUX_ENT(AT_ICACHEBSIZE, icache_bsize); \ NEW_AUX_ENT(AT_UCACHEBSIZE, ucache_bsize); \ - VDSO_AUX_ENT(AT_SYSINFO_EHDR, current->mm->context.vdso_base) \ + VDSO_AUX_ENT(AT_SYSINFO_EHDR, current->mm->context.vdso_base); \ } while (0) /* PowerPC64 relocations defined by the ABIs */ diff --git a/arch/powerpc/include/asm/smu.h b/arch/powerpc/include/asm/smu.h index e3bdada8c542..ae20ce1af4c7 100644 --- a/arch/powerpc/include/asm/smu.h +++ b/arch/powerpc/include/asm/smu.h @@ -547,7 +547,7 @@ struct smu_sdbp_header { * (currently, afaik, this concerns only the FVT partition * (0x12) */ -#define SMU_U16_MIX(x) le16_to_cpu(x); +#define SMU_U16_MIX(x) le16_to_cpu(x) #define SMU_U32_MIX(x) ((((x) & 0xff00ff00u) >> 8)|(((x) & 0x00ff00ffu) << 8)) diff --git a/arch/sparc/include/asm/elf_64.h b/arch/sparc/include/asm/elf_64.h index e67880381b84..cfa9cd2e5519 100644 --- a/arch/sparc/include/asm/elf_64.h +++ b/arch/sparc/include/asm/elf_64.h @@ -186,7 +186,7 @@ static inline unsigned int sparc64_elf_hwcap(void) return cap; } -#define ELF_HWCAP sparc64_elf_hwcap(); +#define ELF_HWCAP sparc64_elf_hwcap() /* This yields a string that ld.so will use to load implementation specific libraries for optimization. This is more specific in diff --git a/arch/um/sys-i386/signal.c b/arch/um/sys-i386/signal.c index 129647375a6c..89a46626bfd8 100644 --- a/arch/um/sys-i386/signal.c +++ b/arch/um/sys-i386/signal.c @@ -58,7 +58,7 @@ static inline unsigned long twd_fxsr_to_i387(struct user_fxsr_struct *fxsave) unsigned long ret = 0xffff0000; int i; -#define FPREG_ADDR(f, n) ((char *)&(f)->st_space + (n) * 16); +#define FPREG_ADDR(f, n) ((char *)&(f)->st_space + (n) * 16) for (i = 0; i < 8; i++) { if (twd & 0x1) { diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index 12aff2537682..739d8598f789 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c @@ -321,7 +321,7 @@ static inline unsigned short twd_i387_to_fxsr(unsigned short twd) return tmp; } -#define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16); +#define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16) #define FP_EXP_TAG_VALID 0 #define FP_EXP_TAG_ZERO 1 #define FP_EXP_TAG_SPECIAL 2 diff --git a/drivers/acpi/ac.c b/drivers/acpi/ac.c index 58c3f74bd84c..6512b20aeccd 100644 --- a/drivers/acpi/ac.c +++ b/drivers/acpi/ac.c @@ -89,7 +89,7 @@ struct acpi_ac { unsigned long long state; }; -#define to_acpi_ac(x) container_of(x, struct acpi_ac, charger); +#define to_acpi_ac(x) container_of(x, struct acpi_ac, charger) #ifdef CONFIG_ACPI_PROCFS_POWER static const struct file_operations acpi_ac_fops = { diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index fcc13ac0aa18..2c661353e8f2 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -132,7 +132,7 @@ struct acpi_battery { unsigned long flags; }; -#define to_acpi_battery(x) container_of(x, struct acpi_battery, bat); +#define to_acpi_battery(x) container_of(x, struct acpi_battery, bat) inline int acpi_battery_present(struct acpi_battery *battery) { diff --git a/drivers/acpi/sbs.c b/drivers/acpi/sbs.c index 51ae3794ec7f..50658ff887d9 100644 --- a/drivers/acpi/sbs.c +++ b/drivers/acpi/sbs.c @@ -112,7 +112,7 @@ struct acpi_battery { u8 have_sysfs_alarm:1; }; -#define to_acpi_battery(x) container_of(x, struct acpi_battery, bat); +#define to_acpi_battery(x) container_of(x, struct acpi_battery, bat) struct acpi_sbs { struct power_supply charger; diff --git a/drivers/gpu/drm/sis/sis_drv.h b/drivers/gpu/drm/sis/sis_drv.h index ef940bad63f7..194303c177ad 100644 --- a/drivers/gpu/drm/sis/sis_drv.h +++ b/drivers/gpu/drm/sis/sis_drv.h @@ -48,8 +48,8 @@ enum sis_family { #define SIS_BASE (dev_priv->mmio) -#define SIS_READ(reg) DRM_READ32(SIS_BASE, reg); -#define SIS_WRITE(reg, val) DRM_WRITE32(SIS_BASE, reg, val); +#define SIS_READ(reg) DRM_READ32(SIS_BASE, reg) +#define SIS_WRITE(reg, val) DRM_WRITE32(SIS_BASE, reg, val) typedef struct drm_sis_private { drm_local_map_t *mmio; diff --git a/drivers/hwmon/gl520sm.c b/drivers/hwmon/gl520sm.c index ec588026f0a9..131ea8625f08 100644 --- a/drivers/hwmon/gl520sm.c +++ b/drivers/hwmon/gl520sm.c @@ -273,7 +273,7 @@ static SENSOR_DEVICE_ATTR(in4_max, S_IRUGO | S_IWUSR, #define DIV_FROM_REG(val) (1 << (val)) #define FAN_FROM_REG(val,div) ((val)==0 ? 0 : (480000/((val) << (div)))) -#define FAN_TO_REG(val,div) ((val)<=0?0:SENSORS_LIMIT((480000 + ((val) << ((div)-1))) / ((val) << (div)), 1, 255)); +#define FAN_TO_REG(val,div) ((val)<=0?0:SENSORS_LIMIT((480000 + ((val) << ((div)-1))) / ((val) << (div)), 1, 255)) static ssize_t get_fan_input(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/drivers/isdn/i4l/isdn_bsdcomp.c b/drivers/isdn/i4l/isdn_bsdcomp.c index 02d9918705dd..aa0b6a6f5ef4 100644 --- a/drivers/isdn/i4l/isdn_bsdcomp.c +++ b/drivers/isdn/i4l/isdn_bsdcomp.c @@ -155,7 +155,7 @@ struct bsd_db { #define LAST 255 #define MAXCODE(b) ((1 << (b)) - 1) -#define BADCODEM1 MAXCODE(MAX_BSD_BITS); +#define BADCODEM1 MAXCODE(MAX_BSD_BITS) #define BSD_HASH(prefix,suffix,hshift) ((((unsigned long)(suffix))<<(hshift)) \ ^ (unsigned long)(prefix)) diff --git a/drivers/net/bsd_comp.c b/drivers/net/bsd_comp.c index 6e99d80ec409..a9b759add187 100644 --- a/drivers/net/bsd_comp.c +++ b/drivers/net/bsd_comp.c @@ -201,7 +201,7 @@ extern void ppp_unregister_compressor (struct compressor *cp); #define LAST 255 #define MAXCODE(b) ((1 << (b)) - 1) -#define BADCODEM1 MAXCODE(MAX_BSD_BITS); +#define BADCODEM1 MAXCODE(MAX_BSD_BITS) #define BSD_HASH(prefix,suffix,hshift) ((((unsigned long)(suffix))<<(hshift)) \ ^ (unsigned long)(prefix)) diff --git a/drivers/net/natsemi.c b/drivers/net/natsemi.c index 68e6b0224edd..c69f82a17c28 100644 --- a/drivers/net/natsemi.c +++ b/drivers/net/natsemi.c @@ -1382,7 +1382,7 @@ static int find_mii(struct net_device *dev) /* WCSR bits [0:4] [9:10] */ #define WCSR_RESET_SAVE 0x61f /* RFCR bits [20] [22] [27:31] */ -#define RFCR_RESET_SAVE 0xf8500000; +#define RFCR_RESET_SAVE 0xf8500000 static void natsemi_reset(struct net_device *dev) { diff --git a/drivers/net/r8169.c b/drivers/net/r8169.c index 5990621fb5cd..6f3630618fa8 100644 --- a/drivers/net/r8169.c +++ b/drivers/net/r8169.c @@ -667,7 +667,7 @@ struct rtl8169_private { u32 saved_wolopts; const struct firmware *fw; -#define RTL_FIRMWARE_UNKNOWN ERR_PTR(-EAGAIN); +#define RTL_FIRMWARE_UNKNOWN ERR_PTR(-EAGAIN) }; MODULE_AUTHOR("Realtek and the Linux r8169 crew "); diff --git a/drivers/net/s2io.h b/drivers/net/s2io.h index 800b3a44e653..57a4dc7c7d7a 100644 --- a/drivers/net/s2io.h +++ b/drivers/net/s2io.h @@ -968,8 +968,8 @@ struct s2io_nic { u8 serial_num[VPD_STRING_LEN]; }; -#define RESET_ERROR 1; -#define CMD_ERROR 2; +#define RESET_ERROR 1 +#define CMD_ERROR 2 /* OS related system calls */ #ifndef readq diff --git a/drivers/net/wireless/iwlegacy/iwl-commands.h b/drivers/net/wireless/iwlegacy/iwl-commands.h index 17a1d504348e..6a5c76e7345f 100644 --- a/drivers/net/wireless/iwlegacy/iwl-commands.h +++ b/drivers/net/wireless/iwlegacy/iwl-commands.h @@ -2624,8 +2624,8 @@ struct iwl_scanstart_notification { __le32 status; } __packed; -#define SCAN_OWNER_STATUS 0x1; -#define MEASURE_OWNER_STATUS 0x2; +#define SCAN_OWNER_STATUS 0x1 +#define MEASURE_OWNER_STATUS 0x2 #define IWL_PROBE_STATUS_OK 0 #define IWL_PROBE_STATUS_TX_FAILED BIT(0) diff --git a/drivers/net/wireless/iwlwifi/iwl-commands.h b/drivers/net/wireless/iwlwifi/iwl-commands.h index 6ee5f1aa555c..6288d1fdfcc6 100644 --- a/drivers/net/wireless/iwlwifi/iwl-commands.h +++ b/drivers/net/wireless/iwlwifi/iwl-commands.h @@ -2457,8 +2457,8 @@ struct iwl_scanstart_notification { __le32 status; } __packed; -#define SCAN_OWNER_STATUS 0x1; -#define MEASURE_OWNER_STATUS 0x2; +#define SCAN_OWNER_STATUS 0x1 +#define MEASURE_OWNER_STATUS 0x2 #define IWL_PROBE_STATUS_OK 0 #define IWL_PROBE_STATUS_TX_FAILED BIT(0) diff --git a/drivers/net/wireless/rtlwifi/rtl8192ce/reg.h b/drivers/net/wireless/rtlwifi/rtl8192ce/reg.h index 598cecc63f41..5b43749031dd 100644 --- a/drivers/net/wireless/rtlwifi/rtl8192ce/reg.h +++ b/drivers/net/wireless/rtlwifi/rtl8192ce/reg.h @@ -1074,10 +1074,10 @@ #define _SRL(x) (((x) & 0x3F) << 8) #define _SIFS_CCK_CTX(x) ((x) & 0xFF) -#define _SIFS_CCK_TRX(x) (((x) & 0xFF) << 8); +#define _SIFS_CCK_TRX(x) (((x) & 0xFF) << 8) #define _SIFS_OFDM_CTX(x) ((x) & 0xFF) -#define _SIFS_OFDM_TRX(x) (((x) & 0xFF) << 8); +#define _SIFS_OFDM_TRX(x) (((x) & 0xFF) << 8) #define _TBTT_PROHIBIT_HOLD(x) (((x) & 0xFF) << 8) diff --git a/drivers/scsi/device_handler/scsi_dh_rdac.c b/drivers/scsi/device_handler/scsi_dh_rdac.c index e7fc70d6b478..2e7c136bb805 100644 --- a/drivers/scsi/device_handler/scsi_dh_rdac.c +++ b/drivers/scsi/device_handler/scsi_dh_rdac.c @@ -35,7 +35,7 @@ * mode page were taken from the LSI RDAC 2.4 GPL'd * driver, and then converted to Linux conventions. */ -#define RDAC_QUIESCENCE_TIME 20; +#define RDAC_QUIESCENCE_TIME 20 /* * Page Codes */ diff --git a/drivers/scsi/lpfc/lpfc_hw.h b/drivers/scsi/lpfc/lpfc_hw.h index 9059524cf225..ab4c4d651d0c 100644 --- a/drivers/scsi/lpfc/lpfc_hw.h +++ b/drivers/scsi/lpfc/lpfc_hw.h @@ -2955,18 +2955,18 @@ typedef struct _SLI2_RDSC { typedef struct _PCB { #ifdef __BIG_ENDIAN_BITFIELD uint32_t type:8; -#define TYPE_NATIVE_SLI2 0x01; +#define TYPE_NATIVE_SLI2 0x01 uint32_t feature:8; -#define FEATURE_INITIAL_SLI2 0x01; +#define FEATURE_INITIAL_SLI2 0x01 uint32_t rsvd:12; uint32_t maxRing:4; #else /* __LITTLE_ENDIAN_BITFIELD */ uint32_t maxRing:4; uint32_t rsvd:12; uint32_t feature:8; -#define FEATURE_INITIAL_SLI2 0x01; +#define FEATURE_INITIAL_SLI2 0x01 uint32_t type:8; -#define TYPE_NATIVE_SLI2 0x01; +#define TYPE_NATIVE_SLI2 0x01 #endif uint32_t mailBoxSize; diff --git a/drivers/usb/otg/twl4030-usb.c b/drivers/usb/otg/twl4030-usb.c index efeb4d1517ff..14f66c358629 100644 --- a/drivers/usb/otg/twl4030-usb.c +++ b/drivers/usb/otg/twl4030-usb.c @@ -166,7 +166,7 @@ struct twl4030_usb { }; /* internal define on top of container_of */ -#define xceiv_to_twl(x) container_of((x), struct twl4030_usb, otg); +#define xceiv_to_twl(x) container_of((x), struct twl4030_usb, otg) /*-------------------------------------------------------------------------*/ diff --git a/drivers/video/i810/i810.h b/drivers/video/i810/i810.h index f37de60ecc59..1414b73ac55b 100644 --- a/drivers/video/i810/i810.h +++ b/drivers/video/i810/i810.h @@ -137,7 +137,7 @@ #define DRAM_ON 0x08 #define DRAM_OFF 0xE7 #define PG_ENABLE_MASK 0x01 -#define RING_SIZE_MASK (RINGBUFFER_SIZE - 1); +#define RING_SIZE_MASK (RINGBUFFER_SIZE - 1) /* defines for restoring registers partially */ #define ADDR_MAP_MASK (0x07 << 5) diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 6365f041745b..563755181c1e 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -35,7 +35,7 @@ #define CEPH_OPT_MYIP (1<<2) /* specified my ip */ #define CEPH_OPT_NOCRC (1<<3) /* no data crc on writes */ -#define CEPH_OPT_DEFAULT (0); +#define CEPH_OPT_DEFAULT (0) #define ceph_set_opt(client, opt) \ (client)->options->flags |= CEPH_OPT_##opt; diff --git a/include/linux/mfd/tps65910.h b/include/linux/mfd/tps65910.h index 8bb85b930c07..73572c65d04f 100644 --- a/include/linux/mfd/tps65910.h +++ b/include/linux/mfd/tps65910.h @@ -269,7 +269,7 @@ #define LDO1_SEL_MASK 0xFC #define LDO3_SEL_MASK 0x7C #define LDO_MIN_VOLT 1000 -#define LDO_MAX_VOLT 3300; +#define LDO_MAX_VOLT 3300 /*Register VDIG1 (0x80) register.RegisterDescription */ diff --git a/include/scsi/scsi.h b/include/scsi/scsi.h index 3668903e397b..8001ae4cd7ba 100644 --- a/include/scsi/scsi.h +++ b/include/scsi/scsi.h @@ -495,7 +495,7 @@ static inline int scsi_is_wlun(unsigned int lun) #define sense_class(sense) (((sense) >> 4) & 0x7) #define sense_error(sense) ((sense) & 0xf) -#define sense_valid(sense) ((sense) & 0x80); +#define sense_valid(sense) ((sense) & 0x80) /* * default timeouts diff --git a/include/sound/soundfont.h b/include/sound/soundfont.h index f95d99ba7f74..679df0574066 100644 --- a/include/sound/soundfont.h +++ b/include/sound/soundfont.h @@ -121,7 +121,7 @@ int snd_soundfont_search_zone(struct snd_sf_list *sflist, int *notep, int vel, int snd_sf_calc_parm_hold(int msec); int snd_sf_calc_parm_attack(int msec); int snd_sf_calc_parm_decay(int msec); -#define snd_sf_calc_parm_delay(msec) (0x8000 - (msec) * 1000 / 725); +#define snd_sf_calc_parm_delay(msec) (0x8000 - (msec) * 1000 / 725) extern int snd_sf_vol_table[128]; int snd_sf_linear_to_log(unsigned int amount, int offset, int ratio); diff --git a/mm/slub.c b/mm/slub.c index 35f351f26193..5b7f7eb680e1 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4058,7 +4058,7 @@ static int any_slab_objects(struct kmem_cache *s) #endif #define to_slab_attr(n) container_of(n, struct slab_attribute, attr) -#define to_slab(n) container_of(n, struct kmem_cache, kobj); +#define to_slab(n) container_of(n, struct kmem_cache, kobj) struct slab_attribute { struct attribute attr; diff --git a/net/mac80211/mesh_hwmp.c b/net/mac80211/mesh_hwmp.c index 2b18053070c1..3460108810d5 100644 --- a/net/mac80211/mesh_hwmp.c +++ b/net/mac80211/mesh_hwmp.c @@ -57,29 +57,29 @@ static inline u32 u16_field_get(u8 *preq_elem, int offset, bool ae) #define PREQ_IE_TTL(x) (*(x + 2)) #define PREQ_IE_PREQ_ID(x) u32_field_get(x, 3, 0) #define PREQ_IE_ORIG_ADDR(x) (x + 7) -#define PREQ_IE_ORIG_SN(x) u32_field_get(x, 13, 0); -#define PREQ_IE_LIFETIME(x) u32_field_get(x, 17, AE_F_SET(x)); -#define PREQ_IE_METRIC(x) u32_field_get(x, 21, AE_F_SET(x)); +#define PREQ_IE_ORIG_SN(x) u32_field_get(x, 13, 0) +#define PREQ_IE_LIFETIME(x) u32_field_get(x, 17, AE_F_SET(x)) +#define PREQ_IE_METRIC(x) u32_field_get(x, 21, AE_F_SET(x)) #define PREQ_IE_TARGET_F(x) (*(AE_F_SET(x) ? x + 32 : x + 26)) #define PREQ_IE_TARGET_ADDR(x) (AE_F_SET(x) ? x + 33 : x + 27) -#define PREQ_IE_TARGET_SN(x) u32_field_get(x, 33, AE_F_SET(x)); +#define PREQ_IE_TARGET_SN(x) u32_field_get(x, 33, AE_F_SET(x)) #define PREP_IE_FLAGS(x) PREQ_IE_FLAGS(x) #define PREP_IE_HOPCOUNT(x) PREQ_IE_HOPCOUNT(x) #define PREP_IE_TTL(x) PREQ_IE_TTL(x) #define PREP_IE_ORIG_ADDR(x) (x + 3) -#define PREP_IE_ORIG_SN(x) u32_field_get(x, 9, 0); -#define PREP_IE_LIFETIME(x) u32_field_get(x, 13, AE_F_SET(x)); -#define PREP_IE_METRIC(x) u32_field_get(x, 17, AE_F_SET(x)); +#define PREP_IE_ORIG_SN(x) u32_field_get(x, 9, 0) +#define PREP_IE_LIFETIME(x) u32_field_get(x, 13, AE_F_SET(x)) +#define PREP_IE_METRIC(x) u32_field_get(x, 17, AE_F_SET(x)) #define PREP_IE_TARGET_ADDR(x) (AE_F_SET(x) ? x + 27 : x + 21) -#define PREP_IE_TARGET_SN(x) u32_field_get(x, 27, AE_F_SET(x)); +#define PREP_IE_TARGET_SN(x) u32_field_get(x, 27, AE_F_SET(x)) #define PERR_IE_TTL(x) (*(x)) #define PERR_IE_TARGET_FLAGS(x) (*(x + 2)) #define PERR_IE_TARGET_ADDR(x) (x + 3) -#define PERR_IE_TARGET_SN(x) u32_field_get(x, 9, 0); -#define PERR_IE_TARGET_RCODE(x) u16_field_get(x, 13, 0); +#define PERR_IE_TARGET_SN(x) u32_field_get(x, 9, 0) +#define PERR_IE_TARGET_RCODE(x) u16_field_get(x, 13, 0) #define MSEC_TO_TU(x) (x*1000/1024) #define SN_GT(x, y) ((long) (y) - (long) (x) < 0) -- cgit v1.2.3 From f53173e47dee5f7514d264796bec58d43ed0f67f Mon Sep 17 00:00:00 2001 From: Cyrill Gorcunov Date: Thu, 21 Jul 2011 20:06:25 +0400 Subject: x86, perf: P4 PMU - Fix typos in comments and style cleanup This patch: - fixes typos in comments and clarifies the text - renames obscure p4_event_alias::original and ::alter members to ::original and ::alternative as appropriate - drops parenthesis from the return of p4_get_alias_event() No functional changes. Reported-by: Ingo Molnar Signed-off-by: Cyrill Gorcunov Link: http://lkml.kernel.org/r/20110721160625.GX7492@sun Signed-off-by: Ingo Molnar --- arch/x86/include/asm/perf_event_p4.h | 2 +- arch/x86/kernel/cpu/perf_event_p4.c | 66 ++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 37 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/perf_event_p4.h b/arch/x86/include/asm/perf_event_p4.h index 4d86c86178f2..4f7e67e2345e 100644 --- a/arch/x86/include/asm/perf_event_p4.h +++ b/arch/x86/include/asm/perf_event_p4.h @@ -133,7 +133,7 @@ /* * In case of event aliasing we need to preserve some - * caller bits otherwise the mapping won't be complete. + * caller bits, otherwise the mapping won't be complete. */ #define P4_CONFIG_EVENT_ALIAS_MASK \ (p4_config_pack_escr(P4_CONFIG_MASK_ESCR) | \ diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c index 8b7a0c306784..7809d2bcb209 100644 --- a/arch/x86/kernel/cpu/perf_event_p4.c +++ b/arch/x86/kernel/cpu/perf_event_p4.c @@ -571,35 +571,34 @@ static __initconst const u64 p4_hw_cache_event_ids }; /* - * Because of Netburst being quite restricted in now - * many same events can run simultaneously, we use - * event aliases, ie different events which have the - * same functionallity but use non-intersected resources - * (ESCR/CCCR/couter registers). This allow us to run - * two or more semi-same events together. It is done - * transparently to a user space. + * Because of Netburst being quite restricted in how many + * identical events may run simultaneously, we introduce event aliases, + * ie the different events which have the same functionality but + * utilize non-intersected resources (ESCR/CCCR/counter registers). * - * Never set any cusom internal bits such as P4_CONFIG_HT, - * P4_CONFIG_ALIASABLE or bits for P4_PEBS_METRIC, they are - * either up-to-dated automatically either not appliable - * at all. + * This allow us to relax restrictions a bit and run two or more + * identical events together. * - * And be really carefull choosing aliases! + * Never set any custom internal bits such as P4_CONFIG_HT, + * P4_CONFIG_ALIASABLE or bits for P4_PEBS_METRIC, they are + * either up to date automatically or not applicable at all. */ struct p4_event_alias { - u64 orig; - u64 alter; + u64 original; + u64 alternative; } p4_event_aliases[] = { { /* - * Non-halted cycles can be substituted with - * non-sleeping cycles (see Intel SDM Vol3b for - * details). + * Non-halted cycles can be substituted with non-sleeping cycles (see + * Intel SDM Vol3b for details). We need this alias to be able + * to run nmi-watchdog and 'perf top' (or any other user space tool + * which is interested in running PERF_COUNT_HW_CPU_CYCLES) + * simultaneously. */ - .orig = + .original = p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_GLOBAL_POWER_EVENTS) | P4_ESCR_EMASK_BIT(P4_EVENT_GLOBAL_POWER_EVENTS, RUNNING)), - .alter = + .alternative = p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_EXECUTION_EVENT) | P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS0)| P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS1)| @@ -620,25 +619,21 @@ static u64 p4_get_alias_event(u64 config) int i; /* - * Probably we're lucky and don't have to do - * matching over all config bits. + * Only event with special mark is allowed, + * we're to be sure it didn't come as malformed + * RAW event. */ if (!(config & P4_CONFIG_ALIASABLE)) return 0; config_match = config & P4_CONFIG_EVENT_ALIAS_MASK; - /* - * If an event was previously swapped to the alter config - * we should swap it back otherwise contnention on registers - * will return back. - */ for (i = 0; i < ARRAY_SIZE(p4_event_aliases); i++) { - if (config_match == p4_event_aliases[i].orig) { - config_match = p4_event_aliases[i].alter; + if (config_match == p4_event_aliases[i].original) { + config_match = p4_event_aliases[i].alternative; break; - } else if (config_match == p4_event_aliases[i].alter) { - config_match = p4_event_aliases[i].orig; + } else if (config_match == p4_event_aliases[i].alternative) { + config_match = p4_event_aliases[i].original; break; } } @@ -646,8 +641,7 @@ static u64 p4_get_alias_event(u64 config) if (i >= ARRAY_SIZE(p4_event_aliases)) return 0; - return config_match | - (config & P4_CONFIG_EVENT_ALIAS_IMMUTABLE_BITS); + return config_match | (config & P4_CONFIG_EVENT_ALIAS_IMMUTABLE_BITS); } static u64 p4_general_events[PERF_COUNT_HW_MAX] = { @@ -1229,9 +1223,9 @@ static int p4_pmu_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign again: /* - * Aliases are swappable so we may hit circular - * lock if both original config and alias need - * resources (MSR registers) which already busy. + * It's possible to hit a circular lock + * between original and alternative events + * if both are scheduled already. */ if (pass > 2) goto done; @@ -1251,7 +1245,7 @@ again: cntr_idx = p4_next_cntr(thread, used_mask, bind); if (cntr_idx == -1 || test_bit(escr_idx, escr_mask)) { /* - * Probably an event alias is still available. + * Check whether an event alias is still available. */ config_alias = p4_get_alias_event(hwc->config); if (!config_alias) -- cgit v1.2.3 From 1ac2e6ca44e13a087eb7438d284f887097ee7e84 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Tue, 7 Jun 2011 11:49:55 +0200 Subject: x86, perf: Make copy_from_user_nmi() a library function copy_from_user_nmi() is used in oprofile and perf. Moving it to other library functions like copy_from_user(). As this is x86 code for 32 and 64 bits, create a new file usercopy.c for unified code. Signed-off-by: Robert Richter Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/r/20110607172413.GJ20052@erda.amd.com Signed-off-by: Ingo Molnar --- arch/x86/include/asm/uaccess.h | 3 +++ arch/x86/kernel/cpu/perf_event.c | 35 -------------------------------- arch/x86/lib/Makefile | 2 +- arch/x86/lib/usercopy.c | 43 ++++++++++++++++++++++++++++++++++++++++ arch/x86/oprofile/backtrace.c | 39 +----------------------------------- 5 files changed, 48 insertions(+), 74 deletions(-) create mode 100644 arch/x86/lib/usercopy.c (limited to 'arch/x86/kernel') diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 99ddd148a760..36361bf6fdd1 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -555,6 +555,9 @@ struct __large_struct { unsigned long buf[100]; }; #endif /* CONFIG_X86_WP_WORKS_OK */ +extern unsigned long +copy_from_user_nmi(void *to, const void __user *from, unsigned long n); + /* * movsl can be slow when source and dest are not both 8-byte aligned */ diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index b7a010fce8c3..4ee3abf20ed6 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -67,40 +66,6 @@ enum extra_reg_type { EXTRA_REG_MAX /* number of entries needed */ }; -/* - * best effort, GUP based copy_from_user() that assumes IRQ or NMI context - */ -static unsigned long -copy_from_user_nmi(void *to, const void __user *from, unsigned long n) -{ - unsigned long offset, addr = (unsigned long)from; - unsigned long size, len = 0; - struct page *page; - void *map; - int ret; - - do { - ret = __get_user_pages_fast(addr, 1, 0, &page); - if (!ret) - break; - - offset = addr & (PAGE_SIZE - 1); - size = min(PAGE_SIZE - offset, n - len); - - map = kmap_atomic(page); - memcpy(to, map+offset, size); - kunmap_atomic(map); - put_page(page); - - len += size; - to += size; - addr += size; - - } while (len < n); - - return len; -} - struct event_constraint { union { unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index f2479f19ddde..6ba477342b8e 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -18,7 +18,7 @@ obj-$(CONFIG_SMP) += msr-smp.o cache-smp.o lib-y := delay.o lib-y += thunk_$(BITS).o -lib-y += usercopy_$(BITS).o getuser.o putuser.o +lib-y += usercopy_$(BITS).o usercopy.o getuser.o putuser.o lib-y += memcpy_$(BITS).o lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c new file mode 100644 index 000000000000..97be9cb54483 --- /dev/null +++ b/arch/x86/lib/usercopy.c @@ -0,0 +1,43 @@ +/* + * User address space access functions. + * + * For licencing details see kernel-base/COPYING + */ + +#include +#include + +/* + * best effort, GUP based copy_from_user() that is NMI-safe + */ +unsigned long +copy_from_user_nmi(void *to, const void __user *from, unsigned long n) +{ + unsigned long offset, addr = (unsigned long)from; + unsigned long size, len = 0; + struct page *page; + void *map; + int ret; + + do { + ret = __get_user_pages_fast(addr, 1, 0, &page); + if (!ret) + break; + + offset = addr & (PAGE_SIZE - 1); + size = min(PAGE_SIZE - offset, n - len); + + map = kmap_atomic(page); + memcpy(to, map+offset, size); + kunmap_atomic(map); + put_page(page); + + len += size; + to += size; + addr += size; + + } while (len < n); + + return len; +} +EXPORT_SYMBOL_GPL(copy_from_user_nmi); diff --git a/arch/x86/oprofile/backtrace.c b/arch/x86/oprofile/backtrace.c index 32f78eb46744..bff89dfe3619 100644 --- a/arch/x86/oprofile/backtrace.c +++ b/arch/x86/oprofile/backtrace.c @@ -12,10 +12,9 @@ #include #include #include -#include +#include #include -#include #include static int backtrace_stack(void *data, char *name) @@ -38,42 +37,6 @@ static struct stacktrace_ops backtrace_ops = { .walk_stack = print_context_stack, }; -/* from arch/x86/kernel/cpu/perf_event.c: */ - -/* - * best effort, GUP based copy_from_user() that assumes IRQ or NMI context - */ -static unsigned long -copy_from_user_nmi(void *to, const void __user *from, unsigned long n) -{ - unsigned long offset, addr = (unsigned long)from; - unsigned long size, len = 0; - struct page *page; - void *map; - int ret; - - do { - ret = __get_user_pages_fast(addr, 1, 0, &page); - if (!ret) - break; - - offset = addr & (PAGE_SIZE - 1); - size = min(PAGE_SIZE - offset, n - len); - - map = kmap_atomic(page); - memcpy(to, map+offset, size); - kunmap_atomic(map); - put_page(page); - - len += size; - to += size; - addr += size; - - } while (len < n); - - return len; -} - #ifdef CONFIG_COMPAT static struct stack_frame_ia32 * dump_user_backtrace_32(struct stack_frame_ia32 *head) -- cgit v1.2.3 From b7798d28ec15d20fd34b70fa57eb13f0cf6d1ecd Mon Sep 17 00:00:00 2001 From: Daniel J Blueman Date: Fri, 13 May 2011 09:04:59 +0800 Subject: x86: Make Dell Latitude E5420 use reboot=pci Rebooting on the Dell E5420 often hangs with the keyboard or ACPI methods, but is reliable via the PCI method. [ hpa: this was deferred because we believed for a long time that the recent reshuffling of the boot priorities in commit 660e34cebf0a11d54f2d5dd8838607452355f321 fixed this platform. Unfortunately that turned out to be incorrect. ] Signed-off-by: Daniel J Blueman Link: http://lkml.kernel.org/r/1305248699-2347-1-git-send-email-daniel.blueman@gmail.com Signed-off-by: H. Peter Anvin Cc: --- arch/x86/kernel/reboot.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index 14eed214b584..3dfd38f122d7 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -427,6 +427,14 @@ static struct dmi_system_id __initdata pci_reboot_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Latitude E6320"), }, }, + { /* Handle problems with rebooting on the Latitude E5420. */ + .callback = set_pci_reboot, + .ident = "Dell Latitude E5420", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Latitude E5420"), + }, + }, { } }; -- cgit v1.2.3 From a536877e77f73ea22d12d94a019fedd9671b6acd Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Thu, 21 Jul 2011 11:22:21 -0700 Subject: x86: Make Dell Latitude E6420 use reboot=pci Yet another variant of the Dell Latitude series which requires reboot=pci. From the E5420 bug report by Daniel J Blueman: > The E6420 is affected also (same platform, different casing and > features), which provides an external confirmation of the issue; I can > submit a patch for that later or include it if you prefer: > http://linux.koolsolutions.com/2009/08/04/howto-fix-linux-hangfreeze-during-reboots-and-restarts/ Reported-by: Daniel J Blueman Signed-off-by: H. Peter Anvin Cc: --- arch/x86/kernel/reboot.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index 3dfd38f122d7..9242436e9937 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -435,6 +435,14 @@ static struct dmi_system_id __initdata pci_reboot_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Latitude E5420"), }, }, + { /* Handle problems with rebooting on the Latitude E6420. */ + .callback = set_pci_reboot, + .ident = "Dell Latitude E6420", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Latitude E6420"), + }, + }, { } }; -- cgit v1.2.3 From 5dea1c88ed11a1221581c4b202f053c4fc138704 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 22 Jul 2011 14:39:48 +0930 Subject: lguest: use a special 1:1 linear pagetable mode until first switch. The Host used to create some page tables for the Guest to use at the top of Guest memory; it would then tell the Guest where this was. In particular, it created linear mappings for 0 and 0xC0000000 addresses because lguest used to switch to its real page tables quite late in boot. However, since d50d8fe19 Linux initialized boot page tables in head_32.S even before the "are we lguest?" boot jump. So, now we can simplify things: the Host pagetable code assumes 1:1 linear mapping until it first calls the LHCALL_NEW_PGTABLE hypercall, which we now do before we reach C code. This also means that the Host doesn't need to know anything about the Guest's PAGE_OFFSET. (Non-Linux guests might not even have such a thing). Signed-off-by: Rusty Russell --- arch/x86/kernel/asm-offsets_32.c | 1 - arch/x86/lguest/boot.c | 11 +- arch/x86/lguest/i386_head.S | 9 +- drivers/lguest/lg.h | 2 + drivers/lguest/page_tables.c | 278 ++++++++++++--------------------------- include/linux/lguest.h | 2 - 6 files changed, 98 insertions(+), 205 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c index c29d631af6fc..395a10e68067 100644 --- a/arch/x86/kernel/asm-offsets_32.c +++ b/arch/x86/kernel/asm-offsets_32.c @@ -63,7 +63,6 @@ void foo(void) BLANK(); OFFSET(LGUEST_DATA_irq_enabled, lguest_data, irq_enabled); OFFSET(LGUEST_DATA_irq_pending, lguest_data, irq_pending); - OFFSET(LGUEST_DATA_pgdir, lguest_data, pgdir); BLANK(); OFFSET(LGUEST_PAGES_host_gdt_desc, lguest_pages, state.host_gdt_desc); diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c index db832fd65ecb..719a32c60516 100644 --- a/arch/x86/lguest/boot.c +++ b/arch/x86/lguest/boot.c @@ -520,17 +520,16 @@ static unsigned long lguest_read_cr2(void) /* See lguest_set_pte() below. */ static bool cr3_changed = false; +static unsigned long current_cr3; /* * cr3 is the current toplevel pagetable page: the principle is the same as - * cr0. Keep a local copy, and tell the Host when it changes. The only - * difference is that our local copy is in lguest_data because the Host needs - * to set it upon our initial hypercall. + * cr0. Keep a local copy, and tell the Host when it changes. */ static void lguest_write_cr3(unsigned long cr3) { - lguest_data.pgdir = cr3; lazy_hcall1(LHCALL_NEW_PGTABLE, cr3); + current_cr3 = cr3; /* These two page tables are simple, linear, and used during boot */ if (cr3 != __pa(swapper_pg_dir) && cr3 != __pa(initial_page_table)) @@ -539,7 +538,7 @@ static void lguest_write_cr3(unsigned long cr3) static unsigned long lguest_read_cr3(void) { - return lguest_data.pgdir; + return current_cr3; } /* cr4 is used to enable and disable PGE, but we don't care. */ @@ -758,7 +757,7 @@ static void lguest_pmd_clear(pmd_t *pmdp) static void lguest_flush_tlb_single(unsigned long addr) { /* Simply set it to zero: if it was not, it will fault back in. */ - lazy_hcall3(LHCALL_SET_PTE, lguest_data.pgdir, addr, 0); + lazy_hcall3(LHCALL_SET_PTE, current_cr3, addr, 0); } /* diff --git a/arch/x86/lguest/i386_head.S b/arch/x86/lguest/i386_head.S index 4f420c2f2d55..863b03a9fbff 100644 --- a/arch/x86/lguest/i386_head.S +++ b/arch/x86/lguest/i386_head.S @@ -27,13 +27,18 @@ .section .init.text, "ax", @progbits ENTRY(lguest_entry) /* - * We make the "initialization" hypercall now to tell the Host about - * us, and also find out where it put our page tables. + * We make the "initialization" hypercall now to tell the Host where + * our lguest_data struct is. */ movl $LHCALL_LGUEST_INIT, %eax movl $lguest_data - __PAGE_OFFSET, %ebx int $LGUEST_TRAP_ENTRY + /* Now turn our pagetables on; setup by arch/x86/kernel/head_32.S. */ + movl $LHCALL_NEW_PGTABLE, %eax + movl $(initial_page_table - __PAGE_OFFSET), %ebx + int $LGUEST_TRAP_ENTRY + /* Set up the initial stack so we can run C code. */ movl $(init_thread_union+THREAD_SIZE),%esp diff --git a/drivers/lguest/lg.h b/drivers/lguest/lg.h index 9136411fadd5..295df06e6590 100644 --- a/drivers/lguest/lg.h +++ b/drivers/lguest/lg.h @@ -59,6 +59,8 @@ struct lg_cpu { struct lguest_pages *last_pages; + /* Initialization mode: linear map everything. */ + bool linear_pages; int cpu_pgd; /* Which pgd this cpu is currently using */ /* If a hypercall was asked for, this points to the arguments. */ diff --git a/drivers/lguest/page_tables.c b/drivers/lguest/page_tables.c index d21578ee95de..00026222bde8 100644 --- a/drivers/lguest/page_tables.c +++ b/drivers/lguest/page_tables.c @@ -17,7 +17,6 @@ #include #include #include -#include #include "lg.h" /*M:008 @@ -325,10 +324,15 @@ bool demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) #endif /* First step: get the top-level Guest page table entry. */ - gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t); - /* Toplevel not present? We can't map it in. */ - if (!(pgd_flags(gpgd) & _PAGE_PRESENT)) - return false; + if (unlikely(cpu->linear_pages)) { + /* Faking up a linear mapping. */ + gpgd = __pgd(CHECK_GPGD_MASK); + } else { + gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t); + /* Toplevel not present? We can't map it in. */ + if (!(pgd_flags(gpgd) & _PAGE_PRESENT)) + return false; + } /* Now look at the matching shadow entry. */ spgd = spgd_addr(cpu, cpu->cpu_pgd, vaddr); @@ -353,10 +357,15 @@ bool demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) } #ifdef CONFIG_X86_PAE - gpmd = lgread(cpu, gpmd_addr(gpgd, vaddr), pmd_t); - /* Middle level not present? We can't map it in. */ - if (!(pmd_flags(gpmd) & _PAGE_PRESENT)) - return false; + if (unlikely(cpu->linear_pages)) { + /* Faking up a linear mapping. */ + gpmd = __pmd(_PAGE_TABLE); + } else { + gpmd = lgread(cpu, gpmd_addr(gpgd, vaddr), pmd_t); + /* Middle level not present? We can't map it in. */ + if (!(pmd_flags(gpmd) & _PAGE_PRESENT)) + return false; + } /* Now look at the matching shadow entry. */ spmd = spmd_addr(cpu, *spgd, vaddr); @@ -397,8 +406,13 @@ bool demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) gpte_ptr = gpte_addr(cpu, gpgd, vaddr); #endif - /* Read the actual PTE value. */ - gpte = lgread(cpu, gpte_ptr, pte_t); + if (unlikely(cpu->linear_pages)) { + /* Linear? Make up a PTE which points to same page. */ + gpte = __pte((vaddr & PAGE_MASK) | _PAGE_RW | _PAGE_PRESENT); + } else { + /* Read the actual PTE value. */ + gpte = lgread(cpu, gpte_ptr, pte_t); + } /* If this page isn't in the Guest page tables, we can't page it in. */ if (!(pte_flags(gpte) & _PAGE_PRESENT)) @@ -454,7 +468,8 @@ bool demand_page(struct lg_cpu *cpu, unsigned long vaddr, int errcode) * Finally, we write the Guest PTE entry back: we've set the * _PAGE_ACCESSED and maybe the _PAGE_DIRTY flags. */ - lgwrite(cpu, gpte_ptr, pte_t, gpte); + if (likely(!cpu->linear_pages)) + lgwrite(cpu, gpte_ptr, pte_t, gpte); /* * The fault is fixed, the page table is populated, the mapping @@ -612,6 +627,11 @@ unsigned long guest_pa(struct lg_cpu *cpu, unsigned long vaddr) #ifdef CONFIG_X86_PAE pmd_t gpmd; #endif + + /* Still not set up? Just map 1:1. */ + if (unlikely(cpu->linear_pages)) + return vaddr; + /* First step: get the top-level Guest page table entry. */ gpgd = lgread(cpu, gpgd_addr(cpu, vaddr), pgd_t); /* Toplevel not present? We can't map it in. */ @@ -708,32 +728,6 @@ static unsigned int new_pgdir(struct lg_cpu *cpu, return next; } -/*H:430 - * (iv) Switching page tables - * - * Now we've seen all the page table setting and manipulation, let's see - * what happens when the Guest changes page tables (ie. changes the top-level - * pgdir). This occurs on almost every context switch. - */ -void guest_new_pagetable(struct lg_cpu *cpu, unsigned long pgtable) -{ - int newpgdir, repin = 0; - - /* Look to see if we have this one already. */ - newpgdir = find_pgdir(cpu->lg, pgtable); - /* - * If not, we allocate or mug an existing one: if it's a fresh one, - * repin gets set to 1. - */ - if (newpgdir == ARRAY_SIZE(cpu->lg->pgdirs)) - newpgdir = new_pgdir(cpu, pgtable, &repin); - /* Change the current pgd index to the new one. */ - cpu->cpu_pgd = newpgdir; - /* If it was completely blank, we map in the Guest kernel stack */ - if (repin) - pin_stack_pages(cpu); -} - /*H:470 * Finally, a routine which throws away everything: all PGD entries in all * the shadow page tables, including the Guest's kernel mappings. This is used @@ -780,6 +774,44 @@ void guest_pagetable_clear_all(struct lg_cpu *cpu) /* We need the Guest kernel stack mapped again. */ pin_stack_pages(cpu); } + +/*H:430 + * (iv) Switching page tables + * + * Now we've seen all the page table setting and manipulation, let's see + * what happens when the Guest changes page tables (ie. changes the top-level + * pgdir). This occurs on almost every context switch. + */ +void guest_new_pagetable(struct lg_cpu *cpu, unsigned long pgtable) +{ + int newpgdir, repin = 0; + + /* + * The very first time they call this, we're actually running without + * any page tables; we've been making it up. Throw them away now. + */ + if (unlikely(cpu->linear_pages)) { + release_all_pagetables(cpu->lg); + cpu->linear_pages = false; + /* Force allocation of a new pgdir. */ + newpgdir = ARRAY_SIZE(cpu->lg->pgdirs); + } else { + /* Look to see if we have this one already. */ + newpgdir = find_pgdir(cpu->lg, pgtable); + } + + /* + * If not, we allocate or mug an existing one: if it's a fresh one, + * repin gets set to 1. + */ + if (newpgdir == ARRAY_SIZE(cpu->lg->pgdirs)) + newpgdir = new_pgdir(cpu, pgtable, &repin); + /* Change the current pgd index to the new one. */ + cpu->cpu_pgd = newpgdir; + /* If it was completely blank, we map in the Guest kernel stack */ + if (repin) + pin_stack_pages(cpu); +} /*:*/ /*M:009 @@ -919,168 +951,26 @@ void guest_set_pmd(struct lguest *lg, unsigned long pmdp, u32 idx) } #endif -/*H:505 - * To get through boot, we construct simple identity page mappings (which - * set virtual == physical) and linear mappings which will get the Guest far - * enough into the boot to create its own. The linear mapping means we - * simplify the Guest boot, but it makes assumptions about their PAGE_OFFSET, - * as you'll see. - * - * We lay them out of the way, just below the initrd (which is why we need to - * know its size here). - */ -static unsigned long setup_pagetables(struct lguest *lg, - unsigned long mem, - unsigned long initrd_size) -{ - pgd_t __user *pgdir; - pte_t __user *linear; - unsigned long mem_base = (unsigned long)lg->mem_base; - unsigned int mapped_pages, i, linear_pages; -#ifdef CONFIG_X86_PAE - pmd_t __user *pmds; - unsigned int j; - pgd_t pgd; - pmd_t pmd; -#else - unsigned int phys_linear; -#endif - - /* - * We have mapped_pages frames to map, so we need linear_pages page - * tables to map them. - */ - mapped_pages = mem / PAGE_SIZE; - linear_pages = (mapped_pages + PTRS_PER_PTE - 1) / PTRS_PER_PTE; - - /* We put the toplevel page directory page at the top of memory. */ - pgdir = (pgd_t *)(mem + mem_base - initrd_size - PAGE_SIZE); - - /* Now we use the next linear_pages pages as pte pages */ - linear = (void *)pgdir - linear_pages * PAGE_SIZE; - -#ifdef CONFIG_X86_PAE - /* - * And the single mid page goes below that. We only use one, but - * that's enough to map 1G, which definitely gets us through boot. - */ - pmds = (void *)linear - PAGE_SIZE; -#endif - /* - * Linear mapping is easy: put every page's address into the - * mapping in order. - */ - for (i = 0; i < mapped_pages; i++) { - pte_t pte; - pte = pfn_pte(i, __pgprot(_PAGE_PRESENT|_PAGE_RW|_PAGE_USER)); - if (copy_to_user(&linear[i], &pte, sizeof(pte)) != 0) - return -EFAULT; - } - -#ifdef CONFIG_X86_PAE - /* - * Make the Guest PMD entries point to the corresponding place in the - * linear mapping (up to one page worth of PMD). - */ - for (i = j = 0; i < mapped_pages && j < PTRS_PER_PMD; - i += PTRS_PER_PTE, j++) { - pmd = pfn_pmd(((unsigned long)&linear[i] - mem_base)/PAGE_SIZE, - __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER)); - - if (copy_to_user(&pmds[j], &pmd, sizeof(pmd)) != 0) - return -EFAULT; - } - - /* One PGD entry, pointing to that PMD page. */ - pgd = __pgd(((unsigned long)pmds - mem_base) | _PAGE_PRESENT); - /* Copy it in as the first PGD entry (ie. addresses 0-1G). */ - if (copy_to_user(&pgdir[0], &pgd, sizeof(pgd)) != 0) - return -EFAULT; - /* - * And the other PGD entry to make the linear mapping at PAGE_OFFSET - */ - if (copy_to_user(&pgdir[KERNEL_PGD_BOUNDARY], &pgd, sizeof(pgd))) - return -EFAULT; -#else - /* - * The top level points to the linear page table pages above. - * We setup the identity and linear mappings here. - */ - phys_linear = (unsigned long)linear - mem_base; - for (i = 0; i < mapped_pages; i += PTRS_PER_PTE) { - pgd_t pgd; - /* - * Create a PGD entry which points to the right part of the - * linear PTE pages. - */ - pgd = __pgd((phys_linear + i * sizeof(pte_t)) | - (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER)); - - /* - * Copy it into the PGD page at 0 and PAGE_OFFSET. - */ - if (copy_to_user(&pgdir[i / PTRS_PER_PTE], &pgd, sizeof(pgd)) - || copy_to_user(&pgdir[pgd_index(PAGE_OFFSET) - + i / PTRS_PER_PTE], - &pgd, sizeof(pgd))) - return -EFAULT; - } -#endif - - /* - * We return the top level (guest-physical) address: we remember where - * this is to write it into lguest_data when the Guest initializes. - */ - return (unsigned long)pgdir - mem_base; -} - /*H:500 * (vii) Setting up the page tables initially. * - * When a Guest is first created, the Launcher tells us where the toplevel of - * its first page table is. We set some things up here: + * When a Guest is first created, set initialize a shadow page table which + * we will populate on future faults. The Guest doesn't have any actual + * pagetables yet, so we set linear_pages to tell demand_page() to fake it + * for the moment. */ int init_guest_pagetable(struct lguest *lg) { - u64 mem; - u32 initrd_size; - struct boot_params __user *boot = (struct boot_params *)lg->mem_base; -#ifdef CONFIG_X86_PAE - pgd_t *pgd; - pmd_t *pmd_table; -#endif - /* - * Get the Guest memory size and the ramdisk size from the boot header - * located at lg->mem_base (Guest address 0). - */ - if (copy_from_user(&mem, &boot->e820_map[0].size, sizeof(mem)) - || get_user(initrd_size, &boot->hdr.ramdisk_size)) - return -EFAULT; + struct lg_cpu *cpu = &lg->cpus[0]; + int allocated = 0; - /* - * We start on the first shadow page table, and give it a blank PGD - * page. - */ - lg->pgdirs[0].gpgdir = setup_pagetables(lg, mem, initrd_size); - if (IS_ERR_VALUE(lg->pgdirs[0].gpgdir)) - return lg->pgdirs[0].gpgdir; - lg->pgdirs[0].pgdir = (pgd_t *)get_zeroed_page(GFP_KERNEL); - if (!lg->pgdirs[0].pgdir) + /* lg (and lg->cpus[]) starts zeroed: this allocates a new pgdir */ + cpu->cpu_pgd = new_pgdir(cpu, 0, &allocated); + if (!allocated) return -ENOMEM; -#ifdef CONFIG_X86_PAE - /* For PAE, we also create the initial mid-level. */ - pgd = lg->pgdirs[0].pgdir; - pmd_table = (pmd_t *) get_zeroed_page(GFP_KERNEL); - if (!pmd_table) - return -ENOMEM; - - set_pgd(pgd + SWITCHER_PGD_INDEX, - __pgd(__pa(pmd_table) | _PAGE_PRESENT)); -#endif - - /* This is the current page table. */ - lg->cpus[0].cpu_pgd = 0; + /* We start with a linear mapping until the initialize. */ + cpu->linear_pages = true; return 0; } @@ -1095,10 +985,10 @@ void page_table_guest_data_init(struct lg_cpu *cpu) * of virtual addresses used by the Switcher. */ || put_user(RESERVE_MEM * 1024 * 1024, - &cpu->lg->lguest_data->reserve_mem) - || put_user(cpu->lg->pgdirs[0].gpgdir, - &cpu->lg->lguest_data->pgdir)) + &cpu->lg->lguest_data->reserve_mem)) { kill_guest(cpu, "bad guest page %p", cpu->lg->lguest_data); + return; + } /* * In flush_user_mappings() we loop from 0 to diff --git a/include/linux/lguest.h b/include/linux/lguest.h index 2fb1dcbcb5aa..9962c6bb1311 100644 --- a/include/linux/lguest.h +++ b/include/linux/lguest.h @@ -59,8 +59,6 @@ struct lguest_data { unsigned long reserve_mem; /* KHz for the TSC clock. */ u32 tsc_khz; - /* Page where the top-level pagetable is */ - unsigned long pgdir; /* Fields initialized by the Guest at boot: */ /* Instruction range to suppress interrupts even if enabled */ -- cgit v1.2.3 From d910f5c1064d7ff09c31b0191564f9f99e210f91 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Mon, 11 Jul 2011 15:28:19 -0400 Subject: KVM guest: KVM Steal time registration This patch implements the kvm bits of the steal time infrastructure. The most important part of it, is the steal time clock. It is an continuous clock that shows the accumulated amount of steal time since vcpu creation. It is supposed to survive cpu offlining/onlining. [marcelo: fix build with CONFIG_KVM_GUEST=n] Signed-off-by: Glauber Costa Acked-by: Rik van Riel Tested-by: Eric B Munson CC: Jeremy Fitzhardinge CC: Peter Zijlstra CC: Avi Kivity CC: Anthony Liguori Signed-off-by: Avi Kivity Signed-off-by: Marcelo Tosatti --- Documentation/kernel-parameters.txt | 4 +++ arch/x86/include/asm/kvm_para.h | 6 ++++ arch/x86/kernel/kvm.c | 72 +++++++++++++++++++++++++++++++++++++ arch/x86/kernel/kvmclock.c | 2 ++ 4 files changed, 84 insertions(+) (limited to 'arch/x86/kernel') diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index fd248a318211..a7225746ed96 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -1737,6 +1737,10 @@ bytes respectively. Such letter suffixes can also be entirely omitted. no-kvmapf [X86,KVM] Disable paravirtualized asynchronous page fault handling. + no-steal-acc [X86,KVM] Disable paravirtualized steal time accounting. + steal time is computed, but won't influence scheduler + behaviour + nolapic [X86-32,APIC] Do not enable or use the local APIC. nolapic_timer [X86-32,APIC] Do not use the local APIC timer. diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index c484ba8e05ea..734c3767cfac 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h @@ -192,6 +192,7 @@ void __init kvm_guest_init(void); void kvm_async_pf_task_wait(u32 token); void kvm_async_pf_task_wake(u32 token); u32 kvm_read_and_reset_pf_reason(void); +extern void kvm_disable_steal_time(void); #else #define kvm_guest_init() do { } while (0) #define kvm_async_pf_task_wait(T) do {} while(0) @@ -200,6 +201,11 @@ static inline u32 kvm_read_and_reset_pf_reason(void) { return 0; } + +static inline void kvm_disable_steal_time(void) +{ + return; +} #endif #endif /* __KERNEL__ */ diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 33c07b0b122e..a9c2116001d6 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -51,6 +51,15 @@ static int parse_no_kvmapf(char *arg) early_param("no-kvmapf", parse_no_kvmapf); +static int steal_acc = 1; +static int parse_no_stealacc(char *arg) +{ + steal_acc = 0; + return 0; +} + +early_param("no-steal-acc", parse_no_stealacc); + struct kvm_para_state { u8 mmu_queue[MMU_QUEUE_SIZE]; int mmu_queue_len; @@ -58,6 +67,8 @@ struct kvm_para_state { static DEFINE_PER_CPU(struct kvm_para_state, para_state); static DEFINE_PER_CPU(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64); +static DEFINE_PER_CPU(struct kvm_steal_time, steal_time) __aligned(64); +static int has_steal_clock = 0; static struct kvm_para_state *kvm_para_state(void) { @@ -441,6 +452,21 @@ static void __init paravirt_ops_setup(void) #endif } +static void kvm_register_steal_time(void) +{ + int cpu = smp_processor_id(); + struct kvm_steal_time *st = &per_cpu(steal_time, cpu); + + if (!has_steal_clock) + return; + + memset(st, 0, sizeof(*st)); + + wrmsrl(MSR_KVM_STEAL_TIME, (__pa(st) | KVM_MSR_ENABLED)); + printk(KERN_INFO "kvm-stealtime: cpu %d, msr %lx\n", + cpu, __pa(st)); +} + void __cpuinit kvm_guest_cpu_init(void) { if (!kvm_para_available()) @@ -457,6 +483,9 @@ void __cpuinit kvm_guest_cpu_init(void) printk(KERN_INFO"KVM setup async PF for cpu %d\n", smp_processor_id()); } + + if (has_steal_clock) + kvm_register_steal_time(); } static void kvm_pv_disable_apf(void *unused) @@ -483,6 +512,31 @@ static struct notifier_block kvm_pv_reboot_nb = { .notifier_call = kvm_pv_reboot_notify, }; +static u64 kvm_steal_clock(int cpu) +{ + u64 steal; + struct kvm_steal_time *src; + int version; + + src = &per_cpu(steal_time, cpu); + do { + version = src->version; + rmb(); + steal = src->steal; + rmb(); + } while ((version & 1) || (version != src->version)); + + return steal; +} + +void kvm_disable_steal_time(void) +{ + if (!has_steal_clock) + return; + + wrmsr(MSR_KVM_STEAL_TIME, 0, 0); +} + #ifdef CONFIG_SMP static void __init kvm_smp_prepare_boot_cpu(void) { @@ -500,6 +554,7 @@ static void __cpuinit kvm_guest_cpu_online(void *dummy) static void kvm_guest_cpu_offline(void *dummy) { + kvm_disable_steal_time(); kvm_pv_disable_apf(NULL); apf_task_wake_all(); } @@ -548,6 +603,11 @@ void __init kvm_guest_init(void) if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF)) x86_init.irqs.trap_init = kvm_apf_trap_init; + if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { + has_steal_clock = 1; + pv_time_ops.steal_clock = kvm_steal_clock; + } + #ifdef CONFIG_SMP smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; register_cpu_notifier(&kvm_cpu_notifier); @@ -555,3 +615,15 @@ void __init kvm_guest_init(void) kvm_guest_cpu_init(); #endif } + +static __init int activate_jump_labels(void) +{ + if (has_steal_clock) { + jump_label_inc(¶virt_steal_enabled); + if (steal_acc) + jump_label_inc(¶virt_steal_rq_enabled); + } + + return 0; +} +arch_initcall(activate_jump_labels); diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 6389a6bca11b..c1a0188e29ae 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -160,6 +160,7 @@ static void __cpuinit kvm_setup_secondary_clock(void) static void kvm_crash_shutdown(struct pt_regs *regs) { native_write_msr(msr_kvm_system_time, 0, 0); + kvm_disable_steal_time(); native_machine_crash_shutdown(regs); } #endif @@ -167,6 +168,7 @@ static void kvm_crash_shutdown(struct pt_regs *regs) static void kvm_shutdown(void) { native_write_msr(msr_kvm_system_time, 0, 0); + kvm_disable_steal_time(); native_machine_shutdown(); } -- cgit v1.2.3 From 66574cc05438dd0907029075d7e6ec5ac0036fbc Mon Sep 17 00:00:00 2001 From: Jonas Bonn Date: Thu, 30 Jun 2011 21:22:12 +0200 Subject: modules: make arch's use default loader hooks This patch removes all the module loader hook implementations in the architecture specific code where the functionality is the same as that now provided by the recently added default hooks. Signed-off-by: Jonas Bonn Acked-by: Mike Frysinger Acked-by: Geert Uytterhoeven Tested-by: Michal Simek Signed-off-by: Rusty Russell --- arch/alpha/kernel/module.c | 34 ----------------------- arch/arm/kernel/module.c | 29 +------------------- arch/avr32/kernel/module.c | 20 -------------- arch/blackfin/kernel/module.c | 21 -------------- arch/cris/kernel/module.c | 43 ++--------------------------- arch/frv/kernel/module.c | 57 ++------------------------------------ arch/h8300/kernel/module.c | 45 ------------------------------ arch/ia64/kernel/module.c | 16 ----------- arch/m32r/kernel/module.c | 38 ------------------------- arch/m68k/kernel/module_mm.c | 27 ------------------ arch/m68k/kernel/module_no.c | 34 ----------------------- arch/microblaze/kernel/module.c | 35 ----------------------- arch/mips/kernel/module.c | 20 ++------------ arch/mn10300/kernel/module.c | 61 ----------------------------------------- arch/parisc/kernel/module.c | 12 -------- arch/powerpc/kernel/module.c | 18 ------------ arch/powerpc/kernel/module_32.c | 11 -------- arch/powerpc/kernel/module_64.c | 10 ------- arch/s390/kernel/module.c | 20 -------------- arch/score/kernel/module.c | 29 ++------------------ arch/sh/kernel/module.c | 35 ----------------------- arch/sparc/kernel/module.c | 28 ------------------- arch/tile/kernel/module.c | 31 --------------------- arch/unicore32/kernel/module.c | 35 ----------------------- arch/x86/kernel/module.c | 37 ------------------------- arch/xtensa/kernel/module.c | 43 ----------------------------- 26 files changed, 12 insertions(+), 777 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/alpha/kernel/module.c b/arch/alpha/kernel/module.c index ebc3c894b5a2..2fd00b7077e4 100644 --- a/arch/alpha/kernel/module.c +++ b/arch/alpha/kernel/module.c @@ -29,20 +29,6 @@ #define DEBUGP(fmt...) #endif -void * -module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc(size); -} - -void -module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - /* Allocate the GOT at the end of the core sections. */ struct got_entry { @@ -155,14 +141,6 @@ module_frob_arch_sections(Elf64_Ehdr *hdr, Elf64_Shdr *sechdrs, return 0; } -int -apply_relocate(Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, - unsigned int relsec, struct module *me) -{ - printk(KERN_ERR "module %s: REL relocation unsupported\n", me->name); - return -ENOEXEC; -} - int apply_relocate_add(Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relsec, @@ -302,15 +280,3 @@ apply_relocate_add(Elf64_Shdr *sechdrs, const char *strtab, return 0; } - -int -module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} - -void -module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index 016d6a0830a3..05b377616fd5 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -43,25 +43,7 @@ void *module_alloc(unsigned long size) GFP_KERNEL, PAGE_KERNEL_EXEC, -1, __builtin_return_address(0)); } -#else /* CONFIG_MMU */ -void *module_alloc(unsigned long size) -{ - return size == 0 ? NULL : vmalloc(size); -} -#endif /* !CONFIG_MMU */ - -void module_free(struct module *module, void *region) -{ - vfree(region); -} - -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} +#endif int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -265,15 +247,6 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, return 0; } -int -apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, - unsigned int symindex, unsigned int relsec, struct module *module) -{ - printk(KERN_ERR "module %s: ADD RELOCATION unsupported\n", - module->name); - return -ENOEXEC; -} - struct mod_unwind_map { const Elf_Shdr *unw_sec; const Elf_Shdr *txt_sec; diff --git a/arch/avr32/kernel/module.c b/arch/avr32/kernel/module.c index a727f54d64d6..596f7305d93f 100644 --- a/arch/avr32/kernel/module.c +++ b/arch/avr32/kernel/module.c @@ -19,13 +19,6 @@ #include #include -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc(size); -} - void module_free(struct module *mod, void *module_region) { vfree(mod->arch.syminfo); @@ -299,15 +292,6 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, return ret; } -int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, - unsigned int symindex, unsigned int relindex, - struct module *module) -{ - printk(KERN_ERR "module %s: REL relocations are not supported\n", - module->name); - return -ENOEXEC; -} - int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *module) { @@ -316,7 +300,3 @@ int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, return 0; } - -void module_arch_cleanup(struct module *module) -{ -} diff --git a/arch/blackfin/kernel/module.c b/arch/blackfin/kernel/module.c index 35e350cad9d9..4489efc52883 100644 --- a/arch/blackfin/kernel/module.c +++ b/arch/blackfin/kernel/module.c @@ -16,19 +16,6 @@ #include #include -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc(size); -} - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - /* Transfer the section to the L1 memory */ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, @@ -150,14 +137,6 @@ module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, return 0; } -int -apply_relocate(Elf_Shdr * sechdrs, const char *strtab, - unsigned int symindex, unsigned int relsec, struct module *mod) -{ - pr_err(".rel unsupported\n"); - return -ENOEXEC; -} - /*************************************************************************/ /* FUNCTION : apply_relocate_add */ /* ABSTRACT : Blackfin specific relocation handling for the loadable */ diff --git a/arch/cris/kernel/module.c b/arch/cris/kernel/module.c index bcd502f74cda..37400f5869e6 100644 --- a/arch/cris/kernel/module.c +++ b/arch/cris/kernel/module.c @@ -30,45 +30,19 @@ #endif #ifdef CONFIG_ETRAX_KMALLOCED_MODULES -#define MALLOC_MODULE(size) kmalloc(size, GFP_KERNEL) -#define FREE_MODULE(region) kfree(region) -#else -#define MALLOC_MODULE(size) vmalloc_exec(size) -#define FREE_MODULE(region) vfree(region) -#endif - void *module_alloc(unsigned long size) { if (size == 0) return NULL; - return MALLOC_MODULE(size); + return kmalloc(size, GFP_KERNEL); } - /* Free memory returned from module_alloc */ void module_free(struct module *mod, void *module_region) { - FREE_MODULE(module_region); -} - -/* We don't need anything special. */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - -int apply_relocate(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "module %s: REL relocation unsupported\n", me->name); - return -ENOEXEC; + kfree(module_region); } +#endif int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, @@ -108,14 +82,3 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, return 0; } - -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/frv/kernel/module.c b/arch/frv/kernel/module.c index 711763c8a6f3..9d9835f1fe2b 100644 --- a/arch/frv/kernel/module.c +++ b/arch/frv/kernel/module.c @@ -22,57 +22,6 @@ #define DEBUGP(fmt...) #endif -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - - return vmalloc_exec(size); -} - - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -/* We don't need anything special. */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - -int apply_relocate(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "module %s: ADD RELOCATION unsupported\n", me->name); - return -ENOEXEC; -} - -int apply_relocate_add(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "module %s: ADD RELOCATION unsupported\n", me->name); - return -ENOEXEC; -} - -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} - -void module_arch_cleanup(struct module *mod) -{ -} +/* TODO: At least one of apply_relocate or apply_relocate_add must be + * implemented in order to get working module support. + */ diff --git a/arch/h8300/kernel/module.c b/arch/h8300/kernel/module.c index db4953dc4e1b..1d526e05db19 100644 --- a/arch/h8300/kernel/module.c +++ b/arch/h8300/kernel/module.c @@ -11,40 +11,6 @@ #define DEBUGP(fmt...) #endif -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc(size); -} - - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -/* We don't need anything special. */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - -int apply_relocate(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "module %s: RELOCATION unsupported\n", - me->name); - return -ENOEXEC; -} - int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -107,14 +73,3 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, me->name, rela[i].r_offset); return -ENOEXEC; } - -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/ia64/kernel/module.c b/arch/ia64/kernel/module.c index 1481b0a28ca0..24603be24c14 100644 --- a/arch/ia64/kernel/module.c +++ b/arch/ia64/kernel/module.c @@ -304,14 +304,6 @@ plt_target (struct plt_entry *plt) #endif /* !USE_BRL */ -void * -module_alloc (unsigned long size) -{ - if (!size) - return NULL; - return vmalloc(size); -} - void module_free (struct module *mod, void *module_region) { @@ -853,14 +845,6 @@ apply_relocate_add (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symind return 0; } -int -apply_relocate (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex, - unsigned int relsec, struct module *mod) -{ - printk(KERN_ERR "module %s: REL relocs in section %u unsupported\n", mod->name, relsec); - return -ENOEXEC; -} - /* * Modules contain a single unwind table which covers both the core and the init text * sections but since the two are not contiguous, we need to split this table up such that diff --git a/arch/m32r/kernel/module.c b/arch/m32r/kernel/module.c index cb5f37d78d49..3071fe83ffc8 100644 --- a/arch/m32r/kernel/module.c +++ b/arch/m32r/kernel/module.c @@ -28,33 +28,6 @@ #define DEBUGP(fmt...) #endif -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; -#ifdef CONFIG_MMU - return vmalloc_exec(size); -#else - return vmalloc(size); -#endif -} - - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -/* We don't need anything special. */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - #define COPY_UNALIGNED_WORD(sw, tw, align) \ { \ void *__s = &(sw), *__t = &(tw); \ @@ -243,14 +216,3 @@ int apply_relocate(Elf32_Shdr *sechdrs, return 0; } - -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/m68k/kernel/module_mm.c b/arch/m68k/kernel/module_mm.c index cd6bcb1c957e..ceafc47c96d5 100644 --- a/arch/m68k/kernel/module_mm.c +++ b/arch/m68k/kernel/module_mm.c @@ -19,29 +19,6 @@ #ifdef CONFIG_MODULES -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc(size); -} - - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -/* We don't need anything special. */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -131,10 +108,6 @@ int module_finalize(const Elf_Ehdr *hdr, return 0; } -void module_arch_cleanup(struct module *mod) -{ -} - #endif /* CONFIG_MODULES */ void module_fixup(struct module *mod, struct m68k_fixup_info *start, diff --git a/arch/m68k/kernel/module_no.c b/arch/m68k/kernel/module_no.c index d11ffae7956a..5a097c6063fa 100644 --- a/arch/m68k/kernel/module_no.c +++ b/arch/m68k/kernel/module_no.c @@ -11,29 +11,6 @@ #define DEBUGP(fmt...) #endif -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc(size); -} - - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -/* We don't need anything special. */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -113,14 +90,3 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, } return 0; } - -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/microblaze/kernel/module.c b/arch/microblaze/kernel/module.c index 0e73f6606547..142426f631bb 100644 --- a/arch/microblaze/kernel/module.c +++ b/arch/microblaze/kernel/module.c @@ -18,37 +18,6 @@ #include #include -void *module_alloc(unsigned long size) -{ - void *ret; - ret = (size == 0) ? NULL : vmalloc(size); - pr_debug("module_alloc (%08lx@%08lx)\n", size, (unsigned long int)ret); - return ret; -} - -void module_free(struct module *module, void *region) -{ - pr_debug("module_free(%s,%08lx)\n", module->name, - (unsigned long)region); - vfree(region); -} - -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - -int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, - unsigned int symindex, unsigned int relsec, struct module *module) -{ - printk(KERN_ERR "module %s: ADD RELOCATION unsupported\n", - module->name); - return -ENOEXEC; -} - int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relsec, struct module *module) { @@ -155,7 +124,3 @@ int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs, flush_dcache(); return 0; } - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/mips/kernel/module.c b/arch/mips/kernel/module.c index dd940b701963..4b930ac4aff2 100644 --- a/arch/mips/kernel/module.c +++ b/arch/mips/kernel/module.c @@ -45,30 +45,14 @@ static struct mips_hi16 *mips_hi16_list; static LIST_HEAD(dbe_list); static DEFINE_SPINLOCK(dbe_lock); +#ifdef MODULE_START void *module_alloc(unsigned long size) { -#ifdef MODULE_START return __vmalloc_node_range(size, 1, MODULE_START, MODULE_END, GFP_KERNEL, PAGE_KERNEL, -1, __builtin_return_address(0)); -#else - if (size == 0) - return NULL; - return vmalloc(size); -#endif -} - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, - char *secstrings, struct module *mod) -{ - return 0; } +#endif static int apply_r_mips_none(struct module *me, u32 *location, Elf_Addr v) { diff --git a/arch/mn10300/kernel/module.c b/arch/mn10300/kernel/module.c index 196a111e2e29..216ad23c9570 100644 --- a/arch/mn10300/kernel/module.c +++ b/arch/mn10300/kernel/module.c @@ -32,36 +32,6 @@ #define DEBUGP(fmt, ...) #endif -/* - * allocate storage for a module - */ -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc_exec(size); -} - -/* - * free memory returned from module_alloc() - */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -/* - * allow the arch to fix up the section table - * - we don't need anything special - */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - static void reloc_put16(uint8_t *p, uint32_t val) { p[0] = val & 0xff; @@ -80,20 +50,6 @@ static void reloc_put32(uint8_t *p, uint32_t val) reloc_put16(p+2, val >> 16); } -/* - * apply a REL relocation - */ -int apply_relocate(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "module %s: RELOCATION unsupported\n", - me->name); - return -ENOEXEC; -} - /* * apply a RELA relocation */ @@ -198,20 +154,3 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, } return 0; } - -/* - * finish loading the module - */ -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} - -/* - * finish clearing the module - */ -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/parisc/kernel/module.c b/arch/parisc/kernel/module.c index cedbbb8b18d9..5e34ccf39a49 100644 --- a/arch/parisc/kernel/module.c +++ b/arch/parisc/kernel/module.c @@ -540,18 +540,6 @@ static Elf_Addr get_stub(struct module *me, unsigned long value, long addend, return (Elf_Addr)stub; } -int apply_relocate(Elf_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - /* parisc should not need this ... */ - printk(KERN_ERR "module %s: RELOCATION unsupported\n", - me->name); - return -ENOEXEC; -} - #ifndef CONFIG_64BIT int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, diff --git a/arch/powerpc/kernel/module.c b/arch/powerpc/kernel/module.c index 49cee9df225b..a1cd701b5753 100644 --- a/arch/powerpc/kernel/module.c +++ b/arch/powerpc/kernel/module.c @@ -31,20 +31,6 @@ LIST_HEAD(module_bug_list); -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - - return vmalloc_exec(size); -} - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - static const Elf_Shdr *find_section(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, const char *name) @@ -93,7 +79,3 @@ int module_finalize(const Elf_Ehdr *hdr, return 0; } - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/powerpc/kernel/module_32.c b/arch/powerpc/kernel/module_32.c index f832773fc28e..0b6d79617d7b 100644 --- a/arch/powerpc/kernel/module_32.c +++ b/arch/powerpc/kernel/module_32.c @@ -174,17 +174,6 @@ int module_frob_arch_sections(Elf32_Ehdr *hdr, return 0; } -int apply_relocate(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *module) -{ - printk(KERN_ERR "%s: Non-ADD RELOCATION unsupported\n", - module->name); - return -ENOEXEC; -} - static inline int entry_matches(struct ppc_plt_entry *entry, Elf32_Addr val) { if (entry->jump[0] == 0x3d600000 + ((val + 0x8000) >> 16) diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c index 8fbb12508bf3..9f44a775a106 100644 --- a/arch/powerpc/kernel/module_64.c +++ b/arch/powerpc/kernel/module_64.c @@ -243,16 +243,6 @@ int module_frob_arch_sections(Elf64_Ehdr *hdr, return 0; } -int apply_relocate(Elf64_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "%s: Non-ADD RELOCATION unsupported\n", me->name); - return -ENOEXEC; -} - /* r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the value maximum span in an instruction which uses a signed offset) */ diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index f7167ee4604c..dfcb3436bad0 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -45,13 +45,6 @@ #define PLT_ENTRY_SIZE 20 #endif /* CONFIG_64BIT */ -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc(size); -} - /* Free memory returned from module_alloc */ void module_free(struct module *mod, void *module_region) { @@ -176,15 +169,6 @@ module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, return 0; } -int -apply_relocate(Elf_Shdr *sechdrs, const char *strtab, unsigned int symindex, - unsigned int relsec, struct module *me) -{ - printk(KERN_ERR "module %s: RELOCATION unsupported\n", - me->name); - return -ENOEXEC; -} - static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, struct module *me) @@ -409,7 +393,3 @@ int module_finalize(const Elf_Ehdr *hdr, me->arch.syminfo = NULL; return 0; } - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/score/kernel/module.c b/arch/score/kernel/module.c index 4de8d47becd3..469e3b64e2f2 100644 --- a/arch/score/kernel/module.c +++ b/arch/score/kernel/module.c @@ -27,23 +27,6 @@ #include #include -void *module_alloc(unsigned long size) -{ - return size ? vmalloc(size) : NULL; -} - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, - char *secstrings, struct module *mod) -{ - return 0; -} - int apply_relocate(Elf_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relindex, struct module *me) @@ -146,6 +129,9 @@ int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relsec, struct module *me) { + /* Non-standard return value... most other arch's return -ENOEXEC + * for an unsupported relocation variant + */ return 0; } @@ -154,12 +140,3 @@ const struct exception_table_entry *search_module_dbetables(unsigned long addr) { return NULL; } - -/* Put in dbe list if necessary. */ -int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} - -void module_arch_cleanup(struct module *mod) {} diff --git a/arch/sh/kernel/module.c b/arch/sh/kernel/module.c index 19b1f8826aef..1b525dedd29a 100644 --- a/arch/sh/kernel/module.c +++ b/arch/sh/kernel/module.c @@ -34,30 +34,6 @@ #include #include -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - - return vmalloc_exec(size); -} - - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -/* We don't need anything special. */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -133,17 +109,6 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, return 0; } -int apply_relocate(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "module %s: REL RELOCATION unsupported\n", - me->name); - return -ENOEXEC; -} - int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *me) diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c index 99ba5baa9497..da0c6c70ccb2 100644 --- a/arch/sparc/kernel/module.c +++ b/arch/sparc/kernel/module.c @@ -68,12 +68,6 @@ void *module_alloc(unsigned long size) return ret; } -/* Free memory returned from module_core_alloc/module_init_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - /* Make generic code ignore STT_REGISTER dummy undefined symbols. */ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, @@ -107,17 +101,6 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, return 0; } -int apply_relocate(Elf_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "module %s: non-ADD RELOCATION unsupported\n", - me->name); - return -ENOEXEC; -} - int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -239,15 +222,4 @@ int module_finalize(const Elf_Ehdr *hdr, return 0; } -#else -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} #endif /* CONFIG_SPARC64 */ - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/tile/kernel/module.c b/arch/tile/kernel/module.c index f68df69f1f67..28fa6ece9d3a 100644 --- a/arch/tile/kernel/module.c +++ b/arch/tile/kernel/module.c @@ -98,25 +98,6 @@ void module_free(struct module *mod, void *module_region) */ } -/* We don't need anything special. */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - -int apply_relocate(Elf_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - pr_err("module %s: .rel relocation unsupported\n", me->name); - return -ENOEXEC; -} - #ifdef __tilegx__ /* * Validate that the high 16 bits of "value" is just the sign-extension of @@ -249,15 +230,3 @@ int apply_relocate_add(Elf_Shdr *sechdrs, } return 0; } - -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *me) -{ - /* FIXME: perhaps remove the "writable" bit from the TLB? */ - return 0; -} - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/unicore32/kernel/module.c b/arch/unicore32/kernel/module.c index 3e5a38d71a1e..8fbe8577f5e6 100644 --- a/arch/unicore32/kernel/module.c +++ b/arch/unicore32/kernel/module.c @@ -37,19 +37,6 @@ void *module_alloc(unsigned long size) return __vmalloc_area(area, GFP_KERNEL, PAGE_KERNEL_EXEC); } -void module_free(struct module *module, void *region) -{ - vfree(region); -} - -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relindex, struct module *module) @@ -128,25 +115,3 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, } return 0; } - -int -apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, - unsigned int symindex, unsigned int relsec, - struct module *module) -{ - printk(KERN_ERR "module %s: ADD RELOCATION unsupported\n", - module->name); - return -ENOEXEC; -} - -int -module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs, - struct module *module) -{ - return 0; -} - -void -module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index 52f256f2cc81..925179f871de 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -45,21 +45,6 @@ void *module_alloc(unsigned long size) -1, __builtin_return_address(0)); } -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -/* We don't need anything special. */ -int module_frob_arch_sections(Elf_Ehdr *hdr, - Elf_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - #ifdef CONFIG_X86_32 int apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, @@ -100,17 +85,6 @@ int apply_relocate(Elf32_Shdr *sechdrs, } return 0; } - -int apply_relocate_add(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "module %s: ADD RELOCATION unsupported\n", - me->name); - return -ENOEXEC; -} #else /*X86_64*/ int apply_relocate_add(Elf64_Shdr *sechdrs, const char *strtab, @@ -181,17 +155,6 @@ overflow: me->name); return -ENOEXEC; } - -int apply_relocate(Elf_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "non add relocation not supported\n"); - return -ENOSYS; -} - #endif int module_finalize(const Elf_Ehdr *hdr, diff --git a/arch/xtensa/kernel/module.c b/arch/xtensa/kernel/module.c index c1accea8cb56..451dda928c93 100644 --- a/arch/xtensa/kernel/module.c +++ b/arch/xtensa/kernel/module.c @@ -24,26 +24,6 @@ #undef DEBUG_RELOCATE -void *module_alloc(unsigned long size) -{ - if (size == 0) - return NULL; - return vmalloc_exec(size); -} - -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -int module_frob_arch_sections(Elf32_Ehdr *hdr, - Elf32_Shdr *sechdrs, - char *secstrings, - struct module *mod) -{ - return 0; -} - static int decode_calln_opcode (unsigned char *location) { @@ -66,18 +46,6 @@ decode_l32r_opcode (unsigned char *location) #endif } -int apply_relocate(Elf32_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *mod) -{ - printk(KERN_ERR "module %s: REL RELOCATION unsupported\n", - mod->name); - return -ENOEXEC; - -} - int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -222,14 +190,3 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, } return 0; } - -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *mod) -{ - return 0; -} - -void module_arch_cleanup(struct module *mod) -{ -} -- cgit v1.2.3 From 60063497a95e716c9a689af3be2687d261f115b4 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Tue, 26 Jul 2011 16:09:06 -0700 Subject: atomic: use This allows us to move duplicated code in (atomic_inc_not_zero() for now) to Signed-off-by: Arun Sharma Reviewed-by: Eric Dumazet Cc: Ingo Molnar Cc: David Miller Cc: Eric Dumazet Acked-by: Mike Frysinger Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/alpha/include/asm/atomic.h | 1 - arch/alpha/include/asm/local.h | 2 +- arch/alpha/kernel/perf_event.c | 2 +- arch/alpha/kernel/smp.c | 2 +- arch/alpha/lib/dec_and_lock.c | 2 +- arch/arm/include/asm/atomic.h | 1 - arch/arm/kernel/smp.c | 2 +- arch/arm/kernel/traps.c | 2 +- arch/arm/mach-at91/pm.c | 2 +- arch/arm/mach-bcmring/dma.c | 2 +- arch/arm/mach-cns3xxx/include/mach/pm.h | 2 +- arch/arm/mach-cns3xxx/pm.c | 2 +- arch/arm/mach-omap1/pm.c | 2 +- arch/arm/mach-s3c2440/clock.c | 2 +- arch/arm/mach-s3c2440/s3c2442.c | 2 +- arch/arm/mach-s3c2440/s3c244x-clock.c | 2 +- arch/avr32/include/asm/atomic.h | 1 - arch/blackfin/include/asm/atomic.h | 1 - arch/blackfin/include/asm/dma.h | 2 +- arch/blackfin/include/asm/ipipe.h | 2 +- arch/blackfin/include/asm/spinlock.h | 2 +- arch/blackfin/kernel/ftrace.c | 2 +- arch/blackfin/kernel/ipipe.c | 2 +- arch/blackfin/kernel/nmi.c | 2 +- arch/blackfin/mach-common/smp.c | 2 +- arch/cris/arch-v32/drivers/cryptocop.c | 2 +- arch/cris/arch-v32/kernel/smp.c | 2 +- arch/cris/include/asm/atomic.h | 1 - arch/cris/include/asm/bitops.h | 2 +- arch/cris/kernel/process.c | 2 +- arch/frv/include/asm/atomic.h | 1 - arch/frv/include/asm/hardirq.h | 2 +- arch/frv/kernel/irq.c | 2 +- arch/h8300/include/asm/atomic.h | 1 - arch/ia64/include/asm/atomic.h | 1 - arch/ia64/include/asm/processor.h | 2 +- arch/ia64/include/asm/spinlock.h | 2 +- arch/ia64/kernel/smp.c | 2 +- arch/ia64/kernel/smpboot.c | 2 +- arch/ia64/kernel/uncached.c | 2 +- arch/m32r/include/asm/atomic.h | 1 - arch/m32r/include/asm/mmu_context.h | 2 +- arch/m32r/include/asm/spinlock.h | 2 +- arch/m32r/kernel/smp.c | 2 +- arch/m32r/kernel/traps.c | 2 +- arch/m68k/include/asm/atomic.h | 1 - arch/microblaze/include/asm/mmu_context_mm.h | 2 +- arch/microblaze/include/asm/prom.h | 2 +- arch/mips/include/asm/atomic.h | 1 - arch/mips/include/asm/hw_irq.h | 2 +- arch/mips/include/asm/local.h | 2 +- arch/mips/include/asm/smp.h | 2 +- arch/mips/kernel/irq.c | 2 +- arch/mips/kernel/mips-mt.c | 2 +- arch/mips/kernel/rtlx.c | 2 +- arch/mips/kernel/smp-cmp.c | 2 +- arch/mips/kernel/smp-mt.c | 2 +- arch/mips/kernel/smp.c | 2 +- arch/mips/kernel/smtc-proc.c | 2 +- arch/mips/kernel/smtc.c | 2 +- arch/mips/kernel/sync-r4k.c | 2 +- arch/mips/kernel/vpe.c | 2 +- arch/mips/mipssim/sim_smtc.c | 2 +- arch/mips/sgi-ip27/ip27-nmi.c | 2 +- arch/mn10300/include/asm/atomic.h | 1 - arch/mn10300/include/asm/mmu_context.h | 2 +- arch/mn10300/include/asm/spinlock.h | 2 +- arch/mn10300/include/asm/system.h | 2 +- arch/mn10300/kernel/mn10300-watchdog.c | 2 +- arch/mn10300/kernel/traps.c | 2 +- arch/mn10300/mm/misalignment.c | 2 +- arch/mn10300/proc-mn2ws0050/proc-init.c | 2 +- arch/parisc/include/asm/atomic.h | 1 - arch/parisc/include/asm/bitops.h | 2 +- arch/parisc/include/asm/mmu_context.h | 2 +- arch/parisc/kernel/parisc_ksyms.c | 2 +- arch/parisc/kernel/smp.c | 2 +- arch/parisc/kernel/traps.c | 2 +- arch/parisc/lib/bitops.c | 2 +- arch/powerpc/include/asm/atomic.h | 1 - arch/powerpc/include/asm/emulated_ops.h | 2 +- arch/powerpc/include/asm/irq.h | 2 +- arch/powerpc/include/asm/local.h | 2 +- arch/powerpc/include/asm/prom.h | 2 +- arch/powerpc/kernel/of_platform.c | 2 +- arch/powerpc/kernel/ppc_ksyms.c | 2 +- arch/powerpc/kernel/rtas.c | 2 +- arch/powerpc/kernel/rtasd.c | 2 +- arch/powerpc/kernel/smp-tbsync.c | 2 +- arch/powerpc/kernel/smp.c | 2 +- arch/powerpc/platforms/83xx/km83xx.c | 2 +- arch/powerpc/platforms/83xx/mpc832x_mds.c | 2 +- arch/powerpc/platforms/83xx/mpc834x_itx.c | 2 +- arch/powerpc/platforms/83xx/mpc834x_mds.c | 2 +- arch/powerpc/platforms/83xx/mpc836x_mds.c | 2 +- arch/powerpc/platforms/83xx/sbc834x.c | 2 +- arch/powerpc/platforms/85xx/mpc85xx_cds.c | 2 +- arch/powerpc/platforms/85xx/mpc85xx_mds.c | 2 +- arch/powerpc/platforms/85xx/sbc8548.c | 2 +- arch/powerpc/platforms/cell/cpufreq_spudemand.c | 2 +- arch/powerpc/platforms/cell/smp.c | 2 +- arch/powerpc/platforms/cell/spufs/context.c | 2 +- arch/powerpc/platforms/chrp/smp.c | 2 +- arch/powerpc/platforms/iseries/smp.c | 2 +- arch/powerpc/platforms/powermac/backlight.c | 2 +- arch/powerpc/platforms/powermac/smp.c | 2 +- arch/powerpc/platforms/pseries/eeh.c | 2 +- arch/powerpc/platforms/pseries/eeh_cache.c | 2 +- arch/powerpc/platforms/pseries/smp.c | 2 +- arch/powerpc/sysdev/fsl_soc.c | 2 +- arch/powerpc/sysdev/tsi108_dev.c | 2 +- arch/s390/include/asm/atomic.h | 1 - arch/s390/kernel/dis.c | 2 +- arch/s390/kernel/traps.c | 2 +- arch/sh/include/asm/atomic.h | 1 - arch/sh/include/asm/hw_irq.h | 2 +- arch/sh/include/asm/smp.h | 2 +- arch/sh/kernel/idle.c | 2 +- arch/sh/kernel/smp.c | 2 +- arch/sh/kernel/traps_64.c | 2 +- arch/sh/kernel/unwinder.c | 2 +- arch/sparc/include/asm/atomic_32.h | 1 - arch/sparc/include/asm/atomic_64.h | 1 - arch/sparc/include/asm/prom.h | 2 +- arch/sparc/include/asm/smp_32.h | 2 +- arch/sparc/include/asm/smp_64.h | 2 +- arch/sparc/kernel/irq_64.c | 2 +- arch/sparc/kernel/leon_smp.c | 2 +- arch/sparc/kernel/perf_event.c | 2 +- arch/sparc/kernel/smp_32.c | 2 +- arch/sparc/kernel/smp_64.c | 2 +- arch/sparc/lib/atomic32.c | 2 +- arch/tile/include/asm/atomic.h | 9 --------- arch/tile/include/asm/atomic_32.h | 4 ++-- arch/tile/include/asm/atomic_64.h | 2 +- arch/tile/include/asm/bitops_32.h | 2 +- arch/tile/include/asm/bitops_64.h | 2 +- arch/tile/include/asm/spinlock_32.h | 2 +- arch/tile/kernel/intvec_32.S | 2 +- arch/tile/lib/atomic_32.c | 2 +- arch/tile/lib/atomic_asm_32.S | 2 +- arch/x86/ia32/sys_ia32.c | 2 +- arch/x86/include/asm/apic.h | 2 +- arch/x86/include/asm/atomic.h | 1 - arch/x86/include/asm/hw_irq.h | 2 +- arch/x86/include/asm/local.h | 2 +- arch/x86/include/asm/mce.h | 2 +- arch/x86/include/asm/mmu_context.h | 2 +- arch/x86/include/asm/prom.h | 2 +- arch/x86/include/asm/spinlock.h | 2 +- arch/x86/include/asm/thread_info.h | 2 +- arch/x86/kernel/amd_gart_64.c | 2 +- arch/x86/kernel/apic/apic.c | 2 +- arch/x86/kernel/apic/es7000_32.c | 2 +- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/i8259.c | 2 +- arch/x86/kernel/irqinit.c | 2 +- arch/x86/kernel/traps.c | 2 +- arch/x86/kvm/lapic.c | 2 +- arch/x86/kvm/timer.c | 2 +- arch/x86/lib/atomic64_32.c | 2 +- arch/x86/mm/mmio-mod.c | 2 +- arch/xtensa/include/asm/atomic.h | 1 - arch/xtensa/kernel/process.c | 2 +- crypto/af_alg.c | 2 +- crypto/proc.c | 2 +- crypto/rng.c | 2 +- drivers/atm/ambassador.c | 2 +- drivers/atm/atmtcp.c | 2 +- drivers/atm/eni.c | 2 +- drivers/atm/eni.h | 2 +- drivers/atm/firestream.c | 2 +- drivers/atm/fore200e.c | 2 +- drivers/atm/horizon.c | 2 +- drivers/atm/idt77252.c | 2 +- drivers/atm/iphase.c | 2 +- drivers/atm/nicstar.c | 2 +- drivers/atm/suni.c | 2 +- drivers/atm/uPD98402.c | 2 +- drivers/atm/zatm.c | 2 +- drivers/base/memory.c | 2 +- drivers/base/power/sysfs.c | 2 +- drivers/block/cciss_scsi.c | 2 +- drivers/char/ipmi/ipmi_watchdog.c | 2 +- drivers/char/mspec.c | 2 +- drivers/connector/cn_proc.c | 3 ++- drivers/edac/edac_stub.c | 2 +- drivers/firewire/core-card.c | 2 +- drivers/firewire/core-device.c | 2 +- drivers/firewire/core-topology.c | 2 +- drivers/firewire/core.h | 2 +- drivers/firewire/nosy.c | 2 +- drivers/gpu/drm/radeon/radeon.h | 2 +- drivers/gpu/drm/radeon/radeon_fence.c | 2 +- drivers/gpu/drm/ttm/ttm_bo.c | 2 +- drivers/gpu/drm/ttm/ttm_lock.c | 2 +- drivers/gpu/drm/ttm/ttm_object.c | 2 +- drivers/gpu/drm/ttm/ttm_page_alloc.c | 2 +- drivers/hwmon/sht15.c | 2 +- drivers/infiniband/hw/cxgb4/mem.c | 2 +- drivers/infiniband/hw/ehca/ehca_tools.h | 2 +- drivers/infiniband/hw/nes/nes_cm.c | 2 +- drivers/infiniband/ulp/ipoib/ipoib.h | 2 +- drivers/infiniband/ulp/srp/ib_srp.c | 2 +- drivers/isdn/gigaset/gigaset.h | 2 +- drivers/md/dm-crypt.c | 2 +- drivers/md/dm-kcopyd.c | 2 +- drivers/md/dm-mpath.c | 2 +- drivers/md/dm-queue-length.c | 2 +- drivers/md/dm-table.c | 2 +- drivers/media/video/hdpvr/hdpvr-core.c | 2 +- drivers/media/video/tlg2300/pd-dvb.c | 2 +- drivers/media/video/uvc/uvc_ctrl.c | 2 +- drivers/media/video/uvc/uvc_queue.c | 2 +- drivers/media/video/uvc/uvc_v4l2.c | 2 +- drivers/media/video/uvc/uvc_video.c | 2 +- drivers/message/i2o/i2o_scsi.c | 2 +- drivers/misc/phantom.c | 2 +- drivers/net/atlx/atl1.c | 2 +- drivers/net/atlx/atl2.c | 2 +- drivers/net/atlx/atl2.h | 2 +- drivers/net/cassini.c | 2 +- drivers/net/cpmac.c | 2 +- drivers/net/cxgb3/cxgb3_offload.c | 2 +- drivers/net/cxgb3/l2t.h | 2 +- drivers/net/cxgb3/t3cdev.h | 2 +- drivers/net/cxgb4/cxgb4_uld.h | 2 +- drivers/net/cxgb4/l2t.h | 2 +- drivers/net/hamradio/6pack.c | 2 +- drivers/net/hamradio/dmascc.c | 2 +- drivers/net/ibmveth.c | 2 +- drivers/net/phy/phy.c | 2 +- drivers/net/ppp_generic.c | 2 +- drivers/net/wimax/i2400m/i2400m.h | 2 +- drivers/net/wireless/b43legacy/b43legacy.h | 2 +- drivers/net/wireless/b43legacy/dma.h | 2 +- drivers/oprofile/oprofile_stats.h | 2 +- drivers/pci/hotplug/cpci_hotplug_core.c | 2 +- drivers/pci/xen-pcifront.c | 2 +- drivers/s390/block/dasd_eer.c | 2 +- drivers/s390/char/sclp_quiesce.c | 2 +- drivers/s390/char/vmlogrdr.c | 2 +- drivers/s390/cio/device.h | 2 +- drivers/s390/cio/qdio_main.c | 2 +- drivers/s390/cio/qdio_thinint.c | 2 +- drivers/s390/crypto/ap_bus.c | 2 +- drivers/s390/crypto/zcrypt_api.c | 2 +- drivers/s390/crypto/zcrypt_cex2a.c | 2 +- drivers/s390/crypto/zcrypt_mono.c | 2 +- drivers/s390/crypto/zcrypt_pcica.c | 2 +- drivers/s390/crypto/zcrypt_pcicc.c | 2 +- drivers/s390/crypto/zcrypt_pcixcc.c | 2 +- drivers/s390/net/fsm.h | 2 +- drivers/s390/scsi/zfcp_scsi.c | 2 +- drivers/sbus/char/display7seg.c | 2 +- drivers/scsi/dpt/dpti_i2o.h | 2 +- drivers/scsi/hpsa.c | 2 +- drivers/scsi/pm8001/pm8001_sas.h | 2 +- drivers/staging/octeon/ethernet-rx.c | 2 +- drivers/staging/octeon/ethernet-tx.c | 2 +- drivers/staging/solo6x10/solo6x10.h | 2 +- drivers/staging/tidspbridge/include/dspbridge/host_os.h | 2 +- drivers/staging/winbond/mds_s.h | 2 +- drivers/staging/winbond/wb35reg_s.h | 2 +- drivers/tty/bfin_jtag_comm.c | 2 +- drivers/tty/rocket.c | 2 +- drivers/tty/serial/dz.c | 2 +- drivers/tty/serial/sb1250-duart.c | 2 +- drivers/tty/serial/zs.c | 2 +- drivers/usb/gadget/f_audio.c | 2 +- drivers/usb/gadget/f_rndis.c | 2 +- drivers/usb/gadget/uvc_queue.c | 2 +- drivers/usb/image/microtek.c | 2 +- drivers/usb/misc/appledisplay.c | 2 +- drivers/usb/serial/garmin_gps.c | 2 +- drivers/usb/wusbcore/wa-rpipe.c | 2 +- drivers/vhost/vhost.h | 2 +- drivers/video/sh_mobile_lcdcfb.c | 2 +- drivers/video/vermilion/vermilion.h | 2 +- drivers/w1/masters/matrox_w1.c | 2 +- drivers/w1/w1.c | 2 +- drivers/w1/w1_family.h | 2 +- drivers/watchdog/intel_scu_watchdog.c | 2 +- drivers/watchdog/sbc7240_wdt.c | 2 +- fs/btrfs/delayed-inode.h | 2 +- fs/direct-io.c | 2 +- fs/eventpoll.c | 2 +- fs/file_table.c | 2 +- fs/gfs2/main.c | 2 +- fs/nfs/cache_lib.h | 2 +- fs/nfs/direct.c | 2 +- fs/notify/group.c | 2 +- fs/notify/inode_mark.c | 2 +- fs/notify/mark.c | 2 +- fs/notify/notification.c | 2 +- fs/notify/vfsmount_mark.c | 2 +- fs/ntfs/inode.h | 2 +- fs/posix_acl.c | 2 +- fs/proc/meminfo.c | 2 +- include/acpi/platform/aclinux.h | 2 +- include/asm-generic/atomic.h | 2 -- include/asm-generic/local.h | 2 +- include/asm-generic/local64.h | 2 +- include/drm/ttm/ttm_lock.h | 2 +- include/linux/aio.h | 2 +- include/linux/atmdev.h | 2 +- include/linux/atomic.h | 9 +++++++++ include/linux/backing-dev.h | 2 +- include/linux/bit_spinlock.h | 2 +- include/linux/buffer_head.h | 2 +- include/linux/configfs.h | 2 +- include/linux/connector.h | 2 +- include/linux/cred.h | 2 +- include/linux/crypto.h | 2 +- include/linux/dcache.h | 2 +- include/linux/debug_locks.h | 2 +- include/linux/device.h | 2 +- include/linux/edac.h | 2 +- include/linux/fault-inject.h | 2 +- include/linux/fdtable.h | 2 +- include/linux/filter.h | 2 +- include/linux/firewire.h | 2 +- include/linux/fsnotify_backend.h | 2 +- include/linux/interrupt.h | 2 +- include/linux/jump_label.h | 2 +- include/linux/kdb.h | 2 +- include/linux/key.h | 2 +- include/linux/kgdb.h | 2 +- include/linux/kobject.h | 2 +- include/linux/mlx4/device.h | 2 +- include/linux/mman.h | 2 +- include/linux/mmzone.h | 2 +- include/linux/mount.h | 2 +- include/linux/mutex.h | 2 +- include/linux/netdevice.h | 2 +- include/linux/nfs_fs_sb.h | 2 +- include/linux/oprofile.h | 2 +- include/linux/pci.h | 2 +- include/linux/perf_event.h | 2 +- include/linux/phy.h | 2 +- include/linux/proc_fs.h | 2 +- include/linux/quota.h | 2 +- include/linux/rwsem.h | 2 +- include/linux/sem.h | 2 +- include/linux/skbuff.h | 2 +- include/linux/sonet.h | 2 +- include/linux/spinlock.h | 2 +- include/linux/sunrpc/auth.h | 2 +- include/linux/sunrpc/cache.h | 2 +- include/linux/sunrpc/timer.h | 2 +- include/linux/swap.h | 2 +- include/linux/sysfs.h | 2 +- include/linux/vmstat.h | 2 +- include/linux/workqueue.h | 2 +- include/net/ax25.h | 2 +- include/net/cipso_ipv4.h | 2 +- include/net/flow.h | 2 +- include/net/inet_hashtables.h | 2 +- include/net/inet_timewait_sock.h | 2 +- include/net/inetpeer.h | 2 +- include/net/ip_vs.h | 2 +- include/net/lib80211.h | 2 +- include/net/llc.h | 2 +- include/net/neighbour.h | 2 +- include/net/net_namespace.h | 2 +- include/net/netfilter/nf_conntrack.h | 2 +- include/net/netlabel.h | 2 +- include/net/netns/conntrack.h | 2 +- include/net/sctp/structs.h | 2 +- include/pcmcia/ds.h | 2 +- include/rdma/ib_sa.h | 2 +- include/rdma/ib_verbs.h | 2 +- include/rxrpc/types.h | 2 +- include/scsi/scsi_device.h | 2 +- kernel/audit.c | 2 +- kernel/auditsc.c | 2 +- kernel/cgroup.c | 2 +- kernel/cpuset.c | 2 +- kernel/debug/debug_core.c | 2 +- kernel/rcupdate.c | 2 +- kernel/rcutorture.c | 2 +- kernel/rcutree_trace.c | 2 +- kernel/rwsem.c | 2 +- kernel/stop_machine.c | 2 +- kernel/taskstats.c | 2 +- kernel/trace/trace.h | 2 +- kernel/trace/trace_mmiotrace.c | 2 +- lib/atomic64.c | 2 +- lib/atomic64_test.c | 2 +- lib/crc32.c | 2 +- lib/dec_and_lock.c | 2 +- mm/init-mm.c | 2 +- mm/kmemleak.c | 2 +- mm/slob.c | 2 +- mm/vmalloc.c | 2 +- net/atm/atm_misc.c | 2 +- net/atm/clip.c | 2 +- net/atm/common.c | 2 +- net/atm/lec.c | 2 +- net/atm/proc.c | 2 +- net/bridge/br_fdb.c | 2 +- net/core/flow.c | 2 +- net/decnet/dn_fib.c | 2 +- net/decnet/dn_neigh.c | 2 +- net/decnet/dn_table.c | 2 +- net/decnet/dn_timer.c | 2 +- net/ipv4/cipso_ipv4.c | 2 +- net/ipv4/raw.c | 2 +- net/ipv6/ip6_tunnel.c | 2 +- net/iucv/iucv.c | 2 +- net/l2tp/l2tp_core.c | 2 +- net/l2tp/l2tp_ppp.c | 2 +- net/netfilter/nfnetlink_log.c | 2 +- net/netfilter/nfnetlink_queue.c | 2 +- net/netlabel/netlabel_cipso_v4.c | 2 +- net/netlabel/netlabel_kapi.c | 2 +- net/netlabel/netlabel_mgmt.c | 2 +- net/netlabel/netlabel_mgmt.h | 2 +- net/netlabel/netlabel_unlabeled.c | 2 +- net/sunrpc/xprtrdma/xprt_rdma.h | 2 +- net/tipc/core.h | 2 +- security/selinux/hooks.c | 2 +- security/selinux/xfrm.c | 2 +- sound/pci/echoaudio/darla20.c | 2 +- sound/pci/echoaudio/darla24.c | 2 +- sound/pci/echoaudio/echo3g.c | 2 +- sound/pci/echoaudio/gina20.c | 2 +- sound/pci/echoaudio/gina24.c | 2 +- sound/pci/echoaudio/indigo.c | 2 +- sound/pci/echoaudio/indigodj.c | 2 +- sound/pci/echoaudio/indigodjx.c | 2 +- sound/pci/echoaudio/indigoio.c | 2 +- sound/pci/echoaudio/indigoiox.c | 2 +- sound/pci/echoaudio/layla20.c | 2 +- sound/pci/echoaudio/layla24.c | 2 +- sound/pci/echoaudio/mia.c | 2 +- sound/pci/echoaudio/mona.c | 2 +- sound/pci/lx6464es/lx6464es.h | 2 +- sound/sparc/dbri.c | 2 +- 439 files changed, 427 insertions(+), 448 deletions(-) (limited to 'arch/x86/kernel') diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h index e756d04b6cd5..88b7491490bc 100644 --- a/arch/alpha/include/asm/atomic.h +++ b/arch/alpha/include/asm/atomic.h @@ -199,7 +199,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /** * atomic64_add_unless - add unless the number is a given value diff --git a/arch/alpha/include/asm/local.h b/arch/alpha/include/asm/local.h index b9e3e3318371..9c94b8456043 100644 --- a/arch/alpha/include/asm/local.h +++ b/arch/alpha/include/asm/local.h @@ -2,7 +2,7 @@ #define _ALPHA_LOCAL_H #include -#include +#include typedef struct { diff --git a/arch/alpha/kernel/perf_event.c b/arch/alpha/kernel/perf_event.c index 8e47709160f8..8143cd7cdbfb 100644 --- a/arch/alpha/kernel/perf_event.c +++ b/arch/alpha/kernel/perf_event.c @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c index d739703608fc..4087a569b43b 100644 --- a/arch/alpha/kernel/smp.c +++ b/arch/alpha/kernel/smp.c @@ -31,7 +31,7 @@ #include #include -#include +#include #include #include diff --git a/arch/alpha/lib/dec_and_lock.c b/arch/alpha/lib/dec_and_lock.c index 0f5520d2f45f..f9f5fe830e9f 100644 --- a/arch/alpha/lib/dec_and_lock.c +++ b/arch/alpha/lib/dec_and_lock.c @@ -6,7 +6,7 @@ */ #include -#include +#include asm (".text \n\ .global _atomic_dec_and_lock \n\ diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 7e79503ab89b..4d501f1bdc9d 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -217,7 +217,6 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) c = old; return c != u; } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_inc(v) atomic_add(1, v) #define atomic_dec(v) atomic_sub(1, v) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 167e3cbe1f2f..d88ff0230e82 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -27,7 +27,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 2d3436e9f71f..bc9f9da782cb 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/mach-at91/pm.c b/arch/arm/mach-at91/pm.c index ea53f4d9b283..4159eca78945 100644 --- a/arch/arm/mach-at91/pm.c +++ b/arch/arm/mach-at91/pm.c @@ -20,7 +20,7 @@ #include #include -#include +#include #include #include diff --git a/arch/arm/mach-bcmring/dma.c b/arch/arm/mach-bcmring/dma.c index 9f2a948e0e72..0ca00050666a 100644 --- a/arch/arm/mach-bcmring/dma.c +++ b/arch/arm/mach-bcmring/dma.c @@ -34,7 +34,7 @@ #include #include -#include +#include #include /* I don't quite understand why dc4 fails when this is set to 1 and DMA is enabled */ diff --git a/arch/arm/mach-cns3xxx/include/mach/pm.h b/arch/arm/mach-cns3xxx/include/mach/pm.h index 6eae7f764d1d..c2588cc991d1 100644 --- a/arch/arm/mach-cns3xxx/include/mach/pm.h +++ b/arch/arm/mach-cns3xxx/include/mach/pm.h @@ -11,7 +11,7 @@ #ifndef __CNS3XXX_PM_H #define __CNS3XXX_PM_H -#include +#include void cns3xxx_pwr_clk_en(unsigned int block); void cns3xxx_pwr_clk_dis(unsigned int block); diff --git a/arch/arm/mach-cns3xxx/pm.c b/arch/arm/mach-cns3xxx/pm.c index 5e579552aa54..0c04678615ce 100644 --- a/arch/arm/mach-cns3xxx/pm.c +++ b/arch/arm/mach-cns3xxx/pm.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/mach-omap1/pm.c b/arch/arm/mach-omap1/pm.c index 98ba9784aa15..495b3987d461 100644 --- a/arch/arm/mach-omap1/pm.c +++ b/arch/arm/mach-omap1/pm.c @@ -44,7 +44,7 @@ #include #include -#include +#include #include #include diff --git a/arch/arm/mach-s3c2440/clock.c b/arch/arm/mach-s3c2440/clock.c index 554e0d3ec70b..f9e6bdaf41d2 100644 --- a/arch/arm/mach-s3c2440/clock.c +++ b/arch/arm/mach-s3c2440/clock.c @@ -36,7 +36,7 @@ #include #include -#include +#include #include #include diff --git a/arch/arm/mach-s3c2440/s3c2442.c b/arch/arm/mach-s3c2440/s3c2442.c index 6224bad4d604..9ad99f8016a1 100644 --- a/arch/arm/mach-s3c2440/s3c2442.c +++ b/arch/arm/mach-s3c2440/s3c2442.c @@ -38,7 +38,7 @@ #include #include -#include +#include #include #include diff --git a/arch/arm/mach-s3c2440/s3c244x-clock.c b/arch/arm/mach-s3c2440/s3c244x-clock.c index f8d96130d1d1..7f5ea0a169a5 100644 --- a/arch/arm/mach-s3c2440/s3c244x-clock.c +++ b/arch/arm/mach-s3c2440/s3c244x-clock.c @@ -35,7 +35,7 @@ #include #include -#include +#include #include #include diff --git a/arch/avr32/include/asm/atomic.h b/arch/avr32/include/asm/atomic.h index bbce6a1c6bb6..f229c3849f03 100644 --- a/arch/avr32/include/asm/atomic.h +++ b/arch/avr32/include/asm/atomic.h @@ -188,7 +188,6 @@ static inline int atomic_sub_if_positive(int i, atomic_t *v) #define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0) #define atomic_add_negative(i, v) (atomic_add_return(i, v) < 0) -#define atomic_inc_not_zero(v) atomic_add_unless(v, 1, 0) #define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v) #define smp_mb__before_atomic_dec() barrier() diff --git a/arch/blackfin/include/asm/atomic.h b/arch/blackfin/include/asm/atomic.h index 4c707dbe1ff9..f2cf5b714ea4 100644 --- a/arch/blackfin/include/asm/atomic.h +++ b/arch/blackfin/include/asm/atomic.h @@ -97,7 +97,6 @@ static inline void atomic_set_mask(int mask, atomic_t *v) c = old; \ c != (u); \ }) -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /* * atomic_inc_and_test - increment and test diff --git a/arch/blackfin/include/asm/dma.h b/arch/blackfin/include/asm/dma.h index d9dbc1a53534..dac0c97242bb 100644 --- a/arch/blackfin/include/asm/dma.h +++ b/arch/blackfin/include/asm/dma.h @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/blackfin/include/asm/ipipe.h b/arch/blackfin/include/asm/ipipe.h index 9e0cc0e2534f..17b5e92e3bc6 100644 --- a/arch/blackfin/include/asm/ipipe.h +++ b/arch/blackfin/include/asm/ipipe.h @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/blackfin/include/asm/spinlock.h b/arch/blackfin/include/asm/spinlock.h index 2336093fca23..490c7caa02d9 100644 --- a/arch/blackfin/include/asm/spinlock.h +++ b/arch/blackfin/include/asm/spinlock.h @@ -11,7 +11,7 @@ # include #else -#include +#include asmlinkage int __raw_spin_is_locked_asm(volatile int *ptr); asmlinkage void __raw_spin_lock_asm(volatile int *ptr); diff --git a/arch/blackfin/kernel/ftrace.c b/arch/blackfin/kernel/ftrace.c index 48808a12b427..9277905b82cf 100644 --- a/arch/blackfin/kernel/ftrace.c +++ b/arch/blackfin/kernel/ftrace.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #ifdef CONFIG_DYNAMIC_FTRACE diff --git a/arch/blackfin/kernel/ipipe.c b/arch/blackfin/kernel/ipipe.c index 486426f8a0d7..dbe11220cc53 100644 --- a/arch/blackfin/kernel/ipipe.c +++ b/arch/blackfin/kernel/ipipe.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs); diff --git a/arch/blackfin/kernel/nmi.c b/arch/blackfin/kernel/nmi.c index 679d0db35256..9919d29287dc 100644 --- a/arch/blackfin/kernel/nmi.c +++ b/arch/blackfin/kernel/nmi.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c index 1c143a4de5f5..107622aacf6b 100644 --- a/arch/blackfin/mach-common/smp.c +++ b/arch/blackfin/mach-common/smp.c @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/cris/arch-v32/drivers/cryptocop.c b/arch/cris/arch-v32/drivers/cryptocop.c index c03bc3bc30c2..642c6fed43d7 100644 --- a/arch/cris/arch-v32/drivers/cryptocop.c +++ b/arch/cris/arch-v32/drivers/cryptocop.c @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include diff --git a/arch/cris/arch-v32/kernel/smp.c b/arch/cris/arch-v32/kernel/smp.c index a0843a71aaee..0b99df72d2a4 100644 --- a/arch/cris/arch-v32/kernel/smp.c +++ b/arch/cris/arch-v32/kernel/smp.c @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/cris/include/asm/atomic.h b/arch/cris/include/asm/atomic.h index 88dc9b9c4ba0..ce9f67e4d977 100644 --- a/arch/cris/include/asm/atomic.h +++ b/arch/cris/include/asm/atomic.h @@ -150,7 +150,6 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) cris_atomic_restore(v, flags); return ret != u; } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /* Atomic operations are already serializing */ #define smp_mb__before_atomic_dec() barrier() diff --git a/arch/cris/include/asm/bitops.h b/arch/cris/include/asm/bitops.h index c0092fc7d846..a78a2d70cd8b 100644 --- a/arch/cris/include/asm/bitops.h +++ b/arch/cris/include/asm/bitops.h @@ -20,7 +20,7 @@ #include #include -#include +#include #include /* diff --git a/arch/cris/kernel/process.c b/arch/cris/kernel/process.c index c99aeab7cef7..aa585e4e979e 100644 --- a/arch/cris/kernel/process.c +++ b/arch/cris/kernel/process.c @@ -12,7 +12,7 @@ * This file handles the architecture-dependent parts of process handling.. */ -#include +#include #include #include #include diff --git a/arch/frv/include/asm/atomic.h b/arch/frv/include/asm/atomic.h index fae32c7fdcb6..b07b75f411f2 100644 --- a/arch/frv/include/asm/atomic.h +++ b/arch/frv/include/asm/atomic.h @@ -256,7 +256,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #include #endif /* _ASM_ATOMIC_H */ diff --git a/arch/frv/include/asm/hardirq.h b/arch/frv/include/asm/hardirq.h index 5fc8b6f5bc55..c62833d6ebbb 100644 --- a/arch/frv/include/asm/hardirq.h +++ b/arch/frv/include/asm/hardirq.h @@ -12,7 +12,7 @@ #ifndef __ASM_HARDIRQ_H #define __ASM_HARDIRQ_H -#include +#include extern atomic_t irq_err_count; static inline void ack_bad_irq(int irq) diff --git a/arch/frv/kernel/irq.c b/arch/frv/kernel/irq.c index a5f624a9f559..3facbc28cbbc 100644 --- a/arch/frv/kernel/irq.c +++ b/arch/frv/kernel/irq.c @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/h8300/include/asm/atomic.h b/arch/h8300/include/asm/atomic.h index 984221abb66d..b641714774ea 100644 --- a/arch/h8300/include/asm/atomic.h +++ b/arch/h8300/include/asm/atomic.h @@ -116,7 +116,6 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) local_irq_restore(flags); return ret != u; } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) static __inline__ void atomic_clear_mask(unsigned long mask, unsigned long *v) { diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h index 446881439675..fdb887005dff 100644 --- a/arch/ia64/include/asm/atomic.h +++ b/arch/ia64/include/asm/atomic.h @@ -105,7 +105,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) static __inline__ long atomic64_add_unless(atomic64_t *v, long a, long u) { diff --git a/arch/ia64/include/asm/processor.h b/arch/ia64/include/asm/processor.h index 03afe7970748..d9f397fae03e 100644 --- a/arch/ia64/include/asm/processor.h +++ b/arch/ia64/include/asm/processor.h @@ -75,7 +75,7 @@ #include #include #include -#include +#include #ifdef CONFIG_NUMA #include #endif diff --git a/arch/ia64/include/asm/spinlock.h b/arch/ia64/include/asm/spinlock.h index 1a91c9121d17..b77768d35f93 100644 --- a/arch/ia64/include/asm/spinlock.h +++ b/arch/ia64/include/asm/spinlock.h @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c index be450a3e9871..0bd537b4ea6b 100644 --- a/arch/ia64/kernel/smp.c +++ b/arch/ia64/kernel/smp.c @@ -32,7 +32,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c index 14ec641003da..559097986672 100644 --- a/arch/ia64/kernel/smpboot.c +++ b/arch/ia64/kernel/smpboot.c @@ -40,7 +40,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c index c4696d217ce0..6a867dc45c05 100644 --- a/arch/ia64/kernel/uncached.c +++ b/arch/ia64/kernel/uncached.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/m32r/include/asm/atomic.h b/arch/m32r/include/asm/atomic.h index d44a51e5271b..d64d894dc549 100644 --- a/arch/m32r/include/asm/atomic.h +++ b/arch/m32r/include/asm/atomic.h @@ -262,7 +262,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) static __inline__ void atomic_clear_mask(unsigned long mask, atomic_t *addr) { diff --git a/arch/m32r/include/asm/mmu_context.h b/arch/m32r/include/asm/mmu_context.h index a70a3df33635..a979a4198168 100644 --- a/arch/m32r/include/asm/mmu_context.h +++ b/arch/m32r/include/asm/mmu_context.h @@ -11,7 +11,7 @@ #ifndef __ASSEMBLY__ -#include +#include #include #include #include diff --git a/arch/m32r/include/asm/spinlock.h b/arch/m32r/include/asm/spinlock.h index 179a06489b10..b0ea2f26da3b 100644 --- a/arch/m32r/include/asm/spinlock.h +++ b/arch/m32r/include/asm/spinlock.h @@ -10,7 +10,7 @@ */ #include -#include +#include #include /* diff --git a/arch/m32r/kernel/smp.c b/arch/m32r/kernel/smp.c index 092d40a6708e..ce7aea34fdf4 100644 --- a/arch/m32r/kernel/smp.c +++ b/arch/m32r/kernel/smp.c @@ -26,7 +26,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/m32r/kernel/traps.c b/arch/m32r/kernel/traps.c index fbd109031df3..ee6a9199561c 100644 --- a/arch/m32r/kernel/traps.c +++ b/arch/m32r/kernel/traps.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h index 307a573881ad..e844a2d2ba23 100644 --- a/arch/m68k/include/asm/atomic.h +++ b/arch/m68k/include/asm/atomic.h @@ -198,7 +198,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /* Atomic operations are already serializing */ #define smp_mb__before_atomic_dec() barrier() diff --git a/arch/microblaze/include/asm/mmu_context_mm.h b/arch/microblaze/include/asm/mmu_context_mm.h index 3e5c254e8d1c..d68647746448 100644 --- a/arch/microblaze/include/asm/mmu_context_mm.h +++ b/arch/microblaze/include/asm/mmu_context_mm.h @@ -11,7 +11,7 @@ #ifndef _ASM_MICROBLAZE_MMU_CONTEXT_H #define _ASM_MICROBLAZE_MMU_CONTEXT_H -#include +#include #include #include #include diff --git a/arch/microblaze/include/asm/prom.h b/arch/microblaze/include/asm/prom.h index 9bd01ecb00d6..9ad567e2d425 100644 --- a/arch/microblaze/include/asm/prom.h +++ b/arch/microblaze/include/asm/prom.h @@ -21,7 +21,7 @@ #include #include -#include +#include #define HAVE_ARCH_DEVTREE_FIXUPS diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h index 4a02fe891ab6..833a4023648a 100644 --- a/arch/mips/include/asm/atomic.h +++ b/arch/mips/include/asm/atomic.h @@ -325,7 +325,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) } return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_dec_return(v) atomic_sub_return(1, (v)) #define atomic_inc_return(v) atomic_add_return(1, (v)) diff --git a/arch/mips/include/asm/hw_irq.h b/arch/mips/include/asm/hw_irq.h index 77adda297ad9..9e8ef5994c9c 100644 --- a/arch/mips/include/asm/hw_irq.h +++ b/arch/mips/include/asm/hw_irq.h @@ -8,7 +8,7 @@ #ifndef __ASM_HW_IRQ_H #define __ASM_HW_IRQ_H -#include +#include extern atomic_t irq_err_count; diff --git a/arch/mips/include/asm/local.h b/arch/mips/include/asm/local.h index fffc8307a80a..94fde8d0fac1 100644 --- a/arch/mips/include/asm/local.h +++ b/arch/mips/include/asm/local.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include diff --git a/arch/mips/include/asm/smp.h b/arch/mips/include/asm/smp.h index af42385245d5..d4fb4d852a6d 100644 --- a/arch/mips/include/asm/smp.h +++ b/arch/mips/include/asm/smp.h @@ -17,7 +17,7 @@ #include #include -#include +#include #include extern int smp_num_siblings; diff --git a/arch/mips/kernel/irq.c b/arch/mips/kernel/irq.c index 9b734d74ae8e..b53970d80991 100644 --- a/arch/mips/kernel/irq.c +++ b/arch/mips/kernel/irq.c @@ -23,7 +23,7 @@ #include #include -#include +#include #include #include diff --git a/arch/mips/kernel/mips-mt.c b/arch/mips/kernel/mips-mt.c index b2259e7cd829..594ca69cb867 100644 --- a/arch/mips/kernel/mips-mt.c +++ b/arch/mips/kernel/mips-mt.c @@ -12,7 +12,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/mips/kernel/rtlx.c b/arch/mips/kernel/rtlx.c index 557ef72472e0..7a80b7cda7cc 100644 --- a/arch/mips/kernel/rtlx.c +++ b/arch/mips/kernel/rtlx.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/mips/kernel/smp-cmp.c b/arch/mips/kernel/smp-cmp.c index cc81771b882c..fe3095160655 100644 --- a/arch/mips/kernel/smp-cmp.c +++ b/arch/mips/kernel/smp-cmp.c @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/mips/kernel/smp-mt.c b/arch/mips/kernel/smp-mt.c index 1ec56e635d04..ce9e286f0a74 100644 --- a/arch/mips/kernel/smp-mt.c +++ b/arch/mips/kernel/smp-mt.c @@ -24,7 +24,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index 32a256101082..32c1e954cd37 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -34,7 +34,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/mips/kernel/smtc-proc.c b/arch/mips/kernel/smtc-proc.c index fe256559c997..928a5a61e1a6 100644 --- a/arch/mips/kernel/smtc-proc.c +++ b/arch/mips/kernel/smtc-proc.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c index cedac4633741..f0895e70e283 100644 --- a/arch/mips/kernel/smtc.c +++ b/arch/mips/kernel/smtc.c @@ -30,7 +30,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/mips/kernel/sync-r4k.c b/arch/mips/kernel/sync-r4k.c index 05dd170a83f7..99f913c8d7a6 100644 --- a/arch/mips/kernel/sync-r4k.c +++ b/arch/mips/kernel/sync-r4k.c @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include diff --git a/arch/mips/kernel/vpe.c b/arch/mips/kernel/vpe.c index dbb6b408f001..2cd50ad0d5c6 100644 --- a/arch/mips/kernel/vpe.c +++ b/arch/mips/kernel/vpe.c @@ -46,7 +46,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/mips/mipssim/sim_smtc.c b/arch/mips/mipssim/sim_smtc.c index 30df47258c2c..915063991f6e 100644 --- a/arch/mips/mipssim/sim_smtc.c +++ b/arch/mips/mipssim/sim_smtc.c @@ -24,7 +24,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/mips/sgi-ip27/ip27-nmi.c b/arch/mips/sgi-ip27/ip27-nmi.c index bc4fa8dd67f3..005c29ed419a 100644 --- a/arch/mips/sgi-ip27/ip27-nmi.c +++ b/arch/mips/sgi-ip27/ip27-nmi.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/mn10300/include/asm/atomic.h b/arch/mn10300/include/asm/atomic.h index 9d773a639513..041b9d69d86c 100644 --- a/arch/mn10300/include/asm/atomic.h +++ b/arch/mn10300/include/asm/atomic.h @@ -269,7 +269,6 @@ static inline void atomic_dec(atomic_t *v) c != (u); \ }) -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /** * atomic_clear_mask - Atomically clear bits in memory diff --git a/arch/mn10300/include/asm/mmu_context.h b/arch/mn10300/include/asm/mmu_context.h index c8f6c82672ad..c67c2b5365a6 100644 --- a/arch/mn10300/include/asm/mmu_context.h +++ b/arch/mn10300/include/asm/mmu_context.h @@ -22,7 +22,7 @@ #ifndef _ASM_MMU_CONTEXT_H #define _ASM_MMU_CONTEXT_H -#include +#include #include #include #include diff --git a/arch/mn10300/include/asm/spinlock.h b/arch/mn10300/include/asm/spinlock.h index 93429154e898..1ae580f38933 100644 --- a/arch/mn10300/include/asm/spinlock.h +++ b/arch/mn10300/include/asm/spinlock.h @@ -11,7 +11,7 @@ #ifndef _ASM_SPINLOCK_H #define _ASM_SPINLOCK_H -#include +#include #include #include diff --git a/arch/mn10300/include/asm/system.h b/arch/mn10300/include/asm/system.h index 8ff3e5aaca41..94b4c5e1491b 100644 --- a/arch/mn10300/include/asm/system.h +++ b/arch/mn10300/include/asm/system.h @@ -19,7 +19,7 @@ #include #include -#include +#include #if !defined(CONFIG_LAZY_SAVE_FPU) struct fpu_state_struct; diff --git a/arch/mn10300/kernel/mn10300-watchdog.c b/arch/mn10300/kernel/mn10300-watchdog.c index c5e12bfd9fcd..a45f0c7549a6 100644 --- a/arch/mn10300/kernel/mn10300-watchdog.c +++ b/arch/mn10300/kernel/mn10300-watchdog.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/mn10300/kernel/traps.c b/arch/mn10300/kernel/traps.c index bd3e5e73826e..9220a75a7b43 100644 --- a/arch/mn10300/kernel/traps.c +++ b/arch/mn10300/kernel/traps.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/mn10300/mm/misalignment.c b/arch/mn10300/mm/misalignment.c index eef989c1d0c1..f9bb8cb1c14a 100644 --- a/arch/mn10300/mm/misalignment.c +++ b/arch/mn10300/mm/misalignment.c @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/mn10300/proc-mn2ws0050/proc-init.c b/arch/mn10300/proc-mn2ws0050/proc-init.c index c58249b9525a..fe6e24906ffc 100644 --- a/arch/mn10300/proc-mn2ws0050/proc-init.c +++ b/arch/mn10300/proc-mn2ws0050/proc-init.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h index f81955934aeb..192488999b63 100644 --- a/arch/parisc/include/asm/atomic.h +++ b/arch/parisc/include/asm/atomic.h @@ -220,7 +220,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_add(i,v) ((void)(__atomic_add_return( (i),(v)))) #define atomic_sub(i,v) ((void)(__atomic_add_return(-(i),(v)))) diff --git a/arch/parisc/include/asm/bitops.h b/arch/parisc/include/asm/bitops.h index 4e833aa05a44..8c9b631d2a78 100644 --- a/arch/parisc/include/asm/bitops.h +++ b/arch/parisc/include/asm/bitops.h @@ -8,7 +8,7 @@ #include #include /* for BITS_PER_LONG/SHIFT_PER_LONG */ #include -#include +#include /* * HP-PARISC specific bit operations diff --git a/arch/parisc/include/asm/mmu_context.h b/arch/parisc/include/asm/mmu_context.h index 354b2aca990e..59be25764433 100644 --- a/arch/parisc/include/asm/mmu_context.h +++ b/arch/parisc/include/asm/mmu_context.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/parisc/kernel/parisc_ksyms.c b/arch/parisc/kernel/parisc_ksyms.c index df653663d3db..a7bb757a5497 100644 --- a/arch/parisc/kernel/parisc_ksyms.c +++ b/arch/parisc/kernel/parisc_ksyms.c @@ -31,7 +31,7 @@ #include EXPORT_SYMBOL(memset); -#include +#include EXPORT_SYMBOL(__xchg8); EXPORT_SYMBOL(__xchg32); EXPORT_SYMBOL(__cmpxchg_u32); diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c index 828305f19cff..32d588488f04 100644 --- a/arch/parisc/kernel/smp.c +++ b/arch/parisc/kernel/smp.c @@ -33,7 +33,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index 8b58bf0b7d5a..f19e6604026a 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/parisc/lib/bitops.c b/arch/parisc/lib/bitops.c index 353963d42059..a8bffd8af77d 100644 --- a/arch/parisc/lib/bitops.c +++ b/arch/parisc/lib/bitops.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #ifdef CONFIG_SMP arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned = { diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index b8f152ece025..b2bcbee622ea 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -212,7 +212,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) return t != u; } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0) #define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0) diff --git a/arch/powerpc/include/asm/emulated_ops.h b/arch/powerpc/include/asm/emulated_ops.h index 2cc41c715d2b..63f2a22e9954 100644 --- a/arch/powerpc/include/asm/emulated_ops.h +++ b/arch/powerpc/include/asm/emulated_ops.h @@ -18,7 +18,7 @@ #ifndef _ASM_POWERPC_EMULATED_OPS_H #define _ASM_POWERPC_EMULATED_OPS_H -#include +#include #include diff --git a/arch/powerpc/include/asm/irq.h b/arch/powerpc/include/asm/irq.h index c57a28e52b64..c0e1bc319e35 100644 --- a/arch/powerpc/include/asm/irq.h +++ b/arch/powerpc/include/asm/irq.h @@ -14,7 +14,7 @@ #include #include -#include +#include /* Define a way to iterate across irqs. */ diff --git a/arch/powerpc/include/asm/local.h b/arch/powerpc/include/asm/local.h index c2410af6bfd9..b8da91363864 100644 --- a/arch/powerpc/include/asm/local.h +++ b/arch/powerpc/include/asm/local.h @@ -2,7 +2,7 @@ #define _ARCH_POWERPC_LOCAL_H #include -#include +#include typedef struct { diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h index b823536375dc..b5c91901e384 100644 --- a/arch/powerpc/include/asm/prom.h +++ b/arch/powerpc/include/asm/prom.h @@ -18,7 +18,7 @@ */ #include #include -#include +#include #define HAVE_ARCH_DEVTREE_FIXUPS diff --git a/arch/powerpc/kernel/of_platform.c b/arch/powerpc/kernel/of_platform.c index 24582181b6ec..59dbf6abaaf3 100644 --- a/arch/powerpc/kernel/of_platform.c +++ b/arch/powerpc/kernel/of_platform.c @@ -26,7 +26,7 @@ #include #include #include -#include +#include #ifdef CONFIG_PPC_OF_PLATFORM_PCI diff --git a/arch/powerpc/kernel/ppc_ksyms.c b/arch/powerpc/kernel/ppc_ksyms.c index 7d28f540200c..f5ae872a2ef0 100644 --- a/arch/powerpc/kernel/ppc_ksyms.c +++ b/arch/powerpc/kernel/ppc_ksyms.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index 0e0ea941156f..d5ca8236315c 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/kernel/rtasd.c b/arch/powerpc/kernel/rtasd.c index 67f6c3b51357..481ef064c8f1 100644 --- a/arch/powerpc/kernel/rtasd.c +++ b/arch/powerpc/kernel/rtasd.c @@ -27,7 +27,7 @@ #include #include #include -#include +#include #include diff --git a/arch/powerpc/kernel/smp-tbsync.c b/arch/powerpc/kernel/smp-tbsync.c index 03e45c4a9ef1..640de836e466 100644 --- a/arch/powerpc/kernel/smp-tbsync.c +++ b/arch/powerpc/kernel/smp-tbsync.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index f932f8a0cf0c..7bf2187dfd99 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -33,7 +33,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/83xx/km83xx.c b/arch/powerpc/platforms/83xx/km83xx.c index f8fa2fc3129f..c55129f5760a 100644 --- a/arch/powerpc/platforms/83xx/km83xx.c +++ b/arch/powerpc/platforms/83xx/km83xx.c @@ -28,7 +28,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/83xx/mpc832x_mds.c b/arch/powerpc/platforms/83xx/mpc832x_mds.c index 93e60f1f21a9..32a52896822f 100644 --- a/arch/powerpc/platforms/83xx/mpc832x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc832x_mds.c @@ -27,7 +27,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/83xx/mpc834x_itx.c b/arch/powerpc/platforms/83xx/mpc834x_itx.c index 81e44fa1c644..6b45969567d4 100644 --- a/arch/powerpc/platforms/83xx/mpc834x_itx.c +++ b/arch/powerpc/platforms/83xx/mpc834x_itx.c @@ -26,7 +26,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/83xx/mpc834x_mds.c b/arch/powerpc/platforms/83xx/mpc834x_mds.c index c1b1dc50b32a..041c5177e737 100644 --- a/arch/powerpc/platforms/83xx/mpc834x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc834x_mds.c @@ -26,7 +26,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/83xx/mpc836x_mds.c b/arch/powerpc/platforms/83xx/mpc836x_mds.c index 81c052b1353e..934cc8c46bbc 100644 --- a/arch/powerpc/platforms/83xx/mpc836x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc836x_mds.c @@ -34,7 +34,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/83xx/sbc834x.c b/arch/powerpc/platforms/83xx/sbc834x.c index 49023dbe1576..af41d8c810a8 100644 --- a/arch/powerpc/platforms/83xx/sbc834x.c +++ b/arch/powerpc/platforms/83xx/sbc834x.c @@ -28,7 +28,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/85xx/mpc85xx_cds.c b/arch/powerpc/platforms/85xx/mpc85xx_cds.c index 6299a2a51ae8..2bf99786d249 100644 --- a/arch/powerpc/platforms/85xx/mpc85xx_cds.c +++ b/arch/powerpc/platforms/85xx/mpc85xx_cds.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/85xx/mpc85xx_mds.c b/arch/powerpc/platforms/85xx/mpc85xx_mds.c index 747d1ee661fd..973b3f4a4b49 100644 --- a/arch/powerpc/platforms/85xx/mpc85xx_mds.c +++ b/arch/powerpc/platforms/85xx/mpc85xx_mds.c @@ -36,7 +36,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/85xx/sbc8548.c b/arch/powerpc/platforms/85xx/sbc8548.c index ecdd8c09e4ed..d07dcb7f4ee9 100644 --- a/arch/powerpc/platforms/85xx/sbc8548.c +++ b/arch/powerpc/platforms/85xx/sbc8548.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/cell/cpufreq_spudemand.c b/arch/powerpc/platforms/cell/cpufreq_spudemand.c index d809836bcf5f..7f92096fe968 100644 --- a/arch/powerpc/platforms/cell/cpufreq_spudemand.c +++ b/arch/powerpc/platforms/cell/cpufreq_spudemand.c @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/powerpc/platforms/cell/smp.c b/arch/powerpc/platforms/cell/smp.c index dbb641ea90dd..f2e1dfe4bf31 100644 --- a/arch/powerpc/platforms/cell/smp.c +++ b/arch/powerpc/platforms/cell/smp.c @@ -28,7 +28,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/cell/spufs/context.c b/arch/powerpc/platforms/cell/spufs/context.c index 0c87bcd2452a..bf4d41d8fa14 100644 --- a/arch/powerpc/platforms/cell/spufs/context.c +++ b/arch/powerpc/platforms/cell/spufs/context.c @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include "spufs.h" diff --git a/arch/powerpc/platforms/chrp/smp.c b/arch/powerpc/platforms/chrp/smp.c index a800122e4dda..feab30bbae23 100644 --- a/arch/powerpc/platforms/chrp/smp.c +++ b/arch/powerpc/platforms/chrp/smp.c @@ -18,7 +18,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/iseries/smp.c b/arch/powerpc/platforms/iseries/smp.c index 2df48c2287bd..8bda9be06fa0 100644 --- a/arch/powerpc/platforms/iseries/smp.c +++ b/arch/powerpc/platforms/iseries/smp.c @@ -29,7 +29,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/powermac/backlight.c b/arch/powerpc/platforms/powermac/backlight.c index d679964ae2ab..c2f3e861f5ea 100644 --- a/arch/powerpc/platforms/powermac/backlight.c +++ b/arch/powerpc/platforms/powermac/backlight.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c index d15fca322978..9a521dc8e485 100644 --- a/arch/powerpc/platforms/powermac/smp.c +++ b/arch/powerpc/platforms/powermac/smp.c @@ -35,7 +35,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/pseries/eeh.c b/arch/powerpc/platforms/pseries/eeh.c index 46b55cf563e3..ada6e07532ec 100644 --- a/arch/powerpc/platforms/pseries/eeh.c +++ b/arch/powerpc/platforms/pseries/eeh.c @@ -31,7 +31,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/platforms/pseries/eeh_cache.c b/arch/powerpc/platforms/pseries/eeh_cache.c index 8ed0d2d0e1b5..fc5ae767989e 100644 --- a/arch/powerpc/platforms/pseries/eeh_cache.c +++ b/arch/powerpc/platforms/pseries/eeh_cache.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/powerpc/platforms/pseries/smp.c b/arch/powerpc/platforms/pseries/smp.c index 1672db2d1b0e..4e44c4dcd11c 100644 --- a/arch/powerpc/platforms/pseries/smp.c +++ b/arch/powerpc/platforms/pseries/smp.c @@ -27,7 +27,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/sysdev/fsl_soc.c b/arch/powerpc/sysdev/fsl_soc.c index 265313e8396b..2d66275e489f 100644 --- a/arch/powerpc/sysdev/fsl_soc.c +++ b/arch/powerpc/sysdev/fsl_soc.c @@ -32,7 +32,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/sysdev/tsi108_dev.c b/arch/powerpc/sysdev/tsi108_dev.c index ee056807b52c..9f51f97abb5d 100644 --- a/arch/powerpc/sysdev/tsi108_dev.c +++ b/arch/powerpc/sysdev/tsi108_dev.c @@ -23,7 +23,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index d9db13810d15..29d756329228 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -108,7 +108,6 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) return c != u; } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #undef __CS_LOOP diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 1ca3d1d6a86c..45df6d456aa1 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -27,7 +27,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index e9372c77cced..ffabcd9d3363 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/sh/include/asm/atomic.h b/arch/sh/include/asm/atomic.h index c7983124d99d..8ddb2635cf92 100644 --- a/arch/sh/include/asm/atomic.h +++ b/arch/sh/include/asm/atomic.h @@ -30,7 +30,6 @@ #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) #define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0) #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_inc(v) atomic_add(1, (v)) #define atomic_dec(v) atomic_sub(1, (v)) diff --git a/arch/sh/include/asm/hw_irq.h b/arch/sh/include/asm/hw_irq.h index 603cdde813d1..693d44184058 100644 --- a/arch/sh/include/asm/hw_irq.h +++ b/arch/sh/include/asm/hw_irq.h @@ -3,7 +3,7 @@ #include #include -#include +#include extern atomic_t irq_err_count; diff --git a/arch/sh/include/asm/smp.h b/arch/sh/include/asm/smp.h index 9070d943ddde..78b0d0f4b24b 100644 --- a/arch/sh/include/asm/smp.h +++ b/arch/sh/include/asm/smp.h @@ -8,7 +8,7 @@ #ifdef CONFIG_SMP #include -#include +#include #include #include diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c index 425d604e3a28..84db0d6ccd0d 100644 --- a/arch/sh/kernel/idle.c +++ b/arch/sh/kernel/idle.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include void (*pm_idle)(void) = NULL; diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c index 6207561ea34a..3147a9a6fb8b 100644 --- a/arch/sh/kernel/smp.c +++ b/arch/sh/kernel/smp.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/sh/kernel/traps_64.c b/arch/sh/kernel/traps_64.c index 67110be83fd7..cd3a40483299 100644 --- a/arch/sh/kernel/traps_64.c +++ b/arch/sh/kernel/traps_64.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/sh/kernel/unwinder.c b/arch/sh/kernel/unwinder.c index 468889d958f4..521b5432471f 100644 --- a/arch/sh/kernel/unwinder.c +++ b/arch/sh/kernel/unwinder.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include /* * This is the most basic stack unwinder an architecture can diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index 7ae128b19d3f..7646f2cef5d0 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -52,7 +52,6 @@ extern void atomic_set(atomic_t *, int); #define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) #define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /* This is the old 24-bit implementation. It's still used internally * by some sparc-specific code, notably the semaphore implementation. diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index bdb2ff880bdd..337139ef91be 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -85,7 +85,6 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic64_cmpxchg(v, o, n) \ ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) diff --git a/arch/sparc/include/asm/prom.h b/arch/sparc/include/asm/prom.h index 56bbaadef646..edd3d3cde460 100644 --- a/arch/sparc/include/asm/prom.h +++ b/arch/sparc/include/asm/prom.h @@ -21,7 +21,7 @@ #include #include #include -#include +#include #define OF_ROOT_NODE_ADDR_CELLS_DEFAULT 2 #define OF_ROOT_NODE_SIZE_CELLS_DEFAULT 1 diff --git a/arch/sparc/include/asm/smp_32.h b/arch/sparc/include/asm/smp_32.h index 093f10843ff2..01c51c704341 100644 --- a/arch/sparc/include/asm/smp_32.h +++ b/arch/sparc/include/asm/smp_32.h @@ -22,7 +22,7 @@ #include #include -#include +#include /* * Private routines/data diff --git a/arch/sparc/include/asm/smp_64.h b/arch/sparc/include/asm/smp_64.h index 20bca8950710..29862a9e9065 100644 --- a/arch/sparc/include/asm/smp_64.h +++ b/arch/sparc/include/asm/smp_64.h @@ -27,7 +27,7 @@ */ #include -#include +#include #include DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 4e78862d12fd..0dd8422a469c 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -26,7 +26,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c index fe8fb44c609c..1210fde18740 100644 --- a/arch/sparc/kernel/leon_smp.c +++ b/arch/sparc/kernel/leon_smp.c @@ -28,7 +28,7 @@ #include #include -#include +#include #include #include diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c index 62a034318b18..171e8d84dc3f 100644 --- a/arch/sparc/kernel/perf_event.c +++ b/arch/sparc/kernel/perf_event.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/sparc/kernel/smp_32.c b/arch/sparc/kernel/smp_32.c index 21b125341bf7..f671e7fd6ddc 100644 --- a/arch/sparc/kernel/smp_32.c +++ b/arch/sparc/kernel/smp_32.c @@ -22,7 +22,7 @@ #include #include -#include +#include #include #include diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index 99cb17251bb5..4a442c32e117 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -28,7 +28,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index d3c7a12ad879..1a371f8ae0b0 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -7,7 +7,7 @@ * Based on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf */ -#include +#include #include #include diff --git a/arch/tile/include/asm/atomic.h b/arch/tile/include/asm/atomic.h index 739cfe0499d1..e3272715c3cb 100644 --- a/arch/tile/include/asm/atomic.h +++ b/arch/tile/include/asm/atomic.h @@ -121,15 +121,6 @@ static inline int atomic_read(const atomic_t *v) */ #define atomic_add_negative(i, v) (atomic_add_return((i), (v)) < 0) -/** - * atomic_inc_not_zero - increment unless the number is zero - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1, so long as @v is non-zero. - * Returns non-zero if @v was non-zero, and zero otherwise. - */ -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) - /* Nonexistent functions intended to cause link errors. */ extern unsigned long __xchg_called_with_bad_pointer(void); extern unsigned long __cmpxchg_called_with_bad_pointer(void); diff --git a/arch/tile/include/asm/atomic_32.h b/arch/tile/include/asm/atomic_32.h index 92a8bee32311..246feed4794d 100644 --- a/arch/tile/include/asm/atomic_32.h +++ b/arch/tile/include/asm/atomic_32.h @@ -11,7 +11,7 @@ * NON INFRINGEMENT. See the GNU General Public License for * more details. * - * Do not include directly; use . + * Do not include directly; use . */ #ifndef _ASM_TILE_ATOMIC_32_H @@ -21,7 +21,7 @@ #ifndef __ASSEMBLY__ -/* Tile-specific routines to support . */ +/* Tile-specific routines to support . */ int _atomic_xchg(atomic_t *v, int n); int _atomic_xchg_add(atomic_t *v, int i); int _atomic_xchg_add_unless(atomic_t *v, int a, int u); diff --git a/arch/tile/include/asm/atomic_64.h b/arch/tile/include/asm/atomic_64.h index 1c1e60d8ccb6..a48dda30cbcc 100644 --- a/arch/tile/include/asm/atomic_64.h +++ b/arch/tile/include/asm/atomic_64.h @@ -11,7 +11,7 @@ * NON INFRINGEMENT. See the GNU General Public License for * more details. * - * Do not include directly; use . + * Do not include directly; use . */ #ifndef _ASM_TILE_ATOMIC_64_H diff --git a/arch/tile/include/asm/bitops_32.h b/arch/tile/include/asm/bitops_32.h index d31ab905cfa7..571b118bfd9b 100644 --- a/arch/tile/include/asm/bitops_32.h +++ b/arch/tile/include/asm/bitops_32.h @@ -16,7 +16,7 @@ #define _ASM_TILE_BITOPS_32_H #include -#include +#include #include /* Tile-specific routines to support . */ diff --git a/arch/tile/include/asm/bitops_64.h b/arch/tile/include/asm/bitops_64.h index 68f8c5bc0679..e9c8e381ee0e 100644 --- a/arch/tile/include/asm/bitops_64.h +++ b/arch/tile/include/asm/bitops_64.h @@ -16,7 +16,7 @@ #define _ASM_TILE_BITOPS_64_H #include -#include +#include #include /* See for API comments. */ diff --git a/arch/tile/include/asm/spinlock_32.h b/arch/tile/include/asm/spinlock_32.h index a8f2c6e31a87..a5e4208d34f9 100644 --- a/arch/tile/include/asm/spinlock_32.h +++ b/arch/tile/include/asm/spinlock_32.h @@ -17,7 +17,7 @@ #ifndef _ASM_TILE_SPINLOCK_32_H #define _ASM_TILE_SPINLOCK_32_H -#include +#include #include #include #include diff --git a/arch/tile/kernel/intvec_32.S b/arch/tile/kernel/intvec_32.S index 72ade79b621b..fc94607f0bd5 100644 --- a/arch/tile/kernel/intvec_32.S +++ b/arch/tile/kernel/intvec_32.S @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/tile/lib/atomic_32.c b/arch/tile/lib/atomic_32.c index 46570211df52..771b251b409d 100644 --- a/arch/tile/lib/atomic_32.c +++ b/arch/tile/lib/atomic_32.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/tile/lib/atomic_asm_32.S b/arch/tile/lib/atomic_asm_32.S index 24448734f6f1..1f75a2a56101 100644 --- a/arch/tile/lib/atomic_asm_32.S +++ b/arch/tile/lib/atomic_asm_32.S @@ -70,7 +70,7 @@ */ #include -#include +#include #include #include diff --git a/arch/x86/ia32/sys_ia32.c b/arch/x86/ia32/sys_ia32.c index 5852519b2d0f..f6f5c53dc903 100644 --- a/arch/x86/ia32/sys_ia32.c +++ b/arch/x86/ia32/sys_ia32.c @@ -43,7 +43,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 4a0b7c7e2cce..7b3ca8324b69 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index 952a826ac4e5..897969bdd4e6 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -244,7 +244,6 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) /* * atomic_dec_if_positive - decrement by 1 if old value positive diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index 13f5504c76c0..09199052060f 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -21,7 +21,7 @@ #include #include -#include +#include #include #include diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h index 2e9972468a5d..9cdae5d47e8f 100644 --- a/arch/x86/include/asm/local.h +++ b/arch/x86/include/asm/local.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include typedef struct { diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index 716b48af7863..c9321f34e55b 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -124,7 +124,7 @@ extern struct atomic_notifier_head x86_mce_decoder_chain; #include #include -#include +#include extern int mce_disabled; extern int mce_p5_enabled; diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index 8b5393ec1080..69021528b43c 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -2,7 +2,7 @@ #define _ASM_X86_MMU_CONTEXT_H #include -#include +#include #include #include #include diff --git a/arch/x86/include/asm/prom.h b/arch/x86/include/asm/prom.h index df1287019e6d..644dd885f05a 100644 --- a/arch/x86/include/asm/prom.h +++ b/arch/x86/include/asm/prom.h @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h index e9e51f710e6c..ee67edf86fdd 100644 --- a/arch/x86/include/asm/spinlock.h +++ b/arch/x86/include/asm/spinlock.h @@ -1,7 +1,7 @@ #ifndef _ASM_X86_SPINLOCK_H #define _ASM_X86_SPINLOCK_H -#include +#include #include #include #include diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index 1f2e61e28981..a1fe5c127b52 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -21,7 +21,7 @@ struct task_struct; struct exec_domain; #include #include -#include +#include struct thread_info { struct task_struct *task; /* main task structure */ diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index b117efd24f71..8a439d364b94 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index b24be38c8cf8..52fa56399a50 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/apic/es7000_32.c b/arch/x86/kernel/apic/es7000_32.c index 9536b3fe43f8..5d513bc47b6b 100644 --- a/arch/x86/kernel/apic/es7000_32.c +++ b/arch/x86/kernel/apic/es7000_32.c @@ -48,7 +48,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 22a073d7fbff..62184390a601 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/i8259.c b/arch/x86/kernel/i8259.c index 65b8f5c2eebf..610485223bdb 100644 --- a/arch/x86/kernel/i8259.c +++ b/arch/x86/kernel/i8259.c @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c index f09d4bbe2d2d..b3300e6bacef 100644 --- a/arch/x86/kernel/irqinit.c +++ b/arch/x86/kernel/irqinit.c @@ -15,7 +15,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index fbc097a085ca..9682ec50180c 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -49,7 +49,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 2b2255b1f04b..57dcbd4308fa 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include "kvm_cache_regs.h" #include "irq.h" #include "trace.h" diff --git a/arch/x86/kvm/timer.c b/arch/x86/kvm/timer.c index abd86e865be3..ae432ea1cd83 100644 --- a/arch/x86/kvm/timer.c +++ b/arch/x86/kvm/timer.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include "kvm_timer.h" static int __kvm_timer_fn(struct kvm_vcpu *vcpu, struct kvm_timer *ktimer) diff --git a/arch/x86/lib/atomic64_32.c b/arch/x86/lib/atomic64_32.c index 540179e8e9fa..042f6826bf57 100644 --- a/arch/x86/lib/atomic64_32.c +++ b/arch/x86/lib/atomic64_32.c @@ -4,7 +4,7 @@ #include #include -#include +#include long long atomic64_read_cx8(long long, const atomic64_t *v); EXPORT_SYMBOL(atomic64_read_cx8); diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c index 3adff7dcc148..67421f38a215 100644 --- a/arch/x86/mm/mmio-mod.c +++ b/arch/x86/mm/mmio-mod.c @@ -34,7 +34,7 @@ #include #include #include /* for ISA_START_ADDRESS */ -#include +#include #include #include diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h index a96a0619d0b7..7cca2fb18baf 100644 --- a/arch/xtensa/include/asm/atomic.h +++ b/arch/xtensa/include/asm/atomic.h @@ -248,7 +248,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) return c != (u); } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) { diff --git a/arch/xtensa/kernel/process.c b/arch/xtensa/kernel/process.c index e3558b9a58ba..47041e7c088c 100644 --- a/arch/xtensa/kernel/process.c +++ b/arch/xtensa/kernel/process.c @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include #include diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 940d70cb5c25..ac33d5f30778 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -12,7 +12,7 @@ * */ -#include +#include #include #include #include diff --git a/crypto/proc.c b/crypto/proc.c index 58fef67d4f4d..3808697814d7 100644 --- a/crypto/proc.c +++ b/crypto/proc.c @@ -13,7 +13,7 @@ * */ -#include +#include #include #include #include diff --git a/crypto/rng.c b/crypto/rng.c index f93cb5311182..45229ae782be 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -12,7 +12,7 @@ * */ -#include +#include #include #include #include diff --git a/drivers/atm/ambassador.c b/drivers/atm/ambassador.c index bb3b016b6ce8..f8f41e0e8a8c 100644 --- a/drivers/atm/ambassador.c +++ b/drivers/atm/ambassador.c @@ -38,7 +38,7 @@ #include #include -#include +#include #include #include diff --git a/drivers/atm/atmtcp.c b/drivers/atm/atmtcp.c index 0b0625054a87..b22d71cac54c 100644 --- a/drivers/atm/atmtcp.c +++ b/drivers/atm/atmtcp.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include extern int atm_init_aal5(struct atm_vcc *vcc); /* "raw" AAL5 transport */ diff --git a/drivers/atm/eni.c b/drivers/atm/eni.c index 3230ea0df83c..93071417315f 100644 --- a/drivers/atm/eni.c +++ b/drivers/atm/eni.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/atm/eni.h b/drivers/atm/eni.h index 493a6932507e..dc9a62cc2605 100644 --- a/drivers/atm/eni.h +++ b/drivers/atm/eni.h @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include "midway.h" diff --git a/drivers/atm/firestream.c b/drivers/atm/firestream.c index 7c7b571647f9..5072f8ac16fd 100644 --- a/drivers/atm/firestream.c +++ b/drivers/atm/firestream.c @@ -52,7 +52,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/atm/fore200e.c b/drivers/atm/fore200e.c index bc9e702186dd..361f5aee3be1 100644 --- a/drivers/atm/fore200e.c +++ b/drivers/atm/fore200e.c @@ -44,7 +44,7 @@ #include #include #include -#include +#include #ifdef CONFIG_SBUS #include diff --git a/drivers/atm/horizon.c b/drivers/atm/horizon.c index 287506183893..b81210330aca 100644 --- a/drivers/atm/horizon.c +++ b/drivers/atm/horizon.c @@ -45,7 +45,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/atm/idt77252.c b/drivers/atm/idt77252.c index be0dbfeb541c..db06f34419cf 100644 --- a/drivers/atm/idt77252.c +++ b/drivers/atm/idt77252.c @@ -46,7 +46,7 @@ #include #include -#include +#include #include #ifdef CONFIG_ATM_IDT77252_USE_SUNI diff --git a/drivers/atm/iphase.c b/drivers/atm/iphase.c index 957106f636ea..cb90f7a3e074 100644 --- a/drivers/atm/iphase.c +++ b/drivers/atm/iphase.c @@ -58,7 +58,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/atm/nicstar.c b/drivers/atm/nicstar.c index 6b313ee9231b..1c70c45fa044 100644 --- a/drivers/atm/nicstar.c +++ b/drivers/atm/nicstar.c @@ -51,7 +51,7 @@ #include #include #include -#include +#include #include "nicstar.h" #ifdef CONFIG_ATM_NICSTAR_USE_SUNI #include "suni.h" diff --git a/drivers/atm/suni.c b/drivers/atm/suni.c index 41c56eae4c81..90f1ccca9e52 100644 --- a/drivers/atm/suni.c +++ b/drivers/atm/suni.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include "suni.h" diff --git a/drivers/atm/uPD98402.c b/drivers/atm/uPD98402.c index c45ae0573bbd..5120a96b3a89 100644 --- a/drivers/atm/uPD98402.c +++ b/drivers/atm/uPD98402.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include "uPD98402.h" diff --git a/drivers/atm/zatm.c b/drivers/atm/zatm.c index 7f8c5132ff32..d889f56e8d8c 100644 --- a/drivers/atm/zatm.c +++ b/drivers/atm/zatm.c @@ -27,7 +27,7 @@ #include #include #include -#include +#include #include #include "uPD98401.h" diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 45d7c8fc73bd..2840ed4668c1 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -24,7 +24,7 @@ #include #include -#include +#include #include static DEFINE_MUTEX(mem_sysfs_mutex); diff --git a/drivers/base/power/sysfs.c b/drivers/base/power/sysfs.c index 942d6a7c9ae1..17b7934f31cb 100644 --- a/drivers/base/power/sysfs.c +++ b/drivers/base/power/sysfs.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include "power.h" diff --git a/drivers/block/cciss_scsi.c b/drivers/block/cciss_scsi.c index 696100241a6f..951a4e33b92b 100644 --- a/drivers/block/cciss_scsi.c +++ b/drivers/block/cciss_scsi.c @@ -33,7 +33,7 @@ #include #include -#include +#include #include #include diff --git a/drivers/char/ipmi/ipmi_watchdog.c b/drivers/char/ipmi/ipmi_watchdog.c index 320668f4c3aa..3302586655c4 100644 --- a/drivers/char/ipmi/ipmi_watchdog.c +++ b/drivers/char/ipmi/ipmi_watchdog.c @@ -52,7 +52,7 @@ #include #include #include -#include +#include #ifdef CONFIG_X86 /* diff --git a/drivers/char/mspec.c b/drivers/char/mspec.c index 25d139c9dbed..5c0d96a820fa 100644 --- a/drivers/char/mspec.c +++ b/drivers/char/mspec.c @@ -46,7 +46,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/connector/cn_proc.c b/drivers/connector/cn_proc.c index 0debc17c8e28..3ee1fdb31ea7 100644 --- a/drivers/connector/cn_proc.c +++ b/drivers/connector/cn_proc.c @@ -29,7 +29,8 @@ #include #include #include -#include +#include + #include #include diff --git a/drivers/edac/edac_stub.c b/drivers/edac/edac_stub.c index aab970760b75..86ad2eee1201 100644 --- a/drivers/edac/edac_stub.c +++ b/drivers/edac/edac_stub.c @@ -14,7 +14,7 @@ */ #include #include -#include +#include #include int edac_op_state = EDAC_OPSTATE_INVAL; diff --git a/drivers/firewire/core-card.c b/drivers/firewire/core-card.c index 29d2423fae6d..85661b060ed7 100644 --- a/drivers/firewire/core-card.c +++ b/drivers/firewire/core-card.c @@ -32,7 +32,7 @@ #include #include -#include +#include #include #include "core.h" diff --git a/drivers/firewire/core-device.c b/drivers/firewire/core-device.c index 95a471401892..8ba7f7928f1f 100644 --- a/drivers/firewire/core-device.c +++ b/drivers/firewire/core-device.c @@ -38,7 +38,7 @@ #include #include -#include +#include #include #include diff --git a/drivers/firewire/core-topology.c b/drivers/firewire/core-topology.c index 193ed9233144..94d3b494ddfb 100644 --- a/drivers/firewire/core-topology.c +++ b/drivers/firewire/core-topology.c @@ -29,7 +29,7 @@ #include #include -#include +#include #include #include diff --git a/drivers/firewire/core.h b/drivers/firewire/core.h index 0fe4e4e6eda7..b45be5767529 100644 --- a/drivers/firewire/core.h +++ b/drivers/firewire/core.h @@ -9,7 +9,7 @@ #include #include -#include +#include struct device; struct fw_card; diff --git a/drivers/firewire/nosy.c b/drivers/firewire/nosy.c index 0618145376ad..763626b739d1 100644 --- a/drivers/firewire/nosy.c +++ b/drivers/firewire/nosy.c @@ -37,7 +37,7 @@ #include #include -#include +#include #include #include "nosy.h" diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index ef37a9b5a3cc..32807baf55e2 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -60,7 +60,7 @@ * are considered as fatal) */ -#include +#include #include #include #include diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c index 021d2b6b556f..7fd4e3e5ad5f 100644 --- a/drivers/gpu/drm/radeon/radeon_fence.c +++ b/drivers/gpu/drm/radeon/radeon_fence.c @@ -29,7 +29,7 @@ * Dave Airlie */ #include -#include +#include #include #include #include diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 2e618b5ac465..56619f64b6bf 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -37,7 +37,7 @@ #include #include #include -#include +#include #define TTM_ASSERT_LOCKED(param) #define TTM_DEBUG(fmt, arg...) diff --git a/drivers/gpu/drm/ttm/ttm_lock.c b/drivers/gpu/drm/ttm/ttm_lock.c index de41e55a944a..075daf44bce4 100644 --- a/drivers/gpu/drm/ttm/ttm_lock.c +++ b/drivers/gpu/drm/ttm/ttm_lock.c @@ -30,7 +30,7 @@ #include "ttm/ttm_lock.h" #include "ttm/ttm_module.h" -#include +#include #include #include #include diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c index ebddd443d91a..93577f2e2954 100644 --- a/drivers/gpu/drm/ttm/ttm_object.c +++ b/drivers/gpu/drm/ttm/ttm_object.c @@ -55,7 +55,7 @@ #include #include #include -#include +#include struct ttm_object_file { struct ttm_object_device *tdev; diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index 170e751c283e..727e93daac3b 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -40,7 +40,7 @@ #include #include -#include +#include #include "ttm/ttm_bo_driver.h" #include "ttm/ttm_page_alloc.h" diff --git a/drivers/hwmon/sht15.c b/drivers/hwmon/sht15.c index 7d231cf5d2ce..fe4104c6b764 100644 --- a/drivers/hwmon/sht15.c +++ b/drivers/hwmon/sht15.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include /* Commands */ #define SHT15_MEASURE_TEMP 0x03 diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index 0347eed4a167..40c835309e49 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -31,7 +31,7 @@ */ #include -#include +#include #include "iw_cxgb4.h" diff --git a/drivers/infiniband/hw/ehca/ehca_tools.h b/drivers/infiniband/hw/ehca/ehca_tools.h index f09914cccf53..54c0d23bad92 100644 --- a/drivers/infiniband/hw/ehca/ehca_tools.h +++ b/drivers/infiniband/hw/ehca/ehca_tools.h @@ -58,7 +58,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c index 73bc18465c9c..c118663e4437 100644 --- a/drivers/infiniband/hw/nes/nes_cm.c +++ b/drivers/infiniband/hw/nes/nes_cm.c @@ -34,7 +34,7 @@ #define TCPOPT_TIMESTAMP 8 -#include +#include #include #include #include diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h index 7b6985a2e652..b3cc1e062b17 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib.h +++ b/drivers/infiniband/ulp/ipoib/ipoib.h @@ -45,7 +45,7 @@ #include -#include +#include #include #include diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index 7d5109bbd1ad..0bfa545675b8 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -39,7 +39,7 @@ #include #include -#include +#include #include #include diff --git a/drivers/isdn/gigaset/gigaset.h b/drivers/isdn/gigaset/gigaset.h index 6dd360734cfd..212efaf9a4e4 100644 --- a/drivers/isdn/gigaset/gigaset.h +++ b/drivers/isdn/gigaset/gigaset.h @@ -34,7 +34,7 @@ #include #include #include -#include +#include #define GIG_VERSION {0, 5, 0, 0} #define GIG_COMPAT {0, 4, 0, 0} diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index c8827ffd85bb..bae6c4e23d3f 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 819e37eaaeba..320401dec104 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -10,7 +10,7 @@ */ #include -#include +#include #include #include #include diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index aa4e570c2cb5..c3547016f0f1 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #define DM_MSG_PREFIX "multipath" #define MESG_STR(x) x, sizeof(x) diff --git a/drivers/md/dm-queue-length.c b/drivers/md/dm-queue-length.c index f92b6cea9d9c..03a837aa5ce6 100644 --- a/drivers/md/dm-queue-length.c +++ b/drivers/md/dm-queue-length.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #define DM_MSG_PREFIX "multipath queue-length" #define QL_MIN_IO 128 diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 451c3bb176d2..bfe9c2333cea 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #define DM_MSG_PREFIX "table" diff --git a/drivers/media/video/hdpvr/hdpvr-core.c b/drivers/media/video/hdpvr/hdpvr-core.c index a27d93b503a5..5f1db46beb4e 100644 --- a/drivers/media/video/hdpvr/hdpvr-core.c +++ b/drivers/media/video/hdpvr/hdpvr-core.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/media/video/tlg2300/pd-dvb.c b/drivers/media/video/tlg2300/pd-dvb.c index edd78f8b1baa..d0da11ae19df 100644 --- a/drivers/media/video/tlg2300/pd-dvb.c +++ b/drivers/media/video/tlg2300/pd-dvb.c @@ -7,7 +7,7 @@ #include "vendorcmds.h" #include -#include +#include static void dvb_urb_cleanup(struct pd_dvb_adapter *pd_dvb); diff --git a/drivers/media/video/uvc/uvc_ctrl.c b/drivers/media/video/uvc/uvc_ctrl.c index a4db26fa2f53..2c8954ec6859 100644 --- a/drivers/media/video/uvc/uvc_ctrl.c +++ b/drivers/media/video/uvc/uvc_ctrl.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include "uvcvideo.h" diff --git a/drivers/media/video/uvc/uvc_queue.c b/drivers/media/video/uvc/uvc_queue.c index f90ce9fce539..677691c44500 100644 --- a/drivers/media/video/uvc/uvc_queue.c +++ b/drivers/media/video/uvc/uvc_queue.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include "uvcvideo.h" diff --git a/drivers/media/video/uvc/uvc_v4l2.c b/drivers/media/video/uvc/uvc_v4l2.c index 543a80395b7f..dde6533e8e6d 100644 --- a/drivers/media/video/uvc/uvc_v4l2.c +++ b/drivers/media/video/uvc/uvc_v4l2.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/media/video/uvc/uvc_video.c b/drivers/media/video/uvc/uvc_video.c index 49994793cc77..8244167c8915 100644 --- a/drivers/media/video/uvc/uvc_video.c +++ b/drivers/media/video/uvc/uvc_video.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/message/i2o/i2o_scsi.c b/drivers/message/i2o/i2o_scsi.c index 74fbe56321ff..c8ed7b63fdf5 100644 --- a/drivers/message/i2o/i2o_scsi.c +++ b/drivers/message/i2o/i2o_scsi.c @@ -59,7 +59,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/misc/phantom.c b/drivers/misc/phantom.c index b05db55c8c8e..21b28fc6d912 100644 --- a/drivers/misc/phantom.c +++ b/drivers/misc/phantom.c @@ -26,7 +26,7 @@ #include #include -#include +#include #include #define PHANTOM_VERSION "n0.9.8" diff --git a/drivers/net/atlx/atl1.c b/drivers/net/atlx/atl1.c index 6f0e9403004b..97e6954304ea 100644 --- a/drivers/net/atlx/atl1.c +++ b/drivers/net/atlx/atl1.c @@ -44,7 +44,7 @@ * SMP torture testing */ -#include +#include #include #include diff --git a/drivers/net/atlx/atl2.c b/drivers/net/atlx/atl2.c index e0f87cf1e2ba..d4f7dda39721 100644 --- a/drivers/net/atlx/atl2.c +++ b/drivers/net/atlx/atl2.c @@ -20,7 +20,7 @@ * Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ -#include +#include #include #include #include diff --git a/drivers/net/atlx/atl2.h b/drivers/net/atlx/atl2.h index 78344ddf4bf0..bf9016ebdd9b 100644 --- a/drivers/net/atlx/atl2.h +++ b/drivers/net/atlx/atl2.h @@ -25,7 +25,7 @@ #ifndef _ATL2_H_ #define _ATL2_H_ -#include +#include #include #ifndef _ATL2_HW_H_ diff --git a/drivers/net/cassini.c b/drivers/net/cassini.c index b414f5ae0da5..646c86bcc545 100644 --- a/drivers/net/cassini.c +++ b/drivers/net/cassini.c @@ -98,7 +98,7 @@ #include -#include +#include #include #include #include diff --git a/drivers/net/cpmac.c b/drivers/net/cpmac.c index 086ce0418b29..e0638cb4b07c 100644 --- a/drivers/net/cpmac.c +++ b/drivers/net/cpmac.c @@ -40,7 +40,7 @@ #include #include #include -#include +#include MODULE_AUTHOR("Eugene Konev "); MODULE_DESCRIPTION("TI AR7 ethernet driver (CPMAC)"); diff --git a/drivers/net/cxgb3/cxgb3_offload.c b/drivers/net/cxgb3/cxgb3_offload.c index 32636a1d62a5..805076c54f1b 100644 --- a/drivers/net/cxgb3/cxgb3_offload.c +++ b/drivers/net/cxgb3/cxgb3_offload.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/net/cxgb3/l2t.h b/drivers/net/cxgb3/l2t.h index fd3eb07e3f40..7a12d52ed4fc 100644 --- a/drivers/net/cxgb3/l2t.h +++ b/drivers/net/cxgb3/l2t.h @@ -34,7 +34,7 @@ #include #include "t3cdev.h" -#include +#include enum { L2T_STATE_VALID, /* entry is up to date */ diff --git a/drivers/net/cxgb3/t3cdev.h b/drivers/net/cxgb3/t3cdev.h index be55e9ae74d1..705713b56636 100644 --- a/drivers/net/cxgb3/t3cdev.h +++ b/drivers/net/cxgb3/t3cdev.h @@ -33,7 +33,7 @@ #define _T3CDEV_H_ #include -#include +#include #include #include #include diff --git a/drivers/net/cxgb4/cxgb4_uld.h b/drivers/net/cxgb4/cxgb4_uld.h index 1b48c0170145..b1d39b8d141a 100644 --- a/drivers/net/cxgb4/cxgb4_uld.h +++ b/drivers/net/cxgb4/cxgb4_uld.h @@ -38,7 +38,7 @@ #include #include #include -#include +#include /* CPL message priority levels */ enum { diff --git a/drivers/net/cxgb4/l2t.h b/drivers/net/cxgb4/l2t.h index 7bd8f42378ff..02b31d0c6410 100644 --- a/drivers/net/cxgb4/l2t.h +++ b/drivers/net/cxgb4/l2t.h @@ -37,7 +37,7 @@ #include #include -#include +#include struct adapter; struct l2t_data; diff --git a/drivers/net/hamradio/6pack.c b/drivers/net/hamradio/6pack.c index 0d283781bc5e..2a5a34d2d67b 100644 --- a/drivers/net/hamradio/6pack.c +++ b/drivers/net/hamradio/6pack.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #define SIXPACK_VERSION "Revision: 0.3.0" diff --git a/drivers/net/hamradio/dmascc.c b/drivers/net/hamradio/dmascc.c index 52b14256e2c0..ce555d9ac02c 100644 --- a/drivers/net/hamradio/dmascc.c +++ b/drivers/net/hamradio/dmascc.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/net/ibmveth.c b/drivers/net/ibmveth.c index 838c5b673767..ba99af05bf62 100644 --- a/drivers/net/ibmveth.c +++ b/drivers/net/ibmveth.c @@ -43,7 +43,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index a47595760751..3cbda0851f83 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -33,7 +33,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/net/ppp_generic.c b/drivers/net/ppp_generic.c index 4609bc0e2f56..10e5d985afa3 100644 --- a/drivers/net/ppp_generic.c +++ b/drivers/net/ppp_generic.c @@ -48,7 +48,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/net/wimax/i2400m/i2400m.h b/drivers/net/wimax/i2400m/i2400m.h index 5eacc653a94d..c421a6141854 100644 --- a/drivers/net/wimax/i2400m/i2400m.h +++ b/drivers/net/wimax/i2400m/i2400m.h @@ -155,7 +155,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/net/wireless/b43legacy/b43legacy.h b/drivers/net/wireless/b43legacy/b43legacy.h index 17a130d18dc9..a610a352102a 100644 --- a/drivers/net/wireless/b43legacy/b43legacy.h +++ b/drivers/net/wireless/b43legacy/b43legacy.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/net/wireless/b43legacy/dma.h b/drivers/net/wireless/b43legacy/dma.h index f89c34226288..686941c242fc 100644 --- a/drivers/net/wireless/b43legacy/dma.h +++ b/drivers/net/wireless/b43legacy/dma.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include "b43legacy.h" diff --git a/drivers/oprofile/oprofile_stats.h b/drivers/oprofile/oprofile_stats.h index 0b54e46c3c14..38b6fc028984 100644 --- a/drivers/oprofile/oprofile_stats.h +++ b/drivers/oprofile/oprofile_stats.h @@ -10,7 +10,7 @@ #ifndef OPROFILE_STATS_H #define OPROFILE_STATS_H -#include +#include struct oprofile_stat_struct { atomic_t sample_lost_no_mm; diff --git a/drivers/pci/hotplug/cpci_hotplug_core.c b/drivers/pci/hotplug/cpci_hotplug_core.c index d703e73fffa7..3fadf2f135e8 100644 --- a/drivers/pci/hotplug/cpci_hotplug_core.c +++ b/drivers/pci/hotplug/cpci_hotplug_core.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include #include "cpci_hotplug.h" diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c index 492b7d807fe8..6fa215a38615 100644 --- a/drivers/pci/xen-pcifront.c +++ b/drivers/pci/xen-pcifront.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/s390/block/dasd_eer.c b/drivers/s390/block/dasd_eer.c index 77f778b7b070..16c5208c3dc7 100644 --- a/drivers/s390/block/dasd_eer.c +++ b/drivers/s390/block/dasd_eer.c @@ -21,7 +21,7 @@ #include #include -#include +#include #include #include "dasd_int.h" diff --git a/drivers/s390/char/sclp_quiesce.c b/drivers/s390/char/sclp_quiesce.c index 05909a7df8b3..a90a02c28d6a 100644 --- a/drivers/s390/char/sclp_quiesce.c +++ b/drivers/s390/char/sclp_quiesce.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/s390/char/vmlogrdr.c b/drivers/s390/char/vmlogrdr.c index c837d7419a6a..524d988d89dd 100644 --- a/drivers/s390/char/vmlogrdr.c +++ b/drivers/s390/char/vmlogrdr.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/s390/cio/device.h b/drivers/s390/cio/device.h index 7e297c7bb5ff..0b7245c72d5e 100644 --- a/drivers/s390/cio/device.h +++ b/drivers/s390/cio/device.h @@ -2,7 +2,7 @@ #define S390_DEVICE_H #include -#include +#include #include #include #include "io_sch.h" diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index 570d4da10696..e58169c32474 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/s390/cio/qdio_thinint.c b/drivers/s390/cio/qdio_thinint.c index 68be6e157126..2a1d4dfaf859 100644 --- a/drivers/s390/cio/qdio_thinint.c +++ b/drivers/s390/cio/qdio_thinint.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index f8134a44cefa..b77ae519d79c 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -41,7 +41,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 8e65447f76b7..88ad33ed5d38 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/s390/crypto/zcrypt_cex2a.c b/drivers/s390/crypto/zcrypt_cex2a.c index 2176d00b395e..da171b5f3996 100644 --- a/drivers/s390/crypto/zcrypt_cex2a.c +++ b/drivers/s390/crypto/zcrypt_cex2a.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include "ap_bus.h" diff --git a/drivers/s390/crypto/zcrypt_mono.c b/drivers/s390/crypto/zcrypt_mono.c index 44253fdd4136..eb313c3fb2d1 100644 --- a/drivers/s390/crypto/zcrypt_mono.c +++ b/drivers/s390/crypto/zcrypt_mono.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include "ap_bus.h" diff --git a/drivers/s390/crypto/zcrypt_pcica.c b/drivers/s390/crypto/zcrypt_pcica.c index 1afb69c75fea..d84816f144df 100644 --- a/drivers/s390/crypto/zcrypt_pcica.c +++ b/drivers/s390/crypto/zcrypt_pcica.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include "ap_bus.h" diff --git a/drivers/s390/crypto/zcrypt_pcicc.c b/drivers/s390/crypto/zcrypt_pcicc.c index aa4c050a5694..bdbdbe192993 100644 --- a/drivers/s390/crypto/zcrypt_pcicc.c +++ b/drivers/s390/crypto/zcrypt_pcicc.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include "ap_bus.h" diff --git a/drivers/s390/crypto/zcrypt_pcixcc.c b/drivers/s390/crypto/zcrypt_pcixcc.c index 4f85eb725f4f..dd4737808e06 100644 --- a/drivers/s390/crypto/zcrypt_pcixcc.c +++ b/drivers/s390/crypto/zcrypt_pcixcc.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include "ap_bus.h" diff --git a/drivers/s390/net/fsm.h b/drivers/s390/net/fsm.h index 1e8b235d95b5..a4510cf59034 100644 --- a/drivers/s390/net/fsm.h +++ b/drivers/s390/net/fsm.h @@ -8,7 +8,7 @@ #include #include #include -#include +#include /** * Define this to get debugging messages. diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c index 2a4991d6d4d5..7cac873c7383 100644 --- a/drivers/s390/scsi/zfcp_scsi.c +++ b/drivers/s390/scsi/zfcp_scsi.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include "zfcp_ext.h" #include "zfcp_dbf.h" #include "zfcp_fc.h" diff --git a/drivers/sbus/char/display7seg.c b/drivers/sbus/char/display7seg.c index 740da4465447..965a1fccd66a 100644 --- a/drivers/sbus/char/display7seg.c +++ b/drivers/sbus/char/display7seg.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include /* put_/get_user */ #include diff --git a/drivers/scsi/dpt/dpti_i2o.h b/drivers/scsi/dpt/dpti_i2o.h index 179ad77f6cc9..bd9e31e16249 100644 --- a/drivers/scsi/dpt/dpti_i2o.h +++ b/drivers/scsi/dpt/dpti_i2o.h @@ -22,7 +22,7 @@ #include #include -#include +#include /* diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index 6bba23a26303..c6f99b1d2383 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -46,7 +46,7 @@ #include #include #include -#include +#include #include #include "hpsa_cmd.h" #include "hpsa.h" diff --git a/drivers/scsi/pm8001/pm8001_sas.h b/drivers/scsi/pm8001/pm8001_sas.h index aa05e661d113..b97c8ab0c20e 100644 --- a/drivers/scsi/pm8001/pm8001_sas.h +++ b/drivers/scsi/pm8001/pm8001_sas.h @@ -54,7 +54,7 @@ #include #include #include -#include +#include #include "pm8001_defs.h" #define DRV_NAME "pm8001" diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c index 0f22f0f47446..1a7c19ae766f 100644 --- a/drivers/staging/octeon/ethernet-rx.c +++ b/drivers/staging/octeon/ethernet-rx.c @@ -42,7 +42,7 @@ #include #endif /* CONFIG_XFRM */ -#include +#include #include diff --git a/drivers/staging/octeon/ethernet-tx.c b/drivers/staging/octeon/ethernet-tx.c index 6227571149f5..b445cd63f901 100644 --- a/drivers/staging/octeon/ethernet-tx.c +++ b/drivers/staging/octeon/ethernet-tx.c @@ -38,7 +38,7 @@ #include #endif /* CONFIG_XFRM */ -#include +#include #include diff --git a/drivers/staging/solo6x10/solo6x10.h b/drivers/staging/solo6x10/solo6x10.h index fd59b093dd4d..17c06bd6cc91 100644 --- a/drivers/staging/solo6x10/solo6x10.h +++ b/drivers/staging/solo6x10/solo6x10.h @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/staging/tidspbridge/include/dspbridge/host_os.h b/drivers/staging/tidspbridge/include/dspbridge/host_os.h index 1a38896f4331..a2f31c69d12e 100644 --- a/drivers/staging/tidspbridge/include/dspbridge/host_os.h +++ b/drivers/staging/tidspbridge/include/dspbridge/host_os.h @@ -18,7 +18,7 @@ #define _HOST_OS_H_ #include -#include +#include #include #include #include diff --git a/drivers/staging/winbond/mds_s.h b/drivers/staging/winbond/mds_s.h index eeedf0186365..07d835b3b706 100644 --- a/drivers/staging/winbond/mds_s.h +++ b/drivers/staging/winbond/mds_s.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include "localpara.h" #include "mac_structures.h" diff --git a/drivers/staging/winbond/wb35reg_s.h b/drivers/staging/winbond/wb35reg_s.h index eb274ffdd1ba..dc79faa4029f 100644 --- a/drivers/staging/winbond/wb35reg_s.h +++ b/drivers/staging/winbond/wb35reg_s.h @@ -3,7 +3,7 @@ #include #include -#include +#include struct hw_data; diff --git a/drivers/tty/bfin_jtag_comm.c b/drivers/tty/bfin_jtag_comm.c index 03c285bb2f18..3a997760ec32 100644 --- a/drivers/tty/bfin_jtag_comm.c +++ b/drivers/tty/bfin_jtag_comm.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #define pr_init(fmt, args...) ({ static const __initconst char __fmt[] = fmt; printk(__fmt, ## args); }) diff --git a/drivers/tty/rocket.c b/drivers/tty/rocket.c index 13043e8d37fe..6a1241c7f841 100644 --- a/drivers/tty/rocket.c +++ b/drivers/tty/rocket.c @@ -83,7 +83,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/tty/serial/dz.c b/drivers/tty/serial/dz.c index 57421d776329..ddc487a2d42f 100644 --- a/drivers/tty/serial/dz.c +++ b/drivers/tty/serial/dz.c @@ -48,7 +48,7 @@ #include #include -#include +#include #include #include #include diff --git a/drivers/tty/serial/sb1250-duart.c b/drivers/tty/serial/sb1250-duart.c index ea2340b814e9..6bc2e3f876f4 100644 --- a/drivers/tty/serial/sb1250-duart.c +++ b/drivers/tty/serial/sb1250-duart.c @@ -39,7 +39,7 @@ #include #include -#include +#include #include #include diff --git a/drivers/tty/serial/zs.c b/drivers/tty/serial/zs.c index 1a7fd3e70315..0aebd7121b56 100644 --- a/drivers/tty/serial/zs.c +++ b/drivers/tty/serial/zs.c @@ -65,7 +65,7 @@ #include #include -#include +#include #include #include diff --git a/drivers/usb/gadget/f_audio.c b/drivers/usb/gadget/f_audio.c index 02a02700b51d..a9a4eade7e80 100644 --- a/drivers/usb/gadget/f_audio.c +++ b/drivers/usb/gadget/f_audio.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include "u_audio.h" diff --git a/drivers/usb/gadget/f_rndis.c b/drivers/usb/gadget/f_rndis.c index 8f3eae90919f..3ea4666be3d0 100644 --- a/drivers/usb/gadget/f_rndis.c +++ b/drivers/usb/gadget/f_rndis.c @@ -29,7 +29,7 @@ #include #include -#include +#include #include "u_ether.h" #include "rndis.h" diff --git a/drivers/usb/gadget/uvc_queue.c b/drivers/usb/gadget/uvc_queue.c index f7395ac5dc17..aa0ad34e0f1f 100644 --- a/drivers/usb/gadget/uvc_queue.c +++ b/drivers/usb/gadget/uvc_queue.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include "uvc.h" diff --git a/drivers/usb/image/microtek.c b/drivers/usb/image/microtek.c index a0037961e5bd..27e209a7222f 100644 --- a/drivers/usb/image/microtek.c +++ b/drivers/usb/image/microtek.c @@ -131,7 +131,7 @@ #include #include -#include +#include #include #include "../../scsi/scsi.h" #include diff --git a/drivers/usb/misc/appledisplay.c b/drivers/usb/misc/appledisplay.c index 68ab460a735c..ac0d75a9005a 100644 --- a/drivers/usb/misc/appledisplay.c +++ b/drivers/usb/misc/appledisplay.c @@ -29,7 +29,7 @@ #include #include #include -#include +#include #define APPLE_VENDOR_ID 0x05AC diff --git a/drivers/usb/serial/garmin_gps.c b/drivers/usb/serial/garmin_gps.c index b0a7a9e909a4..1a49ca9c8ea5 100644 --- a/drivers/usb/serial/garmin_gps.c +++ b/drivers/usb/serial/garmin_gps.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/usb/wusbcore/wa-rpipe.c b/drivers/usb/wusbcore/wa-rpipe.c index ca80171f42c6..2acc7f504c51 100644 --- a/drivers/usb/wusbcore/wa-rpipe.c +++ b/drivers/usb/wusbcore/wa-rpipe.c @@ -58,7 +58,7 @@ * destination address. */ #include -#include +#include #include #include diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 14c9abf0d800..a801e2821d03 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include /* This is for zerocopy, used buffer len is set to 1 when lower device DMA * done */ diff --git a/drivers/video/sh_mobile_lcdcfb.c b/drivers/video/sh_mobile_lcdcfb.c index 019dbd3f12b2..b048417247e8 100644 --- a/drivers/video/sh_mobile_lcdcfb.c +++ b/drivers/video/sh_mobile_lcdcfb.c @@ -24,7 +24,7 @@ #include #include #include