diff options
Diffstat (limited to 'kernel/locking')
| -rw-r--r-- | kernel/locking/lockdep.c | 144 | ||||
| -rw-r--r-- | kernel/locking/mutex.c | 4 | ||||
| -rw-r--r-- | kernel/locking/percpu-rwsem.c | 2 | ||||
| -rw-r--r-- | kernel/locking/qspinlock.c | 22 | ||||
| -rw-r--r-- | kernel/locking/rtmutex_api.c | 2 | ||||
| -rw-r--r-- | kernel/locking/spinlock.c | 31 |
6 files changed, 180 insertions, 25 deletions
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2d4c5bab5af8..c3dc84a7cef2 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -787,17 +787,33 @@ static void lockdep_print_held_locks(struct task_struct *p) { int i, depth = READ_ONCE(p->lockdep_depth); - if (!depth) - printk("no locks held by %s/%d.\n", p->comm, task_pid_nr(p)); - else - printk("%d lock%s held by %s/%d:\n", depth, - str_plural(depth), p->comm, task_pid_nr(p)); /* - * It's not reliable to print a task's held locks if it's not sleeping - * and it's not the current task. + * Note that it's always somewhat unreliable to print held locks + * of a task that is running on another CPU, but we cannot guarantee + * the stability of ->held_locks without actually stopping all active + * remote CPUs, which we absolutely do not want to do because it's + * very intrusive and thus slow. + * + * So we do the next best thing here: we print out the held lock + * array on a best-effort basis, without crashing even if the + * fields are being modified on another CPU. Note the careful + * construction of print_lock() so that it never crashes. + * + * We also print out the CPU the task is or was last running on, with + * the message saying 'on CPU...' if the task is running, and + * 'last CPU' if it's not. + * + * Also note that the task_is_running(p) information is fundamentally + * racy: even if the message says the task is 'on CPU', the task may + * have scheduled out already, or if it says 'last CPU', it may just + * have scheduled in on another CPU. But even with these limitations + * it's still useful debuggining information. */ - if (p != current && task_is_running(p)) - return; + printk("locks held by %s/%d: %d, %s CPU#%d%s\n", + p->comm, task_pid_nr(p), depth, + task_is_running(p) ? "last" : "on", task_cpu(p), + depth > 0 ? ":" : ""); + for (i = 0; i < depth; i++) { printk(" #%d: ", i); print_lock(p->held_locks + i); @@ -947,6 +963,34 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) return NULL; } +static __always_inline bool lock_class_cache_is_valid(const struct lockdep_map *lock, + const struct lock_class *class, + unsigned int subclass) +{ + unsigned int class_subclass; + + if (!class) + return false; + + if (unlikely(class < lock_classes || class >= lock_classes + MAX_LOCKDEP_KEYS)) + return false; + + if (unlikely(!arch_test_bit(class - lock_classes, lock_classes_in_use))) + return false; + + if (unlikely(!lock->key)) + return false; + + class_subclass = subclass ? subclass : class->subclass; + if (unlikely(class_subclass >= MAX_LOCKDEP_SUBCLASSES)) + return false; + + if (unlikely(READ_ONCE(class->key) != lock->key->subkeys + class_subclass)) + return false; + + return true; +} + /* * Static locks do not have their class-keys yet - for them the key is * the lock object itself. If the lock is in the per cpu area, the @@ -1379,9 +1423,9 @@ out_unlock_set: out_set_class_cache: if (!subclass || force) - lock->class_cache[0] = class; + WRITE_ONCE(lock->class_cache[0], class); else if (subclass < NR_LOCKDEP_CACHING_CLASSES) - lock->class_cache[subclass] = class; + WRITE_ONCE(lock->class_cache[subclass], class); /* * Hash collision, did we smoke some? We found a class with a matching @@ -4941,7 +4985,7 @@ void lockdep_init_map_type(struct lockdep_map *lock, const char *name, int i; for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++) - lock->class_cache[i] = NULL; + WRITE_ONCE(lock->class_cache[i], NULL); #ifdef CONFIG_LOCK_STAT lock->cpu = raw_smp_processor_id(); @@ -5006,12 +5050,15 @@ EXPORT_SYMBOL_GPL(__lockdep_no_track__); void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn, lock_print_fn print_fn) { - struct lock_class *class = lock->class_cache[0]; + struct lock_class *class = READ_ONCE(lock->class_cache[0]); unsigned long flags; raw_local_irq_save(flags); lockdep_recursion_inc(); + if (!lock_class_cache_is_valid(lock, class, 0)) + class = NULL; + if (!class) class = register_lock_class(lock, 0, 0); @@ -5077,7 +5124,7 @@ static int __lock_is_held(const struct lockdep_map *lock, int read); static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, int trylock, int read, int check, int hardirqs_off, struct lockdep_map *nest_lock, unsigned long ip, - int references, int pin_count, int sync) + int references, int pin_count, int sync, int seq) { struct task_struct *curr = current; struct lock_class *class = NULL; @@ -5103,8 +5150,11 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, if (DEBUG_LOCKS_WARN_ON(subclass >= MAX_LOCKDEP_SUBCLASSES)) return 0; - if (subclass < NR_LOCKDEP_CACHING_CLASSES) - class = lock->class_cache[subclass]; + if (subclass < NR_LOCKDEP_CACHING_CLASSES) { + class = READ_ONCE(lock->class_cache[subclass]); + if (!lock_class_cache_is_valid(lock, class, subclass)) + class = NULL; + } /* * Not cached? */ @@ -5183,6 +5233,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, hlock->holdtime_stamp = lockstat_clock(); #endif hlock->pin_count = pin_count; + hlock->seq_count = seq; if (check_wait_context(curr, hlock)) return 0; @@ -5307,9 +5358,9 @@ static noinstr int match_held_lock(const struct held_lock *hlock, return 1; if (hlock->references) { - const struct lock_class *class = lock->class_cache[0]; + const struct lock_class *class = READ_ONCE(lock->class_cache[0]); - if (!class) + if (!lock_class_cache_is_valid(lock, class, 0)) class = look_up_lock_class(lock, 0); /* @@ -5388,7 +5439,7 @@ static int reacquire_held_locks(struct task_struct *curr, unsigned int depth, hlock->read, hlock->check, hlock->hardirqs_off, hlock->nest_lock, hlock->acquire_ip, - hlock->references, hlock->pin_count, 0)) { + hlock->references, hlock->pin_count, 0, hlock->seq_count)) { case 0: return 1; case 1: @@ -5437,6 +5488,8 @@ __lock_set_class(struct lockdep_map *lock, const char *name, lock->wait_type_outer, lock->lock_type); class = register_lock_class(lock, subclass, 0); + if (!class) + return 0; hlock->class_idx = class - lock_classes; curr->lockdep_depth = i; @@ -5669,14 +5722,17 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie struct held_lock *hlock = curr->held_locks + i; if (match_held_lock(hlock, lock)) { + int pin_count; + if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n")) return; - hlock->pin_count -= cookie.val; + pin_count = hlock->pin_count - cookie.val; - if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n")) - hlock->pin_count = 0; + if (WARN(pin_count < 0, "pin count corrupted\n")) + pin_count = 0; + hlock->pin_count = pin_count; return; } } @@ -5684,6 +5740,24 @@ static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie WARN(1, "unpinning an unheld lock\n"); } +static u32 __lock_sequence(struct lockdep_map *lock) +{ + struct task_struct *curr = current; + int i; + + if (unlikely(!debug_locks)) + return ~0; + + for (i = 0; i < curr->lockdep_depth; i++) { + struct held_lock *hlock = curr->held_locks + i; + + if (match_held_lock(hlock, lock)) + return hlock->seq_count; + } + + return ~0; +} + /* * Check whether we follow the irq-flags state precisely: */ @@ -5866,7 +5940,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass, lockdep_recursion_inc(); __lock_acquire(lock, subclass, trylock, read, check, - irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0); + irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 0, + ++current->lockdep_seq); lockdep_recursion_finish(); raw_local_irq_restore(flags); } @@ -5914,7 +5989,8 @@ void lock_sync(struct lockdep_map *lock, unsigned subclass, int read, lockdep_recursion_inc(); __lock_acquire(lock, subclass, 0, read, check, - irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1); + irqs_disabled_flags(flags), nest_lock, ip, 0, 0, 1, + ++current->lockdep_seq); check_chain_key(current); lockdep_recursion_finish(); raw_local_irq_restore(flags); @@ -6000,6 +6076,26 @@ void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie) } EXPORT_SYMBOL_GPL(lock_unpin_lock); +u32 lock_sequence(struct lockdep_map *lock) +{ + unsigned long flags; + u32 seq = ~0; + + if (unlikely(!lockdep_enabled())) + return seq; + + raw_local_irq_save(flags); + check_flags(flags); + + lockdep_recursion_inc(); + seq = __lock_sequence(lock); + lockdep_recursion_finish(); + raw_local_irq_restore(flags); + + return seq; +} +EXPORT_SYMBOL_GPL(lock_sequence); + #ifdef CONFIG_LOCK_STAT static void print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock, diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 8a85912d7ee6..942a939cee95 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -1272,6 +1272,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(contention_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(contention_end); EXPORT_TRACEPOINT_SYMBOL_GPL(contended_release); +__weak int arch_contended_release_trace_reg(void) { return 0; } + +__weak void arch_contended_release_trace_unreg(void) { } + /** * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 * @cnt: the atomic which we are to dec diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index f7e152c40d6d..6c78961fe753 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -211,7 +211,7 @@ EXPORT_SYMBOL_GPL(percpu_is_read_locked); */ static bool readers_active_check(struct percpu_rw_semaphore *sem) { - if (per_cpu_sum(*sem->read_count) != 0) + if (data_race(per_cpu_sum(*sem->read_count)) != 0) return false; /* diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c index af8d122bb649..33fe6d437c8f 100644 --- a/kernel/locking/qspinlock.c +++ b/kernel/locking/qspinlock.c @@ -104,6 +104,28 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock, #define queued_spin_lock_slowpath native_queued_spin_lock_slowpath #endif +#if !defined(queued_spin_unlock) && \ + IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE) +/* + * Out-of-line trace-and-release path for queued_spin_unlock(), used when + * the contended_release tracepoint is enabled. + * + * queued_spin_release() is duplicated here on purpose: doing the release + * in this function (rather than tracing here and releasing in the caller) + * lets queued_spin_unlock() return right after the call, so the + * tracepoint-disabled hot path never has to keep lock live across a call + * in a callee-saved register. Keep this release in sync with the one in + * queued_spin_unlock(). + */ +void __lockfunc queued_spin_release_traced(struct qspinlock *lock) +{ + if (queued_spin_is_contended(lock)) + trace_call__contended_release(lock); + queued_spin_release(lock); +} +EXPORT_SYMBOL(queued_spin_release_traced); +#endif + #endif /* _GEN_PV_LOCK_SLOWPATH */ /** diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 5d48d64725b1..eb18b094473c 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -423,6 +423,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, { int ret; + rt_mutex_futex_pre_schedule(); raw_spin_lock_irq(&lock->wait_lock); /* sleep on the mutex */ set_current_state(TASK_INTERRUPTIBLE); @@ -433,6 +434,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, */ fixup_rt_mutex_waiters(lock, true); raw_spin_unlock_irq(&lock->wait_lock); + rt_mutex_futex_post_schedule(); return ret; } diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index b42d293da38b..83a17eaf5717 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ */ BUILD_LOCK_OPS(spin, raw_spinlock, __acquires); +/* No rwlock_t variants for now, so just build this function by hand */ +static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + for (;;) { + preempt_disable(); + local_interrupt_disable(); + if (likely(do_raw_spin_trylock(lock))) + break; + local_interrupt_enable(); + preempt_enable(); + + arch_spin_relax(&lock->raw_lock); + } +} + #ifndef CONFIG_PREEMPT_RT BUILD_LOCK_OPS(read, rwlock, __acquires_shared); BUILD_LOCK_OPS(write, rwlock, __acquires); @@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) EXPORT_SYMBOL(_raw_spin_lock_irq); #endif +#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ +noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + __raw_spin_lock_irq_disable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable); +#endif + #ifndef CONFIG_INLINE_SPIN_LOCK_BH noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) { @@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) EXPORT_SYMBOL(_raw_spin_unlock_irq); #endif +#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ +noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) +{ + __raw_spin_unlock_irq_enable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable); +#endif + #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) { |