diff options
| -rw-r--r-- | Documentation/trace/ftrace.txt | 9 | ||||
| -rw-r--r-- | Documentation/trace/histograms.txt | 112 | ||||
| -rw-r--r-- | drivers/gpu/drm/r128/r128_cce.c | 18 | ||||
| -rw-r--r-- | drivers/gpu/drm/r128/r128_drv.h | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/r128/r128_state.c | 36 | ||||
| -rw-r--r-- | drivers/misc/Kconfig | 2 | ||||
| -rw-r--r-- | drivers/misc/hwlat_detector.c | 12 | ||||
| -rw-r--r-- | fs/pipe.c | 41 | ||||
| -rw-r--r-- | include/linux/sched.h | 3 | ||||
| -rw-r--r-- | include/trace/events/hist.h | 29 | ||||
| -rw-r--r-- | init/main.c | 13 | ||||
| -rw-r--r-- | kernel/hrtimer.c | 10 | ||||
| -rw-r--r-- | kernel/trace/Kconfig | 65 | ||||
| -rw-r--r-- | kernel/trace/Makefile | 1 | ||||
| -rw-r--r-- | kernel/trace/latency_hist.c | 460 | ||||
| -rw-r--r-- | net/sched/cls_api.c | 2 | ||||
| -rw-r--r-- | net/unix/af_unix.c | 2 | ||||
| -rw-r--r-- | security/keys/keyctl.c | 2 |
18 files changed, 589 insertions, 236 deletions
diff --git a/Documentation/trace/ftrace.txt b/Documentation/trace/ftrace.txt index 355d0f1f8c50..a4fcad7b423c 100644 --- a/Documentation/trace/ftrace.txt +++ b/Documentation/trace/ftrace.txt @@ -111,9 +111,14 @@ of ftrace. Here is a list of some of the key files: For example, the time interrupts are disabled. This time is saved in this file. The max trace will also be stored, and displayed by "trace". - A new max trace will only be recorded if the + A new max trace will only be recorded, if the latency is greater than the value in this - file. (in microseconds) + file (in microseconds). Note that the max latency + recorded by the wakeup and the wakeup_rt tracer + do not necessarily reflect the worst-case latency + of the system, but may be erroneously high in + case two or more processes share the maximum + priority of the system. buffer_size_kb: diff --git a/Documentation/trace/histograms.txt b/Documentation/trace/histograms.txt index 664505717219..2f179673710a 100644 --- a/Documentation/trace/histograms.txt +++ b/Documentation/trace/histograms.txt @@ -24,26 +24,43 @@ histograms of potential sources of latency, the kernel stores the time stamp at the start of a critical section, determines the time elapsed when the end of the section is reached, and increments the frequency counter of that latency value - irrespective of whether any concurrently -running process is affected by latency or not. +running process is affected by this latency or not. - Configuration items (in the Kernel hacking/Tracers submenu) - CONFIG_INTERRUPT_OFF_LATENCY - CONFIG_PREEMPT_OFF_LATENCY + CONFIG_INTERRUPT_OFF_HIST + CONFIG_PREEMPT_OFF_HIST * Effective latencies - -Effective latencies are actually occuring during wakeup of a process. To -determine effective latencies, the kernel stores the time stamp when a -process is scheduled to be woken up, and determines the duration of the -wakeup time shortly before control is passed over to this process. Note -that the apparent latency in user space may be considerably longer, -since -i) interrupts may be disabled preventing the scheduler from initiating -the wakeup mechanism, and +There are two types of effective latencies, wakeup latencies and missed +timer latencies + +* Wakeup latencies +Wakeup latencies may occur during wakeup of a process. To determine +wakeup latencies, the kernel stores the time stamp when a process is +scheduled to be woken up, and determines the duration of the wakeup time +shortly before control is passed over to this process. Note that the +apparent latency in user space may be considerably longer, since +i) interrupts may be disabled preventing the timer from waking up a process +in time ii) the process may be interrupted after control is passed over to it but before user space execution takes place. +If a particular wakeup latency is highest so far, details of the task +that is suffering from this latency are stored as well (see below). +- Configuration item (in the Kernel hacking/Tracers submenu) + CONFIG_WAKEUP_LATENCY_HIST + +* Missed timer latencies + +Missed timer latencies occur when a timer interrupt is serviced later +than it should. This is mainly due to disabled interrupts. To determine +the missed timer latency, the expected and the real execution time of a +timer are compared. If the former precedes the latter, the difference is +entered into the missed timer offsets histogram. If the timer is +responsible to wakeup a sleeping process and the latency is highest so +far among previous wakeup timers, details of the related task are stored +as well (see below). - Configuration item (in the Kernel hacking/Tracers submenu) - CONFIG_WAKEUP_LATENCY + CONFIG_MISSED_TIMER_OFFSETS_HIST * Usage @@ -59,30 +76,36 @@ from shell command line level, or add nodev /sys sysfs defaults 0 0 nodev /sys/kernel/debug debugfs defaults 0 0 -to the file /etc/fstab. All latency histogram related files are +to the file /etc/fstab in order to implicitly mount the debug file +system at every reboot. All latency histogram related files are available in the directory /sys/kernel/debug/tracing/latency_hist. A particular histogram type is enabled by writing non-zero to the related variable in the /sys/kernel/debug/tracing/latency_hist/enable directory. -Select "preemptirqsoff" for the histograms of potential sources of -latencies and "wakeup" for histograms of effective latencies. The -histogram data - one per CPU - are available in the files +Select "preemptirqsoff" for histograms of potential sources of +latencies, "wakeup" for histograms of wakeup latencies and +"missed_timer_offsets" for histograms of missed timer offsets, +respectively. +The histogram data - one per CPU - are available in the files /sys/kernel/debug/tracing/latency_hist/preemptoff/CPUx /sys/kernel/debug/tracing/latency_hist/irqsoff/CPUx /sys/kernel/debug/tracing/latency_hist/preemptirqsoff/CPUx /sys/kernel/debug/tracing/latency_hist/wakeup/CPUx. +/sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio/CPUx. +/sys/kernel/debug/tracing/latency_hist/missed_timer_offsets/CPUx. The histograms are reset by writing non-zero to the file "reset" in a particular latency directory. To reset all latency data, use -#!/bin/sh +#!/bin/bash -HISTDIR=/sys/kernel/debug/tracing/latency_hist +TRACINGDIR=/sys/kernel/debug/tracing +HISTDIR=$TRACINGDIR/latency_hist if test -d $HISTDIR then cd $HISTDIR - for i in */reset + for i in `find . | grep /reset$` do echo 1 >$i done @@ -92,19 +115,19 @@ fi * Data format Latency data are stored with a resolution of one microsecond. The -maximum latency is 10,240 microseconds. The data are only valid, if the -overflow register is empty. Every output line contains the latency in -microseconds in the first row and the number of samples in the second -row. To display only lines with a positive latency count, use, for -example, +maximum latency is 10,240 microseconds. Every output line contains the +latency in microseconds in the first row and the number of samples in +the second row. To display only lines with a positive latency count, +use, for example, grep -v " 0$" /sys/kernel/debug/tracing/latency_hist/preemptoff/CPU0 -#Minimum latency: 0 microseconds. -#Average latency: 0 microseconds. -#Maximum latency: 25 microseconds. +#Minimum latency: 0 microseconds +#Average latency: 0 microseconds +#Maximum latency: 25 microseconds #Total samples: 3104770694 -#There are 0 samples greater or equal than 10240 microseconds +#There are 0 samples lower than 0 microseconds. +#There are 0 samples greater or equal than 10240 microseconds. #usecs samples 0 2984486876 1 49843506 @@ -133,6 +156,23 @@ grep -v " 0$" /sys/kernel/debug/tracing/latency_hist/preemptoff/CPU0 25 1 +* Two types of wakeup latency histograms + +Two different algorithms are used to determine the wakeup latency of a +process. One of them only considers processes that exclusively use the +highest priority of the system, the other one records the wakeup latency +of a process even if it shares the highest system latency with other +processes. The former is used to determine the worst-case latency of a +system; if higher than expected, the hardware and or system software +(e.g. the Linux kernel) may need to be debugged and fixed. The latter +reflects the priority design of a given system; if higher than expected, +the system design may need to be re-evaluated - the hardware +manufacturer or the kernel developers must not be blamed for such +latencies. The exclusive-priority wakeup latency histograms are located +in the "wakeup" subdirectory, the shared-priority histograms are located +in the "wakeup/sharedprio" subdirectory. + + * Wakeup latency of a selected process To only collect wakeup latency data of a particular process, write the @@ -143,14 +183,18 @@ PID of the requested process to PIDs are not considered, if this variable is set to 0. -* Details of the process with the highest wakeup latency so far +* Details of processes with the highest wakeup or missed timer +latency so far -Selected data of the process that suffered from the highest wakeup -latency that occurred in a particular CPU are available in the file +Selected data of processes that suffered from the highest wakeup or +missed timer latency that occurred on a particular CPU are available in +the files -/sys/kernel/debug/tracing/latency_hist/wakeup/max_latency-CPUx. +/sys/kernel/debug/tracing/latency_hist/wakeup/max_latency-CPUx +/sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio/max_latency-CPUx +/sys/kernel/debug/tracing/latency_hist/missed_timer_offsets/max_latency-CPUx The format of the data is <PID> <Priority> <Latency> <Command> -These data are also reset when the wakeup histogram ist reset. +These data are also reset when the related histograms are reset. diff --git a/drivers/gpu/drm/r128/r128_cce.c b/drivers/gpu/drm/r128/r128_cce.c index c75fd3564040..ebf9f63e5d49 100644 --- a/drivers/gpu/drm/r128/r128_cce.c +++ b/drivers/gpu/drm/r128/r128_cce.c @@ -353,6 +353,11 @@ static int r128_do_init_cce(struct drm_device * dev, drm_r128_init_t * init) DRM_DEBUG("\n"); + if (dev->dev_private) { + DRM_DEBUG("called when already initialized\n"); + return -EINVAL; + } + dev_priv = kzalloc(sizeof(drm_r128_private_t), GFP_KERNEL); if (dev_priv == NULL) return -ENOMEM; @@ -649,6 +654,8 @@ int r128_cce_start(struct drm_device *dev, void *data, struct drm_file *file_pri LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev_priv); + if (dev_priv->cce_running || dev_priv->cce_mode == R128_PM4_NONPM4) { DRM_DEBUG("while CCE running\n"); return 0; @@ -671,6 +678,8 @@ int r128_cce_stop(struct drm_device *dev, void *data, struct drm_file *file_priv LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev_priv); + /* Flush any pending CCE commands. This ensures any outstanding * commands are exectuted by the engine before we turn it off. */ @@ -708,10 +717,7 @@ int r128_cce_reset(struct drm_device *dev, void *data, struct drm_file *file_pri LOCK_TEST_WITH_RETURN(dev, file_priv); - if (!dev_priv) { - DRM_DEBUG("called before init done\n"); - return -EINVAL; - } + DEV_INIT_TEST_WITH_RETURN(dev_priv); r128_do_cce_reset(dev_priv); @@ -728,6 +734,8 @@ int r128_cce_idle(struct drm_device *dev, void *data, struct drm_file *file_priv LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev_priv); + if (dev_priv->cce_running) { r128_do_cce_flush(dev_priv); } @@ -741,6 +749,8 @@ int r128_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_ LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev->dev_private); + return r128_do_engine_reset(dev); } diff --git a/drivers/gpu/drm/r128/r128_drv.h b/drivers/gpu/drm/r128/r128_drv.h index 797a26c42dab..3c60829d82e9 100644 --- a/drivers/gpu/drm/r128/r128_drv.h +++ b/drivers/gpu/drm/r128/r128_drv.h @@ -422,6 +422,14 @@ static __inline__ void r128_update_ring_snapshot(drm_r128_private_t * dev_priv) * Misc helper macros */ +#define DEV_INIT_TEST_WITH_RETURN(_dev_priv) \ +do { \ + if (!_dev_priv) { \ + DRM_ERROR("called with no initialization\n"); \ + return -EINVAL; \ + } \ +} while (0) + #define RING_SPACE_TEST_WITH_RETURN( dev_priv ) \ do { \ drm_r128_ring_buffer_t *ring = &dev_priv->ring; int i; \ diff --git a/drivers/gpu/drm/r128/r128_state.c b/drivers/gpu/drm/r128/r128_state.c index 026a48c95c8f..af2665cf4718 100644 --- a/drivers/gpu/drm/r128/r128_state.c +++ b/drivers/gpu/drm/r128/r128_state.c @@ -1244,14 +1244,18 @@ static void r128_cce_dispatch_stipple(struct drm_device * dev, u32 * stipple) static int r128_cce_clear(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_r128_private_t *dev_priv = dev->dev_private; - drm_r128_sarea_t *sarea_priv = dev_priv->sarea_priv; + drm_r128_sarea_t *sarea_priv; drm_r128_clear_t *clear = data; DRM_DEBUG("\n"); LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev_priv); + RING_SPACE_TEST_WITH_RETURN(dev_priv); + sarea_priv = dev_priv->sarea_priv; + if (sarea_priv->nbox > R128_NR_SAREA_CLIPRECTS) sarea_priv->nbox = R128_NR_SAREA_CLIPRECTS; @@ -1312,6 +1316,8 @@ static int r128_cce_flip(struct drm_device *dev, void *data, struct drm_file *fi LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev_priv); + RING_SPACE_TEST_WITH_RETURN(dev_priv); if (!dev_priv->page_flipping) @@ -1331,6 +1337,8 @@ static int r128_cce_swap(struct drm_device *dev, void *data, struct drm_file *fi LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev_priv); + RING_SPACE_TEST_WITH_RETURN(dev_priv); if (sarea_priv->nbox > R128_NR_SAREA_CLIPRECTS) @@ -1354,10 +1362,7 @@ static int r128_cce_vertex(struct drm_device *dev, void *data, struct drm_file * LOCK_TEST_WITH_RETURN(dev, file_priv); - if (!dev_priv) { - DRM_ERROR("called with no initialization\n"); - return -EINVAL; - } + DEV_INIT_TEST_WITH_RETURN(dev_priv); DRM_DEBUG("pid=%d index=%d count=%d discard=%d\n", DRM_CURRENTPID, vertex->idx, vertex->count, vertex->discard); @@ -1410,10 +1415,7 @@ static int r128_cce_indices(struct drm_device *dev, void *data, struct drm_file LOCK_TEST_WITH_RETURN(dev, file_priv); - if (!dev_priv) { - DRM_ERROR("called with no initialization\n"); - return -EINVAL; - } + DEV_INIT_TEST_WITH_RETURN(dev_priv); DRM_DEBUG("pid=%d buf=%d s=%d e=%d d=%d\n", DRM_CURRENTPID, elts->idx, elts->start, elts->end, elts->discard); @@ -1476,6 +1478,8 @@ static int r128_cce_blit(struct drm_device *dev, void *data, struct drm_file *fi LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev_priv); + DRM_DEBUG("pid=%d index=%d\n", DRM_CURRENTPID, blit->idx); if (blit->idx < 0 || blit->idx >= dma->buf_count) { @@ -1501,6 +1505,8 @@ static int r128_cce_depth(struct drm_device *dev, void *data, struct drm_file *f LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev_priv); + RING_SPACE_TEST_WITH_RETURN(dev_priv); ret = -EINVAL; @@ -1531,6 +1537,8 @@ static int r128_cce_stipple(struct drm_device *dev, void *data, struct drm_file LOCK_TEST_WITH_RETURN(dev, file_priv); + DEV_INIT_TEST_WITH_RETURN(dev_priv); + if (DRM_COPY_FROM_USER(&mask, stipple->mask, 32 * sizeof(u32))) return -EFAULT; @@ -1555,10 +1563,7 @@ static int r128_cce_indirect(struct drm_device *dev, void *data, struct drm_file LOCK_TEST_WITH_RETURN(dev, file_priv); - if (!dev_priv) { - DRM_ERROR("called with no initialization\n"); - return -EINVAL; - } + DEV_INIT_TEST_WITH_RETURN(dev_priv); DRM_DEBUG("idx=%d s=%d e=%d d=%d\n", indirect->idx, indirect->start, indirect->end, @@ -1620,10 +1625,7 @@ static int r128_getparam(struct drm_device *dev, void *data, struct drm_file *fi drm_r128_getparam_t *param = data; int value; - if (!dev_priv) { - DRM_ERROR("called with no initialization\n"); - return -EINVAL; - } + DEV_INIT_TEST_WITH_RETURN(dev_priv); DRM_DEBUG("pid=%d\n", DRM_CURRENTPID); diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index f51ba7b08f9b..d16b79fcc8fd 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -79,7 +79,7 @@ config IBM_ASM config HWLAT_DETECTOR tristate "Testing module to detect hardware-induced latencies" depends on DEBUG_FS - select RING_BUFFER + depends on RING_BUFFER default m ---help--- A simple hardware latency detector. Use this module to detect diff --git a/drivers/misc/hwlat_detector.c b/drivers/misc/hwlat_detector.c index be6553f2ddfc..d9549e9b58f4 100644 --- a/drivers/misc/hwlat_detector.c +++ b/drivers/misc/hwlat_detector.c @@ -607,7 +607,11 @@ static ssize_t debug_enable_fwrite(struct file *filp, if (!enabled) goto unlock; enabled = 0; - stop_kthread(); + err = stop_kthread(); + if (err) { + printk(KERN_ERR BANNER "cannot stop kthread\n"); + return -EFAULT; + } wake_up(&data.wq); /* reader(s) should return */ } unlock: @@ -1194,9 +1198,13 @@ out: */ static void detector_exit(void) { + int err; + if (enabled) { enabled = 0; - stop_kthread(); + err = stop_kthread(); + if (err) + printk(KERN_ERR BANNER "cannot stop kthread\n"); } free_debugfs(); diff --git a/fs/pipe.c b/fs/pipe.c index 7f30ed2a148e..c085c8f87050 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -789,36 +789,55 @@ pipe_rdwr_release(struct inode *inode, struct file *filp) static int pipe_read_open(struct inode *inode, struct file *filp) { - /* We could have perhaps used atomic_t, but this and friends - below are the only places. So it doesn't seem worthwhile. */ + int ret = -ENOENT; + mutex_lock(&inode->i_mutex); - inode->i_pipe->readers++; + + if (inode->i_pipe) { + ret = 0; + inode->i_pipe->readers++; + } + mutex_unlock(&inode->i_mutex); - return 0; + return ret; } static int pipe_write_open(struct inode *inode, struct file *filp) { + int ret = -ENOENT; + mutex_lock(&inode->i_mutex); - inode->i_pipe->writers++; + + if (inode->i_pipe) { + ret = 0; + inode->i_pipe->writers++; + } + mutex_unlock(&inode->i_mutex); - return 0; + return ret; } static int pipe_rdwr_open(struct inode *inode, struct file *filp) { + int ret = -ENOENT; + mutex_lock(&inode->i_mutex); - if (filp->f_mode & FMODE_READ) - inode->i_pipe->readers++; - if (filp->f_mode & FMODE_WRITE) - inode->i_pipe->writers++; + + if (inode->i_pipe) { + ret = 0; + if (filp->f_mode & FMODE_READ) + inode->i_pipe->readers++; + if (filp->f_mode & FMODE_WRITE) + inode->i_pipe->writers++; + } + mutex_unlock(&inode->i_mutex); - return 0; + return ret; } /* diff --git a/include/linux/sched.h b/include/linux/sched.h index c231a2467a83..676126d56894 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1548,6 +1548,9 @@ struct task_struct { unsigned long trace; /* bitmask of trace recursion */ unsigned long trace_recursion; +#ifdef CONFIG_WAKEUP_LATENCY_HIST + u64 preempt_timestamp_hist; +#endif #endif /* CONFIG_TRACING */ #ifdef CONFIG_PREEMPT_RT /* diff --git a/include/trace/events/hist.h b/include/trace/events/hist.h index 73b0454eb90f..2dc4c70760e1 100644 --- a/include/trace/events/hist.h +++ b/include/trace/events/hist.h @@ -17,8 +17,8 @@ TRACE_EVENT(preemptirqsoff_hist, TP_ARGS(reason, starthist), TP_STRUCT__entry( - __field( int, reason ) - __field( int, starthist ) + __field(int, reason ) + __field(int, starthist ) ), TP_fast_assign( @@ -31,6 +31,31 @@ TRACE_EVENT(preemptirqsoff_hist, ); #endif +#ifndef CONFIG_MISSED_TIMER_OFFSETS_HIST +#define trace_hrtimer_interrupt(a,b,c) +#else +TRACE_EVENT(hrtimer_interrupt, + + TP_PROTO(int cpu, long long offset, struct task_struct *task), + + TP_ARGS(cpu, offset, task), + + TP_STRUCT__entry( + __array(char, comm, TASK_COMM_LEN) + __field(int, cpu ) + __field(long long, offset ) + ), + + TP_fast_assign( + strncpy(__entry->comm, task != NULL ? task->comm : "", TASK_COMM_LEN); + __entry->cpu = cpu; + __entry->offset = offset; + ), + + TP_printk("cpu=%d offset=%lld thread=%s", __entry->cpu, __entry->offset, __entry->comm) +); +#endif + #endif /* _TRACE_HIST_H */ /* This part must be outside protection */ diff --git a/init/main.c b/init/main.c index bdbdaab0469e..ff438b198fe3 100644 --- a/init/main.c +++ b/init/main.c @@ -936,7 +936,15 @@ static int __init kernel_init(void * unused) WARN_ON(irqs_disabled()); #endif -#define DEBUG_COUNT (defined(CONFIG_DEBUG_RT_MUTEXES) + defined(CONFIG_IRQSOFF_TRACER) + defined(CONFIG_PREEMPT_TRACER) + defined(CONFIG_STACK_TRACER) + defined(CONFIG_INTERRUPT_OFF_HIST) + defined(CONFIG_PREEMPT_OFF_HIST) + defined(CONFIG_WAKEUP_LATENCY_HIST) + defined(CONFIG_DEBUG_SLAB) + defined(CONFIG_DEBUG_PAGEALLOC) + defined(CONFIG_LOCKDEP) + (defined(CONFIG_FTRACE) - defined(CONFIG_FTRACE_MCOUNT_RECORD))) +#define DEBUG_COUNT (defined(CONFIG_DEBUG_RT_MUTEXES) + \ + defined(CONFIG_IRQSOFF_TRACER) + defined(CONFIG_PREEMPT_TRACER) + \ + defined(CONFIG_STACK_TRACER) + defined(CONFIG_INTERRUPT_OFF_HIST) + \ + defined(CONFIG_PREEMPT_OFF_HIST) + \ + defined(CONFIG_WAKEUP_LATENCY_HIST) + \ + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) + \ + defined(CONFIG_DEBUG_SLAB) + defined(CONFIG_DEBUG_PAGEALLOC) + \ + defined(CONFIG_LOCKDEP) + \ + (defined(CONFIG_FTRACE) - defined(CONFIG_FTRACE_MCOUNT_RECORD))) #if DEBUG_COUNT > 0 printk(KERN_ERR "*****************************************************************************\n"); @@ -968,6 +976,9 @@ static int __init kernel_init(void * unused) #ifdef CONFIG_WAKEUP_LATENCY_HIST printk(KERN_ERR "* CONFIG_WAKEUP_LATENCY_HIST *\n"); #endif +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST + printk(KERN_ERR "* CONFIG_MISSED_TIMER_OFFSETS_HIST *\n"); +#endif #ifdef CONFIG_DEBUG_SLAB printk(KERN_ERR "* CONFIG_DEBUG_SLAB *\n"); #endif diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 875327a43ad4..1840ba8c8283 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -48,6 +48,8 @@ #include <asm/uaccess.h> +#include <trace/events/hist.h> + /* * The timer bases: * @@ -1349,6 +1351,7 @@ static inline int hrtimer_rt_defer(struct hrtimer *timer) { return 0; } #ifdef CONFIG_HIGH_RES_TIMERS static int force_clock_reprogram; +static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer); /* * After 5 iteration's attempts, we consider that hrtimer_interrupt() @@ -1419,6 +1422,13 @@ void hrtimer_interrupt(struct clock_event_device *dev) timer = rb_entry(node, struct hrtimer, node); + trace_hrtimer_interrupt(raw_smp_processor_id(), + ktime_to_ns(ktime_sub( + hrtimer_get_expires(timer), basenow)), + timer->function == hrtimer_wakeup ? + container_of(timer, struct hrtimer_sleeper, + timer)->task : NULL); + /* * The immediate goal for using the softexpires is * minimizing wakeups, not running timers at the diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index fea2f14a5d62..471c5f19f573 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -143,7 +143,6 @@ config FUNCTION_GRAPH_TRACER the return value. This is done by setting the current return address on the current task structure into a stack of calls. - config IRQSOFF_TRACER bool "Interrupts-off Latency Tracer" default n @@ -171,16 +170,19 @@ config INTERRUPT_OFF_HIST bool "Interrupts-off Latency Histogram" depends on IRQSOFF_TRACER help - This option generates a continuously updated histogram (one per cpu) + This option generates continuously updated histograms (one per cpu) of the duration of time periods with interrupts disabled. The - histogram is disabled by default. To enable it, write a non-zero - number to the related file in + histograms are disabled by default. To enable them, write a non-zero + number to /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff - If PREEMPT_OFF_HIST is also selected, an additional histogram (one - per cpu) is generated that accumulates the duration of time periods - when both interrupts and preemption are disabled. + If PREEMPT_OFF_HIST is also selected, additional histograms (one + per cpu) are generated that accumulate the duration of time periods + when both interrupts and preemption are disabled. The histogram data + will be located in the debug file system at + + /sys/kernel/debug/tracing/latency_hist/irqsoff config PREEMPT_TRACER bool "Preemption-off Latency Tracer" @@ -208,16 +210,19 @@ config PREEMPT_OFF_HIST bool "Preemption-off Latency Histogram" depends on PREEMPT_TRACER help - This option generates a continuously updated histogram (one per cpu) + This option generates continuously updated histograms (one per cpu) of the duration of time periods with preemption disabled. The - histogram is disabled by default. To enable it, write a non-zero + histograms are disabled by default. To enable them, write a non-zero number to /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff - If INTERRUPT_OFF_HIST is also selected, an additional histogram (one - per cpu) is generated that accumulates the duration of time periods - when both interrupts and preemption are disabled. + If INTERRUPT_OFF_HIST is also selected, additional histograms (one + per cpu) are generated that accumulate the duration of time periods + when both interrupts and preemption are disabled. The histogram data + will be located in the debug file system at + + /sys/kernel/debug/tracing/latency_hist/preemptoff config SCHED_TRACER bool "Scheduling Latency Tracer" @@ -232,12 +237,42 @@ config WAKEUP_LATENCY_HIST bool "Scheduling Latency Histogram" depends on SCHED_TRACER help - This option generates a continuously updated histogram (one per cpu) - of the scheduling latency of the highest priority task. The histogram - is disabled by default. To enable it, write a non-zero number to + This option generates continuously updated histograms (one per cpu) + of the scheduling latency of the highest priority task. + The histograms are disabled by default. To enable them, write a + non-zero number to /sys/kernel/debug/tracing/latency_hist/enable/wakeup + Two different algorithms are used, one to determine the latency of + processes that exclusively use the highest priority of the system and + another one to determine the latency of processes that share the + highest system priority with other processes. The former is used to + improve hardware and system software, the latter to optimize the + priority design of a given system. The histogram data will be + located in the debug file system at + + /sys/kernel/debug/tracing/latency_hist/wakeup + + and + + /sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio + +config MISSED_TIMER_OFFSETS_HIST + depends on GENERIC_TIME + select GENERIC_TRACER + bool "Missed timer offsets histogram" + help + Generate a histogram of missed timer offsets in microseconds. The + histograms are disabled by default. To enable them, write a non-zero + number to + + /sys/kernel/debug/tracing/latency_hist/enable/missed_timer_offsets + + The histogram data will be located in the debug file system at + + /sys/kernel/debug/tracing/latency_hist/missed_timer_offsets + config SYSPROF_TRACER bool "Sysprof Tracer" depends on X86 diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index c71519bb516e..8136881c59ac 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -38,6 +38,7 @@ obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o obj-$(CONFIG_INTERRUPT_OFF_HIST) += latency_hist.o obj-$(CONFIG_PREEMPT_OFF_HIST) += latency_hist.o obj-$(CONFIG_WAKEUP_LATENCY_HIST) += latency_hist.o +obj-$(CONFIG_MISSED_TIMER_OFFSETS_HIST) += latency_hist.o obj-$(CONFIG_NOP_TRACER) += trace_nop.o obj-$(CONFIG_STACK_TRACER) += trace_stack.o obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o diff --git a/kernel/trace/latency_hist.c b/kernel/trace/latency_hist.c index c83463e9b37e..8705501ed408 100644 --- a/kernel/trace/latency_hist.c +++ b/kernel/trace/latency_hist.c @@ -34,6 +34,8 @@ enum { PREEMPTOFF_LATENCY, PREEMPTIRQSOFF_LATENCY, WAKEUP_LATENCY, + WAKEUP_LATENCY_SHAREDPRIO, + MISSED_TIMER_OFFSETS, MAX_LATENCY_TYPE, }; @@ -41,9 +43,11 @@ enum { struct hist_data { atomic_t hist_mode; /* 0 log, 1 don't log */ + long offset; /* set it to MAX_ENTRY_NUM/2 for a bipolar scale */ unsigned long min_lat; unsigned long max_lat; - unsigned long long beyond_hist_bound_samples; + unsigned long long below_hist_bound_samples; + unsigned long long above_hist_bound_samples; unsigned long long accumulate_lat; unsigned long long total_samples; unsigned long long hist_array[MAX_ENTRY_NUM]; @@ -59,16 +63,22 @@ static char *latency_hist_dir_root = "latency_hist"; #ifdef CONFIG_INTERRUPT_OFF_HIST static DEFINE_PER_CPU(struct hist_data, irqsoff_hist); static char *irqsoff_hist_dir = "irqsoff"; +static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start); +static DEFINE_PER_CPU(int, hist_irqsoff_counting); #endif #ifdef CONFIG_PREEMPT_OFF_HIST static DEFINE_PER_CPU(struct hist_data, preemptoff_hist); static char *preemptoff_hist_dir = "preemptoff"; +static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start); +static DEFINE_PER_CPU(int, hist_preemptoff_counting); #endif #if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) static DEFINE_PER_CPU(struct hist_data, preemptirqsoff_hist); static char *preemptirqsoff_hist_dir = "preemptirqsoff"; +static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start); +static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting); #endif #if defined(CONFIG_PREEMPT_OFF_HIST) || defined(CONFIG_INTERRUPT_OFF_HIST) @@ -79,9 +89,21 @@ static struct enable_data preemptirqsoff_enabled_data = { }; #endif +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) +struct maxlatproc_data { + char comm[FIELD_SIZEOF(struct task_struct, comm)]; + unsigned int pid; + unsigned int prio; + unsigned long latency; +}; +#endif + #ifdef CONFIG_WAKEUP_LATENCY_HIST static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist); +static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist_sharedprio); static char *wakeup_latency_hist_dir = "wakeup"; +static char *wakeup_latency_hist_dir_sharedprio = "sharedprio"; static notrace void probe_wakeup_latency_hist_start(struct rq *rq, struct task_struct *p, int success); static notrace void probe_wakeup_latency_hist_stop(struct rq *rq, @@ -90,23 +112,34 @@ static struct enable_data wakeup_latency_enabled_data = { .latency_type = WAKEUP_LATENCY, .enabled = 0, }; -static struct task_struct *ts; -struct maxlatproc_data { - char comm[sizeof(ts->comm)]; - unsigned int pid; - unsigned int prio; - unsigned long latency; -}; static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc); -static unsigned wakeup_prio = (unsigned)-1; +static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc_sharedprio); static struct task_struct *wakeup_task; +static int wakeup_sharedprio; static int wakeup_pid; #endif +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST +static DEFINE_PER_CPU(struct hist_data, missed_timer_offsets); +static char *missed_timer_offsets_dir = "missed_timer_offsets"; +static notrace void probe_hrtimer_interrupt(int cpu, + long long offset, struct task_struct *task); +static struct enable_data missed_timer_offsets_enabled_data = { + .latency_type = MISSED_TIMER_OFFSETS, + .enabled = 0, +}; +static DEFINE_PER_CPU(struct maxlatproc_data, + missed_timer_offsets_maxlatproc); +#endif + void notrace latency_hist(int latency_type, int cpu, unsigned long latency, struct task_struct *p) { struct hist_data *my_hist; +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) + struct maxlatproc_data *mp = NULL; +#endif if (cpu < 0 || cpu >= NR_CPUS || latency_type < 0 || latency_type >= MAX_LATENCY_TYPE) @@ -118,22 +151,30 @@ void notrace latency_hist(int latency_type, int cpu, unsigned long latency, my_hist = &per_cpu(irqsoff_hist, cpu); break; #endif - #ifdef CONFIG_PREEMPT_OFF_HIST case PREEMPTOFF_LATENCY: my_hist = &per_cpu(preemptoff_hist, cpu); break; #endif - #if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) case PREEMPTIRQSOFF_LATENCY: my_hist = &per_cpu(preemptirqsoff_hist, cpu); break; #endif - #ifdef CONFIG_WAKEUP_LATENCY_HIST case WAKEUP_LATENCY: my_hist = &per_cpu(wakeup_latency_hist, cpu); + mp = &per_cpu(wakeup_maxlatproc, cpu); + break; + case WAKEUP_LATENCY_SHAREDPRIO: + my_hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); + mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); + break; +#endif +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST + case MISSED_TIMER_OFFSETS: + my_hist = &per_cpu(missed_timer_offsets, cpu); + mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); break; #endif default: @@ -143,18 +184,23 @@ void notrace latency_hist(int latency_type, int cpu, unsigned long latency, if (atomic_read(&my_hist->hist_mode) == 0) return; - if (latency >= MAX_ENTRY_NUM) - my_hist->beyond_hist_bound_samples++; - else + latency += my_hist->offset; + + if (latency < 0 || latency >= MAX_ENTRY_NUM) { + if (latency < 0) + my_hist->below_hist_bound_samples++; + else + my_hist->above_hist_bound_samples++; + } else my_hist->hist_array[latency]++; if (latency < my_hist->min_lat) my_hist->min_lat = latency; else if (latency > my_hist->max_lat) { -#ifdef CONFIG_WAKEUP_LATENCY_HIST - if (latency_type == WAKEUP_LATENCY) { - struct maxlatproc_data *mp = - &per_cpu(wakeup_maxlatproc, cpu); +#if defined (CONFIG_WAKEUP_LATENCY_HIST) || \ + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) + if (latency_type == WAKEUP_LATENCY || + latency_type == WAKEUP_LATENCY_SHAREDPRIO) { strncpy(mp->comm, p->comm, sizeof(mp->comm)); mp->pid = task_pid_nr(p); mp->prio = p->prio; @@ -163,7 +209,15 @@ void notrace latency_hist(int latency_type, int cpu, unsigned long latency, #endif my_hist->max_lat = latency; } - +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST + if (latency_type == MISSED_TIMER_OFFSETS && p != NULL && + latency > mp->latency) { + strncpy(mp->comm, p->comm, sizeof(mp->comm)); + mp->pid = task_pid_nr(p); + mp->prio = p->prio; + mp->latency = latency; + } +#endif my_hist->total_samples++; my_hist->accumulate_lat += latency; return; @@ -179,30 +233,42 @@ static void *l_start(struct seq_file *m, loff_t *pos) return NULL; if (index == 0) { - char avgstr[32]; + char minstr[32], avgstr[32], maxstr[32]; atomic_dec(&my_hist->hist_mode); + if (likely(my_hist->total_samples)) { unsigned long avg = (unsigned long) div64_u64(my_hist->accumulate_lat, my_hist->total_samples); - sprintf(avgstr, "%lu", avg); - } else - strcpy(avgstr, "<undef>"); + snprintf(minstr, sizeof(minstr), "%ld", + (long) my_hist->min_lat - my_hist->offset); + snprintf(avgstr, sizeof(avgstr), "%ld", + (long) avg - my_hist->offset); + snprintf(maxstr, sizeof(minstr), "%ld", + (long) my_hist->max_lat - my_hist->offset); + } else { + strcpy(minstr, "<undef>"); + strcpy(avgstr, minstr); + strcpy(maxstr, minstr); + } - seq_printf(m, "#Minimum latency: %lu microseconds.\n" - "#Average latency: %s microseconds.\n" - "#Maximum latency: %lu microseconds.\n" + seq_printf(m, "#Minimum latency: %s microseconds\n" + "#Average latency: %s microseconds\n" + "#Maximum latency: %s microseconds\n" "#Total samples: %llu\n" + "#There are %llu samples lower than %ld" + " microseconds.\n" "#There are %llu samples greater or equal" - " than %d microseconds\n" - "#usecs\t%16s\n" - , my_hist->min_lat - , avgstr - , my_hist->max_lat - , my_hist->total_samples - , my_hist->beyond_hist_bound_samples - , MAX_ENTRY_NUM, "samples"); + " than %ld microseconds.\n" + "#usecs\t%16s\n", + minstr, avgstr, maxstr, + my_hist->total_samples, + my_hist->below_hist_bound_samples, + -my_hist->offset, + my_hist->above_hist_bound_samples, + MAX_ENTRY_NUM - my_hist->offset, + "samples"); } if (index >= MAX_ENTRY_NUM) return NULL; @@ -233,8 +299,15 @@ static int l_show(struct seq_file *m, void *p) { int index = *(loff_t *) p; struct hist_data *my_hist = m->private; + char *fmt; - seq_printf(m, "%5d\t%16llu\n", index, my_hist->hist_array[index]); + if (my_hist->offset) + fmt = "%6d\t%16llu\n"; + else + fmt = "%5d\t%16llu\n"; + + seq_printf(m, fmt, index - my_hist->offset, + my_hist->hist_array[index]); return 0; } @@ -269,7 +342,8 @@ static void hist_reset(struct hist_data *hist) atomic_dec(&hist->hist_mode); memset(hist->hist_array, 0, sizeof(hist->hist_array)); - hist->beyond_hist_bound_samples = 0ULL; + hist->below_hist_bound_samples = 0ULL; + hist->above_hist_bound_samples = 0ULL; hist->min_lat = 0xFFFFFFFFUL; hist->max_lat = 0UL; hist->total_samples = 0ULL; @@ -284,10 +358,13 @@ latency_hist_reset(struct file *file, const char __user *a, { int cpu; struct hist_data *hist; +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) + struct maxlatproc_data *mp = NULL; +#endif int latency_type = (int) file->private_data; switch (latency_type) { - #ifdef CONFIG_PREEMPT_OFF_HIST case PREEMPTOFF_LATENCY: for_each_online_cpu(cpu) { @@ -296,7 +373,6 @@ latency_hist_reset(struct file *file, const char __user *a, } break; #endif - #ifdef CONFIG_INTERRUPT_OFF_HIST case IRQSOFF_LATENCY: for_each_online_cpu(cpu) { @@ -305,7 +381,6 @@ latency_hist_reset(struct file *file, const char __user *a, } break; #endif - #if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) case PREEMPTIRQSOFF_LATENCY: for_each_online_cpu(cpu) { @@ -314,16 +389,34 @@ latency_hist_reset(struct file *file, const char __user *a, } break; #endif - #ifdef CONFIG_WAKEUP_LATENCY_HIST case WAKEUP_LATENCY: for_each_online_cpu(cpu) { - struct maxlatproc_data *mp = - &per_cpu(wakeup_maxlatproc, cpu); + hist = &per_cpu(wakeup_latency_hist, cpu); + hist_reset(hist); + mp = &per_cpu(wakeup_maxlatproc, cpu); mp->comm[0] = '\0'; mp->prio = mp->pid = mp->latency = 0; - hist = &per_cpu(wakeup_latency_hist, cpu); + } + break; + case WAKEUP_LATENCY_SHAREDPRIO: + for_each_online_cpu(cpu) { + hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); hist_reset(hist); + mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); + mp->comm[0] = '\0'; + mp->prio = mp->pid = mp->latency = 0; + } + break; +#endif +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST + case MISSED_TIMER_OFFSETS: + for_each_online_cpu(cpu) { + hist = &per_cpu(missed_timer_offsets, cpu); + hist_reset(hist); + mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); + mp->comm[0] = '\0'; + mp->prio = mp->pid = mp->latency = 0; } break; #endif @@ -341,8 +434,6 @@ latency_hist_show_pid(struct file *filp, char __user *ubuf, int r; r = snprintf(buf, sizeof(buf), "%u\n", wakeup_pid); - if (r > sizeof(buf)) - r = sizeof(buf); return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); } @@ -367,7 +458,10 @@ latency_hist_pid(struct file *filp, const char __user *ubuf, wakeup_pid = pid; return cnt; } +#endif +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) static ssize_t latency_hist_show_maxlatproc(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) @@ -379,27 +473,11 @@ latency_hist_show_maxlatproc(struct file *filp, char __user *ubuf, r = snprintf(buf, sizeof(buf), "%5d %3d %ld %s\n", mp->pid, mp->prio, mp->latency, mp->comm); - if (r > sizeof(buf)) - r = sizeof(buf); return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); } #endif -#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) -#ifdef CONFIG_INTERRUPT_OFF_HIST -static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start); -static DEFINE_PER_CPU(int, hist_irqsoff_counting); -#endif -#ifdef CONFIG_PREEMPT_OFF_HIST -static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start); -static DEFINE_PER_CPU(int, hist_preemptoff_counting); -#endif -#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start); -static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting); -#endif - static ssize_t latency_hist_show_enable(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) @@ -409,8 +487,6 @@ latency_hist_show_enable(struct file *filp, char __user *ubuf, int r; r = snprintf(buf, sizeof(buf), "%d\n", ed->enabled); - if (r > sizeof(buf)) - r = sizeof(buf); return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); } @@ -440,6 +516,19 @@ latency_hist_enable(struct file *filp, const char __user *ubuf, int ret; switch (ed->latency_type) { +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) + case PREEMPTIRQSOFF_LATENCY: + ret = register_trace_preemptirqsoff_hist( + probe_preemptirqsoff_hist); + if (ret) { + pr_info("wakeup trace: Couldn't assign " + "probe_preemptirqsoff_hist " + "to trace_preemptirqsoff_hist\n"); + return ret; + } + break; +#endif +#ifdef CONFIG_WAKEUP_LATENCY_HIST case WAKEUP_LATENCY: ret = register_trace_sched_wakeup( probe_wakeup_latency_hist_start); @@ -472,23 +561,49 @@ latency_hist_enable(struct file *filp, const char __user *ubuf, return ret; } break; - case PREEMPTIRQSOFF_LATENCY: - ret = register_trace_preemptirqsoff_hist( - probe_preemptirqsoff_hist); +#endif +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST + case MISSED_TIMER_OFFSETS: + ret = register_trace_hrtimer_interrupt( + probe_hrtimer_interrupt); if (ret) { pr_info("wakeup trace: Couldn't assign " - "probe_preemptirqsoff_hist " - "to trace_preemptirqsoff_hist\n"); + "probe_hrtimer_interrupt " + "to trace_hrtimer_interrupt\n"); return ret; } break; +#endif default: break; } } else { - int cpu; - switch (ed->latency_type) { +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) + case PREEMPTIRQSOFF_LATENCY: + { + int cpu; + + unregister_trace_preemptirqsoff_hist( + probe_preemptirqsoff_hist); + for_each_online_cpu(cpu) { +#ifdef CONFIG_INTERRUPT_OFF_HIST + per_cpu(hist_irqsoff_counting, + cpu) = 0; +#endif +#ifdef CONFIG_PREEMPT_OFF_HIST + per_cpu(hist_preemptoff_counting, + cpu) = 0; +#endif +#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) + per_cpu(hist_preemptirqsoff_counting, + cpu) = 0; +#endif + } + } + break; +#endif +#ifdef CONFIG_WAKEUP_LATEHCY_HIST case WAKEUP_LATENCY: unregister_trace_sched_wakeup( probe_wakeup_latency_hist_start); @@ -497,23 +612,15 @@ latency_hist_enable(struct file *filp, const char __user *ubuf, unregister_trace_sched_switch( probe_wakeup_latency_hist_stop); wakeup_task = NULL; - wakeup_prio = (unsigned)-1; + wakeup_sharedprio = 0; break; - case PREEMPTIRQSOFF_LATENCY: - unregister_trace_preemptirqsoff_hist( - probe_preemptirqsoff_hist); - for_each_online_cpu(cpu) { -#ifdef CONFIG_INTERRUPT_OFF_HIST - per_cpu(hist_irqsoff_counting, cpu) = 0; -#endif -#ifdef CONFIG_PREEMPT_OFF_HIST - per_cpu(hist_preemptoff_counting, cpu) = 0; -#endif -#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) - per_cpu(hist_preemptirqsoff_counting, cpu) = 0; #endif - } +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST + case MISSED_TIMER_OFFSETS: + unregister_trace_hrtimer_interrupt( + probe_hrtimer_interrupt); break; +#endif default: break; } @@ -522,28 +629,34 @@ latency_hist_enable(struct file *filp, const char __user *ubuf, return cnt; } -static struct file_operations latency_hist_reset_fops = { +static const struct file_operations latency_hist_reset_fops = { .open = tracing_open_generic, .write = latency_hist_reset, }; -static struct file_operations latency_hist_pid_fops = { +static const struct file_operations latency_hist_enable_fops = { .open = tracing_open_generic, - .read = latency_hist_show_pid, - .write = latency_hist_pid, + .read = latency_hist_show_enable, + .write = latency_hist_enable, }; -static struct file_operations latency_hist_maxlatproc_fops = { +#ifdef CONFIG_WAKEUP_LATENCY_HIST +static const struct file_operations latency_hist_pid_fops = { .open = tracing_open_generic, - .read = latency_hist_show_maxlatproc, + .read = latency_hist_show_pid, + .write = latency_hist_pid, }; +#endif -static struct file_operations latency_hist_enable_fops = { +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) +static const struct file_operations latency_hist_maxlatproc_fops = { .open = tracing_open_generic, - .read = latency_hist_show_enable, - .write = latency_hist_enable, + .read = latency_hist_show_maxlatproc, }; +#endif +#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) notrace void probe_preemptirqsoff_hist(int reason, int starthist) { int cpu = raw_smp_processor_id(); @@ -643,11 +756,9 @@ notrace void probe_preemptirqsoff_hist(int reason, int starthist) #endif } } - #endif #ifdef CONFIG_WAKEUP_LATENCY_HIST -static cycle_t wakeup_start; static DEFINE_ATOMIC_SPINLOCK(wakeup_lock); notrace void probe_wakeup_latency_hist_start(struct rq *rq, @@ -657,23 +768,28 @@ notrace void probe_wakeup_latency_hist_start(struct rq *rq, struct task_struct *curr = rq_curr(rq); if (wakeup_pid) { + if ((wakeup_task && p->prio == wakeup_task->prio) || + p->prio == curr->prio) + wakeup_sharedprio = 1; if (likely(wakeup_pid != task_pid_nr(p))) return; } else { if (likely(!rt_task(p)) || - p->prio >= wakeup_prio || - p->prio >= curr->prio) + (wakeup_task && p->prio > wakeup_task->prio) || + p->prio > curr->prio) return; + if ((wakeup_task && p->prio == wakeup_task->prio) || + p->prio == curr->prio) + wakeup_sharedprio = 1; } atomic_spin_lock_irqsave(&wakeup_lock, flags); if (wakeup_task) put_task_struct(wakeup_task); - get_task_struct(p); wakeup_task = p; - wakeup_prio = p->prio; - wakeup_start = ftrace_now(raw_smp_processor_id()); + wakeup_task->preempt_timestamp_hist = + ftrace_now(raw_smp_processor_id()); atomic_spin_unlock_irqrestore(&wakeup_lock, flags); } @@ -692,31 +808,59 @@ notrace void probe_wakeup_latency_hist_stop(struct rq *rq, stop = ftrace_now(cpu); atomic_spin_lock_irqsave(&wakeup_lock, flags); - if (next != wakeup_task) + + latency = nsecs_to_usecs(stop - next->preempt_timestamp_hist); + if (next != wakeup_task) { + if (wakeup_task && next->prio == wakeup_task->prio) + latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency, + next); goto out; + } - latency = nsecs_to_usecs(stop - wakeup_start); - latency_hist(WAKEUP_LATENCY, cpu, latency, next); + if (wakeup_sharedprio) { + latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency, next); + wakeup_sharedprio = 0; + } else + latency_hist(WAKEUP_LATENCY, cpu, latency, next); put_task_struct(wakeup_task); wakeup_task = NULL; - wakeup_prio = (unsigned)-1; out: atomic_spin_unlock_irqrestore(&wakeup_lock, flags); } +#endif + +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST +notrace void probe_hrtimer_interrupt(int cpu, long long latency_ns, + struct task_struct *task) +{ + if (latency_ns <= 0) { + unsigned long latency; + + latency = (unsigned long) div_s64(-latency_ns, 1000); + latency_hist(MISSED_TIMER_OFFSETS, cpu, latency, task); + } +} #endif static __init int latency_hist_init(void) { struct dentry *latency_hist_root = NULL; struct dentry *dentry; +#ifdef CONFIG_WAKEUP_LATENCY_HIST + struct dentry *dentry_sharedprio; +#endif struct dentry *entry; struct dentry *latency_hist_enable_root; - int i = 0, len = 0; + int i = 0; struct hist_data *my_hist; char name[64]; char *cpufmt = "CPU%d"; +#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ + defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) + char *cpufmt_maxlatproc = "max_latency-CPU%d"; +#endif dentry = tracing_init_dentry(); @@ -729,92 +873,118 @@ static __init int latency_hist_init(void) #ifdef CONFIG_INTERRUPT_OFF_HIST dentry = debugfs_create_dir(irqsoff_hist_dir, latency_hist_root); for_each_possible_cpu(i) { - len = sprintf(name, cpufmt, i); - name[len] = '\0'; + sprintf(name, cpufmt, i); entry = debugfs_create_file(name, 0444, dentry, - &per_cpu(irqsoff_hist, i), - &latency_hist_fops); + &per_cpu(irqsoff_hist, i), &latency_hist_fops); my_hist = &per_cpu(irqsoff_hist, i); atomic_set(&my_hist->hist_mode, 1); my_hist->min_lat = 0xFFFFFFFFUL; } entry = debugfs_create_file("reset", 0644, dentry, - (void *)IRQSOFF_LATENCY, - &latency_hist_reset_fops); + (void *)IRQSOFF_LATENCY, &latency_hist_reset_fops); #endif #ifdef CONFIG_PREEMPT_OFF_HIST dentry = debugfs_create_dir(preemptoff_hist_dir, - latency_hist_root); + latency_hist_root); for_each_possible_cpu(i) { - len = sprintf(name, cpufmt, i); - name[len] = '\0'; + sprintf(name, cpufmt, i); entry = debugfs_create_file(name, 0444, dentry, - &per_cpu(preemptoff_hist, i), - &latency_hist_fops); + &per_cpu(preemptoff_hist, i), &latency_hist_fops); my_hist = &per_cpu(preemptoff_hist, i); atomic_set(&my_hist->hist_mode, 1); my_hist->min_lat = 0xFFFFFFFFUL; } entry = debugfs_create_file("reset", 0644, dentry, - (void *)PREEMPTOFF_LATENCY, - &latency_hist_reset_fops); + (void *)PREEMPTOFF_LATENCY, &latency_hist_reset_fops); #endif #if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) dentry = debugfs_create_dir(preemptirqsoff_hist_dir, - latency_hist_root); + latency_hist_root); for_each_possible_cpu(i) { - len = sprintf(name, cpufmt, i); - name[len] = '\0'; + sprintf(name, cpufmt, i); entry = debugfs_create_file(name, 0444, dentry, - &per_cpu(preemptirqsoff_hist, i), - &latency_hist_fops); + &per_cpu(preemptirqsoff_hist, i), &latency_hist_fops); my_hist = &per_cpu(preemptirqsoff_hist, i); atomic_set(&my_hist->hist_mode, 1); my_hist->min_lat = 0xFFFFFFFFUL; } entry = debugfs_create_file("reset", 0644, dentry, - (void *)PREEMPTIRQSOFF_LATENCY, - &latency_hist_reset_fops); + (void *)PREEMPTIRQSOFF_LATENCY, &latency_hist_reset_fops); #endif #if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) entry = debugfs_create_file("preemptirqsoff", 0644, - latency_hist_enable_root, - (void *)&preemptirqsoff_enabled_data, - &latency_hist_enable_fops); + latency_hist_enable_root, (void *)&preemptirqsoff_enabled_data, + &latency_hist_enable_fops); #endif #ifdef CONFIG_WAKEUP_LATENCY_HIST dentry = debugfs_create_dir(wakeup_latency_hist_dir, - latency_hist_root); + latency_hist_root); + dentry_sharedprio = debugfs_create_dir( + wakeup_latency_hist_dir_sharedprio, dentry); for_each_possible_cpu(i) { - len = sprintf(name, cpufmt, i); - name[len] = '\0'; + sprintf(name, cpufmt, i); entry = debugfs_create_file(name, 0444, dentry, - &per_cpu(wakeup_latency_hist, i), - &latency_hist_fops); + &per_cpu(wakeup_latency_hist, i), + &latency_hist_fops); my_hist = &per_cpu(wakeup_latency_hist, i); atomic_set(&my_hist->hist_mode, 1); my_hist->min_lat = 0xFFFFFFFFUL; - len = sprintf(name, "max_latency-CPU%d", i); - name[len] = '\0'; + sprintf(name, cpufmt, i); + entry = debugfs_create_file(name, 0444, dentry_sharedprio, + &per_cpu(wakeup_latency_hist_sharedprio, i), + &latency_hist_fops); + my_hist = &per_cpu(wakeup_latency_hist_sharedprio, i); + atomic_set(&my_hist->hist_mode, 1); + my_hist->min_lat = 0xFFFFFFFFUL; + + sprintf(name, cpufmt_maxlatproc, i); entry = debugfs_create_file(name, 0444, dentry, - &per_cpu(wakeup_maxlatproc, i), - &latency_hist_maxlatproc_fops); + &per_cpu(wakeup_maxlatproc, i), + &latency_hist_maxlatproc_fops); + + sprintf(name, cpufmt_maxlatproc, i); + entry = debugfs_create_file(name, 0444, dentry_sharedprio, + &per_cpu(wakeup_maxlatproc_sharedprio, i), + &latency_hist_maxlatproc_fops); } entry = debugfs_create_file("pid", 0644, dentry, - (void *)&wakeup_pid, - &latency_hist_pid_fops); + (void *)&wakeup_pid, &latency_hist_pid_fops); entry = debugfs_create_file("reset", 0644, dentry, - (void *)WAKEUP_LATENCY, - &latency_hist_reset_fops); + (void *)WAKEUP_LATENCY, &latency_hist_reset_fops); + entry = debugfs_create_file("reset", 0644, dentry_sharedprio, + (void *)WAKEUP_LATENCY_SHAREDPRIO, &latency_hist_reset_fops); entry = debugfs_create_file("wakeup", 0644, - latency_hist_enable_root, - (void *)&wakeup_latency_enabled_data, - &latency_hist_enable_fops); + latency_hist_enable_root, (void *)&wakeup_latency_enabled_data, + &latency_hist_enable_fops); +#endif + +#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST + dentry = debugfs_create_dir(missed_timer_offsets_dir, + latency_hist_root); + for_each_possible_cpu(i) { + sprintf(name, cpufmt, i); + entry = debugfs_create_file(name, 0444, dentry, + &per_cpu(missed_timer_offsets, i), &latency_hist_fops); + my_hist = &per_cpu(missed_timer_offsets, i); + atomic_set(&my_hist->hist_mode, 1); + my_hist->min_lat = 0xFFFFFFFFUL; + + sprintf(name, cpufmt_maxlatproc, i); + entry = debugfs_create_file(name, 0444, dentry, + &per_cpu(missed_timer_offsets_maxlatproc, i), + &latency_hist_maxlatproc_fops); + } + entry = debugfs_create_file("reset", 0644, dentry, + (void *)MISSED_TIMER_OFFSETS, &latency_hist_reset_fops); + entry = debugfs_create_file("missed_timer_offsets", 0644, + latency_hist_enable_root, + (void *)&missed_timer_offsets_enabled_data, + &latency_hist_enable_fops); #endif return 0; } diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 09cdcdfe7e91..b83ac88a1448 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -348,7 +348,7 @@ static int tcf_fill_node(struct sk_buff *skb, struct tcf_proto *tp, tcm = NLMSG_DATA(nlh); tcm->tcm_family = AF_UNSPEC; tcm->tcm__pad1 = 0; - tcm->tcm__pad1 = 0; + tcm->tcm__pad2 = 0; tcm->tcm_ifindex = qdisc_dev(tp->q)->ifindex; tcm->tcm_parent = tp->classid; tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol); diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 51ab497115eb..fc820cd75453 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1074,6 +1074,8 @@ restart: err = -ECONNREFUSED; if (other->sk_state != TCP_LISTEN) goto out_unlock; + if (other->sk_shutdown & RCV_SHUTDOWN) + goto out_unlock; if (unix_recvq_full(other)) { err = -EAGAIN; diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index 7f09fb897d2b..667aecd0fe52 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -860,7 +860,7 @@ static long get_instantiation_keyring(key_serial_t ringid, /* otherwise specify the destination keyring recorded in the * authorisation key (any KEY_SPEC_*_KEYRING) */ if (ringid >= KEY_SPEC_REQUESTOR_KEYRING) { - *_dest_keyring = rka->dest_keyring; + *_dest_keyring = key_get(rka->dest_keyring); return 0; } |