diff options
| author | Tejun Heo <tj@kernel.org> | 2026-10-07 08:01:02 -1000 |
|---|---|---|
| committer | Tejun Heo <tj@kernel.org> | 2026-10-07 08:01:02 -1000 |
| commit | 92b542db38bf7cb4de583b2ca37febc549db0ce4 (patch) | |
| tree | 30dcfdb338f7fdba597e1ec9314c2006df86f525 /kernel/time | |
| parent | fee1265a725725dac0ff7459f912d69558798b36 (diff) | |
| parent | d1a4549a3645e55cc9ade09e360280590b3c9559 (diff) | |
| download | linux-next-92b542db38bf7cb4de583b2ca37febc549db0ce4.tar.gz linux-next-92b542db38bf7cb4de583b2ca37febc549db0ce4.zip | |
Merge branch 'for-7.3-fixes' into for-next
Diffstat (limited to 'kernel/time')
| -rw-r--r-- | kernel/time/Kconfig | 1 | ||||
| -rw-r--r-- | kernel/time/clockevents.c | 33 | ||||
| -rw-r--r-- | kernel/time/clocksource.c | 9 | ||||
| -rw-r--r-- | kernel/time/hrtimer.c | 79 | ||||
| -rw-r--r-- | kernel/time/itimer.c | 2 | ||||
| -rw-r--r-- | kernel/time/jiffies.c | 37 | ||||
| -rw-r--r-- | kernel/time/namespace.c | 6 | ||||
| -rw-r--r-- | kernel/time/namespace_vdso.c | 7 | ||||
| -rw-r--r-- | kernel/time/ntp.c | 310 | ||||
| -rw-r--r-- | kernel/time/ntp_internal.h | 4 | ||||
| -rw-r--r-- | kernel/time/posix-cpu-timers.c | 222 | ||||
| -rw-r--r-- | kernel/time/tick-broadcast.c | 36 | ||||
| -rw-r--r-- | kernel/time/tick-internal.h | 3 | ||||
| -rw-r--r-- | kernel/time/tick-sched.c | 7 | ||||
| -rw-r--r-- | kernel/time/time.c | 25 | ||||
| -rw-r--r-- | kernel/time/timekeeping.c | 204 | ||||
| -rw-r--r-- | kernel/time/timekeeping.h | 2 | ||||
| -rw-r--r-- | kernel/time/timekeeping_internal.h | 21 | ||||
| -rw-r--r-- | kernel/time/timer.c | 1 | ||||
| -rw-r--r-- | kernel/time/timer_list.c | 78 | ||||
| -rw-r--r-- | kernel/time/timer_migration.c | 4 | ||||
| -rw-r--r-- | kernel/time/timer_migration.h | 2 |
22 files changed, 794 insertions, 299 deletions
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index d098ac39bde4..ddfb6bee0745 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -133,7 +133,6 @@ config NO_HZ_FULL depends on HAVE_VIRT_CPU_ACCOUNTING_GEN select NO_HZ_COMMON select RCU_NOCB_CPU - select VIRT_CPU_ACCOUNTING_GEN select IRQ_WORK select CPU_ISOLATION help diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 0014d163f989..62ad7c000386 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -615,34 +615,41 @@ void clockevents_handle_noop(struct clock_event_device *dev) { } -/** - * clockevents_exchange_device - release and request clock devices - * @old: device to release (can be NULL) - * @new: device to request (can be NULL) - * - * Called from various tick functions with clockevents_lock held and - * interrupts disabled. - */ -void clockevents_exchange_device(struct clock_event_device *old, - struct clock_event_device *new) +void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) { /* * Caller releases a clock event device. We queue it into the * released list and do a notify add later. */ if (old) { - module_put(old->owner); clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED); list_move(&old->list, &clockevents_released); } if (new) { - BUG_ON(!clockevent_state_detached(new)); + WARN_ON(!clockevent_state_detached(new)); clockevents_shutdown(new); } } /** + * clockevents_exchange_device - release and request clock devices + * @old: device to release (can be NULL) + * @new: device to request (can be NULL) + * + * Called from various tick functions with clockevents_lock held and + * interrupts disabled. + */ +void clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) +{ + __clockevents_exchange_device(old, new); + if (old) + module_put(old->owner); +} + +/** * clockevents_suspend - suspend clock devices */ void clockevents_suspend(void) @@ -699,7 +706,7 @@ void tick_offline_cpu(unsigned int cpu) if (cpumask_test_cpu(cpu, dev->cpumask) && cpumask_weight(dev->cpumask) == 1 && !tick_is_broadcast_device(dev)) { - BUG_ON(!clockevent_state_detached(dev)); + WARN_ON(!clockevent_state_detached(dev)); list_del(&dev->list); } } diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index e48c4d379a7c..f1253f5795c6 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -123,7 +123,6 @@ static atomic_t watchdog_reset_pending; /* Watchdog interval: 0.5sec. */ #define WATCHDOG_INTERVAL (HZ >> 1) -#define WATCHDOG_INTERVAL_NS (WATCHDOG_INTERVAL * (NSEC_PER_SEC / HZ)) /* Maximum time between two reference watchdog readouts */ #define WATCHDOG_READOUT_MAX_NS (50U * NSEC_PER_USEC) @@ -1566,8 +1565,12 @@ static int __init init_clocksource_sysfs(void) { int error = subsys_system_register(&clocksource_subsys, NULL); - if (!error) - error = device_register(&device_clocksource); + if (error) + return error; + + error = device_register(&device_clocksource); + if (error) + bus_unregister(&clocksource_subsys); return error; } diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 313dcea127fe..cbf1693c86b3 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -26,6 +26,7 @@ #include <linux/export.h> #include <linux/percpu.h> #include <linux/hrtimer.h> +#include <linux/hrtimer_bases.h> #include <linux/notifier.h> #include <linux/syscalls.h> #include <linux/interrupt.h> @@ -676,6 +677,8 @@ static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base) static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) { + lockdep_assert_held(&base->lock); + ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; @@ -707,7 +710,7 @@ static inline void hrtimer_rearm_event(ktime_t expires_next, bool deferred) tick_program_event(expires_next, 1); } -static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer, +static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next) { cpu_base->expires_next = expires_next; @@ -743,7 +746,7 @@ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip if (skip_equal && expires_next == cpu_base->expires_next) return; - __hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next); + __hrtimer_reprogram(cpu_base, expires_next); } /* High resolution timer related functions */ @@ -896,14 +899,14 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram) cpu_base->next_timer = timer; - __hrtimer_reprogram(cpu_base, timer, expires); + __hrtimer_reprogram(cpu_base, expires); } static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active) { struct hrtimer_clock_base *base; - unsigned int seq; ktime_t expires; + u32 seq; /* * Update the base offsets unconditionally so the following @@ -1040,6 +1043,30 @@ static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned lon } /** + * hrtimer_update_function - Update the timer's callback function + * @timer: Timer to update + * @function: New callback function + * + * Only safe to call if the timer is not enqueued. Can be called in the callback function if the + * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED). + */ +void hrtimer_update_function(struct hrtimer *timer, + enum hrtimer_restart (*function)(struct hrtimer *)) +{ +#ifdef CONFIG_PROVE_LOCKING + guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock); + + if (WARN_ON_ONCE(hrtimer_is_queued(timer))) + return; + + if (WARN_ON_ONCE(!function)) + return; +#endif + ACCESS_PRIVATE(timer, function) = function; +} +EXPORT_SYMBOL_GPL(hrtimer_update_function); + +/** * hrtimer_forward() - forward the timer expiry * @timer: hrtimer to forward * @now: forward past this time @@ -1236,13 +1263,23 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b { bool was_first = false; + /* + * Updating the sort key while @timer is queued can temporarily + * make the tree inconsistent. This is safe under cpu_base->lock: + * no other queue operation can observe that state. + * hrtimer_can_update_in_place() either confirms that the new expiry + * fits between the neighbours or timerqueue_linked_del() removes the + * timer without consulting the expiry. + */ + hrtimer_set_expires_range_ns(timer, expires, delta_ns); + expires = hrtimer_get_expires(timer); + /* Remove it from the timer queue if active */ if (timer->is_queued) { was_first = !timerqueue_linked_prev(&timer->node); /* Try to update in place to avoid the de/enqueue dance */ if (hrtimer_can_update_in_place(timer, base, expires)) { - hrtimer_set_expires_range_ns(timer, expires, delta_ns); trace_hrtimer_start(timer, mode, true); if (was_first) base->expires_next = expires; @@ -1253,9 +1290,6 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b timerqueue_linked_del(&base->active, &timer->node); } - /* Set the new expiry time */ - hrtimer_set_expires_range_ns(timer, expires, delta_ns); - debug_activate(timer, mode, timer->is_queued); base->cpu_base->active_bases |= 1 << base->index; @@ -1786,13 +1820,21 @@ EXPORT_SYMBOL_GPL(__hrtimer_get_remaining); ktime_t hrtimer_get_next_event(void) { struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); - ktime_t expires = KTIME_MAX; - guard(raw_spinlock_irqsave)(&cpu_base->lock); - if (!hrtimer_hres_active(cpu_base)) - expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL); + /* + * When HRES is active cmp_next_hrtimer_event() expects KTIME_MAX. + * + * cpu_base->hres_active is written only by the local CPU in + * hrtimer_switch_to_hres() from hard interrupt context and in + * hrtimers_cpu_starting() during CPU bring-up, and all callers reach + * this with interrupts disabled on the same CPU, so an unlocked read is + * stable without holding the lock. + */ + if (hrtimer_hres_active(cpu_base)) + return KTIME_MAX; - return expires; + guard(raw_spinlock_irqsave)(&cpu_base->lock); + return __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL); } /** @@ -2060,13 +2102,6 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc base->running = NULL; } -static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base) -{ - struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active); - - return next ? hrtimer_from_timerqueue_node(next) : NULL; -} - static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, unsigned long flags, unsigned int active_mask) { @@ -2228,8 +2263,10 @@ retry: expires_next = hrtimer_update_next_event(cpu_base); cpu_base->hang_detected = false; if (expires_next < now) { - if (++retries < 3) + if (++retries < 3) { + cpu_base->nr_retries++; goto retry; + } delta = ktime_sub(now, entry_time); cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta); diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c index 7c6110e964e7..03a32dffc56c 100644 --- a/kernel/time/itimer.c +++ b/kernel/time/itimer.c @@ -100,7 +100,7 @@ static int do_getitimer(int which, struct itimerspec64 *value) static int put_itimerval(struct __kernel_old_itimerval __user *o, const struct itimerspec64 *i) { - struct __kernel_old_itimerval v; + struct __kernel_old_itimerval v = {}; v.it_interval.tv_sec = i->it_interval.tv_sec; v.it_interval.tv_usec = i->it_interval.tv_nsec / NSEC_PER_USEC; diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index d51428867a33..9b3487d40cd6 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c @@ -98,9 +98,11 @@ void __init register_refined_jiffies(long cycles_per_second) __clocksource_register(&refined_jiffies); } -#ifdef CONFIG_PROC_SYSCTL +#ifdef CONFIG_SYSCTL static ulong mult_hz(const ulong val) { + if (val >= ULONG_MAX / HZ) + return ULONG_MAX; return val * HZ; } @@ -136,6 +138,8 @@ static int sysctl_k2u_int_conv_userhz(bool *negp, ulong *u_ptr, const int *k_ptr static ulong sysctl_msecs_to_jiffies(const ulong val) { + if (val > jiffies_to_msecs(MAX_JIFFY_OFFSET)) + return MAX_JIFFY_OFFSET; return msecs_to_jiffies(val); } @@ -181,11 +185,28 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, int *k_ptr, int dir, const struct ctl_table *tbl) { - return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false, + return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, true, sysctl_u2k_int_conv_ms, sysctl_k2u_int_conv_ms); } -#else // CONFIG_PROC_SYSCTL +static int sysctl_u2k_ulong_conv_ms(const ulong *u_ptr, ulong *k_ptr) +{ + return proc_ulong_u2k_conv_uop(u_ptr, k_ptr, sysctl_msecs_to_jiffies); +} + +static int sysctl_k2u_ulong_conv_ms(ulong *u_ptr, const ulong *k_ptr) +{ + return proc_ulong_k2u_conv_kop(u_ptr, k_ptr, sysctl_jiffies_to_msecs); +} + +static int do_proc_ulong_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, ulong *k_ptr, + int dir, const struct ctl_table *tbl) +{ + return proc_ulong_conv(u_ptr, k_ptr, dir, tbl, true, + sysctl_u2k_ulong_conv_ms, sysctl_k2u_ulong_conv_ms); +} + +#else // CONFIG_SYSCTL static int do_proc_int_conv_jiffies(bool *negp, ulong *u_ptr, int *k_ptr, int dir, const struct ctl_table *tbl) { @@ -211,6 +232,12 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, { return -ENOSYS; } + +static int do_proc_ulong_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, ulong *k_ptr, + int dir, const struct ctl_table *tbl) +{ + return -ENOSYS; +} #endif /** @@ -309,8 +336,8 @@ int proc_dointvec_ms_jiffies_minmax(const struct ctl_table *table, int dir, int proc_doulongvec_ms_jiffies_minmax(const struct ctl_table *table, int dir, void *buffer, size_t *lenp, loff_t *ppos) { - return proc_doulongvec_minmax_conv(table, dir, buffer, lenp, ppos, - HZ, 1000l); + return proc_doulongvec_conv(table, dir, buffer, lenp, ppos, + do_proc_ulong_conv_ms_jiffies_minmax); } EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax); diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 5fa0af66cf3f..3aff27bb0a15 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -293,10 +293,12 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p, return -EINVAL; } - if (off->val.tv_sec > KTIME_SEC_MAX || - off->val.tv_sec < -KTIME_SEC_MAX) + if (off->val.tv_sec > KTIME_SEC_MAX || off->val.tv_sec < -KTIME_SEC_MAX) return -ERANGE; + if (off->val.tv_nsec < 0 || off->val.tv_nsec >= NSEC_PER_SEC) + return -EINVAL; + tp = timespec64_add(tp, off->val); /* * KTIME_SEC_MAX is divided by 2 to be sure that KTIME_MAX is diff --git a/kernel/time/namespace_vdso.c b/kernel/time/namespace_vdso.c index 0d74d160eec9..5ac7b6a6d3a8 100644 --- a/kernel/time/namespace_vdso.c +++ b/kernel/time/namespace_vdso.c @@ -74,11 +74,8 @@ struct page *find_timens_vvar_page(struct vm_area_struct *vma) return current->nsproxy->time_ns->vvar_page; /* - * VM_PFNMAP | VM_IO protect .fault() handler from being called - * through interfaces like /proc/$pid/mem or - * process_vm_{readv,writev}() as long as there's no .access() - * in special_mapping_vmops(). - * For more details check_vma_flags() and __access_remote_vm() + * vvar_fault() protects this from being called through remote interfaces like + * /proc/$pid/mem or process_vm_{readv,writev}(). */ WARN(1, "vvar_page accessed remotely"); diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 97fa99b96dd0..d22b532ec536 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -26,11 +26,13 @@ /** * struct ntp_data - Structure holding all NTP related state * @tick_usec: USER_HZ period in microseconds - * @tick_length: Adjusted tick length - * @tick_length_base: Base value for @tick_length + * @tick_length: Tick length in ns << NTP_SCALE_SHIFT * @time_state: State of the clock synchronization * @time_status: Clock status bits * @time_offset: Time adjustment in nanoseconds + * @skew_delta: Per-tick phase slew rate for the coming second, in + * @time_offset units (shifted-ns / HZ). Set by + * second_overflow(). * @time_constant: PLL time constant * @time_maxerror: Maximum error in microseconds holding the NTP sync distance * (NTP dispersion + delay / 2) @@ -38,7 +40,13 @@ * @time_freq: Frequency offset scaled nsecs/secs * @time_reftime: Time at last adjustment in seconds * @time_adjust: Adjustment value + * @time_adjust_frac: Sub-microsecond remainder of @time_adjust being + * delivered, in ns << NTP_SCALE_SHIFT (not divided by HZ). * @ntp_tick_adj: Constant boot-param configurable NTP tick adjustment (upscaled) + * @cs_tick_adj: Fixed per-second adjustment compensating for the difference + * between the nominal NTP interval and the real time taken + * by the clocksource's integer @cycle_interval (upscaled). + * Set by the timekeeping core via ntp_clear(). * @ntp_next_leap_sec: Second value of the next pending leapsecond, or TIME64_MAX if no leap * * @pps_valid: PPS signal watchdog counter @@ -59,17 +67,19 @@ struct ntp_data { unsigned long tick_usec; u64 tick_length; - u64 tick_length_base; int time_state; int time_status; s64 time_offset; + s64 skew_delta; long time_constant; long time_maxerror; long time_esterror; s64 time_freq; time64_t time_reftime; long time_adjust; + s64 time_adjust_frac; s64 ntp_tick_adj; + s64 cs_tick_adj; time64_t ntp_next_leap_sec; #ifdef CONFIG_NTP_PPS int pps_valid; @@ -101,6 +111,9 @@ static struct ntp_data tk_ntp_data[TIMEKEEPERS_MAX] = { #define SECS_PER_DAY 86400 #define MAX_TICKADJ 500LL /* usecs */ +/* One microsecond of phase, in plain shifted-ns (ns << NTP_SCALE_SHIFT) */ +#define ONE_US_NS ((s64)NSEC_PER_USEC << NTP_SCALE_SHIFT) +/* Per-tick MAX_TICKADJ slew, in plain shifted-ns */ #define MAX_TICKADJ_SCALED \ (((MAX_TICKADJ * NSEC_PER_USEC) << NTP_SCALE_SHIFT) / NTP_INTERVAL_FREQ) #define MAX_TAI_OFFSET 100000 @@ -245,8 +258,7 @@ static inline void pps_fill_timex(struct ntp_data *ntpdata, struct __kernel_time #endif /* CONFIG_NTP_PPS */ /* - * Update tick_length and tick_length_base, based on tick_usec, ntp_tick_adj and - * time_freq: + * Update tick_length based on tick_usec, ntp_tick_adj and time_freq: */ static void ntp_update_frequency(struct ntp_data *ntpdata) { @@ -255,6 +267,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata) second_length = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT; second_length += ntpdata->ntp_tick_adj; + second_length += ntpdata->cs_tick_adj; second_length += ntpdata->time_freq; new_base = div_u64(second_length, NTP_INTERVAL_FREQ); @@ -263,8 +276,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata) * Don't wait for the next second_overflow, apply the change to the * tick length immediately: */ - ntpdata->tick_length += new_base - ntpdata->tick_length_base; - ntpdata->tick_length_base = new_base; + ntpdata->tick_length = new_base; } static inline s64 ntp_update_offset_fll(struct ntp_data *ntpdata, s64 offset64, long secs) @@ -335,14 +347,15 @@ static void __ntp_clear(struct ntp_data *ntpdata) { /* Stop active adjtime() */ ntpdata->time_adjust = 0; + ntpdata->time_adjust_frac = 0; ntpdata->time_status |= STA_UNSYNC; ntpdata->time_maxerror = NTP_PHASE_LIMIT; ntpdata->time_esterror = NTP_PHASE_LIMIT; ntp_update_frequency(ntpdata); - ntpdata->tick_length = ntpdata->tick_length_base; ntpdata->time_offset = 0; + ntpdata->skew_delta = 0; ntpdata->ntp_next_leap_sec = TIME64_MAX; /* Clear PPS state variables */ @@ -350,11 +363,26 @@ static void __ntp_clear(struct ntp_data *ntpdata) } /** - * ntp_clear - Clears the NTP state variables - * @tkid: Timekeeper ID to be able to select proper ntp data array member + * ntp_clear - Clear NTP state and set the clocksource quantisation adjustment + * @tkid: Timekeeper ID + * @cs_tick_adj: Per-second adjustment in ns << NTP_SCALE_SHIFT + * + * The timekeeping core uses an integer number of cycles (@cycle_interval) + * per NTP interval, so the real time that interval represents differs from + * the nominal NTP_INTERVAL_LENGTH by up to half a counter period. Folding + * this fixed offset into @cs_tick_adj makes it an explicit part of the NTP + * tick_length computation in ntp.c, instead of being applied during + * timekeeping accumulation where the NTP code never saw it. Like + * @ntp_tick_adj it stays internal to the kernel; userspace still sees the + * nominal tick via adjtimex. NTP retains its full symmetric ±MAXFREQ range + * around the corrected base rate. + * + * Called whenever the clocksource is (re)configured, which is also when the + * rest of the NTP state must be cleared, so the two are done together. */ -void ntp_clear(unsigned int tkid) +void ntp_clear(unsigned int tkid, s64 cs_tick_adj) { + tk_ntp_data[tkid].cs_tick_adj = cs_tick_adj; __ntp_clear(&tk_ntp_data[tkid]); } @@ -364,6 +392,186 @@ u64 ntp_tick_length(unsigned int tkid) return tk_ntp_data[tkid].tick_length; } +s64 ntp_get_skew_delta(unsigned int tkid) +{ + return tk_ntp_data[tkid].skew_delta; +} + +/* Sign of @x as +1 or -1 (zero counts as positive; callers pass nonzero). */ +static inline int signof(s64 x) +{ + return x < 0 ? -1 : 1; +} + +static s64 ntp_drain_time_offset(unsigned int tkid, s64 amount) +{ + struct ntp_data *ntpdata = &tk_ntp_data[tkid]; + + /* Only drain if amount and time_offset have the same sign */ + if (!amount || signof(amount) != signof(ntpdata->time_offset)) + return amount; + + /* Clamp: don't overshoot zero */ + if (abs(amount) > abs(ntpdata->time_offset)) { + s64 undrained = amount - ntpdata->time_offset; + + ntpdata->time_offset = 0; + return undrained; + } + + ntpdata->time_offset -= amount; + return 0; +} + +/* + * Drain the legacy adjtime() correction (time_adjust) as it is delivered. + * + * @amount is the total intentional per-tick skew for this accumulation + * (skew_delta << shift), in time_offset units (shifted_ns / HZ); it covers + * both the exponential time_offset slew and the linear adjtime slew. This + * function claims only the adjtime share — capped at the MAX_TICKADJ rate — + * and returns the remainder for ntp_drain_time_offset(). + * + * time_adjust is in whole µs. The sub-µs remainder being delivered lives in + * time_adjust_frac (plain shifted-ns, i.e. ns << NTP_SCALE_SHIFT -- unlike + * time_offset these are NOT pre-divided by HZ); we top it up by borrowing + * whole microseconds from time_adjust as the drain consumes it. + */ +static s64 ntp_drain_time_adjust(unsigned int tkid, s64 amount, unsigned int shift) +{ + struct ntp_data *ntpdata = &tk_ntp_data[tkid]; + /* Sign reference: time_adjust if any whole us remain, else the drawer */ + s64 ref = ntpdata->time_adjust ? (s64)ntpdata->time_adjust + : ntpdata->time_adjust_frac; + s64 deliver, deficit, claimed; + + if (!amount || !ref || signof(amount) != signof(ref)) + return amount; + + /* + * Phase to deliver this accumulation, in plain shifted-ns. The drain + * @amount is in ÷HZ units, so multiply by HZ first, then clamp to the + * MAX_TICKADJ rate (MAX_TICKADJ_SCALED is the per-tick slew in + * shifted-ns). Multiply-then-clamp avoids an s64 divide for the cap. + */ + deliver = min(abs(amount) * NTP_INTERVAL_FREQ, + (s64)MAX_TICKADJ_SCALED << shift); + + /* Top up the sub-µs drawer from whole-µs time_adjust as needed */ + deficit = deliver - abs(ntpdata->time_adjust_frac); + if (deficit > 0 && ntpdata->time_adjust) { + long borrow = div64_u64(deficit + ONE_US_NS - 1, ONE_US_NS); + + if (ntpdata->time_adjust > 0) { + borrow = min(borrow, ntpdata->time_adjust); + ntpdata->time_adjust -= borrow; + ntpdata->time_adjust_frac += (s64)borrow * ONE_US_NS; + } else { + /* Clamp without negating time_adjust (UB for LONG_MIN) */ + if (ntpdata->time_adjust > -borrow) + borrow = -ntpdata->time_adjust; + ntpdata->time_adjust += borrow; + ntpdata->time_adjust_frac -= (s64)borrow * ONE_US_NS; + } + } + + /* Never deliver more than the drawer holds */ + deliver = min(deliver, abs(ntpdata->time_adjust_frac)); + if (ntpdata->time_adjust_frac > 0) + ntpdata->time_adjust_frac -= deliver; + else + ntpdata->time_adjust_frac += deliver; + + /* Return the unclaimed remainder in ÷HZ drain units for time_offset */ + claimed = div_s64(deliver, NTP_INTERVAL_FREQ); + return amount - signof(amount) * claimed; +} + +/* + * Drain one accumulation's worth of intentional skew as it is delivered. + * + * @amount is the total intentional per-tick skew for this accumulation + * (skew_delta << shift), in time_offset units (shifted_ns / HZ). The + * adjtime() linear share is taken from time_adjust first (capped at the + * MAX_TICKADJ rate, hence @shift), then the exponential remainder from + * time_offset. Returns the amount actually claimed (same ÷HZ units). + */ +s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift) +{ + s64 unclaimed = ntp_drain_time_adjust(tkid, amount, shift); + + unclaimed = ntp_drain_time_offset(tkid, unclaimed); + + /* + * Return the amount actually drained from the intentional + * phase offset in time_offset and/or time_adjust. + */ + return amount - unclaimed; +} + +/* + * time_offset (drained exponentially) and time_adjust (drained linearly at the + * MAX_TICKADJ rate) can be asked to slew the clock in opposite directions. + * second_overflow() only folds their *net* into skew_delta, so the cancelling + * part would never be drained from either tracker via the per-tick code -- and + * if they cancel exactly, skew_delta is zero and neither converges at all. + * + * Settle that cancelling phase directly between the two here. No clock motion + * results (the opposing slews annihilate), but both move toward zero so neither + * stalls. @amount is the phase to take off time_offset, in its (÷HZ) units and + * with its sign; the same real magnitude comes off time_adjust in the opposite + * direction. Clamped so neither tracker is driven past zero. + */ +static void ntp_transfer_offset_adjust(struct ntp_data *ntpdata, s64 amount) +{ + s64 frac_delta, carry; + + /* + * Don't drain time_offset past zero. @amount shares its sign and is + * normally bounded below it by ntp_offset_chunk(), but the ±1 skew_delta + * floor for a tiny time_offset can exceed it, so clamp. + */ + if (abs(amount) > abs(ntpdata->time_offset)) + amount = ntpdata->time_offset; + if (!amount) + return; + + /* + * Remove the matching phase from time_adjust, in plain shifted-ns. No + * clamp against time_adjust's zero is needed: @amount is bounded by the + * adjtime chunk, which second_overflow() never lets exceed time_adjust's + * own pending phase, so this cannot overshoot. + */ + frac_delta = amount * NTP_INTERVAL_FREQ; + + ntpdata->time_offset -= amount; + + /* Add the matching phase to time_adjust, carrying whole µs (O(1)). */ + ntpdata->time_adjust_frac += frac_delta; + if (ntpdata->time_adjust_frac >= ONE_US_NS || + ntpdata->time_adjust_frac <= -ONE_US_NS) { + carry = div64_s64(ntpdata->time_adjust_frac, ONE_US_NS); + ntpdata->time_adjust += carry; + ntpdata->time_adjust_frac -= carry * ONE_US_NS; + } + + /* + * Keep time_adjust and its sub-µs remainder the same sign. The + * truncating carry above can leave them opposed (e.g. +4 µs paired + * with -250 ns), and ntp_drain_time_adjust() treats abs(time_adjust_frac) + * as same-direction drawer capacity -- an opposing remainder there makes + * it over-deliver phase that was never removed from the pile. Borrow or + * repay a single whole µs to realign; the total phase is unchanged. + */ + if (ntpdata->time_adjust > 0 && ntpdata->time_adjust_frac < 0) { + ntpdata->time_adjust--; + ntpdata->time_adjust_frac += ONE_US_NS; + } else if (ntpdata->time_adjust < 0 && ntpdata->time_adjust_frac > 0) { + ntpdata->time_adjust++; + ntpdata->time_adjust_frac -= ONE_US_NS; + } +} + /** * ntp_get_next_leap - Returns the next leapsecond in CLOCK_REALTIME ktime_t * @tkid: Timekeeper ID @@ -398,7 +606,6 @@ ktime_t ntp_get_next_leap(unsigned int tkid) int second_overflow(unsigned int tkid, time64_t secs) { struct ntp_data *ntpdata = &tk_ntp_data[tkid]; - s64 delta; int leap = 0; s32 rem; @@ -458,35 +665,70 @@ int second_overflow(unsigned int tkid, time64_t secs) } /* Compute the phase adjustment for the next second */ - ntpdata->tick_length = ntpdata->tick_length_base; - - delta = ntp_offset_chunk(ntpdata, ntpdata->time_offset); - ntpdata->time_offset -= delta; - ntpdata->tick_length += delta; /* Check PPS signal */ pps_dec_valid(ntpdata); - if (!ntpdata->time_adjust) - goto out; + /* + * Set the per-tick skew rate for the next second. This is in + * the same units as time_offset: (ns << NTP_SCALE_SHIFT) / HZ. + * If the result is so low that the skew imparted would round + * to zero, pass the bare minimum ±1 to ensure that it *does* + * actually drain completely to zero. It won't overshoot because + * logarithmic_accumulation() only drains what it can from + * time_offset or time_adjust, and the rest ends up in ntp_error + * which drives the selection of 'mult' immediately each tick. + */ + if (ntpdata->time_offset || ntpdata->time_adjust || + ntpdata->time_adjust_frac) { + s64 off_chunk = ntp_offset_chunk(ntpdata, ntpdata->time_offset); + s64 adj_chunk = 0, net; - if (ntpdata->time_adjust > MAX_TICKADJ) { - ntpdata->time_adjust -= MAX_TICKADJ; - ntpdata->tick_length += MAX_TICKADJ_SCALED; - goto out; - } + /* + * Once the exponential chunk rounds to zero, deliver the last + * remaining offset this second so it converges to zero instead + * of stalling just above it. + */ + if (!off_chunk) + off_chunk = ntpdata->time_offset; + + if (ntpdata->time_adjust || ntpdata->time_adjust_frac) { + s64 adj; + + if (ntpdata->time_adjust >= MAX_TICKADJ) + adj = MAX_TICKADJ * ONE_US_NS; + else if (ntpdata->time_adjust <= -MAX_TICKADJ) + adj = -MAX_TICKADJ * ONE_US_NS; + else + adj = ntpdata->time_adjust * ONE_US_NS + + ntpdata->time_adjust_frac; + + adj_chunk = div_s64(adj, NTP_INTERVAL_FREQ); + if (!adj_chunk) + adj_chunk = signof(ntpdata->time_adjust_frac); + } - if (ntpdata->time_adjust < -MAX_TICKADJ) { - ntpdata->time_adjust += MAX_TICKADJ; - ntpdata->tick_length -= MAX_TICKADJ_SCALED; - goto out; - } + /* + * If the two slews oppose, only their net would drive the + * per-tick drain, so the cancelling part would never drain from + * either tracker and an exact cancellation would stall both. + * Settle that overlap directly between them (no clock motion). + */ + if (off_chunk && adj_chunk && signof(off_chunk) != signof(adj_chunk)) { + s64 conflict = min(abs(off_chunk), abs(adj_chunk)); - ntpdata->tick_length += (s64)(ntpdata->time_adjust * NSEC_PER_USEC / NTP_INTERVAL_FREQ) - << NTP_SCALE_SHIFT; - ntpdata->time_adjust = 0; + ntp_transfer_offset_adjust(ntpdata, signof(off_chunk) * conflict); + } + + /* Net is what the clock delivers; reduce to per-tick, then floor. */ + net = off_chunk + adj_chunk; + ntpdata->skew_delta = div_s64(net, NTP_INTERVAL_FREQ); + if (!ntpdata->skew_delta && net) + ntpdata->skew_delta = signof(net); + } else { + ntpdata->skew_delta = 0; + } -out: return leap; } @@ -779,6 +1021,7 @@ int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct tim if (!(txc->modes & ADJ_OFFSET_READONLY)) { /* adjtime() is independent from ntp_adjtime() */ ntpdata->time_adjust = txc->offset; + ntpdata->time_adjust_frac = 0; ntp_update_frequency(ntpdata); audit_ntp_set_old(ad, AUDIT_NTP_ADJUST, save_adjust); @@ -1020,6 +1263,7 @@ static void hardpps_update_phase(struct ntp_data *ntpdata, long error) NTP_INTERVAL_FREQ); /* Cancel running adjtime() */ ntpdata->time_adjust = 0; + ntpdata->time_adjust_frac = 0; } /* Update jitter */ ntpdata->pps_jitter += (jitter - ntpdata->pps_jitter) >> PPS_INTMIN; diff --git a/kernel/time/ntp_internal.h b/kernel/time/ntp_internal.h index 7084d839c207..0474a761bafc 100644 --- a/kernel/time/ntp_internal.h +++ b/kernel/time/ntp_internal.h @@ -3,9 +3,11 @@ #define _LINUX_NTP_INTERNAL_H extern void ntp_init(void); -extern void ntp_clear(unsigned int tkid); +extern void ntp_clear(unsigned int tkid, s64 cs_tick_adj); /* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */ extern u64 ntp_tick_length(unsigned int tkid); +extern s64 ntp_get_skew_delta(unsigned int tkid); +extern s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift); extern ktime_t ntp_get_next_leap(unsigned int tkid); extern int second_overflow(unsigned int tkid, time64_t secs); extern int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct timespec64 *ts, diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 5e633d8750d1..0bf4fcd969c8 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -408,6 +408,7 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer) new_timer->kclock = &clock_posix_cpu; timerqueue_init(&new_timer->it.cpu.node); + INIT_LIST_HEAD(&new_timer->it.cpu.elist); new_timer->it.cpu.pid = get_pid(pid); rcu_read_unlock(); return 0; @@ -461,6 +462,127 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p) trigger_base_recalc_expires(timer, p); } +/* + * Lookup the task via timer->it.cpu.pid and attempt to lock the task's sighand. + * + * This can race with the reaping of the task: + * + * CPU0 CPU1 + * + * // Finds task + * p = pid_task(pid, pid_type); __exit_signal(p) + * lock(p, sighand); + * posix_cpu_timers*_exit(); + * sighand = lock_task_sighand(p); unhash_task(p); + * p->sighand = NULL; + * unlock(sighand); + * + * In this case sighand is NULL, which means the task and the associated timer + * queue cannot be longer accessed safely. + * + * __exit_signal() invokes posix_cpu_timers_exit() and if the thread group is + * dead it also invokes posix_cpu_timers_group_exit(). These functions delete + * all pending timers from the related timer queues. The POSIX timers (k_itimer) + * themself are still accessible, but not longer connected to the task. + * + * exec() works slightly differently. The task which exec()'s terminates all + * other threads in the thread group and runs __exit_signal() on them. As the + * thread group is not dead they only clean up the per task timers via + * posix_cpu_timers_exit(). + * + * As the TGID on exec() stays the same per process timers stay queued, if they + * are armed. This works without a problem when exec() is done by the thread + * group leader. If a non-leader thread exec()'s this can end up in the + * following scenario: + * + * CPU0 CPU1 + * // Returns old leader + * p = pid_task(pid, pid_type); de_thread() + * switch_leader() + * release_task(old leader) + * __exit_signal() + * old_leader->sighand = NULL; + * // Returns NULL + * sighand = lock_task_sighand(p) + * + * That's problematic for several functions: + * + * - posix_cpu_timer_del(): If the timer is still enqueued on the task the + * underlying k_itimer will be freed which results in a UAF in + * run_posix_cpu_timers() or on timerqueue related add/delete operations. + * If the timer is not enqueued, the failure is harmless + * + * - posix_cpu_timer_set(): Independent of the enqueued state that results in a + * transient failure which is user space visible (-ESRCH) for regular posix + * timers. But for the use case in do_cpu_nanosleep() it's the same UAF + * problem just that the timer is allocated on the stack. + * + * - posix_cpu_timer_rearm(): Timer is not enqueued at that point, but this + * silently ignores the rearm request, which is a functional problem as the + * timer wont expire anymore. + */ +static struct task_struct *timer_lock_sighand(struct k_itimer *timer, unsigned long *flags) +{ + enum pid_type type = clock_pid_type(timer->it_clock); + struct cpu_timer *ctmr = &timer->it.cpu; + + guard(rcu)(); + + for (;;) { + struct task_struct *t = pid_task(timer->it.cpu.pid, type); + + /* Fail if the task cannot be found. */ + if (!t) + break; + + /* Try to lock the task's sighand */ + if (lock_task_sighand(t, flags)) + return t; + + /* + * The next PID lookup might either fail or return the new + * leader. This is correct for both exit() and exec(). + */ + } + + /* + * If the timer is still enqueued, warn. There is nothing safe to do + * here as there might be two timers in there which are removed in + * parallel and that will cause more damage than good. This should never + * happen! + * + * Ensure that the stores to the timer and timerqueue are visible: + * + * __exit_signal() + * posix_cpu_timers*_exit() + * write_seqlock(seqlock) + * smp_wmb(); <------- + * __unhash_process() | !pid_task() + * ----> smp_rmb(); + * WARN_ON_ONCE(...) + */ + smp_rmb(); + WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node)); + return NULL; +} + +/* + * If the timer is queued on the expiry list, then it cannot be dequeued because + * the firing list is not protected by sighand->lock. The delivery path is + * waiting for the timer lock. So go back, unlock and retry. + */ +static bool posix_cpu_timer_on_expiry_list(struct k_itimer *timer) +{ + if (list_empty(&timer->it.cpu.elist)) + return false; + + /* + * Prevent signal delivery as there is no point in delivering a signal + * which is made obsolete right away. + */ + timer->it.cpu.firing = false; + return true; +} /* * Clean up a CPU-clock timer that is about to be destroyed. @@ -470,49 +592,22 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p) */ static int posix_cpu_timer_del(struct k_itimer *timer) { - struct cpu_timer *ctmr = &timer->it.cpu; - struct sighand_struct *sighand; struct task_struct *p; unsigned long flags; int ret = 0; - rcu_read_lock(); - p = cpu_timer_task_rcu(timer); - if (!p) - goto out; + p = timer_lock_sighand(timer, &flags); - /* - * Protect against sighand release/switch in exit/exec and process/ - * thread timer list entry concurrent read/writes. - */ - sighand = lock_task_sighand(p, &flags); - if (unlikely(sighand == NULL)) { - /* - * This raced with the reaping of the task. The exit cleanup - * should have removed this timer from the timer queue. - */ - WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node)); - } else { - if (timer->it.cpu.firing) { - /* - * Prevent signal delivery. The timer cannot be dequeued - * because it is on the firing list which is not protected - * by sighand->lock. The delivery path is waiting for - * the timer lock. So go back, unlock and retry. - */ - timer->it.cpu.firing = false; + if (likely(p)) { + if (posix_cpu_timer_on_expiry_list(timer)) ret = TIMER_RETRY; - } else { + else disarm_timer(timer, p); - } unlock_task_sighand(p, &flags); } -out: - rcu_read_unlock(); - if (!ret) { - put_pid(ctmr->pid); + put_pid(timer->it.cpu.pid); timer->it_status = POSIX_TIMER_DISARMED; } return ret; @@ -626,21 +721,17 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); struct cpu_timer *ctmr = &timer->it.cpu; u64 old_expires, new_expires, now; - struct sighand_struct *sighand; struct task_struct *p; unsigned long flags; int ret = 0; - rcu_read_lock(); - p = cpu_timer_task_rcu(timer); - if (!p) { - /* - * If p has just been reaped, we can no - * longer get any information about it at all. - */ - rcu_read_unlock(); + p = timer_lock_sighand(timer, &flags); + /* + * If p has just been reaped, we can no longer get any information about + * it at all. + */ + if (!p) return -ESRCH; - } /* * Use the to_ktime conversion because that clamps the maximum @@ -648,31 +739,10 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, */ new_expires = ktime_to_ns(timespec64_to_ktime(new->it_value)); - /* - * Protect against sighand release/switch in exit/exec and p->cpu_timers - * and p->signal->cpu_timers read/write in arm_timer() - */ - sighand = lock_task_sighand(p, &flags); - /* - * If p has just been reaped, we can no - * longer get any information about it at all. - */ - if (unlikely(sighand == NULL)) { - rcu_read_unlock(); - return -ESRCH; - } - /* Retrieve the current expiry time before disarming the timer */ old_expires = cpu_timer_getexpires(ctmr); - if (unlikely(timer->it.cpu.firing)) { - /* - * Prevent signal delivery. The timer cannot be dequeued - * because it is on the firing list which is not protected - * by sighand->lock. The delivery path is waiting for - * the timer lock. So go back, unlock and retry. - */ - timer->it.cpu.firing = false; + if (posix_cpu_timer_on_expiry_list(timer)) { ret = TIMER_RETRY; } else { cpu_timer_dequeue(ctmr); @@ -698,7 +768,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, /* Retry if the timer expiry is running concurrently */ if (unlikely(ret)) { unlock_task_sighand(p, &flags); - goto out; + return ret; } /* Convert relative expiry time to absolute */ @@ -733,8 +803,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, */ if (!sigev_none && new_expires && now >= new_expires) cpu_timer_fire(timer); -out: - rcu_read_unlock(); return ret; } @@ -1018,19 +1086,12 @@ static void check_process_timers(struct task_struct *tsk, static bool posix_cpu_timer_rearm(struct k_itimer *timer) { clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); - struct sighand_struct *sighand; struct task_struct *p; unsigned long flags; u64 now; - guard(rcu)(); - p = cpu_timer_task_rcu(timer); - if (!p) - return true; - - /* Protect timer list r/w in arm_timer() */ - sighand = lock_task_sighand(p, &flags); - if (unlikely(sighand == NULL)) + p = timer_lock_sighand(timer, &flags); + if (unlikely(!p)) return true; /* @@ -1300,8 +1361,11 @@ static void handle_posix_cpu_timers(struct task_struct *tsk) unsigned long flags, start; LIST_HEAD(firing); - if (!lock_task_sighand(tsk, &flags)) - return; + /* + * tsk is current and ->sighand is stable, see the + * tsk->exit_state check in run_posix_cpu_timers() + */ + spin_lock_irqsave(&tsk->sighand->siglock, flags); do { /* @@ -1361,7 +1425,7 @@ static void handle_posix_cpu_timers(struct task_struct *tsk) * that gets the timer lock before we do will give it up and * spin until we've taken care of that timer below. */ - unlock_task_sighand(tsk, &flags); + spin_unlock_irqrestore(&tsk->sighand->siglock, flags); /* * Now that all the timers on our list have the firing flag, diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 115e0bf01276..bda3d2391a60 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -165,23 +165,31 @@ static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev, */ void tick_install_broadcast_device(struct clock_event_device *dev, int cpu) { - struct clock_event_device *cur = tick_broadcast_device.evtdev; + struct clock_event_device *cur; - if (tick_set_oneshot_wakeup_device(dev, cpu)) - return; + scoped_guard(raw_spinlock_irqsave, &tick_broadcast_lock) { - if (!tick_check_broadcast_device(cur, dev)) - return; + if (tick_set_oneshot_wakeup_device(dev, cpu)) + return; - if (!try_module_get(dev->owner)) - return; + cur = tick_broadcast_device.evtdev; + if (!tick_check_broadcast_device(cur, dev)) + return; - clockevents_exchange_device(cur, dev); + if (!try_module_get(dev->owner)) + return; + + __clockevents_exchange_device(cur, dev); + if (cur) + cur->event_handler = clockevents_handle_noop; + WRITE_ONCE(tick_broadcast_device.evtdev, dev); + if (!cpumask_empty(tick_broadcast_mask)) + tick_broadcast_start_periodic(dev); + } + + /* Module release must be outside of the lock */ if (cur) - cur->event_handler = clockevents_handle_noop; - tick_broadcast_device.evtdev = dev; - if (!cpumask_empty(tick_broadcast_mask)) - tick_broadcast_start_periodic(dev); + module_put(cur->owner); if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT)) return; @@ -1218,7 +1226,7 @@ int tick_broadcast_oneshot_active(void) */ bool tick_broadcast_oneshot_available(void) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false; } @@ -1226,7 +1234,7 @@ bool tick_broadcast_oneshot_available(void) #else int __tick_broadcast_oneshot_control(enum tick_broadcast_state state) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER)) return -EBUSY; diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 597d816d22e8..65680db95053 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -3,6 +3,7 @@ * tick internal variable and functions used by low/high res code */ #include <linux/hrtimer.h> +#include <linux/hrtimer_bases.h> #include <linux/tick.h> #include "timekeeping.h" @@ -54,6 +55,8 @@ static inline void clockevent_set_state(struct clock_event_device *dev, } extern void clockevents_shutdown(struct clock_event_device *dev); +extern void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new); extern void clockevents_exchange_device(struct clock_event_device *old, struct clock_event_device *new); extern void clockevents_switch_state(struct clock_event_device *dev, diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 98a9cae915c0..6c3fea386713 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -828,8 +828,7 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) ts->timer_expires_base = basemono; /* - * Keep the periodic tick, when RCU, architecture or irq_work - * requests it. + * Keep the periodic tick, when RCU or irq_work requests it. * Aside of that, check whether the local timer softirq is * pending. If so, its a bad idea to call get_next_timer_interrupt(), * because there is an already expired timer, so it will request @@ -837,8 +836,8 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu) * minimal delta, which brings us back to this place * immediately. Lather, rinse and repeat... */ - if (rcu_needs_cpu() || arch_needs_cpu() || - irq_work_needs_cpu() || local_timer_softirq_pending()) { + if (rcu_needs_cpu() || irq_work_needs_cpu() || + local_timer_softirq_pending()) { next_tick = basemono + TICK_NSEC; } else { /* diff --git a/kernel/time/time.c b/kernel/time/time.c index 0dd63a91e7c5..079ab34f61db 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -42,6 +42,13 @@ #include <generated/timeconst.h> #include "timekeeping.h" +#include "timekeeping_internal.h" + +#if defined(CONFIG_64BIT) || defined(CONFIG_COMPAT_32BIT_TIME) +#define __WANT_OLD_TIME_TYPE_SYSCALL 1 +#endif + +static_assert(sizeof(__kernel_old_time_t) == 8 ? IS_ENABLED(__WANT_OLD_TIME_TYPE_SYSCALL) : true); /* * The timezone where the local system is located. Used as a default by some @@ -51,7 +58,7 @@ struct timezone sys_tz; EXPORT_SYMBOL(sys_tz); -#ifdef __ARCH_WANT_SYS_TIME +#if defined(__ARCH_WANT_SYS_TIME) && defined(__WANT_OLD_TIME_TYPE_SYSCALL) /* * sys_time() can be implemented in user-level using @@ -96,7 +103,7 @@ SYSCALL_DEFINE1(stime, __kernel_old_time_t __user *, tptr) return 0; } -#endif /* __ARCH_WANT_SYS_TIME */ +#endif /* __ARCH_WANT_SYS_TIME && __WANT_OLD_TIME_TYPE_SYSCALL */ #ifdef CONFIG_COMPAT_32BIT_TIME #ifdef __ARCH_WANT_SYS_TIME32 @@ -137,6 +144,7 @@ SYSCALL_DEFINE1(stime32, old_time32_t __user *, tptr) #endif /* __ARCH_WANT_SYS_TIME32 */ #endif +#ifdef __WANT_OLD_TIME_TYPE_SYSCALL SYSCALL_DEFINE2(gettimeofday, struct __kernel_old_timeval __user *, tv, struct timezone __user *, tz) { @@ -154,6 +162,7 @@ SYSCALL_DEFINE2(gettimeofday, struct __kernel_old_timeval __user *, tv, } return 0; } +#endif /* __WANT_OLD_TIME_TYPE_SYSCALL */ /* * In case for some reason the CMOS clock has not already been running @@ -203,6 +212,9 @@ SYSCALL_DEFINE2(settimeofday, struct __kernel_old_timeval __user *, tv, struct timezone new_tz; if (tv) { + if (!IS_ENABLED(__WANT_OLD_TIME_TYPE_SYSCALL)) + return -EINVAL; + if (get_user(new_ts.tv_sec, &tv->tv_sec) || get_user(new_ts.tv_nsec, &tv->tv_usec)) return -EFAULT; @@ -220,7 +232,7 @@ SYSCALL_DEFINE2(settimeofday, struct __kernel_old_timeval __user *, tv, return do_sys_settimeofday64(tv ? &new_ts : NULL, tz ? &new_tz : NULL); } -#ifdef CONFIG_COMPAT +#ifdef CONFIG_COMPAT_32BIT_TIME COMPAT_SYSCALL_DEFINE2(gettimeofday, struct old_timeval32 __user *, tv, struct timezone __user *, tz) { @@ -239,7 +251,9 @@ COMPAT_SYSCALL_DEFINE2(gettimeofday, struct old_timeval32 __user *, tv, return 0; } +#endif /* CONFIG_COMPAT_32BIT_TIME */ +#ifdef CONFIG_COMPAT COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv, struct timezone __user *, tz) { @@ -247,6 +261,9 @@ COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv, struct timezone new_tz; if (tv) { + if (!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)) + return -EINVAL; + if (get_user(new_ts.tv_sec, &tv->tv_sec) || get_user(new_ts.tv_nsec, &tv->tv_usec)) return -EFAULT; @@ -263,7 +280,7 @@ COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv, return do_sys_settimeofday64(tv ? &new_ts : NULL, tz ? &new_tz : NULL); } -#endif +#endif /* CONFIG_COMPAT */ #ifdef CONFIG_64BIT SYSCALL_DEFINE1(adjtimex, struct __kernel_timex __user *, txc_p) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index b1b5ec43c0f2..ea2e6e55f37b 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -339,7 +339,6 @@ static inline void clocksource_enable_inline_read(void) { } static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) { u64 interval; - u64 tmp, ntpinterval; struct clocksource *old_clock; ++tk->cs_was_changed_seq; @@ -353,20 +352,16 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; /* Do the ns -> cycle conversion first, using original mult */ - tmp = NTP_INTERVAL_LENGTH; - tmp <<= clock->shift; - ntpinterval = tmp; - tmp += clock->mult/2; - do_div(tmp, clock->mult); - if (tmp == 0) - tmp = 1; - - interval = (u64) tmp; + interval = (u64)NTP_INTERVAL_LENGTH << clock->shift; + interval += clock->mult / 2; + do_div(interval, clock->mult); + if (interval == 0) + interval = 1; + tk->cycle_interval = interval; /* Go back from cycles -> shifted ns */ tk->xtime_interval = interval * clock->mult; - tk->xtime_remainder = ntpinterval - tk->xtime_interval; tk->raw_interval = interval * clock->mult; /* if changing clocks, convert xtime_nsec shift units */ @@ -386,7 +381,38 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) tk->ntp_error = 0; tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; - tk->ntp_tick = ntpinterval << tk->ntp_error_shift; + + /* + * ntp_tick is the tick length that NTP disciplines (its ±500 PPM + * scales only this part), in NTP-shifted ns: the real interval of + * a whole number of counter cycles. Because cycle_interval is + * rounded to an integer number of cycles, this ntp_tick differs + * from the true intended 1/HZ tick length by up to half a cycle + * period. + */ + tk->ntp_tick = (u64)tk->xtime_interval << tk->ntp_error_shift; + + /* + * cs_tick_adj is the constant difference between the disciplined + * ntp_tick above and the true 1/HZ tick, expressed per-second to + * match the ntp_update_frequency() addends and handed to NTP via + * ntp_clear() to be explicitly included in its tick_length. + * + * Worked example: HZ=1000, ACPI PM timer at 3.579545 MHz, which + * has 3579.545 cycles in 1ms, rounded to cycle_interval = 3580. + * + * So ntp_tick is actually 1.000127ms, as that is the amount of + * time that 3580 cycles will take at the nominal frequency. This + * is the part that NTP disciplines, causing each 3580 counts to + * advance the clock by up to NTP's ±500PPM of that amount. + * + * The "extra" 127ns/tick is what's stored in cs_tick_adj and + * applied as a constant correction by ntp_update_frequency() so + * that NTP *believes* it's disciplining a 1ms tick. + */ + tk->cs_tick_adj = (s64)tk->ntp_tick - + ((s64)NTP_INTERVAL_LENGTH << NTP_SCALE_SHIFT); + tk->cs_tick_adj *= NTP_INTERVAL_FREQ; /* * The timekeeper keeps its own mult values for the currently @@ -397,6 +423,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) tk->tkr_raw.mult = clock->mult; tk->ntp_err_mult = 0; tk->skip_second_overflow = 0; + tk->skew_delta = 0; tk->cs_id = clock->id; @@ -803,7 +830,7 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act if (action & TK_CLEAR_NTP) { tk->ntp_error = 0; - ntp_clear(tk->id); + ntp_clear(tk->id, tk->cs_tick_adj); } tk_update_leap_state(tk); @@ -831,7 +858,11 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act * the downside that the reader side does not longer benefit from * the cacheline optimized data layout of the timekeeper and requires * another indirection. + * + * Write xtime_sec first so that even if the memcpy() tears the store + * data integrity is provided for ktime_get_real_seconds(). */ + WRITE_ONCE(tkd->timekeeper.xtime_sec, tk->xtime_sec); memcpy(&tkd->timekeeper, tk, sizeof(*tk)); write_seqcount_end(&tkd->seq); } @@ -1159,11 +1190,11 @@ time64_t ktime_get_real_seconds(void) unsigned int seq; if (IS_ENABLED(CONFIG_64BIT)) - return tk->xtime_sec; + return READ_ONCE(tk->xtime_sec); do { seq = read_seqcount_begin(&tk_core.seq); - seconds = tk->xtime_sec; + seconds = READ_ONCE(tk->xtime_sec); } while (read_seqcount_retry(&tk_core.seq, seq)); @@ -1185,7 +1216,7 @@ noinstr time64_t __ktime_get_real_seconds(void) { struct timekeeper *tk = &tk_core.timekeeper; - return tk->xtime_sec; + return READ_ONCE(tk->xtime_sec); } static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr, @@ -1202,10 +1233,21 @@ static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr, /** * ktime_get_snapshot_id - Simultaneously snapshot a given clock ID with - * CLOCK_MONOTONIC_RAW and the underlying + * the corresponding monotonic raw and the underlying * clocksource counter value. * @clock_id: The clock ID to snapshot * @systime_snapshot: Pointer to struct receiving the system time snapshot + * + * For the system time keeping clocks (REALTIME, MONOTONIC and BOOTTIME) the + * monotonic raw clock is CLOCK_MONOTONIC_RAW. For AUX clocks this is the + * monotonic raw clock related to the AUX clock. These AUX clock related + * monotonic raw clocks have a strict linear offset to the system time + * CLOCK_MONOTONIC_RAW: + * + * MONOTONIC_RAW(AUX$N) = CLOCK_MONOTONIC_RAW(system) + offset(AUX$N) + * + * The offset is established when a AUX clock is initialized, but it is + * currently not accessible. */ void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot) { @@ -1512,6 +1554,9 @@ EXPORT_SYMBOL_GPL(ktime_real_to_base_clock); * @xtstamp: Receives simultaneously captured system and device time * * Reads a timestamp from a device and correlates it to system time + * + * See documentation for ktime_get_snapshot_id() for information about the raw + * monotonic time stamp which is used here. */ int get_device_system_crosststamp(int (*get_time_fn) (ktime_t *device_time, @@ -1522,10 +1567,11 @@ int get_device_system_crosststamp(int (*get_time_fn) struct system_device_crosststamp *xtstamp) { u64 syscnt_cycles, cycles, now, interval_start; - unsigned int seq, clock_was_set_seq = 0; ktime_t base_sys, base_raw, *offs; + u32 clock_was_set_seq = 0; u64 nsec_sys, nsec_raw; u8 cs_was_changed_seq; + unsigned int seq; bool do_interp; struct timekeeper *tk; struct tk_data *tkd; @@ -1897,40 +1943,6 @@ void ktime_get_raw_ts64(struct timespec64 *ts) EXPORT_SYMBOL(ktime_get_raw_ts64); /** - * ktime_get_clock_ts64 - Returns time of a clock in a timespec - * @id: POSIX clock ID of the clock to read - * @ts: Pointer to the timespec64 to be set - * - * The timestamp is invalidated (@ts->sec is set to -1) if the - * clock @id is not available. - */ -void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts) -{ - /* Invalidate time stamp */ - ts->tv_sec = -1; - ts->tv_nsec = 0; - - switch (id) { - case CLOCK_REALTIME: - ktime_get_real_ts64(ts); - return; - case CLOCK_MONOTONIC: - ktime_get_ts64(ts); - return; - case CLOCK_MONOTONIC_RAW: - ktime_get_raw_ts64(ts); - return; - case CLOCK_AUX ... CLOCK_AUX_LAST: - if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) - ktime_get_aux_ts64(id, ts); - return; - default: - WARN_ON_ONCE(1); - } -} -EXPORT_SYMBOL_GPL(ktime_get_clock_ts64); - -/** * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres */ int timekeeping_valid_for_hres(void) @@ -2076,7 +2088,12 @@ void __init timekeeping_init(void) tk_set_wall_to_mono(tks, wall_to_mono); - timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET); + /* + * Use TK_UPDATE_ALL so the NTP layer picks up the clocksource's + * cs_tick_adj via ntp_clear(). Clearing NTP here is otherwise + * redundant as ntp_init() already initialised it above. + */ + timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL); } /* time in seconds when suspend began for persistent clock */ @@ -2390,6 +2407,11 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, * xtime_nsec_2 = xtime_nsec_1 - offset * Which simplifies to: * xtime_nsec -= offset + * + * When subtracting offset from xtime_nsec, the same amount + * (in appropriate units) has to be added to ntp_error, in + * order to correctly track the delta between the time + * reported in xtime_nsec, and the intended time. */ if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { /* NTP adjustment caused clocksource mult overflow */ @@ -2400,6 +2422,7 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, tk->tkr_mono.mult += mult_adj; tk->xtime_interval += interval; tk->tkr_mono.xtime_nsec -= offset; + tk->ntp_error += offset << tk->ntp_error_shift; } /* @@ -2409,18 +2432,27 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, static void timekeeping_adjust(struct timekeeper *tk, s64 offset) { u64 ntp_tl = ntp_tick_length(tk->id); + s64 skew = ntp_get_skew_delta(tk->id); u32 mult; /* - * Determine the multiplier from the current NTP tick length. - * Avoid expensive division when the tick length doesn't change. + * Determine the multiplier from the current NTP tick length plus + * skew_delta. The skew biases mult so that ±1 dithering can deliver + * the time_offset slew rate. Recompute when either changes. */ - if (likely(tk->ntp_tick == ntp_tl)) { + if (likely(tk->ntp_tick == ntp_tl && tk->skew_delta == skew)) { + /* Revert to the base mult rate. */ mult = tk->tkr_mono.mult - tk->ntp_err_mult; } else { tk->ntp_tick = ntp_tl; - mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) - - tk->xtime_remainder, tk->cycle_interval); + tk->skew_delta = skew; + /* + * skew_delta is stored pre-divided by HZ (matching time_offset); + * scale it back up to the full per-tick rate for the mult bias. + */ + skew *= NTP_INTERVAL_FREQ; + mult = div64_u64((tk->ntp_tick + skew) >> tk->ntp_error_shift, + tk->cycle_interval); } /* @@ -2545,8 +2577,25 @@ static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset, /* Accumulate error between NTP and clock interval */ tk->ntp_error += tk->ntp_tick << shift; - tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << - (tk->ntp_error_shift + shift); + tk->ntp_error -= tk->xtime_interval << (tk->ntp_error_shift + shift); + + /* + * When skewing, do so by adjusting ntp_error to impart an extra + * target delta into ntp_error per tick, limited to what can be + * drained from time_offset / time_adjust to avoid overshoot. + * + * The base 'mult' value was calculated with the skew taken into + * account, such that the per-tick choice of 'mult' vs. 'mult+1' + * allows for the desired effective rate and ntp_error does not + * grow unbounded. + * + * Once the full desired phase offset is delivered, any remaining + * skew imparted by the adjusted 'mult', accounted above, remains + * in ntp_error and will be compensated by the dithering over time. + */ + if (tk->skew_delta) + tk->ntp_error += ntp_drain_skew(tk->id, tk->skew_delta << shift, + shift) * NTP_INTERVAL_FREQ; return offset; } @@ -2795,7 +2844,7 @@ void do_timer(unsigned long ticks) * * Called from hrtimer_interrupt() or retrigger_next_event() */ -ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real, +ktime_t ktime_get_update_offsets_now(u32 *cwsseq, ktime_t *offs_real, ktime_t *offs_boot, ktime_t *offs_tai) { struct timekeeper *tk = &tk_core.timekeeper; @@ -2943,10 +2992,12 @@ static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc, return ret; add_device_randomness(txc, sizeof(*txc)); - if (!aux_clock) + if (!aux_clock) { ktime_get_real_ts64(&ts); - else - tk_get_aux_ts64(tkd->timekeeper.id, &ts); + } else { + if (!tk_get_aux_ts64(tkd->timekeeper.id, &ts)) + return -ENODEV; + } add_device_randomness(&ts, sizeof(ts)); @@ -3051,7 +3102,7 @@ static inline unsigned int clockid_to_tkid(unsigned int id) static inline struct tk_data *aux_get_tk_data(clockid_t id) { - if (!clockid_aux_valid(id)) + if (!clockid_is_aux_clock(id)) return NULL; return &timekeeper_data[clockid_to_tkid(id)]; } @@ -3146,7 +3197,7 @@ EXPORT_SYMBOL_GPL(ktime_get_aux_ts64); static int aux_get_res(clockid_t id, struct timespec64 *tp) { - if (!clockid_aux_valid(id)) + if (!clockid_is_aux_clock(id)) return -ENODEV; tp->tv_sec = aux_clock_resolution_ns() / NSEC_PER_SEC; @@ -3313,7 +3364,9 @@ static const struct attribute_group aux_clock_enable_attr_group = { static int __init tk_aux_sysfs_init(void) { struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj); + struct kobject *clks[MAX_AUX_CLOCKS]; int ret = -ENOMEM; + int i; if (!tko) return ret; @@ -3322,21 +3375,28 @@ static int __init tk_aux_sysfs_init(void) if (!auxo) goto err_clean; - for (int i = 0; i < MAX_AUX_CLOCKS; i++) { + for (i = 0; i < MAX_AUX_CLOCKS; i++) { char id[2] = { [0] = '0' + i, }; - struct kobject *clk = kobject_create_and_add(id, auxo); + clks[i] = kobject_create_and_add(id, auxo); - if (!clk) { + if (!clks[i]) { ret = -ENOMEM; - goto err_clean; + goto err_clks; } - ret = sysfs_create_group(clk, &aux_clock_enable_attr_group); + ret = sysfs_create_group(clks[i], &aux_clock_enable_attr_group); if (ret) - goto err_clean; + goto err_clk; } return 0; +err_clk: + kobject_put(clks[i]); +err_clks: + while (--i >= 0) { + sysfs_remove_group(clks[i], &aux_clock_enable_attr_group); + kobject_put(clks[i]); + } err_clean: kobject_put(auxo); kobject_put(tko); diff --git a/kernel/time/timekeeping.h b/kernel/time/timekeeping.h index 198d0608db74..4201f9e90813 100644 --- a/kernel/time/timekeeping.h +++ b/kernel/time/timekeeping.h @@ -4,7 +4,7 @@ /* * Internal interfaces for kernel/time/ */ -extern ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, +extern ktime_t ktime_get_update_offsets_now(u32 *cwsseq, ktime_t *offs_real, ktime_t *offs_boot, ktime_t *offs_tai); diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h index 973ede670a36..6d719b8e5ea2 100644 --- a/kernel/time/timekeeping_internal.h +++ b/kernel/time/timekeeping_internal.h @@ -6,6 +6,8 @@ #include <linux/spinlock.h> #include <linux/time.h> +struct timekeeper; + /* * timekeeping debug functions */ @@ -48,4 +50,23 @@ void timekeeper_unlock_irqrestore(unsigned long flags); /* NTP specific interface to access the current seconds value */ long ktime_get_ntp_seconds(unsigned int id); +#ifdef CONFIG_GENERIC_GETTIMEOFDAY + +extern void update_vsyscall(struct timekeeper *tk); +extern void update_vsyscall_tz(void); +extern void vdso_time_update_aux(struct timekeeper *tk); + +#else + +static inline void update_vsyscall(struct timekeeper *tk) +{ +} +static inline void update_vsyscall_tz(void) +{ +} +static inline void vdso_time_update_aux(struct timekeeper *tk) +{ +} +#endif + #endif /* _TIMEKEEPING_INTERNAL_H */ diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 655a8c6cd84d..ae9abf14688e 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -2492,6 +2492,7 @@ static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *h timer = hlist_entry(head->first, struct timer_list, entry); detach_timer(timer, false); timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu; + debug_timer_activate(timer); internal_add_timer(new_base, timer); } } diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index 514802def1e0..0406bf4488e9 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c @@ -20,7 +20,7 @@ struct timer_list_iter { int cpu; bool second_pass; - u64 now; + ktime_t now; }; /* @@ -44,19 +44,19 @@ static void SEQ_printf(struct seq_file *m, const char *fmt, ...) static void print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer, - int idx, u64 now) + int idx, ktime_t now) { SEQ_printf(m, " #%d: <%p>, %ps", idx, taddr, ACCESS_PRIVATE(timer, function)); SEQ_printf(m, ", S:%02x", timer->is_queued); SEQ_printf(m, "\n"); - SEQ_printf(m, " # expires at %Lu-%Lu nsecs [in %Ld to %Ld nsecs]\n", - (unsigned long long)ktime_to_ns(hrtimer_get_softexpires(timer)), - (unsigned long long)ktime_to_ns(hrtimer_get_expires(timer)), - (long long)(ktime_to_ns(hrtimer_get_softexpires(timer)) - now), - (long long)(ktime_to_ns(hrtimer_get_expires(timer)) - now)); + SEQ_printf(m, " # expires at %lld-%lld nsecs [in %lld to %lld nsecs]\n", + (long long)hrtimer_get_softexpires(timer), + (long long)hrtimer_get_expires(timer), + (long long)ktime_sub(hrtimer_get_softexpires(timer), now), + (long long)ktime_sub(hrtimer_get_expires(timer), now)); } -static void print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, u64 now) +static void print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, ktime_t now) { struct timerqueue_linked_node *curr; struct hrtimer *timer, tmp; @@ -94,21 +94,21 @@ next_one: } static void -print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now) +print_base(struct seq_file *m, struct hrtimer_clock_base *base, ktime_t now) { SEQ_printf(m, " .base: %p\n", base); SEQ_printf(m, " .index: %d\n", base->index); SEQ_printf(m, " .resolution: %u nsecs\n", hrtimer_resolution); #ifdef CONFIG_HIGH_RES_TIMERS - SEQ_printf(m, " .offset: %Ld nsecs\n", + SEQ_printf(m, " .offset: %lld nsecs\n", (long long) base->offset); #endif SEQ_printf(m, "active timers:\n"); - print_active_timers(m, base, now + ktime_to_ns(base->offset)); + print_active_timers(m, base, ktime_add(now, base->offset)); } -static void print_cpu(struct seq_file *m, int cpu, u64 now) +static void print_cpu(struct seq_file *m, int cpu, ktime_t now) { struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); int i; @@ -118,15 +118,17 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now) SEQ_printf(m, " clock %d:\n", i); print_base(m, cpu_base->clock_base + i, now); } -#define P(x) \ - SEQ_printf(m, " .%-15s: %Lu\n", #x, \ - (unsigned long long)(cpu_base->x)) -#define P_ns(x) \ - SEQ_printf(m, " .%-15s: %Lu nsecs\n", #x, \ - (unsigned long long)(ktime_to_ns(cpu_base->x))) + +#define DIAG_READ(x) data_race(READ_ONCE(x)) + +#define P(x) \ + SEQ_printf(m, " .%-15s: %llu\n", #x, \ + (unsigned long long)DIAG_READ(cpu_base->x)) +#define P_ktime(x) \ + SEQ_printf(m, " .%-15s: %lld nsecs\n", #x, (long long)DIAG_READ(cpu_base->x)) #ifdef CONFIG_HIGH_RES_TIMERS - P_ns(expires_next); + P_ktime(expires_next); P(hres_active); P(nr_events); P(nr_retries); @@ -134,38 +136,39 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now) P(max_hang_time); #endif #undef P -#undef P_ns +#undef P_ktime #ifdef CONFIG_TICK_ONESHOT # define P(x) \ - SEQ_printf(m, " .%-15s: %Lu\n", #x, \ - (unsigned long long)(ts->x)) -# define P_ns(x) \ - SEQ_printf(m, " .%-15s: %Lu nsecs\n", #x, \ - (unsigned long long)(ktime_to_ns(ts->x))) + SEQ_printf(m, " .%-15s: %llu\n", #x, \ + (unsigned long long)DIAG_READ(ts->x)) +# define P_ktime(x) \ + SEQ_printf(m, " .%-15s: %lld nsecs\n", #x, (long long)DIAG_READ(ts->x)) # define P_flag(x, f) \ - SEQ_printf(m, " .%-15s: %d\n", #x, !!(ts->flags & (f))) + SEQ_printf(m, " .%-15s: %d\n", #x, !!(DIAG_READ(ts->flags) & (f))) { struct tick_sched *ts = tick_get_tick_sched(cpu); P_flag(nohz, TS_FLAG_NOHZ); P_flag(highres, TS_FLAG_HIGHRES); - P_ns(last_tick); + P_ktime(last_tick); P_flag(tick_stopped, TS_FLAG_STOPPED); P(idle_calls); P(idle_sleeps); - P_ns(idle_entrytime); - P_ns(idle_waketime); + P_ktime(idle_entrytime); + P_ktime(idle_waketime); P(last_jiffies); P(next_timer); - P_ns(idle_expires); - SEQ_printf(m, "jiffies: %Lu\n", + P_ktime(idle_expires); + SEQ_printf(m, "jiffies: %llu\n", (unsigned long long)jiffies); } #endif #undef P -#undef P_ns +#undef P_ktime +#undef P_flag +#undef DIAG_READ SEQ_printf(m, "\n"); } @@ -196,8 +199,7 @@ print_tickdevice(struct seq_file *m, struct tick_device *td, int cpu) SEQ_printf(m, " mult: %u\n", dev->mult); SEQ_printf(m, " shift: %u\n", dev->shift); SEQ_printf(m, " mode: %d\n", clockevent_get_state(dev)); - SEQ_printf(m, " next_event: %Ld nsecs\n", - (unsigned long long) ktime_to_ns(dev->next_event)); + SEQ_printf(m, " next_event: %lld nsecs\n", (long long)dev->next_event); SEQ_printf(m, " set_next_event: %ps\n", dev->set_next_event); @@ -250,17 +252,17 @@ static void timer_list_show_tickdevices_header(struct seq_file *m) } #endif -static inline void timer_list_header(struct seq_file *m, u64 now) +static inline void timer_list_header(struct seq_file *m, ktime_t now) { SEQ_printf(m, "Timer List Version: v0.11\n"); SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES); - SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now); + SEQ_printf(m, "now at %lld nsecs\n", (long long)now); SEQ_printf(m, "\n"); } void sysrq_timer_list_show(void) { - u64 now = ktime_to_ns(ktime_get()); + ktime_t now = ktime_get(); int cpu; timer_list_header(NULL, now); @@ -318,7 +320,7 @@ static void *timer_list_start(struct seq_file *file, loff_t *offset) struct timer_list_iter *iter = file->private; if (!*offset) - iter->now = ktime_to_ns(ktime_get()); + iter->now = ktime_get(); iter->cpu = -1; iter->second_pass = false; return move_iter(iter, *offset); diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 806c23cf71fc..059d43355e65 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1847,8 +1847,10 @@ static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu, } /* Assert single root without parent */ - if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) + if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) { + kfree(stack); return -EINVAL; + } for (; i >= start_lvl; i--) { group = stack[i]; diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h index 31735dd52327..c9c1c29f011d 100644 --- a/kernel/time/timer_migration.h +++ b/kernel/time/timer_migration.h @@ -103,7 +103,7 @@ struct tmigr_group { * before the timer migration hierarchy hotplug callback is * reached. During this phase, the CPU has to handle the * global timers on its own and must not act as a migrator. - + * * @idle: Indicates whether the CPU is idle in the timer migration * hierarchy * @remote: Is set when timers of the CPU are expired remotely |
