summaryrefslogtreecommitdiff
path: root/kernel/time
diff options
context:
space:
mode:
authorTejun Heo <tj@kernel.org>2026-10-07 08:01:02 -1000
committerTejun Heo <tj@kernel.org>2026-10-07 08:01:02 -1000
commit92b542db38bf7cb4de583b2ca37febc549db0ce4 (patch)
tree30dcfdb338f7fdba597e1ec9314c2006df86f525 /kernel/time
parentfee1265a725725dac0ff7459f912d69558798b36 (diff)
parentd1a4549a3645e55cc9ade09e360280590b3c9559 (diff)
downloadlinux-next-92b542db38bf7cb4de583b2ca37febc549db0ce4.tar.gz
linux-next-92b542db38bf7cb4de583b2ca37febc549db0ce4.zip
Merge branch 'for-7.3-fixes' into for-next
Diffstat (limited to 'kernel/time')
-rw-r--r--kernel/time/Kconfig1
-rw-r--r--kernel/time/clockevents.c33
-rw-r--r--kernel/time/clocksource.c9
-rw-r--r--kernel/time/hrtimer.c79
-rw-r--r--kernel/time/itimer.c2
-rw-r--r--kernel/time/jiffies.c37
-rw-r--r--kernel/time/namespace.c6
-rw-r--r--kernel/time/namespace_vdso.c7
-rw-r--r--kernel/time/ntp.c310
-rw-r--r--kernel/time/ntp_internal.h4
-rw-r--r--kernel/time/posix-cpu-timers.c222
-rw-r--r--kernel/time/tick-broadcast.c36
-rw-r--r--kernel/time/tick-internal.h3
-rw-r--r--kernel/time/tick-sched.c7
-rw-r--r--kernel/time/time.c25
-rw-r--r--kernel/time/timekeeping.c204
-rw-r--r--kernel/time/timekeeping.h2
-rw-r--r--kernel/time/timekeeping_internal.h21
-rw-r--r--kernel/time/timer.c1
-rw-r--r--kernel/time/timer_list.c78
-rw-r--r--kernel/time/timer_migration.c4
-rw-r--r--kernel/time/timer_migration.h2
22 files changed, 794 insertions, 299 deletions
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index d098ac39bde4..ddfb6bee0745 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -133,7 +133,6 @@ config NO_HZ_FULL
depends on HAVE_VIRT_CPU_ACCOUNTING_GEN
select NO_HZ_COMMON
select RCU_NOCB_CPU
- select VIRT_CPU_ACCOUNTING_GEN
select IRQ_WORK
select CPU_ISOLATION
help
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 0014d163f989..62ad7c000386 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -615,34 +615,41 @@ void clockevents_handle_noop(struct clock_event_device *dev)
{
}
-/**
- * clockevents_exchange_device - release and request clock devices
- * @old: device to release (can be NULL)
- * @new: device to request (can be NULL)
- *
- * Called from various tick functions with clockevents_lock held and
- * interrupts disabled.
- */
-void clockevents_exchange_device(struct clock_event_device *old,
- struct clock_event_device *new)
+void __clockevents_exchange_device(struct clock_event_device *old,
+ struct clock_event_device *new)
{
/*
* Caller releases a clock event device. We queue it into the
* released list and do a notify add later.
*/
if (old) {
- module_put(old->owner);
clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED);
list_move(&old->list, &clockevents_released);
}
if (new) {
- BUG_ON(!clockevent_state_detached(new));
+ WARN_ON(!clockevent_state_detached(new));
clockevents_shutdown(new);
}
}
/**
+ * clockevents_exchange_device - release and request clock devices
+ * @old: device to release (can be NULL)
+ * @new: device to request (can be NULL)
+ *
+ * Called from various tick functions with clockevents_lock held and
+ * interrupts disabled.
+ */
+void clockevents_exchange_device(struct clock_event_device *old,
+ struct clock_event_device *new)
+{
+ __clockevents_exchange_device(old, new);
+ if (old)
+ module_put(old->owner);
+}
+
+/**
* clockevents_suspend - suspend clock devices
*/
void clockevents_suspend(void)
@@ -699,7 +706,7 @@ void tick_offline_cpu(unsigned int cpu)
if (cpumask_test_cpu(cpu, dev->cpumask) &&
cpumask_weight(dev->cpumask) == 1 &&
!tick_is_broadcast_device(dev)) {
- BUG_ON(!clockevent_state_detached(dev));
+ WARN_ON(!clockevent_state_detached(dev));
list_del(&dev->list);
}
}
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index e48c4d379a7c..f1253f5795c6 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -123,7 +123,6 @@ static atomic_t watchdog_reset_pending;
/* Watchdog interval: 0.5sec. */
#define WATCHDOG_INTERVAL (HZ >> 1)
-#define WATCHDOG_INTERVAL_NS (WATCHDOG_INTERVAL * (NSEC_PER_SEC / HZ))
/* Maximum time between two reference watchdog readouts */
#define WATCHDOG_READOUT_MAX_NS (50U * NSEC_PER_USEC)
@@ -1566,8 +1565,12 @@ static int __init init_clocksource_sysfs(void)
{
int error = subsys_system_register(&clocksource_subsys, NULL);
- if (!error)
- error = device_register(&device_clocksource);
+ if (error)
+ return error;
+
+ error = device_register(&device_clocksource);
+ if (error)
+ bus_unregister(&clocksource_subsys);
return error;
}
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 313dcea127fe..cbf1693c86b3 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -26,6 +26,7 @@
#include <linux/export.h>
#include <linux/percpu.h>
#include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
#include <linux/notifier.h>
#include <linux/syscalls.h>
#include <linux/interrupt.h>
@@ -676,6 +677,8 @@ static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
{
+ lockdep_assert_held(&base->lock);
+
ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
@@ -707,7 +710,7 @@ static inline void hrtimer_rearm_event(ktime_t expires_next, bool deferred)
tick_program_event(expires_next, 1);
}
-static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer,
+static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
ktime_t expires_next)
{
cpu_base->expires_next = expires_next;
@@ -743,7 +746,7 @@ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip
if (skip_equal && expires_next == cpu_base->expires_next)
return;
- __hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);
+ __hrtimer_reprogram(cpu_base, expires_next);
}
/* High resolution timer related functions */
@@ -896,14 +899,14 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
cpu_base->next_timer = timer;
- __hrtimer_reprogram(cpu_base, timer, expires);
+ __hrtimer_reprogram(cpu_base, expires);
}
static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active)
{
struct hrtimer_clock_base *base;
- unsigned int seq;
ktime_t expires;
+ u32 seq;
/*
* Update the base offsets unconditionally so the following
@@ -1040,6 +1043,30 @@ static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned lon
}
/**
+ * hrtimer_update_function - Update the timer's callback function
+ * @timer: Timer to update
+ * @function: New callback function
+ *
+ * Only safe to call if the timer is not enqueued. Can be called in the callback function if the
+ * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED).
+ */
+void hrtimer_update_function(struct hrtimer *timer,
+ enum hrtimer_restart (*function)(struct hrtimer *))
+{
+#ifdef CONFIG_PROVE_LOCKING
+ guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock);
+
+ if (WARN_ON_ONCE(hrtimer_is_queued(timer)))
+ return;
+
+ if (WARN_ON_ONCE(!function))
+ return;
+#endif
+ ACCESS_PRIVATE(timer, function) = function;
+}
+EXPORT_SYMBOL_GPL(hrtimer_update_function);
+
+/**
* hrtimer_forward() - forward the timer expiry
* @timer: hrtimer to forward
* @now: forward past this time
@@ -1236,13 +1263,23 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b
{
bool was_first = false;
+ /*
+ * Updating the sort key while @timer is queued can temporarily
+ * make the tree inconsistent. This is safe under cpu_base->lock:
+ * no other queue operation can observe that state.
+ * hrtimer_can_update_in_place() either confirms that the new expiry
+ * fits between the neighbours or timerqueue_linked_del() removes the
+ * timer without consulting the expiry.
+ */
+ hrtimer_set_expires_range_ns(timer, expires, delta_ns);
+ expires = hrtimer_get_expires(timer);
+
/* Remove it from the timer queue if active */
if (timer->is_queued) {
was_first = !timerqueue_linked_prev(&timer->node);
/* Try to update in place to avoid the de/enqueue dance */
if (hrtimer_can_update_in_place(timer, base, expires)) {
- hrtimer_set_expires_range_ns(timer, expires, delta_ns);
trace_hrtimer_start(timer, mode, true);
if (was_first)
base->expires_next = expires;
@@ -1253,9 +1290,6 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b
timerqueue_linked_del(&base->active, &timer->node);
}
- /* Set the new expiry time */
- hrtimer_set_expires_range_ns(timer, expires, delta_ns);
-
debug_activate(timer, mode, timer->is_queued);
base->cpu_base->active_bases |= 1 << base->index;
@@ -1786,13 +1820,21 @@ EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
ktime_t hrtimer_get_next_event(void)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
- ktime_t expires = KTIME_MAX;
- guard(raw_spinlock_irqsave)(&cpu_base->lock);
- if (!hrtimer_hres_active(cpu_base))
- expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
+ /*
+ * When HRES is active cmp_next_hrtimer_event() expects KTIME_MAX.
+ *
+ * cpu_base->hres_active is written only by the local CPU in
+ * hrtimer_switch_to_hres() from hard interrupt context and in
+ * hrtimers_cpu_starting() during CPU bring-up, and all callers reach
+ * this with interrupts disabled on the same CPU, so an unlocked read is
+ * stable without holding the lock.
+ */
+ if (hrtimer_hres_active(cpu_base))
+ return KTIME_MAX;
- return expires;
+ guard(raw_spinlock_irqsave)(&cpu_base->lock);
+ return __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
}
/**
@@ -2060,13 +2102,6 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc
base->running = NULL;
}
-static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base)
-{
- struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
-
- return next ? hrtimer_from_timerqueue_node(next) : NULL;
-}
-
static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
unsigned long flags, unsigned int active_mask)
{
@@ -2228,8 +2263,10 @@ retry:
expires_next = hrtimer_update_next_event(cpu_base);
cpu_base->hang_detected = false;
if (expires_next < now) {
- if (++retries < 3)
+ if (++retries < 3) {
+ cpu_base->nr_retries++;
goto retry;
+ }
delta = ktime_sub(now, entry_time);
cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta);
diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c
index 7c6110e964e7..03a32dffc56c 100644
--- a/kernel/time/itimer.c
+++ b/kernel/time/itimer.c
@@ -100,7 +100,7 @@ static int do_getitimer(int which, struct itimerspec64 *value)
static int put_itimerval(struct __kernel_old_itimerval __user *o,
const struct itimerspec64 *i)
{
- struct __kernel_old_itimerval v;
+ struct __kernel_old_itimerval v = {};
v.it_interval.tv_sec = i->it_interval.tv_sec;
v.it_interval.tv_usec = i->it_interval.tv_nsec / NSEC_PER_USEC;
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index d51428867a33..9b3487d40cd6 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -98,9 +98,11 @@ void __init register_refined_jiffies(long cycles_per_second)
__clocksource_register(&refined_jiffies);
}
-#ifdef CONFIG_PROC_SYSCTL
+#ifdef CONFIG_SYSCTL
static ulong mult_hz(const ulong val)
{
+ if (val >= ULONG_MAX / HZ)
+ return ULONG_MAX;
return val * HZ;
}
@@ -136,6 +138,8 @@ static int sysctl_k2u_int_conv_userhz(bool *negp, ulong *u_ptr, const int *k_ptr
static ulong sysctl_msecs_to_jiffies(const ulong val)
{
+ if (val > jiffies_to_msecs(MAX_JIFFY_OFFSET))
+ return MAX_JIFFY_OFFSET;
return msecs_to_jiffies(val);
}
@@ -181,11 +185,28 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr,
int *k_ptr, int dir,
const struct ctl_table *tbl)
{
- return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, false,
+ return proc_int_conv(negp, u_ptr, k_ptr, dir, tbl, true,
sysctl_u2k_int_conv_ms, sysctl_k2u_int_conv_ms);
}
-#else // CONFIG_PROC_SYSCTL
+static int sysctl_u2k_ulong_conv_ms(const ulong *u_ptr, ulong *k_ptr)
+{
+ return proc_ulong_u2k_conv_uop(u_ptr, k_ptr, sysctl_msecs_to_jiffies);
+}
+
+static int sysctl_k2u_ulong_conv_ms(ulong *u_ptr, const ulong *k_ptr)
+{
+ return proc_ulong_k2u_conv_kop(u_ptr, k_ptr, sysctl_jiffies_to_msecs);
+}
+
+static int do_proc_ulong_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, ulong *k_ptr,
+ int dir, const struct ctl_table *tbl)
+{
+ return proc_ulong_conv(u_ptr, k_ptr, dir, tbl, true,
+ sysctl_u2k_ulong_conv_ms, sysctl_k2u_ulong_conv_ms);
+}
+
+#else // CONFIG_SYSCTL
static int do_proc_int_conv_jiffies(bool *negp, ulong *u_ptr, int *k_ptr,
int dir, const struct ctl_table *tbl)
{
@@ -211,6 +232,12 @@ static int do_proc_int_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr,
{
return -ENOSYS;
}
+
+static int do_proc_ulong_conv_ms_jiffies_minmax(bool *negp, ulong *u_ptr, ulong *k_ptr,
+ int dir, const struct ctl_table *tbl)
+{
+ return -ENOSYS;
+}
#endif
/**
@@ -309,8 +336,8 @@ int proc_dointvec_ms_jiffies_minmax(const struct ctl_table *table, int dir,
int proc_doulongvec_ms_jiffies_minmax(const struct ctl_table *table, int dir,
void *buffer, size_t *lenp, loff_t *ppos)
{
- return proc_doulongvec_minmax_conv(table, dir, buffer, lenp, ppos,
- HZ, 1000l);
+ return proc_doulongvec_conv(table, dir, buffer, lenp, ppos,
+ do_proc_ulong_conv_ms_jiffies_minmax);
}
EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax);
diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c
index 5fa0af66cf3f..3aff27bb0a15 100644
--- a/kernel/time/namespace.c
+++ b/kernel/time/namespace.c
@@ -293,10 +293,12 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p,
return -EINVAL;
}
- if (off->val.tv_sec > KTIME_SEC_MAX ||
- off->val.tv_sec < -KTIME_SEC_MAX)
+ if (off->val.tv_sec > KTIME_SEC_MAX || off->val.tv_sec < -KTIME_SEC_MAX)
return -ERANGE;
+ if (off->val.tv_nsec < 0 || off->val.tv_nsec >= NSEC_PER_SEC)
+ return -EINVAL;
+
tp = timespec64_add(tp, off->val);
/*
* KTIME_SEC_MAX is divided by 2 to be sure that KTIME_MAX is
diff --git a/kernel/time/namespace_vdso.c b/kernel/time/namespace_vdso.c
index 0d74d160eec9..5ac7b6a6d3a8 100644
--- a/kernel/time/namespace_vdso.c
+++ b/kernel/time/namespace_vdso.c
@@ -74,11 +74,8 @@ struct page *find_timens_vvar_page(struct vm_area_struct *vma)
return current->nsproxy->time_ns->vvar_page;
/*
- * VM_PFNMAP | VM_IO protect .fault() handler from being called
- * through interfaces like /proc/$pid/mem or
- * process_vm_{readv,writev}() as long as there's no .access()
- * in special_mapping_vmops().
- * For more details check_vma_flags() and __access_remote_vm()
+ * vvar_fault() protects this from being called through remote interfaces like
+ * /proc/$pid/mem or process_vm_{readv,writev}().
*/
WARN(1, "vvar_page accessed remotely");
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 97fa99b96dd0..d22b532ec536 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -26,11 +26,13 @@
/**
* struct ntp_data - Structure holding all NTP related state
* @tick_usec: USER_HZ period in microseconds
- * @tick_length: Adjusted tick length
- * @tick_length_base: Base value for @tick_length
+ * @tick_length: Tick length in ns << NTP_SCALE_SHIFT
* @time_state: State of the clock synchronization
* @time_status: Clock status bits
* @time_offset: Time adjustment in nanoseconds
+ * @skew_delta: Per-tick phase slew rate for the coming second, in
+ * @time_offset units (shifted-ns / HZ). Set by
+ * second_overflow().
* @time_constant: PLL time constant
* @time_maxerror: Maximum error in microseconds holding the NTP sync distance
* (NTP dispersion + delay / 2)
@@ -38,7 +40,13 @@
* @time_freq: Frequency offset scaled nsecs/secs
* @time_reftime: Time at last adjustment in seconds
* @time_adjust: Adjustment value
+ * @time_adjust_frac: Sub-microsecond remainder of @time_adjust being
+ * delivered, in ns << NTP_SCALE_SHIFT (not divided by HZ).
* @ntp_tick_adj: Constant boot-param configurable NTP tick adjustment (upscaled)
+ * @cs_tick_adj: Fixed per-second adjustment compensating for the difference
+ * between the nominal NTP interval and the real time taken
+ * by the clocksource's integer @cycle_interval (upscaled).
+ * Set by the timekeeping core via ntp_clear().
* @ntp_next_leap_sec: Second value of the next pending leapsecond, or TIME64_MAX if no leap
*
* @pps_valid: PPS signal watchdog counter
@@ -59,17 +67,19 @@
struct ntp_data {
unsigned long tick_usec;
u64 tick_length;
- u64 tick_length_base;
int time_state;
int time_status;
s64 time_offset;
+ s64 skew_delta;
long time_constant;
long time_maxerror;
long time_esterror;
s64 time_freq;
time64_t time_reftime;
long time_adjust;
+ s64 time_adjust_frac;
s64 ntp_tick_adj;
+ s64 cs_tick_adj;
time64_t ntp_next_leap_sec;
#ifdef CONFIG_NTP_PPS
int pps_valid;
@@ -101,6 +111,9 @@ static struct ntp_data tk_ntp_data[TIMEKEEPERS_MAX] = {
#define SECS_PER_DAY 86400
#define MAX_TICKADJ 500LL /* usecs */
+/* One microsecond of phase, in plain shifted-ns (ns << NTP_SCALE_SHIFT) */
+#define ONE_US_NS ((s64)NSEC_PER_USEC << NTP_SCALE_SHIFT)
+/* Per-tick MAX_TICKADJ slew, in plain shifted-ns */
#define MAX_TICKADJ_SCALED \
(((MAX_TICKADJ * NSEC_PER_USEC) << NTP_SCALE_SHIFT) / NTP_INTERVAL_FREQ)
#define MAX_TAI_OFFSET 100000
@@ -245,8 +258,7 @@ static inline void pps_fill_timex(struct ntp_data *ntpdata, struct __kernel_time
#endif /* CONFIG_NTP_PPS */
/*
- * Update tick_length and tick_length_base, based on tick_usec, ntp_tick_adj and
- * time_freq:
+ * Update tick_length based on tick_usec, ntp_tick_adj and time_freq:
*/
static void ntp_update_frequency(struct ntp_data *ntpdata)
{
@@ -255,6 +267,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata)
second_length = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT;
second_length += ntpdata->ntp_tick_adj;
+ second_length += ntpdata->cs_tick_adj;
second_length += ntpdata->time_freq;
new_base = div_u64(second_length, NTP_INTERVAL_FREQ);
@@ -263,8 +276,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata)
* Don't wait for the next second_overflow, apply the change to the
* tick length immediately:
*/
- ntpdata->tick_length += new_base - ntpdata->tick_length_base;
- ntpdata->tick_length_base = new_base;
+ ntpdata->tick_length = new_base;
}
static inline s64 ntp_update_offset_fll(struct ntp_data *ntpdata, s64 offset64, long secs)
@@ -335,14 +347,15 @@ static void __ntp_clear(struct ntp_data *ntpdata)
{
/* Stop active adjtime() */
ntpdata->time_adjust = 0;
+ ntpdata->time_adjust_frac = 0;
ntpdata->time_status |= STA_UNSYNC;
ntpdata->time_maxerror = NTP_PHASE_LIMIT;
ntpdata->time_esterror = NTP_PHASE_LIMIT;
ntp_update_frequency(ntpdata);
- ntpdata->tick_length = ntpdata->tick_length_base;
ntpdata->time_offset = 0;
+ ntpdata->skew_delta = 0;
ntpdata->ntp_next_leap_sec = TIME64_MAX;
/* Clear PPS state variables */
@@ -350,11 +363,26 @@ static void __ntp_clear(struct ntp_data *ntpdata)
}
/**
- * ntp_clear - Clears the NTP state variables
- * @tkid: Timekeeper ID to be able to select proper ntp data array member
+ * ntp_clear - Clear NTP state and set the clocksource quantisation adjustment
+ * @tkid: Timekeeper ID
+ * @cs_tick_adj: Per-second adjustment in ns << NTP_SCALE_SHIFT
+ *
+ * The timekeeping core uses an integer number of cycles (@cycle_interval)
+ * per NTP interval, so the real time that interval represents differs from
+ * the nominal NTP_INTERVAL_LENGTH by up to half a counter period. Folding
+ * this fixed offset into @cs_tick_adj makes it an explicit part of the NTP
+ * tick_length computation in ntp.c, instead of being applied during
+ * timekeeping accumulation where the NTP code never saw it. Like
+ * @ntp_tick_adj it stays internal to the kernel; userspace still sees the
+ * nominal tick via adjtimex. NTP retains its full symmetric ±MAXFREQ range
+ * around the corrected base rate.
+ *
+ * Called whenever the clocksource is (re)configured, which is also when the
+ * rest of the NTP state must be cleared, so the two are done together.
*/
-void ntp_clear(unsigned int tkid)
+void ntp_clear(unsigned int tkid, s64 cs_tick_adj)
{
+ tk_ntp_data[tkid].cs_tick_adj = cs_tick_adj;
__ntp_clear(&tk_ntp_data[tkid]);
}
@@ -364,6 +392,186 @@ u64 ntp_tick_length(unsigned int tkid)
return tk_ntp_data[tkid].tick_length;
}
+s64 ntp_get_skew_delta(unsigned int tkid)
+{
+ return tk_ntp_data[tkid].skew_delta;
+}
+
+/* Sign of @x as +1 or -1 (zero counts as positive; callers pass nonzero). */
+static inline int signof(s64 x)
+{
+ return x < 0 ? -1 : 1;
+}
+
+static s64 ntp_drain_time_offset(unsigned int tkid, s64 amount)
+{
+ struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+
+ /* Only drain if amount and time_offset have the same sign */
+ if (!amount || signof(amount) != signof(ntpdata->time_offset))
+ return amount;
+
+ /* Clamp: don't overshoot zero */
+ if (abs(amount) > abs(ntpdata->time_offset)) {
+ s64 undrained = amount - ntpdata->time_offset;
+
+ ntpdata->time_offset = 0;
+ return undrained;
+ }
+
+ ntpdata->time_offset -= amount;
+ return 0;
+}
+
+/*
+ * Drain the legacy adjtime() correction (time_adjust) as it is delivered.
+ *
+ * @amount is the total intentional per-tick skew for this accumulation
+ * (skew_delta << shift), in time_offset units (shifted_ns / HZ); it covers
+ * both the exponential time_offset slew and the linear adjtime slew. This
+ * function claims only the adjtime share — capped at the MAX_TICKADJ rate —
+ * and returns the remainder for ntp_drain_time_offset().
+ *
+ * time_adjust is in whole µs. The sub-µs remainder being delivered lives in
+ * time_adjust_frac (plain shifted-ns, i.e. ns << NTP_SCALE_SHIFT -- unlike
+ * time_offset these are NOT pre-divided by HZ); we top it up by borrowing
+ * whole microseconds from time_adjust as the drain consumes it.
+ */
+static s64 ntp_drain_time_adjust(unsigned int tkid, s64 amount, unsigned int shift)
+{
+ struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+ /* Sign reference: time_adjust if any whole us remain, else the drawer */
+ s64 ref = ntpdata->time_adjust ? (s64)ntpdata->time_adjust
+ : ntpdata->time_adjust_frac;
+ s64 deliver, deficit, claimed;
+
+ if (!amount || !ref || signof(amount) != signof(ref))
+ return amount;
+
+ /*
+ * Phase to deliver this accumulation, in plain shifted-ns. The drain
+ * @amount is in ÷HZ units, so multiply by HZ first, then clamp to the
+ * MAX_TICKADJ rate (MAX_TICKADJ_SCALED is the per-tick slew in
+ * shifted-ns). Multiply-then-clamp avoids an s64 divide for the cap.
+ */
+ deliver = min(abs(amount) * NTP_INTERVAL_FREQ,
+ (s64)MAX_TICKADJ_SCALED << shift);
+
+ /* Top up the sub-µs drawer from whole-µs time_adjust as needed */
+ deficit = deliver - abs(ntpdata->time_adjust_frac);
+ if (deficit > 0 && ntpdata->time_adjust) {
+ long borrow = div64_u64(deficit + ONE_US_NS - 1, ONE_US_NS);
+
+ if (ntpdata->time_adjust > 0) {
+ borrow = min(borrow, ntpdata->time_adjust);
+ ntpdata->time_adjust -= borrow;
+ ntpdata->time_adjust_frac += (s64)borrow * ONE_US_NS;
+ } else {
+ /* Clamp without negating time_adjust (UB for LONG_MIN) */
+ if (ntpdata->time_adjust > -borrow)
+ borrow = -ntpdata->time_adjust;
+ ntpdata->time_adjust += borrow;
+ ntpdata->time_adjust_frac -= (s64)borrow * ONE_US_NS;
+ }
+ }
+
+ /* Never deliver more than the drawer holds */
+ deliver = min(deliver, abs(ntpdata->time_adjust_frac));
+ if (ntpdata->time_adjust_frac > 0)
+ ntpdata->time_adjust_frac -= deliver;
+ else
+ ntpdata->time_adjust_frac += deliver;
+
+ /* Return the unclaimed remainder in ÷HZ drain units for time_offset */
+ claimed = div_s64(deliver, NTP_INTERVAL_FREQ);
+ return amount - signof(amount) * claimed;
+}
+
+/*
+ * Drain one accumulation's worth of intentional skew as it is delivered.
+ *
+ * @amount is the total intentional per-tick skew for this accumulation
+ * (skew_delta << shift), in time_offset units (shifted_ns / HZ). The
+ * adjtime() linear share is taken from time_adjust first (capped at the
+ * MAX_TICKADJ rate, hence @shift), then the exponential remainder from
+ * time_offset. Returns the amount actually claimed (same ÷HZ units).
+ */
+s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift)
+{
+ s64 unclaimed = ntp_drain_time_adjust(tkid, amount, shift);
+
+ unclaimed = ntp_drain_time_offset(tkid, unclaimed);
+
+ /*
+ * Return the amount actually drained from the intentional
+ * phase offset in time_offset and/or time_adjust.
+ */
+ return amount - unclaimed;
+}
+
+/*
+ * time_offset (drained exponentially) and time_adjust (drained linearly at the
+ * MAX_TICKADJ rate) can be asked to slew the clock in opposite directions.
+ * second_overflow() only folds their *net* into skew_delta, so the cancelling
+ * part would never be drained from either tracker via the per-tick code -- and
+ * if they cancel exactly, skew_delta is zero and neither converges at all.
+ *
+ * Settle that cancelling phase directly between the two here. No clock motion
+ * results (the opposing slews annihilate), but both move toward zero so neither
+ * stalls. @amount is the phase to take off time_offset, in its (÷HZ) units and
+ * with its sign; the same real magnitude comes off time_adjust in the opposite
+ * direction. Clamped so neither tracker is driven past zero.
+ */
+static void ntp_transfer_offset_adjust(struct ntp_data *ntpdata, s64 amount)
+{
+ s64 frac_delta, carry;
+
+ /*
+ * Don't drain time_offset past zero. @amount shares its sign and is
+ * normally bounded below it by ntp_offset_chunk(), but the ±1 skew_delta
+ * floor for a tiny time_offset can exceed it, so clamp.
+ */
+ if (abs(amount) > abs(ntpdata->time_offset))
+ amount = ntpdata->time_offset;
+ if (!amount)
+ return;
+
+ /*
+ * Remove the matching phase from time_adjust, in plain shifted-ns. No
+ * clamp against time_adjust's zero is needed: @amount is bounded by the
+ * adjtime chunk, which second_overflow() never lets exceed time_adjust's
+ * own pending phase, so this cannot overshoot.
+ */
+ frac_delta = amount * NTP_INTERVAL_FREQ;
+
+ ntpdata->time_offset -= amount;
+
+ /* Add the matching phase to time_adjust, carrying whole µs (O(1)). */
+ ntpdata->time_adjust_frac += frac_delta;
+ if (ntpdata->time_adjust_frac >= ONE_US_NS ||
+ ntpdata->time_adjust_frac <= -ONE_US_NS) {
+ carry = div64_s64(ntpdata->time_adjust_frac, ONE_US_NS);
+ ntpdata->time_adjust += carry;
+ ntpdata->time_adjust_frac -= carry * ONE_US_NS;
+ }
+
+ /*
+ * Keep time_adjust and its sub-µs remainder the same sign. The
+ * truncating carry above can leave them opposed (e.g. +4 µs paired
+ * with -250 ns), and ntp_drain_time_adjust() treats abs(time_adjust_frac)
+ * as same-direction drawer capacity -- an opposing remainder there makes
+ * it over-deliver phase that was never removed from the pile. Borrow or
+ * repay a single whole µs to realign; the total phase is unchanged.
+ */
+ if (ntpdata->time_adjust > 0 && ntpdata->time_adjust_frac < 0) {
+ ntpdata->time_adjust--;
+ ntpdata->time_adjust_frac += ONE_US_NS;
+ } else if (ntpdata->time_adjust < 0 && ntpdata->time_adjust_frac > 0) {
+ ntpdata->time_adjust++;
+ ntpdata->time_adjust_frac -= ONE_US_NS;
+ }
+}
+
/**
* ntp_get_next_leap - Returns the next leapsecond in CLOCK_REALTIME ktime_t
* @tkid: Timekeeper ID
@@ -398,7 +606,6 @@ ktime_t ntp_get_next_leap(unsigned int tkid)
int second_overflow(unsigned int tkid, time64_t secs)
{
struct ntp_data *ntpdata = &tk_ntp_data[tkid];
- s64 delta;
int leap = 0;
s32 rem;
@@ -458,35 +665,70 @@ int second_overflow(unsigned int tkid, time64_t secs)
}
/* Compute the phase adjustment for the next second */
- ntpdata->tick_length = ntpdata->tick_length_base;
-
- delta = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
- ntpdata->time_offset -= delta;
- ntpdata->tick_length += delta;
/* Check PPS signal */
pps_dec_valid(ntpdata);
- if (!ntpdata->time_adjust)
- goto out;
+ /*
+ * Set the per-tick skew rate for the next second. This is in
+ * the same units as time_offset: (ns << NTP_SCALE_SHIFT) / HZ.
+ * If the result is so low that the skew imparted would round
+ * to zero, pass the bare minimum ±1 to ensure that it *does*
+ * actually drain completely to zero. It won't overshoot because
+ * logarithmic_accumulation() only drains what it can from
+ * time_offset or time_adjust, and the rest ends up in ntp_error
+ * which drives the selection of 'mult' immediately each tick.
+ */
+ if (ntpdata->time_offset || ntpdata->time_adjust ||
+ ntpdata->time_adjust_frac) {
+ s64 off_chunk = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
+ s64 adj_chunk = 0, net;
- if (ntpdata->time_adjust > MAX_TICKADJ) {
- ntpdata->time_adjust -= MAX_TICKADJ;
- ntpdata->tick_length += MAX_TICKADJ_SCALED;
- goto out;
- }
+ /*
+ * Once the exponential chunk rounds to zero, deliver the last
+ * remaining offset this second so it converges to zero instead
+ * of stalling just above it.
+ */
+ if (!off_chunk)
+ off_chunk = ntpdata->time_offset;
+
+ if (ntpdata->time_adjust || ntpdata->time_adjust_frac) {
+ s64 adj;
+
+ if (ntpdata->time_adjust >= MAX_TICKADJ)
+ adj = MAX_TICKADJ * ONE_US_NS;
+ else if (ntpdata->time_adjust <= -MAX_TICKADJ)
+ adj = -MAX_TICKADJ * ONE_US_NS;
+ else
+ adj = ntpdata->time_adjust * ONE_US_NS +
+ ntpdata->time_adjust_frac;
+
+ adj_chunk = div_s64(adj, NTP_INTERVAL_FREQ);
+ if (!adj_chunk)
+ adj_chunk = signof(ntpdata->time_adjust_frac);
+ }
- if (ntpdata->time_adjust < -MAX_TICKADJ) {
- ntpdata->time_adjust += MAX_TICKADJ;
- ntpdata->tick_length -= MAX_TICKADJ_SCALED;
- goto out;
- }
+ /*
+ * If the two slews oppose, only their net would drive the
+ * per-tick drain, so the cancelling part would never drain from
+ * either tracker and an exact cancellation would stall both.
+ * Settle that overlap directly between them (no clock motion).
+ */
+ if (off_chunk && adj_chunk && signof(off_chunk) != signof(adj_chunk)) {
+ s64 conflict = min(abs(off_chunk), abs(adj_chunk));
- ntpdata->tick_length += (s64)(ntpdata->time_adjust * NSEC_PER_USEC / NTP_INTERVAL_FREQ)
- << NTP_SCALE_SHIFT;
- ntpdata->time_adjust = 0;
+ ntp_transfer_offset_adjust(ntpdata, signof(off_chunk) * conflict);
+ }
+
+ /* Net is what the clock delivers; reduce to per-tick, then floor. */
+ net = off_chunk + adj_chunk;
+ ntpdata->skew_delta = div_s64(net, NTP_INTERVAL_FREQ);
+ if (!ntpdata->skew_delta && net)
+ ntpdata->skew_delta = signof(net);
+ } else {
+ ntpdata->skew_delta = 0;
+ }
-out:
return leap;
}
@@ -779,6 +1021,7 @@ int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct tim
if (!(txc->modes & ADJ_OFFSET_READONLY)) {
/* adjtime() is independent from ntp_adjtime() */
ntpdata->time_adjust = txc->offset;
+ ntpdata->time_adjust_frac = 0;
ntp_update_frequency(ntpdata);
audit_ntp_set_old(ad, AUDIT_NTP_ADJUST, save_adjust);
@@ -1020,6 +1263,7 @@ static void hardpps_update_phase(struct ntp_data *ntpdata, long error)
NTP_INTERVAL_FREQ);
/* Cancel running adjtime() */
ntpdata->time_adjust = 0;
+ ntpdata->time_adjust_frac = 0;
}
/* Update jitter */
ntpdata->pps_jitter += (jitter - ntpdata->pps_jitter) >> PPS_INTMIN;
diff --git a/kernel/time/ntp_internal.h b/kernel/time/ntp_internal.h
index 7084d839c207..0474a761bafc 100644
--- a/kernel/time/ntp_internal.h
+++ b/kernel/time/ntp_internal.h
@@ -3,9 +3,11 @@
#define _LINUX_NTP_INTERNAL_H
extern void ntp_init(void);
-extern void ntp_clear(unsigned int tkid);
+extern void ntp_clear(unsigned int tkid, s64 cs_tick_adj);
/* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */
extern u64 ntp_tick_length(unsigned int tkid);
+extern s64 ntp_get_skew_delta(unsigned int tkid);
+extern s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift);
extern ktime_t ntp_get_next_leap(unsigned int tkid);
extern int second_overflow(unsigned int tkid, time64_t secs);
extern int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct timespec64 *ts,
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 5e633d8750d1..0bf4fcd969c8 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -408,6 +408,7 @@ static int posix_cpu_timer_create(struct k_itimer *new_timer)
new_timer->kclock = &clock_posix_cpu;
timerqueue_init(&new_timer->it.cpu.node);
+ INIT_LIST_HEAD(&new_timer->it.cpu.elist);
new_timer->it.cpu.pid = get_pid(pid);
rcu_read_unlock();
return 0;
@@ -461,6 +462,127 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p)
trigger_base_recalc_expires(timer, p);
}
+/*
+ * Lookup the task via timer->it.cpu.pid and attempt to lock the task's sighand.
+ *
+ * This can race with the reaping of the task:
+ *
+ * CPU0 CPU1
+ *
+ * // Finds task
+ * p = pid_task(pid, pid_type); __exit_signal(p)
+ * lock(p, sighand);
+ * posix_cpu_timers*_exit();
+ * sighand = lock_task_sighand(p); unhash_task(p);
+ * p->sighand = NULL;
+ * unlock(sighand);
+ *
+ * In this case sighand is NULL, which means the task and the associated timer
+ * queue cannot be longer accessed safely.
+ *
+ * __exit_signal() invokes posix_cpu_timers_exit() and if the thread group is
+ * dead it also invokes posix_cpu_timers_group_exit(). These functions delete
+ * all pending timers from the related timer queues. The POSIX timers (k_itimer)
+ * themself are still accessible, but not longer connected to the task.
+ *
+ * exec() works slightly differently. The task which exec()'s terminates all
+ * other threads in the thread group and runs __exit_signal() on them. As the
+ * thread group is not dead they only clean up the per task timers via
+ * posix_cpu_timers_exit().
+ *
+ * As the TGID on exec() stays the same per process timers stay queued, if they
+ * are armed. This works without a problem when exec() is done by the thread
+ * group leader. If a non-leader thread exec()'s this can end up in the
+ * following scenario:
+ *
+ * CPU0 CPU1
+ * // Returns old leader
+ * p = pid_task(pid, pid_type); de_thread()
+ * switch_leader()
+ * release_task(old leader)
+ * __exit_signal()
+ * old_leader->sighand = NULL;
+ * // Returns NULL
+ * sighand = lock_task_sighand(p)
+ *
+ * That's problematic for several functions:
+ *
+ * - posix_cpu_timer_del(): If the timer is still enqueued on the task the
+ * underlying k_itimer will be freed which results in a UAF in
+ * run_posix_cpu_timers() or on timerqueue related add/delete operations.
+ * If the timer is not enqueued, the failure is harmless
+ *
+ * - posix_cpu_timer_set(): Independent of the enqueued state that results in a
+ * transient failure which is user space visible (-ESRCH) for regular posix
+ * timers. But for the use case in do_cpu_nanosleep() it's the same UAF
+ * problem just that the timer is allocated on the stack.
+ *
+ * - posix_cpu_timer_rearm(): Timer is not enqueued at that point, but this
+ * silently ignores the rearm request, which is a functional problem as the
+ * timer wont expire anymore.
+ */
+static struct task_struct *timer_lock_sighand(struct k_itimer *timer, unsigned long *flags)
+{
+ enum pid_type type = clock_pid_type(timer->it_clock);
+ struct cpu_timer *ctmr = &timer->it.cpu;
+
+ guard(rcu)();
+
+ for (;;) {
+ struct task_struct *t = pid_task(timer->it.cpu.pid, type);
+
+ /* Fail if the task cannot be found. */
+ if (!t)
+ break;
+
+ /* Try to lock the task's sighand */
+ if (lock_task_sighand(t, flags))
+ return t;
+
+ /*
+ * The next PID lookup might either fail or return the new
+ * leader. This is correct for both exit() and exec().
+ */
+ }
+
+ /*
+ * If the timer is still enqueued, warn. There is nothing safe to do
+ * here as there might be two timers in there which are removed in
+ * parallel and that will cause more damage than good. This should never
+ * happen!
+ *
+ * Ensure that the stores to the timer and timerqueue are visible:
+ *
+ * __exit_signal()
+ * posix_cpu_timers*_exit()
+ * write_seqlock(seqlock)
+ * smp_wmb(); <-------
+ * __unhash_process() | !pid_task()
+ * ----> smp_rmb();
+ * WARN_ON_ONCE(...)
+ */
+ smp_rmb();
+ WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node));
+ return NULL;
+}
+
+/*
+ * If the timer is queued on the expiry list, then it cannot be dequeued because
+ * the firing list is not protected by sighand->lock. The delivery path is
+ * waiting for the timer lock. So go back, unlock and retry.
+ */
+static bool posix_cpu_timer_on_expiry_list(struct k_itimer *timer)
+{
+ if (list_empty(&timer->it.cpu.elist))
+ return false;
+
+ /*
+ * Prevent signal delivery as there is no point in delivering a signal
+ * which is made obsolete right away.
+ */
+ timer->it.cpu.firing = false;
+ return true;
+}
/*
* Clean up a CPU-clock timer that is about to be destroyed.
@@ -470,49 +592,22 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p)
*/
static int posix_cpu_timer_del(struct k_itimer *timer)
{
- struct cpu_timer *ctmr = &timer->it.cpu;
- struct sighand_struct *sighand;
struct task_struct *p;
unsigned long flags;
int ret = 0;
- rcu_read_lock();
- p = cpu_timer_task_rcu(timer);
- if (!p)
- goto out;
+ p = timer_lock_sighand(timer, &flags);
- /*
- * Protect against sighand release/switch in exit/exec and process/
- * thread timer list entry concurrent read/writes.
- */
- sighand = lock_task_sighand(p, &flags);
- if (unlikely(sighand == NULL)) {
- /*
- * This raced with the reaping of the task. The exit cleanup
- * should have removed this timer from the timer queue.
- */
- WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node));
- } else {
- if (timer->it.cpu.firing) {
- /*
- * Prevent signal delivery. The timer cannot be dequeued
- * because it is on the firing list which is not protected
- * by sighand->lock. The delivery path is waiting for
- * the timer lock. So go back, unlock and retry.
- */
- timer->it.cpu.firing = false;
+ if (likely(p)) {
+ if (posix_cpu_timer_on_expiry_list(timer))
ret = TIMER_RETRY;
- } else {
+ else
disarm_timer(timer, p);
- }
unlock_task_sighand(p, &flags);
}
-out:
- rcu_read_unlock();
-
if (!ret) {
- put_pid(ctmr->pid);
+ put_pid(timer->it.cpu.pid);
timer->it_status = POSIX_TIMER_DISARMED;
}
return ret;
@@ -626,21 +721,17 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
struct cpu_timer *ctmr = &timer->it.cpu;
u64 old_expires, new_expires, now;
- struct sighand_struct *sighand;
struct task_struct *p;
unsigned long flags;
int ret = 0;
- rcu_read_lock();
- p = cpu_timer_task_rcu(timer);
- if (!p) {
- /*
- * If p has just been reaped, we can no
- * longer get any information about it at all.
- */
- rcu_read_unlock();
+ p = timer_lock_sighand(timer, &flags);
+ /*
+ * If p has just been reaped, we can no longer get any information about
+ * it at all.
+ */
+ if (!p)
return -ESRCH;
- }
/*
* Use the to_ktime conversion because that clamps the maximum
@@ -648,31 +739,10 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
*/
new_expires = ktime_to_ns(timespec64_to_ktime(new->it_value));
- /*
- * Protect against sighand release/switch in exit/exec and p->cpu_timers
- * and p->signal->cpu_timers read/write in arm_timer()
- */
- sighand = lock_task_sighand(p, &flags);
- /*
- * If p has just been reaped, we can no
- * longer get any information about it at all.
- */
- if (unlikely(sighand == NULL)) {
- rcu_read_unlock();
- return -ESRCH;
- }
-
/* Retrieve the current expiry time before disarming the timer */
old_expires = cpu_timer_getexpires(ctmr);
- if (unlikely(timer->it.cpu.firing)) {
- /*
- * Prevent signal delivery. The timer cannot be dequeued
- * because it is on the firing list which is not protected
- * by sighand->lock. The delivery path is waiting for
- * the timer lock. So go back, unlock and retry.
- */
- timer->it.cpu.firing = false;
+ if (posix_cpu_timer_on_expiry_list(timer)) {
ret = TIMER_RETRY;
} else {
cpu_timer_dequeue(ctmr);
@@ -698,7 +768,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
/* Retry if the timer expiry is running concurrently */
if (unlikely(ret)) {
unlock_task_sighand(p, &flags);
- goto out;
+ return ret;
}
/* Convert relative expiry time to absolute */
@@ -733,8 +803,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
*/
if (!sigev_none && new_expires && now >= new_expires)
cpu_timer_fire(timer);
-out:
- rcu_read_unlock();
return ret;
}
@@ -1018,19 +1086,12 @@ static void check_process_timers(struct task_struct *tsk,
static bool posix_cpu_timer_rearm(struct k_itimer *timer)
{
clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
- struct sighand_struct *sighand;
struct task_struct *p;
unsigned long flags;
u64 now;
- guard(rcu)();
- p = cpu_timer_task_rcu(timer);
- if (!p)
- return true;
-
- /* Protect timer list r/w in arm_timer() */
- sighand = lock_task_sighand(p, &flags);
- if (unlikely(sighand == NULL))
+ p = timer_lock_sighand(timer, &flags);
+ if (unlikely(!p))
return true;
/*
@@ -1300,8 +1361,11 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
unsigned long flags, start;
LIST_HEAD(firing);
- if (!lock_task_sighand(tsk, &flags))
- return;
+ /*
+ * tsk is current and ->sighand is stable, see the
+ * tsk->exit_state check in run_posix_cpu_timers()
+ */
+ spin_lock_irqsave(&tsk->sighand->siglock, flags);
do {
/*
@@ -1361,7 +1425,7 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
* that gets the timer lock before we do will give it up and
* spin until we've taken care of that timer below.
*/
- unlock_task_sighand(tsk, &flags);
+ spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
/*
* Now that all the timers on our list have the firing flag,
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 115e0bf01276..bda3d2391a60 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -165,23 +165,31 @@ static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev,
*/
void tick_install_broadcast_device(struct clock_event_device *dev, int cpu)
{
- struct clock_event_device *cur = tick_broadcast_device.evtdev;
+ struct clock_event_device *cur;
- if (tick_set_oneshot_wakeup_device(dev, cpu))
- return;
+ scoped_guard(raw_spinlock_irqsave, &tick_broadcast_lock) {
- if (!tick_check_broadcast_device(cur, dev))
- return;
+ if (tick_set_oneshot_wakeup_device(dev, cpu))
+ return;
- if (!try_module_get(dev->owner))
- return;
+ cur = tick_broadcast_device.evtdev;
+ if (!tick_check_broadcast_device(cur, dev))
+ return;
- clockevents_exchange_device(cur, dev);
+ if (!try_module_get(dev->owner))
+ return;
+
+ __clockevents_exchange_device(cur, dev);
+ if (cur)
+ cur->event_handler = clockevents_handle_noop;
+ WRITE_ONCE(tick_broadcast_device.evtdev, dev);
+ if (!cpumask_empty(tick_broadcast_mask))
+ tick_broadcast_start_periodic(dev);
+ }
+
+ /* Module release must be outside of the lock */
if (cur)
- cur->event_handler = clockevents_handle_noop;
- tick_broadcast_device.evtdev = dev;
- if (!cpumask_empty(tick_broadcast_mask))
- tick_broadcast_start_periodic(dev);
+ module_put(cur->owner);
if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))
return;
@@ -1218,7 +1226,7 @@ int tick_broadcast_oneshot_active(void)
*/
bool tick_broadcast_oneshot_available(void)
{
- struct clock_event_device *bc = tick_broadcast_device.evtdev;
+ struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev);
return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
}
@@ -1226,7 +1234,7 @@ bool tick_broadcast_oneshot_available(void)
#else
int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
{
- struct clock_event_device *bc = tick_broadcast_device.evtdev;
+ struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev);
if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))
return -EBUSY;
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 597d816d22e8..65680db95053 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -3,6 +3,7 @@
* tick internal variable and functions used by low/high res code
*/
#include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
#include <linux/tick.h>
#include "timekeeping.h"
@@ -54,6 +55,8 @@ static inline void clockevent_set_state(struct clock_event_device *dev,
}
extern void clockevents_shutdown(struct clock_event_device *dev);
+extern void __clockevents_exchange_device(struct clock_event_device *old,
+ struct clock_event_device *new);
extern void clockevents_exchange_device(struct clock_event_device *old,
struct clock_event_device *new);
extern void clockevents_switch_state(struct clock_event_device *dev,
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 98a9cae915c0..6c3fea386713 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -828,8 +828,7 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
ts->timer_expires_base = basemono;
/*
- * Keep the periodic tick, when RCU, architecture or irq_work
- * requests it.
+ * Keep the periodic tick, when RCU or irq_work requests it.
* Aside of that, check whether the local timer softirq is
* pending. If so, its a bad idea to call get_next_timer_interrupt(),
* because there is an already expired timer, so it will request
@@ -837,8 +836,8 @@ static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
* minimal delta, which brings us back to this place
* immediately. Lather, rinse and repeat...
*/
- if (rcu_needs_cpu() || arch_needs_cpu() ||
- irq_work_needs_cpu() || local_timer_softirq_pending()) {
+ if (rcu_needs_cpu() || irq_work_needs_cpu() ||
+ local_timer_softirq_pending()) {
next_tick = basemono + TICK_NSEC;
} else {
/*
diff --git a/kernel/time/time.c b/kernel/time/time.c
index 0dd63a91e7c5..079ab34f61db 100644
--- a/kernel/time/time.c
+++ b/kernel/time/time.c
@@ -42,6 +42,13 @@
#include <generated/timeconst.h>
#include "timekeeping.h"
+#include "timekeeping_internal.h"
+
+#if defined(CONFIG_64BIT) || defined(CONFIG_COMPAT_32BIT_TIME)
+#define __WANT_OLD_TIME_TYPE_SYSCALL 1
+#endif
+
+static_assert(sizeof(__kernel_old_time_t) == 8 ? IS_ENABLED(__WANT_OLD_TIME_TYPE_SYSCALL) : true);
/*
* The timezone where the local system is located. Used as a default by some
@@ -51,7 +58,7 @@ struct timezone sys_tz;
EXPORT_SYMBOL(sys_tz);
-#ifdef __ARCH_WANT_SYS_TIME
+#if defined(__ARCH_WANT_SYS_TIME) && defined(__WANT_OLD_TIME_TYPE_SYSCALL)
/*
* sys_time() can be implemented in user-level using
@@ -96,7 +103,7 @@ SYSCALL_DEFINE1(stime, __kernel_old_time_t __user *, tptr)
return 0;
}
-#endif /* __ARCH_WANT_SYS_TIME */
+#endif /* __ARCH_WANT_SYS_TIME && __WANT_OLD_TIME_TYPE_SYSCALL */
#ifdef CONFIG_COMPAT_32BIT_TIME
#ifdef __ARCH_WANT_SYS_TIME32
@@ -137,6 +144,7 @@ SYSCALL_DEFINE1(stime32, old_time32_t __user *, tptr)
#endif /* __ARCH_WANT_SYS_TIME32 */
#endif
+#ifdef __WANT_OLD_TIME_TYPE_SYSCALL
SYSCALL_DEFINE2(gettimeofday, struct __kernel_old_timeval __user *, tv,
struct timezone __user *, tz)
{
@@ -154,6 +162,7 @@ SYSCALL_DEFINE2(gettimeofday, struct __kernel_old_timeval __user *, tv,
}
return 0;
}
+#endif /* __WANT_OLD_TIME_TYPE_SYSCALL */
/*
* In case for some reason the CMOS clock has not already been running
@@ -203,6 +212,9 @@ SYSCALL_DEFINE2(settimeofday, struct __kernel_old_timeval __user *, tv,
struct timezone new_tz;
if (tv) {
+ if (!IS_ENABLED(__WANT_OLD_TIME_TYPE_SYSCALL))
+ return -EINVAL;
+
if (get_user(new_ts.tv_sec, &tv->tv_sec) ||
get_user(new_ts.tv_nsec, &tv->tv_usec))
return -EFAULT;
@@ -220,7 +232,7 @@ SYSCALL_DEFINE2(settimeofday, struct __kernel_old_timeval __user *, tv,
return do_sys_settimeofday64(tv ? &new_ts : NULL, tz ? &new_tz : NULL);
}
-#ifdef CONFIG_COMPAT
+#ifdef CONFIG_COMPAT_32BIT_TIME
COMPAT_SYSCALL_DEFINE2(gettimeofday, struct old_timeval32 __user *, tv,
struct timezone __user *, tz)
{
@@ -239,7 +251,9 @@ COMPAT_SYSCALL_DEFINE2(gettimeofday, struct old_timeval32 __user *, tv,
return 0;
}
+#endif /* CONFIG_COMPAT_32BIT_TIME */
+#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv,
struct timezone __user *, tz)
{
@@ -247,6 +261,9 @@ COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv,
struct timezone new_tz;
if (tv) {
+ if (!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME))
+ return -EINVAL;
+
if (get_user(new_ts.tv_sec, &tv->tv_sec) ||
get_user(new_ts.tv_nsec, &tv->tv_usec))
return -EFAULT;
@@ -263,7 +280,7 @@ COMPAT_SYSCALL_DEFINE2(settimeofday, struct old_timeval32 __user *, tv,
return do_sys_settimeofday64(tv ? &new_ts : NULL, tz ? &new_tz : NULL);
}
-#endif
+#endif /* CONFIG_COMPAT */
#ifdef CONFIG_64BIT
SYSCALL_DEFINE1(adjtimex, struct __kernel_timex __user *, txc_p)
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index b1b5ec43c0f2..ea2e6e55f37b 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -339,7 +339,6 @@ static inline void clocksource_enable_inline_read(void) { }
static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
{
u64 interval;
- u64 tmp, ntpinterval;
struct clocksource *old_clock;
++tk->cs_was_changed_seq;
@@ -353,20 +352,16 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
/* Do the ns -> cycle conversion first, using original mult */
- tmp = NTP_INTERVAL_LENGTH;
- tmp <<= clock->shift;
- ntpinterval = tmp;
- tmp += clock->mult/2;
- do_div(tmp, clock->mult);
- if (tmp == 0)
- tmp = 1;
-
- interval = (u64) tmp;
+ interval = (u64)NTP_INTERVAL_LENGTH << clock->shift;
+ interval += clock->mult / 2;
+ do_div(interval, clock->mult);
+ if (interval == 0)
+ interval = 1;
+
tk->cycle_interval = interval;
/* Go back from cycles -> shifted ns */
tk->xtime_interval = interval * clock->mult;
- tk->xtime_remainder = ntpinterval - tk->xtime_interval;
tk->raw_interval = interval * clock->mult;
/* if changing clocks, convert xtime_nsec shift units */
@@ -386,7 +381,38 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
tk->ntp_error = 0;
tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
- tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
+
+ /*
+ * ntp_tick is the tick length that NTP disciplines (its ±500 PPM
+ * scales only this part), in NTP-shifted ns: the real interval of
+ * a whole number of counter cycles. Because cycle_interval is
+ * rounded to an integer number of cycles, this ntp_tick differs
+ * from the true intended 1/HZ tick length by up to half a cycle
+ * period.
+ */
+ tk->ntp_tick = (u64)tk->xtime_interval << tk->ntp_error_shift;
+
+ /*
+ * cs_tick_adj is the constant difference between the disciplined
+ * ntp_tick above and the true 1/HZ tick, expressed per-second to
+ * match the ntp_update_frequency() addends and handed to NTP via
+ * ntp_clear() to be explicitly included in its tick_length.
+ *
+ * Worked example: HZ=1000, ACPI PM timer at 3.579545 MHz, which
+ * has 3579.545 cycles in 1ms, rounded to cycle_interval = 3580.
+ *
+ * So ntp_tick is actually 1.000127ms, as that is the amount of
+ * time that 3580 cycles will take at the nominal frequency. This
+ * is the part that NTP disciplines, causing each 3580 counts to
+ * advance the clock by up to NTP's ±500PPM of that amount.
+ *
+ * The "extra" 127ns/tick is what's stored in cs_tick_adj and
+ * applied as a constant correction by ntp_update_frequency() so
+ * that NTP *believes* it's disciplining a 1ms tick.
+ */
+ tk->cs_tick_adj = (s64)tk->ntp_tick -
+ ((s64)NTP_INTERVAL_LENGTH << NTP_SCALE_SHIFT);
+ tk->cs_tick_adj *= NTP_INTERVAL_FREQ;
/*
* The timekeeper keeps its own mult values for the currently
@@ -397,6 +423,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
tk->tkr_raw.mult = clock->mult;
tk->ntp_err_mult = 0;
tk->skip_second_overflow = 0;
+ tk->skew_delta = 0;
tk->cs_id = clock->id;
@@ -803,7 +830,7 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act
if (action & TK_CLEAR_NTP) {
tk->ntp_error = 0;
- ntp_clear(tk->id);
+ ntp_clear(tk->id, tk->cs_tick_adj);
}
tk_update_leap_state(tk);
@@ -831,7 +858,11 @@ static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int act
* the downside that the reader side does not longer benefit from
* the cacheline optimized data layout of the timekeeper and requires
* another indirection.
+ *
+ * Write xtime_sec first so that even if the memcpy() tears the store
+ * data integrity is provided for ktime_get_real_seconds().
*/
+ WRITE_ONCE(tkd->timekeeper.xtime_sec, tk->xtime_sec);
memcpy(&tkd->timekeeper, tk, sizeof(*tk));
write_seqcount_end(&tkd->seq);
}
@@ -1159,11 +1190,11 @@ time64_t ktime_get_real_seconds(void)
unsigned int seq;
if (IS_ENABLED(CONFIG_64BIT))
- return tk->xtime_sec;
+ return READ_ONCE(tk->xtime_sec);
do {
seq = read_seqcount_begin(&tk_core.seq);
- seconds = tk->xtime_sec;
+ seconds = READ_ONCE(tk->xtime_sec);
} while (read_seqcount_retry(&tk_core.seq, seq));
@@ -1185,7 +1216,7 @@ noinstr time64_t __ktime_get_real_seconds(void)
{
struct timekeeper *tk = &tk_core.timekeeper;
- return tk->xtime_sec;
+ return READ_ONCE(tk->xtime_sec);
}
static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr,
@@ -1202,10 +1233,21 @@ static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr,
/**
* ktime_get_snapshot_id - Simultaneously snapshot a given clock ID with
- * CLOCK_MONOTONIC_RAW and the underlying
+ * the corresponding monotonic raw and the underlying
* clocksource counter value.
* @clock_id: The clock ID to snapshot
* @systime_snapshot: Pointer to struct receiving the system time snapshot
+ *
+ * For the system time keeping clocks (REALTIME, MONOTONIC and BOOTTIME) the
+ * monotonic raw clock is CLOCK_MONOTONIC_RAW. For AUX clocks this is the
+ * monotonic raw clock related to the AUX clock. These AUX clock related
+ * monotonic raw clocks have a strict linear offset to the system time
+ * CLOCK_MONOTONIC_RAW:
+ *
+ * MONOTONIC_RAW(AUX$N) = CLOCK_MONOTONIC_RAW(system) + offset(AUX$N)
+ *
+ * The offset is established when a AUX clock is initialized, but it is
+ * currently not accessible.
*/
void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot)
{
@@ -1512,6 +1554,9 @@ EXPORT_SYMBOL_GPL(ktime_real_to_base_clock);
* @xtstamp: Receives simultaneously captured system and device time
*
* Reads a timestamp from a device and correlates it to system time
+ *
+ * See documentation for ktime_get_snapshot_id() for information about the raw
+ * monotonic time stamp which is used here.
*/
int get_device_system_crosststamp(int (*get_time_fn)
(ktime_t *device_time,
@@ -1522,10 +1567,11 @@ int get_device_system_crosststamp(int (*get_time_fn)
struct system_device_crosststamp *xtstamp)
{
u64 syscnt_cycles, cycles, now, interval_start;
- unsigned int seq, clock_was_set_seq = 0;
ktime_t base_sys, base_raw, *offs;
+ u32 clock_was_set_seq = 0;
u64 nsec_sys, nsec_raw;
u8 cs_was_changed_seq;
+ unsigned int seq;
bool do_interp;
struct timekeeper *tk;
struct tk_data *tkd;
@@ -1897,40 +1943,6 @@ void ktime_get_raw_ts64(struct timespec64 *ts)
EXPORT_SYMBOL(ktime_get_raw_ts64);
/**
- * ktime_get_clock_ts64 - Returns time of a clock in a timespec
- * @id: POSIX clock ID of the clock to read
- * @ts: Pointer to the timespec64 to be set
- *
- * The timestamp is invalidated (@ts->sec is set to -1) if the
- * clock @id is not available.
- */
-void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts)
-{
- /* Invalidate time stamp */
- ts->tv_sec = -1;
- ts->tv_nsec = 0;
-
- switch (id) {
- case CLOCK_REALTIME:
- ktime_get_real_ts64(ts);
- return;
- case CLOCK_MONOTONIC:
- ktime_get_ts64(ts);
- return;
- case CLOCK_MONOTONIC_RAW:
- ktime_get_raw_ts64(ts);
- return;
- case CLOCK_AUX ... CLOCK_AUX_LAST:
- if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS))
- ktime_get_aux_ts64(id, ts);
- return;
- default:
- WARN_ON_ONCE(1);
- }
-}
-EXPORT_SYMBOL_GPL(ktime_get_clock_ts64);
-
-/**
* timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
*/
int timekeeping_valid_for_hres(void)
@@ -2076,7 +2088,12 @@ void __init timekeeping_init(void)
tk_set_wall_to_mono(tks, wall_to_mono);
- timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
+ /*
+ * Use TK_UPDATE_ALL so the NTP layer picks up the clocksource's
+ * cs_tick_adj via ntp_clear(). Clearing NTP here is otherwise
+ * redundant as ntp_init() already initialised it above.
+ */
+ timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
}
/* time in seconds when suspend began for persistent clock */
@@ -2390,6 +2407,11 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
* xtime_nsec_2 = xtime_nsec_1 - offset
* Which simplifies to:
* xtime_nsec -= offset
+ *
+ * When subtracting offset from xtime_nsec, the same amount
+ * (in appropriate units) has to be added to ntp_error, in
+ * order to correctly track the delta between the time
+ * reported in xtime_nsec, and the intended time.
*/
if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
/* NTP adjustment caused clocksource mult overflow */
@@ -2400,6 +2422,7 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
tk->tkr_mono.mult += mult_adj;
tk->xtime_interval += interval;
tk->tkr_mono.xtime_nsec -= offset;
+ tk->ntp_error += offset << tk->ntp_error_shift;
}
/*
@@ -2409,18 +2432,27 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
{
u64 ntp_tl = ntp_tick_length(tk->id);
+ s64 skew = ntp_get_skew_delta(tk->id);
u32 mult;
/*
- * Determine the multiplier from the current NTP tick length.
- * Avoid expensive division when the tick length doesn't change.
+ * Determine the multiplier from the current NTP tick length plus
+ * skew_delta. The skew biases mult so that ±1 dithering can deliver
+ * the time_offset slew rate. Recompute when either changes.
*/
- if (likely(tk->ntp_tick == ntp_tl)) {
+ if (likely(tk->ntp_tick == ntp_tl && tk->skew_delta == skew)) {
+ /* Revert to the base mult rate. */
mult = tk->tkr_mono.mult - tk->ntp_err_mult;
} else {
tk->ntp_tick = ntp_tl;
- mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
- tk->xtime_remainder, tk->cycle_interval);
+ tk->skew_delta = skew;
+ /*
+ * skew_delta is stored pre-divided by HZ (matching time_offset);
+ * scale it back up to the full per-tick rate for the mult bias.
+ */
+ skew *= NTP_INTERVAL_FREQ;
+ mult = div64_u64((tk->ntp_tick + skew) >> tk->ntp_error_shift,
+ tk->cycle_interval);
}
/*
@@ -2545,8 +2577,25 @@ static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
/* Accumulate error between NTP and clock interval */
tk->ntp_error += tk->ntp_tick << shift;
- tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
- (tk->ntp_error_shift + shift);
+ tk->ntp_error -= tk->xtime_interval << (tk->ntp_error_shift + shift);
+
+ /*
+ * When skewing, do so by adjusting ntp_error to impart an extra
+ * target delta into ntp_error per tick, limited to what can be
+ * drained from time_offset / time_adjust to avoid overshoot.
+ *
+ * The base 'mult' value was calculated with the skew taken into
+ * account, such that the per-tick choice of 'mult' vs. 'mult+1'
+ * allows for the desired effective rate and ntp_error does not
+ * grow unbounded.
+ *
+ * Once the full desired phase offset is delivered, any remaining
+ * skew imparted by the adjusted 'mult', accounted above, remains
+ * in ntp_error and will be compensated by the dithering over time.
+ */
+ if (tk->skew_delta)
+ tk->ntp_error += ntp_drain_skew(tk->id, tk->skew_delta << shift,
+ shift) * NTP_INTERVAL_FREQ;
return offset;
}
@@ -2795,7 +2844,7 @@ void do_timer(unsigned long ticks)
*
* Called from hrtimer_interrupt() or retrigger_next_event()
*/
-ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
+ktime_t ktime_get_update_offsets_now(u32 *cwsseq, ktime_t *offs_real,
ktime_t *offs_boot, ktime_t *offs_tai)
{
struct timekeeper *tk = &tk_core.timekeeper;
@@ -2943,10 +2992,12 @@ static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc,
return ret;
add_device_randomness(txc, sizeof(*txc));
- if (!aux_clock)
+ if (!aux_clock) {
ktime_get_real_ts64(&ts);
- else
- tk_get_aux_ts64(tkd->timekeeper.id, &ts);
+ } else {
+ if (!tk_get_aux_ts64(tkd->timekeeper.id, &ts))
+ return -ENODEV;
+ }
add_device_randomness(&ts, sizeof(ts));
@@ -3051,7 +3102,7 @@ static inline unsigned int clockid_to_tkid(unsigned int id)
static inline struct tk_data *aux_get_tk_data(clockid_t id)
{
- if (!clockid_aux_valid(id))
+ if (!clockid_is_aux_clock(id))
return NULL;
return &timekeeper_data[clockid_to_tkid(id)];
}
@@ -3146,7 +3197,7 @@ EXPORT_SYMBOL_GPL(ktime_get_aux_ts64);
static int aux_get_res(clockid_t id, struct timespec64 *tp)
{
- if (!clockid_aux_valid(id))
+ if (!clockid_is_aux_clock(id))
return -ENODEV;
tp->tv_sec = aux_clock_resolution_ns() / NSEC_PER_SEC;
@@ -3313,7 +3364,9 @@ static const struct attribute_group aux_clock_enable_attr_group = {
static int __init tk_aux_sysfs_init(void)
{
struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj);
+ struct kobject *clks[MAX_AUX_CLOCKS];
int ret = -ENOMEM;
+ int i;
if (!tko)
return ret;
@@ -3322,21 +3375,28 @@ static int __init tk_aux_sysfs_init(void)
if (!auxo)
goto err_clean;
- for (int i = 0; i < MAX_AUX_CLOCKS; i++) {
+ for (i = 0; i < MAX_AUX_CLOCKS; i++) {
char id[2] = { [0] = '0' + i, };
- struct kobject *clk = kobject_create_and_add(id, auxo);
+ clks[i] = kobject_create_and_add(id, auxo);
- if (!clk) {
+ if (!clks[i]) {
ret = -ENOMEM;
- goto err_clean;
+ goto err_clks;
}
- ret = sysfs_create_group(clk, &aux_clock_enable_attr_group);
+ ret = sysfs_create_group(clks[i], &aux_clock_enable_attr_group);
if (ret)
- goto err_clean;
+ goto err_clk;
}
return 0;
+err_clk:
+ kobject_put(clks[i]);
+err_clks:
+ while (--i >= 0) {
+ sysfs_remove_group(clks[i], &aux_clock_enable_attr_group);
+ kobject_put(clks[i]);
+ }
err_clean:
kobject_put(auxo);
kobject_put(tko);
diff --git a/kernel/time/timekeeping.h b/kernel/time/timekeeping.h
index 198d0608db74..4201f9e90813 100644
--- a/kernel/time/timekeeping.h
+++ b/kernel/time/timekeeping.h
@@ -4,7 +4,7 @@
/*
* Internal interfaces for kernel/time/
*/
-extern ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq,
+extern ktime_t ktime_get_update_offsets_now(u32 *cwsseq,
ktime_t *offs_real,
ktime_t *offs_boot,
ktime_t *offs_tai);
diff --git a/kernel/time/timekeeping_internal.h b/kernel/time/timekeeping_internal.h
index 973ede670a36..6d719b8e5ea2 100644
--- a/kernel/time/timekeeping_internal.h
+++ b/kernel/time/timekeeping_internal.h
@@ -6,6 +6,8 @@
#include <linux/spinlock.h>
#include <linux/time.h>
+struct timekeeper;
+
/*
* timekeeping debug functions
*/
@@ -48,4 +50,23 @@ void timekeeper_unlock_irqrestore(unsigned long flags);
/* NTP specific interface to access the current seconds value */
long ktime_get_ntp_seconds(unsigned int id);
+#ifdef CONFIG_GENERIC_GETTIMEOFDAY
+
+extern void update_vsyscall(struct timekeeper *tk);
+extern void update_vsyscall_tz(void);
+extern void vdso_time_update_aux(struct timekeeper *tk);
+
+#else
+
+static inline void update_vsyscall(struct timekeeper *tk)
+{
+}
+static inline void update_vsyscall_tz(void)
+{
+}
+static inline void vdso_time_update_aux(struct timekeeper *tk)
+{
+}
+#endif
+
#endif /* _TIMEKEEPING_INTERNAL_H */
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 655a8c6cd84d..ae9abf14688e 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -2492,6 +2492,7 @@ static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *h
timer = hlist_entry(head->first, struct timer_list, entry);
detach_timer(timer, false);
timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;
+ debug_timer_activate(timer);
internal_add_timer(new_base, timer);
}
}
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index 514802def1e0..0406bf4488e9 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -20,7 +20,7 @@
struct timer_list_iter {
int cpu;
bool second_pass;
- u64 now;
+ ktime_t now;
};
/*
@@ -44,19 +44,19 @@ static void SEQ_printf(struct seq_file *m, const char *fmt, ...)
static void
print_timer(struct seq_file *m, struct hrtimer *taddr, struct hrtimer *timer,
- int idx, u64 now)
+ int idx, ktime_t now)
{
SEQ_printf(m, " #%d: <%p>, %ps", idx, taddr, ACCESS_PRIVATE(timer, function));
SEQ_printf(m, ", S:%02x", timer->is_queued);
SEQ_printf(m, "\n");
- SEQ_printf(m, " # expires at %Lu-%Lu nsecs [in %Ld to %Ld nsecs]\n",
- (unsigned long long)ktime_to_ns(hrtimer_get_softexpires(timer)),
- (unsigned long long)ktime_to_ns(hrtimer_get_expires(timer)),
- (long long)(ktime_to_ns(hrtimer_get_softexpires(timer)) - now),
- (long long)(ktime_to_ns(hrtimer_get_expires(timer)) - now));
+ SEQ_printf(m, " # expires at %lld-%lld nsecs [in %lld to %lld nsecs]\n",
+ (long long)hrtimer_get_softexpires(timer),
+ (long long)hrtimer_get_expires(timer),
+ (long long)ktime_sub(hrtimer_get_softexpires(timer), now),
+ (long long)ktime_sub(hrtimer_get_expires(timer), now));
}
-static void print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
+static void print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, ktime_t now)
{
struct timerqueue_linked_node *curr;
struct hrtimer *timer, tmp;
@@ -94,21 +94,21 @@ next_one:
}
static void
-print_base(struct seq_file *m, struct hrtimer_clock_base *base, u64 now)
+print_base(struct seq_file *m, struct hrtimer_clock_base *base, ktime_t now)
{
SEQ_printf(m, " .base: %p\n", base);
SEQ_printf(m, " .index: %d\n", base->index);
SEQ_printf(m, " .resolution: %u nsecs\n", hrtimer_resolution);
#ifdef CONFIG_HIGH_RES_TIMERS
- SEQ_printf(m, " .offset: %Ld nsecs\n",
+ SEQ_printf(m, " .offset: %lld nsecs\n",
(long long) base->offset);
#endif
SEQ_printf(m, "active timers:\n");
- print_active_timers(m, base, now + ktime_to_ns(base->offset));
+ print_active_timers(m, base, ktime_add(now, base->offset));
}
-static void print_cpu(struct seq_file *m, int cpu, u64 now)
+static void print_cpu(struct seq_file *m, int cpu, ktime_t now)
{
struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
int i;
@@ -118,15 +118,17 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
SEQ_printf(m, " clock %d:\n", i);
print_base(m, cpu_base->clock_base + i, now);
}
-#define P(x) \
- SEQ_printf(m, " .%-15s: %Lu\n", #x, \
- (unsigned long long)(cpu_base->x))
-#define P_ns(x) \
- SEQ_printf(m, " .%-15s: %Lu nsecs\n", #x, \
- (unsigned long long)(ktime_to_ns(cpu_base->x)))
+
+#define DIAG_READ(x) data_race(READ_ONCE(x))
+
+#define P(x) \
+ SEQ_printf(m, " .%-15s: %llu\n", #x, \
+ (unsigned long long)DIAG_READ(cpu_base->x))
+#define P_ktime(x) \
+ SEQ_printf(m, " .%-15s: %lld nsecs\n", #x, (long long)DIAG_READ(cpu_base->x))
#ifdef CONFIG_HIGH_RES_TIMERS
- P_ns(expires_next);
+ P_ktime(expires_next);
P(hres_active);
P(nr_events);
P(nr_retries);
@@ -134,38 +136,39 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
P(max_hang_time);
#endif
#undef P
-#undef P_ns
+#undef P_ktime
#ifdef CONFIG_TICK_ONESHOT
# define P(x) \
- SEQ_printf(m, " .%-15s: %Lu\n", #x, \
- (unsigned long long)(ts->x))
-# define P_ns(x) \
- SEQ_printf(m, " .%-15s: %Lu nsecs\n", #x, \
- (unsigned long long)(ktime_to_ns(ts->x)))
+ SEQ_printf(m, " .%-15s: %llu\n", #x, \
+ (unsigned long long)DIAG_READ(ts->x))
+# define P_ktime(x) \
+ SEQ_printf(m, " .%-15s: %lld nsecs\n", #x, (long long)DIAG_READ(ts->x))
# define P_flag(x, f) \
- SEQ_printf(m, " .%-15s: %d\n", #x, !!(ts->flags & (f)))
+ SEQ_printf(m, " .%-15s: %d\n", #x, !!(DIAG_READ(ts->flags) & (f)))
{
struct tick_sched *ts = tick_get_tick_sched(cpu);
P_flag(nohz, TS_FLAG_NOHZ);
P_flag(highres, TS_FLAG_HIGHRES);
- P_ns(last_tick);
+ P_ktime(last_tick);
P_flag(tick_stopped, TS_FLAG_STOPPED);
P(idle_calls);
P(idle_sleeps);
- P_ns(idle_entrytime);
- P_ns(idle_waketime);
+ P_ktime(idle_entrytime);
+ P_ktime(idle_waketime);
P(last_jiffies);
P(next_timer);
- P_ns(idle_expires);
- SEQ_printf(m, "jiffies: %Lu\n",
+ P_ktime(idle_expires);
+ SEQ_printf(m, "jiffies: %llu\n",
(unsigned long long)jiffies);
}
#endif
#undef P
-#undef P_ns
+#undef P_ktime
+#undef P_flag
+#undef DIAG_READ
SEQ_printf(m, "\n");
}
@@ -196,8 +199,7 @@ print_tickdevice(struct seq_file *m, struct tick_device *td, int cpu)
SEQ_printf(m, " mult: %u\n", dev->mult);
SEQ_printf(m, " shift: %u\n", dev->shift);
SEQ_printf(m, " mode: %d\n", clockevent_get_state(dev));
- SEQ_printf(m, " next_event: %Ld nsecs\n",
- (unsigned long long) ktime_to_ns(dev->next_event));
+ SEQ_printf(m, " next_event: %lld nsecs\n", (long long)dev->next_event);
SEQ_printf(m, " set_next_event: %ps\n", dev->set_next_event);
@@ -250,17 +252,17 @@ static void timer_list_show_tickdevices_header(struct seq_file *m)
}
#endif
-static inline void timer_list_header(struct seq_file *m, u64 now)
+static inline void timer_list_header(struct seq_file *m, ktime_t now)
{
SEQ_printf(m, "Timer List Version: v0.11\n");
SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES);
- SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now);
+ SEQ_printf(m, "now at %lld nsecs\n", (long long)now);
SEQ_printf(m, "\n");
}
void sysrq_timer_list_show(void)
{
- u64 now = ktime_to_ns(ktime_get());
+ ktime_t now = ktime_get();
int cpu;
timer_list_header(NULL, now);
@@ -318,7 +320,7 @@ static void *timer_list_start(struct seq_file *file, loff_t *offset)
struct timer_list_iter *iter = file->private;
if (!*offset)
- iter->now = ktime_to_ns(ktime_get());
+ iter->now = ktime_get();
iter->cpu = -1;
iter->second_pass = false;
return move_iter(iter, *offset);
diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c
index 806c23cf71fc..059d43355e65 100644
--- a/kernel/time/timer_migration.c
+++ b/kernel/time/timer_migration.c
@@ -1847,8 +1847,10 @@ static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu,
}
/* Assert single root without parent */
- if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels))
+ if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) {
+ kfree(stack);
return -EINVAL;
+ }
for (; i >= start_lvl; i--) {
group = stack[i];
diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h
index 31735dd52327..c9c1c29f011d 100644
--- a/kernel/time/timer_migration.h
+++ b/kernel/time/timer_migration.h
@@ -103,7 +103,7 @@ struct tmigr_group {
* before the timer migration hierarchy hotplug callback is
* reached. During this phase, the CPU has to handle the
* global timers on its own and must not act as a migrator.
-
+ *
* @idle: Indicates whether the CPU is idle in the timer migration
* hierarchy
* @remote: Is set when timers of the CPU are expired remotely