From c8e3a8b6f2e62661d838ae222774121ae23777a4 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Mon, 25 Mar 2024 08:40:05 +0200 Subject: vdso: Consolidate vdso_calc_delta() Consolidate vdso_calc_delta(), in preparation for further simplification. Suggested-by: Thomas Gleixner Signed-off-by: Adrian Hunter Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240325064023.2997-2-adrian.hunter@intel.com --- lib/vdso/gettimeofday.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index ce2f69552003..faccf12f7c03 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -6,6 +6,13 @@ #include #ifndef vdso_calc_delta + +#ifdef VDSO_DELTA_NOMASK +# define VDSO_DELTA_MASK(mask) U64_MAX +#else +# define VDSO_DELTA_MASK(mask) (mask) +#endif + /* * Default implementation which works for all sane clocksources. That * obviously excludes x86/TSC. @@ -13,7 +20,7 @@ static __always_inline u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) { - return ((cycles - last) & mask) * mult; + return ((cycles - last) & VDSO_DELTA_MASK(mask)) * mult; } #endif -- cgit v1.2.3 From 5b26ef660a690e424d9548fdf0565d4172d5d88f Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Mon, 25 Mar 2024 08:40:06 +0200 Subject: vdso: Consolidate nanoseconds calculation Consolidate nanoseconds calculation to simplify and reduce code duplication. Suggested-by: Thomas Gleixner Signed-off-by: Adrian Hunter Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240325064023.2997-3-adrian.hunter@intel.com --- arch/x86/include/asm/vdso/gettimeofday.h | 17 ++++++------- lib/vdso/gettimeofday.c | 43 ++++++++++++++------------------ 2 files changed, 27 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h index 8e048ca980df..5727dedd3549 100644 --- a/arch/x86/include/asm/vdso/gettimeofday.h +++ b/arch/x86/include/asm/vdso/gettimeofday.h @@ -300,7 +300,7 @@ static inline bool arch_vdso_cycles_ok(u64 cycles) #define vdso_cycles_ok arch_vdso_cycles_ok /* - * x86 specific delta calculation. + * x86 specific calculation of nanoseconds for the current cycle count * * The regular implementation assumes that clocksource reads are globally * monotonic. The TSC can be slightly off across sockets which can cause @@ -308,8 +308,8 @@ static inline bool arch_vdso_cycles_ok(u64 cycles) * jump. * * Therefore it needs to be verified that @cycles are greater than - * @last. If not then use @last, which is the base time of the current - * conversion period. + * @vd->cycles_last. If not then use @vd->cycles_last, which is the base + * time of the current conversion period. * * This variant also uses a custom mask because while the clocksource mask of * all the VDSO capable clocksources on x86 is U64_MAX, the above code uses @@ -317,25 +317,24 @@ static inline bool arch_vdso_cycles_ok(u64 cycles) * declares everything with the MSB/Sign-bit set as invalid. Therefore the * effective mask is S64_MAX. */ -static __always_inline -u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) +static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base) { /* * Due to the MSB/Sign-bit being used as invalid marker (see * arch_vdso_cycles_valid() above), the effective mask is S64_MAX. */ - u64 delta = (cycles - last) & S64_MAX; + u64 delta = (cycles - vd->cycle_last) & S64_MAX; /* * Due to the above mentioned TSC wobbles, filter out negative motion. * Per the above masking, the effective sign bit is now bit 62. */ if (unlikely(delta & (1ULL << 62))) - return 0; + return base >> vd->shift; - return delta * mult; + return ((delta * vd->mult) + base) >> vd->shift; } -#define vdso_calc_delta vdso_calc_delta +#define vdso_calc_ns vdso_calc_ns #endif /* !__ASSEMBLY__ */ diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index faccf12f7c03..9fa90e0794c9 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -5,31 +5,32 @@ #include #include -#ifndef vdso_calc_delta +#ifndef vdso_calc_ns #ifdef VDSO_DELTA_NOMASK -# define VDSO_DELTA_MASK(mask) U64_MAX +# define VDSO_DELTA_MASK(vd) U64_MAX #else -# define VDSO_DELTA_MASK(mask) (mask) +# define VDSO_DELTA_MASK(vd) (vd->mask) +#endif + +#ifndef vdso_shift_ns +static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) +{ + return ns >> shift; +} #endif /* * Default implementation which works for all sane clocksources. That * obviously excludes x86/TSC. */ -static __always_inline -u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) +static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base) { - return ((cycles - last) & VDSO_DELTA_MASK(mask)) * mult; -} -#endif + u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd); -#ifndef vdso_shift_ns -static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) -{ - return ns >> shift; + return vdso_shift_ns((delta * vd->mult) + base, vd->shift); } -#endif +#endif /* vdso_calc_ns */ #ifndef __arch_vdso_hres_capable static inline bool __arch_vdso_hres_capable(void) @@ -56,10 +57,10 @@ static inline bool vdso_cycles_ok(u64 cycles) static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, struct __kernel_timespec *ts) { - const struct vdso_data *vd; const struct timens_offset *offs = &vdns->offset[clk]; const struct vdso_timestamp *vdso_ts; - u64 cycles, last, ns; + const struct vdso_data *vd; + u64 cycles, ns; u32 seq; s64 sec; @@ -80,10 +81,7 @@ static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_ cycles = __arch_get_hw_counter(vd->clock_mode, vd); if (unlikely(!vdso_cycles_ok(cycles))) return -1; - ns = vdso_ts->nsec; - last = vd->cycle_last; - ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); - ns = vdso_shift_ns(ns, vd->shift); + ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec); sec = vdso_ts->sec; } while (unlikely(vdso_read_retry(vd, seq))); @@ -118,7 +116,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, struct __kernel_timespec *ts) { const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; - u64 cycles, last, sec, ns; + u64 cycles, sec, ns; u32 seq; /* Allows to compile the high resolution parts out */ @@ -151,10 +149,7 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, cycles = __arch_get_hw_counter(vd->clock_mode, vd); if (unlikely(!vdso_cycles_ok(cycles))) return -1; - ns = vdso_ts->nsec; - last = vd->cycle_last; - ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); - ns = vdso_shift_ns(ns, vd->shift); + ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec); sec = vdso_ts->sec; } while (unlikely(vdso_read_retry(vd, seq))); -- cgit v1.2.3 From 0c68458b0a5878d735572b4f4d91219a1db7c784 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Mon, 25 Mar 2024 08:40:07 +0200 Subject: vdso: Add CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT Add CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT in preparation to add multiplication overflow protection to the VDSO time getter functions. Suggested-by: Thomas Gleixner Signed-off-by: Adrian Hunter Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240325064023.2997-4-adrian.hunter@intel.com --- lib/vdso/Kconfig | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig index d883ac299508..c46c2300517c 100644 --- a/lib/vdso/Kconfig +++ b/lib/vdso/Kconfig @@ -30,4 +30,11 @@ config GENERIC_VDSO_TIME_NS Selected by architectures which support time namespaces in the VDSO +config GENERIC_VDSO_OVERFLOW_PROTECT + bool + help + Select to add multiplication overflow protection to the VDSO + time getter functions for the price of an extra conditional + in the hotpath. + endif -- cgit v1.2.3 From 456e3788bc7164c1c8298045e04068b8e3d8e413 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Mon, 25 Mar 2024 08:40:11 +0200 Subject: vdso: Make delta calculation overflow safe Kernel timekeeping is designed to keep the change in cycles (since the last timer interrupt) below max_cycles, which prevents multiplication overflow when converting cycles to nanoseconds. However, if timer interrupts stop, the calculation will eventually overflow. Add protection against that, enabled by config option CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT. Check against max_cycles, falling back to a slower higher precision calculation. Suggested-by: Thomas Gleixner Signed-off-by: Adrian Hunter Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240325064023.2997-8-adrian.hunter@intel.com --- lib/vdso/gettimeofday.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 9fa90e0794c9..9c3a8d2440c9 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -13,6 +13,18 @@ # define VDSO_DELTA_MASK(vd) (vd->mask) #endif +#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT +static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta) +{ + return delta < vd->max_cycles; +} +#else +static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta) +{ + return true; +} +#endif + #ifndef vdso_shift_ns static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) { @@ -28,7 +40,10 @@ static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, { u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd); - return vdso_shift_ns((delta * vd->mult) + base, vd->shift); + if (likely(vdso_delta_ok(vd, delta))) + return vdso_shift_ns((delta * vd->mult) + base, vd->shift); + + return mul_u64_u32_add_u64_shr(delta, vd->mult, base, vd->shift); } #endif /* vdso_calc_ns */ -- cgit v1.2.3 From 8ff1e6c5aca5fd908e81c33c460c45f9555e1c22 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Tue, 9 Apr 2024 09:26:39 +0300 Subject: vdso: Fix powerpc build U64_MAX undeclared error U64_MAX is not in include/vdso/limits.h, although that isn't noticed on x86 because x86 includes include/linux/limits.h indirectly. However powerpc is more selective, resulting in the following build error: In file included from : lib/vdso/gettimeofday.c: In function 'vdso_calc_ns': lib/vdso/gettimeofday.c:11:33: error: 'U64_MAX' undeclared 11 | # define VDSO_DELTA_MASK(vd) U64_MAX | ^~~~~~~ Use ULLONG_MAX instead which will work just as well and is in include/vdso/limits.h. Fixes: c8e3a8b6f2e6 ("vdso: Consolidate vdso_calc_delta()") Reported-by: Stephen Rothwell Signed-off-by: Adrian Hunter Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20240409062639.3393-1-adrian.hunter@intel.com Closes: https://lore.kernel.org/all/20240409124905.6816db37@canb.auug.org.au/ --- lib/vdso/gettimeofday.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index 9c3a8d2440c9..899850bd6f0b 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -8,7 +8,7 @@ #ifndef vdso_calc_ns #ifdef VDSO_DELTA_NOMASK -# define VDSO_DELTA_MASK(vd) U64_MAX +# define VDSO_DELTA_MASK(vd) ULLONG_MAX #else # define VDSO_DELTA_MASK(vd) (vd->mask) #endif -- cgit v1.2.3