lib/vdso: Provide generic VDSO implementation
authorVincenzo Frascino <vincenzo.frascino@arm.com>
Fri, 21 Jun 2019 09:52:29 +0000 (10:52 +0100)
committerThomas Gleixner <tglx@linutronix.de>
Sat, 22 Jun 2019 19:21:05 +0000 (21:21 +0200)
In the last few years the kernel gained quite some architecture specific
vdso implementations which contain very similar code.

Introduce a generic VDSO implementation of gettimeofday() which will be
shareable between architectures once they are converted over.

The implementation is based on the current x86 VDSO code.

[ tglx: Massaged changelog and made the kernel doc tabular ]

Signed-off-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Shijith Thotton <sthotton@marvell.com>
Tested-by: Andre Przywara <andre.przywara@arm.com>
Cc: linux-arch@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-mips@vger.kernel.org
Cc: linux-kselftest@vger.kernel.org
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Russell King <linux@armlinux.org.uk>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Paul Burton <paul.burton@mips.com>
Cc: Daniel Lezcano <daniel.lezcano@linaro.org>
Cc: Mark Salyzyn <salyzyn@android.com>
Cc: Peter Collingbourne <pcc@google.com>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Dmitry Safonov <0x7f454c46@gmail.com>
Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk>
Cc: Huw Davies <huw@codeweavers.com>
Link: https://lkml.kernel.org/r/20190621095252.32307-3-vincenzo.frascino@arm.com
include/vdso/helpers.h [new file with mode: 0644]
lib/Kconfig
lib/vdso/Kconfig [new file with mode: 0644]
lib/vdso/Makefile [new file with mode: 0644]
lib/vdso/gettimeofday.c [new file with mode: 0644]

diff --git a/include/vdso/helpers.h b/include/vdso/helpers.h
new file mode 100644 (file)
index 0000000..01641db
--- /dev/null
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __VDSO_HELPERS_H
+#define __VDSO_HELPERS_H
+
+#ifndef __ASSEMBLY__
+
+#include <vdso/datapage.h>
+
+static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
+{
+       u32 seq;
+
+       while ((seq = READ_ONCE(vd->seq)) & 1)
+               cpu_relax();
+
+       smp_rmb();
+       return seq;
+}
+
+static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
+                                          u32 start)
+{
+       u32 seq;
+
+       smp_rmb();
+       seq = READ_ONCE(vd->seq);
+       return seq != start;
+}
+
+static __always_inline void vdso_write_begin(struct vdso_data *vd)
+{
+       /*
+        * WRITE_ONCE it is required otherwise the compiler can validly tear
+        * updates to vd[x].seq and it is possible that the value seen by the
+        * reader it is inconsistent.
+        */
+       WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
+       WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
+       smp_wmb();
+}
+
+static __always_inline void vdso_write_end(struct vdso_data *vd)
+{
+       smp_wmb();
+       /*
+        * WRITE_ONCE it is required otherwise the compiler can validly tear
+        * updates to vd[x].seq and it is possible that the value seen by the
+        * reader it is inconsistent.
+        */
+       WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
+       WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
+}
+
+#endif /* !__ASSEMBLY__ */
+
+#endif /* __VDSO_HELPERS_H */
index 90623a0e1942b28994b4dce60abe599d35cc3d43..8c8eefc5e54c01b0bcb92ad0eda4b4fec39cde7c 100644 (file)
@@ -576,6 +576,11 @@ config OID_REGISTRY
 config UCS2_STRING
         tristate
 
+#
+# generic vdso
+#
+source "lib/vdso/Kconfig"
+
 source "lib/fonts/Kconfig"
 
 config SG_SPLIT
diff --git a/lib/vdso/Kconfig b/lib/vdso/Kconfig
new file mode 100644 (file)
index 0000000..cc00364
--- /dev/null
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config HAVE_GENERIC_VDSO
+       bool
+
+if HAVE_GENERIC_VDSO
+
+config GENERIC_GETTIMEOFDAY
+       bool
+       help
+         This is a generic implementation of gettimeofday vdso.
+         Each architecture that enables this feature has to
+         provide the fallback implementation.
+
+config GENERIC_VDSO_32
+       bool
+       depends on GENERIC_GETTIMEOFDAY && !64BIT
+       help
+         This config option helps to avoid possible performance issues
+         in 32 bit only architectures.
+
+config GENERIC_COMPAT_VDSO
+       bool
+       help
+         This config option enables the compat VDSO layer.
+
+config CROSS_COMPILE_COMPAT_VDSO
+       string "32 bit Toolchain prefix for compat vDSO"
+       default ""
+       depends on GENERIC_COMPAT_VDSO
+       help
+         Defines the cross-compiler prefix for compiling compat vDSO.
+         If a 64 bit compiler (i.e. x86_64) can compile the VDSO for
+         32 bit, it does not need to define this parameter.
+
+endif
diff --git a/lib/vdso/Makefile b/lib/vdso/Makefile
new file mode 100644 (file)
index 0000000..c415a68
--- /dev/null
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: GPL-2.0
+
+GENERIC_VDSO_MK_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+GENERIC_VDSO_DIR := $(dir $(GENERIC_VDSO_MK_PATH))
+
+c-gettimeofday-$(CONFIG_GENERIC_GETTIMEOFDAY) := $(addprefix $(GENERIC_VDSO_DIR), gettimeofday.c)
+
+# This cmd checks that the vdso library does not contain absolute relocation
+# It has to be called after the linking of the vdso library and requires it
+# as a parameter.
+#
+# $(ARCH_REL_TYPE_ABS) is defined in the arch specific makefile and corresponds
+# to the absolute relocation types printed by "objdump -R" and accepted by the
+# dynamic linker.
+ifndef ARCH_REL_TYPE_ABS
+$(error ARCH_REL_TYPE_ABS is not set)
+endif
+
+quiet_cmd_vdso_check = VDSOCHK $@
+      cmd_vdso_check = if $(OBJDUMP) -R $@ | egrep -h "$(ARCH_REL_TYPE_ABS)"; \
+                      then (echo >&2 "$@: dynamic relocations are not supported"; \
+                            rm -f $@; /bin/false); fi
diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c
new file mode 100644 (file)
index 0000000..767d3a0
--- /dev/null
@@ -0,0 +1,224 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Generic userspace implementations of gettimeofday() and similar.
+ */
+#include <linux/compiler.h>
+#include <linux/math64.h>
+#include <linux/time.h>
+#include <linux/kernel.h>
+#include <linux/hrtimer_defs.h>
+#include <vdso/datapage.h>
+#include <vdso/helpers.h>
+
+/*
+ * The generic vDSO implementation requires that gettimeofday.h
+ * provides:
+ * - __arch_get_vdso_data(): to get the vdso datapage.
+ * - __arch_get_hw_counter(): to get the hw counter based on the
+ *   clock_mode.
+ * - gettimeofday_fallback(): fallback for gettimeofday.
+ * - clock_gettime_fallback(): fallback for clock_gettime.
+ * - clock_getres_fallback(): fallback for clock_getres.
+ */
+#include <asm/vdso/gettimeofday.h>
+
+static int do_hres(const struct vdso_data *vd, clockid_t clk,
+                  struct __kernel_timespec *ts)
+{
+       const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
+       u64 cycles, last, sec, ns;
+       u32 seq;
+
+       do {
+               seq = vdso_read_begin(vd);
+               cycles = __arch_get_hw_counter(vd->clock_mode) &
+                       vd->mask;
+               ns = vdso_ts->nsec;
+               last = vd->cycle_last;
+               if (unlikely((s64)cycles < 0))
+                       return clock_gettime_fallback(clk, ts);
+               if (cycles > last)
+                       ns += (cycles - last) * vd->mult;
+               ns >>= vd->shift;
+               sec = vdso_ts->sec;
+       } while (unlikely(vdso_read_retry(vd, seq)));
+
+       /*
+        * Do this outside the loop: a race inside the loop could result
+        * in __iter_div_u64_rem() being extremely slow.
+        */
+       ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
+       ts->tv_nsec = ns;
+
+       return 0;
+}
+
+static void do_coarse(const struct vdso_data *vd, clockid_t clk,
+                     struct __kernel_timespec *ts)
+{
+       const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
+       u32 seq;
+
+       do {
+               seq = vdso_read_begin(vd);
+               ts->tv_sec = vdso_ts->sec;
+               ts->tv_nsec = vdso_ts->nsec;
+       } while (unlikely(vdso_read_retry(vd, seq)));
+}
+
+static __maybe_unused int
+__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
+{
+       const struct vdso_data *vd = __arch_get_vdso_data();
+       u32 msk;
+
+       /* Check for negative values or invalid clocks */
+       if (unlikely((u32) clock >= MAX_CLOCKS))
+               goto fallback;
+
+       /*
+        * Convert the clockid to a bitmask and use it to check which
+        * clocks are handled in the VDSO directly.
+        */
+       msk = 1U << clock;
+       if (likely(msk & VDSO_HRES)) {
+               return do_hres(&vd[CS_HRES_COARSE], clock, ts);
+       } else if (msk & VDSO_COARSE) {
+               do_coarse(&vd[CS_HRES_COARSE], clock, ts);
+               return 0;
+       } else if (msk & VDSO_RAW) {
+               return do_hres(&vd[CS_RAW], clock, ts);
+       }
+
+fallback:
+       return clock_gettime_fallback(clock, ts);
+}
+
+static __maybe_unused int
+__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
+{
+       struct __kernel_timespec ts;
+       int ret;
+
+       if (res == NULL)
+               goto fallback;
+
+       ret = __cvdso_clock_gettime(clock, &ts);
+
+       if (ret == 0) {
+               res->tv_sec = ts.tv_sec;
+               res->tv_nsec = ts.tv_nsec;
+       }
+
+       return ret;
+
+fallback:
+       return clock_gettime_fallback(clock, (struct __kernel_timespec *)res);
+}
+
+static __maybe_unused int
+__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
+{
+       const struct vdso_data *vd = __arch_get_vdso_data();
+
+       if (likely(tv != NULL)) {
+               struct __kernel_timespec ts;
+
+               if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
+                       return gettimeofday_fallback(tv, tz);
+
+               tv->tv_sec = ts.tv_sec;
+               tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
+       }
+
+       if (unlikely(tz != NULL)) {
+               tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
+               tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
+       }
+
+       return 0;
+}
+
+#ifdef VDSO_HAS_TIME
+static __maybe_unused time_t __cvdso_time(time_t *time)
+{
+       const struct vdso_data *vd = __arch_get_vdso_data();
+       time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
+
+       if (time)
+               *time = t;
+
+       return t;
+}
+#endif /* VDSO_HAS_TIME */
+
+#ifdef VDSO_HAS_CLOCK_GETRES
+static __maybe_unused
+int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
+{
+       const struct vdso_data *vd = __arch_get_vdso_data();
+       u64 ns;
+       u32 msk;
+       u64 hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
+
+       /* Check for negative values or invalid clocks */
+       if (unlikely((u32) clock >= MAX_CLOCKS))
+               goto fallback;
+
+       /*
+        * Convert the clockid to a bitmask and use it to check which
+        * clocks are handled in the VDSO directly.
+        */
+       msk = 1U << clock;
+       if (msk & VDSO_HRES) {
+               /*
+                * Preserves the behaviour of posix_get_hrtimer_res().
+                */
+               ns = hrtimer_res;
+       } else if (msk & VDSO_COARSE) {
+               /*
+                * Preserves the behaviour of posix_get_coarse_res().
+                */
+               ns = LOW_RES_NSEC;
+       } else if (msk & VDSO_RAW) {
+               /*
+                * Preserves the behaviour of posix_get_hrtimer_res().
+                */
+               ns = hrtimer_res;
+       } else {
+               goto fallback;
+       }
+
+       if (res) {
+               res->tv_sec = 0;
+               res->tv_nsec = ns;
+       }
+
+       return 0;
+
+fallback:
+       return clock_getres_fallback(clock, res);
+}
+
+static __maybe_unused int
+__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
+{
+       struct __kernel_timespec ts;
+       int ret;
+
+       if (res == NULL)
+               goto fallback;
+
+       ret = __cvdso_clock_getres(clock, &ts);
+
+       if (ret == 0) {
+               res->tv_sec = ts.tv_sec;
+               res->tv_nsec = ts.tv_nsec;
+       }
+
+       return ret;
+
+fallback:
+       return clock_getres_fallback(clock, (struct __kernel_timespec *)res);
+}
+#endif /* VDSO_HAS_CLOCK_GETRES */