x86/vdso: Collapse coarse functions

[linux-block.git] / arch / x86 / entry / vdso / vclock_gettime.c

diff --git a/arch/x86/entry/vdso/vclock_gettime.c b/arch/x86/entry/vdso/vclock_gettime.c
index f19856d95c60919c92d1679e0037d9339c4c2a65..b27dea0e23af058d599466849235e03ad36a3231 100644 (file)
--- a/arch/x86/entry/vdso/vclock_gettime.c
+++ b/arch/x86/entry/vdso/vclock_gettime.c
@@ -43,8 +43,9 @@ extern u8 hvclock_page
 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
 {
        long ret;
-       asm("syscall" : "=a" (ret) :
-           "0" (__NR_clock_gettime), "D" (clock), "S" (ts) : "memory");
+       asm ("syscall" : "=a" (ret), "=m" (*ts) :
+            "0" (__NR_clock_gettime), "D" (clock), "S" (ts) :
+            "memory", "rcx", "r11");
        return ret;
 }
 
@@ -52,8 +53,9 @@ notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
 {
        long ret;
 
-       asm("syscall" : "=a" (ret) :
-           "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
+       asm ("syscall" : "=a" (ret), "=m" (*tv), "=m" (*tz) :
+            "0" (__NR_gettimeofday), "D" (tv), "S" (tz) :
+            "memory", "rcx", "r11");
        return ret;
 }
 
@@ -64,13 +66,13 @@ notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
 {
        long ret;
 
-       asm(
+       asm (
                "mov %%ebx, %%edx \n"
-               "mov %2, %%ebx \n"
+               "mov %[clock], %%ebx \n"
                "call __kernel_vsyscall \n"
                "mov %%edx, %%ebx \n"
-               : "=a" (ret)
-               : "0" (__NR_clock_gettime), "g" (clock), "c" (ts)
+               : "=a" (ret), "=m" (*ts)
+               : "0" (__NR_clock_gettime), [clock] "g" (clock), "c" (ts)
                : "memory", "edx");
        return ret;
 }
@@ -79,13 +81,13 @@ notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
 {
        long ret;
 
-       asm(
+       asm (
                "mov %%ebx, %%edx \n"
-               "mov %2, %%ebx \n"
+               "mov %[tv], %%ebx \n"
                "call __kernel_vsyscall \n"
                "mov %%edx, %%ebx \n"
-               : "=a" (ret)
-               : "0" (__NR_gettimeofday), "g" (tv), "c" (tz)
+               : "=a" (ret), "=m" (*tv), "=m" (*tz)
+               : "0" (__NR_gettimeofday), [tv] "g" (tv), "c" (tz)
                : "memory", "edx");
        return ret;
 }
@@ -199,43 +201,22 @@ notrace static inline u64 vgetsns(int *mode)
 #endif
        else
                return 0;
-       v = (cycles - gtod->cycle_last) & gtod->mask;
+       v = cycles - gtod->cycle_last;
        return v * gtod->mult;
 }
 
-/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
-notrace static int __always_inline do_realtime(struct timespec *ts)
+notrace static int do_hres(clockid_t clk, struct timespec *ts)
 {
-       unsigned long seq;
-       u64 ns;
+       struct vgtod_ts *base = &gtod->basetime[clk];
+       unsigned int seq;
        int mode;
-
-       do {
-               seq = gtod_read_begin(gtod);
-               mode = gtod->vclock_mode;
-               ts->tv_sec = gtod->wall_time_sec;
-               ns = gtod->wall_time_snsec;
-               ns += vgetsns(&mode);
-               ns >>= gtod->shift;
-       } while (unlikely(gtod_read_retry(gtod, seq)));
-
-       ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
-       ts->tv_nsec = ns;
-
-       return mode;
-}
-
-notrace static int __always_inline do_monotonic(struct timespec *ts)
-{
-       unsigned long seq;
        u64 ns;
-       int mode;
 
        do {
                seq = gtod_read_begin(gtod);
                mode = gtod->vclock_mode;
-               ts->tv_sec = gtod->monotonic_time_sec;
-               ns = gtod->monotonic_time_snsec;
+               ts->tv_sec = base->sec;
+               ns = base->nsec;
                ns += vgetsns(&mode);
                ns >>= gtod->shift;
        } while (unlikely(gtod_read_retry(gtod, seq)));
@@ -246,23 +227,15 @@ notrace static int __always_inline do_monotonic(struct timespec *ts)
        return mode;
 }
 
-notrace static void do_realtime_coarse(struct timespec *ts)
+notrace static void do_coarse(clockid_t clk, struct timespec *ts)
 {
-       unsigned long seq;
-       do {
-               seq = gtod_read_begin(gtod);
-               ts->tv_sec = gtod->wall_time_coarse_sec;
-               ts->tv_nsec = gtod->wall_time_coarse_nsec;
-       } while (unlikely(gtod_read_retry(gtod, seq)));
-}
+       struct vgtod_ts *base = &gtod->basetime[clk];
+       unsigned int seq;
 
-notrace static void do_monotonic_coarse(struct timespec *ts)
-{
-       unsigned long seq;
        do {
                seq = gtod_read_begin(gtod);
-               ts->tv_sec = gtod->monotonic_time_coarse_sec;
-               ts->tv_nsec = gtod->monotonic_time_coarse_nsec;
+               ts->tv_sec = base->sec;
+               ts->tv_nsec = base->nsec;
        } while (unlikely(gtod_read_retry(gtod, seq)));
 }
 
@@ -270,18 +243,18 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
 {
        switch (clock) {
        case CLOCK_REALTIME:
-               if (do_realtime(ts) == VCLOCK_NONE)
+               if (do_hres(CLOCK_REALTIME, ts) == VCLOCK_NONE)
                        goto fallback;
                break;
        case CLOCK_MONOTONIC:
-               if (do_monotonic(ts) == VCLOCK_NONE)
+               if (do_hres(CLOCK_MONOTONIC, ts) == VCLOCK_NONE)
                        goto fallback;
                break;
        case CLOCK_REALTIME_COARSE:
-               do_realtime_coarse(ts);
+               do_coarse(CLOCK_REALTIME_COARSE, ts);
                break;
        case CLOCK_MONOTONIC_COARSE:
-               do_monotonic_coarse(ts);
+               do_coarse(CLOCK_MONOTONIC_COARSE, ts);
                break;
        default:
                goto fallback;
@@ -297,7 +270,9 @@ int clock_gettime(clockid_t, struct timespec *)
 notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
 {
        if (likely(tv != NULL)) {
-               if (unlikely(do_realtime((struct timespec *)tv) == VCLOCK_NONE))
+               struct timespec *ts = (struct timespec *) tv;
+
+               if (unlikely(do_hres(CLOCK_REALTIME, ts) == VCLOCK_NONE))
                        return vdso_fallback_gtod(tv, tz);
                tv->tv_usec /= 1000;
        }
@@ -318,7 +293,7 @@ int gettimeofday(struct timeval *, struct timezone *)
 notrace time_t __vdso_time(time_t *t)
 {
        /* This is atomic on x86 so we don't need any locks. */
-       time_t result = READ_ONCE(gtod->wall_time_sec);
+       time_t result = READ_ONCE(gtod->basetime[CLOCK_REALTIME].sec);
 
        if (t)
                *t = result;