Commit | Line | Data |
---|---|---|
00b26474 VF |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Generic userspace implementations of gettimeofday() and similar. | |
4 | */ | |
00b26474 VF |
5 | #include <vdso/datapage.h> |
6 | #include <vdso/helpers.h> | |
7 | ||
9d90b93b TG |
8 | #ifndef vdso_calc_delta |
9 | /* | |
10 | * Default implementation which works for all sane clocksources. That | |
11 | * obviously excludes x86/TSC. | |
12 | */ | |
13 | static __always_inline | |
14 | u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) | |
15 | { | |
16 | return ((cycles - last) & mask) * mult; | |
17 | } | |
18 | #endif | |
19 | ||
8345228c CL |
20 | #ifndef vdso_shift_ns |
21 | static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) | |
22 | { | |
23 | return ns >> shift; | |
24 | } | |
25 | #endif | |
26 | ||
1dff4156 TG |
27 | #ifndef __arch_vdso_hres_capable |
28 | static inline bool __arch_vdso_hres_capable(void) | |
29 | { | |
30 | return true; | |
31 | } | |
32 | #endif | |
33 | ||
ae12e085 CL |
34 | #ifndef vdso_clocksource_ok |
35 | static inline bool vdso_clocksource_ok(const struct vdso_data *vd) | |
36 | { | |
37 | return vd->clock_mode != VDSO_CLOCKMODE_NONE; | |
38 | } | |
39 | #endif | |
40 | ||
72ce7780 TG |
41 | #ifndef vdso_cycles_ok |
42 | static inline bool vdso_cycles_ok(u64 cycles) | |
43 | { | |
44 | return true; | |
45 | } | |
46 | #endif | |
47 | ||
660fd04f | 48 | #ifdef CONFIG_TIME_NS |
58efe9f6 CL |
49 | static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, |
50 | struct __kernel_timespec *ts) | |
660fd04f | 51 | { |
808094fc | 52 | const struct vdso_data *vd; |
660fd04f TG |
53 | const struct timens_offset *offs = &vdns->offset[clk]; |
54 | const struct vdso_timestamp *vdso_ts; | |
55 | u64 cycles, last, ns; | |
56 | u32 seq; | |
57 | s64 sec; | |
58 | ||
808094fc CL |
59 | vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE); |
60 | vd = __arch_get_timens_vdso_data(vd); | |
660fd04f TG |
61 | if (clk != CLOCK_MONOTONIC_RAW) |
62 | vd = &vd[CS_HRES_COARSE]; | |
63 | else | |
64 | vd = &vd[CS_RAW]; | |
65 | vdso_ts = &vd->basetime[clk]; | |
66 | ||
67 | do { | |
68 | seq = vdso_read_begin(vd); | |
f86fd32d | 69 | |
ae12e085 | 70 | if (unlikely(!vdso_clocksource_ok(vd))) |
5d51bee7 | 71 | return -1; |
f86fd32d | 72 | |
4c5a116a | 73 | cycles = __arch_get_hw_counter(vd->clock_mode, vd); |
72ce7780 TG |
74 | if (unlikely(!vdso_cycles_ok(cycles))) |
75 | return -1; | |
660fd04f TG |
76 | ns = vdso_ts->nsec; |
77 | last = vd->cycle_last; | |
660fd04f | 78 | ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); |
8345228c | 79 | ns = vdso_shift_ns(ns, vd->shift); |
660fd04f TG |
80 | sec = vdso_ts->sec; |
81 | } while (unlikely(vdso_read_retry(vd, seq))); | |
82 | ||
83 | /* Add the namespace offset */ | |
84 | sec += offs->sec; | |
85 | ns += offs->nsec; | |
86 | ||
87 | /* | |
88 | * Do this outside the loop: a race inside the loop could result | |
89 | * in __iter_div_u64_rem() being extremely slow. | |
90 | */ | |
91 | ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); | |
92 | ts->tv_nsec = ns; | |
93 | ||
94 | return 0; | |
95 | } | |
96 | #else | |
808094fc CL |
97 | static __always_inline |
98 | const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) | |
660fd04f TG |
99 | { |
100 | return NULL; | |
101 | } | |
102 | ||
58efe9f6 CL |
103 | static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, |
104 | struct __kernel_timespec *ts) | |
660fd04f TG |
105 | { |
106 | return -EINVAL; | |
107 | } | |
108 | #endif | |
109 | ||
c966533f | 110 | static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, |
660fd04f | 111 | struct __kernel_timespec *ts) |
00b26474 VF |
112 | { |
113 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
114 | u64 cycles, last, sec, ns; | |
115 | u32 seq; | |
116 | ||
1dff4156 TG |
117 | /* Allows to compile the high resolution parts out */ |
118 | if (!__arch_vdso_hres_capable()) | |
119 | return -1; | |
120 | ||
00b26474 | 121 | do { |
660fd04f | 122 | /* |
2d6b01bd | 123 | * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace |
660fd04f TG |
124 | * enabled tasks have a special VVAR page installed which |
125 | * has vd->seq set to 1 and vd->clock_mode set to | |
2d6b01bd | 126 | * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks |
660fd04f TG |
127 | * this does not affect performance because if vd->seq is |
128 | * odd, i.e. a concurrent update is in progress the extra | |
129 | * check for vd->clock_mode is just a few extra | |
130 | * instructions while spin waiting for vd->seq to become | |
131 | * even again. | |
132 | */ | |
133 | while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { | |
134 | if (IS_ENABLED(CONFIG_TIME_NS) && | |
2d6b01bd | 135 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) |
660fd04f TG |
136 | return do_hres_timens(vd, clk, ts); |
137 | cpu_relax(); | |
138 | } | |
139 | smp_rmb(); | |
140 | ||
ae12e085 | 141 | if (unlikely(!vdso_clocksource_ok(vd))) |
5d51bee7 | 142 | return -1; |
f86fd32d | 143 | |
4c5a116a | 144 | cycles = __arch_get_hw_counter(vd->clock_mode, vd); |
72ce7780 TG |
145 | if (unlikely(!vdso_cycles_ok(cycles))) |
146 | return -1; | |
00b26474 VF |
147 | ns = vdso_ts->nsec; |
148 | last = vd->cycle_last; | |
9d90b93b | 149 | ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); |
8345228c | 150 | ns = vdso_shift_ns(ns, vd->shift); |
00b26474 VF |
151 | sec = vdso_ts->sec; |
152 | } while (unlikely(vdso_read_retry(vd, seq))); | |
153 | ||
154 | /* | |
155 | * Do this outside the loop: a race inside the loop could result | |
156 | * in __iter_div_u64_rem() being extremely slow. | |
157 | */ | |
158 | ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); | |
159 | ts->tv_nsec = ns; | |
160 | ||
161 | return 0; | |
162 | } | |
163 | ||
660fd04f | 164 | #ifdef CONFIG_TIME_NS |
58efe9f6 CL |
165 | static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, |
166 | struct __kernel_timespec *ts) | |
660fd04f | 167 | { |
808094fc | 168 | const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns); |
660fd04f TG |
169 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; |
170 | const struct timens_offset *offs = &vdns->offset[clk]; | |
171 | u64 nsec; | |
172 | s64 sec; | |
173 | s32 seq; | |
174 | ||
175 | do { | |
176 | seq = vdso_read_begin(vd); | |
177 | sec = vdso_ts->sec; | |
178 | nsec = vdso_ts->nsec; | |
179 | } while (unlikely(vdso_read_retry(vd, seq))); | |
180 | ||
181 | /* Add the namespace offset */ | |
182 | sec += offs->sec; | |
183 | nsec += offs->nsec; | |
184 | ||
185 | /* | |
186 | * Do this outside the loop: a race inside the loop could result | |
187 | * in __iter_div_u64_rem() being extremely slow. | |
188 | */ | |
189 | ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); | |
190 | ts->tv_nsec = nsec; | |
191 | return 0; | |
192 | } | |
193 | #else | |
58efe9f6 CL |
194 | static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, |
195 | struct __kernel_timespec *ts) | |
660fd04f TG |
196 | { |
197 | return -1; | |
198 | } | |
199 | #endif | |
200 | ||
c966533f AV |
201 | static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, |
202 | struct __kernel_timespec *ts) | |
00b26474 VF |
203 | { |
204 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
205 | u32 seq; | |
206 | ||
207 | do { | |
660fd04f | 208 | /* |
2d6b01bd | 209 | * Open coded to handle VDSO_CLOCK_TIMENS. See comment in |
660fd04f TG |
210 | * do_hres(). |
211 | */ | |
212 | while ((seq = READ_ONCE(vd->seq)) & 1) { | |
213 | if (IS_ENABLED(CONFIG_TIME_NS) && | |
2d6b01bd | 214 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) |
660fd04f TG |
215 | return do_coarse_timens(vd, clk, ts); |
216 | cpu_relax(); | |
217 | } | |
218 | smp_rmb(); | |
219 | ||
00b26474 VF |
220 | ts->tv_sec = vdso_ts->sec; |
221 | ts->tv_nsec = vdso_ts->nsec; | |
222 | } while (unlikely(vdso_read_retry(vd, seq))); | |
8463cf80 CL |
223 | |
224 | return 0; | |
00b26474 VF |
225 | } |
226 | ||
b91c8c42 | 227 | static __always_inline int |
e876f0b6 CL |
228 | __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, |
229 | struct __kernel_timespec *ts) | |
00b26474 | 230 | { |
00b26474 VF |
231 | u32 msk; |
232 | ||
233 | /* Check for negative values or invalid clocks */ | |
234 | if (unlikely((u32) clock >= MAX_CLOCKS)) | |
502a590a | 235 | return -1; |
00b26474 VF |
236 | |
237 | /* | |
238 | * Convert the clockid to a bitmask and use it to check which | |
239 | * clocks are handled in the VDSO directly. | |
240 | */ | |
241 | msk = 1U << clock; | |
8463cf80 | 242 | if (likely(msk & VDSO_HRES)) |
c966533f | 243 | vd = &vd[CS_HRES_COARSE]; |
8463cf80 CL |
244 | else if (msk & VDSO_COARSE) |
245 | return do_coarse(&vd[CS_HRES_COARSE], clock, ts); | |
246 | else if (msk & VDSO_RAW) | |
c966533f AV |
247 | vd = &vd[CS_RAW]; |
248 | else | |
249 | return -1; | |
8463cf80 | 250 | |
c966533f | 251 | return do_hres(vd, clock, ts); |
502a590a | 252 | } |
00b26474 | 253 | |
502a590a | 254 | static __maybe_unused int |
e876f0b6 CL |
255 | __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, |
256 | struct __kernel_timespec *ts) | |
502a590a | 257 | { |
e876f0b6 | 258 | int ret = __cvdso_clock_gettime_common(vd, clock, ts); |
502a590a TG |
259 | |
260 | if (unlikely(ret)) | |
261 | return clock_gettime_fallback(clock, ts); | |
262 | return 0; | |
00b26474 VF |
263 | } |
264 | ||
e876f0b6 CL |
265 | static __maybe_unused int |
266 | __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) | |
267 | { | |
268 | return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); | |
269 | } | |
270 | ||
bf279849 | 271 | #ifdef BUILD_VDSO32 |
00b26474 | 272 | static __maybe_unused int |
e876f0b6 CL |
273 | __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, |
274 | struct old_timespec32 *res) | |
00b26474 VF |
275 | { |
276 | struct __kernel_timespec ts; | |
277 | int ret; | |
278 | ||
e876f0b6 | 279 | ret = __cvdso_clock_gettime_common(vd, clock, &ts); |
00b26474 | 280 | |
c60a32ea TG |
281 | if (unlikely(ret)) |
282 | return clock_gettime32_fallback(clock, res); | |
502a590a | 283 | |
a279235d VF |
284 | /* For ret == 0 */ |
285 | res->tv_sec = ts.tv_sec; | |
286 | res->tv_nsec = ts.tv_nsec; | |
287 | ||
00b26474 | 288 | return ret; |
00b26474 | 289 | } |
e876f0b6 CL |
290 | |
291 | static __maybe_unused int | |
292 | __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) | |
293 | { | |
294 | return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); | |
295 | } | |
bf279849 | 296 | #endif /* BUILD_VDSO32 */ |
00b26474 VF |
297 | |
298 | static __maybe_unused int | |
e876f0b6 CL |
299 | __cvdso_gettimeofday_data(const struct vdso_data *vd, |
300 | struct __kernel_old_timeval *tv, struct timezone *tz) | |
00b26474 | 301 | { |
00b26474 VF |
302 | |
303 | if (likely(tv != NULL)) { | |
304 | struct __kernel_timespec ts; | |
305 | ||
306 | if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) | |
307 | return gettimeofday_fallback(tv, tz); | |
308 | ||
309 | tv->tv_sec = ts.tv_sec; | |
310 | tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; | |
311 | } | |
312 | ||
313 | if (unlikely(tz != NULL)) { | |
660fd04f | 314 | if (IS_ENABLED(CONFIG_TIME_NS) && |
2d6b01bd | 315 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) |
808094fc | 316 | vd = __arch_get_timens_vdso_data(vd); |
660fd04f | 317 | |
00b26474 VF |
318 | tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; |
319 | tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; | |
320 | } | |
321 | ||
322 | return 0; | |
323 | } | |
324 | ||
e876f0b6 CL |
325 | static __maybe_unused int |
326 | __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) | |
327 | { | |
328 | return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); | |
329 | } | |
330 | ||
00b26474 | 331 | #ifdef VDSO_HAS_TIME |
e876f0b6 CL |
332 | static __maybe_unused __kernel_old_time_t |
333 | __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) | |
00b26474 | 334 | { |
660fd04f TG |
335 | __kernel_old_time_t t; |
336 | ||
2d6b01bd TG |
337 | if (IS_ENABLED(CONFIG_TIME_NS) && |
338 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) | |
808094fc | 339 | vd = __arch_get_timens_vdso_data(vd); |
660fd04f TG |
340 | |
341 | t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); | |
00b26474 VF |
342 | |
343 | if (time) | |
344 | *time = t; | |
345 | ||
346 | return t; | |
347 | } | |
e876f0b6 CL |
348 | |
349 | static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) | |
350 | { | |
351 | return __cvdso_time_data(__arch_get_vdso_data(), time); | |
352 | } | |
00b26474 VF |
353 | #endif /* VDSO_HAS_TIME */ |
354 | ||
355 | #ifdef VDSO_HAS_CLOCK_GETRES | |
356 | static __maybe_unused | |
e876f0b6 CL |
357 | int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, |
358 | struct __kernel_timespec *res) | |
00b26474 | 359 | { |
00b26474 | 360 | u32 msk; |
502a590a | 361 | u64 ns; |
00b26474 VF |
362 | |
363 | /* Check for negative values or invalid clocks */ | |
364 | if (unlikely((u32) clock >= MAX_CLOCKS)) | |
502a590a | 365 | return -1; |
00b26474 | 366 | |
2d6b01bd TG |
367 | if (IS_ENABLED(CONFIG_TIME_NS) && |
368 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) | |
808094fc | 369 | vd = __arch_get_timens_vdso_data(vd); |
660fd04f | 370 | |
00b26474 VF |
371 | /* |
372 | * Convert the clockid to a bitmask and use it to check which | |
373 | * clocks are handled in the VDSO directly. | |
374 | */ | |
375 | msk = 1U << clock; | |
cdb7c5a9 | 376 | if (msk & (VDSO_HRES | VDSO_RAW)) { |
00b26474 VF |
377 | /* |
378 | * Preserves the behaviour of posix_get_hrtimer_res(). | |
379 | */ | |
49a101d7 | 380 | ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); |
00b26474 VF |
381 | } else if (msk & VDSO_COARSE) { |
382 | /* | |
383 | * Preserves the behaviour of posix_get_coarse_res(). | |
384 | */ | |
385 | ns = LOW_RES_NSEC; | |
00b26474 | 386 | } else { |
502a590a | 387 | return -1; |
00b26474 VF |
388 | } |
389 | ||
1638b8f0 TG |
390 | if (likely(res)) { |
391 | res->tv_sec = 0; | |
392 | res->tv_nsec = ns; | |
393 | } | |
00b26474 | 394 | return 0; |
502a590a TG |
395 | } |
396 | ||
ffd08731 | 397 | static __maybe_unused |
e876f0b6 CL |
398 | int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, |
399 | struct __kernel_timespec *res) | |
502a590a | 400 | { |
e876f0b6 | 401 | int ret = __cvdso_clock_getres_common(vd, clock, res); |
00b26474 | 402 | |
502a590a TG |
403 | if (unlikely(ret)) |
404 | return clock_getres_fallback(clock, res); | |
405 | return 0; | |
00b26474 VF |
406 | } |
407 | ||
e876f0b6 CL |
408 | static __maybe_unused |
409 | int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) | |
410 | { | |
411 | return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); | |
412 | } | |
413 | ||
bf279849 | 414 | #ifdef BUILD_VDSO32 |
00b26474 | 415 | static __maybe_unused int |
e876f0b6 CL |
416 | __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, |
417 | struct old_timespec32 *res) | |
00b26474 VF |
418 | { |
419 | struct __kernel_timespec ts; | |
420 | int ret; | |
421 | ||
e876f0b6 | 422 | ret = __cvdso_clock_getres_common(vd, clock, &ts); |
c60a32ea | 423 | |
c60a32ea TG |
424 | if (unlikely(ret)) |
425 | return clock_getres32_fallback(clock, res); | |
00b26474 | 426 | |
a279235d | 427 | if (likely(res)) { |
00b26474 VF |
428 | res->tv_sec = ts.tv_sec; |
429 | res->tv_nsec = ts.tv_nsec; | |
430 | } | |
00b26474 | 431 | return ret; |
00b26474 | 432 | } |
e876f0b6 CL |
433 | |
434 | static __maybe_unused int | |
435 | __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) | |
436 | { | |
437 | return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), | |
438 | clock, res); | |
439 | } | |
bf279849 | 440 | #endif /* BUILD_VDSO32 */ |
00b26474 | 441 | #endif /* VDSO_HAS_CLOCK_GETRES */ |