Commit | Line | Data |
---|---|---|
00b26474 VF |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Generic userspace implementations of gettimeofday() and similar. | |
4 | */ | |
00b26474 VF |
5 | #include <vdso/datapage.h> |
6 | #include <vdso/helpers.h> | |
7 | ||
9d90b93b TG |
8 | #ifndef vdso_calc_delta |
9 | /* | |
10 | * Default implementation which works for all sane clocksources. That | |
11 | * obviously excludes x86/TSC. | |
12 | */ | |
13 | static __always_inline | |
14 | u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) | |
15 | { | |
16 | return ((cycles - last) & mask) * mult; | |
17 | } | |
18 | #endif | |
19 | ||
8345228c CL |
20 | #ifndef vdso_shift_ns |
21 | static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) | |
22 | { | |
23 | return ns >> shift; | |
24 | } | |
25 | #endif | |
26 | ||
1dff4156 TG |
27 | #ifndef __arch_vdso_hres_capable |
28 | static inline bool __arch_vdso_hres_capable(void) | |
29 | { | |
30 | return true; | |
31 | } | |
32 | #endif | |
33 | ||
ae12e085 CL |
34 | #ifndef vdso_clocksource_ok |
35 | static inline bool vdso_clocksource_ok(const struct vdso_data *vd) | |
36 | { | |
37 | return vd->clock_mode != VDSO_CLOCKMODE_NONE; | |
38 | } | |
39 | #endif | |
40 | ||
660fd04f TG |
41 | #ifdef CONFIG_TIME_NS |
42 | static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, | |
43 | struct __kernel_timespec *ts) | |
44 | { | |
45 | const struct vdso_data *vd = __arch_get_timens_vdso_data(); | |
46 | const struct timens_offset *offs = &vdns->offset[clk]; | |
47 | const struct vdso_timestamp *vdso_ts; | |
48 | u64 cycles, last, ns; | |
49 | u32 seq; | |
50 | s64 sec; | |
51 | ||
52 | if (clk != CLOCK_MONOTONIC_RAW) | |
53 | vd = &vd[CS_HRES_COARSE]; | |
54 | else | |
55 | vd = &vd[CS_RAW]; | |
56 | vdso_ts = &vd->basetime[clk]; | |
57 | ||
58 | do { | |
59 | seq = vdso_read_begin(vd); | |
f86fd32d | 60 | |
ae12e085 | 61 | if (unlikely(!vdso_clocksource_ok(vd))) |
5d51bee7 | 62 | return -1; |
f86fd32d | 63 | |
660fd04f TG |
64 | cycles = __arch_get_hw_counter(vd->clock_mode); |
65 | ns = vdso_ts->nsec; | |
66 | last = vd->cycle_last; | |
660fd04f | 67 | ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); |
8345228c | 68 | ns = vdso_shift_ns(ns, vd->shift); |
660fd04f TG |
69 | sec = vdso_ts->sec; |
70 | } while (unlikely(vdso_read_retry(vd, seq))); | |
71 | ||
72 | /* Add the namespace offset */ | |
73 | sec += offs->sec; | |
74 | ns += offs->nsec; | |
75 | ||
76 | /* | |
77 | * Do this outside the loop: a race inside the loop could result | |
78 | * in __iter_div_u64_rem() being extremely slow. | |
79 | */ | |
80 | ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); | |
81 | ts->tv_nsec = ns; | |
82 | ||
83 | return 0; | |
84 | } | |
85 | #else | |
86 | static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void) | |
87 | { | |
88 | return NULL; | |
89 | } | |
90 | ||
91 | static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, | |
92 | struct __kernel_timespec *ts) | |
93 | { | |
94 | return -EINVAL; | |
95 | } | |
96 | #endif | |
97 | ||
c966533f | 98 | static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, |
660fd04f | 99 | struct __kernel_timespec *ts) |
00b26474 VF |
100 | { |
101 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
102 | u64 cycles, last, sec, ns; | |
103 | u32 seq; | |
104 | ||
1dff4156 TG |
105 | /* Allows to compile the high resolution parts out */ |
106 | if (!__arch_vdso_hres_capable()) | |
107 | return -1; | |
108 | ||
00b26474 | 109 | do { |
660fd04f | 110 | /* |
2d6b01bd | 111 | * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace |
660fd04f TG |
112 | * enabled tasks have a special VVAR page installed which |
113 | * has vd->seq set to 1 and vd->clock_mode set to | |
2d6b01bd | 114 | * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks |
660fd04f TG |
115 | * this does not affect performance because if vd->seq is |
116 | * odd, i.e. a concurrent update is in progress the extra | |
117 | * check for vd->clock_mode is just a few extra | |
118 | * instructions while spin waiting for vd->seq to become | |
119 | * even again. | |
120 | */ | |
121 | while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { | |
122 | if (IS_ENABLED(CONFIG_TIME_NS) && | |
2d6b01bd | 123 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) |
660fd04f TG |
124 | return do_hres_timens(vd, clk, ts); |
125 | cpu_relax(); | |
126 | } | |
127 | smp_rmb(); | |
128 | ||
ae12e085 | 129 | if (unlikely(!vdso_clocksource_ok(vd))) |
5d51bee7 | 130 | return -1; |
f86fd32d | 131 | |
9d90b93b | 132 | cycles = __arch_get_hw_counter(vd->clock_mode); |
00b26474 VF |
133 | ns = vdso_ts->nsec; |
134 | last = vd->cycle_last; | |
9d90b93b | 135 | ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); |
8345228c | 136 | ns = vdso_shift_ns(ns, vd->shift); |
00b26474 VF |
137 | sec = vdso_ts->sec; |
138 | } while (unlikely(vdso_read_retry(vd, seq))); | |
139 | ||
140 | /* | |
141 | * Do this outside the loop: a race inside the loop could result | |
142 | * in __iter_div_u64_rem() being extremely slow. | |
143 | */ | |
144 | ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); | |
145 | ts->tv_nsec = ns; | |
146 | ||
147 | return 0; | |
148 | } | |
149 | ||
660fd04f TG |
150 | #ifdef CONFIG_TIME_NS |
151 | static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, | |
152 | struct __kernel_timespec *ts) | |
153 | { | |
154 | const struct vdso_data *vd = __arch_get_timens_vdso_data(); | |
155 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
156 | const struct timens_offset *offs = &vdns->offset[clk]; | |
157 | u64 nsec; | |
158 | s64 sec; | |
159 | s32 seq; | |
160 | ||
161 | do { | |
162 | seq = vdso_read_begin(vd); | |
163 | sec = vdso_ts->sec; | |
164 | nsec = vdso_ts->nsec; | |
165 | } while (unlikely(vdso_read_retry(vd, seq))); | |
166 | ||
167 | /* Add the namespace offset */ | |
168 | sec += offs->sec; | |
169 | nsec += offs->nsec; | |
170 | ||
171 | /* | |
172 | * Do this outside the loop: a race inside the loop could result | |
173 | * in __iter_div_u64_rem() being extremely slow. | |
174 | */ | |
175 | ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); | |
176 | ts->tv_nsec = nsec; | |
177 | return 0; | |
178 | } | |
179 | #else | |
180 | static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, | |
181 | struct __kernel_timespec *ts) | |
182 | { | |
183 | return -1; | |
184 | } | |
185 | #endif | |
186 | ||
c966533f AV |
187 | static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, |
188 | struct __kernel_timespec *ts) | |
00b26474 VF |
189 | { |
190 | const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; | |
191 | u32 seq; | |
192 | ||
193 | do { | |
660fd04f | 194 | /* |
2d6b01bd | 195 | * Open coded to handle VDSO_CLOCK_TIMENS. See comment in |
660fd04f TG |
196 | * do_hres(). |
197 | */ | |
198 | while ((seq = READ_ONCE(vd->seq)) & 1) { | |
199 | if (IS_ENABLED(CONFIG_TIME_NS) && | |
2d6b01bd | 200 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) |
660fd04f TG |
201 | return do_coarse_timens(vd, clk, ts); |
202 | cpu_relax(); | |
203 | } | |
204 | smp_rmb(); | |
205 | ||
00b26474 VF |
206 | ts->tv_sec = vdso_ts->sec; |
207 | ts->tv_nsec = vdso_ts->nsec; | |
208 | } while (unlikely(vdso_read_retry(vd, seq))); | |
8463cf80 CL |
209 | |
210 | return 0; | |
00b26474 VF |
211 | } |
212 | ||
213 | static __maybe_unused int | |
e876f0b6 CL |
214 | __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, |
215 | struct __kernel_timespec *ts) | |
00b26474 | 216 | { |
00b26474 VF |
217 | u32 msk; |
218 | ||
219 | /* Check for negative values or invalid clocks */ | |
220 | if (unlikely((u32) clock >= MAX_CLOCKS)) | |
502a590a | 221 | return -1; |
00b26474 VF |
222 | |
223 | /* | |
224 | * Convert the clockid to a bitmask and use it to check which | |
225 | * clocks are handled in the VDSO directly. | |
226 | */ | |
227 | msk = 1U << clock; | |
8463cf80 | 228 | if (likely(msk & VDSO_HRES)) |
c966533f | 229 | vd = &vd[CS_HRES_COARSE]; |
8463cf80 CL |
230 | else if (msk & VDSO_COARSE) |
231 | return do_coarse(&vd[CS_HRES_COARSE], clock, ts); | |
232 | else if (msk & VDSO_RAW) | |
c966533f AV |
233 | vd = &vd[CS_RAW]; |
234 | else | |
235 | return -1; | |
8463cf80 | 236 | |
c966533f | 237 | return do_hres(vd, clock, ts); |
502a590a | 238 | } |
00b26474 | 239 | |
502a590a | 240 | static __maybe_unused int |
e876f0b6 CL |
241 | __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, |
242 | struct __kernel_timespec *ts) | |
502a590a | 243 | { |
e876f0b6 | 244 | int ret = __cvdso_clock_gettime_common(vd, clock, ts); |
502a590a TG |
245 | |
246 | if (unlikely(ret)) | |
247 | return clock_gettime_fallback(clock, ts); | |
248 | return 0; | |
00b26474 VF |
249 | } |
250 | ||
e876f0b6 CL |
251 | static __maybe_unused int |
252 | __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) | |
253 | { | |
254 | return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); | |
255 | } | |
256 | ||
bf279849 | 257 | #ifdef BUILD_VDSO32 |
00b26474 | 258 | static __maybe_unused int |
e876f0b6 CL |
259 | __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, |
260 | struct old_timespec32 *res) | |
00b26474 VF |
261 | { |
262 | struct __kernel_timespec ts; | |
263 | int ret; | |
264 | ||
e876f0b6 | 265 | ret = __cvdso_clock_gettime_common(vd, clock, &ts); |
00b26474 | 266 | |
c60a32ea TG |
267 | if (unlikely(ret)) |
268 | return clock_gettime32_fallback(clock, res); | |
502a590a | 269 | |
a279235d VF |
270 | /* For ret == 0 */ |
271 | res->tv_sec = ts.tv_sec; | |
272 | res->tv_nsec = ts.tv_nsec; | |
273 | ||
00b26474 | 274 | return ret; |
00b26474 | 275 | } |
e876f0b6 CL |
276 | |
277 | static __maybe_unused int | |
278 | __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) | |
279 | { | |
280 | return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); | |
281 | } | |
bf279849 | 282 | #endif /* BUILD_VDSO32 */ |
00b26474 VF |
283 | |
284 | static __maybe_unused int | |
e876f0b6 CL |
285 | __cvdso_gettimeofday_data(const struct vdso_data *vd, |
286 | struct __kernel_old_timeval *tv, struct timezone *tz) | |
00b26474 | 287 | { |
00b26474 VF |
288 | |
289 | if (likely(tv != NULL)) { | |
290 | struct __kernel_timespec ts; | |
291 | ||
292 | if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) | |
293 | return gettimeofday_fallback(tv, tz); | |
294 | ||
295 | tv->tv_sec = ts.tv_sec; | |
296 | tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; | |
297 | } | |
298 | ||
299 | if (unlikely(tz != NULL)) { | |
660fd04f | 300 | if (IS_ENABLED(CONFIG_TIME_NS) && |
2d6b01bd | 301 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) |
660fd04f TG |
302 | vd = __arch_get_timens_vdso_data(); |
303 | ||
00b26474 VF |
304 | tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; |
305 | tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; | |
306 | } | |
307 | ||
308 | return 0; | |
309 | } | |
310 | ||
e876f0b6 CL |
311 | static __maybe_unused int |
312 | __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) | |
313 | { | |
314 | return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); | |
315 | } | |
316 | ||
00b26474 | 317 | #ifdef VDSO_HAS_TIME |
e876f0b6 CL |
318 | static __maybe_unused __kernel_old_time_t |
319 | __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) | |
00b26474 | 320 | { |
660fd04f TG |
321 | __kernel_old_time_t t; |
322 | ||
2d6b01bd TG |
323 | if (IS_ENABLED(CONFIG_TIME_NS) && |
324 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) | |
660fd04f TG |
325 | vd = __arch_get_timens_vdso_data(); |
326 | ||
327 | t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); | |
00b26474 VF |
328 | |
329 | if (time) | |
330 | *time = t; | |
331 | ||
332 | return t; | |
333 | } | |
e876f0b6 CL |
334 | |
335 | static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) | |
336 | { | |
337 | return __cvdso_time_data(__arch_get_vdso_data(), time); | |
338 | } | |
00b26474 VF |
339 | #endif /* VDSO_HAS_TIME */ |
340 | ||
341 | #ifdef VDSO_HAS_CLOCK_GETRES | |
342 | static __maybe_unused | |
e876f0b6 CL |
343 | int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, |
344 | struct __kernel_timespec *res) | |
00b26474 | 345 | { |
00b26474 | 346 | u32 msk; |
502a590a | 347 | u64 ns; |
00b26474 VF |
348 | |
349 | /* Check for negative values or invalid clocks */ | |
350 | if (unlikely((u32) clock >= MAX_CLOCKS)) | |
502a590a | 351 | return -1; |
00b26474 | 352 | |
2d6b01bd TG |
353 | if (IS_ENABLED(CONFIG_TIME_NS) && |
354 | vd->clock_mode == VDSO_CLOCKMODE_TIMENS) | |
660fd04f TG |
355 | vd = __arch_get_timens_vdso_data(); |
356 | ||
00b26474 VF |
357 | /* |
358 | * Convert the clockid to a bitmask and use it to check which | |
359 | * clocks are handled in the VDSO directly. | |
360 | */ | |
361 | msk = 1U << clock; | |
cdb7c5a9 | 362 | if (msk & (VDSO_HRES | VDSO_RAW)) { |
00b26474 VF |
363 | /* |
364 | * Preserves the behaviour of posix_get_hrtimer_res(). | |
365 | */ | |
49a101d7 | 366 | ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); |
00b26474 VF |
367 | } else if (msk & VDSO_COARSE) { |
368 | /* | |
369 | * Preserves the behaviour of posix_get_coarse_res(). | |
370 | */ | |
371 | ns = LOW_RES_NSEC; | |
00b26474 | 372 | } else { |
502a590a | 373 | return -1; |
00b26474 VF |
374 | } |
375 | ||
1638b8f0 TG |
376 | if (likely(res)) { |
377 | res->tv_sec = 0; | |
378 | res->tv_nsec = ns; | |
379 | } | |
00b26474 | 380 | return 0; |
502a590a TG |
381 | } |
382 | ||
ffd08731 | 383 | static __maybe_unused |
e876f0b6 CL |
384 | int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, |
385 | struct __kernel_timespec *res) | |
502a590a | 386 | { |
e876f0b6 | 387 | int ret = __cvdso_clock_getres_common(vd, clock, res); |
00b26474 | 388 | |
502a590a TG |
389 | if (unlikely(ret)) |
390 | return clock_getres_fallback(clock, res); | |
391 | return 0; | |
00b26474 VF |
392 | } |
393 | ||
e876f0b6 CL |
394 | static __maybe_unused |
395 | int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) | |
396 | { | |
397 | return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); | |
398 | } | |
399 | ||
bf279849 | 400 | #ifdef BUILD_VDSO32 |
00b26474 | 401 | static __maybe_unused int |
e876f0b6 CL |
402 | __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, |
403 | struct old_timespec32 *res) | |
00b26474 VF |
404 | { |
405 | struct __kernel_timespec ts; | |
406 | int ret; | |
407 | ||
e876f0b6 | 408 | ret = __cvdso_clock_getres_common(vd, clock, &ts); |
c60a32ea | 409 | |
c60a32ea TG |
410 | if (unlikely(ret)) |
411 | return clock_getres32_fallback(clock, res); | |
00b26474 | 412 | |
a279235d | 413 | if (likely(res)) { |
00b26474 VF |
414 | res->tv_sec = ts.tv_sec; |
415 | res->tv_nsec = ts.tv_nsec; | |
416 | } | |
00b26474 | 417 | return ret; |
00b26474 | 418 | } |
e876f0b6 CL |
419 | |
420 | static __maybe_unused int | |
421 | __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) | |
422 | { | |
423 | return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), | |
424 | clock, res); | |
425 | } | |
bf279849 | 426 | #endif /* BUILD_VDSO32 */ |
00b26474 | 427 | #endif /* VDSO_HAS_CLOCK_GETRES */ |