[linux-2.6-block.git] / lib / vdso / gettimeofday.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Generic userspace implementations of gettimeofday() and similar.
 */
#include <linux/compiler.h>
#include <linux/math64.h>
#include <linux/time.h>
#include <linux/kernel.h>
#include <linux/hrtimer_defs.h>
#include <vdso/datapage.h>
#include <vdso/helpers.h>

/*
 * The generic vDSO implementation requires that gettimeofday.h
 * provides:
 * - __arch_get_vdso_data(): to get the vdso datapage.
 * - __arch_get_hw_counter(): to get the hw counter based on the
 *   clock_mode.
 * - gettimeofday_fallback(): fallback for gettimeofday.
 * - clock_gettime_fallback(): fallback for clock_gettime.
 * - clock_getres_fallback(): fallback for clock_getres.
 */
#ifdef ENABLE_COMPAT_VDSO
#include <asm/vdso/compat_gettimeofday.h>
#else
#include <asm/vdso/gettimeofday.h>
#endif /* ENABLE_COMPAT_VDSO */

#ifndef vdso_calc_delta
/*
 * Default implementation which works for all sane clocksources. That
 * obviously excludes x86/TSC.
 */
static __always_inline
u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
{
	return ((cycles - last) & mask) * mult;
}
#endif

static int do_hres(const struct vdso_data *vd, clockid_t clk,
		   struct __kernel_timespec *ts)
{
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	u64 cycles, last, sec, ns;
	u32 seq;

	do {
		seq = vdso_read_begin(vd);
		cycles = __arch_get_hw_counter(vd->clock_mode);
		ns = vdso_ts->nsec;
		last = vd->cycle_last;
		if (unlikely((s64)cycles < 0))
			return clock_gettime_fallback(clk, ts);

		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
		ns >>= vd->shift;
		sec = vdso_ts->sec;
	} while (unlikely(vdso_read_retry(vd, seq)));

	/*
	 * Do this outside the loop: a race inside the loop could result
	 * in __iter_div_u64_rem() being extremely slow.
	 */
	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	ts->tv_nsec = ns;

	return 0;
}

static void do_coarse(const struct vdso_data *vd, clockid_t clk,
		      struct __kernel_timespec *ts)
{
	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	u32 seq;

	do {
		seq = vdso_read_begin(vd);
		ts->tv_sec = vdso_ts->sec;
		ts->tv_nsec = vdso_ts->nsec;
	} while (unlikely(vdso_read_retry(vd, seq)));
}

static __maybe_unused int
__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
{
	const struct vdso_data *vd = __arch_get_vdso_data();
	u32 msk;

	/* Check for negative values or invalid clocks */
	if (unlikely((u32) clock >= MAX_CLOCKS))
		goto fallback;

	/*
	 * Convert the clockid to a bitmask and use it to check which
	 * clocks are handled in the VDSO directly.
	 */
	msk = 1U << clock;
	if (likely(msk & VDSO_HRES)) {
		return do_hres(&vd[CS_HRES_COARSE], clock, ts);
	} else if (msk & VDSO_COARSE) {
		do_coarse(&vd[CS_HRES_COARSE], clock, ts);
		return 0;
	} else if (msk & VDSO_RAW) {
		return do_hres(&vd[CS_RAW], clock, ts);
	}

fallback:
	return clock_gettime_fallback(clock, ts);
}

static __maybe_unused int
__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
{
	struct __kernel_timespec ts;
	int ret;

	if (res == NULL)
		goto fallback;

	ret = __cvdso_clock_gettime(clock, &ts);

	if (ret == 0) {
		res->tv_sec = ts.tv_sec;
		res->tv_nsec = ts.tv_nsec;
	}

	return ret;

fallback:
	return clock_gettime_fallback(clock, (struct __kernel_timespec *)res);
}

static __maybe_unused int
__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
	const struct vdso_data *vd = __arch_get_vdso_data();

	if (likely(tv != NULL)) {
		struct __kernel_timespec ts;

		if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
			return gettimeofday_fallback(tv, tz);

		tv->tv_sec = ts.tv_sec;
		tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
	}

	if (unlikely(tz != NULL)) {
		tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
		tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
	}

	return 0;
}

#ifdef VDSO_HAS_TIME
static __maybe_unused time_t __cvdso_time(time_t *time)
{
	const struct vdso_data *vd = __arch_get_vdso_data();
	time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);

	if (time)
		*time = t;

	return t;
}
#endif /* VDSO_HAS_TIME */

#ifdef VDSO_HAS_CLOCK_GETRES
static __maybe_unused
int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
{
	const struct vdso_data *vd = __arch_get_vdso_data();
	u64 ns;
	u32 msk;
	u64 hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);

	/* Check for negative values or invalid clocks */
	if (unlikely((u32) clock >= MAX_CLOCKS))
		goto fallback;

	/*
	 * Convert the clockid to a bitmask and use it to check which
	 * clocks are handled in the VDSO directly.
	 */
	msk = 1U << clock;
	if (msk & VDSO_HRES) {
		/*
		 * Preserves the behaviour of posix_get_hrtimer_res().
		 */
		ns = hrtimer_res;
	} else if (msk & VDSO_COARSE) {
		/*
		 * Preserves the behaviour of posix_get_coarse_res().
		 */
		ns = LOW_RES_NSEC;
	} else if (msk & VDSO_RAW) {
		/*
		 * Preserves the behaviour of posix_get_hrtimer_res().
		 */
		ns = hrtimer_res;
	} else {
		goto fallback;
	}

	if (res) {
		res->tv_sec = 0;
		res->tv_nsec = ns;
	}

	return 0;

fallback:
	return clock_getres_fallback(clock, res);
}

static __maybe_unused int
__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
{
	struct __kernel_timespec ts;
	int ret;

	if (res == NULL)
		goto fallback;

	ret = __cvdso_clock_getres(clock, &ts);

	if (ret == 0) {
		res->tv_sec = ts.tv_sec;
		res->tv_nsec = ts.tv_nsec;
	}

	return ret;

fallback:
	return clock_getres_fallback(clock, (struct __kernel_timespec *)res);
}
#endif /* VDSO_HAS_CLOCK_GETRES */
Commit	Line	Data
00b26474 VF	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Generic userspace implementations of gettimeofday() and similar.
	4	*/
	5	#include <linux/compiler.h>
	6	#include <linux/math64.h>
	7	#include <linux/time.h>
	8	#include <linux/kernel.h>
	9	#include <linux/hrtimer_defs.h>
	10	#include <vdso/datapage.h>
	11	#include <vdso/helpers.h>
	12
	13	/*
	14	* The generic vDSO implementation requires that gettimeofday.h
	15	* provides:
	16	* - __arch_get_vdso_data(): to get the vdso datapage.
	17	* - __arch_get_hw_counter(): to get the hw counter based on the
	18	* clock_mode.
	19	* - gettimeofday_fallback(): fallback for gettimeofday.
	20	* - clock_gettime_fallback(): fallback for clock_gettime.
	21	* - clock_getres_fallback(): fallback for clock_getres.
	22	*/
629fdf77 VF	23	#ifdef ENABLE_COMPAT_VDSO
	24	#include <asm/vdso/compat_gettimeofday.h>
	25	#else
00b26474	26	#include <asm/vdso/gettimeofday.h>
629fdf77	27	#endif /* ENABLE_COMPAT_VDSO */
00b26474	28
9d90b93b TG	29	#ifndef vdso_calc_delta
	30	/*
	31	* Default implementation which works for all sane clocksources. That
	32	* obviously excludes x86/TSC.
	33	*/
	34	static __always_inline
	35	u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
	36	{
	37	return ((cycles - last) & mask) * mult;
	38	}
	39	#endif
	40
00b26474 VF	41	static int do_hres(const struct vdso_data *vd, clockid_t clk,
	42	struct __kernel_timespec *ts)
	43	{
	44	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	45	u64 cycles, last, sec, ns;
	46	u32 seq;
	47
	48	do {
	49	seq = vdso_read_begin(vd);
9d90b93b	50	cycles = __arch_get_hw_counter(vd->clock_mode);
00b26474 VF	51	ns = vdso_ts->nsec;
	52	last = vd->cycle_last;
	53	if (unlikely((s64)cycles < 0))
	54	return clock_gettime_fallback(clk, ts);
9d90b93b TG	55
9d90b93b TG	56	ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
00b26474 VF	57	ns >>= vd->shift;
	58	sec = vdso_ts->sec;
	59	} while (unlikely(vdso_read_retry(vd, seq)));
	60
	61	/*
	62	* Do this outside the loop: a race inside the loop could result
	63	* in __iter_div_u64_rem() being extremely slow.
	64	*/
	65	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
	66	ts->tv_nsec = ns;
	67
	68	return 0;
	69	}
	70
	71	static void do_coarse(const struct vdso_data *vd, clockid_t clk,
	72	struct __kernel_timespec *ts)
	73	{
	74	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
	75	u32 seq;
	76
	77	do {
	78	seq = vdso_read_begin(vd);
	79	ts->tv_sec = vdso_ts->sec;
	80	ts->tv_nsec = vdso_ts->nsec;
	81	} while (unlikely(vdso_read_retry(vd, seq)));
	82	}
	83
	84	static __maybe_unused int
	85	__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
	86	{
	87	const struct vdso_data *vd = __arch_get_vdso_data();
	88	u32 msk;
	89
	90	/* Check for negative values or invalid clocks */
	91	if (unlikely((u32) clock >= MAX_CLOCKS))
	92	goto fallback;
	93
	94	/*
	95	* Convert the clockid to a bitmask and use it to check which
	96	* clocks are handled in the VDSO directly.
	97	*/
	98	msk = 1U << clock;
	99	if (likely(msk & VDSO_HRES)) {
	100	return do_hres(&vd[CS_HRES_COARSE], clock, ts);
	101	} else if (msk & VDSO_COARSE) {
	102	do_coarse(&vd[CS_HRES_COARSE], clock, ts);
	103	return 0;
	104	} else if (msk & VDSO_RAW) {
	105	return do_hres(&vd[CS_RAW], clock, ts);
	106	}
	107
	108	fallback:
	109	return clock_gettime_fallback(clock, ts);
	110	}
	111
	112	static __maybe_unused int
	113	__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
	114	{
	115	struct __kernel_timespec ts;
	116	int ret;
	117
	118	if (res == NULL)
	119	goto fallback;
	120
121	ret = __cvdso_clock_gettime(clock, &ts);
122
123	if (ret == 0) {
124	res->tv_sec = ts.tv_sec;
125	res->tv_nsec = ts.tv_nsec;
126	}
127
128	return ret;
129
130	fallback:
131	return clock_gettime_fallback(clock, (struct __kernel_timespec *)res);
132	}
133
134	static __maybe_unused int
135	__cvdso_gettimeofday(struct __kernel_old_timeval tv, struct timezone tz)
136	{
137	const struct vdso_data *vd = __arch_get_vdso_data();
138
139	if (likely(tv != NULL)) {
140	struct __kernel_timespec ts;
141
142	if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
143	return gettimeofday_fallback(tv, tz);
144
145	tv->tv_sec = ts.tv_sec;
146	tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
147	}
148
149	if (unlikely(tz != NULL)) {
150	tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
151	tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
152	}
153
154	return 0;
155	}
156
157	#ifdef VDSO_HAS_TIME
158	static __maybe_unused time_t __cvdso_time(time_t *time)
159	{
160	const struct vdso_data *vd = __arch_get_vdso_data();
161	time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
162
163	if (time)
164	*time = t;
165
166	return t;
167	}
168	#endif /* VDSO_HAS_TIME */
169
170	#ifdef VDSO_HAS_CLOCK_GETRES
171	static __maybe_unused
172	int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
173	{
174	const struct vdso_data *vd = __arch_get_vdso_data();
175	u64 ns;
176	u32 msk;
177	u64 hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
178
179	/* Check for negative values or invalid clocks */
180	if (unlikely((u32) clock >= MAX_CLOCKS))
181	goto fallback;
182
183	/*
184	* Convert the clockid to a bitmask and use it to check which
185	* clocks are handled in the VDSO directly.
186	*/
187	msk = 1U << clock;
188	if (msk & VDSO_HRES) {
189	/*
190	* Preserves the behaviour of posix_get_hrtimer_res().
191	*/
192	ns = hrtimer_res;
193	} else if (msk & VDSO_COARSE) {
194	/*
195	* Preserves the behaviour of posix_get_coarse_res().
196	*/
197	ns = LOW_RES_NSEC;
198	} else if (msk & VDSO_RAW) {
199	/*
200	* Preserves the behaviour of posix_get_hrtimer_res().
201	*/
202	ns = hrtimer_res;
203	} else {
204	goto fallback;
205	}
206
207	if (res) {
208	res->tv_sec = 0;
209	res->tv_nsec = ns;
210	}
211
212	return 0;
213
214	fallback:
215	return clock_getres_fallback(clock, res);
216	}
217
218	static __maybe_unused int
219	__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
220	{
221	struct __kernel_timespec ts;
222	int ret;
223
224	if (res == NULL)
225	goto fallback;
226
227	ret = __cvdso_clock_getres(clock, &ts);
228
229	if (ret == 0) {
230	res->tv_sec = ts.tv_sec;
231	res->tv_nsec = ts.tv_nsec;
232	}
233
234	return ret;
235
236	fallback:
237	return clock_getres_fallback(clock, (struct __kernel_timespec *)res);
238	}
239	#endif /* VDSO_HAS_CLOCK_GETRES */