[linux-2.6-block.git] / arch / x86 / vdso / vclock_gettime.c

/*
 * Copyright 2006 Andi Kleen, SUSE Labs.
 * Subject to the GNU Public License, v.2
 *
 * Fast user context implementation of clock_gettime and gettimeofday.
 *
 * The code should have no internal unresolved relocations.
 * Check with readelf after changing.
 * Also alternative() doesn't work.
 */

/* Disable profiling for userspace code: */
#define DISABLE_BRANCH_PROFILING

#include <linux/kernel.h>
#include <linux/posix-timers.h>
#include <linux/time.h>
#include <linux/string.h>
#include <asm/vsyscall.h>
#include <asm/vgtod.h>
#include <asm/timex.h>
#include <asm/hpet.h>
#include <asm/unistd.h>
#include <asm/io.h>
#include "vextern.h"

#define gtod vdso_vsyscall_gtod_data

notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
{
	long ret;
	asm("syscall" : "=a" (ret) :
	    "0" (__NR_clock_gettime),"D" (clock), "S" (ts) : "memory");
	return ret;
}

notrace static inline long vgetns(void)
{
	long v;
	cycles_t (*vread)(void);
	vread = gtod->clock.vread;
	v = (vread() - gtod->clock.cycle_last) & gtod->clock.mask;
	return (v * gtod->clock.mult) >> gtod->clock.shift;
}

notrace static noinline int do_realtime(struct timespec *ts)
{
	unsigned long seq, ns;
	do {
		seq = read_seqbegin(&gtod->lock);
		ts->tv_sec = gtod->wall_time_sec;
		ts->tv_nsec = gtod->wall_time_nsec;
		ns = vgetns();
	} while (unlikely(read_seqretry(&gtod->lock, seq)));
	timespec_add_ns(ts, ns);
	return 0;
}

/* Copy of the version in kernel/time.c which we cannot directly access */
notrace static void
vset_normalized_timespec(struct timespec *ts, long sec, long nsec)
{
	while (nsec >= NSEC_PER_SEC) {
		nsec -= NSEC_PER_SEC;
		++sec;
	}
	while (nsec < 0) {
		nsec += NSEC_PER_SEC;
		--sec;
	}
	ts->tv_sec = sec;
	ts->tv_nsec = nsec;
}

notrace static noinline int do_monotonic(struct timespec *ts)
{
	unsigned long seq, ns, secs;
	do {
		seq = read_seqbegin(&gtod->lock);
		secs = gtod->wall_time_sec;
		ns = gtod->wall_time_nsec + vgetns();
		secs += gtod->wall_to_monotonic.tv_sec;
		ns += gtod->wall_to_monotonic.tv_nsec;
	} while (unlikely(read_seqretry(&gtod->lock, seq)));
	vset_normalized_timespec(ts, secs, ns);
	return 0;
}

notrace static noinline int do_realtime_coarse(struct timespec *ts)
{
	unsigned long seq;
	do {
		seq = read_seqbegin(&gtod->lock);
		ts->tv_sec = gtod->wall_time_coarse.tv_sec;
		ts->tv_nsec = gtod->wall_time_coarse.tv_nsec;
	} while (unlikely(read_seqretry(&gtod->lock, seq)));
	return 0;
}

notrace static noinline int do_monotonic_coarse(struct timespec *ts)
{
	unsigned long seq, ns, secs;
	do {
		seq = read_seqbegin(&gtod->lock);
		secs = gtod->wall_time_coarse.tv_sec;
		ns = gtod->wall_time_coarse.tv_nsec;
		secs += gtod->wall_to_monotonic.tv_sec;
		ns += gtod->wall_to_monotonic.tv_nsec;
	} while (unlikely(read_seqretry(&gtod->lock, seq)));
	vset_normalized_timespec(ts, secs, ns);
	return 0;
}

notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
{
	if (likely(gtod->sysctl_enabled))
		switch (clock) {
		case CLOCK_REALTIME:
			if (likely(gtod->clock.vread))
				return do_realtime(ts);
			break;
		case CLOCK_MONOTONIC:
			if (likely(gtod->clock.vread))
				return do_monotonic(ts);
			break;
		case CLOCK_REALTIME_COARSE:
			return do_realtime_coarse(ts);
		case CLOCK_MONOTONIC_COARSE:
			return do_monotonic_coarse(ts);
		}
	return vdso_fallback_gettime(clock, ts);
}
int clock_gettime(clockid_t, struct timespec *)
	__attribute__((weak, alias("__vdso_clock_gettime")));

notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
{
	long ret;
	if (likely(gtod->sysctl_enabled && gtod->clock.vread)) {
		if (likely(tv != NULL)) {
			BUILD_BUG_ON(offsetof(struct timeval, tv_usec) !=
				     offsetof(struct timespec, tv_nsec) ||
				     sizeof(*tv) != sizeof(struct timespec));
			do_realtime((struct timespec *)tv);
			tv->tv_usec /= 1000;
		}
		if (unlikely(tz != NULL)) {
			/* Avoid memcpy. Some old compilers fail to inline it */
			tz->tz_minuteswest = gtod->sys_tz.tz_minuteswest;
			tz->tz_dsttime = gtod->sys_tz.tz_dsttime;
		}
		return 0;
	}
	asm("syscall" : "=a" (ret) :
	    "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
	return ret;
}
int gettimeofday(struct timeval *, struct timezone *)
	__attribute__((weak, alias("__vdso_gettimeofday")));
Commit	Line	Data
2aae950b AK	1	/*
	2	* Copyright 2006 Andi Kleen, SUSE Labs.
	3	* Subject to the GNU Public License, v.2
	4	*
	5	* Fast user context implementation of clock_gettime and gettimeofday.
	6	*
	7	* The code should have no internal unresolved relocations.
	8	* Check with readelf after changing.
	9	* Also alternative() doesn't work.
	10	*/
	11
2b7d0390	12	/* Disable profiling for userspace code: */
2ed84eeb	13	#define DISABLE_BRANCH_PROFILING
2b7d0390	14
2aae950b AK	15	#include <linux/kernel.h>
	16	#include <linux/posix-timers.h>
	17	#include <linux/time.h>
	18	#include <linux/string.h>
	19	#include <asm/vsyscall.h>
	20	#include <asm/vgtod.h>
	21	#include <asm/timex.h>
	22	#include <asm/hpet.h>
	23	#include <asm/unistd.h>
	24	#include <asm/io.h>
2aae950b AK	25	#include "vextern.h"
	26
	27	#define gtod vdso_vsyscall_gtod_data
	28
23adec55	29	notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
2aae950b AK	30	{
	31	long ret;
	32	asm("syscall" : "=a" (ret) :
	33	"0" (__NR_clock_gettime),"D" (clock), "S" (ts) : "memory");
	34	return ret;
	35	}
	36
23adec55	37	notrace static inline long vgetns(void)
2aae950b	38	{
95b08679	39	long v;
2aae950b AK	40	cycles_t (*vread)(void);
2aae950b AK	41	vread = gtod->clock.vread;
95b08679 AK	42	v = (vread() - gtod->clock.cycle_last) & gtod->clock.mask;
95b08679 AK	43	return (v * gtod->clock.mult) >> gtod->clock.shift;
2aae950b AK	44	}
2aae950b AK	45
23adec55	46	notrace static noinline int do_realtime(struct timespec *ts)
2aae950b AK	47	{
	48	unsigned long seq, ns;
	49	do {
	50	seq = read_seqbegin(&gtod->lock);
	51	ts->tv_sec = gtod->wall_time_sec;
	52	ts->tv_nsec = gtod->wall_time_nsec;
	53	ns = vgetns();
	54	} while (unlikely(read_seqretry(&gtod->lock, seq)));
	55	timespec_add_ns(ts, ns);
	56	return 0;
	57	}
	58
	59	/* Copy of the version in kernel/time.c which we cannot directly access */
23adec55 SR	60	notrace static void
23adec55 SR	61	vset_normalized_timespec(struct timespec *ts, long sec, long nsec)
2aae950b AK	62	{
	63	while (nsec >= NSEC_PER_SEC) {
	64	nsec -= NSEC_PER_SEC;
	65	++sec;
	66	}
	67	while (nsec < 0) {
	68	nsec += NSEC_PER_SEC;
	69	--sec;
	70	}
	71	ts->tv_sec = sec;
	72	ts->tv_nsec = nsec;
	73	}
	74
23adec55	75	notrace static noinline int do_monotonic(struct timespec *ts)
2aae950b AK	76	{
	77	unsigned long seq, ns, secs;
	78	do {
	79	seq = read_seqbegin(&gtod->lock);
	80	secs = gtod->wall_time_sec;
	81	ns = gtod->wall_time_nsec + vgetns();
	82	secs += gtod->wall_to_monotonic.tv_sec;
	83	ns += gtod->wall_to_monotonic.tv_nsec;
	84	} while (unlikely(read_seqretry(&gtod->lock, seq)));
	85	vset_normalized_timespec(ts, secs, ns);
	86	return 0;
	87	}
	88
da15cfda	89	notrace static noinline int do_realtime_coarse(struct timespec *ts)
	90	{
	91	unsigned long seq;
	92	do {
	93	seq = read_seqbegin(&gtod->lock);
	94	ts->tv_sec = gtod->wall_time_coarse.tv_sec;
	95	ts->tv_nsec = gtod->wall_time_coarse.tv_nsec;
	96	} while (unlikely(read_seqretry(&gtod->lock, seq)));
	97	return 0;
	98	}
	99
	100	notrace static noinline int do_monotonic_coarse(struct timespec *ts)
	101	{
	102	unsigned long seq, ns, secs;
	103	do {
	104	seq = read_seqbegin(&gtod->lock);
	105	secs = gtod->wall_time_coarse.tv_sec;
	106	ns = gtod->wall_time_coarse.tv_nsec;
	107	secs += gtod->wall_to_monotonic.tv_sec;
	108	ns += gtod->wall_to_monotonic.tv_nsec;
	109	} while (unlikely(read_seqretry(&gtod->lock, seq)));
	110	vset_normalized_timespec(ts, secs, ns);
	111	return 0;
	112	}
	113
23adec55	114	notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
2aae950b	115	{
da15cfda	116	if (likely(gtod->sysctl_enabled))
2aae950b AK	117	switch (clock) {
2aae950b AK	118	case CLOCK_REALTIME:
da15cfda	119	if (likely(gtod->clock.vread))
	120	return do_realtime(ts);
	121	break;
2aae950b	122	case CLOCK_MONOTONIC:
da15cfda	123	if (likely(gtod->clock.vread))
	124	return do_monotonic(ts);
	125	break;
	126	case CLOCK_REALTIME_COARSE:
	127	return do_realtime_coarse(ts);
	128	case CLOCK_MONOTONIC_COARSE:
	129	return do_monotonic_coarse(ts);
2aae950b AK	130	}
	131	return vdso_fallback_gettime(clock, ts);
	132	}
	133	int clock_gettime(clockid_t, struct timespec *)
	134	__attribute__((weak, alias("__vdso_clock_gettime")));
	135
23adec55	136	notrace int __vdso_gettimeofday(struct timeval tv, struct timezone tz)
2aae950b AK	137	{
	138	long ret;
	139	if (likely(gtod->sysctl_enabled && gtod->clock.vread)) {
2f65dd47 JW	140	if (likely(tv != NULL)) {
	141	BUILD_BUG_ON(offsetof(struct timeval, tv_usec) !=
	142	offsetof(struct timespec, tv_nsec) \|\|
	143	sizeof(*tv) != sizeof(struct timespec));
	144	do_realtime((struct timespec *)tv);
	145	tv->tv_usec /= 1000;
	146	}
2aae950b	147	if (unlikely(tz != NULL)) {
a1289643 AK	148	/* Avoid memcpy. Some old compilers fail to inline it */
	149	tz->tz_minuteswest = gtod->sys_tz.tz_minuteswest;
	150	tz->tz_dsttime = gtod->sys_tz.tz_dsttime;
2aae950b AK	151	}
	152	return 0;
	153	}
	154	asm("syscall" : "=a" (ret) :
	155	"0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory");
	156	return ret;
	157	}
	158	int gettimeofday(struct timeval , struct timezone )
	159	__attribute__((weak, alias("__vdso_gettimeofday")));