[linux-block.git] / arch / x86 / kernel / vsyscall_64.c

/*
 *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
 *  Copyright 2003 Andi Kleen, SuSE Labs.
 *
 *  Thanks to hpa@transmeta.com for some useful hint.
 *  Special thanks to Ingo Molnar for his early experience with
 *  a different vsyscall implementation for Linux/IA32 and for the name.
 *
 *  vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
 *  at virtual address -10Mbyte+1024bytes etc... There are at max 4
 *  vsyscalls. One vsyscall can reserve more than 1 slot to avoid
 *  jumping out of line if necessary. We cannot add more with this
 *  mechanism because older kernels won't return -ENOSYS.
 *  If we want more than four we need a vDSO.
 *
 *  Note: the concept clashes with user mode linux. If you use UML and
 *  want per guest time just set the kernel.vsyscall64 sysctl to 0.
 */

/* Disable profiling for userspace code: */
#define DISABLE_BRANCH_PROFILING

#include <linux/time.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/seqlock.h>
#include <linux/jiffies.h>
#include <linux/sysctl.h>
#include <linux/clocksource.h>
#include <linux/getcpu.h>
#include <linux/cpu.h>
#include <linux/smp.h>
#include <linux/notifier.h>

#include <asm/vsyscall.h>
#include <asm/pgtable.h>
#include <asm/page.h>
#include <asm/unistd.h>
#include <asm/fixmap.h>
#include <asm/errno.h>
#include <asm/io.h>
#include <asm/segment.h>
#include <asm/desc.h>
#include <asm/topology.h>
#include <asm/vgtod.h>

#define __vsyscall(nr) \
		__attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
#define __syscall_clobber "r11","cx","memory"

/*
 * vsyscall_gtod_data contains data that is :
 * - readonly from vsyscalls
 * - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
 * Try to keep this structure as small as possible to avoid cache line ping pongs
 */
int __vgetcpu_mode __section_vgetcpu_mode;

struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
{
	.lock = SEQLOCK_UNLOCKED,
	.sysctl_enabled = 1,
};

void update_vsyscall_tz(void)
{
	unsigned long flags;

	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
	/* sys_tz has changed */
	vsyscall_gtod_data.sys_tz = sys_tz;
	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
}

void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
{
	unsigned long flags;

	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
	/* copy vsyscall data */
	vsyscall_gtod_data.clock.vread = clock->vread;
	vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
	vsyscall_gtod_data.clock.mask = clock->mask;
	vsyscall_gtod_data.clock.mult = clock->mult;
	vsyscall_gtod_data.clock.shift = clock->shift;
	vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
	vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
	vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
	vsyscall_gtod_data.wall_time_coarse = __current_kernel_time();
	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
}

/* RED-PEN may want to readd seq locking, but then the variable should be
 * write-once.
 */
static __always_inline void do_get_tz(struct timezone * tz)
{
	*tz = __vsyscall_gtod_data.sys_tz;
}

static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
{
	int ret;
	asm volatile("syscall"
		: "=a" (ret)
		: "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
		: __syscall_clobber );
	return ret;
}

static __always_inline long time_syscall(long *t)
{
	long secs;
	asm volatile("syscall"
		: "=a" (secs)
		: "0" (__NR_time),"D" (t) : __syscall_clobber);
	return secs;
}

static __always_inline void do_vgettimeofday(struct timeval * tv)
{
	cycle_t now, base, mask, cycle_delta;
	unsigned seq;
	unsigned long mult, shift, nsec;
	cycle_t (*vread)(void);
	do {
		seq = read_seqbegin(&__vsyscall_gtod_data.lock);

		vread = __vsyscall_gtod_data.clock.vread;
		if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
			gettimeofday(tv,NULL);
			return;
		}

		now = vread();
		base = __vsyscall_gtod_data.clock.cycle_last;
		mask = __vsyscall_gtod_data.clock.mask;
		mult = __vsyscall_gtod_data.clock.mult;
		shift = __vsyscall_gtod_data.clock.shift;

		tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
		nsec = __vsyscall_gtod_data.wall_time_nsec;
	} while (read_seqretry(&__vsyscall_gtod_data.lock, seq));

	/* calculate interval: */
	cycle_delta = (now - base) & mask;
	/* convert to nsecs: */
	nsec += (cycle_delta * mult) >> shift;

	while (nsec >= NSEC_PER_SEC) {
		tv->tv_sec += 1;
		nsec -= NSEC_PER_SEC;
	}
	tv->tv_usec = nsec / NSEC_PER_USEC;
}

int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
{
	if (tv)
		do_vgettimeofday(tv);
	if (tz)
		do_get_tz(tz);
	return 0;
}

/* This will break when the xtime seconds get inaccurate, but that is
 * unlikely */
time_t __vsyscall(1) vtime(time_t *t)
{
	struct timeval tv;
	time_t result;
	if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
		return time_syscall(t);

	vgettimeofday(&tv, NULL);
	result = tv.tv_sec;
	if (t)
		*t = result;
	return result;
}

/* Fast way to get current CPU and node.
   This helps to do per node and per CPU caches in user space.
   The result is not guaranteed without CPU affinity, but usually
   works out because the scheduler tries to keep a thread on the same
   CPU.

   tcache must point to a two element sized long array.
   All arguments can be NULL. */
long __vsyscall(2)
vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
{
	unsigned int p;
	unsigned long j = 0;

	/* Fast cache - only recompute value once per jiffies and avoid
	   relatively costly rdtscp/cpuid otherwise.
	   This works because the scheduler usually keeps the process
	   on the same CPU and this syscall doesn't guarantee its
	   results anyways.
	   We do this here because otherwise user space would do it on
	   its own in a likely inferior way (no access to jiffies).
	   If you don't like it pass NULL. */
	if (tcache && tcache->blob[0] == (j = __jiffies)) {
		p = tcache->blob[1];
	} else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
		/* Load per CPU data from RDTSCP */
		native_read_tscp(&p);
	} else {
		/* Load per CPU data from GDT */
		asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
	}
	if (tcache) {
		tcache->blob[0] = j;
		tcache->blob[1] = p;
	}
	if (cpu)
		*cpu = p & 0xfff;
	if (node)
		*node = p >> 12;
	return 0;
}

static long __vsyscall(3) venosys_1(void)
{
	return -ENOSYS;
}

#ifdef CONFIG_SYSCTL

static int
vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
		       void __user *buffer, size_t *lenp, loff_t *ppos)
{
	return proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
}

static ctl_table kernel_table2[] = {
	{ .procname = "vsyscall64",
	  .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
	  .mode = 0644,
	  .proc_handler = vsyscall_sysctl_change },
	{}
};

static ctl_table kernel_root_table2[] = {
	{ .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
	  .child = kernel_table2 },
	{}
};
#endif

/* Assume __initcall executes before all user space. Hopefully kmod
   doesn't violate that. We'll find out if it does. */
static void __cpuinit vsyscall_set_cpu(int cpu)
{
	unsigned long d;
	unsigned long node = 0;
#ifdef CONFIG_NUMA
	node = cpu_to_node(cpu);
#endif
	if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
		write_rdtscp_aux((node << 12) | cpu);

	/* Store cpu number in limit so that it can be loaded quickly
	   in user space in vgetcpu.
	   12 bits for the CPU and 8 bits for the node. */
	d = 0x0f40000000000ULL;
	d |= cpu;
	d |= (node & 0xf) << 12;
	d |= (node >> 4) << 48;
	write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
}

static void __cpuinit cpu_vsyscall_init(void *arg)
{
	/* preemption should be already off */
	vsyscall_set_cpu(raw_smp_processor_id());
}

static int __cpuinit
cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
{
	long cpu = (long)arg;
	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
		smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
	return NOTIFY_DONE;
}

void __init map_vsyscall(void)
{
	extern char __vsyscall_0;
	unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);

	/* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
	__set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
}

static int __init vsyscall_init(void)
{
	BUG_ON(((unsigned long) &vgettimeofday !=
			VSYSCALL_ADDR(__NR_vgettimeofday)));
	BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
	BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
	BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
#ifdef CONFIG_SYSCTL
	register_sysctl_table(kernel_root_table2);
#endif
	on_each_cpu(cpu_vsyscall_init, NULL, 1);
	hotcpu_notifier(cpu_vsyscall_notifier, 0);
	return 0;
}

__initcall(vsyscall_init);
Commit	Line	Data
1da177e4	1	/*
1da177e4 LT	2	* Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
	3	* Copyright 2003 Andi Kleen, SuSE Labs.
	4	*
	5	* Thanks to hpa@transmeta.com for some useful hint.
	6	* Special thanks to Ingo Molnar for his early experience with
	7	* a different vsyscall implementation for Linux/IA32 and for the name.
	8	*
	9	* vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
	10	* at virtual address -10Mbyte+1024bytes etc... There are at max 4
	11	* vsyscalls. One vsyscall can reserve more than 1 slot to avoid
	12	* jumping out of line if necessary. We cannot add more with this
	13	* mechanism because older kernels won't return -ENOSYS.
	14	* If we want more than four we need a vDSO.
	15	*
	16	* Note: the concept clashes with user mode linux. If you use UML and
	17	* want per guest time just set the kernel.vsyscall64 sysctl to 0.
	18	*/
	19
2b7d0390	20	/* Disable profiling for userspace code: */
2ed84eeb	21	#define DISABLE_BRANCH_PROFILING
1f0d69a9	22
1da177e4 LT	23	#include <linux/time.h>
	24	#include <linux/init.h>
	25	#include <linux/kernel.h>
	26	#include <linux/timer.h>
	27	#include <linux/seqlock.h>
	28	#include <linux/jiffies.h>
	29	#include <linux/sysctl.h>
7460ed28	30	#include <linux/clocksource.h>
c08c8205	31	#include <linux/getcpu.h>
8c131af1 AK	32	#include <linux/cpu.h>
	33	#include <linux/smp.h>
	34	#include <linux/notifier.h>
1da177e4 LT	35
	36	#include <asm/vsyscall.h>
	37	#include <asm/pgtable.h>
	38	#include <asm/page.h>
7460ed28	39	#include <asm/unistd.h>
1da177e4 LT	40	#include <asm/fixmap.h>
	41	#include <asm/errno.h>
	42	#include <asm/io.h>
c08c8205 VP	43	#include <asm/segment.h>
	44	#include <asm/desc.h>
	45	#include <asm/topology.h>
2aae950b	46	#include <asm/vgtod.h>
1da177e4	47
23adec55 SR	48	#define __vsyscall(nr) \
23adec55 SR	49	__attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
65ea5b03	50	#define __syscall_clobber "r11","cx","memory"
1da177e4	51
c8118c6c ED	52	/*
	53	* vsyscall_gtod_data contains data that is :
	54	* - readonly from vsyscalls
676b1855	55	* - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
c8118c6c ED	56	* Try to keep this structure as small as possible to avoid cache line ping pongs
c8118c6c ED	57	*/
c08c8205	58	int __vgetcpu_mode __section_vgetcpu_mode;
1da177e4	59
2aae950b	60	struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
1da177e4	61	{
7460ed28	62	.lock = SEQLOCK_UNLOCKED,
	63	.sysctl_enabled = 1,
	64	};
1da177e4	65
2c622148 TB	66	void update_vsyscall_tz(void)
	67	{
	68	unsigned long flags;
	69
	70	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
	71	/* sys_tz has changed */
	72	vsyscall_gtod_data.sys_tz = sys_tz;
	73	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
	74	}
	75
7460ed28	76	void update_vsyscall(struct timespec wall_time, struct clocksource clock)
1da177e4	77	{
7460ed28	78	unsigned long flags;
1da177e4	79
7460ed28	80	write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
7460ed28	81	/* copy vsyscall data */
c8118c6c ED	82	vsyscall_gtod_data.clock.vread = clock->vread;
	83	vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
	84	vsyscall_gtod_data.clock.mask = clock->mask;
	85	vsyscall_gtod_data.clock.mult = clock->mult;
	86	vsyscall_gtod_data.clock.shift = clock->shift;
	87	vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
	88	vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
2aae950b	89	vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
da15cfda	90	vsyscall_gtod_data.wall_time_coarse = __current_kernel_time();
7460ed28	91	write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
1da177e4 LT	92	}
1da177e4 LT	93
7460ed28	94	/* RED-PEN may want to readd seq locking, but then the variable should be
	95	* write-once.
	96	*/
2c8bc944	97	static __always_inline void do_get_tz(struct timezone * tz)
1da177e4	98	{
7460ed28	99	*tz = __vsyscall_gtod_data.sys_tz;
1da177e4 LT	100	}
1da177e4 LT	101
2c8bc944	102	static __always_inline int gettimeofday(struct timeval tv, struct timezone tz)
1da177e4 LT	103	{
1da177e4 LT	104	int ret;
ce28b986	105	asm volatile("syscall"
1da177e4	106	: "=a" (ret)
7460ed28	107	: "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
7460ed28	108	: __syscall_clobber );
1da177e4 LT	109	return ret;
	110	}
	111
2c8bc944	112	static __always_inline long time_syscall(long *t)
1da177e4 LT	113	{
1da177e4 LT	114	long secs;
ce28b986	115	asm volatile("syscall"
1da177e4 LT	116	: "=a" (secs)
	117	: "0" (__NR_time),"D" (t) : __syscall_clobber);
	118	return secs;
	119	}
	120
7460ed28	121	static __always_inline void do_vgettimeofday(struct timeval * tv)
	122	{
	123	cycle_t now, base, mask, cycle_delta;
c8118c6c ED	124	unsigned seq;
c8118c6c ED	125	unsigned long mult, shift, nsec;
7460ed28	126	cycle_t (*vread)(void);
	127	do {
	128	seq = read_seqbegin(&__vsyscall_gtod_data.lock);
	129
	130	vread = __vsyscall_gtod_data.clock.vread;
	131	if (unlikely(!__vsyscall_gtod_data.sysctl_enabled \|\| !vread)) {
89952d13	132	gettimeofday(tv,NULL);
7460ed28	133	return;
7460ed28	134	}
cb9e35dc	135
7460ed28	136	now = vread();
	137	base = __vsyscall_gtod_data.clock.cycle_last;
	138	mask = __vsyscall_gtod_data.clock.mask;
	139	mult = __vsyscall_gtod_data.clock.mult;
	140	shift = __vsyscall_gtod_data.clock.shift;
	141
c8118c6c ED	142	tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
c8118c6c ED	143	nsec = __vsyscall_gtod_data.wall_time_nsec;
7460ed28	144	} while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
	145
	146	/* calculate interval: */
	147	cycle_delta = (now - base) & mask;
	148	/* convert to nsecs: */
c8118c6c	149	nsec += (cycle_delta * mult) >> shift;
7460ed28	150
c8118c6c	151	while (nsec >= NSEC_PER_SEC) {
7460ed28	152	tv->tv_sec += 1;
c8118c6c	153	nsec -= NSEC_PER_SEC;
7460ed28	154	}
c8118c6c	155	tv->tv_usec = nsec / NSEC_PER_USEC;
7460ed28	156	}
7460ed28	157
2e8ad43e	158	int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
1da177e4	159	{
1da177e4 LT	160	if (tv)
	161	do_vgettimeofday(tv);
	162	if (tz)
	163	do_get_tz(tz);
	164	return 0;
	165	}
	166
	167	/* This will break when the xtime seconds get inaccurate, but that is
	168	* unlikely */
2e8ad43e	169	time_t __vsyscall(1) vtime(time_t *t)
1da177e4	170	{
d0aff6e6	171	struct timeval tv;
272a3713	172	time_t result;
7460ed28	173	if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
1da177e4	174	return time_syscall(t);
d0aff6e6	175
c80544dc	176	vgettimeofday(&tv, NULL);
d0aff6e6	177	result = tv.tv_sec;
272a3713 ED	178	if (t)
	179	*t = result;
	180	return result;
1da177e4 LT	181	}
1da177e4 LT	182
c08c8205 VP	183	/* Fast way to get current CPU and node.
	184	This helps to do per node and per CPU caches in user space.
	185	The result is not guaranteed without CPU affinity, but usually
	186	works out because the scheduler tries to keep a thread on the same
	187	CPU.
	188
	189	tcache must point to a two element sized long array.
	190	All arguments can be NULL. */
	191	long __vsyscall(2)
	192	vgetcpu(unsigned cpu, unsigned node, struct getcpu_cache *tcache)
1da177e4	193	{
8f12dea6	194	unsigned int p;
c08c8205 VP	195	unsigned long j = 0;
	196
	197	/* Fast cache - only recompute value once per jiffies and avoid
	198	relatively costly rdtscp/cpuid otherwise.
	199	This works because the scheduler usually keeps the process
	200	on the same CPU and this syscall doesn't guarantee its
	201	results anyways.
	202	We do this here because otherwise user space would do it on
	203	its own in a likely inferior way (no access to jiffies).
	204	If you don't like it pass NULL. */
34596dc9 AK	205	if (tcache && tcache->blob[0] == (j = __jiffies)) {
34596dc9 AK	206	p = tcache->blob[1];
c08c8205 VP	207	} else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
c08c8205 VP	208	/* Load per CPU data from RDTSCP */
8f12dea6	209	native_read_tscp(&p);
c08c8205 VP	210	} else {
	211	/* Load per CPU data from GDT */
	212	asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
	213	}
	214	if (tcache) {
34596dc9 AK	215	tcache->blob[0] = j;
34596dc9 AK	216	tcache->blob[1] = p;
c08c8205 VP	217	}
	218	if (cpu)
	219	*cpu = p & 0xfff;
	220	if (node)
	221	*node = p >> 12;
	222	return 0;
1da177e4 LT	223	}
1da177e4 LT	224
a4928cff	225	static long __vsyscall(3) venosys_1(void)
1da177e4 LT	226	{
	227	return -ENOSYS;
	228	}
	229
	230	#ifdef CONFIG_SYSCTL
d67bbacb TG	231
	232	static int
	233	vsyscall_sysctl_change(ctl_table ctl, int write, struct file filp,
	234	void __user buffer, size_t lenp, loff_t *ppos)
	235	{
	236	return proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
	237	}
	238
1da177e4	239	static ctl_table kernel_table2[] = {
282a821f	240	{ .procname = "vsyscall64",
7460ed28	241	.data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
d67bbacb TG	242	.mode = 0644,
d67bbacb TG	243	.proc_handler = vsyscall_sysctl_change },
7a44d37d	244	{}
1da177e4 LT	245	};
	246
	247	static ctl_table kernel_root_table2[] = {
	248	{ .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
	249	.child = kernel_table2 },
7a44d37d	250	{}
1da177e4	251	};
1da177e4 LT	252	#endif
1da177e4 LT	253
8c131af1 AK	254	/* Assume __initcall executes before all user space. Hopefully kmod
	255	doesn't violate that. We'll find out if it does. */
	256	static void __cpuinit vsyscall_set_cpu(int cpu)
c08c8205	257	{
fc8b8a60	258	unsigned long d;
c08c8205 VP	259	unsigned long node = 0;
c08c8205 VP	260	#ifdef CONFIG_NUMA
98c9e27a	261	node = cpu_to_node(cpu);
c08c8205	262	#endif
92cb7612	263	if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
8c131af1	264	write_rdtscp_aux((node << 12) \| cpu);
c08c8205 VP	265
	266	/* Store cpu number in limit so that it can be loaded quickly
	267	in user space in vgetcpu.
	268	12 bits for the CPU and 8 bits for the node. */
fc8b8a60 JF	269	d = 0x0f40000000000ULL;
	270	d \|= cpu;
	271	d \|= (node & 0xf) << 12;
	272	d \|= (node >> 4) << 48;
	273	write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
c08c8205 VP	274	}
c08c8205 VP	275
8c131af1 AK	276	static void __cpuinit cpu_vsyscall_init(void *arg)
	277	{
	278	/* preemption should be already off */
	279	vsyscall_set_cpu(raw_smp_processor_id());
	280	}
	281
	282	static int __cpuinit
	283	cpu_vsyscall_notifier(struct notifier_block n, unsigned long action, void arg)
	284	{
	285	long cpu = (long)arg;
8bb78442	286	if (action == CPU_ONLINE \|\| action == CPU_ONLINE_FROZEN)
8691e5a8	287	smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
8c131af1 AK	288	return NOTIFY_DONE;
	289	}
	290
e4026440	291	void __init map_vsyscall(void)
1da177e4 LT	292	{
	293	extern char __vsyscall_0;
	294	unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
	295
103efcd9	296	/* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
1da177e4 LT	297	__set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
	298	}
	299
	300	static int __init vsyscall_init(void)
	301	{
	302	BUG_ON(((unsigned long) &vgettimeofday !=
	303	VSYSCALL_ADDR(__NR_vgettimeofday)));
	304	BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
	305	BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
c08c8205	306	BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
f3c5f5e7	307	#ifdef CONFIG_SYSCTL
0b4d4147	308	register_sysctl_table(kernel_root_table2);
f3c5f5e7	309	#endif
15c8b6c1	310	on_each_cpu(cpu_vsyscall_init, NULL, 1);
8c131af1	311	hotcpu_notifier(cpu_vsyscall_notifier, 0);
1da177e4 LT	312	return 0;
	313	}
	314
	315	__initcall(vsyscall_init);