[linux-2.6-block.git] / arch / x86 / kernel / alternative.c

#include <linux/module.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/kprobes.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <asm/alternative.h>
#include <asm/sections.h>
#include <asm/pgtable.h>
#include <asm/mce.h>
#include <asm/nmi.h>
#include <asm/vsyscall.h>

#define MAX_PATCH_LEN (255-1)

#ifdef CONFIG_HOTPLUG_CPU
static int smp_alt_once;

static int __init bootonly(char *str)
{
	smp_alt_once = 1;
	return 1;
}
__setup("smp-alt-boot", bootonly);
#else
#define smp_alt_once 1
#endif

static int debug_alternative;

static int __init debug_alt(char *str)
{
	debug_alternative = 1;
	return 1;
}
__setup("debug-alternative", debug_alt);

static int noreplace_smp;

static int __init setup_noreplace_smp(char *str)
{
	noreplace_smp = 1;
	return 1;
}
__setup("noreplace-smp", setup_noreplace_smp);

#ifdef CONFIG_PARAVIRT
static int noreplace_paravirt = 0;

static int __init setup_noreplace_paravirt(char *str)
{
	noreplace_paravirt = 1;
	return 1;
}
__setup("noreplace-paravirt", setup_noreplace_paravirt);
#endif

#define DPRINTK(fmt, args...) if (debug_alternative) \
	printk(KERN_DEBUG fmt, args)

#ifdef GENERIC_NOP1
/* Use inline assembly to define this because the nops are defined
   as inline assembly strings in the include files and we cannot
   get them easily into strings. */
asm("\t.section .rodata, \"a\"\nintelnops: "
	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	GENERIC_NOP7 GENERIC_NOP8);
extern const unsigned char intelnops[];
static const unsigned char *const intel_nops[ASM_NOP_MAX+1] = {
	NULL,
	intelnops,
	intelnops + 1,
	intelnops + 1 + 2,
	intelnops + 1 + 2 + 3,
	intelnops + 1 + 2 + 3 + 4,
	intelnops + 1 + 2 + 3 + 4 + 5,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K8_NOP1
asm("\t.section .rodata, \"a\"\nk8nops: "
	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	K8_NOP7 K8_NOP8);
extern const unsigned char k8nops[];
static const unsigned char *const k8_nops[ASM_NOP_MAX+1] = {
	NULL,
	k8nops,
	k8nops + 1,
	k8nops + 1 + 2,
	k8nops + 1 + 2 + 3,
	k8nops + 1 + 2 + 3 + 4,
	k8nops + 1 + 2 + 3 + 4 + 5,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K7_NOP1
asm("\t.section .rodata, \"a\"\nk7nops: "
	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	K7_NOP7 K7_NOP8);
extern const unsigned char k7nops[];
static const unsigned char *const k7_nops[ASM_NOP_MAX+1] = {
	NULL,
	k7nops,
	k7nops + 1,
	k7nops + 1 + 2,
	k7nops + 1 + 2 + 3,
	k7nops + 1 + 2 + 3 + 4,
	k7nops + 1 + 2 + 3 + 4 + 5,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef CONFIG_X86_64

extern char __vsyscall_0;
static inline const unsigned char*const * find_nop_table(void)
{
	return k8_nops;
}

#else /* CONFIG_X86_64 */

static const struct nop {
	int cpuid;
	const unsigned char *const *noptable;
} noptypes[] = {
	{ X86_FEATURE_K8, k8_nops },
	{ X86_FEATURE_K7, k7_nops },
	{ -1, NULL }
};

static const unsigned char*const * find_nop_table(void)
{
	const unsigned char *const *noptable = intel_nops;
	int i;

	for (i = 0; noptypes[i].cpuid >= 0; i++) {
		if (boot_cpu_has(noptypes[i].cpuid)) {
			noptable = noptypes[i].noptable;
			break;
		}
	}
	return noptable;
}

#endif /* CONFIG_X86_64 */

/* Use this to add nops to a buffer, then text_poke the whole buffer. */
static void add_nops(void *insns, unsigned int len)
{
	const unsigned char *const *noptable = find_nop_table();

	while (len > 0) {
		unsigned int noplen = len;
		if (noplen > ASM_NOP_MAX)
			noplen = ASM_NOP_MAX;
		memcpy(insns, noptable[noplen], noplen);
		insns += noplen;
		len -= noplen;
	}
}

extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern u8 *__smp_locks[], *__smp_locks_end[];

/* Replace instructions with better alternatives for this CPU type.
   This runs before SMP is initialized to avoid SMP problems with
   self modifying code. This implies that assymetric systems where
   APs have less capabilities than the boot processor are not handled.
   Tough. Make sure you disable such features by hand. */

void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
{
	struct alt_instr *a;
	char insnbuf[MAX_PATCH_LEN];

	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {
		u8 *instr = a->instr;
		BUG_ON(a->replacementlen > a->instrlen);
		BUG_ON(a->instrlen > sizeof(insnbuf));
		if (!boot_cpu_has(a->cpuid))
			continue;
#ifdef CONFIG_X86_64
		/* vsyscall code is not mapped yet. resolve it manually. */
		if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
			instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
			DPRINTK("%s: vsyscall fixup: %p => %p\n",
				__FUNCTION__, a->instr, instr);
		}
#endif
		memcpy(insnbuf, a->replacement, a->replacementlen);
		add_nops(insnbuf + a->replacementlen,
			 a->instrlen - a->replacementlen);
		text_poke(instr, insnbuf, a->instrlen);
	}
}

#ifdef CONFIG_SMP

static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		text_poke(*ptr, ((unsigned char []){0xf0}), 1); /* add lock prefix */
	};
}

static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;
	char insn[1];

	if (noreplace_smp)
		return;

	add_nops(insn, 1);
	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		text_poke(*ptr, insn, 1);
	};
}

struct smp_alt_module {
	/* what is this ??? */
	struct module	*mod;
	char		*name;

	/* ptrs to lock prefixes */
	u8		**locks;
	u8		**locks_end;

	/* .text segment, needed to avoid patching init code ;) */
	u8		*text;
	u8		*text_end;

	struct list_head next;
};
static LIST_HEAD(smp_alt_modules);
static DEFINE_SPINLOCK(smp_alt);

void alternatives_smp_module_add(struct module *mod, char *name,
				 void *locks, void *locks_end,
				 void *text,  void *text_end)
{
	struct smp_alt_module *smp;
	unsigned long flags;

	if (noreplace_smp)
		return;

	if (smp_alt_once) {
		if (boot_cpu_has(X86_FEATURE_UP))
			alternatives_smp_unlock(locks, locks_end,
						text, text_end);
		return;
	}

	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	if (NULL == smp)
		return; /* we'll run the (safe but slow) SMP code then ... */

	smp->mod	= mod;
	smp->name	= name;
	smp->locks	= locks;
	smp->locks_end	= locks_end;
	smp->text	= text;
	smp->text_end	= text_end;
	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
		__FUNCTION__, smp->locks, smp->locks_end,
		smp->text, smp->text_end, smp->name);

	spin_lock_irqsave(&smp_alt, flags);
	list_add_tail(&smp->next, &smp_alt_modules);
	if (boot_cpu_has(X86_FEATURE_UP))
		alternatives_smp_unlock(smp->locks, smp->locks_end,
					smp->text, smp->text_end);
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_module_del(struct module *mod)
{
	struct smp_alt_module *item;
	unsigned long flags;

	if (smp_alt_once || noreplace_smp)
		return;

	spin_lock_irqsave(&smp_alt, flags);
	list_for_each_entry(item, &smp_alt_modules, next) {
		if (mod != item->mod)
			continue;
		list_del(&item->next);
		spin_unlock_irqrestore(&smp_alt, flags);
		DPRINTK("%s: %s\n", __FUNCTION__, item->name);
		kfree(item);
		return;
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_switch(int smp)
{
	struct smp_alt_module *mod;
	unsigned long flags;

#ifdef CONFIG_LOCKDEP
	/*
	 * A not yet fixed binutils section handling bug prevents
	 * alternatives-replacement from working reliably, so turn
	 * it off:
	 */
	printk("lockdep: not fixing up alternatives.\n");
	return;
#endif

	if (noreplace_smp || smp_alt_once)
		return;
	BUG_ON(!smp && (num_online_cpus() > 1));

	spin_lock_irqsave(&smp_alt, flags);
	if (smp) {
		printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
		clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_lock(mod->locks, mod->locks_end,
					      mod->text, mod->text_end);
	} else {
		printk(KERN_INFO "SMP alternatives: switching to UP code\n");
		set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_unlock(mod->locks, mod->locks_end,
						mod->text, mod->text_end);
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

#endif

#ifdef CONFIG_PARAVIRT
void apply_paravirt(struct paravirt_patch_site *start,
		    struct paravirt_patch_site *end)
{
	struct paravirt_patch_site *p;
	char insnbuf[MAX_PATCH_LEN];

	if (noreplace_paravirt)
		return;

	for (p = start; p < end; p++) {
		unsigned int used;

		BUG_ON(p->len > MAX_PATCH_LEN);
		/* prep the buffer with the original instructions */
		memcpy(insnbuf, p->instr, p->len);
		used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf,
					  (unsigned long)p->instr, p->len);

		BUG_ON(used > p->len);

		/* Pad the rest with nops */
		add_nops(insnbuf + used, p->len - used);
		text_poke(p->instr, insnbuf, p->len);
	}
}
extern struct paravirt_patch_site __start_parainstructions[],
	__stop_parainstructions[];
#endif	/* CONFIG_PARAVIRT */

void __init alternative_instructions(void)
{
	unsigned long flags;

	/* The patching is not fully atomic, so try to avoid local interruptions
	   that might execute the to be patched code.
	   Other CPUs are not running. */
	stop_nmi();
#ifdef CONFIG_X86_MCE
	stop_mce();
#endif

	local_irq_save(flags);
	apply_alternatives(__alt_instructions, __alt_instructions_end);

	/* switch to patch-once-at-boottime-only mode and free the
	 * tables in case we know the number of CPUs will never ever
	 * change */
#ifdef CONFIG_HOTPLUG_CPU
	if (num_possible_cpus() < 2)
		smp_alt_once = 1;
#endif

#ifdef CONFIG_SMP
	if (smp_alt_once) {
		if (1 == num_possible_cpus()) {
			printk(KERN_INFO "SMP alternatives: switching to UP code\n");
			set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
			set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
			alternatives_smp_unlock(__smp_locks, __smp_locks_end,
						_text, _etext);
		}
	} else {
		alternatives_smp_module_add(NULL, "core kernel",
					    __smp_locks, __smp_locks_end,
					    _text, _etext);
		alternatives_smp_switch(0);
	}
#endif
 	apply_paravirt(__parainstructions, __parainstructions_end);
	local_irq_restore(flags);

	if (smp_alt_once)
		free_init_pages("SMP alternatives",
				(unsigned long)__smp_locks,
				(unsigned long)__smp_locks_end);

	restart_nmi();
#ifdef CONFIG_X86_MCE
	restart_mce();
#endif
}

/*
 * Warning:
 * When you use this code to patch more than one byte of an instruction
 * you need to make sure that other CPUs cannot execute this code in parallel.
 * Also no thread must be currently preempted in the middle of these instructions.
 * And on the local CPU you need to be protected again NMI or MCE handlers
 * seeing an inconsistent instruction while you patch.
 */
void __kprobes text_poke(void *addr, unsigned char *opcode, int len)
{
	memcpy(addr, opcode, len);
	sync_core();
	/* Could also do a CLFLUSH here to speed up CPU recovery; but
	   that causes hangs on some VIA CPUs. */
}
Commit	Line	Data
9a0b5817	1	#include <linux/module.h>
f6a57033	2	#include <linux/sched.h>
9a0b5817 GH	3	#include <linux/spinlock.h>
9a0b5817 GH	4	#include <linux/list.h>
19d36ccd AK	5	#include <linux/kprobes.h>
	6	#include <linux/mm.h>
	7	#include <linux/vmalloc.h>
9a0b5817 GH	8	#include <asm/alternative.h>
9a0b5817 GH	9	#include <asm/sections.h>
19d36ccd	10	#include <asm/pgtable.h>
8f4e956b AK	11	#include <asm/mce.h>
8f4e956b AK	12	#include <asm/nmi.h>
b097976e	13	#include <asm/vsyscall.h>
9a0b5817	14
ab144f5e AK	15	#define MAX_PATCH_LEN (255-1)
ab144f5e AK	16
09488165 JB	17	#ifdef CONFIG_HOTPLUG_CPU
09488165 JB	18	static int smp_alt_once;
9a0b5817	19
d167a518 GH	20	static int __init bootonly(char *str)
	21	{
	22	smp_alt_once = 1;
	23	return 1;
	24	}
b7fb4af0	25	__setup("smp-alt-boot", bootonly);
09488165 JB	26	#else
	27	#define smp_alt_once 1
	28	#endif
	29
	30	static int debug_alternative;
b7fb4af0	31
d167a518 GH	32	static int __init debug_alt(char *str)
	33	{
	34	debug_alternative = 1;
	35	return 1;
	36	}
d167a518 GH	37	__setup("debug-alternative", debug_alt);
d167a518 GH	38
09488165 JB	39	static int noreplace_smp;
09488165 JB	40
b7fb4af0 JF	41	static int __init setup_noreplace_smp(char *str)
	42	{
	43	noreplace_smp = 1;
	44	return 1;
	45	}
	46	__setup("noreplace-smp", setup_noreplace_smp);
	47
959b4fdf JF	48	#ifdef CONFIG_PARAVIRT
	49	static int noreplace_paravirt = 0;
	50
	51	static int __init setup_noreplace_paravirt(char *str)
	52	{
	53	noreplace_paravirt = 1;
	54	return 1;
	55	}
	56	__setup("noreplace-paravirt", setup_noreplace_paravirt);
	57	#endif
b7fb4af0	58
d167a518 GH	59	#define DPRINTK(fmt, args...) if (debug_alternative) \
	60	printk(KERN_DEBUG fmt, args)
	61
	62	#ifdef GENERIC_NOP1
9a0b5817 GH	63	/* Use inline assembly to define this because the nops are defined
	64	as inline assembly strings in the include files and we cannot
	65	get them easily into strings. */
121d7bf5	66	asm("\t.section .rodata, \"a\"\nintelnops: "
9a0b5817 GH	67	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
9a0b5817 GH	68	GENERIC_NOP7 GENERIC_NOP8);
121d7bf5 JB	69	extern const unsigned char intelnops[];
121d7bf5 JB	70	static const unsigned char *const intel_nops[ASM_NOP_MAX+1] = {
9a0b5817 GH	71	NULL,
	72	intelnops,
	73	intelnops + 1,
	74	intelnops + 1 + 2,
	75	intelnops + 1 + 2 + 3,
	76	intelnops + 1 + 2 + 3 + 4,
	77	intelnops + 1 + 2 + 3 + 4 + 5,
	78	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	79	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	80	};
d167a518 GH	81	#endif
	82
	83	#ifdef K8_NOP1
121d7bf5	84	asm("\t.section .rodata, \"a\"\nk8nops: "
d167a518 GH	85	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
d167a518 GH	86	K8_NOP7 K8_NOP8);
121d7bf5 JB	87	extern const unsigned char k8nops[];
121d7bf5 JB	88	static const unsigned char *const k8_nops[ASM_NOP_MAX+1] = {
9a0b5817 GH	89	NULL,
	90	k8nops,
	91	k8nops + 1,
	92	k8nops + 1 + 2,
	93	k8nops + 1 + 2 + 3,
	94	k8nops + 1 + 2 + 3 + 4,
	95	k8nops + 1 + 2 + 3 + 4 + 5,
	96	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	97	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	98	};
d167a518 GH	99	#endif
	100
	101	#ifdef K7_NOP1
121d7bf5	102	asm("\t.section .rodata, \"a\"\nk7nops: "
d167a518 GH	103	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
d167a518 GH	104	K7_NOP7 K7_NOP8);
121d7bf5 JB	105	extern const unsigned char k7nops[];
121d7bf5 JB	106	static const unsigned char *const k7_nops[ASM_NOP_MAX+1] = {
9a0b5817 GH	107	NULL,
	108	k7nops,
	109	k7nops + 1,
	110	k7nops + 1 + 2,
	111	k7nops + 1 + 2 + 3,
	112	k7nops + 1 + 2 + 3 + 4,
	113	k7nops + 1 + 2 + 3 + 4 + 5,
	114	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	115	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	116	};
d167a518 GH	117	#endif
	118
	119	#ifdef CONFIG_X86_64
	120
	121	extern char __vsyscall_0;
121d7bf5	122	static inline const unsigned charconst find_nop_table(void)
d167a518 GH	123	{
	124	return k8_nops;
	125	}
	126
	127	#else /* CONFIG_X86_64 */
	128
121d7bf5	129	static const struct nop {
9a0b5817	130	int cpuid;
121d7bf5	131	const unsigned char const noptable;
9a0b5817 GH	132	} noptypes[] = {
	133	{ X86_FEATURE_K8, k8_nops },
	134	{ X86_FEATURE_K7, k7_nops },
	135	{ -1, NULL }
	136	};
	137
121d7bf5	138	static const unsigned charconst find_nop_table(void)
9a0b5817	139	{
121d7bf5	140	const unsigned char const noptable = intel_nops;
9a0b5817 GH	141	int i;
	142
	143	for (i = 0; noptypes[i].cpuid >= 0; i++) {
	144	if (boot_cpu_has(noptypes[i].cpuid)) {
	145	noptable = noptypes[i].noptable;
	146	break;
	147	}
	148	}
	149	return noptable;
	150	}
	151
d167a518 GH	152	#endif /* CONFIG_X86_64 */
d167a518 GH	153
ab144f5e AK	154	/* Use this to add nops to a buffer, then text_poke the whole buffer. */
ab144f5e AK	155	static void add_nops(void *insns, unsigned int len)
139ec7c4	156	{
121d7bf5	157	const unsigned char const noptable = find_nop_table();
139ec7c4 RR	158
	159	while (len > 0) {
	160	unsigned int noplen = len;
	161	if (noplen > ASM_NOP_MAX)
	162	noplen = ASM_NOP_MAX;
ab144f5e	163	memcpy(insns, noptable[noplen], noplen);
139ec7c4 RR	164	insns += noplen;
	165	len -= noplen;
	166	}
	167	}
	168
d167a518	169	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
d167a518 GH	170	extern u8 __smp_locks[], __smp_locks_end[];
d167a518 GH	171
9a0b5817 GH	172	/* Replace instructions with better alternatives for this CPU type.
	173	This runs before SMP is initialized to avoid SMP problems with
	174	self modifying code. This implies that assymetric systems where
	175	APs have less capabilities than the boot processor are not handled.
	176	Tough. Make sure you disable such features by hand. */
	177
	178	void apply_alternatives(struct alt_instr start, struct alt_instr end)
	179	{
9a0b5817	180	struct alt_instr *a;
ab144f5e	181	char insnbuf[MAX_PATCH_LEN];
9a0b5817 GH	182
	183	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	184	for (a = start; a < end; a++) {
ab144f5e	185	u8 *instr = a->instr;
9a0b5817	186	BUG_ON(a->replacementlen > a->instrlen);
ab144f5e	187	BUG_ON(a->instrlen > sizeof(insnbuf));
9a0b5817 GH	188	if (!boot_cpu_has(a->cpuid))
9a0b5817 GH	189	continue;
d167a518 GH	190	#ifdef CONFIG_X86_64
	191	/* vsyscall code is not mapped yet. resolve it manually. */
	192	if (instr >= (u8 )VSYSCALL_START && instr < (u8)VSYSCALL_END) {
	193	instr = __va(instr - (u8)VSYSCALL_START + (u8)__pa_symbol(&__vsyscall_0));
	194	DPRINTK("%s: vsyscall fixup: %p => %p\n",
	195	__FUNCTION__, a->instr, instr);
	196	}
	197	#endif
ab144f5e AK	198	memcpy(insnbuf, a->replacement, a->replacementlen);
	199	add_nops(insnbuf + a->replacementlen,
	200	a->instrlen - a->replacementlen);
	201	text_poke(instr, insnbuf, a->instrlen);
9a0b5817 GH	202	}
	203	}
	204
8ec4d41f GH	205	#ifdef CONFIG_SMP
8ec4d41f GH	206
9a0b5817 GH	207	static void alternatives_smp_lock(u8 start, u8 end, u8 text, u8 text_end)
	208	{
	209	u8 **ptr;
	210
	211	for (ptr = start; ptr < end; ptr++) {
	212	if (*ptr < text)
	213	continue;
	214	if (*ptr > text_end)
	215	continue;
19d36ccd	216	text_poke(ptr, ((unsigned char []){0xf0}), 1); / add lock prefix */
9a0b5817 GH	217	};
	218	}
	219
	220	static void alternatives_smp_unlock(u8 start, u8 end, u8 text, u8 text_end)
	221	{
9a0b5817	222	u8 **ptr;
ab144f5e	223	char insn[1];
9a0b5817	224
b7fb4af0 JF	225	if (noreplace_smp)
	226	return;
	227
ab144f5e	228	add_nops(insn, 1);
9a0b5817 GH	229	for (ptr = start; ptr < end; ptr++) {
	230	if (*ptr < text)
	231	continue;
	232	if (*ptr > text_end)
	233	continue;
ab144f5e	234	text_poke(*ptr, insn, 1);
9a0b5817 GH	235	};
	236	}
	237
	238	struct smp_alt_module {
	239	/* what is this ??? */
	240	struct module *mod;
	241	char *name;
	242
	243	/* ptrs to lock prefixes */
	244	u8 **locks;
	245	u8 **locks_end;
	246
	247	/* .text segment, needed to avoid patching init code ;) */
	248	u8 *text;
	249	u8 *text_end;
	250
	251	struct list_head next;
	252	};
	253	static LIST_HEAD(smp_alt_modules);
	254	static DEFINE_SPINLOCK(smp_alt);
	255
9a0b5817 GH	256	void alternatives_smp_module_add(struct module mod, char name,
	257	void locks, void locks_end,
	258	void text, void text_end)
	259	{
	260	struct smp_alt_module *smp;
	261	unsigned long flags;
	262
b7fb4af0 JF	263	if (noreplace_smp)
	264	return;
	265
9a0b5817 GH	266	if (smp_alt_once) {
	267	if (boot_cpu_has(X86_FEATURE_UP))
	268	alternatives_smp_unlock(locks, locks_end,
	269	text, text_end);
	270	return;
	271	}
	272
	273	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	274	if (NULL == smp)
	275	return; /* we'll run the (safe but slow) SMP code then ... */
	276
	277	smp->mod = mod;
	278	smp->name = name;
	279	smp->locks = locks;
	280	smp->locks_end = locks_end;
	281	smp->text = text;
	282	smp->text_end = text_end;
	283	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
	284	__FUNCTION__, smp->locks, smp->locks_end,
	285	smp->text, smp->text_end, smp->name);
	286
	287	spin_lock_irqsave(&smp_alt, flags);
	288	list_add_tail(&smp->next, &smp_alt_modules);
	289	if (boot_cpu_has(X86_FEATURE_UP))
	290	alternatives_smp_unlock(smp->locks, smp->locks_end,
	291	smp->text, smp->text_end);
	292	spin_unlock_irqrestore(&smp_alt, flags);
	293	}
	294
	295	void alternatives_smp_module_del(struct module *mod)
	296	{
	297	struct smp_alt_module *item;
	298	unsigned long flags;
	299
b7fb4af0	300	if (smp_alt_once \|\| noreplace_smp)
9a0b5817 GH	301	return;
	302
	303	spin_lock_irqsave(&smp_alt, flags);
	304	list_for_each_entry(item, &smp_alt_modules, next) {
	305	if (mod != item->mod)
	306	continue;
	307	list_del(&item->next);
	308	spin_unlock_irqrestore(&smp_alt, flags);
	309	DPRINTK("%s: %s\n", __FUNCTION__, item->name);
	310	kfree(item);
	311	return;
	312	}
	313	spin_unlock_irqrestore(&smp_alt, flags);
	314	}
	315
	316	void alternatives_smp_switch(int smp)
	317	{
	318	struct smp_alt_module *mod;
	319	unsigned long flags;
	320
3047e99e IM	321	#ifdef CONFIG_LOCKDEP
	322	/*
	323	* A not yet fixed binutils section handling bug prevents
	324	* alternatives-replacement from working reliably, so turn
	325	* it off:
	326	*/
	327	printk("lockdep: not fixing up alternatives.\n");
	328	return;
	329	#endif
	330
b7fb4af0	331	if (noreplace_smp \|\| smp_alt_once)
9a0b5817 GH	332	return;
	333	BUG_ON(!smp && (num_online_cpus() > 1));
	334
	335	spin_lock_irqsave(&smp_alt, flags);
	336	if (smp) {
	337	printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
	338	clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	339	clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
9a0b5817 GH	340	list_for_each_entry(mod, &smp_alt_modules, next)
	341	alternatives_smp_lock(mod->locks, mod->locks_end,
	342	mod->text, mod->text_end);
	343	} else {
	344	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	345	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	346	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
9a0b5817 GH	347	list_for_each_entry(mod, &smp_alt_modules, next)
	348	alternatives_smp_unlock(mod->locks, mod->locks_end,
	349	mod->text, mod->text_end);
	350	}
	351	spin_unlock_irqrestore(&smp_alt, flags);
	352	}
	353
8ec4d41f GH	354	#endif
8ec4d41f GH	355
139ec7c4	356	#ifdef CONFIG_PARAVIRT
98de032b JF	357	void apply_paravirt(struct paravirt_patch_site *start,
98de032b JF	358	struct paravirt_patch_site *end)
139ec7c4	359	{
98de032b	360	struct paravirt_patch_site *p;
ab144f5e	361	char insnbuf[MAX_PATCH_LEN];
139ec7c4	362
959b4fdf JF	363	if (noreplace_paravirt)
	364	return;
	365
139ec7c4 RR	366	for (p = start; p < end; p++) {
	367	unsigned int used;
	368
ab144f5e	369	BUG_ON(p->len > MAX_PATCH_LEN);
d34fda4a CW	370	/* prep the buffer with the original instructions */
d34fda4a CW	371	memcpy(insnbuf, p->instr, p->len);
ab144f5e AK	372	used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf,
ab144f5e AK	373	(unsigned long)p->instr, p->len);
7f63c41c	374
63f70270 JF	375	BUG_ON(used > p->len);
63f70270 JF	376
139ec7c4	377	/* Pad the rest with nops */
ab144f5e AK	378	add_nops(insnbuf + used, p->len - used);
ab144f5e AK	379	text_poke(p->instr, insnbuf, p->len);
139ec7c4	380	}
139ec7c4	381	}
98de032b	382	extern struct paravirt_patch_site __start_parainstructions[],
139ec7c4 RR	383	__stop_parainstructions[];
	384	#endif /* CONFIG_PARAVIRT */
	385
9a0b5817 GH	386	void __init alternative_instructions(void)
9a0b5817 GH	387	{
e51959fa	388	unsigned long flags;
e51959fa	389
8f4e956b AK	390	/* The patching is not fully atomic, so try to avoid local interruptions
	391	that might execute the to be patched code.
	392	Other CPUs are not running. */
	393	stop_nmi();
d2d0251f	394	#ifdef CONFIG_X86_MCE
8f4e956b AK	395	stop_mce();
	396	#endif
	397
e51959fa	398	local_irq_save(flags);
9a0b5817 GH	399	apply_alternatives(__alt_instructions, __alt_instructions_end);
	400
	401	/* switch to patch-once-at-boottime-only mode and free the
	402	* tables in case we know the number of CPUs will never ever
	403	* change */
	404	#ifdef CONFIG_HOTPLUG_CPU
	405	if (num_possible_cpus() < 2)
	406	smp_alt_once = 1;
9a0b5817 GH	407	#endif
9a0b5817 GH	408
8ec4d41f	409	#ifdef CONFIG_SMP
9a0b5817 GH	410	if (smp_alt_once) {
	411	if (1 == num_possible_cpus()) {
	412	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	413	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	414	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
9a0b5817 GH	415	alternatives_smp_unlock(__smp_locks, __smp_locks_end,
	416	_text, _etext);
	417	}
9a0b5817	418	} else {
9a0b5817 GH	419	alternatives_smp_module_add(NULL, "core kernel",
	420	__smp_locks, __smp_locks_end,
	421	_text, _etext);
	422	alternatives_smp_switch(0);
	423	}
8ec4d41f	424	#endif
441d40dc	425	apply_paravirt(__parainstructions, __parainstructions_end);
e51959fa	426	local_irq_restore(flags);
8f4e956b	427
f68fd5f4 FW	428	if (smp_alt_once)
	429	free_init_pages("SMP alternatives",
	430	(unsigned long)__smp_locks,
	431	(unsigned long)__smp_locks_end);
	432
8f4e956b	433	restart_nmi();
d2d0251f	434	#ifdef CONFIG_X86_MCE
8f4e956b AK	435	restart_mce();
8f4e956b AK	436	#endif
9a0b5817	437	}
19d36ccd AK	438
	439	/*
	440	* Warning:
	441	* When you use this code to patch more than one byte of an instruction
	442	* you need to make sure that other CPUs cannot execute this code in parallel.
	443	* Also no thread must be currently preempted in the middle of these instructions.
	444	* And on the local CPU you need to be protected again NMI or MCE handlers
	445	* seeing an inconsistent instruction while you patch.
	446	*/
602033ed	447	void __kprobes text_poke(void addr, unsigned char opcode, int len)
19d36ccd	448	{
19d36ccd AK	449	memcpy(addr, opcode, len);
19d36ccd AK	450	sync_core();
a534b679 AK	451	/* Could also do a CLFLUSH here to speed up CPU recovery; but
a534b679 AK	452	that causes hangs on some VIA CPUs. */
19d36ccd	453	}