[linux-2.6-block.git] / arch / i386 / kernel / alternative.c

#include <linux/module.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <asm/alternative.h>
#include <asm/sections.h>

static int no_replacement    = 0;
static int smp_alt_once      = 0;
static int debug_alternative = 0;

static int __init noreplacement_setup(char *s)
{
	no_replacement = 1;
	return 1;
}
static int __init bootonly(char *str)
{
	smp_alt_once = 1;
	return 1;
}
static int __init debug_alt(char *str)
{
	debug_alternative = 1;
	return 1;
}

__setup("noreplacement", noreplacement_setup);
__setup("smp-alt-boot", bootonly);
__setup("debug-alternative", debug_alt);

#define DPRINTK(fmt, args...) if (debug_alternative) \
	printk(KERN_DEBUG fmt, args)

#ifdef GENERIC_NOP1
/* Use inline assembly to define this because the nops are defined
   as inline assembly strings in the include files and we cannot
   get them easily into strings. */
asm("\t.data\nintelnops: "
	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	GENERIC_NOP7 GENERIC_NOP8);
extern unsigned char intelnops[];
static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	NULL,
	intelnops,
	intelnops + 1,
	intelnops + 1 + 2,
	intelnops + 1 + 2 + 3,
	intelnops + 1 + 2 + 3 + 4,
	intelnops + 1 + 2 + 3 + 4 + 5,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K8_NOP1
asm("\t.data\nk8nops: "
	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	K8_NOP7 K8_NOP8);
extern unsigned char k8nops[];
static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	NULL,
	k8nops,
	k8nops + 1,
	k8nops + 1 + 2,
	k8nops + 1 + 2 + 3,
	k8nops + 1 + 2 + 3 + 4,
	k8nops + 1 + 2 + 3 + 4 + 5,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef K7_NOP1
asm("\t.data\nk7nops: "
	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	K7_NOP7 K7_NOP8);
extern unsigned char k7nops[];
static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	NULL,
	k7nops,
	k7nops + 1,
	k7nops + 1 + 2,
	k7nops + 1 + 2 + 3,
	k7nops + 1 + 2 + 3 + 4,
	k7nops + 1 + 2 + 3 + 4 + 5,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
#endif

#ifdef CONFIG_X86_64

extern char __vsyscall_0;
static inline unsigned char** find_nop_table(void)
{
	return k8_nops;
}

#else /* CONFIG_X86_64 */

static struct nop {
	int cpuid;
	unsigned char **noptable;
} noptypes[] = {
	{ X86_FEATURE_K8, k8_nops },
	{ X86_FEATURE_K7, k7_nops },
	{ -1, NULL }
};

static unsigned char** find_nop_table(void)
{
	unsigned char **noptable = intel_nops;
	int i;

	for (i = 0; noptypes[i].cpuid >= 0; i++) {
		if (boot_cpu_has(noptypes[i].cpuid)) {
			noptable = noptypes[i].noptable;
			break;
		}
	}
	return noptable;
}

#endif /* CONFIG_X86_64 */

extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
extern u8 *__smp_locks[], *__smp_locks_end[];

extern u8 __smp_alt_begin[], __smp_alt_end[];

/* Replace instructions with better alternatives for this CPU type.
   This runs before SMP is initialized to avoid SMP problems with
   self modifying code. This implies that assymetric systems where
   APs have less capabilities than the boot processor are not handled.
   Tough. Make sure you disable such features by hand. */

void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
{
	unsigned char **noptable = find_nop_table();
	struct alt_instr *a;
	u8 *instr;
	int diff, i, k;

	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {
		BUG_ON(a->replacementlen > a->instrlen);
		if (!boot_cpu_has(a->cpuid))
			continue;
		instr = a->instr;
#ifdef CONFIG_X86_64
		/* vsyscall code is not mapped yet. resolve it manually. */
		if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
			instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
			DPRINTK("%s: vsyscall fixup: %p => %p\n",
				__FUNCTION__, a->instr, instr);
		}
#endif
		memcpy(instr, a->replacement, a->replacementlen);
		diff = a->instrlen - a->replacementlen;
		/* Pad the rest with nops */
		for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
			k = diff;
			if (k > ASM_NOP_MAX)
				k = ASM_NOP_MAX;
			memcpy(a->instr + i, noptable[k], k);
		}
	}
}

#ifdef CONFIG_SMP

static void alternatives_smp_save(struct alt_instr *start, struct alt_instr *end)
{
	struct alt_instr *a;

	DPRINTK("%s: alt table %p-%p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {
		memcpy(a->replacement + a->replacementlen,
		       a->instr,
		       a->instrlen);
	}
}

static void alternatives_smp_apply(struct alt_instr *start, struct alt_instr *end)
{
	struct alt_instr *a;

	for (a = start; a < end; a++) {
		memcpy(a->instr,
		       a->replacement + a->replacementlen,
		       a->instrlen);
	}
}

static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		**ptr = 0xf0; /* lock prefix */
	};
}

static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	unsigned char **noptable = find_nop_table();
	u8 **ptr;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		**ptr = noptable[1][0];
	};
}

struct smp_alt_module {
	/* what is this ??? */
	struct module	*mod;
	char		*name;

	/* ptrs to lock prefixes */
	u8		**locks;
	u8		**locks_end;

	/* .text segment, needed to avoid patching init code ;) */
	u8		*text;
	u8		*text_end;

	struct list_head next;
};
static LIST_HEAD(smp_alt_modules);
static DEFINE_SPINLOCK(smp_alt);

void alternatives_smp_module_add(struct module *mod, char *name,
				 void *locks, void *locks_end,
				 void *text,  void *text_end)
{
	struct smp_alt_module *smp;
	unsigned long flags;

	if (no_replacement)
		return;

	if (smp_alt_once) {
		if (boot_cpu_has(X86_FEATURE_UP))
			alternatives_smp_unlock(locks, locks_end,
						text, text_end);
		return;
	}

	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	if (NULL == smp)
		return; /* we'll run the (safe but slow) SMP code then ... */

	smp->mod	= mod;
	smp->name	= name;
	smp->locks	= locks;
	smp->locks_end	= locks_end;
	smp->text	= text;
	smp->text_end	= text_end;
	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
		__FUNCTION__, smp->locks, smp->locks_end,
		smp->text, smp->text_end, smp->name);

	spin_lock_irqsave(&smp_alt, flags);
	list_add_tail(&smp->next, &smp_alt_modules);
	if (boot_cpu_has(X86_FEATURE_UP))
		alternatives_smp_unlock(smp->locks, smp->locks_end,
					smp->text, smp->text_end);
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_module_del(struct module *mod)
{
	struct smp_alt_module *item;
	unsigned long flags;

	if (no_replacement || smp_alt_once)
		return;

	spin_lock_irqsave(&smp_alt, flags);
	list_for_each_entry(item, &smp_alt_modules, next) {
		if (mod != item->mod)
			continue;
		list_del(&item->next);
		spin_unlock_irqrestore(&smp_alt, flags);
		DPRINTK("%s: %s\n", __FUNCTION__, item->name);
		kfree(item);
		return;
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_switch(int smp)
{
	struct smp_alt_module *mod;
	unsigned long flags;

#ifdef CONFIG_LOCKDEP
	/*
	 * A not yet fixed binutils section handling bug prevents
	 * alternatives-replacement from working reliably, so turn
	 * it off:
	 */
	printk("lockdep: not fixing up alternatives.\n");
	return;
#endif

	if (no_replacement || smp_alt_once)
		return;
	BUG_ON(!smp && (num_online_cpus() > 1));

	spin_lock_irqsave(&smp_alt, flags);
	if (smp) {
		printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
		clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		alternatives_smp_apply(__smp_alt_instructions,
				       __smp_alt_instructions_end);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_lock(mod->locks, mod->locks_end,
					      mod->text, mod->text_end);
	} else {
		printk(KERN_INFO "SMP alternatives: switching to UP code\n");
		set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		apply_alternatives(__smp_alt_instructions,
				   __smp_alt_instructions_end);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_unlock(mod->locks, mod->locks_end,
						mod->text, mod->text_end);
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

#endif

void __init alternative_instructions(void)
{
	unsigned long flags;
	if (no_replacement) {
		printk(KERN_INFO "(SMP-)alternatives turned off\n");
		free_init_pages("SMP alternatives",
				(unsigned long)__smp_alt_begin,
				(unsigned long)__smp_alt_end);
		return;
	}

	local_irq_save(flags);
	apply_alternatives(__alt_instructions, __alt_instructions_end);

	/* switch to patch-once-at-boottime-only mode and free the
	 * tables in case we know the number of CPUs will never ever
	 * change */
#ifdef CONFIG_HOTPLUG_CPU
	if (num_possible_cpus() < 2)
		smp_alt_once = 1;
#else
	smp_alt_once = 1;
#endif

#ifdef CONFIG_SMP
	if (smp_alt_once) {
		if (1 == num_possible_cpus()) {
			printk(KERN_INFO "SMP alternatives: switching to UP code\n");
			set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
			set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
			apply_alternatives(__smp_alt_instructions,
					   __smp_alt_instructions_end);
			alternatives_smp_unlock(__smp_locks, __smp_locks_end,
						_text, _etext);
		}
		free_init_pages("SMP alternatives",
				(unsigned long)__smp_alt_begin,
				(unsigned long)__smp_alt_end);
	} else {
		alternatives_smp_save(__smp_alt_instructions,
				      __smp_alt_instructions_end);
		alternatives_smp_module_add(NULL, "core kernel",
					    __smp_locks, __smp_locks_end,
					    _text, _etext);
		alternatives_smp_switch(0);
	}
#endif
	local_irq_restore(flags);
}
Commit	Line	Data
9a0b5817	1	#include <linux/module.h>
f6a57033	2	#include <linux/sched.h>
9a0b5817 GH	3	#include <linux/spinlock.h>
	4	#include <linux/list.h>
	5	#include <asm/alternative.h>
	6	#include <asm/sections.h>
	7
d167a518 GH	8	static int no_replacement = 0;
	9	static int smp_alt_once = 0;
	10	static int debug_alternative = 0;
9a0b5817	11
d167a518 GH	12	static int __init noreplacement_setup(char *s)
	13	{
	14	no_replacement = 1;
	15	return 1;
	16	}
	17	static int __init bootonly(char *str)
	18	{
	19	smp_alt_once = 1;
	20	return 1;
	21	}
	22	static int __init debug_alt(char *str)
	23	{
	24	debug_alternative = 1;
	25	return 1;
	26	}
	27
	28	__setup("noreplacement", noreplacement_setup);
	29	__setup("smp-alt-boot", bootonly);
	30	__setup("debug-alternative", debug_alt);
	31
	32	#define DPRINTK(fmt, args...) if (debug_alternative) \
	33	printk(KERN_DEBUG fmt, args)
	34
	35	#ifdef GENERIC_NOP1
9a0b5817 GH	36	/* Use inline assembly to define this because the nops are defined
	37	as inline assembly strings in the include files and we cannot
	38	get them easily into strings. */
	39	asm("\t.data\nintelnops: "
	40	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	41	GENERIC_NOP7 GENERIC_NOP8);
d167a518	42	extern unsigned char intelnops[];
9a0b5817 GH	43	static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	44	NULL,
	45	intelnops,
	46	intelnops + 1,
	47	intelnops + 1 + 2,
	48	intelnops + 1 + 2 + 3,
	49	intelnops + 1 + 2 + 3 + 4,
	50	intelnops + 1 + 2 + 3 + 4 + 5,
	51	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	52	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	53	};
d167a518 GH	54	#endif
	55
	56	#ifdef K8_NOP1
	57	asm("\t.data\nk8nops: "
	58	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	59	K8_NOP7 K8_NOP8);
	60	extern unsigned char k8nops[];
9a0b5817 GH	61	static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	62	NULL,
	63	k8nops,
	64	k8nops + 1,
	65	k8nops + 1 + 2,
	66	k8nops + 1 + 2 + 3,
	67	k8nops + 1 + 2 + 3 + 4,
	68	k8nops + 1 + 2 + 3 + 4 + 5,
	69	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	70	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	71	};
d167a518 GH	72	#endif
	73
	74	#ifdef K7_NOP1
	75	asm("\t.data\nk7nops: "
	76	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	77	K7_NOP7 K7_NOP8);
	78	extern unsigned char k7nops[];
9a0b5817 GH	79	static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	80	NULL,
	81	k7nops,
	82	k7nops + 1,
	83	k7nops + 1 + 2,
	84	k7nops + 1 + 2 + 3,
	85	k7nops + 1 + 2 + 3 + 4,
	86	k7nops + 1 + 2 + 3 + 4 + 5,
	87	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	88	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	89	};
d167a518 GH	90	#endif
	91
	92	#ifdef CONFIG_X86_64
	93
	94	extern char __vsyscall_0;
	95	static inline unsigned char** find_nop_table(void)
	96	{
	97	return k8_nops;
	98	}
	99
	100	#else /* CONFIG_X86_64 */
	101
9a0b5817 GH	102	static struct nop {
	103	int cpuid;
	104	unsigned char **noptable;
	105	} noptypes[] = {
	106	{ X86_FEATURE_K8, k8_nops },
	107	{ X86_FEATURE_K7, k7_nops },
	108	{ -1, NULL }
	109	};
	110
9a0b5817 GH	111	static unsigned char** find_nop_table(void)
	112	{
	113	unsigned char **noptable = intel_nops;
	114	int i;
	115
	116	for (i = 0; noptypes[i].cpuid >= 0; i++) {
	117	if (boot_cpu_has(noptypes[i].cpuid)) {
	118	noptable = noptypes[i].noptable;
	119	break;
	120	}
	121	}
	122	return noptable;
	123	}
	124
d167a518 GH	125	#endif /* CONFIG_X86_64 */
	126
	127	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
	128	extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
	129	extern u8 __smp_locks[], __smp_locks_end[];
	130
	131	extern u8 __smp_alt_begin[], __smp_alt_end[];
	132
9a0b5817 GH	133	/* Replace instructions with better alternatives for this CPU type.
	134	This runs before SMP is initialized to avoid SMP problems with
	135	self modifying code. This implies that assymetric systems where
	136	APs have less capabilities than the boot processor are not handled.
	137	Tough. Make sure you disable such features by hand. */
	138
	139	void apply_alternatives(struct alt_instr start, struct alt_instr end)
	140	{
	141	unsigned char **noptable = find_nop_table();
	142	struct alt_instr *a;
d167a518	143	u8 *instr;
9a0b5817 GH	144	int diff, i, k;
	145
	146	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	147	for (a = start; a < end; a++) {
	148	BUG_ON(a->replacementlen > a->instrlen);
	149	if (!boot_cpu_has(a->cpuid))
	150	continue;
d167a518 GH	151	instr = a->instr;
	152	#ifdef CONFIG_X86_64
	153	/* vsyscall code is not mapped yet. resolve it manually. */
	154	if (instr >= (u8 )VSYSCALL_START && instr < (u8)VSYSCALL_END) {
	155	instr = __va(instr - (u8)VSYSCALL_START + (u8)__pa_symbol(&__vsyscall_0));
	156	DPRINTK("%s: vsyscall fixup: %p => %p\n",
	157	__FUNCTION__, a->instr, instr);
	158	}
	159	#endif
	160	memcpy(instr, a->replacement, a->replacementlen);
9a0b5817 GH	161	diff = a->instrlen - a->replacementlen;
	162	/* Pad the rest with nops */
	163	for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
	164	k = diff;
	165	if (k > ASM_NOP_MAX)
	166	k = ASM_NOP_MAX;
	167	memcpy(a->instr + i, noptable[k], k);
	168	}
	169	}
	170	}
	171
8ec4d41f GH	172	#ifdef CONFIG_SMP
8ec4d41f GH	173
9a0b5817 GH	174	static void alternatives_smp_save(struct alt_instr start, struct alt_instr end)
	175	{
	176	struct alt_instr *a;
	177
	178	DPRINTK("%s: alt table %p-%p\n", __FUNCTION__, start, end);
	179	for (a = start; a < end; a++) {
	180	memcpy(a->replacement + a->replacementlen,
	181	a->instr,
	182	a->instrlen);
	183	}
	184	}
	185
	186	static void alternatives_smp_apply(struct alt_instr start, struct alt_instr end)
	187	{
	188	struct alt_instr *a;
	189
	190	for (a = start; a < end; a++) {
	191	memcpy(a->instr,
	192	a->replacement + a->replacementlen,
	193	a->instrlen);
	194	}
	195	}
	196
	197	static void alternatives_smp_lock(u8 start, u8 end, u8 text, u8 text_end)
	198	{
	199	u8 **ptr;
	200
	201	for (ptr = start; ptr < end; ptr++) {
	202	if (*ptr < text)
	203	continue;
	204	if (*ptr > text_end)
	205	continue;
	206	*ptr = 0xf0; / lock prefix */
	207	};
	208	}
	209
	210	static void alternatives_smp_unlock(u8 start, u8 end, u8 text, u8 text_end)
	211	{
	212	unsigned char **noptable = find_nop_table();
	213	u8 **ptr;
	214
	215	for (ptr = start; ptr < end; ptr++) {
	216	if (*ptr < text)
	217	continue;
	218	if (*ptr > text_end)
	219	continue;
	220	**ptr = noptable[1][0];
	221	};
	222	}
	223
	224	struct smp_alt_module {
	225	/* what is this ??? */
	226	struct module *mod;
	227	char *name;
	228
	229	/* ptrs to lock prefixes */
	230	u8 **locks;
	231	u8 **locks_end;
	232
	233	/* .text segment, needed to avoid patching init code ;) */
	234	u8 *text;
	235	u8 *text_end;
	236
	237	struct list_head next;
238	};
239	static LIST_HEAD(smp_alt_modules);
240	static DEFINE_SPINLOCK(smp_alt);
241
9a0b5817 GH	242	void alternatives_smp_module_add(struct module mod, char name,
	243	void locks, void locks_end,
	244	void text, void text_end)
	245	{
	246	struct smp_alt_module *smp;
	247	unsigned long flags;
	248
d167a518 GH	249	if (no_replacement)
	250	return;
	251
9a0b5817 GH	252	if (smp_alt_once) {
	253	if (boot_cpu_has(X86_FEATURE_UP))
	254	alternatives_smp_unlock(locks, locks_end,
	255	text, text_end);
	256	return;
	257	}
	258
	259	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	260	if (NULL == smp)
	261	return; /* we'll run the (safe but slow) SMP code then ... */
	262
	263	smp->mod = mod;
	264	smp->name = name;
	265	smp->locks = locks;
	266	smp->locks_end = locks_end;
	267	smp->text = text;
	268	smp->text_end = text_end;
	269	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
	270	__FUNCTION__, smp->locks, smp->locks_end,
	271	smp->text, smp->text_end, smp->name);
	272
	273	spin_lock_irqsave(&smp_alt, flags);
	274	list_add_tail(&smp->next, &smp_alt_modules);
	275	if (boot_cpu_has(X86_FEATURE_UP))
	276	alternatives_smp_unlock(smp->locks, smp->locks_end,
	277	smp->text, smp->text_end);
	278	spin_unlock_irqrestore(&smp_alt, flags);
	279	}
	280
	281	void alternatives_smp_module_del(struct module *mod)
	282	{
	283	struct smp_alt_module *item;
	284	unsigned long flags;
	285
d167a518	286	if (no_replacement \|\| smp_alt_once)
9a0b5817 GH	287	return;
	288
	289	spin_lock_irqsave(&smp_alt, flags);
	290	list_for_each_entry(item, &smp_alt_modules, next) {
	291	if (mod != item->mod)
	292	continue;
	293	list_del(&item->next);
	294	spin_unlock_irqrestore(&smp_alt, flags);
	295	DPRINTK("%s: %s\n", __FUNCTION__, item->name);
	296	kfree(item);
	297	return;
	298	}
	299	spin_unlock_irqrestore(&smp_alt, flags);
	300	}
	301
	302	void alternatives_smp_switch(int smp)
	303	{
	304	struct smp_alt_module *mod;
	305	unsigned long flags;
	306
3047e99e IM	307	#ifdef CONFIG_LOCKDEP
	308	/*
	309	* A not yet fixed binutils section handling bug prevents
	310	* alternatives-replacement from working reliably, so turn
	311	* it off:
	312	*/
	313	printk("lockdep: not fixing up alternatives.\n");
	314	return;
	315	#endif
	316
d167a518	317	if (no_replacement \|\| smp_alt_once)
9a0b5817 GH	318	return;
	319	BUG_ON(!smp && (num_online_cpus() > 1));
	320
	321	spin_lock_irqsave(&smp_alt, flags);
	322	if (smp) {
	323	printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
	324	clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	325	clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
	326	alternatives_smp_apply(__smp_alt_instructions,
	327	__smp_alt_instructions_end);
	328	list_for_each_entry(mod, &smp_alt_modules, next)
	329	alternatives_smp_lock(mod->locks, mod->locks_end,
	330	mod->text, mod->text_end);
	331	} else {
	332	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	333	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	334	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
	335	apply_alternatives(__smp_alt_instructions,
	336	__smp_alt_instructions_end);
	337	list_for_each_entry(mod, &smp_alt_modules, next)
	338	alternatives_smp_unlock(mod->locks, mod->locks_end,
	339	mod->text, mod->text_end);
	340	}
	341	spin_unlock_irqrestore(&smp_alt, flags);
	342	}
	343
8ec4d41f GH	344	#endif
8ec4d41f GH	345
9a0b5817 GH	346	void __init alternative_instructions(void)
9a0b5817 GH	347	{
e51959fa	348	unsigned long flags;
d167a518 GH	349	if (no_replacement) {
	350	printk(KERN_INFO "(SMP-)alternatives turned off\n");
	351	free_init_pages("SMP alternatives",
	352	(unsigned long)__smp_alt_begin,
	353	(unsigned long)__smp_alt_end);
	354	return;
	355	}
e51959fa ZA	356
e51959fa ZA	357	local_irq_save(flags);
9a0b5817 GH	358	apply_alternatives(__alt_instructions, __alt_instructions_end);
	359
	360	/* switch to patch-once-at-boottime-only mode and free the
	361	* tables in case we know the number of CPUs will never ever
	362	* change */
	363	#ifdef CONFIG_HOTPLUG_CPU
	364	if (num_possible_cpus() < 2)
	365	smp_alt_once = 1;
	366	#else
	367	smp_alt_once = 1;
	368	#endif
	369
8ec4d41f	370	#ifdef CONFIG_SMP
9a0b5817 GH	371	if (smp_alt_once) {
	372	if (1 == num_possible_cpus()) {
	373	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
	374	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
	375	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
	376	apply_alternatives(__smp_alt_instructions,
	377	__smp_alt_instructions_end);
	378	alternatives_smp_unlock(__smp_locks, __smp_locks_end,
	379	_text, _etext);
	380	}
	381	free_init_pages("SMP alternatives",
	382	(unsigned long)__smp_alt_begin,
	383	(unsigned long)__smp_alt_end);
	384	} else {
	385	alternatives_smp_save(__smp_alt_instructions,
	386	__smp_alt_instructions_end);
	387	alternatives_smp_module_add(NULL, "core kernel",
	388	__smp_locks, __smp_locks_end,
	389	_text, _etext);
	390	alternatives_smp_switch(0);
	391	}
8ec4d41f	392	#endif
e51959fa	393	local_irq_restore(flags);
9a0b5817	394	}