[linux-2.6-block.git] / arch / x86 / kernel / tlb_32.c

#include <linux/spinlock.h>
#include <linux/cpu.h>
#include <linux/interrupt.h>

#include <asm/tlbflush.h>

DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate)
			____cacheline_aligned = { &init_mm, 0, };

/* must come after the send_IPI functions above for inlining */
#include <mach_ipi.h>

/*
 *	Smarter SMP flushing macros.
 *		c/o Linus Torvalds.
 *
 *	These mean you can really definitely utterly forget about
 *	writing to user space from interrupts. (Its not allowed anyway).
 *
 *	Optimizations Manfred Spraul <manfred@colorfullife.com>
 */

static cpumask_t flush_cpumask;
static struct mm_struct *flush_mm;
static unsigned long flush_va;
static DEFINE_SPINLOCK(tlbstate_lock);

/*
 * We cannot call mmdrop() because we are in interrupt context,
 * instead update mm->cpu_vm_mask.
 *
 * We need to reload %cr3 since the page tables may be going
 * away from under us..
 */
void leave_mm(int cpu)
{
	BUG_ON(x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK);
	cpu_clear(cpu, x86_read_percpu(cpu_tlbstate.active_mm)->cpu_vm_mask);
	load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);

/*
 *
 * The flush IPI assumes that a thread switch happens in this order:
 * [cpu0: the cpu that switches]
 * 1) switch_mm() either 1a) or 1b)
 * 1a) thread switch to a different mm
 * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
 * 	Stop ipi delivery for the old mm. This is not synchronized with
 * 	the other cpus, but smp_invalidate_interrupt ignore flush ipis
 * 	for the wrong mm, and in the worst case we perform a superfluous
 * 	tlb flush.
 * 1a2) set cpu_tlbstate to TLBSTATE_OK
 * 	Now the smp_invalidate_interrupt won't call leave_mm if cpu0
 *	was in lazy tlb mode.
 * 1a3) update cpu_tlbstate[].active_mm
 * 	Now cpu0 accepts tlb flushes for the new mm.
 * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
 * 	Now the other cpus will send tlb flush ipis.
 * 1a4) change cr3.
 * 1b) thread switch without mm change
 *	cpu_tlbstate[].active_mm is correct, cpu0 already handles
 *	flush ipis.
 * 1b1) set cpu_tlbstate to TLBSTATE_OK
 * 1b2) test_and_set the cpu bit in cpu_vm_mask.
 * 	Atomically set the bit [other cpus will start sending flush ipis],
 * 	and test the bit.
 * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
 * 2) switch %%esp, ie current
 *
 * The interrupt must handle 2 special cases:
 * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
 * - the cpu performs speculative tlb reads, i.e. even if the cpu only
 *   runs in kernel space, the cpu could load tlb entries for user space
 *   pages.
 *
 * The good news is that cpu_tlbstate is local to each cpu, no
 * write/read ordering problems.
 */

/*
 * TLB flush IPI:
 *
 * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
 * 2) Leave the mm if we are in the lazy tlb mode.
 */

void smp_invalidate_interrupt(struct pt_regs *regs)
{
	unsigned long cpu;

	cpu = get_cpu();

	if (!cpu_isset(cpu, flush_cpumask))
		goto out;
		/*
		 * This was a BUG() but until someone can quote me the
		 * line from the intel manual that guarantees an IPI to
		 * multiple CPUs is retried _only_ on the erroring CPUs
		 * its staying as a return
		 *
		 * BUG();
		 */

	if (flush_mm == x86_read_percpu(cpu_tlbstate.active_mm)) {
		if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK) {
			if (flush_va == TLB_FLUSH_ALL)
				local_flush_tlb();
			else
				__flush_tlb_one(flush_va);
		} else
			leave_mm(cpu);
	}
	ack_APIC_irq();
	smp_mb__before_clear_bit();
	cpu_clear(cpu, flush_cpumask);
	smp_mb__after_clear_bit();
out:
	put_cpu_no_resched();
	inc_irq_stat(irq_tlb_count);
}

void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
			     unsigned long va)
{
	cpumask_t cpumask = *cpumaskp;

	/*
	 * A couple of (to be removed) sanity checks:
	 *
	 * - current CPU must not be in mask
	 * - mask must exist :)
	 */
	BUG_ON(cpus_empty(cpumask));
	BUG_ON(cpu_isset(smp_processor_id(), cpumask));
	BUG_ON(!mm);

#ifdef CONFIG_HOTPLUG_CPU
	/* If a CPU which we ran on has gone down, OK. */
	cpus_and(cpumask, cpumask, cpu_online_map);
	if (unlikely(cpus_empty(cpumask)))
		return;
#endif

	/*
	 * i'm not happy about this global shared spinlock in the
	 * MM hot path, but we'll see how contended it is.
	 * AK: x86-64 has a faster method that could be ported.
	 */
	spin_lock(&tlbstate_lock);

	flush_mm = mm;
	flush_va = va;
	cpus_or(flush_cpumask, cpumask, flush_cpumask);

	/*
	 * Make the above memory operations globally visible before
	 * sending the IPI.
	 */
	smp_mb();
	/*
	 * We have to send the IPI only to
	 * CPUs affected.
	 */
	send_IPI_mask(&cpumask, INVALIDATE_TLB_VECTOR);

	while (!cpus_empty(flush_cpumask))
		/* nothing. lockup detection does not belong here */
		cpu_relax();

	flush_mm = NULL;
	flush_va = 0;
	spin_unlock(&tlbstate_lock);
}

void flush_tlb_current_task(void)
{
	struct mm_struct *mm = current->mm;
	cpumask_t cpu_mask;

	preempt_disable();
	cpu_mask = mm->cpu_vm_mask;
	cpu_clear(smp_processor_id(), cpu_mask);

	local_flush_tlb();
	if (!cpus_empty(cpu_mask))
		flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
	preempt_enable();
}

void flush_tlb_mm(struct mm_struct *mm)
{
	cpumask_t cpu_mask;

	preempt_disable();
	cpu_mask = mm->cpu_vm_mask;
	cpu_clear(smp_processor_id(), cpu_mask);

	if (current->active_mm == mm) {
		if (current->mm)
			local_flush_tlb();
		else
			leave_mm(smp_processor_id());
	}
	if (!cpus_empty(cpu_mask))
		flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);

	preempt_enable();
}

void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
{
	struct mm_struct *mm = vma->vm_mm;
	cpumask_t cpu_mask;

	preempt_disable();
	cpu_mask = mm->cpu_vm_mask;
	cpu_clear(smp_processor_id(), cpu_mask);

	if (current->active_mm == mm) {
		if (current->mm)
			__flush_tlb_one(va);
		 else
			leave_mm(smp_processor_id());
	}

	if (!cpus_empty(cpu_mask))
		flush_tlb_others(cpu_mask, mm, va);

	preempt_enable();
}
EXPORT_SYMBOL(flush_tlb_page);

static void do_flush_tlb_all(void *info)
{
	unsigned long cpu = smp_processor_id();

	__flush_tlb_all();
	if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_LAZY)
		leave_mm(cpu);
}

void flush_tlb_all(void)
{
	on_each_cpu(do_flush_tlb_all, NULL, 1);
}

void reset_lazy_tlbstate(void)
{
	int cpu = raw_smp_processor_id();

	per_cpu(cpu_tlbstate, cpu).state = 0;
	per_cpu(cpu_tlbstate, cpu).active_mm = &init_mm;
}
Commit	Line	Data
c048fdfe GC	1	#include <linux/spinlock.h>
	2	#include <linux/cpu.h>
	3	#include <linux/interrupt.h>
	4
	5	#include <asm/tlbflush.h>
	6
	7	DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate)
	8	____cacheline_aligned = { &init_mm, 0, };
	9
	10	/* must come after the send_IPI functions above for inlining */
	11	#include <mach_ipi.h>
	12
	13	/*
	14	* Smarter SMP flushing macros.
	15	* c/o Linus Torvalds.
	16	*
	17	* These mean you can really definitely utterly forget about
	18	* writing to user space from interrupts. (Its not allowed anyway).
	19	*
	20	* Optimizations Manfred Spraul <manfred@colorfullife.com>
	21	*/
	22
	23	static cpumask_t flush_cpumask;
	24	static struct mm_struct *flush_mm;
	25	static unsigned long flush_va;
	26	static DEFINE_SPINLOCK(tlbstate_lock);
	27
	28	/*
	29	* We cannot call mmdrop() because we are in interrupt context,
	30	* instead update mm->cpu_vm_mask.
	31	*
	32	* We need to reload %cr3 since the page tables may be going
	33	* away from under us..
	34	*/
	35	void leave_mm(int cpu)
	36	{
cfc31983 JB	37	BUG_ON(x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK);
cfc31983 JB	38	cpu_clear(cpu, x86_read_percpu(cpu_tlbstate.active_mm)->cpu_vm_mask);
c048fdfe GC	39	load_cr3(swapper_pg_dir);
	40	}
	41	EXPORT_SYMBOL_GPL(leave_mm);
	42
	43	/*
	44	*
	45	* The flush IPI assumes that a thread switch happens in this order:
	46	* [cpu0: the cpu that switches]
	47	* 1) switch_mm() either 1a) or 1b)
	48	* 1a) thread switch to a different mm
	49	* 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
	50	* Stop ipi delivery for the old mm. This is not synchronized with
	51	* the other cpus, but smp_invalidate_interrupt ignore flush ipis
	52	* for the wrong mm, and in the worst case we perform a superfluous
	53	* tlb flush.
	54	* 1a2) set cpu_tlbstate to TLBSTATE_OK
	55	* Now the smp_invalidate_interrupt won't call leave_mm if cpu0
	56	* was in lazy tlb mode.
	57	* 1a3) update cpu_tlbstate[].active_mm
	58	* Now cpu0 accepts tlb flushes for the new mm.
	59	* 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
	60	* Now the other cpus will send tlb flush ipis.
	61	* 1a4) change cr3.
	62	* 1b) thread switch without mm change
	63	* cpu_tlbstate[].active_mm is correct, cpu0 already handles
	64	* flush ipis.
	65	* 1b1) set cpu_tlbstate to TLBSTATE_OK
	66	* 1b2) test_and_set the cpu bit in cpu_vm_mask.
	67	* Atomically set the bit [other cpus will start sending flush ipis],
	68	* and test the bit.
	69	* 1b3) if the bit was 0: leave_mm was called, flush the tlb.
	70	* 2) switch %%esp, ie current
	71	*
	72	* The interrupt must handle 2 special cases:
	73	* - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
	74	* - the cpu performs speculative tlb reads, i.e. even if the cpu only
	75	* runs in kernel space, the cpu could load tlb entries for user space
	76	* pages.
	77	*
	78	* The good news is that cpu_tlbstate is local to each cpu, no
	79	* write/read ordering problems.
	80	*/
	81
	82	/*
	83	* TLB flush IPI:
	84	*
	85	* 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
	86	* 2) Leave the mm if we are in the lazy tlb mode.
	87	*/
	88
	89	void smp_invalidate_interrupt(struct pt_regs *regs)
	90	{
	91	unsigned long cpu;
	92
	93	cpu = get_cpu();
	94
	95	if (!cpu_isset(cpu, flush_cpumask))
	96	goto out;
	97	/*
	98	* This was a BUG() but until someone can quote me the
	99	* line from the intel manual that guarantees an IPI to
	100	* multiple CPUs is retried _only_ on the erroring CPUs
	101	* its staying as a return
	102	*
103	* BUG();
104	*/
105
cfc31983 JB	106	if (flush_mm == x86_read_percpu(cpu_tlbstate.active_mm)) {
cfc31983 JB	107	if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK) {
c048fdfe GC	108	if (flush_va == TLB_FLUSH_ALL)
	109	local_flush_tlb();
	110	else
	111	__flush_tlb_one(flush_va);
	112	} else
	113	leave_mm(cpu);
	114	}
	115	ack_APIC_irq();
	116	smp_mb__before_clear_bit();
	117	cpu_clear(cpu, flush_cpumask);
	118	smp_mb__after_clear_bit();
	119	out:
	120	put_cpu_no_resched();
8ae93669	121	inc_irq_stat(irq_tlb_count);
c048fdfe GC	122	}
	123
	124	void native_flush_tlb_others(const cpumask_t cpumaskp, struct mm_struct mm,
	125	unsigned long va)
	126	{
	127	cpumask_t cpumask = *cpumaskp;
	128
	129	/*
	130	* A couple of (to be removed) sanity checks:
	131	*
	132	* - current CPU must not be in mask
	133	* - mask must exist :)
	134	*/
	135	BUG_ON(cpus_empty(cpumask));
	136	BUG_ON(cpu_isset(smp_processor_id(), cpumask));
	137	BUG_ON(!mm);
	138
	139	#ifdef CONFIG_HOTPLUG_CPU
	140	/* If a CPU which we ran on has gone down, OK. */
	141	cpus_and(cpumask, cpumask, cpu_online_map);
	142	if (unlikely(cpus_empty(cpumask)))
	143	return;
	144	#endif
	145
	146	/*
	147	* i'm not happy about this global shared spinlock in the
	148	* MM hot path, but we'll see how contended it is.
	149	* AK: x86-64 has a faster method that could be ported.
	150	*/
	151	spin_lock(&tlbstate_lock);
	152
	153	flush_mm = mm;
	154	flush_va = va;
	155	cpus_or(flush_cpumask, cpumask, flush_cpumask);
d6f0f39b SS	156
	157	/*
	158	* Make the above memory operations globally visible before
	159	* sending the IPI.
	160	*/
	161	smp_mb();
c048fdfe GC	162	/*
	163	* We have to send the IPI only to
	164	* CPUs affected.
	165	*/
e7986739	166	send_IPI_mask(&cpumask, INVALIDATE_TLB_VECTOR);
c048fdfe GC	167
	168	while (!cpus_empty(flush_cpumask))
	169	/* nothing. lockup detection does not belong here */
	170	cpu_relax();
	171
	172	flush_mm = NULL;
	173	flush_va = 0;
	174	spin_unlock(&tlbstate_lock);
	175	}
	176
	177	void flush_tlb_current_task(void)
	178	{
	179	struct mm_struct *mm = current->mm;
	180	cpumask_t cpu_mask;
	181
	182	preempt_disable();
	183	cpu_mask = mm->cpu_vm_mask;
	184	cpu_clear(smp_processor_id(), cpu_mask);
	185
	186	local_flush_tlb();
	187	if (!cpus_empty(cpu_mask))
	188	flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
	189	preempt_enable();
	190	}
	191
	192	void flush_tlb_mm(struct mm_struct *mm)
	193	{
	194	cpumask_t cpu_mask;
	195
	196	preempt_disable();
	197	cpu_mask = mm->cpu_vm_mask;
	198	cpu_clear(smp_processor_id(), cpu_mask);
	199
	200	if (current->active_mm == mm) {
	201	if (current->mm)
	202	local_flush_tlb();
	203	else
	204	leave_mm(smp_processor_id());
	205	}
	206	if (!cpus_empty(cpu_mask))
	207	flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
	208
	209	preempt_enable();
	210	}
	211
	212	void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
	213	{
	214	struct mm_struct *mm = vma->vm_mm;
	215	cpumask_t cpu_mask;
	216
	217	preempt_disable();
	218	cpu_mask = mm->cpu_vm_mask;
	219	cpu_clear(smp_processor_id(), cpu_mask);
	220
	221	if (current->active_mm == mm) {
	222	if (current->mm)
	223	__flush_tlb_one(va);
	224	else
	225	leave_mm(smp_processor_id());
	226	}
	227
	228	if (!cpus_empty(cpu_mask))
	229	flush_tlb_others(cpu_mask, mm, va);
	230
231	preempt_enable();
232	}
233	EXPORT_SYMBOL(flush_tlb_page);
234
235	static void do_flush_tlb_all(void *info)
236	{
237	unsigned long cpu = smp_processor_id();
238
239	__flush_tlb_all();
cfc31983	240	if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_LAZY)
c048fdfe GC	241	leave_mm(cpu);
	242	}
	243
	244	void flush_tlb_all(void)
	245	{
15c8b6c1	246	on_each_cpu(do_flush_tlb_all, NULL, 1);
c048fdfe GC	247	}
c048fdfe GC	248
913da64b AN	249	void reset_lazy_tlbstate(void)
	250	{
	251	int cpu = raw_smp_processor_id();
	252
	253	per_cpu(cpu_tlbstate, cpu).state = 0;
	254	per_cpu(cpu_tlbstate, cpu).active_mm = &init_mm;
	255	}
	256