Commit | Line | Data |
---|---|---|
965ea4bb AT |
1 | /* SMP TLB support routines. |
2 | * | |
3 | * Copyright (C) 2006-2008 Panasonic Corporation | |
4 | * All Rights Reserved. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * version 2 as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | */ | |
15 | #include <linux/interrupt.h> | |
16 | #include <linux/spinlock.h> | |
17 | #include <linux/init.h> | |
18 | #include <linux/jiffies.h> | |
19 | #include <linux/cpumask.h> | |
20 | #include <linux/err.h> | |
21 | #include <linux/kernel.h> | |
22 | #include <linux/delay.h> | |
23 | #include <linux/sched.h> | |
24 | #include <linux/profile.h> | |
25 | #include <linux/smp.h> | |
26 | #include <asm/tlbflush.h> | |
27 | #include <asm/system.h> | |
28 | #include <asm/bitops.h> | |
29 | #include <asm/processor.h> | |
30 | #include <asm/bug.h> | |
31 | #include <asm/exceptions.h> | |
32 | #include <asm/hardirq.h> | |
33 | #include <asm/fpu.h> | |
34 | #include <asm/mmu_context.h> | |
35 | #include <asm/thread_info.h> | |
36 | #include <asm/cpu-regs.h> | |
37 | #include <asm/intctl-regs.h> | |
38 | ||
39 | /* | |
40 | * For flush TLB | |
41 | */ | |
42 | #define FLUSH_ALL 0xffffffff | |
43 | ||
44 | static cpumask_t flush_cpumask; | |
45 | static struct mm_struct *flush_mm; | |
46 | static unsigned long flush_va; | |
47 | static DEFINE_SPINLOCK(tlbstate_lock); | |
48 | ||
49 | DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = { | |
50 | &init_mm, 0 | |
51 | }; | |
52 | ||
53 | static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, | |
54 | unsigned long va); | |
55 | static void do_flush_tlb_all(void *info); | |
56 | ||
57 | /** | |
58 | * smp_flush_tlb - Callback to invalidate the TLB. | |
59 | * @unused: Callback context (ignored). | |
60 | */ | |
61 | void smp_flush_tlb(void *unused) | |
62 | { | |
63 | unsigned long cpu_id; | |
64 | ||
65 | cpu_id = get_cpu(); | |
66 | ||
67 | if (!cpu_isset(cpu_id, flush_cpumask)) | |
68 | /* This was a BUG() but until someone can quote me the line | |
69 | * from the intel manual that guarantees an IPI to multiple | |
70 | * CPUs is retried _only_ on the erroring CPUs its staying as a | |
71 | * return | |
72 | * | |
73 | * BUG(); | |
74 | */ | |
75 | goto out; | |
76 | ||
77 | if (flush_va == FLUSH_ALL) | |
78 | local_flush_tlb(); | |
79 | else | |
80 | local_flush_tlb_page(flush_mm, flush_va); | |
81 | ||
82 | smp_mb__before_clear_bit(); | |
83 | cpu_clear(cpu_id, flush_cpumask); | |
84 | smp_mb__after_clear_bit(); | |
85 | out: | |
86 | put_cpu(); | |
87 | } | |
88 | ||
89 | /** | |
90 | * flush_tlb_others - Tell the specified CPUs to invalidate their TLBs | |
91 | * @cpumask: The list of CPUs to target. | |
92 | * @mm: The VM context to flush from (if va!=FLUSH_ALL). | |
93 | * @va: Virtual address to flush or FLUSH_ALL to flush everything. | |
94 | */ | |
95 | static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, | |
96 | unsigned long va) | |
97 | { | |
98 | cpumask_t tmp; | |
99 | ||
100 | /* A couple of sanity checks (to be removed): | |
101 | * - mask must not be empty | |
102 | * - current CPU must not be in mask | |
103 | * - we do not send IPIs to as-yet unbooted CPUs. | |
104 | */ | |
105 | BUG_ON(!mm); | |
106 | BUG_ON(cpus_empty(cpumask)); | |
107 | BUG_ON(cpu_isset(smp_processor_id(), cpumask)); | |
108 | ||
109 | cpus_and(tmp, cpumask, cpu_online_map); | |
110 | BUG_ON(!cpus_equal(cpumask, tmp)); | |
111 | ||
112 | /* I'm not happy about this global shared spinlock in the MM hot path, | |
113 | * but we'll see how contended it is. | |
114 | * | |
115 | * Temporarily this turns IRQs off, so that lockups are detected by the | |
116 | * NMI watchdog. | |
117 | */ | |
118 | spin_lock(&tlbstate_lock); | |
119 | ||
120 | flush_mm = mm; | |
121 | flush_va = va; | |
122 | #if NR_CPUS <= BITS_PER_LONG | |
123 | atomic_set_mask(cpumask.bits[0], &flush_cpumask.bits[0]); | |
124 | #else | |
125 | #error Not supported. | |
126 | #endif | |
127 | ||
128 | /* FIXME: if NR_CPUS>=3, change send_IPI_mask */ | |
129 | smp_call_function(smp_flush_tlb, NULL, 1); | |
130 | ||
131 | while (!cpus_empty(flush_cpumask)) | |
132 | /* Lockup detection does not belong here */ | |
133 | smp_mb(); | |
134 | ||
135 | flush_mm = NULL; | |
136 | flush_va = 0; | |
137 | spin_unlock(&tlbstate_lock); | |
138 | } | |
139 | ||
140 | /** | |
141 | * flush_tlb_mm - Invalidate TLB of specified VM context | |
142 | * @mm: The VM context to invalidate. | |
143 | */ | |
144 | void flush_tlb_mm(struct mm_struct *mm) | |
145 | { | |
146 | cpumask_t cpu_mask; | |
147 | ||
148 | preempt_disable(); | |
149 | cpu_mask = mm->cpu_vm_mask; | |
150 | cpu_clear(smp_processor_id(), cpu_mask); | |
151 | ||
152 | local_flush_tlb(); | |
153 | if (!cpus_empty(cpu_mask)) | |
154 | flush_tlb_others(cpu_mask, mm, FLUSH_ALL); | |
155 | ||
156 | preempt_enable(); | |
157 | } | |
158 | ||
159 | /** | |
160 | * flush_tlb_current_task - Invalidate TLB of current task | |
161 | */ | |
162 | void flush_tlb_current_task(void) | |
163 | { | |
164 | struct mm_struct *mm = current->mm; | |
165 | cpumask_t cpu_mask; | |
166 | ||
167 | preempt_disable(); | |
168 | cpu_mask = mm->cpu_vm_mask; | |
169 | cpu_clear(smp_processor_id(), cpu_mask); | |
170 | ||
171 | local_flush_tlb(); | |
172 | if (!cpus_empty(cpu_mask)) | |
173 | flush_tlb_others(cpu_mask, mm, FLUSH_ALL); | |
174 | ||
175 | preempt_enable(); | |
176 | } | |
177 | ||
178 | /** | |
179 | * flush_tlb_page - Invalidate TLB of page | |
180 | * @vma: The VM context to invalidate the page for. | |
181 | * @va: The virtual address of the page to invalidate. | |
182 | */ | |
183 | void flush_tlb_page(struct vm_area_struct *vma, unsigned long va) | |
184 | { | |
185 | struct mm_struct *mm = vma->vm_mm; | |
186 | cpumask_t cpu_mask; | |
187 | ||
188 | preempt_disable(); | |
189 | cpu_mask = mm->cpu_vm_mask; | |
190 | cpu_clear(smp_processor_id(), cpu_mask); | |
191 | ||
192 | local_flush_tlb_page(mm, va); | |
193 | if (!cpus_empty(cpu_mask)) | |
194 | flush_tlb_others(cpu_mask, mm, va); | |
195 | ||
196 | preempt_enable(); | |
197 | } | |
198 | ||
199 | /** | |
200 | * do_flush_tlb_all - Callback to completely invalidate a TLB | |
201 | * @unused: Callback context (ignored). | |
202 | */ | |
203 | static void do_flush_tlb_all(void *unused) | |
204 | { | |
205 | local_flush_tlb_all(); | |
206 | } | |
207 | ||
208 | /** | |
209 | * flush_tlb_all - Completely invalidate TLBs on all CPUs | |
210 | */ | |
211 | void flush_tlb_all(void) | |
212 | { | |
213 | on_each_cpu(do_flush_tlb_all, 0, 1); | |
214 | } |