Commit | Line | Data |
---|---|---|
965ea4bb AT |
1 | /* SMP TLB support routines. |
2 | * | |
3 | * Copyright (C) 2006-2008 Panasonic Corporation | |
4 | * All Rights Reserved. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * version 2 as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | */ | |
15 | #include <linux/interrupt.h> | |
16 | #include <linux/spinlock.h> | |
17 | #include <linux/init.h> | |
18 | #include <linux/jiffies.h> | |
19 | #include <linux/cpumask.h> | |
20 | #include <linux/err.h> | |
21 | #include <linux/kernel.h> | |
22 | #include <linux/delay.h> | |
589ee628 | 23 | #include <linux/sched/mm.h> |
965ea4bb AT |
24 | #include <linux/profile.h> |
25 | #include <linux/smp.h> | |
26 | #include <asm/tlbflush.h> | |
965ea4bb AT |
27 | #include <asm/bitops.h> |
28 | #include <asm/processor.h> | |
29 | #include <asm/bug.h> | |
30 | #include <asm/exceptions.h> | |
31 | #include <asm/hardirq.h> | |
32 | #include <asm/fpu.h> | |
33 | #include <asm/mmu_context.h> | |
34 | #include <asm/thread_info.h> | |
35 | #include <asm/cpu-regs.h> | |
36 | #include <asm/intctl-regs.h> | |
37 | ||
38 | /* | |
39 | * For flush TLB | |
40 | */ | |
41 | #define FLUSH_ALL 0xffffffff | |
42 | ||
43 | static cpumask_t flush_cpumask; | |
44 | static struct mm_struct *flush_mm; | |
45 | static unsigned long flush_va; | |
46 | static DEFINE_SPINLOCK(tlbstate_lock); | |
47 | ||
48 | DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = { | |
49 | &init_mm, 0 | |
50 | }; | |
51 | ||
52 | static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, | |
53 | unsigned long va); | |
54 | static void do_flush_tlb_all(void *info); | |
55 | ||
56 | /** | |
57 | * smp_flush_tlb - Callback to invalidate the TLB. | |
58 | * @unused: Callback context (ignored). | |
59 | */ | |
60 | void smp_flush_tlb(void *unused) | |
61 | { | |
62 | unsigned long cpu_id; | |
63 | ||
64 | cpu_id = get_cpu(); | |
65 | ||
8ea9716f | 66 | if (!cpumask_test_cpu(cpu_id, &flush_cpumask)) |
965ea4bb AT |
67 | /* This was a BUG() but until someone can quote me the line |
68 | * from the intel manual that guarantees an IPI to multiple | |
69 | * CPUs is retried _only_ on the erroring CPUs its staying as a | |
70 | * return | |
71 | * | |
72 | * BUG(); | |
73 | */ | |
74 | goto out; | |
75 | ||
76 | if (flush_va == FLUSH_ALL) | |
77 | local_flush_tlb(); | |
78 | else | |
79 | local_flush_tlb_page(flush_mm, flush_va); | |
80 | ||
9424cdf0 | 81 | smp_mb__before_atomic(); |
8ea9716f | 82 | cpumask_clear_cpu(cpu_id, &flush_cpumask); |
9424cdf0 | 83 | smp_mb__after_atomic(); |
965ea4bb AT |
84 | out: |
85 | put_cpu(); | |
86 | } | |
87 | ||
88 | /** | |
89 | * flush_tlb_others - Tell the specified CPUs to invalidate their TLBs | |
90 | * @cpumask: The list of CPUs to target. | |
91 | * @mm: The VM context to flush from (if va!=FLUSH_ALL). | |
92 | * @va: Virtual address to flush or FLUSH_ALL to flush everything. | |
93 | */ | |
94 | static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, | |
95 | unsigned long va) | |
96 | { | |
97 | cpumask_t tmp; | |
98 | ||
99 | /* A couple of sanity checks (to be removed): | |
100 | * - mask must not be empty | |
101 | * - current CPU must not be in mask | |
102 | * - we do not send IPIs to as-yet unbooted CPUs. | |
103 | */ | |
104 | BUG_ON(!mm); | |
8ea9716f KM |
105 | BUG_ON(cpumask_empty(&cpumask)); |
106 | BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask)); | |
965ea4bb | 107 | |
8ea9716f KM |
108 | cpumask_and(&tmp, &cpumask, cpu_online_mask); |
109 | BUG_ON(!cpumask_equal(&cpumask, &tmp)); | |
965ea4bb AT |
110 | |
111 | /* I'm not happy about this global shared spinlock in the MM hot path, | |
112 | * but we'll see how contended it is. | |
113 | * | |
114 | * Temporarily this turns IRQs off, so that lockups are detected by the | |
115 | * NMI watchdog. | |
116 | */ | |
117 | spin_lock(&tlbstate_lock); | |
118 | ||
119 | flush_mm = mm; | |
120 | flush_va = va; | |
121 | #if NR_CPUS <= BITS_PER_LONG | |
805de8f4 | 122 | atomic_or(cpumask.bits[0], (atomic_t *)&flush_cpumask.bits[0]); |
965ea4bb AT |
123 | #else |
124 | #error Not supported. | |
125 | #endif | |
126 | ||
127 | /* FIXME: if NR_CPUS>=3, change send_IPI_mask */ | |
128 | smp_call_function(smp_flush_tlb, NULL, 1); | |
129 | ||
8ea9716f | 130 | while (!cpumask_empty(&flush_cpumask)) |
965ea4bb AT |
131 | /* Lockup detection does not belong here */ |
132 | smp_mb(); | |
133 | ||
134 | flush_mm = NULL; | |
135 | flush_va = 0; | |
136 | spin_unlock(&tlbstate_lock); | |
137 | } | |
138 | ||
139 | /** | |
140 | * flush_tlb_mm - Invalidate TLB of specified VM context | |
141 | * @mm: The VM context to invalidate. | |
142 | */ | |
143 | void flush_tlb_mm(struct mm_struct *mm) | |
144 | { | |
145 | cpumask_t cpu_mask; | |
146 | ||
147 | preempt_disable(); | |
8ea9716f KM |
148 | cpumask_copy(&cpu_mask, mm_cpumask(mm)); |
149 | cpumask_clear_cpu(smp_processor_id(), &cpu_mask); | |
965ea4bb AT |
150 | |
151 | local_flush_tlb(); | |
8ea9716f | 152 | if (!cpumask_empty(&cpu_mask)) |
965ea4bb AT |
153 | flush_tlb_others(cpu_mask, mm, FLUSH_ALL); |
154 | ||
155 | preempt_enable(); | |
156 | } | |
157 | ||
158 | /** | |
159 | * flush_tlb_current_task - Invalidate TLB of current task | |
160 | */ | |
161 | void flush_tlb_current_task(void) | |
162 | { | |
163 | struct mm_struct *mm = current->mm; | |
164 | cpumask_t cpu_mask; | |
165 | ||
166 | preempt_disable(); | |
8ea9716f KM |
167 | cpumask_copy(&cpu_mask, mm_cpumask(mm)); |
168 | cpumask_clear_cpu(smp_processor_id(), &cpu_mask); | |
965ea4bb AT |
169 | |
170 | local_flush_tlb(); | |
8ea9716f | 171 | if (!cpumask_empty(&cpu_mask)) |
965ea4bb AT |
172 | flush_tlb_others(cpu_mask, mm, FLUSH_ALL); |
173 | ||
174 | preempt_enable(); | |
175 | } | |
176 | ||
177 | /** | |
178 | * flush_tlb_page - Invalidate TLB of page | |
179 | * @vma: The VM context to invalidate the page for. | |
180 | * @va: The virtual address of the page to invalidate. | |
181 | */ | |
182 | void flush_tlb_page(struct vm_area_struct *vma, unsigned long va) | |
183 | { | |
184 | struct mm_struct *mm = vma->vm_mm; | |
185 | cpumask_t cpu_mask; | |
186 | ||
187 | preempt_disable(); | |
8ea9716f KM |
188 | cpumask_copy(&cpu_mask, mm_cpumask(mm)); |
189 | cpumask_clear_cpu(smp_processor_id(), &cpu_mask); | |
965ea4bb AT |
190 | |
191 | local_flush_tlb_page(mm, va); | |
8ea9716f | 192 | if (!cpumask_empty(&cpu_mask)) |
965ea4bb AT |
193 | flush_tlb_others(cpu_mask, mm, va); |
194 | ||
195 | preempt_enable(); | |
196 | } | |
197 | ||
198 | /** | |
199 | * do_flush_tlb_all - Callback to completely invalidate a TLB | |
200 | * @unused: Callback context (ignored). | |
201 | */ | |
202 | static void do_flush_tlb_all(void *unused) | |
203 | { | |
204 | local_flush_tlb_all(); | |
205 | } | |
206 | ||
207 | /** | |
208 | * flush_tlb_all - Completely invalidate TLBs on all CPUs | |
209 | */ | |
210 | void flush_tlb_all(void) | |
211 | { | |
212 | on_each_cpu(do_flush_tlb_all, 0, 1); | |
213 | } |