Commit | Line | Data |
---|---|---|
196d9d8b PZ |
1 | #include <linux/gfp.h> |
2 | #include <linux/highmem.h> | |
3 | #include <linux/kernel.h> | |
4 | #include <linux/mmdebug.h> | |
5 | #include <linux/mm_types.h> | |
6 | #include <linux/pagemap.h> | |
7 | #include <linux/rcupdate.h> | |
8 | #include <linux/smp.h> | |
9 | #include <linux/swap.h> | |
10 | ||
11 | #include <asm/pgalloc.h> | |
12 | #include <asm/tlb.h> | |
13 | ||
952a31c9 MS |
14 | #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER |
15 | ||
196d9d8b PZ |
16 | static bool tlb_next_batch(struct mmu_gather *tlb) |
17 | { | |
18 | struct mmu_gather_batch *batch; | |
19 | ||
20 | batch = tlb->active; | |
21 | if (batch->next) { | |
22 | tlb->active = batch->next; | |
23 | return true; | |
24 | } | |
25 | ||
26 | if (tlb->batch_count == MAX_GATHER_BATCH_COUNT) | |
27 | return false; | |
28 | ||
29 | batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); | |
30 | if (!batch) | |
31 | return false; | |
32 | ||
33 | tlb->batch_count++; | |
34 | batch->next = NULL; | |
35 | batch->nr = 0; | |
36 | batch->max = MAX_GATHER_BATCH; | |
37 | ||
38 | tlb->active->next = batch; | |
39 | tlb->active = batch; | |
40 | ||
41 | return true; | |
42 | } | |
43 | ||
952a31c9 MS |
44 | static void tlb_batch_pages_flush(struct mmu_gather *tlb) |
45 | { | |
46 | struct mmu_gather_batch *batch; | |
47 | ||
48 | for (batch = &tlb->local; batch && batch->nr; batch = batch->next) { | |
49 | free_pages_and_swap_cache(batch->pages, batch->nr); | |
50 | batch->nr = 0; | |
51 | } | |
52 | tlb->active = &tlb->local; | |
53 | } | |
54 | ||
55 | static void tlb_batch_list_free(struct mmu_gather *tlb) | |
56 | { | |
57 | struct mmu_gather_batch *batch, *next; | |
58 | ||
59 | for (batch = tlb->local.next; batch; batch = next) { | |
60 | next = batch->next; | |
61 | free_pages((unsigned long)batch, 0); | |
62 | } | |
63 | tlb->local.next = NULL; | |
64 | } | |
65 | ||
66 | bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) | |
67 | { | |
68 | struct mmu_gather_batch *batch; | |
69 | ||
70 | VM_BUG_ON(!tlb->end); | |
71 | ||
72 | #ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE | |
73 | VM_WARN_ON(tlb->page_size != page_size); | |
74 | #endif | |
75 | ||
76 | batch = tlb->active; | |
77 | /* | |
78 | * Add the page and check if we are full. If so | |
79 | * force a flush. | |
80 | */ | |
81 | batch->pages[batch->nr++] = page; | |
82 | if (batch->nr == batch->max) { | |
83 | if (!tlb_next_batch(tlb)) | |
84 | return true; | |
85 | batch = tlb->active; | |
86 | } | |
87 | VM_BUG_ON_PAGE(batch->nr > batch->max, page); | |
88 | ||
89 | return false; | |
90 | } | |
91 | ||
92 | #endif /* HAVE_MMU_GATHER_NO_GATHER */ | |
93 | ||
196d9d8b PZ |
94 | #ifdef CONFIG_HAVE_RCU_TABLE_FREE |
95 | ||
96 | /* | |
97 | * See the comment near struct mmu_table_batch. | |
98 | */ | |
99 | ||
100 | /* | |
101 | * If we want tlb_remove_table() to imply TLB invalidates. | |
102 | */ | |
103 | static inline void tlb_table_invalidate(struct mmu_gather *tlb) | |
104 | { | |
96bc9567 | 105 | #ifndef CONFIG_HAVE_RCU_TABLE_NO_INVALIDATE |
196d9d8b PZ |
106 | /* |
107 | * Invalidate page-table caches used by hardware walkers. Then we still | |
108 | * need to RCU-sched wait while freeing the pages because software | |
109 | * walkers can still be in-flight. | |
110 | */ | |
111 | tlb_flush_mmu_tlbonly(tlb); | |
112 | #endif | |
113 | } | |
114 | ||
115 | static void tlb_remove_table_smp_sync(void *arg) | |
116 | { | |
117 | /* Simply deliver the interrupt */ | |
118 | } | |
119 | ||
120 | static void tlb_remove_table_one(void *table) | |
121 | { | |
122 | /* | |
123 | * This isn't an RCU grace period and hence the page-tables cannot be | |
124 | * assumed to be actually RCU-freed. | |
125 | * | |
126 | * It is however sufficient for software page-table walkers that rely on | |
127 | * IRQ disabling. See the comment near struct mmu_table_batch. | |
128 | */ | |
129 | smp_call_function(tlb_remove_table_smp_sync, NULL, 1); | |
130 | __tlb_remove_table(table); | |
131 | } | |
132 | ||
133 | static void tlb_remove_table_rcu(struct rcu_head *head) | |
134 | { | |
135 | struct mmu_table_batch *batch; | |
136 | int i; | |
137 | ||
138 | batch = container_of(head, struct mmu_table_batch, rcu); | |
139 | ||
140 | for (i = 0; i < batch->nr; i++) | |
141 | __tlb_remove_table(batch->tables[i]); | |
142 | ||
143 | free_page((unsigned long)batch); | |
144 | } | |
145 | ||
0a8caf21 | 146 | static void tlb_table_flush(struct mmu_gather *tlb) |
196d9d8b PZ |
147 | { |
148 | struct mmu_table_batch **batch = &tlb->batch; | |
149 | ||
150 | if (*batch) { | |
151 | tlb_table_invalidate(tlb); | |
b401ec18 | 152 | call_rcu(&(*batch)->rcu, tlb_remove_table_rcu); |
196d9d8b PZ |
153 | *batch = NULL; |
154 | } | |
155 | } | |
156 | ||
157 | void tlb_remove_table(struct mmu_gather *tlb, void *table) | |
158 | { | |
159 | struct mmu_table_batch **batch = &tlb->batch; | |
160 | ||
161 | if (*batch == NULL) { | |
162 | *batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN); | |
163 | if (*batch == NULL) { | |
164 | tlb_table_invalidate(tlb); | |
165 | tlb_remove_table_one(table); | |
166 | return; | |
167 | } | |
168 | (*batch)->nr = 0; | |
169 | } | |
170 | ||
171 | (*batch)->tables[(*batch)->nr++] = table; | |
172 | if ((*batch)->nr == MAX_TABLE_BATCH) | |
173 | tlb_table_flush(tlb); | |
174 | } | |
175 | ||
176 | #endif /* CONFIG_HAVE_RCU_TABLE_FREE */ | |
177 | ||
0a8caf21 PZ |
178 | static void tlb_flush_mmu_free(struct mmu_gather *tlb) |
179 | { | |
180 | #ifdef CONFIG_HAVE_RCU_TABLE_FREE | |
181 | tlb_table_flush(tlb); | |
182 | #endif | |
183 | #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER | |
184 | tlb_batch_pages_flush(tlb); | |
185 | #endif | |
186 | } | |
187 | ||
188 | void tlb_flush_mmu(struct mmu_gather *tlb) | |
189 | { | |
190 | tlb_flush_mmu_tlbonly(tlb); | |
191 | tlb_flush_mmu_free(tlb); | |
192 | } | |
193 | ||
196d9d8b PZ |
194 | /** |
195 | * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down | |
196 | * @tlb: the mmu_gather structure to initialize | |
197 | * @mm: the mm_struct of the target address space | |
198 | * @start: start of the region that will be removed from the page-table | |
199 | * @end: end of the region that will be removed from the page-table | |
200 | * | |
201 | * Called to initialize an (on-stack) mmu_gather structure for page-table | |
202 | * tear-down from @mm. The @start and @end are set to 0 and -1 | |
203 | * respectively when @mm is without users and we're going to destroy | |
204 | * the full address space (exit/execve). | |
205 | */ | |
206 | void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, | |
207 | unsigned long start, unsigned long end) | |
208 | { | |
1808d65b PZ |
209 | tlb->mm = mm; |
210 | ||
211 | /* Is it from 0 to ~0? */ | |
212 | tlb->fullmm = !(start | (end+1)); | |
213 | ||
214 | #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER | |
215 | tlb->need_flush_all = 0; | |
216 | tlb->local.next = NULL; | |
217 | tlb->local.nr = 0; | |
218 | tlb->local.max = ARRAY_SIZE(tlb->__pages); | |
219 | tlb->active = &tlb->local; | |
220 | tlb->batch_count = 0; | |
221 | #endif | |
222 | ||
223 | #ifdef CONFIG_HAVE_RCU_TABLE_FREE | |
224 | tlb->batch = NULL; | |
225 | #endif | |
226 | #ifdef CONFIG_HAVE_MMU_GATHER_PAGE_SIZE | |
227 | tlb->page_size = 0; | |
228 | #endif | |
229 | ||
230 | __tlb_reset_range(tlb); | |
196d9d8b PZ |
231 | inc_tlb_flush_pending(tlb->mm); |
232 | } | |
233 | ||
1808d65b PZ |
234 | /** |
235 | * tlb_finish_mmu - finish an mmu_gather structure | |
236 | * @tlb: the mmu_gather structure to finish | |
237 | * @start: start of the region that will be removed from the page-table | |
238 | * @end: end of the region that will be removed from the page-table | |
239 | * | |
240 | * Called at the end of the shootdown operation to free up any resources that | |
241 | * were required. | |
242 | */ | |
196d9d8b PZ |
243 | void tlb_finish_mmu(struct mmu_gather *tlb, |
244 | unsigned long start, unsigned long end) | |
245 | { | |
246 | /* | |
247 | * If there are parallel threads are doing PTE changes on same range | |
248 | * under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB | |
249 | * flush by batching, a thread has stable TLB entry can fail to flush | |
250 | * the TLB by observing pte_none|!pte_dirty, for example so flush TLB | |
251 | * forcefully if we detect parallel PTE batching threads. | |
252 | */ | |
1808d65b PZ |
253 | if (mm_tlb_flush_nested(tlb->mm)) { |
254 | __tlb_reset_range(tlb); | |
255 | __tlb_adjust_range(tlb, start, end - start); | |
256 | } | |
196d9d8b | 257 | |
1808d65b PZ |
258 | tlb_flush_mmu(tlb); |
259 | ||
260 | /* keep the page table cache within bounds */ | |
261 | check_pgt_cache(); | |
262 | #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER | |
263 | tlb_batch_list_free(tlb); | |
264 | #endif | |
196d9d8b PZ |
265 | dec_tlb_flush_pending(tlb->mm); |
266 | } |