Commit | Line | Data |
---|---|---|
9c92ab61 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
0c972a05 TK |
2 | /* binder_alloc.c |
3 | * | |
4 | * Android IPC Subsystem | |
5 | * | |
6 | * Copyright (C) 2007-2017 Google, Inc. | |
0c972a05 TK |
7 | */ |
8 | ||
9 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
10 | ||
0c972a05 TK |
11 | #include <linux/list.h> |
12 | #include <linux/sched/mm.h> | |
13 | #include <linux/module.h> | |
14 | #include <linux/rtmutex.h> | |
15 | #include <linux/rbtree.h> | |
16 | #include <linux/seq_file.h> | |
17 | #include <linux/vmalloc.h> | |
18 | #include <linux/slab.h> | |
19 | #include <linux/sched.h> | |
f2517eb7 | 20 | #include <linux/list_lru.h> |
128f3804 | 21 | #include <linux/ratelimit.h> |
1e81c57b | 22 | #include <asm/cacheflush.h> |
1a7c3d9b TK |
23 | #include <linux/uaccess.h> |
24 | #include <linux/highmem.h> | |
45d02f79 | 25 | #include <linux/sizes.h> |
0c972a05 TK |
26 | #include "binder_alloc.h" |
27 | #include "binder_trace.h" | |
28 | ||
ea9cdbf0 | 29 | struct list_lru binder_freelist; |
f2517eb7 | 30 | |
0c972a05 TK |
31 | static DEFINE_MUTEX(binder_alloc_mmap_lock); |
32 | ||
33 | enum { | |
128f3804 | 34 | BINDER_DEBUG_USER_ERROR = 1U << 0, |
0c972a05 TK |
35 | BINDER_DEBUG_OPEN_CLOSE = 1U << 1, |
36 | BINDER_DEBUG_BUFFER_ALLOC = 1U << 2, | |
37 | BINDER_DEBUG_BUFFER_ALLOC_ASYNC = 1U << 3, | |
38 | }; | |
128f3804 | 39 | static uint32_t binder_alloc_debug_mask = BINDER_DEBUG_USER_ERROR; |
0c972a05 TK |
40 | |
41 | module_param_named(debug_mask, binder_alloc_debug_mask, | |
42 | uint, 0644); | |
43 | ||
44 | #define binder_alloc_debug(mask, x...) \ | |
45 | do { \ | |
46 | if (binder_alloc_debug_mask & mask) \ | |
128f3804 | 47 | pr_info_ratelimited(x); \ |
0c972a05 TK |
48 | } while (0) |
49 | ||
e2176219 SY |
50 | static struct binder_buffer *binder_buffer_next(struct binder_buffer *buffer) |
51 | { | |
52 | return list_entry(buffer->entry.next, struct binder_buffer, entry); | |
53 | } | |
54 | ||
55 | static struct binder_buffer *binder_buffer_prev(struct binder_buffer *buffer) | |
56 | { | |
57 | return list_entry(buffer->entry.prev, struct binder_buffer, entry); | |
58 | } | |
59 | ||
0c972a05 TK |
60 | static size_t binder_alloc_buffer_size(struct binder_alloc *alloc, |
61 | struct binder_buffer *buffer) | |
62 | { | |
63 | if (list_is_last(&buffer->entry, &alloc->buffers)) | |
bde4a19f TK |
64 | return alloc->buffer + alloc->buffer_size - buffer->user_data; |
65 | return binder_buffer_next(buffer)->user_data - buffer->user_data; | |
0c972a05 TK |
66 | } |
67 | ||
68 | static void binder_insert_free_buffer(struct binder_alloc *alloc, | |
69 | struct binder_buffer *new_buffer) | |
70 | { | |
71 | struct rb_node **p = &alloc->free_buffers.rb_node; | |
72 | struct rb_node *parent = NULL; | |
73 | struct binder_buffer *buffer; | |
74 | size_t buffer_size; | |
75 | size_t new_buffer_size; | |
76 | ||
77 | BUG_ON(!new_buffer->free); | |
78 | ||
79 | new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer); | |
80 | ||
81 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, | |
82 | "%d: add free buffer, size %zd, at %pK\n", | |
83 | alloc->pid, new_buffer_size, new_buffer); | |
84 | ||
85 | while (*p) { | |
86 | parent = *p; | |
87 | buffer = rb_entry(parent, struct binder_buffer, rb_node); | |
88 | BUG_ON(!buffer->free); | |
89 | ||
90 | buffer_size = binder_alloc_buffer_size(alloc, buffer); | |
91 | ||
92 | if (new_buffer_size < buffer_size) | |
93 | p = &parent->rb_left; | |
94 | else | |
95 | p = &parent->rb_right; | |
96 | } | |
97 | rb_link_node(&new_buffer->rb_node, parent, p); | |
98 | rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers); | |
99 | } | |
100 | ||
101 | static void binder_insert_allocated_buffer_locked( | |
102 | struct binder_alloc *alloc, struct binder_buffer *new_buffer) | |
103 | { | |
104 | struct rb_node **p = &alloc->allocated_buffers.rb_node; | |
105 | struct rb_node *parent = NULL; | |
106 | struct binder_buffer *buffer; | |
107 | ||
108 | BUG_ON(new_buffer->free); | |
109 | ||
110 | while (*p) { | |
111 | parent = *p; | |
112 | buffer = rb_entry(parent, struct binder_buffer, rb_node); | |
113 | BUG_ON(buffer->free); | |
114 | ||
bde4a19f | 115 | if (new_buffer->user_data < buffer->user_data) |
0c972a05 | 116 | p = &parent->rb_left; |
bde4a19f | 117 | else if (new_buffer->user_data > buffer->user_data) |
0c972a05 TK |
118 | p = &parent->rb_right; |
119 | else | |
120 | BUG(); | |
121 | } | |
122 | rb_link_node(&new_buffer->rb_node, parent, p); | |
123 | rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers); | |
124 | } | |
125 | ||
53d311cf | 126 | static struct binder_buffer *binder_alloc_prepare_to_free_locked( |
0c972a05 | 127 | struct binder_alloc *alloc, |
df9aabea | 128 | unsigned long user_ptr) |
0c972a05 TK |
129 | { |
130 | struct rb_node *n = alloc->allocated_buffers.rb_node; | |
131 | struct binder_buffer *buffer; | |
0c972a05 TK |
132 | |
133 | while (n) { | |
134 | buffer = rb_entry(n, struct binder_buffer, rb_node); | |
135 | BUG_ON(buffer->free); | |
136 | ||
df9aabea | 137 | if (user_ptr < buffer->user_data) { |
0c972a05 | 138 | n = n->rb_left; |
df9aabea | 139 | } else if (user_ptr > buffer->user_data) { |
0c972a05 | 140 | n = n->rb_right; |
df9aabea | 141 | } else { |
53d311cf TK |
142 | /* |
143 | * Guard against user threads attempting to | |
7bada55a TK |
144 | * free the buffer when in use by kernel or |
145 | * after it's already been freed. | |
53d311cf | 146 | */ |
7bada55a TK |
147 | if (!buffer->allow_user_free) |
148 | return ERR_PTR(-EPERM); | |
149 | buffer->allow_user_free = 0; | |
0c972a05 | 150 | return buffer; |
53d311cf | 151 | } |
0c972a05 TK |
152 | } |
153 | return NULL; | |
154 | } | |
155 | ||
156 | /** | |
5dc54a06 | 157 | * binder_alloc_prepare_to_free() - get buffer given user ptr |
0c972a05 TK |
158 | * @alloc: binder_alloc for this proc |
159 | * @user_ptr: User pointer to buffer data | |
160 | * | |
161 | * Validate userspace pointer to buffer data and return buffer corresponding to | |
162 | * that user pointer. Search the rb tree for buffer that matches user data | |
163 | * pointer. | |
164 | * | |
165 | * Return: Pointer to buffer or NULL | |
166 | */ | |
53d311cf | 167 | struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc, |
df9aabea | 168 | unsigned long user_ptr) |
0c972a05 TK |
169 | { |
170 | struct binder_buffer *buffer; | |
171 | ||
172 | mutex_lock(&alloc->mutex); | |
53d311cf | 173 | buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr); |
0c972a05 TK |
174 | mutex_unlock(&alloc->mutex); |
175 | return buffer; | |
176 | } | |
177 | ||
37ebbb4f CL |
178 | static inline void |
179 | binder_set_installed_page(struct binder_lru_page *lru_page, | |
180 | struct page *page) | |
181 | { | |
182 | /* Pairs with acquire in binder_get_installed_page() */ | |
183 | smp_store_release(&lru_page->page_ptr, page); | |
184 | } | |
185 | ||
186 | static inline struct page * | |
187 | binder_get_installed_page(struct binder_lru_page *lru_page) | |
188 | { | |
189 | /* Pairs with release in binder_set_installed_page() */ | |
190 | return smp_load_acquire(&lru_page->page_ptr); | |
191 | } | |
192 | ||
ea9cdbf0 CL |
193 | static void binder_lru_freelist_add(struct binder_alloc *alloc, |
194 | unsigned long start, unsigned long end) | |
0d35bf3b CL |
195 | { |
196 | struct binder_lru_page *page; | |
197 | unsigned long page_addr; | |
198 | ||
199 | trace_binder_update_page_range(alloc, false, start, end); | |
200 | ||
201 | for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { | |
202 | size_t index; | |
203 | int ret; | |
204 | ||
205 | index = (page_addr - alloc->buffer) / PAGE_SIZE; | |
206 | page = &alloc->pages[index]; | |
207 | ||
37ebbb4f CL |
208 | if (!binder_get_installed_page(page)) |
209 | continue; | |
210 | ||
0d35bf3b CL |
211 | trace_binder_free_lru_start(alloc, index); |
212 | ||
ea9cdbf0 | 213 | ret = list_lru_add(&binder_freelist, &page->lru); |
0d35bf3b CL |
214 | WARN_ON(!ret); |
215 | ||
216 | trace_binder_free_lru_end(alloc, index); | |
217 | } | |
218 | } | |
219 | ||
ea2735ce CL |
220 | static int binder_install_single_page(struct binder_alloc *alloc, |
221 | struct binder_lru_page *lru_page, | |
222 | unsigned long addr) | |
223 | { | |
224 | struct page *page; | |
225 | int ret = 0; | |
226 | ||
227 | if (!mmget_not_zero(alloc->mm)) | |
228 | return -ESRCH; | |
229 | ||
37ebbb4f CL |
230 | /* |
231 | * Protected with mmap_sem in write mode as multiple tasks | |
232 | * might race to install the same page. | |
233 | */ | |
ea2735ce | 234 | mmap_write_lock(alloc->mm); |
37ebbb4f CL |
235 | if (binder_get_installed_page(lru_page)) |
236 | goto out; | |
237 | ||
ea2735ce CL |
238 | if (!alloc->vma) { |
239 | pr_err("%d: %s failed, no vma\n", alloc->pid, __func__); | |
240 | ret = -ESRCH; | |
241 | goto out; | |
242 | } | |
243 | ||
244 | page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); | |
245 | if (!page) { | |
246 | pr_err("%d: failed to allocate page\n", alloc->pid); | |
247 | ret = -ENOMEM; | |
248 | goto out; | |
249 | } | |
250 | ||
251 | ret = vm_insert_page(alloc->vma, addr, page); | |
252 | if (ret) { | |
162c7973 CL |
253 | pr_err("%d: %s failed to insert page at offset %lx with %d\n", |
254 | alloc->pid, __func__, addr - alloc->buffer, ret); | |
ea2735ce CL |
255 | __free_page(page); |
256 | ret = -ENOMEM; | |
257 | goto out; | |
258 | } | |
259 | ||
37ebbb4f CL |
260 | /* Mark page installation complete and safe to use */ |
261 | binder_set_installed_page(lru_page, page); | |
ea2735ce CL |
262 | out: |
263 | mmap_write_unlock(alloc->mm); | |
264 | mmput_async(alloc->mm); | |
265 | return ret; | |
266 | } | |
267 | ||
37ebbb4f CL |
268 | static int binder_install_buffer_pages(struct binder_alloc *alloc, |
269 | struct binder_buffer *buffer, | |
270 | size_t size) | |
271 | { | |
272 | struct binder_lru_page *page; | |
273 | unsigned long start, final; | |
274 | unsigned long page_addr; | |
275 | ||
276 | start = buffer->user_data & PAGE_MASK; | |
277 | final = PAGE_ALIGN(buffer->user_data + size); | |
278 | ||
279 | for (page_addr = start; page_addr < final; page_addr += PAGE_SIZE) { | |
280 | unsigned long index; | |
281 | int ret; | |
282 | ||
283 | index = (page_addr - alloc->buffer) / PAGE_SIZE; | |
284 | page = &alloc->pages[index]; | |
285 | ||
286 | if (binder_get_installed_page(page)) | |
287 | continue; | |
288 | ||
289 | trace_binder_alloc_page_start(alloc, index); | |
290 | ||
291 | ret = binder_install_single_page(alloc, page, page_addr); | |
292 | if (ret) | |
293 | return ret; | |
294 | ||
295 | trace_binder_alloc_page_end(alloc, index); | |
296 | } | |
297 | ||
298 | return 0; | |
299 | } | |
300 | ||
301 | /* The range of pages should exclude those shared with other buffers */ | |
ea9cdbf0 CL |
302 | static void binder_lru_freelist_del(struct binder_alloc *alloc, |
303 | unsigned long start, unsigned long end) | |
0c972a05 | 304 | { |
df9aabea | 305 | struct binder_lru_page *page; |
df9aabea | 306 | unsigned long page_addr; |
0c972a05 | 307 | |
0d35bf3b | 308 | trace_binder_update_page_range(alloc, true, start, end); |
f2517eb7 SY |
309 | |
310 | for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { | |
ea2735ce | 311 | unsigned long index; |
f2517eb7 | 312 | bool on_lru; |
0c972a05 | 313 | |
e41e164c SY |
314 | index = (page_addr - alloc->buffer) / PAGE_SIZE; |
315 | page = &alloc->pages[index]; | |
0c972a05 | 316 | |
f2517eb7 | 317 | if (page->page_ptr) { |
e41e164c SY |
318 | trace_binder_alloc_lru_start(alloc, index); |
319 | ||
ea9cdbf0 | 320 | on_lru = list_lru_del(&binder_freelist, &page->lru); |
f2517eb7 | 321 | WARN_ON(!on_lru); |
e41e164c SY |
322 | |
323 | trace_binder_alloc_lru_end(alloc, index); | |
f2517eb7 SY |
324 | continue; |
325 | } | |
326 | ||
8d9a3ab6 MC |
327 | if (index + 1 > alloc->pages_high) |
328 | alloc->pages_high = index + 1; | |
0c972a05 | 329 | } |
0c972a05 TK |
330 | } |
331 | ||
0fa53349 CL |
332 | static inline void binder_alloc_set_vma(struct binder_alloc *alloc, |
333 | struct vm_area_struct *vma) | |
334 | { | |
335 | /* pairs with smp_load_acquire in binder_alloc_get_vma() */ | |
336 | smp_store_release(&alloc->vma, vma); | |
337 | } | |
338 | ||
da1b9564 MK |
339 | static inline struct vm_area_struct *binder_alloc_get_vma( |
340 | struct binder_alloc *alloc) | |
341 | { | |
0fa53349 CL |
342 | /* pairs with smp_store_release in binder_alloc_set_vma() */ |
343 | return smp_load_acquire(&alloc->vma); | |
da1b9564 MK |
344 | } |
345 | ||
9409af24 CL |
346 | static void debug_no_space_locked(struct binder_alloc *alloc) |
347 | { | |
348 | size_t largest_alloc_size = 0; | |
349 | struct binder_buffer *buffer; | |
350 | size_t allocated_buffers = 0; | |
351 | size_t largest_free_size = 0; | |
352 | size_t total_alloc_size = 0; | |
353 | size_t total_free_size = 0; | |
354 | size_t free_buffers = 0; | |
355 | size_t buffer_size; | |
356 | struct rb_node *n; | |
357 | ||
358 | for (n = rb_first(&alloc->allocated_buffers); n; n = rb_next(n)) { | |
359 | buffer = rb_entry(n, struct binder_buffer, rb_node); | |
360 | buffer_size = binder_alloc_buffer_size(alloc, buffer); | |
361 | allocated_buffers++; | |
362 | total_alloc_size += buffer_size; | |
363 | if (buffer_size > largest_alloc_size) | |
364 | largest_alloc_size = buffer_size; | |
365 | } | |
366 | ||
367 | for (n = rb_first(&alloc->free_buffers); n; n = rb_next(n)) { | |
368 | buffer = rb_entry(n, struct binder_buffer, rb_node); | |
369 | buffer_size = binder_alloc_buffer_size(alloc, buffer); | |
370 | free_buffers++; | |
371 | total_free_size += buffer_size; | |
372 | if (buffer_size > largest_free_size) | |
373 | largest_free_size = buffer_size; | |
374 | } | |
375 | ||
376 | binder_alloc_debug(BINDER_DEBUG_USER_ERROR, | |
377 | "allocated: %zd (num: %zd largest: %zd), free: %zd (num: %zd largest: %zd)\n", | |
378 | total_alloc_size, allocated_buffers, | |
379 | largest_alloc_size, total_free_size, | |
380 | free_buffers, largest_free_size); | |
381 | } | |
382 | ||
89f71743 | 383 | static bool debug_low_async_space_locked(struct binder_alloc *alloc) |
261e7818 MC |
384 | { |
385 | /* | |
386 | * Find the amount and size of buffers allocated by the current caller; | |
387 | * The idea is that once we cross the threshold, whoever is responsible | |
388 | * for the low async space is likely to try to send another async txn, | |
389 | * and at some point we'll catch them in the act. This is more efficient | |
390 | * than keeping a map per pid. | |
391 | */ | |
261e7818 MC |
392 | struct binder_buffer *buffer; |
393 | size_t total_alloc_size = 0; | |
89f71743 | 394 | int pid = current->tgid; |
261e7818 | 395 | size_t num_buffers = 0; |
89f71743 | 396 | struct rb_node *n; |
261e7818 | 397 | |
c13500ea CL |
398 | /* |
399 | * Only start detecting spammers once we have less than 20% of async | |
400 | * space left (which is less than 10% of total buffer size). | |
401 | */ | |
402 | if (alloc->free_async_space >= alloc->buffer_size / 10) { | |
403 | alloc->oneway_spam_detected = false; | |
404 | return false; | |
405 | } | |
406 | ||
261e7818 MC |
407 | for (n = rb_first(&alloc->allocated_buffers); n != NULL; |
408 | n = rb_next(n)) { | |
409 | buffer = rb_entry(n, struct binder_buffer, rb_node); | |
410 | if (buffer->pid != pid) | |
411 | continue; | |
412 | if (!buffer->async_transaction) | |
413 | continue; | |
c6d05e07 | 414 | total_alloc_size += binder_alloc_buffer_size(alloc, buffer); |
261e7818 MC |
415 | num_buffers++; |
416 | } | |
417 | ||
418 | /* | |
419 | * Warn if this pid has more than 50 transactions, or more than 50% of | |
a7dc1e6f HL |
420 | * async space (which is 25% of total buffer size). Oneway spam is only |
421 | * detected when the threshold is exceeded. | |
261e7818 MC |
422 | */ |
423 | if (num_buffers > 50 || total_alloc_size > alloc->buffer_size / 4) { | |
424 | binder_alloc_debug(BINDER_DEBUG_USER_ERROR, | |
425 | "%d: pid %d spamming oneway? %zd buffers allocated for a total size of %zd\n", | |
426 | alloc->pid, pid, num_buffers, total_alloc_size); | |
a7dc1e6f HL |
427 | if (!alloc->oneway_spam_detected) { |
428 | alloc->oneway_spam_detected = true; | |
429 | return true; | |
430 | } | |
261e7818 | 431 | } |
a7dc1e6f | 432 | return false; |
261e7818 MC |
433 | } |
434 | ||
c7ac30fa | 435 | /* Callers preallocate @new_buffer, it is freed by this function if unused */ |
3f827245 XS |
436 | static struct binder_buffer *binder_alloc_new_buf_locked( |
437 | struct binder_alloc *alloc, | |
c7ac30fa | 438 | struct binder_buffer *new_buffer, |
377e1684 | 439 | size_t size, |
89f71743 | 440 | int is_async) |
0c972a05 TK |
441 | { |
442 | struct rb_node *n = alloc->free_buffers.rb_node; | |
0c972a05 | 443 | struct rb_node *best_fit = NULL; |
c7ac30fa | 444 | struct binder_buffer *buffer; |
67dcc880 CL |
445 | unsigned long next_used_page; |
446 | unsigned long curr_last_page; | |
c7ac30fa | 447 | size_t buffer_size; |
0c972a05 | 448 | |
c6d05e07 | 449 | if (is_async && alloc->free_async_space < size) { |
0c972a05 TK |
450 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, |
451 | "%d: binder_alloc_buf size %zd failed, no async space left\n", | |
452 | alloc->pid, size); | |
c7ac30fa CL |
453 | buffer = ERR_PTR(-ENOSPC); |
454 | goto out; | |
0c972a05 TK |
455 | } |
456 | ||
457 | while (n) { | |
458 | buffer = rb_entry(n, struct binder_buffer, rb_node); | |
459 | BUG_ON(!buffer->free); | |
460 | buffer_size = binder_alloc_buffer_size(alloc, buffer); | |
461 | ||
462 | if (size < buffer_size) { | |
463 | best_fit = n; | |
464 | n = n->rb_left; | |
377e1684 | 465 | } else if (size > buffer_size) { |
0c972a05 | 466 | n = n->rb_right; |
377e1684 | 467 | } else { |
0c972a05 TK |
468 | best_fit = n; |
469 | break; | |
470 | } | |
471 | } | |
377e1684 | 472 | |
9409af24 | 473 | if (unlikely(!best_fit)) { |
128f3804 SY |
474 | binder_alloc_debug(BINDER_DEBUG_USER_ERROR, |
475 | "%d: binder_alloc_buf size %zd failed, no address space\n", | |
476 | alloc->pid, size); | |
9409af24 | 477 | debug_no_space_locked(alloc); |
c7ac30fa CL |
478 | buffer = ERR_PTR(-ENOSPC); |
479 | goto out; | |
0c972a05 | 480 | } |
9409af24 | 481 | |
de0e6573 CL |
482 | if (buffer_size != size) { |
483 | /* Found an oversized buffer and needs to be split */ | |
0c972a05 TK |
484 | buffer = rb_entry(best_fit, struct binder_buffer, rb_node); |
485 | buffer_size = binder_alloc_buffer_size(alloc, buffer); | |
de0e6573 CL |
486 | |
487 | WARN_ON(n || buffer_size == size); | |
488 | new_buffer->user_data = buffer->user_data + size; | |
489 | list_add(&new_buffer->entry, &buffer->entry); | |
490 | new_buffer->free = 1; | |
491 | binder_insert_free_buffer(alloc, new_buffer); | |
492 | new_buffer = NULL; | |
0c972a05 TK |
493 | } |
494 | ||
495 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, | |
496 | "%d: binder_alloc_buf size %zd got buffer %pK size %zd\n", | |
497 | alloc->pid, size, buffer, buffer_size); | |
498 | ||
67dcc880 CL |
499 | /* |
500 | * Now we remove the pages from the freelist. A clever calculation | |
501 | * with buffer_size determines if the last page is shared with an | |
502 | * adjacent in-use buffer. In such case, the page has been already | |
503 | * removed from the freelist so we trim our range short. | |
504 | */ | |
505 | next_used_page = (buffer->user_data + buffer_size) & PAGE_MASK; | |
506 | curr_last_page = PAGE_ALIGN(buffer->user_data + size); | |
ea9cdbf0 | 507 | binder_lru_freelist_del(alloc, PAGE_ALIGN(buffer->user_data), |
67dcc880 | 508 | min(next_used_page, curr_last_page)); |
74310e06 | 509 | |
de0e6573 | 510 | rb_erase(&buffer->rb_node, &alloc->free_buffers); |
74310e06 | 511 | buffer->free = 0; |
7bada55a | 512 | buffer->allow_user_free = 0; |
74310e06 | 513 | binder_insert_allocated_buffer_locked(alloc, buffer); |
0c972a05 | 514 | buffer->async_transaction = is_async; |
a7dc1e6f | 515 | buffer->oneway_spam_suspect = false; |
0c972a05 | 516 | if (is_async) { |
c6d05e07 | 517 | alloc->free_async_space -= size; |
0c972a05 TK |
518 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, |
519 | "%d: binder_alloc_buf size %zd async free %zd\n", | |
520 | alloc->pid, size, alloc->free_async_space); | |
c13500ea CL |
521 | if (debug_low_async_space_locked(alloc)) |
522 | buffer->oneway_spam_suspect = true; | |
0c972a05 | 523 | } |
377e1684 | 524 | |
c7ac30fa CL |
525 | out: |
526 | /* Discard possibly unused new_buffer */ | |
527 | kfree(new_buffer); | |
0c972a05 TK |
528 | return buffer; |
529 | } | |
530 | ||
377e1684 CL |
531 | /* Calculate the sanitized total size, returns 0 for invalid request */ |
532 | static inline size_t sanitized_size(size_t data_size, | |
533 | size_t offsets_size, | |
534 | size_t extra_buffers_size) | |
535 | { | |
536 | size_t total, tmp; | |
537 | ||
538 | /* Align to pointer size and check for overflows */ | |
539 | tmp = ALIGN(data_size, sizeof(void *)) + | |
540 | ALIGN(offsets_size, sizeof(void *)); | |
541 | if (tmp < data_size || tmp < offsets_size) | |
542 | return 0; | |
543 | total = tmp + ALIGN(extra_buffers_size, sizeof(void *)); | |
544 | if (total < tmp || total < extra_buffers_size) | |
545 | return 0; | |
546 | ||
547 | /* Pad 0-sized buffers so they get a unique address */ | |
548 | total = max(total, sizeof(void *)); | |
549 | ||
550 | return total; | |
551 | } | |
552 | ||
0c972a05 TK |
553 | /** |
554 | * binder_alloc_new_buf() - Allocate a new binder buffer | |
555 | * @alloc: binder_alloc for this proc | |
556 | * @data_size: size of user data buffer | |
557 | * @offsets_size: user specified buffer offset | |
558 | * @extra_buffers_size: size of extra space for meta-data (eg, security context) | |
559 | * @is_async: buffer for async transaction | |
560 | * | |
561 | * Allocate a new buffer given the requested sizes. Returns | |
562 | * the kernel version of the buffer pointer. The size allocated | |
563 | * is the sum of the three given sizes (each rounded up to | |
564 | * pointer-sized boundary) | |
565 | * | |
e1090371 | 566 | * Return: The allocated buffer or %ERR_PTR(-errno) if error |
0c972a05 TK |
567 | */ |
568 | struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc, | |
569 | size_t data_size, | |
570 | size_t offsets_size, | |
571 | size_t extra_buffers_size, | |
89f71743 | 572 | int is_async) |
0c972a05 | 573 | { |
c7ac30fa | 574 | struct binder_buffer *buffer, *next; |
377e1684 | 575 | size_t size; |
37ebbb4f | 576 | int ret; |
377e1684 CL |
577 | |
578 | /* Check binder_alloc is fully initialized */ | |
579 | if (!binder_alloc_get_vma(alloc)) { | |
580 | binder_alloc_debug(BINDER_DEBUG_USER_ERROR, | |
581 | "%d: binder_alloc_buf, no vma\n", | |
582 | alloc->pid); | |
583 | return ERR_PTR(-ESRCH); | |
584 | } | |
585 | ||
586 | size = sanitized_size(data_size, offsets_size, extra_buffers_size); | |
587 | if (unlikely(!size)) { | |
588 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, | |
589 | "%d: got transaction with invalid size %zd-%zd-%zd\n", | |
590 | alloc->pid, data_size, offsets_size, | |
591 | extra_buffers_size); | |
592 | return ERR_PTR(-EINVAL); | |
593 | } | |
0c972a05 | 594 | |
c7ac30fa CL |
595 | /* Preallocate the next buffer */ |
596 | next = kzalloc(sizeof(*next), GFP_KERNEL); | |
597 | if (!next) | |
598 | return ERR_PTR(-ENOMEM); | |
599 | ||
0c972a05 | 600 | mutex_lock(&alloc->mutex); |
c7ac30fa | 601 | buffer = binder_alloc_new_buf_locked(alloc, next, size, is_async); |
377e1684 CL |
602 | if (IS_ERR(buffer)) { |
603 | mutex_unlock(&alloc->mutex); | |
604 | goto out; | |
605 | } | |
606 | ||
607 | buffer->data_size = data_size; | |
608 | buffer->offsets_size = offsets_size; | |
609 | buffer->extra_buffers_size = extra_buffers_size; | |
89f71743 | 610 | buffer->pid = current->tgid; |
0c972a05 | 611 | mutex_unlock(&alloc->mutex); |
377e1684 | 612 | |
37ebbb4f CL |
613 | ret = binder_install_buffer_pages(alloc, buffer, size); |
614 | if (ret) { | |
615 | binder_alloc_free_buf(alloc, buffer); | |
616 | buffer = ERR_PTR(ret); | |
617 | } | |
377e1684 | 618 | out: |
0c972a05 TK |
619 | return buffer; |
620 | } | |
621 | ||
df9aabea | 622 | static unsigned long buffer_start_page(struct binder_buffer *buffer) |
0c972a05 | 623 | { |
df9aabea | 624 | return buffer->user_data & PAGE_MASK; |
0c972a05 TK |
625 | } |
626 | ||
df9aabea | 627 | static unsigned long prev_buffer_end_page(struct binder_buffer *buffer) |
0c972a05 | 628 | { |
df9aabea | 629 | return (buffer->user_data - 1) & PAGE_MASK; |
0c972a05 TK |
630 | } |
631 | ||
632 | static void binder_delete_free_buffer(struct binder_alloc *alloc, | |
633 | struct binder_buffer *buffer) | |
634 | { | |
f07b83a4 CL |
635 | struct binder_buffer *prev, *next; |
636 | ||
637 | if (PAGE_ALIGNED(buffer->user_data)) | |
638 | goto skip_freelist; | |
4df9772c | 639 | |
0c972a05 | 640 | BUG_ON(alloc->buffers.next == &buffer->entry); |
e2176219 | 641 | prev = binder_buffer_prev(buffer); |
0c972a05 | 642 | BUG_ON(!prev->free); |
f07b83a4 CL |
643 | if (prev_buffer_end_page(prev) == buffer_start_page(buffer)) |
644 | goto skip_freelist; | |
0c972a05 TK |
645 | |
646 | if (!list_is_last(&buffer->entry, &alloc->buffers)) { | |
e2176219 | 647 | next = binder_buffer_next(buffer); |
f07b83a4 CL |
648 | if (buffer_start_page(next) == buffer_start_page(buffer)) |
649 | goto skip_freelist; | |
0c972a05 | 650 | } |
74310e06 | 651 | |
f07b83a4 CL |
652 | binder_lru_freelist_add(alloc, buffer_start_page(buffer), |
653 | buffer_start_page(buffer) + PAGE_SIZE); | |
654 | skip_freelist: | |
74310e06 SY |
655 | list_del(&buffer->entry); |
656 | kfree(buffer); | |
0c972a05 TK |
657 | } |
658 | ||
659 | static void binder_free_buf_locked(struct binder_alloc *alloc, | |
660 | struct binder_buffer *buffer) | |
661 | { | |
662 | size_t size, buffer_size; | |
663 | ||
664 | buffer_size = binder_alloc_buffer_size(alloc, buffer); | |
665 | ||
666 | size = ALIGN(buffer->data_size, sizeof(void *)) + | |
667 | ALIGN(buffer->offsets_size, sizeof(void *)) + | |
668 | ALIGN(buffer->extra_buffers_size, sizeof(void *)); | |
669 | ||
670 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, | |
671 | "%d: binder_free_buf %pK size %zd buffer_size %zd\n", | |
672 | alloc->pid, buffer, size, buffer_size); | |
673 | ||
674 | BUG_ON(buffer->free); | |
675 | BUG_ON(size > buffer_size); | |
676 | BUG_ON(buffer->transaction != NULL); | |
bde4a19f TK |
677 | BUG_ON(buffer->user_data < alloc->buffer); |
678 | BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size); | |
0c972a05 TK |
679 | |
680 | if (buffer->async_transaction) { | |
c6d05e07 | 681 | alloc->free_async_space += buffer_size; |
0c972a05 TK |
682 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC, |
683 | "%d: binder_free_buf size %zd async free %zd\n", | |
684 | alloc->pid, size, alloc->free_async_space); | |
685 | } | |
686 | ||
ea9cdbf0 CL |
687 | binder_lru_freelist_add(alloc, PAGE_ALIGN(buffer->user_data), |
688 | (buffer->user_data + buffer_size) & PAGE_MASK); | |
0c972a05 TK |
689 | |
690 | rb_erase(&buffer->rb_node, &alloc->allocated_buffers); | |
691 | buffer->free = 1; | |
692 | if (!list_is_last(&buffer->entry, &alloc->buffers)) { | |
e2176219 | 693 | struct binder_buffer *next = binder_buffer_next(buffer); |
0c972a05 TK |
694 | |
695 | if (next->free) { | |
696 | rb_erase(&next->rb_node, &alloc->free_buffers); | |
697 | binder_delete_free_buffer(alloc, next); | |
698 | } | |
699 | } | |
700 | if (alloc->buffers.next != &buffer->entry) { | |
e2176219 | 701 | struct binder_buffer *prev = binder_buffer_prev(buffer); |
0c972a05 TK |
702 | |
703 | if (prev->free) { | |
704 | binder_delete_free_buffer(alloc, buffer); | |
705 | rb_erase(&prev->rb_node, &alloc->free_buffers); | |
706 | buffer = prev; | |
707 | } | |
708 | } | |
709 | binder_insert_free_buffer(alloc, buffer); | |
710 | } | |
711 | ||
cbc174a6 CL |
712 | /** |
713 | * binder_alloc_get_page() - get kernel pointer for given buffer offset | |
714 | * @alloc: binder_alloc for this proc | |
715 | * @buffer: binder buffer to be accessed | |
716 | * @buffer_offset: offset into @buffer data | |
717 | * @pgoffp: address to copy final page offset to | |
718 | * | |
719 | * Lookup the struct page corresponding to the address | |
720 | * at @buffer_offset into @buffer->user_data. If @pgoffp is not | |
721 | * NULL, the byte-offset into the page is written there. | |
722 | * | |
723 | * The caller is responsible to ensure that the offset points | |
724 | * to a valid address within the @buffer and that @buffer is | |
725 | * not freeable by the user. Since it can't be freed, we are | |
726 | * guaranteed that the corresponding elements of @alloc->pages[] | |
727 | * cannot change. | |
728 | * | |
729 | * Return: struct page | |
730 | */ | |
731 | static struct page *binder_alloc_get_page(struct binder_alloc *alloc, | |
732 | struct binder_buffer *buffer, | |
733 | binder_size_t buffer_offset, | |
734 | pgoff_t *pgoffp) | |
735 | { | |
736 | binder_size_t buffer_space_offset = buffer_offset + | |
737 | (buffer->user_data - alloc->buffer); | |
738 | pgoff_t pgoff = buffer_space_offset & ~PAGE_MASK; | |
739 | size_t index = buffer_space_offset >> PAGE_SHIFT; | |
740 | struct binder_lru_page *lru_page; | |
741 | ||
742 | lru_page = &alloc->pages[index]; | |
743 | *pgoffp = pgoff; | |
744 | return lru_page->page_ptr; | |
745 | } | |
746 | ||
747 | /** | |
748 | * binder_alloc_clear_buf() - zero out buffer | |
749 | * @alloc: binder_alloc for this proc | |
750 | * @buffer: binder buffer to be cleared | |
751 | * | |
752 | * memset the given buffer to 0 | |
753 | */ | |
0f966cba | 754 | static void binder_alloc_clear_buf(struct binder_alloc *alloc, |
cbc174a6 CL |
755 | struct binder_buffer *buffer) |
756 | { | |
757 | size_t bytes = binder_alloc_buffer_size(alloc, buffer); | |
758 | binder_size_t buffer_offset = 0; | |
759 | ||
760 | while (bytes) { | |
761 | unsigned long size; | |
762 | struct page *page; | |
763 | pgoff_t pgoff; | |
764 | ||
765 | page = binder_alloc_get_page(alloc, buffer, | |
766 | buffer_offset, &pgoff); | |
767 | size = min_t(size_t, bytes, PAGE_SIZE - pgoff); | |
768 | memset_page(page, pgoff, 0, size); | |
769 | bytes -= size; | |
770 | buffer_offset += size; | |
771 | } | |
772 | } | |
773 | ||
0c972a05 TK |
774 | /** |
775 | * binder_alloc_free_buf() - free a binder buffer | |
776 | * @alloc: binder_alloc for this proc | |
777 | * @buffer: kernel pointer to buffer | |
778 | * | |
4b463822 | 779 | * Free the buffer allocated via binder_alloc_new_buf() |
0c972a05 TK |
780 | */ |
781 | void binder_alloc_free_buf(struct binder_alloc *alloc, | |
782 | struct binder_buffer *buffer) | |
783 | { | |
0f966cba TK |
784 | /* |
785 | * We could eliminate the call to binder_alloc_clear_buf() | |
786 | * from binder_alloc_deferred_release() by moving this to | |
122a3c1c | 787 | * binder_free_buf_locked(). However, that could |
0f966cba TK |
788 | * increase contention for the alloc mutex if clear_on_free |
789 | * is used frequently for large buffers. The mutex is not | |
790 | * needed for correctness here. | |
791 | */ | |
792 | if (buffer->clear_on_free) { | |
793 | binder_alloc_clear_buf(alloc, buffer); | |
794 | buffer->clear_on_free = false; | |
795 | } | |
0c972a05 TK |
796 | mutex_lock(&alloc->mutex); |
797 | binder_free_buf_locked(alloc, buffer); | |
798 | mutex_unlock(&alloc->mutex); | |
799 | } | |
800 | ||
801 | /** | |
802 | * binder_alloc_mmap_handler() - map virtual address space for proc | |
803 | * @alloc: alloc structure for this proc | |
804 | * @vma: vma passed to mmap() | |
805 | * | |
806 | * Called by binder_mmap() to initialize the space specified in | |
807 | * vma for allocating binder buffers | |
808 | * | |
809 | * Return: | |
810 | * 0 = success | |
811 | * -EBUSY = address space already mapped | |
812 | * -ENOMEM = failed to map memory to given address space | |
813 | */ | |
814 | int binder_alloc_mmap_handler(struct binder_alloc *alloc, | |
815 | struct vm_area_struct *vma) | |
816 | { | |
0c972a05 | 817 | struct binder_buffer *buffer; |
68aef12d CL |
818 | const char *failure_string; |
819 | int ret, i; | |
0c972a05 | 820 | |
3ce00bb7 CL |
821 | if (unlikely(vma->vm_mm != alloc->mm)) { |
822 | ret = -EINVAL; | |
823 | failure_string = "invalid vma->vm_mm"; | |
824 | goto err_invalid_mm; | |
825 | } | |
826 | ||
0c972a05 | 827 | mutex_lock(&binder_alloc_mmap_lock); |
a7a74d7f | 828 | if (alloc->buffer_size) { |
0c972a05 TK |
829 | ret = -EBUSY; |
830 | failure_string = "already mapped"; | |
831 | goto err_already_mapped; | |
832 | } | |
a7a74d7f JH |
833 | alloc->buffer_size = min_t(unsigned long, vma->vm_end - vma->vm_start, |
834 | SZ_4M); | |
835 | mutex_unlock(&binder_alloc_mmap_lock); | |
0c972a05 | 836 | |
df9aabea | 837 | alloc->buffer = vma->vm_start; |
0c972a05 | 838 | |
45d02f79 | 839 | alloc->pages = kcalloc(alloc->buffer_size / PAGE_SIZE, |
6396bb22 | 840 | sizeof(alloc->pages[0]), |
0c972a05 TK |
841 | GFP_KERNEL); |
842 | if (alloc->pages == NULL) { | |
843 | ret = -ENOMEM; | |
844 | failure_string = "alloc page array"; | |
845 | goto err_alloc_pages_failed; | |
846 | } | |
0c972a05 | 847 | |
68aef12d CL |
848 | for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { |
849 | alloc->pages[i].alloc = alloc; | |
850 | INIT_LIST_HEAD(&alloc->pages[i].lru); | |
851 | } | |
852 | ||
74310e06 SY |
853 | buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); |
854 | if (!buffer) { | |
0c972a05 | 855 | ret = -ENOMEM; |
74310e06 SY |
856 | failure_string = "alloc buffer struct"; |
857 | goto err_alloc_buf_struct_failed; | |
0c972a05 | 858 | } |
74310e06 | 859 | |
bde4a19f | 860 | buffer->user_data = alloc->buffer; |
0c972a05 TK |
861 | list_add(&buffer->entry, &alloc->buffers); |
862 | buffer->free = 1; | |
863 | binder_insert_free_buffer(alloc, buffer); | |
864 | alloc->free_async_space = alloc->buffer_size / 2; | |
0fa53349 CL |
865 | |
866 | /* Signal binder_alloc is fully initialized */ | |
867 | binder_alloc_set_vma(alloc, vma); | |
0c972a05 TK |
868 | |
869 | return 0; | |
870 | ||
74310e06 | 871 | err_alloc_buf_struct_failed: |
0c972a05 TK |
872 | kfree(alloc->pages); |
873 | alloc->pages = NULL; | |
874 | err_alloc_pages_failed: | |
df9aabea | 875 | alloc->buffer = 0; |
a7a74d7f JH |
876 | mutex_lock(&binder_alloc_mmap_lock); |
877 | alloc->buffer_size = 0; | |
0c972a05 TK |
878 | err_already_mapped: |
879 | mutex_unlock(&binder_alloc_mmap_lock); | |
3ce00bb7 | 880 | err_invalid_mm: |
128f3804 SY |
881 | binder_alloc_debug(BINDER_DEBUG_USER_ERROR, |
882 | "%s: %d %lx-%lx %s failed %d\n", __func__, | |
883 | alloc->pid, vma->vm_start, vma->vm_end, | |
884 | failure_string, ret); | |
0c972a05 TK |
885 | return ret; |
886 | } | |
887 | ||
888 | ||
889 | void binder_alloc_deferred_release(struct binder_alloc *alloc) | |
890 | { | |
891 | struct rb_node *n; | |
892 | int buffers, page_count; | |
74310e06 | 893 | struct binder_buffer *buffer; |
0c972a05 | 894 | |
0c972a05 TK |
895 | buffers = 0; |
896 | mutex_lock(&alloc->mutex); | |
c0fd2101 | 897 | BUG_ON(alloc->vma); |
da1b9564 | 898 | |
0c972a05 | 899 | while ((n = rb_first(&alloc->allocated_buffers))) { |
0c972a05 TK |
900 | buffer = rb_entry(n, struct binder_buffer, rb_node); |
901 | ||
902 | /* Transaction should already have been freed */ | |
903 | BUG_ON(buffer->transaction); | |
904 | ||
0f966cba TK |
905 | if (buffer->clear_on_free) { |
906 | binder_alloc_clear_buf(alloc, buffer); | |
907 | buffer->clear_on_free = false; | |
908 | } | |
0c972a05 TK |
909 | binder_free_buf_locked(alloc, buffer); |
910 | buffers++; | |
911 | } | |
912 | ||
74310e06 SY |
913 | while (!list_empty(&alloc->buffers)) { |
914 | buffer = list_first_entry(&alloc->buffers, | |
915 | struct binder_buffer, entry); | |
916 | WARN_ON(!buffer->free); | |
917 | ||
918 | list_del(&buffer->entry); | |
919 | WARN_ON_ONCE(!list_empty(&alloc->buffers)); | |
920 | kfree(buffer); | |
921 | } | |
922 | ||
0c972a05 TK |
923 | page_count = 0; |
924 | if (alloc->pages) { | |
925 | int i; | |
926 | ||
927 | for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { | |
df9aabea | 928 | unsigned long page_addr; |
f2517eb7 | 929 | bool on_lru; |
0c972a05 | 930 | |
f2517eb7 | 931 | if (!alloc->pages[i].page_ptr) |
0c972a05 TK |
932 | continue; |
933 | ||
ea9cdbf0 | 934 | on_lru = list_lru_del(&binder_freelist, |
f2517eb7 | 935 | &alloc->pages[i].lru); |
0c972a05 TK |
936 | page_addr = alloc->buffer + i * PAGE_SIZE; |
937 | binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC, | |
162c7973 CL |
938 | "%s: %d: page %d %s\n", |
939 | __func__, alloc->pid, i, | |
f2517eb7 | 940 | on_lru ? "on lru" : "active"); |
f2517eb7 | 941 | __free_page(alloc->pages[i].page_ptr); |
0c972a05 TK |
942 | page_count++; |
943 | } | |
944 | kfree(alloc->pages); | |
0c972a05 TK |
945 | } |
946 | mutex_unlock(&alloc->mutex); | |
e66b77e5 CL |
947 | if (alloc->mm) |
948 | mmdrop(alloc->mm); | |
0c972a05 TK |
949 | |
950 | binder_alloc_debug(BINDER_DEBUG_OPEN_CLOSE, | |
951 | "%s: %d buffers %d, pages %d\n", | |
952 | __func__, alloc->pid, buffers, page_count); | |
953 | } | |
954 | ||
0c972a05 TK |
955 | /** |
956 | * binder_alloc_print_allocated() - print buffer info | |
957 | * @m: seq_file for output via seq_printf() | |
958 | * @alloc: binder_alloc for this proc | |
959 | * | |
960 | * Prints information about every buffer associated with | |
961 | * the binder_alloc state to the given seq_file | |
962 | */ | |
963 | void binder_alloc_print_allocated(struct seq_file *m, | |
964 | struct binder_alloc *alloc) | |
965 | { | |
8e905217 | 966 | struct binder_buffer *buffer; |
0c972a05 TK |
967 | struct rb_node *n; |
968 | ||
969 | mutex_lock(&alloc->mutex); | |
8e905217 CL |
970 | for (n = rb_first(&alloc->allocated_buffers); n; n = rb_next(n)) { |
971 | buffer = rb_entry(n, struct binder_buffer, rb_node); | |
972 | seq_printf(m, " buffer %d: %lx size %zd:%zd:%zd %s\n", | |
162c7973 CL |
973 | buffer->debug_id, |
974 | buffer->user_data - alloc->buffer, | |
8e905217 CL |
975 | buffer->data_size, buffer->offsets_size, |
976 | buffer->extra_buffers_size, | |
977 | buffer->transaction ? "active" : "delivered"); | |
978 | } | |
0c972a05 TK |
979 | mutex_unlock(&alloc->mutex); |
980 | } | |
981 | ||
8ef4665a SY |
982 | /** |
983 | * binder_alloc_print_pages() - print page usage | |
984 | * @m: seq_file for output via seq_printf() | |
985 | * @alloc: binder_alloc for this proc | |
986 | */ | |
987 | void binder_alloc_print_pages(struct seq_file *m, | |
988 | struct binder_alloc *alloc) | |
989 | { | |
990 | struct binder_lru_page *page; | |
991 | int i; | |
992 | int active = 0; | |
993 | int lru = 0; | |
994 | int free = 0; | |
995 | ||
996 | mutex_lock(&alloc->mutex); | |
8eb52a1e JH |
997 | /* |
998 | * Make sure the binder_alloc is fully initialized, otherwise we might | |
999 | * read inconsistent state. | |
1000 | */ | |
b15655b1 CL |
1001 | if (binder_alloc_get_vma(alloc) != NULL) { |
1002 | for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { | |
1003 | page = &alloc->pages[i]; | |
1004 | if (!page->page_ptr) | |
1005 | free++; | |
1006 | else if (list_empty(&page->lru)) | |
1007 | active++; | |
1008 | else | |
1009 | lru++; | |
1010 | } | |
44e602b4 | 1011 | } |
8ef4665a SY |
1012 | mutex_unlock(&alloc->mutex); |
1013 | seq_printf(m, " pages: %d:%d:%d\n", active, lru, free); | |
8d9a3ab6 | 1014 | seq_printf(m, " pages high watermark: %zu\n", alloc->pages_high); |
8ef4665a SY |
1015 | } |
1016 | ||
0c972a05 TK |
1017 | /** |
1018 | * binder_alloc_get_allocated_count() - return count of buffers | |
1019 | * @alloc: binder_alloc for this proc | |
1020 | * | |
1021 | * Return: count of allocated buffers | |
1022 | */ | |
1023 | int binder_alloc_get_allocated_count(struct binder_alloc *alloc) | |
1024 | { | |
1025 | struct rb_node *n; | |
1026 | int count = 0; | |
1027 | ||
1028 | mutex_lock(&alloc->mutex); | |
1029 | for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n)) | |
1030 | count++; | |
1031 | mutex_unlock(&alloc->mutex); | |
1032 | return count; | |
1033 | } | |
1034 | ||
1035 | ||
1036 | /** | |
1037 | * binder_alloc_vma_close() - invalidate address space | |
1038 | * @alloc: binder_alloc for this proc | |
1039 | * | |
1040 | * Called from binder_vma_close() when releasing address space. | |
1041 | * Clears alloc->vma to prevent new incoming transactions from | |
1042 | * allocating more buffers. | |
1043 | */ | |
1044 | void binder_alloc_vma_close(struct binder_alloc *alloc) | |
1045 | { | |
0fa53349 | 1046 | binder_alloc_set_vma(alloc, NULL); |
0c972a05 TK |
1047 | } |
1048 | ||
f2517eb7 SY |
1049 | /** |
1050 | * binder_alloc_free_page() - shrinker callback to free pages | |
1051 | * @item: item to free | |
1052 | * @lock: lock protecting the item | |
1053 | * @cb_arg: callback argument | |
1054 | * | |
1055 | * Called from list_lru_walk() in binder_shrink_scan() to free | |
1056 | * up pages when the system is under memory pressure. | |
1057 | */ | |
1058 | enum lru_status binder_alloc_free_page(struct list_head *item, | |
1059 | struct list_lru_one *lru, | |
1060 | spinlock_t *lock, | |
1061 | void *cb_arg) | |
324fa64c | 1062 | __must_hold(lock) |
f2517eb7 SY |
1063 | { |
1064 | struct mm_struct *mm = NULL; | |
1065 | struct binder_lru_page *page = container_of(item, | |
1066 | struct binder_lru_page, | |
1067 | lru); | |
1068 | struct binder_alloc *alloc; | |
df9aabea | 1069 | unsigned long page_addr; |
f2517eb7 | 1070 | size_t index; |
a1b2289c | 1071 | struct vm_area_struct *vma; |
f2517eb7 SY |
1072 | |
1073 | alloc = page->alloc; | |
1074 | if (!mutex_trylock(&alloc->mutex)) | |
1075 | goto err_get_alloc_mutex_failed; | |
1076 | ||
1077 | if (!page->page_ptr) | |
1078 | goto err_page_already_freed; | |
1079 | ||
1080 | index = page - alloc->pages; | |
df9aabea | 1081 | page_addr = alloc->buffer + index * PAGE_SIZE; |
5cec2d2e | 1082 | |
e66b77e5 | 1083 | mm = alloc->mm; |
5cec2d2e TK |
1084 | if (!mmget_not_zero(mm)) |
1085 | goto err_mmget; | |
d8ed45c5 | 1086 | if (!mmap_read_trylock(mm)) |
3e4e28c5 | 1087 | goto err_mmap_read_lock_failed; |
3f489c20 CL |
1088 | vma = vma_lookup(mm, page_addr); |
1089 | if (vma && vma != binder_alloc_get_vma(alloc)) | |
1090 | goto err_invalid_vma; | |
a1b2289c SY |
1091 | |
1092 | list_lru_isolate(lru, item); | |
1093 | spin_unlock(lock); | |
f2517eb7 | 1094 | |
a1b2289c | 1095 | if (vma) { |
e41e164c SY |
1096 | trace_binder_unmap_user_start(alloc, index); |
1097 | ||
e9adcfec | 1098 | zap_page_range_single(vma, page_addr, PAGE_SIZE, NULL); |
f2517eb7 | 1099 | |
e41e164c | 1100 | trace_binder_unmap_user_end(alloc, index); |
f2517eb7 | 1101 | } |
d8ed45c5 | 1102 | mmap_read_unlock(mm); |
f867c771 | 1103 | mmput_async(mm); |
f2517eb7 | 1104 | |
e41e164c SY |
1105 | trace_binder_unmap_kernel_start(alloc, index); |
1106 | ||
f2517eb7 SY |
1107 | __free_page(page->page_ptr); |
1108 | page->page_ptr = NULL; | |
1109 | ||
e41e164c SY |
1110 | trace_binder_unmap_kernel_end(alloc, index); |
1111 | ||
a1b2289c | 1112 | spin_lock(lock); |
f2517eb7 | 1113 | mutex_unlock(&alloc->mutex); |
a1b2289c | 1114 | return LRU_REMOVED_RETRY; |
f2517eb7 | 1115 | |
3f489c20 CL |
1116 | err_invalid_vma: |
1117 | mmap_read_unlock(mm); | |
3e4e28c5 | 1118 | err_mmap_read_lock_failed: |
a1b2289c | 1119 | mmput_async(mm); |
a0c2baaf | 1120 | err_mmget: |
f2517eb7 SY |
1121 | err_page_already_freed: |
1122 | mutex_unlock(&alloc->mutex); | |
1123 | err_get_alloc_mutex_failed: | |
1124 | return LRU_SKIP; | |
1125 | } | |
1126 | ||
1127 | static unsigned long | |
1128 | binder_shrink_count(struct shrinker *shrink, struct shrink_control *sc) | |
1129 | { | |
ea9cdbf0 | 1130 | return list_lru_count(&binder_freelist); |
f2517eb7 SY |
1131 | } |
1132 | ||
1133 | static unsigned long | |
1134 | binder_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) | |
1135 | { | |
ea9cdbf0 | 1136 | return list_lru_walk(&binder_freelist, binder_alloc_free_page, |
f2517eb7 | 1137 | NULL, sc->nr_to_scan); |
f2517eb7 SY |
1138 | } |
1139 | ||
95a542da | 1140 | static struct shrinker *binder_shrinker; |
f2517eb7 | 1141 | |
0c972a05 TK |
1142 | /** |
1143 | * binder_alloc_init() - called by binder_open() for per-proc initialization | |
1144 | * @alloc: binder_alloc for this proc | |
1145 | * | |
1146 | * Called from binder_open() to initialize binder_alloc fields for | |
1147 | * new binder proc | |
1148 | */ | |
1149 | void binder_alloc_init(struct binder_alloc *alloc) | |
1150 | { | |
0c972a05 | 1151 | alloc->pid = current->group_leader->pid; |
e66b77e5 CL |
1152 | alloc->mm = current->mm; |
1153 | mmgrab(alloc->mm); | |
0c972a05 | 1154 | mutex_init(&alloc->mutex); |
957ccc2b | 1155 | INIT_LIST_HEAD(&alloc->buffers); |
0c972a05 TK |
1156 | } |
1157 | ||
533dfb25 | 1158 | int binder_alloc_shrinker_init(void) |
f2517eb7 | 1159 | { |
95a542da | 1160 | int ret; |
533dfb25 | 1161 | |
ea9cdbf0 | 1162 | ret = list_lru_init(&binder_freelist); |
95a542da QZ |
1163 | if (ret) |
1164 | return ret; | |
1165 | ||
1166 | binder_shrinker = shrinker_alloc(0, "android-binder"); | |
1167 | if (!binder_shrinker) { | |
ea9cdbf0 | 1168 | list_lru_destroy(&binder_freelist); |
95a542da | 1169 | return -ENOMEM; |
533dfb25 | 1170 | } |
95a542da QZ |
1171 | |
1172 | binder_shrinker->count_objects = binder_shrink_count; | |
1173 | binder_shrinker->scan_objects = binder_shrink_scan; | |
1174 | ||
1175 | shrinker_register(binder_shrinker); | |
1176 | ||
1177 | return 0; | |
f2517eb7 | 1178 | } |
1a7c3d9b | 1179 | |
adb9743d QZ |
1180 | void binder_alloc_shrinker_exit(void) |
1181 | { | |
95a542da | 1182 | shrinker_free(binder_shrinker); |
ea9cdbf0 | 1183 | list_lru_destroy(&binder_freelist); |
adb9743d QZ |
1184 | } |
1185 | ||
1a7c3d9b TK |
1186 | /** |
1187 | * check_buffer() - verify that buffer/offset is safe to access | |
1188 | * @alloc: binder_alloc for this proc | |
1189 | * @buffer: binder buffer to be accessed | |
1190 | * @offset: offset into @buffer data | |
1191 | * @bytes: bytes to access from offset | |
1192 | * | |
1193 | * Check that the @offset/@bytes are within the size of the given | |
1194 | * @buffer and that the buffer is currently active and not freeable. | |
1195 | * Offsets must also be multiples of sizeof(u32). The kernel is | |
1196 | * allowed to touch the buffer in two cases: | |
1197 | * | |
1198 | * 1) when the buffer is being created: | |
1199 | * (buffer->free == 0 && buffer->allow_user_free == 0) | |
1200 | * 2) when the buffer is being torn down: | |
1201 | * (buffer->free == 0 && buffer->transaction == NULL). | |
1202 | * | |
1203 | * Return: true if the buffer is safe to access | |
1204 | */ | |
1205 | static inline bool check_buffer(struct binder_alloc *alloc, | |
1206 | struct binder_buffer *buffer, | |
1207 | binder_size_t offset, size_t bytes) | |
1208 | { | |
1209 | size_t buffer_size = binder_alloc_buffer_size(alloc, buffer); | |
1210 | ||
1211 | return buffer_size >= bytes && | |
1212 | offset <= buffer_size - bytes && | |
1213 | IS_ALIGNED(offset, sizeof(u32)) && | |
1214 | !buffer->free && | |
1215 | (!buffer->allow_user_free || !buffer->transaction); | |
1216 | } | |
1217 | ||
1a7c3d9b TK |
1218 | /** |
1219 | * binder_alloc_copy_user_to_buffer() - copy src user to tgt user | |
1220 | * @alloc: binder_alloc for this proc | |
1221 | * @buffer: binder buffer to be accessed | |
1222 | * @buffer_offset: offset into @buffer data | |
1223 | * @from: userspace pointer to source buffer | |
1224 | * @bytes: bytes to copy | |
1225 | * | |
1226 | * Copy bytes from source userspace to target buffer. | |
1227 | * | |
1228 | * Return: bytes remaining to be copied | |
1229 | */ | |
1230 | unsigned long | |
1231 | binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc, | |
1232 | struct binder_buffer *buffer, | |
1233 | binder_size_t buffer_offset, | |
1234 | const void __user *from, | |
1235 | size_t bytes) | |
1236 | { | |
1237 | if (!check_buffer(alloc, buffer, buffer_offset, bytes)) | |
1238 | return bytes; | |
1239 | ||
1240 | while (bytes) { | |
1241 | unsigned long size; | |
1242 | unsigned long ret; | |
1243 | struct page *page; | |
1244 | pgoff_t pgoff; | |
1245 | void *kptr; | |
1246 | ||
1247 | page = binder_alloc_get_page(alloc, buffer, | |
1248 | buffer_offset, &pgoff); | |
1249 | size = min_t(size_t, bytes, PAGE_SIZE - pgoff); | |
1d625960 | 1250 | kptr = kmap_local_page(page) + pgoff; |
1a7c3d9b | 1251 | ret = copy_from_user(kptr, from, size); |
1d625960 | 1252 | kunmap_local(kptr); |
1a7c3d9b TK |
1253 | if (ret) |
1254 | return bytes - size + ret; | |
1255 | bytes -= size; | |
1256 | from += size; | |
1257 | buffer_offset += size; | |
1258 | } | |
1259 | return 0; | |
1260 | } | |
8ced0c62 | 1261 | |
bb4a2e48 TK |
1262 | static int binder_alloc_do_buffer_copy(struct binder_alloc *alloc, |
1263 | bool to_buffer, | |
1264 | struct binder_buffer *buffer, | |
1265 | binder_size_t buffer_offset, | |
1266 | void *ptr, | |
1267 | size_t bytes) | |
8ced0c62 TK |
1268 | { |
1269 | /* All copies must be 32-bit aligned and 32-bit size */ | |
bb4a2e48 TK |
1270 | if (!check_buffer(alloc, buffer, buffer_offset, bytes)) |
1271 | return -EINVAL; | |
8ced0c62 TK |
1272 | |
1273 | while (bytes) { | |
1274 | unsigned long size; | |
1275 | struct page *page; | |
1276 | pgoff_t pgoff; | |
8ced0c62 TK |
1277 | |
1278 | page = binder_alloc_get_page(alloc, buffer, | |
1279 | buffer_offset, &pgoff); | |
1280 | size = min_t(size_t, bytes, PAGE_SIZE - pgoff); | |
8ced0c62 | 1281 | if (to_buffer) |
e88a6a8f | 1282 | memcpy_to_page(page, pgoff, ptr, size); |
8ced0c62 | 1283 | else |
e88a6a8f | 1284 | memcpy_from_page(ptr, page, pgoff, size); |
8ced0c62 TK |
1285 | bytes -= size; |
1286 | pgoff = 0; | |
1287 | ptr = ptr + size; | |
1288 | buffer_offset += size; | |
1289 | } | |
bb4a2e48 | 1290 | return 0; |
8ced0c62 TK |
1291 | } |
1292 | ||
bb4a2e48 TK |
1293 | int binder_alloc_copy_to_buffer(struct binder_alloc *alloc, |
1294 | struct binder_buffer *buffer, | |
1295 | binder_size_t buffer_offset, | |
1296 | void *src, | |
1297 | size_t bytes) | |
8ced0c62 | 1298 | { |
bb4a2e48 TK |
1299 | return binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset, |
1300 | src, bytes); | |
8ced0c62 TK |
1301 | } |
1302 | ||
bb4a2e48 TK |
1303 | int binder_alloc_copy_from_buffer(struct binder_alloc *alloc, |
1304 | void *dest, | |
1305 | struct binder_buffer *buffer, | |
1306 | binder_size_t buffer_offset, | |
1307 | size_t bytes) | |
8ced0c62 | 1308 | { |
bb4a2e48 TK |
1309 | return binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset, |
1310 | dest, bytes); | |
8ced0c62 TK |
1311 | } |
1312 |