Commit | Line | Data |
---|---|---|
c30707be RSZ |
1 | /* |
2 | * drivers/staging/android/ion/ion_system_heap.c | |
3 | * | |
4 | * Copyright (C) 2011 Google, Inc. | |
5 | * | |
6 | * This software is licensed under the terms of the GNU General Public | |
7 | * License version 2, as published by the Free Software Foundation, and | |
8 | * may be copied, distributed, and modified under those terms. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | */ | |
16 | ||
bd5d6bda RSZ |
17 | #include <asm/page.h> |
18 | #include <linux/dma-mapping.h> | |
c30707be | 19 | #include <linux/err.h> |
bd5d6bda | 20 | #include <linux/highmem.h> |
c30707be RSZ |
21 | #include <linux/mm.h> |
22 | #include <linux/scatterlist.h> | |
45b17a80 | 23 | #include <linux/seq_file.h> |
c30707be RSZ |
24 | #include <linux/slab.h> |
25 | #include <linux/vmalloc.h> | |
26 | #include "ion.h" | |
27 | #include "ion_priv.h" | |
28 | ||
f63958d8 CC |
29 | static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN | |
30 | __GFP_NORETRY) & ~__GFP_WAIT; | |
31 | static gfp_t low_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN); | |
45b17a80 RSZ |
32 | static const unsigned int orders[] = {8, 4, 0}; |
33 | static const int num_orders = ARRAY_SIZE(orders); | |
34 | static int order_to_index(unsigned int order) | |
35 | { | |
36 | int i; | |
37 | for (i = 0; i < num_orders; i++) | |
38 | if (order == orders[i]) | |
39 | return i; | |
40 | BUG(); | |
41 | return -1; | |
42 | } | |
43 | ||
44 | static unsigned int order_to_size(int order) | |
45 | { | |
46 | return PAGE_SIZE << order; | |
47 | } | |
48 | ||
49 | struct ion_system_heap { | |
50 | struct ion_heap heap; | |
51 | struct ion_page_pool **pools; | |
52 | }; | |
53 | ||
bd5d6bda RSZ |
54 | struct page_info { |
55 | struct page *page; | |
45b17a80 | 56 | unsigned int order; |
bd5d6bda RSZ |
57 | struct list_head list; |
58 | }; | |
59 | ||
45b17a80 RSZ |
60 | static struct page *alloc_buffer_page(struct ion_system_heap *heap, |
61 | struct ion_buffer *buffer, | |
62 | unsigned long order) | |
63 | { | |
64 | bool cached = ion_buffer_cached(buffer); | |
45b17a80 RSZ |
65 | struct ion_page_pool *pool = heap->pools[order_to_index(order)]; |
66 | struct page *page; | |
67 | ||
ee4a4986 | 68 | if (!cached) { |
45b17a80 | 69 | page = ion_page_pool_alloc(pool); |
ee4a4986 RSZ |
70 | } else { |
71 | gfp_t gfp_flags = low_order_gfp_flags; | |
72 | ||
73 | if (order > 4) | |
74 | gfp_flags = high_order_gfp_flags; | |
a3056906 | 75 | page = alloc_pages(gfp_flags, order); |
8fae8312 | 76 | if (!page) |
f63958d8 | 77 | return NULL; |
e946b209 CC |
78 | ion_pages_sync_for_device(NULL, page, PAGE_SIZE << order, |
79 | DMA_BIDIRECTIONAL); | |
ee4a4986 | 80 | } |
45b17a80 | 81 | if (!page) |
f63958d8 | 82 | return NULL; |
8fae8312 | 83 | |
45b17a80 RSZ |
84 | return page; |
85 | } | |
86 | ||
87 | static void free_buffer_page(struct ion_system_heap *heap, | |
88 | struct ion_buffer *buffer, struct page *page, | |
0b6b2cde | 89 | unsigned int order) |
45b17a80 RSZ |
90 | { |
91 | bool cached = ion_buffer_cached(buffer); | |
45b17a80 RSZ |
92 | |
93 | if (!cached) { | |
94 | struct ion_page_pool *pool = heap->pools[order_to_index(order)]; | |
45b17a80 | 95 | ion_page_pool_free(pool, page); |
45b17a80 RSZ |
96 | } else { |
97 | __free_pages(page, order); | |
98 | } | |
99 | } | |
100 | ||
b0599c01 | 101 | |
45b17a80 RSZ |
102 | static struct page_info *alloc_largest_available(struct ion_system_heap *heap, |
103 | struct ion_buffer *buffer, | |
104 | unsigned long size, | |
ba96a2ee | 105 | unsigned int max_order) |
bd5d6bda | 106 | { |
bd5d6bda RSZ |
107 | struct page *page; |
108 | struct page_info *info; | |
109 | int i; | |
110 | ||
f4ea823b JS |
111 | info = kmalloc(sizeof(struct page_info), GFP_KERNEL); |
112 | if (!info) | |
113 | return NULL; | |
114 | ||
45b17a80 RSZ |
115 | for (i = 0; i < num_orders; i++) { |
116 | if (size < order_to_size(orders[i])) | |
bd5d6bda | 117 | continue; |
ba96a2ee RSZ |
118 | if (max_order < orders[i]) |
119 | continue; | |
45b17a80 RSZ |
120 | |
121 | page = alloc_buffer_page(heap, buffer, orders[i]); | |
bd5d6bda RSZ |
122 | if (!page) |
123 | continue; | |
45b17a80 | 124 | |
bd5d6bda RSZ |
125 | info->page = page; |
126 | info->order = orders[i]; | |
c9e8440e | 127 | INIT_LIST_HEAD(&info->list); |
bd5d6bda RSZ |
128 | return info; |
129 | } | |
f4ea823b JS |
130 | kfree(info); |
131 | ||
bd5d6bda RSZ |
132 | return NULL; |
133 | } | |
134 | ||
c30707be RSZ |
135 | static int ion_system_heap_allocate(struct ion_heap *heap, |
136 | struct ion_buffer *buffer, | |
137 | unsigned long size, unsigned long align, | |
138 | unsigned long flags) | |
c30707be | 139 | { |
45b17a80 RSZ |
140 | struct ion_system_heap *sys_heap = container_of(heap, |
141 | struct ion_system_heap, | |
142 | heap); | |
4d5ca329 RSZ |
143 | struct sg_table *table; |
144 | struct scatterlist *sg; | |
bd5d6bda RSZ |
145 | int ret; |
146 | struct list_head pages; | |
147 | struct page_info *info, *tmp_info; | |
13ba7805 | 148 | int i = 0; |
c9e8440e | 149 | unsigned long size_remaining = PAGE_ALIGN(size); |
ba96a2ee RSZ |
150 | unsigned int max_order = orders[0]; |
151 | ||
c13d1df9 CC |
152 | if (align > PAGE_SIZE) |
153 | return -EINVAL; | |
154 | ||
c9e8440e CC |
155 | if (size / PAGE_SIZE > totalram_pages / 2) |
156 | return -ENOMEM; | |
157 | ||
bd5d6bda RSZ |
158 | INIT_LIST_HEAD(&pages); |
159 | while (size_remaining > 0) { | |
e1d855b0 JS |
160 | info = alloc_largest_available(sys_heap, buffer, size_remaining, |
161 | max_order); | |
bd5d6bda RSZ |
162 | if (!info) |
163 | goto err; | |
164 | list_add_tail(&info->list, &pages); | |
165 | size_remaining -= (1 << info->order) * PAGE_SIZE; | |
ba96a2ee | 166 | max_order = info->order; |
13ba7805 | 167 | i++; |
bd5d6bda | 168 | } |
ea725ec8 | 169 | table = kzalloc(sizeof(struct sg_table), GFP_KERNEL); |
4d5ca329 | 170 | if (!table) |
bd5d6bda RSZ |
171 | goto err; |
172 | ||
c13bd1c4 | 173 | ret = sg_alloc_table(table, i, GFP_KERNEL); |
bd5d6bda RSZ |
174 | if (ret) |
175 | goto err1; | |
176 | ||
177 | sg = table->sgl; | |
178 | list_for_each_entry_safe(info, tmp_info, &pages, list) { | |
179 | struct page *page = info->page; | |
c13bd1c4 RSZ |
180 | sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 0); |
181 | sg = sg_next(sg); | |
bd5d6bda | 182 | list_del(&info->list); |
708f0cac | 183 | kfree(info); |
c30707be | 184 | } |
bd5d6bda | 185 | |
b15934b6 RSZ |
186 | buffer->priv_virt = table; |
187 | return 0; | |
4d5ca329 | 188 | err1: |
4d5ca329 | 189 | kfree(table); |
bd5d6bda | 190 | err: |
ea725ec8 | 191 | list_for_each_entry_safe(info, tmp_info, &pages, list) { |
0b6b2cde | 192 | free_buffer_page(sys_heap, buffer, info->page, info->order); |
708f0cac | 193 | kfree(info); |
bd5d6bda | 194 | } |
b15934b6 | 195 | return -ENOMEM; |
c30707be RSZ |
196 | } |
197 | ||
f63958d8 | 198 | static void ion_system_heap_free(struct ion_buffer *buffer) |
c30707be | 199 | { |
45b17a80 RSZ |
200 | struct ion_heap *heap = buffer->heap; |
201 | struct ion_system_heap *sys_heap = container_of(heap, | |
202 | struct ion_system_heap, | |
203 | heap); | |
8898227e | 204 | struct sg_table *table = buffer->sg_table; |
0b6b2cde | 205 | bool cached = ion_buffer_cached(buffer); |
45b17a80 RSZ |
206 | struct scatterlist *sg; |
207 | LIST_HEAD(pages); | |
208 | int i; | |
b15934b6 | 209 | |
0b6b2cde RSZ |
210 | /* uncached pages come from the page pools, zero them before returning |
211 | for security purposes (other allocations are zerod at alloc time */ | |
212 | if (!cached) | |
213 | ion_heap_buffer_zero(buffer); | |
77cbe828 | 214 | |
b15934b6 | 215 | for_each_sg(table->sgl, sg, table->nents, i) |
77cbe828 | 216 | free_buffer_page(sys_heap, buffer, sg_page(sg), |
06e0dcae | 217 | get_order(sg->length)); |
45b17a80 RSZ |
218 | sg_free_table(table); |
219 | kfree(table); | |
c30707be RSZ |
220 | } |
221 | ||
f63958d8 CC |
222 | static struct sg_table *ion_system_heap_map_dma(struct ion_heap *heap, |
223 | struct ion_buffer *buffer) | |
b15934b6 RSZ |
224 | { |
225 | return buffer->priv_virt; | |
226 | } | |
227 | ||
f63958d8 CC |
228 | static void ion_system_heap_unmap_dma(struct ion_heap *heap, |
229 | struct ion_buffer *buffer) | |
b15934b6 RSZ |
230 | { |
231 | return; | |
232 | } | |
233 | ||
45b17a80 | 234 | static struct ion_heap_ops system_heap_ops = { |
c30707be RSZ |
235 | .allocate = ion_system_heap_allocate, |
236 | .free = ion_system_heap_free, | |
237 | .map_dma = ion_system_heap_map_dma, | |
238 | .unmap_dma = ion_system_heap_unmap_dma, | |
8898227e RSZ |
239 | .map_kernel = ion_heap_map_kernel, |
240 | .unmap_kernel = ion_heap_unmap_kernel, | |
241 | .map_user = ion_heap_map_user, | |
c30707be RSZ |
242 | }; |
243 | ||
b1aced6f JS |
244 | static unsigned long ion_system_heap_shrink_count(struct shrinker *shrinker, |
245 | struct shrink_control *sc) | |
246 | { | |
ea313b5f RSZ |
247 | struct ion_heap *heap = container_of(shrinker, struct ion_heap, |
248 | shrinker); | |
249 | struct ion_system_heap *sys_heap = container_of(heap, | |
250 | struct ion_system_heap, | |
251 | heap); | |
252 | int nr_total = 0; | |
b1aced6f JS |
253 | int i; |
254 | ||
255 | /* total number of items is whatever the page pools are holding | |
256 | plus whatever's in the freelist */ | |
257 | for (i = 0; i < num_orders; i++) { | |
258 | struct ion_page_pool *pool = sys_heap->pools[i]; | |
259 | nr_total += ion_page_pool_shrink(pool, sc->gfp_mask, 0); | |
260 | } | |
261 | nr_total += ion_heap_freelist_size(heap) / PAGE_SIZE; | |
262 | return nr_total; | |
263 | ||
264 | } | |
265 | ||
266 | static unsigned long ion_system_heap_shrink_scan(struct shrinker *shrinker, | |
267 | struct shrink_control *sc) | |
268 | { | |
269 | ||
270 | struct ion_heap *heap = container_of(shrinker, struct ion_heap, | |
271 | shrinker); | |
272 | struct ion_system_heap *sys_heap = container_of(heap, | |
273 | struct ion_system_heap, | |
274 | heap); | |
ea313b5f RSZ |
275 | int nr_freed = 0; |
276 | int i; | |
277 | ||
278 | if (sc->nr_to_scan == 0) | |
279 | goto end; | |
280 | ||
281 | /* shrink the free list first, no point in zeroing the memory if | |
282 | we're just going to reclaim it */ | |
283 | nr_freed += ion_heap_freelist_drain(heap, sc->nr_to_scan * PAGE_SIZE) / | |
284 | PAGE_SIZE; | |
285 | ||
286 | if (nr_freed >= sc->nr_to_scan) | |
287 | goto end; | |
288 | ||
289 | for (i = 0; i < num_orders; i++) { | |
290 | struct ion_page_pool *pool = sys_heap->pools[i]; | |
291 | ||
292 | nr_freed += ion_page_pool_shrink(pool, sc->gfp_mask, | |
293 | sc->nr_to_scan); | |
294 | if (nr_freed >= sc->nr_to_scan) | |
295 | break; | |
296 | } | |
297 | ||
298 | end: | |
b1aced6f | 299 | return nr_freed; |
ea313b5f RSZ |
300 | |
301 | } | |
302 | ||
45b17a80 RSZ |
303 | static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s, |
304 | void *unused) | |
305 | { | |
306 | ||
307 | struct ion_system_heap *sys_heap = container_of(heap, | |
308 | struct ion_system_heap, | |
309 | heap); | |
310 | int i; | |
311 | for (i = 0; i < num_orders; i++) { | |
312 | struct ion_page_pool *pool = sys_heap->pools[i]; | |
0fb9b815 RSZ |
313 | seq_printf(s, "%d order %u highmem pages in pool = %lu total\n", |
314 | pool->high_count, pool->order, | |
315 | (1 << pool->order) * PAGE_SIZE * pool->high_count); | |
316 | seq_printf(s, "%d order %u lowmem pages in pool = %lu total\n", | |
317 | pool->low_count, pool->order, | |
318 | (1 << pool->order) * PAGE_SIZE * pool->low_count); | |
45b17a80 RSZ |
319 | } |
320 | return 0; | |
321 | } | |
322 | ||
c30707be RSZ |
323 | struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused) |
324 | { | |
45b17a80 RSZ |
325 | struct ion_system_heap *heap; |
326 | int i; | |
c30707be | 327 | |
45b17a80 | 328 | heap = kzalloc(sizeof(struct ion_system_heap), GFP_KERNEL); |
c30707be RSZ |
329 | if (!heap) |
330 | return ERR_PTR(-ENOMEM); | |
45b17a80 RSZ |
331 | heap->heap.ops = &system_heap_ops; |
332 | heap->heap.type = ION_HEAP_TYPE_SYSTEM; | |
fe2faea7 | 333 | heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE; |
45b17a80 RSZ |
334 | heap->pools = kzalloc(sizeof(struct ion_page_pool *) * num_orders, |
335 | GFP_KERNEL); | |
336 | if (!heap->pools) | |
337 | goto err_alloc_pools; | |
338 | for (i = 0; i < num_orders; i++) { | |
339 | struct ion_page_pool *pool; | |
ee4a4986 RSZ |
340 | gfp_t gfp_flags = low_order_gfp_flags; |
341 | ||
342 | if (orders[i] > 4) | |
343 | gfp_flags = high_order_gfp_flags; | |
344 | pool = ion_page_pool_create(gfp_flags, orders[i]); | |
45b17a80 RSZ |
345 | if (!pool) |
346 | goto err_create_pool; | |
347 | heap->pools[i] = pool; | |
348 | } | |
ea313b5f | 349 | |
b1aced6f JS |
350 | heap->heap.shrinker.scan_objects = ion_system_heap_shrink_scan; |
351 | heap->heap.shrinker.count_objects = ion_system_heap_shrink_count; | |
ea313b5f RSZ |
352 | heap->heap.shrinker.seeks = DEFAULT_SEEKS; |
353 | heap->heap.shrinker.batch = 0; | |
354 | register_shrinker(&heap->heap.shrinker); | |
45b17a80 RSZ |
355 | heap->heap.debug_show = ion_system_heap_debug_show; |
356 | return &heap->heap; | |
357 | err_create_pool: | |
358 | for (i = 0; i < num_orders; i++) | |
359 | if (heap->pools[i]) | |
360 | ion_page_pool_destroy(heap->pools[i]); | |
361 | kfree(heap->pools); | |
362 | err_alloc_pools: | |
363 | kfree(heap); | |
364 | return ERR_PTR(-ENOMEM); | |
c30707be RSZ |
365 | } |
366 | ||
367 | void ion_system_heap_destroy(struct ion_heap *heap) | |
368 | { | |
45b17a80 RSZ |
369 | struct ion_system_heap *sys_heap = container_of(heap, |
370 | struct ion_system_heap, | |
371 | heap); | |
372 | int i; | |
373 | ||
374 | for (i = 0; i < num_orders; i++) | |
375 | ion_page_pool_destroy(sys_heap->pools[i]); | |
376 | kfree(sys_heap->pools); | |
377 | kfree(sys_heap); | |
c30707be RSZ |
378 | } |
379 | ||
380 | static int ion_system_contig_heap_allocate(struct ion_heap *heap, | |
381 | struct ion_buffer *buffer, | |
382 | unsigned long len, | |
383 | unsigned long align, | |
384 | unsigned long flags) | |
385 | { | |
c13d1df9 | 386 | int order = get_order(len); |
5c6a4705 CC |
387 | struct page *page; |
388 | struct sg_table *table; | |
389 | unsigned long i; | |
390 | int ret; | |
c13d1df9 CC |
391 | |
392 | if (align > (PAGE_SIZE << order)) | |
393 | return -EINVAL; | |
394 | ||
5c6a4705 CC |
395 | page = alloc_pages(low_order_gfp_flags, order); |
396 | if (!page) | |
c30707be | 397 | return -ENOMEM; |
5c6a4705 CC |
398 | |
399 | split_page(page, order); | |
400 | ||
401 | len = PAGE_ALIGN(len); | |
402 | for (i = len >> PAGE_SHIFT; i < (1 << order); i++) | |
403 | __free_page(page + i); | |
404 | ||
405 | table = kzalloc(sizeof(struct sg_table), GFP_KERNEL); | |
406 | if (!table) { | |
407 | ret = -ENOMEM; | |
408 | goto out; | |
409 | } | |
410 | ||
411 | ret = sg_alloc_table(table, 1, GFP_KERNEL); | |
412 | if (ret) | |
413 | goto out; | |
414 | ||
415 | sg_set_page(table->sgl, page, len, 0); | |
416 | ||
417 | buffer->priv_virt = table; | |
418 | ||
419 | ion_pages_sync_for_device(NULL, page, len, DMA_BIDIRECTIONAL); | |
420 | ||
c30707be | 421 | return 0; |
5c6a4705 CC |
422 | |
423 | out: | |
424 | for (i = 0; i < len >> PAGE_SHIFT; i++) | |
425 | __free_page(page + i); | |
426 | kfree(table); | |
427 | return ret; | |
c30707be RSZ |
428 | } |
429 | ||
f63958d8 | 430 | static void ion_system_contig_heap_free(struct ion_buffer *buffer) |
c30707be | 431 | { |
5c6a4705 CC |
432 | struct sg_table *table = buffer->priv_virt; |
433 | struct page *page = sg_page(table->sgl); | |
434 | unsigned long pages = PAGE_ALIGN(buffer->size) >> PAGE_SHIFT; | |
435 | unsigned long i; | |
436 | ||
437 | for (i = 0; i < pages; i++) | |
438 | __free_page(page + i); | |
439 | sg_free_table(table); | |
440 | kfree(table); | |
c30707be RSZ |
441 | } |
442 | ||
443 | static int ion_system_contig_heap_phys(struct ion_heap *heap, | |
444 | struct ion_buffer *buffer, | |
445 | ion_phys_addr_t *addr, size_t *len) | |
446 | { | |
5c6a4705 CC |
447 | struct sg_table *table = buffer->priv_virt; |
448 | struct page *page = sg_page(table->sgl); | |
449 | *addr = page_to_phys(page); | |
c30707be RSZ |
450 | *len = buffer->size; |
451 | return 0; | |
452 | } | |
453 | ||
f63958d8 | 454 | static struct sg_table *ion_system_contig_heap_map_dma(struct ion_heap *heap, |
56a7c185 | 455 | struct ion_buffer *buffer) |
c30707be | 456 | { |
5c6a4705 | 457 | return buffer->priv_virt; |
c30707be RSZ |
458 | } |
459 | ||
f63958d8 CC |
460 | static void ion_system_contig_heap_unmap_dma(struct ion_heap *heap, |
461 | struct ion_buffer *buffer) | |
56a7c185 | 462 | { |
56a7c185 RSZ |
463 | } |
464 | ||
c30707be RSZ |
465 | static struct ion_heap_ops kmalloc_ops = { |
466 | .allocate = ion_system_contig_heap_allocate, | |
467 | .free = ion_system_contig_heap_free, | |
468 | .phys = ion_system_contig_heap_phys, | |
469 | .map_dma = ion_system_contig_heap_map_dma, | |
56a7c185 | 470 | .unmap_dma = ion_system_contig_heap_unmap_dma, |
8898227e RSZ |
471 | .map_kernel = ion_heap_map_kernel, |
472 | .unmap_kernel = ion_heap_unmap_kernel, | |
a82130f4 | 473 | .map_user = ion_heap_map_user, |
c30707be RSZ |
474 | }; |
475 | ||
476 | struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *unused) | |
477 | { | |
478 | struct ion_heap *heap; | |
479 | ||
480 | heap = kzalloc(sizeof(struct ion_heap), GFP_KERNEL); | |
481 | if (!heap) | |
482 | return ERR_PTR(-ENOMEM); | |
483 | heap->ops = &kmalloc_ops; | |
484 | heap->type = ION_HEAP_TYPE_SYSTEM_CONTIG; | |
485 | return heap; | |
486 | } | |
487 | ||
488 | void ion_system_contig_heap_destroy(struct ion_heap *heap) | |
489 | { | |
490 | kfree(heap); | |
491 | } | |
492 |