Commit | Line | Data |
---|---|---|
1297bf2e | 1 | /* SPDX-License-Identifier: GPL-2.0 OR MIT */ |
ba4e7d97 TH |
2 | /************************************************************************** |
3 | * | |
4 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA | |
5 | * All Rights Reserved. | |
6 | * | |
7 | * Permission is hereby granted, free of charge, to any person obtaining a | |
8 | * copy of this software and associated documentation files (the | |
9 | * "Software"), to deal in the Software without restriction, including | |
10 | * without limitation the rights to use, copy, modify, merge, publish, | |
11 | * distribute, sub license, and/or sell copies of the Software, and to | |
12 | * permit persons to whom the Software is furnished to do so, subject to | |
13 | * the following conditions: | |
14 | * | |
15 | * The above copyright notice and this permission notice (including the | |
16 | * next paragraph) shall be included in all copies or substantial portions | |
17 | * of the Software. | |
18 | * | |
19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
20 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
21 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
22 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
23 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
24 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
25 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
26 | * | |
27 | **************************************************************************/ | |
28 | /* | |
29 | * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> | |
30 | */ | |
31 | ||
25d0479a JP |
32 | #define pr_fmt(fmt) "[TTM] " fmt |
33 | ||
ba4e7d97 | 34 | #include <linux/sched.h> |
ba4e7d97 | 35 | #include <linux/pagemap.h> |
3142b651 | 36 | #include <linux/shmem_fs.h> |
ba4e7d97 | 37 | #include <linux/file.h> |
760285e7 | 38 | #include <drm/drm_cache.h> |
760285e7 | 39 | #include <drm/ttm/ttm_bo_driver.h> |
ba4e7d97 | 40 | |
78616b88 | 41 | /* |
97b7e1b8 CK |
42 | * Allocates a ttm structure for the given BO. |
43 | */ | |
44 | int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) | |
45 | { | |
46 | struct ttm_bo_device *bdev = bo->bdev; | |
97b7e1b8 CK |
47 | uint32_t page_flags = 0; |
48 | ||
52791eee | 49 | dma_resv_assert_held(bo->base.resv); |
97b7e1b8 | 50 | |
0b062865 CK |
51 | if (bo->ttm) |
52 | return 0; | |
53 | ||
97b7e1b8 CK |
54 | switch (bo->type) { |
55 | case ttm_bo_type_device: | |
56 | if (zero_alloc) | |
57 | page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; | |
45a9d154 | 58 | break; |
97b7e1b8 | 59 | case ttm_bo_type_kernel: |
97b7e1b8 CK |
60 | break; |
61 | case ttm_bo_type_sg: | |
45a9d154 | 62 | page_flags |= TTM_PAGE_FLAG_SG; |
97b7e1b8 CK |
63 | break; |
64 | default: | |
65 | pr_err("Illegal buffer object type\n"); | |
45a9d154 | 66 | return -EINVAL; |
97b7e1b8 CK |
67 | } |
68 | ||
dde5da23 | 69 | bo->ttm = bdev->driver->ttm_tt_create(bo, page_flags); |
45a9d154 CK |
70 | if (unlikely(bo->ttm == NULL)) |
71 | return -ENOMEM; | |
72 | ||
45a9d154 | 73 | return 0; |
97b7e1b8 CK |
74 | } |
75 | ||
78616b88 | 76 | /* |
ba4e7d97 | 77 | * Allocates storage for pointers to the pages that back the ttm. |
ba4e7d97 | 78 | */ |
5b4262d7 | 79 | static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm) |
ba4e7d97 | 80 | { |
2098105e MH |
81 | ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*), |
82 | GFP_KERNEL | __GFP_ZERO); | |
5b4262d7 TSD |
83 | if (!ttm->pages) |
84 | return -ENOMEM; | |
85 | return 0; | |
ba4e7d97 TH |
86 | } |
87 | ||
e34b8fee | 88 | static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm) |
ba4e7d97 | 89 | { |
e34b8fee CK |
90 | ttm->pages = kvmalloc_array(ttm->num_pages, |
91 | sizeof(*ttm->pages) + | |
92 | sizeof(*ttm->dma_address), | |
93 | GFP_KERNEL | __GFP_ZERO); | |
94 | if (!ttm->pages) | |
5b4262d7 | 95 | return -ENOMEM; |
e34b8fee CK |
96 | |
97 | ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); | |
5b4262d7 | 98 | return 0; |
ba4e7d97 TH |
99 | } |
100 | ||
e34b8fee | 101 | static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm) |
75a57669 | 102 | { |
e34b8fee | 103 | ttm->dma_address = kvmalloc_array(ttm->num_pages, |
75a57669 CK |
104 | sizeof(*ttm->dma_address), |
105 | GFP_KERNEL | __GFP_ZERO); | |
106 | if (!ttm->dma_address) | |
107 | return -ENOMEM; | |
108 | return 0; | |
109 | } | |
110 | ||
7626168f | 111 | void ttm_tt_destroy_common(struct ttm_bo_device *bdev, struct ttm_tt *ttm) |
ba4e7d97 | 112 | { |
7eec9151 | 113 | ttm_tt_unpopulate(bdev, ttm); |
ba4e7d97 | 114 | |
ab861424 | 115 | if (ttm->swap_storage) |
ba4e7d97 TH |
116 | fput(ttm->swap_storage); |
117 | ||
649bf3ca | 118 | ttm->swap_storage = NULL; |
7626168f DA |
119 | } |
120 | EXPORT_SYMBOL(ttm_tt_destroy_common); | |
121 | ||
122 | void ttm_tt_destroy(struct ttm_bo_device *bdev, struct ttm_tt *ttm) | |
123 | { | |
04e89ff3 | 124 | bdev->driver->ttm_tt_destroy(bdev, ttm); |
ba4e7d97 TH |
125 | } |
126 | ||
2869e82e CK |
127 | static void ttm_tt_init_fields(struct ttm_tt *ttm, |
128 | struct ttm_buffer_object *bo, | |
1b4ea4c5 CK |
129 | uint32_t page_flags, |
130 | enum ttm_caching caching) | |
ba4e7d97 | 131 | { |
dde5da23 | 132 | ttm->num_pages = bo->num_pages; |
1b4ea4c5 | 133 | ttm->caching = ttm_cached; |
ba4e7d97 | 134 | ttm->page_flags = page_flags; |
e34b8fee | 135 | ttm->dma_address = NULL; |
dea7e0ac | 136 | ttm->swap_storage = NULL; |
536bbeba | 137 | ttm->sg = bo->sg; |
1b4ea4c5 | 138 | ttm->caching = caching; |
75a57669 CK |
139 | } |
140 | ||
dde5da23 | 141 | int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, |
1b4ea4c5 | 142 | uint32_t page_flags, enum ttm_caching caching) |
75a57669 | 143 | { |
1b4ea4c5 | 144 | ttm_tt_init_fields(ttm, bo, page_flags, caching); |
ba4e7d97 | 145 | |
5b4262d7 | 146 | if (ttm_tt_alloc_page_directory(ttm)) { |
25d0479a | 147 | pr_err("Failed allocating page table\n"); |
649bf3ca | 148 | return -ENOMEM; |
ba4e7d97 | 149 | } |
649bf3ca | 150 | return 0; |
ba4e7d97 | 151 | } |
649bf3ca | 152 | EXPORT_SYMBOL(ttm_tt_init); |
ba4e7d97 | 153 | |
8e7e7052 JG |
154 | void ttm_tt_fini(struct ttm_tt *ttm) |
155 | { | |
e34b8fee CK |
156 | if (ttm->pages) |
157 | kvfree(ttm->pages); | |
158 | else | |
159 | kvfree(ttm->dma_address); | |
8e7e7052 | 160 | ttm->pages = NULL; |
e34b8fee | 161 | ttm->dma_address = NULL; |
8e7e7052 JG |
162 | } |
163 | EXPORT_SYMBOL(ttm_tt_fini); | |
164 | ||
e34b8fee | 165 | int ttm_dma_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, |
1b4ea4c5 | 166 | uint32_t page_flags, enum ttm_caching caching) |
8e7e7052 | 167 | { |
1b4ea4c5 | 168 | ttm_tt_init_fields(ttm, bo, page_flags, caching); |
8e7e7052 | 169 | |
e34b8fee | 170 | if (ttm_dma_tt_alloc_page_directory(ttm)) { |
25d0479a | 171 | pr_err("Failed allocating page table\n"); |
8e7e7052 JG |
172 | return -ENOMEM; |
173 | } | |
174 | return 0; | |
175 | } | |
176 | EXPORT_SYMBOL(ttm_dma_tt_init); | |
177 | ||
e34b8fee | 178 | int ttm_sg_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, |
1b4ea4c5 | 179 | uint32_t page_flags, enum ttm_caching caching) |
75a57669 | 180 | { |
75a57669 CK |
181 | int ret; |
182 | ||
1b4ea4c5 | 183 | ttm_tt_init_fields(ttm, bo, page_flags, caching); |
75a57669 | 184 | |
75a57669 | 185 | if (page_flags & TTM_PAGE_FLAG_SG) |
e34b8fee | 186 | ret = ttm_sg_tt_alloc_page_directory(ttm); |
75a57669 | 187 | else |
e34b8fee | 188 | ret = ttm_dma_tt_alloc_page_directory(ttm); |
75a57669 | 189 | if (ret) { |
75a57669 CK |
190 | pr_err("Failed allocating page table\n"); |
191 | return -ENOMEM; | |
192 | } | |
193 | return 0; | |
194 | } | |
195 | EXPORT_SYMBOL(ttm_sg_tt_init); | |
196 | ||
b1e5f172 | 197 | int ttm_tt_swapin(struct ttm_tt *ttm) |
ba4e7d97 TH |
198 | { |
199 | struct address_space *swap_space; | |
200 | struct file *swap_storage; | |
201 | struct page *from_page; | |
202 | struct page *to_page; | |
ab861424 CK |
203 | gfp_t gfp_mask; |
204 | int i, ret; | |
ba4e7d97 | 205 | |
ba4e7d97 TH |
206 | swap_storage = ttm->swap_storage; |
207 | BUG_ON(swap_storage == NULL); | |
208 | ||
93c76a3d | 209 | swap_space = swap_storage->f_mapping; |
ab861424 | 210 | gfp_mask = mapping_gfp_mask(swap_space); |
ba4e7d97 TH |
211 | |
212 | for (i = 0; i < ttm->num_pages; ++i) { | |
ab861424 CK |
213 | from_page = shmem_read_mapping_page_gfp(swap_space, i, |
214 | gfp_mask); | |
290e5505 MM |
215 | if (IS_ERR(from_page)) { |
216 | ret = PTR_ERR(from_page); | |
ba4e7d97 | 217 | goto out_err; |
290e5505 | 218 | } |
b1e5f172 | 219 | to_page = ttm->pages[i]; |
ab861424 CK |
220 | if (unlikely(to_page == NULL)) { |
221 | ret = -ENOMEM; | |
ba4e7d97 | 222 | goto out_err; |
ab861424 | 223 | } |
ba4e7d97 | 224 | |
259a290e | 225 | copy_highpage(to_page, from_page); |
09cbfeaf | 226 | put_page(from_page); |
ba4e7d97 TH |
227 | } |
228 | ||
ab861424 | 229 | fput(swap_storage); |
ba4e7d97 TH |
230 | ttm->swap_storage = NULL; |
231 | ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; | |
232 | ||
233 | return 0; | |
ab861424 | 234 | |
ba4e7d97 | 235 | out_err: |
290e5505 | 236 | return ret; |
ba4e7d97 TH |
237 | } |
238 | ||
ab861424 | 239 | int ttm_tt_swapout(struct ttm_bo_device *bdev, struct ttm_tt *ttm) |
ba4e7d97 TH |
240 | { |
241 | struct address_space *swap_space; | |
242 | struct file *swap_storage; | |
243 | struct page *from_page; | |
244 | struct page *to_page; | |
ab861424 CK |
245 | gfp_t gfp_mask; |
246 | int i, ret; | |
247 | ||
248 | swap_storage = shmem_file_setup("ttm swap", | |
249 | ttm->num_pages << PAGE_SHIFT, | |
250 | 0); | |
251 | if (IS_ERR(swap_storage)) { | |
252 | pr_err("Failed allocating swap storage\n"); | |
253 | return PTR_ERR(swap_storage); | |
ddde985c | 254 | } |
ba4e7d97 | 255 | |
93c76a3d | 256 | swap_space = swap_storage->f_mapping; |
ab861424 | 257 | gfp_mask = mapping_gfp_mask(swap_space); |
ba4e7d97 TH |
258 | |
259 | for (i = 0; i < ttm->num_pages; ++i) { | |
260 | from_page = ttm->pages[i]; | |
261 | if (unlikely(from_page == NULL)) | |
262 | continue; | |
cb5f1a52 AG |
263 | |
264 | to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); | |
55579cfe | 265 | if (IS_ERR(to_page)) { |
290e5505 | 266 | ret = PTR_ERR(to_page); |
ba4e7d97 | 267 | goto out_err; |
290e5505 | 268 | } |
259a290e | 269 | copy_highpage(to_page, from_page); |
ba4e7d97 TH |
270 | set_page_dirty(to_page); |
271 | mark_page_accessed(to_page); | |
09cbfeaf | 272 | put_page(to_page); |
ba4e7d97 TH |
273 | } |
274 | ||
0a667b50 | 275 | ttm_tt_unpopulate(bdev, ttm); |
ba4e7d97 TH |
276 | ttm->swap_storage = swap_storage; |
277 | ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; | |
ba4e7d97 TH |
278 | |
279 | return 0; | |
ab861424 | 280 | |
ba4e7d97 | 281 | out_err: |
ab861424 | 282 | fput(swap_storage); |
ba4e7d97 | 283 | |
290e5505 | 284 | return ret; |
ba4e7d97 | 285 | } |
58aa6622 | 286 | |
0a667b50 | 287 | static void ttm_tt_add_mapping(struct ttm_bo_device *bdev, struct ttm_tt *ttm) |
ec929370 CK |
288 | { |
289 | pgoff_t i; | |
290 | ||
291 | if (ttm->page_flags & TTM_PAGE_FLAG_SG) | |
292 | return; | |
293 | ||
294 | for (i = 0; i < ttm->num_pages; ++i) | |
0a667b50 | 295 | ttm->pages[i]->mapping = bdev->dev_mapping; |
ec929370 CK |
296 | } |
297 | ||
0a667b50 DA |
298 | int ttm_tt_populate(struct ttm_bo_device *bdev, |
299 | struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) | |
25893a14 | 300 | { |
ec929370 CK |
301 | int ret; |
302 | ||
2040ec97 DA |
303 | if (!ttm) |
304 | return -EINVAL; | |
305 | ||
7eec9151 | 306 | if (ttm_tt_is_populated(ttm)) |
25893a14 CK |
307 | return 0; |
308 | ||
0a667b50 DA |
309 | if (bdev->driver->ttm_tt_populate) |
310 | ret = bdev->driver->ttm_tt_populate(bdev, ttm, ctx); | |
e44fcf71 | 311 | else |
ee5d2a8e | 312 | ret = ttm_pool_alloc(&bdev->pool, ttm, ctx); |
d1cb1f25 CK |
313 | if (ret) |
314 | return ret; | |
315 | ||
316 | ttm_tt_add_mapping(bdev, ttm); | |
317 | ttm->page_flags |= TTM_PAGE_FLAG_PRIV_POPULATED; | |
05f8d250 CK |
318 | if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { |
319 | ret = ttm_tt_swapin(ttm); | |
320 | if (unlikely(ret != 0)) { | |
321 | ttm_tt_unpopulate(bdev, ttm); | |
322 | return ret; | |
323 | } | |
324 | } | |
325 | ||
d1cb1f25 | 326 | return 0; |
25893a14 | 327 | } |
2040ec97 | 328 | EXPORT_SYMBOL(ttm_tt_populate); |
25893a14 | 329 | |
58aa6622 TH |
330 | static void ttm_tt_clear_mapping(struct ttm_tt *ttm) |
331 | { | |
332 | pgoff_t i; | |
333 | struct page **page = ttm->pages; | |
334 | ||
1b76af5c TH |
335 | if (ttm->page_flags & TTM_PAGE_FLAG_SG) |
336 | return; | |
337 | ||
58aa6622 TH |
338 | for (i = 0; i < ttm->num_pages; ++i) { |
339 | (*page)->mapping = NULL; | |
340 | (*page++)->index = 0; | |
341 | } | |
342 | } | |
343 | ||
0a667b50 DA |
344 | void ttm_tt_unpopulate(struct ttm_bo_device *bdev, |
345 | struct ttm_tt *ttm) | |
58aa6622 | 346 | { |
7eec9151 | 347 | if (!ttm_tt_is_populated(ttm)) |
58aa6622 TH |
348 | return; |
349 | ||
350 | ttm_tt_clear_mapping(ttm); | |
0a667b50 DA |
351 | if (bdev->driver->ttm_tt_unpopulate) |
352 | bdev->driver->ttm_tt_unpopulate(bdev, ttm); | |
e44fcf71 | 353 | else |
ee5d2a8e | 354 | ttm_pool_free(&bdev->pool, ttm); |
d1cb1f25 | 355 | ttm->page_flags &= ~TTM_PAGE_FLAG_PRIV_POPULATED; |
58aa6622 | 356 | } |