Commit | Line | Data |
---|---|---|
dff96888 | 1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
fb1d9738 JB |
2 | /************************************************************************** |
3 | * | |
dff96888 | 4 | * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA |
fb1d9738 JB |
5 | * |
6 | * Permission is hereby granted, free of charge, to any person obtaining a | |
7 | * copy of this software and associated documentation files (the | |
8 | * "Software"), to deal in the Software without restriction, including | |
9 | * without limitation the rights to use, copy, modify, merge, publish, | |
10 | * distribute, sub license, and/or sell copies of the Software, and to | |
11 | * permit persons to whom the Software is furnished to do so, subject to | |
12 | * the following conditions: | |
13 | * | |
14 | * The above copyright notice and this permission notice (including the | |
15 | * next paragraph) shall be included in all copies or substantial portions | |
16 | * of the Software. | |
17 | * | |
18 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
19 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
20 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
21 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
22 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
23 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
24 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
25 | * | |
26 | **************************************************************************/ | |
27 | ||
28 | #include "vmwgfx_drv.h" | |
760285e7 DH |
29 | #include <drm/ttm/ttm_bo_driver.h> |
30 | #include <drm/ttm/ttm_placement.h> | |
31 | #include <drm/ttm/ttm_page_alloc.h> | |
fb1d9738 | 32 | |
9036f8c7 | 33 | static const struct ttm_place vram_placement_flags = { |
f1217ed0 CK |
34 | .fpfn = 0, |
35 | .lpfn = 0, | |
48e07c23 | 36 | .mem_type = TTM_PL_VRAM, |
ce65b874 | 37 | .flags = 0 |
f1217ed0 | 38 | }; |
fb1d9738 | 39 | |
9036f8c7 | 40 | static const struct ttm_place sys_placement_flags = { |
f1217ed0 CK |
41 | .fpfn = 0, |
42 | .lpfn = 0, | |
48e07c23 | 43 | .mem_type = TTM_PL_SYSTEM, |
ce65b874 | 44 | .flags = 0 |
f1217ed0 | 45 | }; |
3530bdc3 | 46 | |
9036f8c7 | 47 | static const struct ttm_place gmr_placement_flags = { |
f1217ed0 CK |
48 | .fpfn = 0, |
49 | .lpfn = 0, | |
48e07c23 | 50 | .mem_type = VMW_PL_GMR, |
ce65b874 | 51 | .flags = 0 |
f1217ed0 | 52 | }; |
d991ef03 | 53 | |
9036f8c7 | 54 | static const struct ttm_place mob_placement_flags = { |
fb1d9738 JB |
55 | .fpfn = 0, |
56 | .lpfn = 0, | |
48e07c23 | 57 | .mem_type = VMW_PL_MOB, |
ce65b874 | 58 | .flags = 0 |
f1217ed0 CK |
59 | }; |
60 | ||
61 | struct ttm_placement vmw_vram_placement = { | |
fb1d9738 JB |
62 | .num_placement = 1, |
63 | .placement = &vram_placement_flags, | |
64 | .num_busy_placement = 1, | |
65 | .busy_placement = &vram_placement_flags | |
66 | }; | |
67 | ||
9036f8c7 | 68 | static const struct ttm_place vram_gmr_placement_flags[] = { |
f1217ed0 CK |
69 | { |
70 | .fpfn = 0, | |
71 | .lpfn = 0, | |
48e07c23 | 72 | .mem_type = TTM_PL_VRAM, |
ce65b874 | 73 | .flags = 0 |
f1217ed0 CK |
74 | }, { |
75 | .fpfn = 0, | |
76 | .lpfn = 0, | |
48e07c23 | 77 | .mem_type = VMW_PL_GMR, |
ce65b874 | 78 | .flags = 0 |
f1217ed0 | 79 | } |
135cba0d TH |
80 | }; |
81 | ||
9036f8c7 | 82 | static const struct ttm_place gmr_vram_placement_flags[] = { |
f1217ed0 CK |
83 | { |
84 | .fpfn = 0, | |
85 | .lpfn = 0, | |
48e07c23 | 86 | .mem_type = VMW_PL_GMR, |
ce65b874 | 87 | .flags = 0 |
f1217ed0 CK |
88 | }, { |
89 | .fpfn = 0, | |
90 | .lpfn = 0, | |
48e07c23 | 91 | .mem_type = TTM_PL_VRAM, |
ce65b874 | 92 | .flags = 0 |
f1217ed0 | 93 | } |
5bb39e81 TH |
94 | }; |
95 | ||
135cba0d | 96 | struct ttm_placement vmw_vram_gmr_placement = { |
135cba0d TH |
97 | .num_placement = 2, |
98 | .placement = vram_gmr_placement_flags, | |
99 | .num_busy_placement = 1, | |
100 | .busy_placement = &gmr_placement_flags | |
101 | }; | |
102 | ||
8ba5152a | 103 | struct ttm_placement vmw_vram_sys_placement = { |
8ba5152a TH |
104 | .num_placement = 1, |
105 | .placement = &vram_placement_flags, | |
106 | .num_busy_placement = 1, | |
107 | .busy_placement = &sys_placement_flags | |
108 | }; | |
109 | ||
fb1d9738 | 110 | struct ttm_placement vmw_sys_placement = { |
fb1d9738 JB |
111 | .num_placement = 1, |
112 | .placement = &sys_placement_flags, | |
113 | .num_busy_placement = 1, | |
114 | .busy_placement = &sys_placement_flags | |
115 | }; | |
116 | ||
9036f8c7 | 117 | static const struct ttm_place evictable_placement_flags[] = { |
f1217ed0 CK |
118 | { |
119 | .fpfn = 0, | |
120 | .lpfn = 0, | |
48e07c23 | 121 | .mem_type = TTM_PL_SYSTEM, |
ce65b874 | 122 | .flags = 0 |
f1217ed0 CK |
123 | }, { |
124 | .fpfn = 0, | |
125 | .lpfn = 0, | |
48e07c23 | 126 | .mem_type = TTM_PL_VRAM, |
ce65b874 | 127 | .flags = 0 |
f1217ed0 CK |
128 | }, { |
129 | .fpfn = 0, | |
130 | .lpfn = 0, | |
48e07c23 | 131 | .mem_type = VMW_PL_GMR, |
ce65b874 | 132 | .flags = 0 |
f1217ed0 CK |
133 | }, { |
134 | .fpfn = 0, | |
135 | .lpfn = 0, | |
48e07c23 | 136 | .mem_type = VMW_PL_MOB, |
ce65b874 | 137 | .flags = 0 |
f1217ed0 | 138 | } |
d991ef03 JB |
139 | }; |
140 | ||
ef86cfee TH |
141 | static const struct ttm_place nonfixed_placement_flags[] = { |
142 | { | |
143 | .fpfn = 0, | |
144 | .lpfn = 0, | |
48e07c23 | 145 | .mem_type = TTM_PL_SYSTEM, |
ce65b874 | 146 | .flags = 0 |
ef86cfee TH |
147 | }, { |
148 | .fpfn = 0, | |
149 | .lpfn = 0, | |
48e07c23 | 150 | .mem_type = VMW_PL_GMR, |
ce65b874 | 151 | .flags = 0 |
ef86cfee TH |
152 | }, { |
153 | .fpfn = 0, | |
154 | .lpfn = 0, | |
48e07c23 | 155 | .mem_type = VMW_PL_MOB, |
ce65b874 | 156 | .flags = 0 |
ef86cfee TH |
157 | } |
158 | }; | |
159 | ||
d991ef03 | 160 | struct ttm_placement vmw_evictable_placement = { |
6da768aa | 161 | .num_placement = 4, |
d991ef03 JB |
162 | .placement = evictable_placement_flags, |
163 | .num_busy_placement = 1, | |
164 | .busy_placement = &sys_placement_flags | |
165 | }; | |
166 | ||
5bb39e81 | 167 | struct ttm_placement vmw_srf_placement = { |
5bb39e81 TH |
168 | .num_placement = 1, |
169 | .num_busy_placement = 2, | |
170 | .placement = &gmr_placement_flags, | |
171 | .busy_placement = gmr_vram_placement_flags | |
172 | }; | |
173 | ||
6da768aa | 174 | struct ttm_placement vmw_mob_placement = { |
6da768aa TH |
175 | .num_placement = 1, |
176 | .num_busy_placement = 1, | |
177 | .placement = &mob_placement_flags, | |
178 | .busy_placement = &mob_placement_flags | |
179 | }; | |
180 | ||
ef86cfee TH |
181 | struct ttm_placement vmw_nonfixed_placement = { |
182 | .num_placement = 3, | |
183 | .placement = nonfixed_placement_flags, | |
184 | .num_busy_placement = 1, | |
185 | .busy_placement = &sys_placement_flags | |
186 | }; | |
187 | ||
649bf3ca | 188 | struct vmw_ttm_tt { |
d92d9851 | 189 | struct ttm_dma_tt dma_ttm; |
135cba0d TH |
190 | struct vmw_private *dev_priv; |
191 | int gmr_id; | |
6da768aa TH |
192 | struct vmw_mob *mob; |
193 | int mem_type; | |
d92d9851 TH |
194 | struct sg_table sgt; |
195 | struct vmw_sg_table vsgt; | |
196 | uint64_t sg_alloc_size; | |
197 | bool mapped; | |
0b988ca1 | 198 | bool bound; |
fb1d9738 JB |
199 | }; |
200 | ||
308d17ef TH |
201 | const size_t vmw_tt_size = sizeof(struct vmw_ttm_tt); |
202 | ||
d92d9851 TH |
203 | /** |
204 | * Helper functions to advance a struct vmw_piter iterator. | |
205 | * | |
206 | * @viter: Pointer to the iterator. | |
207 | * | |
208 | * These functions return false if past the end of the list, | |
209 | * true otherwise. Functions are selected depending on the current | |
210 | * DMA mapping mode. | |
211 | */ | |
212 | static bool __vmw_piter_non_sg_next(struct vmw_piter *viter) | |
213 | { | |
214 | return ++(viter->i) < viter->num_pages; | |
215 | } | |
216 | ||
217 | static bool __vmw_piter_sg_next(struct vmw_piter *viter) | |
218 | { | |
8dc39cfc TH |
219 | bool ret = __vmw_piter_non_sg_next(viter); |
220 | ||
221 | return __sg_page_iter_dma_next(&viter->iter) && ret; | |
d92d9851 TH |
222 | } |
223 | ||
224 | ||
225 | /** | |
226 | * Helper functions to return a pointer to the current page. | |
227 | * | |
228 | * @viter: Pointer to the iterator | |
229 | * | |
230 | * These functions return a pointer to the page currently | |
231 | * pointed to by @viter. Functions are selected depending on the | |
232 | * current mapping mode. | |
233 | */ | |
234 | static struct page *__vmw_piter_non_sg_page(struct vmw_piter *viter) | |
235 | { | |
236 | return viter->pages[viter->i]; | |
237 | } | |
238 | ||
d92d9851 TH |
239 | /** |
240 | * Helper functions to return the DMA address of the current page. | |
241 | * | |
242 | * @viter: Pointer to the iterator | |
243 | * | |
244 | * These functions return the DMA address of the page currently | |
245 | * pointed to by @viter. Functions are selected depending on the | |
246 | * current mapping mode. | |
247 | */ | |
248 | static dma_addr_t __vmw_piter_phys_addr(struct vmw_piter *viter) | |
249 | { | |
250 | return page_to_phys(viter->pages[viter->i]); | |
251 | } | |
252 | ||
253 | static dma_addr_t __vmw_piter_dma_addr(struct vmw_piter *viter) | |
254 | { | |
255 | return viter->addrs[viter->i]; | |
256 | } | |
257 | ||
258 | static dma_addr_t __vmw_piter_sg_addr(struct vmw_piter *viter) | |
259 | { | |
8dc39cfc | 260 | return sg_page_iter_dma_address(&viter->iter); |
d92d9851 TH |
261 | } |
262 | ||
263 | ||
264 | /** | |
265 | * vmw_piter_start - Initialize a struct vmw_piter. | |
266 | * | |
267 | * @viter: Pointer to the iterator to initialize | |
268 | * @vsgt: Pointer to a struct vmw_sg_table to initialize from | |
269 | * | |
270 | * Note that we're following the convention of __sg_page_iter_start, so that | |
271 | * the iterator doesn't point to a valid page after initialization; it has | |
272 | * to be advanced one step first. | |
273 | */ | |
274 | void vmw_piter_start(struct vmw_piter *viter, const struct vmw_sg_table *vsgt, | |
275 | unsigned long p_offset) | |
276 | { | |
277 | viter->i = p_offset - 1; | |
278 | viter->num_pages = vsgt->num_pages; | |
8dc39cfc TH |
279 | viter->page = &__vmw_piter_non_sg_page; |
280 | viter->pages = vsgt->pages; | |
d92d9851 TH |
281 | switch (vsgt->mode) { |
282 | case vmw_dma_phys: | |
283 | viter->next = &__vmw_piter_non_sg_next; | |
284 | viter->dma_address = &__vmw_piter_phys_addr; | |
d92d9851 TH |
285 | break; |
286 | case vmw_dma_alloc_coherent: | |
287 | viter->next = &__vmw_piter_non_sg_next; | |
288 | viter->dma_address = &__vmw_piter_dma_addr; | |
d92d9851 TH |
289 | viter->addrs = vsgt->addrs; |
290 | break; | |
291 | case vmw_dma_map_populate: | |
292 | case vmw_dma_map_bind: | |
293 | viter->next = &__vmw_piter_sg_next; | |
294 | viter->dma_address = &__vmw_piter_sg_addr; | |
8dc39cfc | 295 | __sg_page_iter_start(&viter->iter.base, vsgt->sgt->sgl, |
d92d9851 TH |
296 | vsgt->sgt->orig_nents, p_offset); |
297 | break; | |
298 | default: | |
299 | BUG(); | |
300 | } | |
301 | } | |
302 | ||
303 | /** | |
304 | * vmw_ttm_unmap_from_dma - unmap device addresses previsouly mapped for | |
305 | * TTM pages | |
306 | * | |
307 | * @vmw_tt: Pointer to a struct vmw_ttm_backend | |
308 | * | |
309 | * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma. | |
310 | */ | |
311 | static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt *vmw_tt) | |
312 | { | |
313 | struct device *dev = vmw_tt->dev_priv->dev->dev; | |
314 | ||
315 | dma_unmap_sg(dev, vmw_tt->sgt.sgl, vmw_tt->sgt.nents, | |
316 | DMA_BIDIRECTIONAL); | |
317 | vmw_tt->sgt.nents = vmw_tt->sgt.orig_nents; | |
318 | } | |
319 | ||
320 | /** | |
321 | * vmw_ttm_map_for_dma - map TTM pages to get device addresses | |
322 | * | |
323 | * @vmw_tt: Pointer to a struct vmw_ttm_backend | |
324 | * | |
325 | * This function is used to get device addresses from the kernel DMA layer. | |
326 | * However, it's violating the DMA API in that when this operation has been | |
327 | * performed, it's illegal for the CPU to write to the pages without first | |
328 | * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is | |
329 | * therefore only legal to call this function if we know that the function | |
330 | * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most | |
331 | * a CPU write buffer flush. | |
332 | */ | |
333 | static int vmw_ttm_map_for_dma(struct vmw_ttm_tt *vmw_tt) | |
334 | { | |
335 | struct device *dev = vmw_tt->dev_priv->dev->dev; | |
336 | int ret; | |
337 | ||
338 | ret = dma_map_sg(dev, vmw_tt->sgt.sgl, vmw_tt->sgt.orig_nents, | |
339 | DMA_BIDIRECTIONAL); | |
340 | if (unlikely(ret == 0)) | |
341 | return -ENOMEM; | |
342 | ||
343 | vmw_tt->sgt.nents = ret; | |
344 | ||
345 | return 0; | |
346 | } | |
347 | ||
348 | /** | |
349 | * vmw_ttm_map_dma - Make sure TTM pages are visible to the device | |
350 | * | |
351 | * @vmw_tt: Pointer to a struct vmw_ttm_tt | |
352 | * | |
353 | * Select the correct function for and make sure the TTM pages are | |
354 | * visible to the device. Allocate storage for the device mappings. | |
355 | * If a mapping has already been performed, indicated by the storage | |
356 | * pointer being non NULL, the function returns success. | |
357 | */ | |
358 | static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt) | |
359 | { | |
360 | struct vmw_private *dev_priv = vmw_tt->dev_priv; | |
361 | struct ttm_mem_global *glob = vmw_mem_glob(dev_priv); | |
362 | struct vmw_sg_table *vsgt = &vmw_tt->vsgt; | |
279c01f6 RH |
363 | struct ttm_operation_ctx ctx = { |
364 | .interruptible = true, | |
365 | .no_wait_gpu = false | |
366 | }; | |
d92d9851 TH |
367 | struct vmw_piter iter; |
368 | dma_addr_t old; | |
369 | int ret = 0; | |
370 | static size_t sgl_size; | |
371 | static size_t sgt_size; | |
372 | ||
373 | if (vmw_tt->mapped) | |
374 | return 0; | |
375 | ||
376 | vsgt->mode = dev_priv->map_mode; | |
377 | vsgt->pages = vmw_tt->dma_ttm.ttm.pages; | |
378 | vsgt->num_pages = vmw_tt->dma_ttm.ttm.num_pages; | |
379 | vsgt->addrs = vmw_tt->dma_ttm.dma_address; | |
380 | vsgt->sgt = &vmw_tt->sgt; | |
381 | ||
382 | switch (dev_priv->map_mode) { | |
383 | case vmw_dma_map_bind: | |
384 | case vmw_dma_map_populate: | |
385 | if (unlikely(!sgl_size)) { | |
386 | sgl_size = ttm_round_pot(sizeof(struct scatterlist)); | |
387 | sgt_size = ttm_round_pot(sizeof(struct sg_table)); | |
388 | } | |
389 | vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages; | |
279c01f6 | 390 | ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, &ctx); |
d92d9851 TH |
391 | if (unlikely(ret != 0)) |
392 | return ret; | |
393 | ||
bde15555 TH |
394 | ret = __sg_alloc_table_from_pages |
395 | (&vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0, | |
396 | (unsigned long) vsgt->num_pages << PAGE_SHIFT, | |
397 | dma_get_max_seg_size(dev_priv->dev->dev), | |
398 | GFP_KERNEL); | |
d92d9851 TH |
399 | if (unlikely(ret != 0)) |
400 | goto out_sg_alloc_fail; | |
401 | ||
402 | if (vsgt->num_pages > vmw_tt->sgt.nents) { | |
403 | uint64_t over_alloc = | |
404 | sgl_size * (vsgt->num_pages - | |
405 | vmw_tt->sgt.nents); | |
406 | ||
407 | ttm_mem_global_free(glob, over_alloc); | |
408 | vmw_tt->sg_alloc_size -= over_alloc; | |
409 | } | |
410 | ||
411 | ret = vmw_ttm_map_for_dma(vmw_tt); | |
412 | if (unlikely(ret != 0)) | |
413 | goto out_map_fail; | |
414 | ||
415 | break; | |
416 | default: | |
417 | break; | |
418 | } | |
419 | ||
420 | old = ~((dma_addr_t) 0); | |
421 | vmw_tt->vsgt.num_regions = 0; | |
422 | for (vmw_piter_start(&iter, vsgt, 0); vmw_piter_next(&iter);) { | |
423 | dma_addr_t cur = vmw_piter_dma_addr(&iter); | |
424 | ||
425 | if (cur != old + PAGE_SIZE) | |
426 | vmw_tt->vsgt.num_regions++; | |
427 | old = cur; | |
428 | } | |
429 | ||
430 | vmw_tt->mapped = true; | |
431 | return 0; | |
432 | ||
433 | out_map_fail: | |
434 | sg_free_table(vmw_tt->vsgt.sgt); | |
435 | vmw_tt->vsgt.sgt = NULL; | |
436 | out_sg_alloc_fail: | |
437 | ttm_mem_global_free(glob, vmw_tt->sg_alloc_size); | |
438 | return ret; | |
439 | } | |
440 | ||
441 | /** | |
442 | * vmw_ttm_unmap_dma - Tear down any TTM page device mappings | |
443 | * | |
444 | * @vmw_tt: Pointer to a struct vmw_ttm_tt | |
445 | * | |
446 | * Tear down any previously set up device DMA mappings and free | |
447 | * any storage space allocated for them. If there are no mappings set up, | |
448 | * this function is a NOP. | |
449 | */ | |
450 | static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt) | |
451 | { | |
452 | struct vmw_private *dev_priv = vmw_tt->dev_priv; | |
453 | ||
454 | if (!vmw_tt->vsgt.sgt) | |
455 | return; | |
456 | ||
457 | switch (dev_priv->map_mode) { | |
458 | case vmw_dma_map_bind: | |
459 | case vmw_dma_map_populate: | |
460 | vmw_ttm_unmap_from_dma(vmw_tt); | |
461 | sg_free_table(vmw_tt->vsgt.sgt); | |
462 | vmw_tt->vsgt.sgt = NULL; | |
463 | ttm_mem_global_free(vmw_mem_glob(dev_priv), | |
464 | vmw_tt->sg_alloc_size); | |
465 | break; | |
466 | default: | |
467 | break; | |
468 | } | |
469 | vmw_tt->mapped = false; | |
470 | } | |
471 | ||
0fd53cfb TH |
472 | /** |
473 | * vmw_bo_sg_table - Return a struct vmw_sg_table object for a | |
474 | * TTM buffer object | |
475 | * | |
476 | * @bo: Pointer to a struct ttm_buffer_object | |
477 | * | |
478 | * Returns a pointer to a struct vmw_sg_table object. The object should | |
479 | * not be freed after use. | |
480 | * Note that for the device addresses to be valid, the buffer object must | |
481 | * either be reserved or pinned. | |
482 | */ | |
483 | const struct vmw_sg_table *vmw_bo_sg_table(struct ttm_buffer_object *bo) | |
484 | { | |
485 | struct vmw_ttm_tt *vmw_tt = | |
486 | container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm); | |
487 | ||
488 | return &vmw_tt->vsgt; | |
489 | } | |
490 | ||
491 | ||
0a667b50 DA |
492 | static int vmw_ttm_bind(struct ttm_bo_device *bdev, |
493 | struct ttm_tt *ttm, struct ttm_resource *bo_mem) | |
fb1d9738 | 494 | { |
d92d9851 TH |
495 | struct vmw_ttm_tt *vmw_be = |
496 | container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm); | |
0b988ca1 DA |
497 | int ret = 0; |
498 | ||
499 | if (!bo_mem) | |
500 | return -EINVAL; | |
501 | ||
502 | if (vmw_be->bound) | |
503 | return 0; | |
d92d9851 TH |
504 | |
505 | ret = vmw_ttm_map_dma(vmw_be); | |
506 | if (unlikely(ret != 0)) | |
507 | return ret; | |
135cba0d TH |
508 | |
509 | vmw_be->gmr_id = bo_mem->start; | |
6da768aa | 510 | vmw_be->mem_type = bo_mem->mem_type; |
135cba0d | 511 | |
6da768aa TH |
512 | switch (bo_mem->mem_type) { |
513 | case VMW_PL_GMR: | |
0b988ca1 | 514 | ret = vmw_gmr_bind(vmw_be->dev_priv, &vmw_be->vsgt, |
6da768aa | 515 | ttm->num_pages, vmw_be->gmr_id); |
0b988ca1 | 516 | break; |
6da768aa TH |
517 | case VMW_PL_MOB: |
518 | if (unlikely(vmw_be->mob == NULL)) { | |
519 | vmw_be->mob = | |
520 | vmw_mob_create(ttm->num_pages); | |
521 | if (unlikely(vmw_be->mob == NULL)) | |
522 | return -ENOMEM; | |
523 | } | |
524 | ||
0b988ca1 | 525 | ret = vmw_mob_bind(vmw_be->dev_priv, vmw_be->mob, |
0fd53cfb | 526 | &vmw_be->vsgt, ttm->num_pages, |
6da768aa | 527 | vmw_be->gmr_id); |
0b988ca1 | 528 | break; |
6da768aa TH |
529 | default: |
530 | BUG(); | |
531 | } | |
0b988ca1 DA |
532 | vmw_be->bound = true; |
533 | return ret; | |
fb1d9738 JB |
534 | } |
535 | ||
0a667b50 DA |
536 | static void vmw_ttm_unbind(struct ttm_bo_device *bdev, |
537 | struct ttm_tt *ttm) | |
fb1d9738 | 538 | { |
d92d9851 TH |
539 | struct vmw_ttm_tt *vmw_be = |
540 | container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm); | |
135cba0d | 541 | |
0b988ca1 DA |
542 | if (!vmw_be->bound) |
543 | return; | |
544 | ||
6da768aa TH |
545 | switch (vmw_be->mem_type) { |
546 | case VMW_PL_GMR: | |
547 | vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id); | |
548 | break; | |
549 | case VMW_PL_MOB: | |
550 | vmw_mob_unbind(vmw_be->dev_priv, vmw_be->mob); | |
551 | break; | |
552 | default: | |
553 | BUG(); | |
554 | } | |
d92d9851 TH |
555 | |
556 | if (vmw_be->dev_priv->map_mode == vmw_dma_map_bind) | |
557 | vmw_ttm_unmap_dma(vmw_be); | |
0b988ca1 | 558 | vmw_be->bound = false; |
fb1d9738 JB |
559 | } |
560 | ||
6da768aa | 561 | |
0a667b50 | 562 | static void vmw_ttm_destroy(struct ttm_bo_device *bdev, struct ttm_tt *ttm) |
fb1d9738 | 563 | { |
d92d9851 TH |
564 | struct vmw_ttm_tt *vmw_be = |
565 | container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm); | |
566 | ||
37bff654 | 567 | vmw_ttm_unbind(bdev, ttm); |
7626168f | 568 | ttm_tt_destroy_common(bdev, ttm); |
d92d9851 TH |
569 | vmw_ttm_unmap_dma(vmw_be); |
570 | if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent) | |
571 | ttm_dma_tt_fini(&vmw_be->dma_ttm); | |
572 | else | |
573 | ttm_tt_fini(ttm); | |
6da768aa TH |
574 | |
575 | if (vmw_be->mob) | |
576 | vmw_mob_destroy(vmw_be->mob); | |
577 | ||
fb1d9738 JB |
578 | kfree(vmw_be); |
579 | } | |
580 | ||
0fd53cfb | 581 | |
0a667b50 DA |
582 | static int vmw_ttm_populate(struct ttm_bo_device *bdev, |
583 | struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) | |
d92d9851 TH |
584 | { |
585 | struct vmw_ttm_tt *vmw_tt = | |
586 | container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm); | |
587 | struct vmw_private *dev_priv = vmw_tt->dev_priv; | |
588 | struct ttm_mem_global *glob = vmw_mem_glob(dev_priv); | |
589 | int ret; | |
590 | ||
7eec9151 | 591 | if (ttm_tt_is_populated(ttm)) |
d92d9851 TH |
592 | return 0; |
593 | ||
594 | if (dev_priv->map_mode == vmw_dma_alloc_coherent) { | |
595 | size_t size = | |
596 | ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t)); | |
d0cef9fa | 597 | ret = ttm_mem_global_alloc(glob, size, ctx); |
d92d9851 TH |
598 | if (unlikely(ret != 0)) |
599 | return ret; | |
600 | ||
d0cef9fa RH |
601 | ret = ttm_dma_populate(&vmw_tt->dma_ttm, dev_priv->dev->dev, |
602 | ctx); | |
d92d9851 TH |
603 | if (unlikely(ret != 0)) |
604 | ttm_mem_global_free(glob, size); | |
605 | } else | |
d0cef9fa | 606 | ret = ttm_pool_populate(ttm, ctx); |
d92d9851 TH |
607 | |
608 | return ret; | |
609 | } | |
610 | ||
0a667b50 DA |
611 | static void vmw_ttm_unpopulate(struct ttm_bo_device *bdev, |
612 | struct ttm_tt *ttm) | |
d92d9851 TH |
613 | { |
614 | struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt, | |
615 | dma_ttm.ttm); | |
616 | struct vmw_private *dev_priv = vmw_tt->dev_priv; | |
617 | struct ttm_mem_global *glob = vmw_mem_glob(dev_priv); | |
618 | ||
6da768aa TH |
619 | |
620 | if (vmw_tt->mob) { | |
621 | vmw_mob_destroy(vmw_tt->mob); | |
622 | vmw_tt->mob = NULL; | |
623 | } | |
624 | ||
d92d9851 TH |
625 | vmw_ttm_unmap_dma(vmw_tt); |
626 | if (dev_priv->map_mode == vmw_dma_alloc_coherent) { | |
627 | size_t size = | |
628 | ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t)); | |
629 | ||
630 | ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev); | |
631 | ttm_mem_global_free(glob, size); | |
632 | } else | |
633 | ttm_pool_unpopulate(ttm); | |
634 | } | |
635 | ||
dde5da23 CK |
636 | static struct ttm_tt *vmw_ttm_tt_create(struct ttm_buffer_object *bo, |
637 | uint32_t page_flags) | |
fb1d9738 | 638 | { |
649bf3ca | 639 | struct vmw_ttm_tt *vmw_be; |
d92d9851 | 640 | int ret; |
fb1d9738 | 641 | |
d92d9851 | 642 | vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL); |
fb1d9738 JB |
643 | if (!vmw_be) |
644 | return NULL; | |
645 | ||
dde5da23 | 646 | vmw_be->dev_priv = container_of(bo->bdev, struct vmw_private, bdev); |
6da768aa | 647 | vmw_be->mob = NULL; |
fb1d9738 | 648 | |
d92d9851 | 649 | if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent) |
1b4ea4c5 CK |
650 | ret = ttm_dma_tt_init(&vmw_be->dma_ttm, bo, page_flags, |
651 | ttm_cached); | |
d92d9851 | 652 | else |
1b4ea4c5 CK |
653 | ret = ttm_tt_init(&vmw_be->dma_ttm.ttm, bo, page_flags, |
654 | ttm_cached); | |
d92d9851 TH |
655 | if (unlikely(ret != 0)) |
656 | goto out_no_init; | |
657 | ||
658 | return &vmw_be->dma_ttm.ttm; | |
659 | out_no_init: | |
660 | kfree(vmw_be); | |
661 | return NULL; | |
fb1d9738 JB |
662 | } |
663 | ||
8227622f | 664 | static void vmw_evict_flags(struct ttm_buffer_object *bo, |
fb1d9738 JB |
665 | struct ttm_placement *placement) |
666 | { | |
667 | *placement = vmw_sys_placement; | |
668 | } | |
669 | ||
fb1d9738 JB |
670 | static int vmw_verify_access(struct ttm_buffer_object *bo, struct file *filp) |
671 | { | |
d08a9b9c TH |
672 | struct ttm_object_file *tfile = |
673 | vmw_fpriv((struct drm_file *)filp->private_data)->tfile; | |
674 | ||
f1d34bfd | 675 | return vmw_user_bo_verify_access(bo, tfile); |
fb1d9738 JB |
676 | } |
677 | ||
2966141a | 678 | static int vmw_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_resource *mem) |
96bf8b87 | 679 | { |
96bf8b87 JG |
680 | struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev); |
681 | ||
96bf8b87 JG |
682 | switch (mem->mem_type) { |
683 | case TTM_PL_SYSTEM: | |
135cba0d | 684 | case VMW_PL_GMR: |
6da768aa | 685 | case VMW_PL_MOB: |
96bf8b87 JG |
686 | return 0; |
687 | case TTM_PL_VRAM: | |
54d04ea8 CK |
688 | mem->bus.offset = (mem->start << PAGE_SHIFT) + |
689 | dev_priv->vram_start; | |
96bf8b87 | 690 | mem->bus.is_iomem = true; |
1cf65c45 | 691 | mem->bus.caching = ttm_cached; |
96bf8b87 JG |
692 | break; |
693 | default: | |
694 | return -EINVAL; | |
695 | } | |
696 | return 0; | |
697 | } | |
698 | ||
6da768aa TH |
699 | /** |
700 | * vmw_move_notify - TTM move_notify_callback | |
701 | * | |
fd11a3c0 | 702 | * @bo: The TTM buffer object about to move. |
2966141a | 703 | * @mem: The struct ttm_resource indicating to what memory |
fd11a3c0 | 704 | * region the move is taking place. |
6da768aa TH |
705 | * |
706 | * Calls move_notify for all subsystems needing it. | |
707 | * (currently only resources). | |
708 | */ | |
709 | static void vmw_move_notify(struct ttm_buffer_object *bo, | |
66257db7 | 710 | bool evict, |
2966141a | 711 | struct ttm_resource *mem) |
6da768aa | 712 | { |
279a3010 DA |
713 | if (!mem) |
714 | return; | |
e9431ea5 | 715 | vmw_bo_move_notify(bo, mem); |
fd11a3c0 | 716 | vmw_query_move_notify(bo, mem); |
6da768aa TH |
717 | } |
718 | ||
719 | ||
720 | /** | |
721 | * vmw_swap_notify - TTM move_notify_callback | |
722 | * | |
fd11a3c0 | 723 | * @bo: The TTM buffer object about to be swapped out. |
6da768aa TH |
724 | */ |
725 | static void vmw_swap_notify(struct ttm_buffer_object *bo) | |
726 | { | |
e9431ea5 | 727 | vmw_bo_swap_notify(bo); |
f08c86c3 | 728 | (void) ttm_bo_wait(bo, false, false); |
6da768aa TH |
729 | } |
730 | ||
bcff5d3e DA |
731 | static int vmw_move(struct ttm_buffer_object *bo, |
732 | bool evict, | |
733 | struct ttm_operation_ctx *ctx, | |
734 | struct ttm_resource *new_mem) | |
735 | { | |
736 | struct ttm_resource_manager *old_man = ttm_manager_type(bo->bdev, bo->mem.mem_type); | |
737 | struct ttm_resource_manager *new_man = ttm_manager_type(bo->bdev, new_mem->mem_type); | |
c37d951c | 738 | int ret; |
bcff5d3e DA |
739 | |
740 | if (old_man->use_tt && new_man->use_tt) { | |
741 | if (bo->mem.mem_type == TTM_PL_SYSTEM) { | |
742 | ttm_bo_assign_mem(bo, new_mem); | |
743 | return 0; | |
744 | } | |
29a1d482 | 745 | ret = ttm_bo_wait_ctx(bo, ctx); |
c37d951c DA |
746 | if (ret) |
747 | return ret; | |
29a1d482 DA |
748 | |
749 | vmw_ttm_unbind(bo->bdev, bo->ttm); | |
750 | ttm_resource_free(bo, &bo->mem); | |
c37d951c DA |
751 | ttm_bo_assign_mem(bo, new_mem); |
752 | return 0; | |
bcff5d3e DA |
753 | } else { |
754 | return ttm_bo_move_memcpy(bo, ctx, new_mem); | |
755 | } | |
756 | } | |
6da768aa | 757 | |
fb1d9738 | 758 | struct ttm_bo_driver vmw_bo_driver = { |
649bf3ca | 759 | .ttm_tt_create = &vmw_ttm_tt_create, |
d92d9851 TH |
760 | .ttm_tt_populate = &vmw_ttm_populate, |
761 | .ttm_tt_unpopulate = &vmw_ttm_unpopulate, | |
debf8ab9 | 762 | .ttm_tt_bind = &vmw_ttm_bind, |
debf8ab9 | 763 | .ttm_tt_destroy = &vmw_ttm_destroy, |
a2ab19fe | 764 | .eviction_valuable = ttm_bo_eviction_valuable, |
fb1d9738 | 765 | .evict_flags = vmw_evict_flags, |
bcff5d3e | 766 | .move = vmw_move, |
fb1d9738 | 767 | .verify_access = vmw_verify_access, |
6da768aa TH |
768 | .move_notify = vmw_move_notify, |
769 | .swap_notify = vmw_swap_notify, | |
96bf8b87 | 770 | .io_mem_reserve = &vmw_ttm_io_mem_reserve, |
fb1d9738 | 771 | }; |
56dc01f1 DA |
772 | |
773 | int vmw_bo_create_and_populate(struct vmw_private *dev_priv, | |
774 | unsigned long bo_size, | |
775 | struct ttm_buffer_object **bo_p) | |
776 | { | |
777 | struct ttm_operation_ctx ctx = { | |
778 | .interruptible = false, | |
779 | .no_wait_gpu = false | |
780 | }; | |
781 | struct ttm_buffer_object *bo; | |
782 | int ret; | |
783 | ||
b254557c CK |
784 | ret = vmw_bo_create_kernel(dev_priv, bo_size, |
785 | &vmw_sys_placement, | |
786 | &bo); | |
56dc01f1 DA |
787 | if (unlikely(ret != 0)) |
788 | return ret; | |
789 | ||
790 | ret = ttm_bo_reserve(bo, false, true, NULL); | |
791 | BUG_ON(ret != 0); | |
0a667b50 | 792 | ret = vmw_ttm_populate(bo->bdev, bo->ttm, &ctx); |
a2d6ddc4 DA |
793 | if (likely(ret == 0)) { |
794 | struct vmw_ttm_tt *vmw_tt = | |
795 | container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm.ttm); | |
796 | ret = vmw_ttm_map_dma(vmw_tt); | |
797 | } | |
56dc01f1 DA |
798 | |
799 | ttm_bo_unreserve(bo); | |
800 | ||
801 | if (likely(ret == 0)) | |
802 | *bo_p = bo; | |
803 | return ret; | |
804 | } |