Commit | Line | Data |
---|---|---|
85438a8d TZ |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | ||
3 | #include <drm/drm_gem_vram_helper.h> | |
4 | #include <drm/ttm/ttm_page_alloc.h> | |
5 | ||
6 | /** | |
7 | * DOC: overview | |
8 | * | |
9 | * This library provides a GEM buffer object that is backed by video RAM | |
10 | * (VRAM). It can be used for framebuffer devices with dedicated memory. | |
11 | */ | |
12 | ||
13 | /* | |
14 | * Buffer-objects helpers | |
15 | */ | |
16 | ||
17 | static void drm_gem_vram_cleanup(struct drm_gem_vram_object *gbo) | |
18 | { | |
19 | /* We got here via ttm_bo_put(), which means that the | |
20 | * TTM buffer object in 'bo' has already been cleaned | |
21 | * up; only release the GEM object. | |
22 | */ | |
23 | drm_gem_object_release(&gbo->gem); | |
24 | } | |
25 | ||
26 | static void drm_gem_vram_destroy(struct drm_gem_vram_object *gbo) | |
27 | { | |
28 | drm_gem_vram_cleanup(gbo); | |
29 | kfree(gbo); | |
30 | } | |
31 | ||
32 | static void ttm_buffer_object_destroy(struct ttm_buffer_object *bo) | |
33 | { | |
34 | struct drm_gem_vram_object *gbo = drm_gem_vram_of_bo(bo); | |
35 | ||
36 | drm_gem_vram_destroy(gbo); | |
37 | } | |
38 | ||
39 | static void drm_gem_vram_placement(struct drm_gem_vram_object *gbo, | |
40 | unsigned long pl_flag) | |
41 | { | |
42 | unsigned int i; | |
43 | unsigned int c = 0; | |
44 | ||
45 | gbo->placement.placement = gbo->placements; | |
46 | gbo->placement.busy_placement = gbo->placements; | |
47 | ||
48 | if (pl_flag & TTM_PL_FLAG_VRAM) | |
49 | gbo->placements[c++].flags = TTM_PL_FLAG_WC | | |
50 | TTM_PL_FLAG_UNCACHED | | |
51 | TTM_PL_FLAG_VRAM; | |
52 | ||
53 | if (pl_flag & TTM_PL_FLAG_SYSTEM) | |
54 | gbo->placements[c++].flags = TTM_PL_MASK_CACHING | | |
55 | TTM_PL_FLAG_SYSTEM; | |
56 | ||
57 | if (!c) | |
58 | gbo->placements[c++].flags = TTM_PL_MASK_CACHING | | |
59 | TTM_PL_FLAG_SYSTEM; | |
60 | ||
61 | gbo->placement.num_placement = c; | |
62 | gbo->placement.num_busy_placement = c; | |
63 | ||
64 | for (i = 0; i < c; ++i) { | |
65 | gbo->placements[i].fpfn = 0; | |
66 | gbo->placements[i].lpfn = 0; | |
67 | } | |
68 | } | |
69 | ||
70 | static int drm_gem_vram_init(struct drm_device *dev, | |
71 | struct ttm_bo_device *bdev, | |
72 | struct drm_gem_vram_object *gbo, | |
73 | size_t size, unsigned long pg_align, | |
74 | bool interruptible) | |
75 | { | |
76 | int ret; | |
77 | size_t acc_size; | |
78 | ||
79 | ret = drm_gem_object_init(dev, &gbo->gem, size); | |
80 | if (ret) | |
81 | return ret; | |
82 | ||
83 | acc_size = ttm_bo_dma_acc_size(bdev, size, sizeof(*gbo)); | |
84 | ||
85 | gbo->bo.bdev = bdev; | |
86 | drm_gem_vram_placement(gbo, TTM_PL_FLAG_VRAM | TTM_PL_FLAG_SYSTEM); | |
87 | ||
88 | ret = ttm_bo_init(bdev, &gbo->bo, size, ttm_bo_type_device, | |
89 | &gbo->placement, pg_align, interruptible, acc_size, | |
90 | NULL, NULL, ttm_buffer_object_destroy); | |
91 | if (ret) | |
92 | goto err_drm_gem_object_release; | |
93 | ||
94 | return 0; | |
95 | ||
96 | err_drm_gem_object_release: | |
97 | drm_gem_object_release(&gbo->gem); | |
98 | return ret; | |
99 | } | |
100 | ||
101 | /** | |
102 | * drm_gem_vram_create() - Creates a VRAM-backed GEM object | |
103 | * @dev: the DRM device | |
104 | * @bdev: the TTM BO device backing the object | |
105 | * @size: the buffer size in bytes | |
106 | * @pg_align: the buffer's alignment in multiples of the page size | |
107 | * @interruptible: sleep interruptible if waiting for memory | |
108 | * | |
109 | * Returns: | |
110 | * A new instance of &struct drm_gem_vram_object on success, or | |
111 | * an ERR_PTR()-encoded error code otherwise. | |
112 | */ | |
113 | struct drm_gem_vram_object *drm_gem_vram_create(struct drm_device *dev, | |
114 | struct ttm_bo_device *bdev, | |
115 | size_t size, | |
116 | unsigned long pg_align, | |
117 | bool interruptible) | |
118 | { | |
119 | struct drm_gem_vram_object *gbo; | |
120 | int ret; | |
121 | ||
122 | gbo = kzalloc(sizeof(*gbo), GFP_KERNEL); | |
123 | if (!gbo) | |
124 | return ERR_PTR(-ENOMEM); | |
125 | ||
126 | ret = drm_gem_vram_init(dev, bdev, gbo, size, pg_align, interruptible); | |
127 | if (ret < 0) | |
128 | goto err_kfree; | |
129 | ||
130 | return gbo; | |
131 | ||
132 | err_kfree: | |
133 | kfree(gbo); | |
134 | return ERR_PTR(ret); | |
135 | } | |
136 | EXPORT_SYMBOL(drm_gem_vram_create); | |
137 | ||
138 | /** | |
139 | * drm_gem_vram_put() - Releases a reference to a VRAM-backed GEM object | |
140 | * @gbo: the GEM VRAM object | |
141 | * | |
142 | * See ttm_bo_put() for more information. | |
143 | */ | |
144 | void drm_gem_vram_put(struct drm_gem_vram_object *gbo) | |
145 | { | |
146 | ttm_bo_put(&gbo->bo); | |
147 | } | |
148 | EXPORT_SYMBOL(drm_gem_vram_put); | |
149 | ||
150 | /** | |
151 | * drm_gem_vram_reserve() - Reserves a VRAM-backed GEM object | |
152 | * @gbo: the GEM VRAM object | |
153 | * @no_wait: don't wait for buffer object to become available | |
154 | * | |
155 | * See ttm_bo_reserve() for more information. | |
156 | * | |
157 | * Returns: | |
158 | * 0 on success, or | |
159 | * a negative error code otherwise | |
160 | */ | |
161 | int drm_gem_vram_reserve(struct drm_gem_vram_object *gbo, bool no_wait) | |
162 | { | |
163 | return ttm_bo_reserve(&gbo->bo, true, no_wait, NULL); | |
164 | } | |
165 | EXPORT_SYMBOL(drm_gem_vram_reserve); | |
166 | ||
167 | /** | |
168 | * drm_gem_vram_unreserve() - \ | |
169 | Release a reservation acquired by drm_gem_vram_reserve() | |
170 | * @gbo: the GEM VRAM object | |
171 | * | |
172 | * See ttm_bo_unreserve() for more information. | |
173 | */ | |
174 | void drm_gem_vram_unreserve(struct drm_gem_vram_object *gbo) | |
175 | { | |
176 | ttm_bo_unreserve(&gbo->bo); | |
177 | } | |
178 | EXPORT_SYMBOL(drm_gem_vram_unreserve); | |
179 | ||
180 | /** | |
181 | * drm_gem_vram_mmap_offset() - Returns a GEM VRAM object's mmap offset | |
182 | * @gbo: the GEM VRAM object | |
183 | * | |
184 | * See drm_vma_node_offset_addr() for more information. | |
185 | * | |
186 | * Returns: | |
187 | * The buffer object's offset for userspace mappings on success, or | |
188 | * 0 if no offset is allocated. | |
189 | */ | |
190 | u64 drm_gem_vram_mmap_offset(struct drm_gem_vram_object *gbo) | |
191 | { | |
192 | return drm_vma_node_offset_addr(&gbo->bo.vma_node); | |
193 | } | |
194 | EXPORT_SYMBOL(drm_gem_vram_mmap_offset); | |
195 | ||
196 | /** | |
197 | * drm_gem_vram_offset() - \ | |
198 | Returns a GEM VRAM object's offset in video memory | |
199 | * @gbo: the GEM VRAM object | |
200 | * | |
201 | * This function returns the buffer object's offset in the device's video | |
202 | * memory. The buffer object has to be pinned to %TTM_PL_VRAM. | |
203 | * | |
204 | * Returns: | |
205 | * The buffer object's offset in video memory on success, or | |
206 | * a negative errno code otherwise. | |
207 | */ | |
208 | s64 drm_gem_vram_offset(struct drm_gem_vram_object *gbo) | |
209 | { | |
210 | if (WARN_ON_ONCE(!gbo->pin_count)) | |
211 | return (s64)-ENODEV; | |
212 | return gbo->bo.offset; | |
213 | } | |
214 | EXPORT_SYMBOL(drm_gem_vram_offset); | |
215 | ||
216 | /** | |
217 | * drm_gem_vram_pin() - Pins a GEM VRAM object in a region. | |
218 | * @gbo: the GEM VRAM object | |
219 | * @pl_flag: a bitmask of possible memory regions | |
220 | * | |
221 | * Pinning a buffer object ensures that it is not evicted from | |
222 | * a memory region. A pinned buffer object has to be unpinned before | |
223 | * it can be pinned to another region. | |
224 | * | |
225 | * Returns: | |
226 | * 0 on success, or | |
227 | * a negative error code otherwise. | |
228 | */ | |
229 | int drm_gem_vram_pin(struct drm_gem_vram_object *gbo, unsigned long pl_flag) | |
230 | { | |
231 | int i, ret; | |
232 | struct ttm_operation_ctx ctx = { false, false }; | |
233 | ||
234 | if (gbo->pin_count) { | |
235 | ++gbo->pin_count; | |
236 | return 0; | |
237 | } | |
238 | ||
239 | drm_gem_vram_placement(gbo, pl_flag); | |
240 | for (i = 0; i < gbo->placement.num_placement; ++i) | |
241 | gbo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; | |
242 | ||
243 | ret = ttm_bo_validate(&gbo->bo, &gbo->placement, &ctx); | |
244 | if (ret < 0) | |
245 | return ret; | |
246 | ||
247 | gbo->pin_count = 1; | |
248 | ||
249 | return 0; | |
250 | } | |
251 | EXPORT_SYMBOL(drm_gem_vram_pin); | |
252 | ||
253 | /** | |
254 | * drm_gem_vram_unpin() - Unpins a GEM VRAM object | |
255 | * @gbo: the GEM VRAM object | |
256 | * | |
257 | * Returns: | |
258 | * 0 on success, or | |
259 | * a negative error code otherwise. | |
260 | */ | |
261 | int drm_gem_vram_unpin(struct drm_gem_vram_object *gbo) | |
262 | { | |
263 | int i, ret; | |
264 | struct ttm_operation_ctx ctx = { false, false }; | |
265 | ||
266 | if (WARN_ON_ONCE(!gbo->pin_count)) | |
267 | return 0; | |
268 | ||
269 | --gbo->pin_count; | |
270 | if (gbo->pin_count) | |
271 | return 0; | |
272 | ||
273 | for (i = 0; i < gbo->placement.num_placement ; ++i) | |
274 | gbo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT; | |
275 | ||
276 | ret = ttm_bo_validate(&gbo->bo, &gbo->placement, &ctx); | |
277 | if (ret < 0) | |
278 | return ret; | |
279 | ||
280 | return 0; | |
281 | } | |
282 | EXPORT_SYMBOL(drm_gem_vram_unpin); | |
283 | ||
284 | /** | |
285 | * drm_gem_vram_push_to_system() - \ | |
286 | Unpins a GEM VRAM object and moves it to system memory | |
287 | * @gbo: the GEM VRAM object | |
288 | * | |
289 | * This operation only works if the caller holds the final pin on the | |
290 | * buffer object. | |
291 | * | |
292 | * Returns: | |
293 | * 0 on success, or | |
294 | * a negative error code otherwise. | |
295 | */ | |
296 | int drm_gem_vram_push_to_system(struct drm_gem_vram_object *gbo) | |
297 | { | |
298 | int i, ret; | |
299 | struct ttm_operation_ctx ctx = { false, false }; | |
300 | ||
301 | if (WARN_ON_ONCE(!gbo->pin_count)) | |
302 | return 0; | |
303 | ||
304 | --gbo->pin_count; | |
305 | if (gbo->pin_count) | |
306 | return 0; | |
307 | ||
308 | if (gbo->kmap.virtual) | |
309 | ttm_bo_kunmap(&gbo->kmap); | |
310 | ||
311 | drm_gem_vram_placement(gbo, TTM_PL_FLAG_SYSTEM); | |
312 | for (i = 0; i < gbo->placement.num_placement ; ++i) | |
313 | gbo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; | |
314 | ||
315 | ret = ttm_bo_validate(&gbo->bo, &gbo->placement, &ctx); | |
316 | if (ret) | |
317 | return ret; | |
318 | ||
319 | return 0; | |
320 | } | |
321 | EXPORT_SYMBOL(drm_gem_vram_push_to_system); | |
322 | ||
323 | /** | |
324 | * drm_gem_vram_kmap_at() - Maps a GEM VRAM object into kernel address space | |
325 | * @gbo: the GEM VRAM object | |
326 | * @map: establish a mapping if necessary | |
327 | * @is_iomem: returns true if the mapped memory is I/O memory, or false \ | |
328 | otherwise; can be NULL | |
329 | * @kmap: the mapping's kmap object | |
330 | * | |
331 | * This function maps the buffer object into the kernel's address space | |
332 | * or returns the current mapping. If the parameter map is false, the | |
333 | * function only queries the current mapping, but does not establish a | |
334 | * new one. | |
335 | * | |
336 | * Returns: | |
337 | * The buffers virtual address if mapped, or | |
338 | * NULL if not mapped, or | |
339 | * an ERR_PTR()-encoded error code otherwise. | |
340 | */ | |
341 | void *drm_gem_vram_kmap_at(struct drm_gem_vram_object *gbo, bool map, | |
342 | bool *is_iomem, struct ttm_bo_kmap_obj *kmap) | |
343 | { | |
344 | int ret; | |
345 | ||
346 | if (kmap->virtual || !map) | |
347 | goto out; | |
348 | ||
349 | ret = ttm_bo_kmap(&gbo->bo, 0, gbo->bo.num_pages, kmap); | |
350 | if (ret) | |
351 | return ERR_PTR(ret); | |
352 | ||
353 | out: | |
354 | if (!is_iomem) | |
355 | return kmap->virtual; | |
356 | if (!kmap->virtual) { | |
357 | *is_iomem = false; | |
358 | return NULL; | |
359 | } | |
360 | return ttm_kmap_obj_virtual(kmap, is_iomem); | |
361 | } | |
362 | EXPORT_SYMBOL(drm_gem_vram_kmap_at); | |
363 | ||
364 | /** | |
365 | * drm_gem_vram_kmap() - Maps a GEM VRAM object into kernel address space | |
366 | * @gbo: the GEM VRAM object | |
367 | * @map: establish a mapping if necessary | |
368 | * @is_iomem: returns true if the mapped memory is I/O memory, or false \ | |
369 | otherwise; can be NULL | |
370 | * | |
371 | * This function maps the buffer object into the kernel's address space | |
372 | * or returns the current mapping. If the parameter map is false, the | |
373 | * function only queries the current mapping, but does not establish a | |
374 | * new one. | |
375 | * | |
376 | * Returns: | |
377 | * The buffers virtual address if mapped, or | |
378 | * NULL if not mapped, or | |
379 | * an ERR_PTR()-encoded error code otherwise. | |
380 | */ | |
381 | void *drm_gem_vram_kmap(struct drm_gem_vram_object *gbo, bool map, | |
382 | bool *is_iomem) | |
383 | { | |
384 | return drm_gem_vram_kmap_at(gbo, map, is_iomem, &gbo->kmap); | |
385 | } | |
386 | EXPORT_SYMBOL(drm_gem_vram_kmap); | |
387 | ||
388 | /** | |
389 | * drm_gem_vram_kunmap_at() - Unmaps a GEM VRAM object | |
390 | * @gbo: the GEM VRAM object | |
391 | * @kmap: the mapping's kmap object | |
392 | */ | |
393 | void drm_gem_vram_kunmap_at(struct drm_gem_vram_object *gbo, | |
394 | struct ttm_bo_kmap_obj *kmap) | |
395 | { | |
396 | if (!kmap->virtual) | |
397 | return; | |
398 | ||
399 | ttm_bo_kunmap(kmap); | |
400 | kmap->virtual = NULL; | |
401 | } | |
402 | EXPORT_SYMBOL(drm_gem_vram_kunmap_at); | |
403 | ||
404 | /** | |
405 | * drm_gem_vram_kunmap() - Unmaps a GEM VRAM object | |
406 | * @gbo: the GEM VRAM object | |
407 | */ | |
408 | void drm_gem_vram_kunmap(struct drm_gem_vram_object *gbo) | |
409 | { | |
410 | drm_gem_vram_kunmap_at(gbo, &gbo->kmap); | |
411 | } | |
412 | EXPORT_SYMBOL(drm_gem_vram_kunmap); | |
6c812bc5 TZ |
413 | |
414 | /* | |
415 | * Helpers for struct ttm_bo_driver | |
416 | */ | |
417 | ||
418 | static bool drm_is_gem_vram(struct ttm_buffer_object *bo) | |
419 | { | |
420 | return (bo->destroy == ttm_buffer_object_destroy); | |
421 | } | |
422 | ||
423 | /** | |
424 | * drm_gem_vram_bo_driver_evict_flags() - \ | |
425 | Implements &struct ttm_bo_driver.evict_flags | |
426 | * @bo: TTM buffer object. Refers to &struct drm_gem_vram_object.bo | |
427 | * @pl: TTM placement information. | |
428 | */ | |
429 | void drm_gem_vram_bo_driver_evict_flags(struct ttm_buffer_object *bo, | |
430 | struct ttm_placement *pl) | |
431 | { | |
432 | struct drm_gem_vram_object *gbo; | |
433 | ||
434 | /* TTM may pass BOs that are not GEM VRAM BOs. */ | |
435 | if (!drm_is_gem_vram(bo)) | |
436 | return; | |
437 | ||
438 | gbo = drm_gem_vram_of_bo(bo); | |
439 | drm_gem_vram_placement(gbo, TTM_PL_FLAG_SYSTEM); | |
440 | *pl = gbo->placement; | |
441 | } | |
442 | EXPORT_SYMBOL(drm_gem_vram_bo_driver_evict_flags); | |
443 | ||
444 | /** | |
445 | * drm_gem_vram_bo_driver_verify_access() - \ | |
446 | Implements &struct ttm_bo_driver.verify_access | |
447 | * @bo: TTM buffer object. Refers to &struct drm_gem_vram_object.bo | |
448 | * @filp: File pointer. | |
449 | * | |
450 | * Returns: | |
451 | * 0 on success, or | |
452 | * a negative errno code otherwise. | |
453 | */ | |
454 | int drm_gem_vram_bo_driver_verify_access(struct ttm_buffer_object *bo, | |
455 | struct file *filp) | |
456 | { | |
457 | struct drm_gem_vram_object *gbo = drm_gem_vram_of_bo(bo); | |
458 | ||
459 | return drm_vma_node_verify_access(&gbo->gem.vma_node, | |
460 | filp->private_data); | |
461 | } | |
462 | EXPORT_SYMBOL(drm_gem_vram_bo_driver_verify_access); | |
737000fd TZ |
463 | |
464 | /* | |
465 | * Helpers for struct drm_driver | |
466 | */ | |
467 | ||
468 | /** | |
469 | * drm_gem_vram_driver_gem_free_object_unlocked() - \ | |
470 | Implements &struct drm_driver.gem_free_object_unlocked | |
471 | * @gem: GEM object. Refers to &struct drm_gem_vram_object.gem | |
472 | */ | |
473 | void drm_gem_vram_driver_gem_free_object_unlocked(struct drm_gem_object *gem) | |
474 | { | |
475 | struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem); | |
476 | ||
477 | drm_gem_vram_put(gbo); | |
478 | } | |
479 | EXPORT_SYMBOL(drm_gem_vram_driver_gem_free_object_unlocked); | |
480 | ||
481 | /** | |
482 | * drm_gem_vram_driver_dumb_mmap_offset() - \ | |
483 | Implements &struct drm_driver.dumb_mmap_offset | |
484 | * @file: DRM file pointer. | |
485 | * @dev: DRM device. | |
486 | * @handle: GEM handle | |
487 | * @offset: Returns the mapping's memory offset on success | |
488 | * | |
489 | * Returns: | |
490 | * 0 on success, or | |
491 | * a negative errno code otherwise. | |
492 | */ | |
493 | int drm_gem_vram_driver_dumb_mmap_offset(struct drm_file *file, | |
494 | struct drm_device *dev, | |
495 | uint32_t handle, uint64_t *offset) | |
496 | { | |
497 | struct drm_gem_object *gem; | |
498 | struct drm_gem_vram_object *gbo; | |
499 | ||
500 | gem = drm_gem_object_lookup(file, handle); | |
501 | if (!gem) | |
502 | return -ENOENT; | |
503 | ||
504 | gbo = drm_gem_vram_of_gem(gem); | |
505 | *offset = drm_gem_vram_mmap_offset(gbo); | |
506 | ||
507 | drm_gem_object_put_unlocked(gem); | |
508 | ||
509 | return 0; | |
510 | } | |
511 | EXPORT_SYMBOL(drm_gem_vram_driver_dumb_mmap_offset); |