2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
28 #include <linux/ktime.h>
30 #include <drm/amdgpu_drm.h>
33 void amdgpu_gem_object_free(struct drm_gem_object *gobj)
35 struct amdgpu_bo *robj = gem_to_amdgpu_bo(gobj);
38 if (robj->gem_base.import_attach)
39 drm_prime_gem_destroy(&robj->gem_base, robj->tbo.sg);
40 amdgpu_bo_unref(&robj);
44 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
45 int alignment, u32 initial_domain,
46 u64 flags, bool kernel,
47 struct drm_gem_object **obj)
49 struct amdgpu_bo *robj;
50 unsigned long max_size;
54 /* At least align on page size */
55 if (alignment < PAGE_SIZE) {
56 alignment = PAGE_SIZE;
59 if (!(initial_domain & (AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA))) {
60 /* Maximum bo size is the unpinned gtt size since we use the gtt to
61 * handle vram to system pool migrations.
63 max_size = adev->mc.gtt_size - adev->gart_pin_size;
64 if (size > max_size) {
65 DRM_DEBUG("Allocation size %ldMb bigger than %ldMb limit\n",
66 size >> 20, max_size >> 20);
71 r = amdgpu_bo_create(adev, size, alignment, kernel, initial_domain, flags, NULL, &robj);
73 if (r != -ERESTARTSYS) {
74 if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
75 initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
78 DRM_ERROR("Failed to allocate GEM object (%ld, %d, %u, %d)\n",
79 size, initial_domain, alignment, r);
83 *obj = &robj->gem_base;
84 robj->pid = task_pid_nr(current);
86 mutex_lock(&adev->gem.mutex);
87 list_add_tail(&robj->list, &adev->gem.objects);
88 mutex_unlock(&adev->gem.mutex);
93 int amdgpu_gem_init(struct amdgpu_device *adev)
95 INIT_LIST_HEAD(&adev->gem.objects);
99 void amdgpu_gem_fini(struct amdgpu_device *adev)
101 amdgpu_bo_force_delete(adev);
105 * Call from drm_gem_handle_create which appear in both new and open ioctl
108 int amdgpu_gem_object_open(struct drm_gem_object *obj, struct drm_file *file_priv)
110 struct amdgpu_bo *rbo = gem_to_amdgpu_bo(obj);
111 struct amdgpu_device *adev = rbo->adev;
112 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
113 struct amdgpu_vm *vm = &fpriv->vm;
114 struct amdgpu_bo_va *bo_va;
117 r = amdgpu_bo_reserve(rbo, false);
122 bo_va = amdgpu_vm_bo_find(vm, rbo);
124 bo_va = amdgpu_vm_bo_add(adev, vm, rbo);
128 amdgpu_bo_unreserve(rbo);
133 void amdgpu_gem_object_close(struct drm_gem_object *obj,
134 struct drm_file *file_priv)
136 struct amdgpu_bo *rbo = gem_to_amdgpu_bo(obj);
137 struct amdgpu_device *adev = rbo->adev;
138 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
139 struct amdgpu_vm *vm = &fpriv->vm;
140 struct amdgpu_bo_va *bo_va;
143 r = amdgpu_bo_reserve(rbo, true);
145 dev_err(adev->dev, "leaking bo va because "
146 "we fail to reserve bo (%d)\n", r);
149 bo_va = amdgpu_vm_bo_find(vm, rbo);
151 if (--bo_va->ref_count == 0) {
152 amdgpu_vm_bo_rmv(adev, bo_va);
155 amdgpu_bo_unreserve(rbo);
158 static int amdgpu_gem_handle_lockup(struct amdgpu_device *adev, int r)
161 r = amdgpu_gpu_reset(adev);
171 int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
172 struct drm_file *filp)
174 struct amdgpu_device *adev = dev->dev_private;
175 union drm_amdgpu_gem_create *args = data;
176 uint64_t size = args->in.bo_size;
177 struct drm_gem_object *gobj;
182 down_read(&adev->exclusive_lock);
183 /* create a gem object to contain this object in */
184 if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
185 AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
187 if (args->in.domains == AMDGPU_GEM_DOMAIN_GDS)
188 size = size << AMDGPU_GDS_SHIFT;
189 else if (args->in.domains == AMDGPU_GEM_DOMAIN_GWS)
190 size = size << AMDGPU_GWS_SHIFT;
191 else if (args->in.domains == AMDGPU_GEM_DOMAIN_OA)
192 size = size << AMDGPU_OA_SHIFT;
198 size = roundup(size, PAGE_SIZE);
200 r = amdgpu_gem_object_create(adev, size, args->in.alignment,
201 (u32)(0xffffffff & args->in.domains),
202 args->in.domain_flags,
207 r = drm_gem_handle_create(filp, gobj, &handle);
208 /* drop reference from allocate - handle holds it now */
209 drm_gem_object_unreference_unlocked(gobj);
213 memset(args, 0, sizeof(*args));
214 args->out.handle = handle;
215 up_read(&adev->exclusive_lock);
219 up_read(&adev->exclusive_lock);
220 r = amdgpu_gem_handle_lockup(adev, r);
224 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
225 struct drm_file *filp)
227 struct amdgpu_device *adev = dev->dev_private;
228 struct drm_amdgpu_gem_userptr *args = data;
229 struct drm_gem_object *gobj;
230 struct amdgpu_bo *bo;
234 if (offset_in_page(args->addr | args->size))
237 /* reject unknown flag values */
238 if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
239 AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
240 AMDGPU_GEM_USERPTR_REGISTER))
243 if (!(args->flags & AMDGPU_GEM_USERPTR_ANONONLY) ||
244 !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
246 /* if we want to write to it we must require anonymous
247 memory and install a MMU notifier */
251 down_read(&adev->exclusive_lock);
253 /* create a gem object to contain this object in */
254 r = amdgpu_gem_object_create(adev, args->size, 0,
255 AMDGPU_GEM_DOMAIN_CPU, 0,
260 bo = gem_to_amdgpu_bo(gobj);
261 r = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, args->addr, args->flags);
265 if (args->flags & AMDGPU_GEM_USERPTR_REGISTER) {
266 r = amdgpu_mn_register(bo, args->addr);
271 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
272 down_read(¤t->mm->mmap_sem);
273 r = amdgpu_bo_reserve(bo, true);
275 up_read(¤t->mm->mmap_sem);
279 amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
280 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
281 amdgpu_bo_unreserve(bo);
282 up_read(¤t->mm->mmap_sem);
287 r = drm_gem_handle_create(filp, gobj, &handle);
288 /* drop reference from allocate - handle holds it now */
289 drm_gem_object_unreference_unlocked(gobj);
293 args->handle = handle;
294 up_read(&adev->exclusive_lock);
298 drm_gem_object_unreference_unlocked(gobj);
301 up_read(&adev->exclusive_lock);
302 r = amdgpu_gem_handle_lockup(adev, r);
307 int amdgpu_mode_dumb_mmap(struct drm_file *filp,
308 struct drm_device *dev,
309 uint32_t handle, uint64_t *offset_p)
311 struct drm_gem_object *gobj;
312 struct amdgpu_bo *robj;
314 gobj = drm_gem_object_lookup(dev, filp, handle);
318 robj = gem_to_amdgpu_bo(gobj);
319 if (amdgpu_ttm_tt_has_userptr(robj->tbo.ttm)) {
320 drm_gem_object_unreference_unlocked(gobj);
323 *offset_p = amdgpu_bo_mmap_offset(robj);
324 drm_gem_object_unreference_unlocked(gobj);
328 int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
329 struct drm_file *filp)
331 union drm_amdgpu_gem_mmap *args = data;
332 uint32_t handle = args->in.handle;
333 memset(args, 0, sizeof(*args));
334 return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);
338 * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
340 * @timeout_ns: timeout in ns
342 * Calculate the timeout in jiffies from an absolute timeout in ns.
344 unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
346 unsigned long timeout_jiffies;
349 /* clamp timeout if it's to large */
350 if (((int64_t)timeout_ns) < 0)
351 return MAX_SCHEDULE_TIMEOUT;
353 timeout = ktime_sub_ns(ktime_get(), timeout_ns);
354 if (ktime_to_ns(timeout) < 0)
357 timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));
358 /* clamp timeout to avoid unsigned-> signed overflow */
359 if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT )
360 return MAX_SCHEDULE_TIMEOUT - 1;
362 return timeout_jiffies;
365 int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
366 struct drm_file *filp)
368 struct amdgpu_device *adev = dev->dev_private;
369 union drm_amdgpu_gem_wait_idle *args = data;
370 struct drm_gem_object *gobj;
371 struct amdgpu_bo *robj;
372 uint32_t handle = args->in.handle;
373 unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
377 gobj = drm_gem_object_lookup(dev, filp, handle);
381 robj = gem_to_amdgpu_bo(gobj);
383 ret = reservation_object_test_signaled_rcu(robj->tbo.resv, true);
385 ret = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true, timeout);
387 /* ret == 0 means not signaled,
388 * ret > 0 means signaled
389 * ret < 0 means interrupted before timeout
392 memset(args, 0, sizeof(*args));
393 args->out.status = (ret == 0);
397 drm_gem_object_unreference_unlocked(gobj);
398 r = amdgpu_gem_handle_lockup(adev, r);
402 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
403 struct drm_file *filp)
405 struct drm_amdgpu_gem_metadata *args = data;
406 struct drm_gem_object *gobj;
407 struct amdgpu_bo *robj;
410 DRM_DEBUG("%d \n", args->handle);
411 gobj = drm_gem_object_lookup(dev, filp, args->handle);
414 robj = gem_to_amdgpu_bo(gobj);
416 r = amdgpu_bo_reserve(robj, false);
417 if (unlikely(r != 0))
420 if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
421 amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
422 r = amdgpu_bo_get_metadata(robj, args->data.data,
423 sizeof(args->data.data),
424 &args->data.data_size_bytes,
426 } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
427 r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
429 r = amdgpu_bo_set_metadata(robj, args->data.data,
430 args->data.data_size_bytes,
434 amdgpu_bo_unreserve(robj);
436 drm_gem_object_unreference_unlocked(gobj);
441 * amdgpu_gem_va_update_vm -update the bo_va in its VM
443 * @adev: amdgpu_device pointer
444 * @bo_va: bo_va to update
446 * Update the bo_va directly after setting it's address. Errors are not
447 * vital here, so they are not reported back to userspace.
449 static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
450 struct amdgpu_bo_va *bo_va)
452 struct ttm_validate_buffer tv, *entry;
453 struct amdgpu_bo_list_entry *vm_bos;
454 struct ww_acquire_ctx ticket;
455 struct list_head list;
459 INIT_LIST_HEAD(&list);
461 tv.bo = &bo_va->bo->tbo;
463 list_add(&tv.head, &list);
465 vm_bos = amdgpu_vm_get_bos(adev, bo_va->vm, &list);
469 r = ttm_eu_reserve_buffers(&ticket, &list, true, NULL);
473 list_for_each_entry(entry, &list, head) {
474 domain = amdgpu_mem_type_to_domain(entry->bo->mem.mem_type);
475 /* if anything is swapped out don't swap it in here,
476 just abort and wait for the next CS */
477 if (domain == AMDGPU_GEM_DOMAIN_CPU)
478 goto error_unreserve;
481 mutex_lock(&bo_va->vm->mutex);
482 r = amdgpu_vm_clear_freed(adev, bo_va->vm);
486 r = amdgpu_vm_bo_update(adev, bo_va, &bo_va->bo->tbo.mem);
489 mutex_unlock(&bo_va->vm->mutex);
492 ttm_eu_backoff_reservation(&ticket, &list);
495 drm_free_large(vm_bos);
498 DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
503 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
504 struct drm_file *filp)
506 union drm_amdgpu_gem_va *args = data;
507 struct drm_gem_object *gobj;
508 struct amdgpu_device *adev = dev->dev_private;
509 struct amdgpu_fpriv *fpriv = filp->driver_priv;
510 struct amdgpu_bo *rbo;
511 struct amdgpu_bo_va *bo_va;
512 uint32_t invalid_flags, va_flags = 0;
515 if (!adev->vm_manager.enabled) {
516 memset(args, 0, sizeof(*args));
517 args->out.result = AMDGPU_VA_RESULT_ERROR;
521 if (args->in.va_address < AMDGPU_VA_RESERVED_SIZE) {
522 dev_err(&dev->pdev->dev,
523 "va_address 0x%lX is in reserved area 0x%X\n",
524 (unsigned long)args->in.va_address,
525 AMDGPU_VA_RESERVED_SIZE);
526 memset(args, 0, sizeof(*args));
527 args->out.result = AMDGPU_VA_RESULT_ERROR;
531 invalid_flags = ~(AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
532 AMDGPU_VM_PAGE_EXECUTABLE);
533 if ((args->in.flags & invalid_flags)) {
534 dev_err(&dev->pdev->dev, "invalid flags 0x%08X vs 0x%08X\n",
535 args->in.flags, invalid_flags);
536 memset(args, 0, sizeof(*args));
537 args->out.result = AMDGPU_VA_RESULT_ERROR;
541 switch (args->in.operation) {
542 case AMDGPU_VA_OP_MAP:
543 case AMDGPU_VA_OP_UNMAP:
546 dev_err(&dev->pdev->dev, "unsupported operation %d\n",
548 memset(args, 0, sizeof(*args));
549 args->out.result = AMDGPU_VA_RESULT_ERROR;
553 gobj = drm_gem_object_lookup(dev, filp, args->in.handle);
555 memset(args, 0, sizeof(*args));
556 args->out.result = AMDGPU_VA_RESULT_ERROR;
559 rbo = gem_to_amdgpu_bo(gobj);
560 r = amdgpu_bo_reserve(rbo, false);
562 if (r != -ERESTARTSYS) {
563 memset(args, 0, sizeof(*args));
564 args->out.result = AMDGPU_VA_RESULT_ERROR;
566 drm_gem_object_unreference_unlocked(gobj);
569 bo_va = amdgpu_vm_bo_find(&fpriv->vm, rbo);
571 memset(args, 0, sizeof(*args));
572 args->out.result = AMDGPU_VA_RESULT_ERROR;
573 drm_gem_object_unreference_unlocked(gobj);
577 switch (args->in.operation) {
578 case AMDGPU_VA_OP_MAP:
579 if (args->in.flags & AMDGPU_VM_PAGE_READABLE)
580 va_flags |= AMDGPU_PTE_READABLE;
581 if (args->in.flags & AMDGPU_VM_PAGE_WRITEABLE)
582 va_flags |= AMDGPU_PTE_WRITEABLE;
583 if (args->in.flags & AMDGPU_VM_PAGE_EXECUTABLE)
584 va_flags |= AMDGPU_PTE_EXECUTABLE;
585 r = amdgpu_vm_bo_map(adev, bo_va, args->in.va_address, 0,
586 amdgpu_bo_size(bo_va->bo), va_flags);
588 case AMDGPU_VA_OP_UNMAP:
589 r = amdgpu_vm_bo_unmap(adev, bo_va, args->in.va_address);
596 amdgpu_gem_va_update_vm(adev, bo_va);
597 memset(args, 0, sizeof(*args));
598 args->out.result = AMDGPU_VA_RESULT_OK;
600 memset(args, 0, sizeof(*args));
601 args->out.result = AMDGPU_VA_RESULT_ERROR;
604 drm_gem_object_unreference_unlocked(gobj);
608 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
609 struct drm_file *filp)
611 struct drm_amdgpu_gem_op *args = data;
612 struct drm_gem_object *gobj;
613 struct amdgpu_bo *robj;
616 gobj = drm_gem_object_lookup(dev, filp, args->handle);
620 robj = gem_to_amdgpu_bo(gobj);
622 r = amdgpu_bo_reserve(robj, false);
627 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
628 struct drm_amdgpu_gem_create_in info;
629 void __user *out = (void __user *)(long)args->value;
631 info.bo_size = robj->gem_base.size;
632 info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT;
633 info.domains = robj->initial_domain;
634 info.domain_flags = robj->flags;
635 if (copy_to_user(out, &info, sizeof(info)))
639 case AMDGPU_GEM_OP_SET_INITIAL_DOMAIN:
640 if (amdgpu_ttm_tt_has_userptr(robj->tbo.ttm)) {
644 robj->initial_domain = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
645 AMDGPU_GEM_DOMAIN_GTT |
646 AMDGPU_GEM_DOMAIN_CPU);
652 amdgpu_bo_unreserve(robj);
654 drm_gem_object_unreference_unlocked(gobj);
658 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
659 struct drm_device *dev,
660 struct drm_mode_create_dumb *args)
662 struct amdgpu_device *adev = dev->dev_private;
663 struct drm_gem_object *gobj;
667 args->pitch = amdgpu_align_pitch(adev, args->width, args->bpp, 0) * ((args->bpp + 1) / 8);
668 args->size = args->pitch * args->height;
669 args->size = ALIGN(args->size, PAGE_SIZE);
671 r = amdgpu_gem_object_create(adev, args->size, 0,
672 AMDGPU_GEM_DOMAIN_VRAM,
673 0, ttm_bo_type_device,
678 r = drm_gem_handle_create(file_priv, gobj, &handle);
679 /* drop reference from allocate - handle holds it now */
680 drm_gem_object_unreference_unlocked(gobj);
684 args->handle = handle;
688 #if defined(CONFIG_DEBUG_FS)
689 static int amdgpu_debugfs_gem_info(struct seq_file *m, void *data)
691 struct drm_info_node *node = (struct drm_info_node *)m->private;
692 struct drm_device *dev = node->minor->dev;
693 struct amdgpu_device *adev = dev->dev_private;
694 struct amdgpu_bo *rbo;
697 mutex_lock(&adev->gem.mutex);
698 list_for_each_entry(rbo, &adev->gem.objects, list) {
700 const char *placement;
702 domain = amdgpu_mem_type_to_domain(rbo->tbo.mem.mem_type);
704 case AMDGPU_GEM_DOMAIN_VRAM:
707 case AMDGPU_GEM_DOMAIN_GTT:
710 case AMDGPU_GEM_DOMAIN_CPU:
715 seq_printf(m, "bo[0x%08x] %8ldkB %8ldMB %s pid %8ld\n",
716 i, amdgpu_bo_size(rbo) >> 10, amdgpu_bo_size(rbo) >> 20,
717 placement, (unsigned long)rbo->pid);
720 mutex_unlock(&adev->gem.mutex);
724 static struct drm_info_list amdgpu_debugfs_gem_list[] = {
725 {"amdgpu_gem_info", &amdgpu_debugfs_gem_info, 0, NULL},
729 int amdgpu_gem_debugfs_init(struct amdgpu_device *adev)
731 #if defined(CONFIG_DEBUG_FS)
732 return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_gem_list, 1);