drivers/gpu/drm/radeon/radeon_object.c

   1 /*
   2  * Copyright 2009 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sub license, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * The above copyright notice and this permission notice (including the
  22  * next paragraph) shall be included in all copies or substantial portions
  23  * of the Software.
  24  *
  25  */
  26 /*
  27  * Authors:
  28  *    Jerome Glisse <glisse@freedesktop.org>
  29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30  *    Dave Airlie
  31  */
  32 #include <linux/list.h>
  33 #include <linux/slab.h>
  34 #include <drm/drmP.h>
  35 #include <drm/radeon_drm.h>
  36 #include "radeon.h"
  37 #include "radeon_trace.h"
  38
  39
  40 int radeon_ttm_init(struct radeon_device *rdev);
  41 void radeon_ttm_fini(struct radeon_device *rdev);
  42 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo);
  43
  44 /*
  45  * To exclude mutual BO access we rely on bo_reserve exclusion, as all
  46  * function are calling it.
  47  */
  48
  49 static void radeon_bo_clear_va(struct radeon_bo *bo)
  50 {
  51         struct radeon_bo_va *bo_va, *tmp;
  52
  53         list_for_each_entry_safe(bo_va, tmp, &bo->va, bo_list) {
  54                 /* remove from all vm address space */
  55                 radeon_vm_bo_rmv(bo->rdev, bo_va);
  56         }
  57 }
  58
  59 static void radeon_update_memory_usage(struct radeon_bo *bo,
  60                                        unsigned mem_type, int sign)
  61 {
  62         struct radeon_device *rdev = bo->rdev;
  63         u64 size = (u64)bo->tbo.num_pages << PAGE_SHIFT;
  64
  65         switch (mem_type) {
  66         case TTM_PL_TT:
  67                 if (sign > 0)
  68                         atomic64_add(size, &rdev->gtt_usage);
  69                 else
  70                         atomic64_sub(size, &rdev->gtt_usage);
  71                 break;
  72         case TTM_PL_VRAM:
  73                 if (sign > 0)
  74                         atomic64_add(size, &rdev->vram_usage);
  75                 else
  76                         atomic64_sub(size, &rdev->vram_usage);
  77                 break;
  78         }
  79 }
  80
  81 static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo)
  82 {
  83         struct radeon_bo *bo;
  84
  85         bo = container_of(tbo, struct radeon_bo, tbo);
  86
  87         radeon_update_memory_usage(bo, bo->tbo.mem.mem_type, -1);
  88
  89         mutex_lock(&bo->rdev->gem.mutex);
  90         list_del_init(&bo->list);
  91         mutex_unlock(&bo->rdev->gem.mutex);
  92         radeon_bo_clear_surface_reg(bo);
  93         radeon_bo_clear_va(bo);
  94         drm_gem_object_release(&bo->gem_base);
  95         kfree(bo);
  96 }
  97
  98 bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo)
  99 {
 100         if (bo->destroy == &radeon_ttm_bo_destroy)
 101                 return true;
 102         return false;
 103 }
 104
 105 void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
 106 {
 107         u32 c = 0, i;
 108
 109         rbo->placement.fpfn = 0;
 110         rbo->placement.lpfn = 0;
 111         rbo->placement.placement = rbo->placements;
 112         rbo->placement.busy_placement = rbo->placements;
 113         if (domain & RADEON_GEM_DOMAIN_VRAM)
 114                 rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
 115                                         TTM_PL_FLAG_VRAM;
 116         if (domain & RADEON_GEM_DOMAIN_GTT) {
 117                 if (rbo->rdev->flags & RADEON_IS_AGP) {
 118                         rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT;
 119                 } else {
 120                         rbo->placements[c++] = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT;
 121                 }
 122         }
 123         if (domain & RADEON_GEM_DOMAIN_CPU) {
 124                 if (rbo->rdev->flags & RADEON_IS_AGP) {
 125                         rbo->placements[c++] = TTM_PL_FLAG_WC | TTM_PL_FLAG_SYSTEM;
 126                 } else {
 127                         rbo->placements[c++] = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM;
 128                 }
 129         }
 130         if (!c)
 131                 rbo->placements[c++] = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
 132         rbo->placement.num_placement = c;
 133         rbo->placement.num_busy_placement = c;
 134
 135         /*
 136          * Use two-ended allocation depending on the buffer size to
 137          * improve fragmentation quality.
 138          * 512kb was measured as the most optimal number.
 139          */
 140         if (rbo->tbo.mem.size > 512 * 1024) {
 141                 for (i = 0; i < c; i++) {
 142                         rbo->placements[i] |= TTM_PL_FLAG_TOPDOWN;
 143                 }
 144         }
 145 }
 146
 147 int radeon_bo_create(struct radeon_device *rdev,
 148                      unsigned long size, int byte_align, bool kernel, u32 domain,
 149                      struct sg_table *sg, struct radeon_bo **bo_ptr)
 150 {
 151         struct radeon_bo *bo;
 152         enum ttm_bo_type type;
 153         unsigned long page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
 154         size_t acc_size;
 155         int r;
 156
 157         size = ALIGN(size, PAGE_SIZE);
 158
 159         if (kernel) {
 160                 type = ttm_bo_type_kernel;
 161         } else if (sg) {
 162                 type = ttm_bo_type_sg;
 163         } else {
 164                 type = ttm_bo_type_device;
 165         }
 166         *bo_ptr = NULL;
 167
 168         acc_size = ttm_bo_dma_acc_size(&rdev->mman.bdev, size,
 169                                        sizeof(struct radeon_bo));
 170
 171         bo = kzalloc(sizeof(struct radeon_bo), GFP_KERNEL);
 172         if (bo == NULL)
 173                 return -ENOMEM;
 174         r = drm_gem_object_init(rdev->ddev, &bo->gem_base, size);
 175         if (unlikely(r)) {
 176                 kfree(bo);
 177                 return r;
 178         }
 179         bo->rdev = rdev;
 180         bo->surface_reg = -1;
 181         INIT_LIST_HEAD(&bo->list);
 182         INIT_LIST_HEAD(&bo->va);
 183         bo->initial_domain = domain & (RADEON_GEM_DOMAIN_VRAM |
 184                                        RADEON_GEM_DOMAIN_GTT |
 185                                        RADEON_GEM_DOMAIN_CPU);
 186         radeon_ttm_placement_from_domain(bo, domain);
 187         /* Kernel allocation are uninterruptible */
 188         down_read(&rdev->pm.mclk_lock);
 189         r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type,
 190                         &bo->placement, page_align, !kernel, NULL,
 191                         acc_size, sg, &radeon_ttm_bo_destroy);
 192         up_read(&rdev->pm.mclk_lock);
 193         if (unlikely(r != 0)) {
 194                 return r;
 195         }
 196         *bo_ptr = bo;
 197
 198         trace_radeon_bo_create(bo);
 199
 200         return 0;
 201 }
 202
 203 int radeon_bo_kmap(struct radeon_bo *bo, void **ptr)
 204 {
 205         bool is_iomem;
 206         int r;
 207
 208         if (bo->kptr) {
 209                 if (ptr) {
 210                         *ptr = bo->kptr;
 211                 }
 212                 return 0;
 213         }
 214         r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
 215         if (r) {
 216                 return r;
 217         }
 218         bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
 219         if (ptr) {
 220                 *ptr = bo->kptr;
 221         }
 222         radeon_bo_check_tiling(bo, 0, 0);
 223         return 0;
 224 }
 225
 226 void radeon_bo_kunmap(struct radeon_bo *bo)
 227 {
 228         if (bo->kptr == NULL)
 229                 return;
 230         bo->kptr = NULL;
 231         radeon_bo_check_tiling(bo, 0, 0);
 232         ttm_bo_kunmap(&bo->kmap);
 233 }
 234
 235 void radeon_bo_unref(struct radeon_bo **bo)
 236 {
 237         struct ttm_buffer_object *tbo;
 238         struct radeon_device *rdev;
 239
 240         if ((*bo) == NULL)
 241                 return;
 242         rdev = (*bo)->rdev;
 243         tbo = &((*bo)->tbo);
 244         down_read(&rdev->pm.mclk_lock);
 245         ttm_bo_unref(&tbo);
 246         up_read(&rdev->pm.mclk_lock);
 247         if (tbo == NULL)
 248                 *bo = NULL;
 249 }
 250
 251 int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset,
 252                              u64 *gpu_addr)
 253 {
 254         int r, i;
 255
 256         if (bo->pin_count) {
 257                 bo->pin_count++;
 258                 if (gpu_addr)
 259                         *gpu_addr = radeon_bo_gpu_offset(bo);
 260
 261                 if (max_offset != 0) {
 262                         u64 domain_start;
 263
 264                         if (domain == RADEON_GEM_DOMAIN_VRAM)
 265                                 domain_start = bo->rdev->mc.vram_start;
 266                         else
 267                                 domain_start = bo->rdev->mc.gtt_start;
 268                         WARN_ON_ONCE(max_offset <
 269                                      (radeon_bo_gpu_offset(bo) - domain_start));
 270                 }
 271
 272                 return 0;
 273         }
 274         radeon_ttm_placement_from_domain(bo, domain);
 275         if (domain == RADEON_GEM_DOMAIN_VRAM) {
 276                 /* force to pin into visible video ram */
 277                 bo->placement.lpfn = bo->rdev->mc.visible_vram_size >> PAGE_SHIFT;
 278         }
 279         if (max_offset) {
 280                 u64 lpfn = max_offset >> PAGE_SHIFT;
 281
 282                 if (!bo->placement.lpfn)
 283                         bo->placement.lpfn = bo->rdev->mc.gtt_size >> PAGE_SHIFT;
 284
 285                 if (lpfn < bo->placement.lpfn)
 286                         bo->placement.lpfn = lpfn;
 287         }
 288         for (i = 0; i < bo->placement.num_placement; i++)
 289                 bo->placements[i] |= TTM_PL_FLAG_NO_EVICT;
 290         r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
 291         if (likely(r == 0)) {
 292                 bo->pin_count = 1;
 293                 if (gpu_addr != NULL)
 294                         *gpu_addr = radeon_bo_gpu_offset(bo);
 295         }
 296         if (unlikely(r != 0))
 297                 dev_err(bo->rdev->dev, "%p pin failed\n", bo);
 298         return r;
 299 }
 300
 301 int radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr)
 302 {
 303         return radeon_bo_pin_restricted(bo, domain, 0, gpu_addr);
 304 }
 305
 306 int radeon_bo_unpin(struct radeon_bo *bo)
 307 {
 308         int r, i;
 309
 310         if (!bo->pin_count) {
 311                 dev_warn(bo->rdev->dev, "%p unpin not necessary\n", bo);
 312                 return 0;
 313         }
 314         bo->pin_count--;
 315         if (bo->pin_count)
 316                 return 0;
 317         for (i = 0; i < bo->placement.num_placement; i++)
 318                 bo->placements[i] &= ~TTM_PL_FLAG_NO_EVICT;
 319         r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
 320         if (unlikely(r != 0))
 321                 dev_err(bo->rdev->dev, "%p validate failed for unpin\n", bo);
 322         return r;
 323 }
 324
 325 int radeon_bo_evict_vram(struct radeon_device *rdev)
 326 {
 327         /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
 328         if (0 && (rdev->flags & RADEON_IS_IGP)) {
 329                 if (rdev->mc.igp_sideport_enabled == false)
 330                         /* Useless to evict on IGP chips */
 331                         return 0;
 332         }
 333         return ttm_bo_evict_mm(&rdev->mman.bdev, TTM_PL_VRAM);
 334 }
 335
 336 void radeon_bo_force_delete(struct radeon_device *rdev)
 337 {
 338         struct radeon_bo *bo, *n;
 339
 340         if (list_empty(&rdev->gem.objects)) {
 341                 return;
 342         }
 343         dev_err(rdev->dev, "Userspace still has active objects !\n");
 344         list_for_each_entry_safe(bo, n, &rdev->gem.objects, list) {
 345                 mutex_lock(&rdev->ddev->struct_mutex);
 346                 dev_err(rdev->dev, "%p %p %lu %lu force free\n",
 347                         &bo->gem_base, bo, (unsigned long)bo->gem_base.size,
 348                         *((unsigned long *)&bo->gem_base.refcount));
 349                 mutex_lock(&bo->rdev->gem.mutex);
 350                 list_del_init(&bo->list);
 351                 mutex_unlock(&bo->rdev->gem.mutex);
 352                 /* this should unref the ttm bo */
 353                 drm_gem_object_unreference(&bo->gem_base);
 354                 mutex_unlock(&rdev->ddev->struct_mutex);
 355         }
 356 }
 357
 358 int radeon_bo_init(struct radeon_device *rdev)
 359 {
 360         /* Add an MTRR for the VRAM */
 361         if (!rdev->fastfb_working) {
 362                 rdev->mc.vram_mtrr = arch_phys_wc_add(rdev->mc.aper_base,
 363                                                       rdev->mc.aper_size);
 364         }
 365         DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
 366                 rdev->mc.mc_vram_size >> 20,
 367                 (unsigned long long)rdev->mc.aper_size >> 20);
 368         DRM_INFO("RAM width %dbits %cDR\n",
 369                         rdev->mc.vram_width, rdev->mc.vram_is_ddr ? 'D' : 'S');
 370         return radeon_ttm_init(rdev);
 371 }
 372
 373 void radeon_bo_fini(struct radeon_device *rdev)
 374 {
 375         radeon_ttm_fini(rdev);
 376         arch_phys_wc_del(rdev->mc.vram_mtrr);
 377 }
 378
 379 /* Returns how many bytes TTM can move per IB.
 380  */
 381 static u64 radeon_bo_get_threshold_for_moves(struct radeon_device *rdev)
 382 {
 383         u64 real_vram_size = rdev->mc.real_vram_size;
 384         u64 vram_usage = atomic64_read(&rdev->vram_usage);
 385
 386         /* This function is based on the current VRAM usage.
 387          *
 388          * - If all of VRAM is free, allow relocating the number of bytes that
 389          *   is equal to 1/4 of the size of VRAM for this IB.
 390
 391          * - If more than one half of VRAM is occupied, only allow relocating
 392          *   1 MB of data for this IB.
 393          *
 394          * - From 0 to one half of used VRAM, the threshold decreases
 395          *   linearly.
 396          *         __________________
 397          * 1/4 of -|\               |
 398          * VRAM    | \              |
 399          *         |  \             |
 400          *         |   \            |
 401          *         |    \           |
 402          *         |     \          |
 403          *         |      \         |
 404          *         |       \________|1 MB
 405          *         |----------------|
 406          *    VRAM 0 %             100 %
 407          *         used            used
 408          *
 409          * Note: It's a threshold, not a limit. The threshold must be crossed
 410          * for buffer relocations to stop, so any buffer of an arbitrary size
 411          * can be moved as long as the threshold isn't crossed before
 412          * the relocation takes place. We don't want to disable buffer
 413          * relocations completely.
 414          *
 415          * The idea is that buffers should be placed in VRAM at creation time
 416          * and TTM should only do a minimum number of relocations during
 417          * command submission. In practice, you need to submit at least
 418          * a dozen IBs to move all buffers to VRAM if they are in GTT.
 419          *
 420          * Also, things can get pretty crazy under memory pressure and actual
 421          * VRAM usage can change a lot, so playing safe even at 50% does
 422          * consistently increase performance.
 423          */
 424
 425         u64 half_vram = real_vram_size >> 1;
 426         u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage;
 427         u64 bytes_moved_threshold = half_free_vram >> 1;
 428         return max(bytes_moved_threshold, 1024*1024ull);
 429 }
 430
 431 int radeon_bo_list_validate(struct radeon_device *rdev,
 432                             struct ww_acquire_ctx *ticket,
 433                             struct list_head *head, int ring)
 434 {
 435         struct radeon_cs_reloc *lobj;
 436         struct radeon_bo *bo;
 437         int r;
 438         u64 bytes_moved = 0, initial_bytes_moved;
 439         u64 bytes_moved_threshold = radeon_bo_get_threshold_for_moves(rdev);
 440
 441         r = ttm_eu_reserve_buffers(ticket, head);
 442         if (unlikely(r != 0)) {
 443                 return r;
 444         }
 445
 446         list_for_each_entry(lobj, head, tv.head) {
 447                 bo = lobj->robj;
 448                 if (!bo->pin_count) {
 449                         u32 domain = lobj->prefered_domains;
 450                         u32 current_domain =
 451                                 radeon_mem_type_to_domain(bo->tbo.mem.mem_type);
 452
 453                         /* Check if this buffer will be moved and don't move it
 454                          * if we have moved too many buffers for this IB already.
 455                          *
 456                          * Note that this allows moving at least one buffer of
 457                          * any size, because it doesn't take the current "bo"
 458                          * into account. We don't want to disallow buffer moves
 459                          * completely.
 460                          */
 461                         if ((lobj->allowed_domains & current_domain) != 0 &&
 462                             (domain & current_domain) == 0 && /* will be moved */
 463                             bytes_moved > bytes_moved_threshold) {
 464                                 /* don't move it */
 465                                 domain = current_domain;
 466                         }
 467
 468                 retry:
 469                         radeon_ttm_placement_from_domain(bo, domain);
 470                         if (ring == R600_RING_TYPE_UVD_INDEX)
 471                                 radeon_uvd_force_into_uvd_segment(bo);
 472
 473                         initial_bytes_moved = atomic64_read(&rdev->num_bytes_moved);
 474                         r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
 475                         bytes_moved += atomic64_read(&rdev->num_bytes_moved) -
 476                                        initial_bytes_moved;
 477
 478                         if (unlikely(r)) {
 479                                 if (r != -ERESTARTSYS &&
 480                                     domain != lobj->allowed_domains) {
 481                                         domain = lobj->allowed_domains;
 482                                         goto retry;
 483                                 }
 484                                 ttm_eu_backoff_reservation(ticket, head);
 485                                 return r;
 486                         }
 487                 }
 488                 lobj->gpu_offset = radeon_bo_gpu_offset(bo);
 489                 lobj->tiling_flags = bo->tiling_flags;
 490         }
 491         return 0;
 492 }
 493
 494 int radeon_bo_fbdev_mmap(struct radeon_bo *bo,
 495                              struct vm_area_struct *vma)
 496 {
 497         return ttm_fbdev_mmap(vma, &bo->tbo);
 498 }
 499
 500 int radeon_bo_get_surface_reg(struct radeon_bo *bo)
 501 {
 502         struct radeon_device *rdev = bo->rdev;
 503         struct radeon_surface_reg *reg;
 504         struct radeon_bo *old_object;
 505         int steal;
 506         int i;
 507
 508         lockdep_assert_held(&bo->tbo.resv->lock.base);
 509
 510         if (!bo->tiling_flags)
 511                 return 0;
 512
 513         if (bo->surface_reg >= 0) {
 514                 reg = &rdev->surface_regs[bo->surface_reg];
 515                 i = bo->surface_reg;
 516                 goto out;
 517         }
 518
 519         steal = -1;
 520         for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) {
 521
 522                 reg = &rdev->surface_regs[i];
 523                 if (!reg->bo)
 524                         break;
 525
 526                 old_object = reg->bo;
 527                 if (old_object->pin_count == 0)
 528                         steal = i;
 529         }
 530
 531         /* if we are all out */
 532         if (i == RADEON_GEM_MAX_SURFACES) {
 533                 if (steal == -1)
 534                         return -ENOMEM;
 535                 /* find someone with a surface reg and nuke their BO */
 536                 reg = &rdev->surface_regs[steal];
 537                 old_object = reg->bo;
 538                 /* blow away the mapping */
 539                 DRM_DEBUG("stealing surface reg %d from %p\n", steal, old_object);
 540                 ttm_bo_unmap_virtual(&old_object->tbo);
 541                 old_object->surface_reg = -1;
 542                 i = steal;
 543         }
 544
 545         bo->surface_reg = i;
 546         reg->bo = bo;
 547
 548 out:
 549         radeon_set_surface_reg(rdev, i, bo->tiling_flags, bo->pitch,
 550                                bo->tbo.mem.start << PAGE_SHIFT,
 551                                bo->tbo.num_pages << PAGE_SHIFT);
 552         return 0;
 553 }
 554
 555 static void radeon_bo_clear_surface_reg(struct radeon_bo *bo)
 556 {
 557         struct radeon_device *rdev = bo->rdev;
 558         struct radeon_surface_reg *reg;
 559
 560         if (bo->surface_reg == -1)
 561                 return;
 562
 563         reg = &rdev->surface_regs[bo->surface_reg];
 564         radeon_clear_surface_reg(rdev, bo->surface_reg);
 565
 566         reg->bo = NULL;
 567         bo->surface_reg = -1;
 568 }
 569
 570 int radeon_bo_set_tiling_flags(struct radeon_bo *bo,
 571                                 uint32_t tiling_flags, uint32_t pitch)
 572 {
 573         struct radeon_device *rdev = bo->rdev;
 574         int r;
 575
 576         if (rdev->family >= CHIP_CEDAR) {
 577                 unsigned bankw, bankh, mtaspect, tilesplit, stilesplit;
 578
 579                 bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
 580                 bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
 581                 mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
 582                 tilesplit = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
 583                 stilesplit = (tiling_flags >> RADEON_TILING_EG_STENCIL_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_STENCIL_TILE_SPLIT_MASK;
 584                 switch (bankw) {
 585                 case 0:
 586                 case 1:
 587                 case 2:
 588                 case 4:
 589                 case 8:
 590                         break;
 591                 default:
 592                         return -EINVAL;
 593                 }
 594                 switch (bankh) {
 595                 case 0:
 596                 case 1:
 597                 case 2:
 598                 case 4:
 599                 case 8:
 600                         break;
 601                 default:
 602                         return -EINVAL;
 603                 }
 604                 switch (mtaspect) {
 605                 case 0:
 606                 case 1:
 607                 case 2:
 608                 case 4:
 609                 case 8:
 610                         break;
 611                 default:
 612                         return -EINVAL;
 613                 }
 614                 if (tilesplit > 6) {
 615                         return -EINVAL;
 616                 }
 617                 if (stilesplit > 6) {
 618                         return -EINVAL;
 619                 }
 620         }
 621         r = radeon_bo_reserve(bo, false);
 622         if (unlikely(r != 0))
 623                 return r;
 624         bo->tiling_flags = tiling_flags;
 625         bo->pitch = pitch;
 626         radeon_bo_unreserve(bo);
 627         return 0;
 628 }
 629
 630 void radeon_bo_get_tiling_flags(struct radeon_bo *bo,
 631                                 uint32_t *tiling_flags,
 632                                 uint32_t *pitch)
 633 {
 634         lockdep_assert_held(&bo->tbo.resv->lock.base);
 635
 636         if (tiling_flags)
 637                 *tiling_flags = bo->tiling_flags;
 638         if (pitch)
 639                 *pitch = bo->pitch;
 640 }
 641
 642 int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,
 643                                 bool force_drop)
 644 {
 645         if (!force_drop)
 646                 lockdep_assert_held(&bo->tbo.resv->lock.base);
 647
 648         if (!(bo->tiling_flags & RADEON_TILING_SURFACE))
 649                 return 0;
 650
 651         if (force_drop) {
 652                 radeon_bo_clear_surface_reg(bo);
 653                 return 0;
 654         }
 655
 656         if (bo->tbo.mem.mem_type != TTM_PL_VRAM) {
 657                 if (!has_moved)
 658                         return 0;
 659
 660                 if (bo->surface_reg >= 0)
 661                         radeon_bo_clear_surface_reg(bo);
 662                 return 0;
 663         }
 664
 665         if ((bo->surface_reg >= 0) && !has_moved)
 666                 return 0;
 667
 668         return radeon_bo_get_surface_reg(bo);
 669 }
 670
 671 void radeon_bo_move_notify(struct ttm_buffer_object *bo,
 672                            struct ttm_mem_reg *new_mem)
 673 {
 674         struct radeon_bo *rbo;
 675
 676         if (!radeon_ttm_bo_is_radeon_bo(bo))
 677                 return;
 678
 679         rbo = container_of(bo, struct radeon_bo, tbo);
 680         radeon_bo_check_tiling(rbo, 0, 1);
 681         radeon_vm_bo_invalidate(rbo->rdev, rbo);
 682
 683         /* update statistics */
 684         if (!new_mem)
 685                 return;
 686
 687         radeon_update_memory_usage(rbo, bo->mem.mem_type, -1);
 688         radeon_update_memory_usage(rbo, new_mem->mem_type, 1);
 689 }
 690
 691 int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 692 {
 693         struct radeon_device *rdev;
 694         struct radeon_bo *rbo;
 695         unsigned long offset, size;
 696         int r;
 697
 698         if (!radeon_ttm_bo_is_radeon_bo(bo))
 699                 return 0;
 700         rbo = container_of(bo, struct radeon_bo, tbo);
 701         radeon_bo_check_tiling(rbo, 0, 0);
 702         rdev = rbo->rdev;
 703         if (bo->mem.mem_type != TTM_PL_VRAM)
 704                 return 0;
 705
 706         size = bo->mem.num_pages << PAGE_SHIFT;
 707         offset = bo->mem.start << PAGE_SHIFT;
 708         if ((offset + size) <= rdev->mc.visible_vram_size)
 709                 return 0;
 710
 711         /* hurrah the memory is not visible ! */
 712         radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM);
 713         rbo->placement.lpfn = rdev->mc.visible_vram_size >> PAGE_SHIFT;
 714         r = ttm_bo_validate(bo, &rbo->placement, false, false);
 715         if (unlikely(r == -ENOMEM)) {
 716                 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT);
 717                 return ttm_bo_validate(bo, &rbo->placement, false, false);
 718         } else if (unlikely(r != 0)) {
 719                 return r;
 720         }
 721
 722         offset = bo->mem.start << PAGE_SHIFT;
 723         /* this should never happen */
 724         if ((offset + size) > rdev->mc.visible_vram_size)
 725                 return -EINVAL;
 726
 727         return 0;
 728 }
 729
 730 int radeon_bo_wait(struct radeon_bo *bo, u32 *mem_type, bool no_wait)
 731 {
 732         int r;
 733
 734         r = ttm_bo_reserve(&bo->tbo, true, no_wait, false, NULL);
 735         if (unlikely(r != 0))
 736                 return r;
 737         spin_lock(&bo->tbo.bdev->fence_lock);
 738         if (mem_type)
 739                 *mem_type = bo->tbo.mem.mem_type;
 740         if (bo->tbo.sync_obj)
 741                 r = ttm_bo_wait(&bo->tbo, true, true, no_wait);
 742         spin_unlock(&bo->tbo.bdev->fence_lock);
 743         ttm_bo_unreserve(&bo->tbo);
 744         return r;
 745 }