drivers/gpu/drm/i915/i915_gem_execbuffer.c

   1 /*
   2  * Copyright © 2008,2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Chris Wilson <chris@chris-wilson.co.uk>
  26  *
  27  */
  28
  29 #include <drm/drmP.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/dma_remapping.h>
  35 #include <linux/uaccess.h>
  36
  37 #define  __EXEC_OBJECT_HAS_PIN          (1<<31)
  38 #define  __EXEC_OBJECT_HAS_FENCE        (1<<30)
  39 #define  __EXEC_OBJECT_NEEDS_MAP        (1<<29)
  40 #define  __EXEC_OBJECT_NEEDS_BIAS       (1<<28)
  41 #define  __EXEC_OBJECT_INTERNAL_FLAGS (0xf<<28) /* all of the above */
  42
  43 #define BATCH_OFFSET_BIAS (256*1024)
  44
  45 struct eb_vmas {
  46         struct list_head vmas;
  47         int and;
  48         union {
  49                 struct i915_vma *lut[0];
  50                 struct hlist_head buckets[0];
  51         };
  52 };
  53
  54 static struct eb_vmas *
  55 eb_create(struct drm_i915_gem_execbuffer2 *args)
  56 {
  57         struct eb_vmas *eb = NULL;
  58
  59         if (args->flags & I915_EXEC_HANDLE_LUT) {
  60                 unsigned size = args->buffer_count;
  61                 size *= sizeof(struct i915_vma *);
  62                 size += sizeof(struct eb_vmas);
  63                 eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
  64         }
  65
  66         if (eb == NULL) {
  67                 unsigned size = args->buffer_count;
  68                 unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
  69                 BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
  70                 while (count > 2*size)
  71                         count >>= 1;
  72                 eb = kzalloc(count*sizeof(struct hlist_head) +
  73                              sizeof(struct eb_vmas),
  74                              GFP_TEMPORARY);
  75                 if (eb == NULL)
  76                         return eb;
  77
  78                 eb->and = count - 1;
  79         } else
  80                 eb->and = -args->buffer_count;
  81
  82         INIT_LIST_HEAD(&eb->vmas);
  83         return eb;
  84 }
  85
  86 static void
  87 eb_reset(struct eb_vmas *eb)
  88 {
  89         if (eb->and >= 0)
  90                 memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
  91 }
  92
  93 static int
  94 eb_lookup_vmas(struct eb_vmas *eb,
  95                struct drm_i915_gem_exec_object2 *exec,
  96                const struct drm_i915_gem_execbuffer2 *args,
  97                struct i915_address_space *vm,
  98                struct drm_file *file)
  99 {
 100         struct drm_i915_gem_object *obj;
 101         struct list_head objects;
 102         int i, ret;
 103
 104         INIT_LIST_HEAD(&objects);
 105         spin_lock(&file->table_lock);
 106         /* Grab a reference to the object and release the lock so we can lookup
 107          * or create the VMA without using GFP_ATOMIC */
 108         for (i = 0; i < args->buffer_count; i++) {
 109                 obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
 110                 if (obj == NULL) {
 111                         spin_unlock(&file->table_lock);
 112                         DRM_DEBUG("Invalid object handle %d at index %d\n",
 113                                    exec[i].handle, i);
 114                         ret = -ENOENT;
 115                         goto err;
 116                 }
 117
 118                 if (!list_empty(&obj->obj_exec_link)) {
 119                         spin_unlock(&file->table_lock);
 120                         DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
 121                                    obj, exec[i].handle, i);
 122                         ret = -EINVAL;
 123                         goto err;
 124                 }
 125
 126                 i915_gem_object_get(obj);
 127                 list_add_tail(&obj->obj_exec_link, &objects);
 128         }
 129         spin_unlock(&file->table_lock);
 130
 131         i = 0;
 132         while (!list_empty(&objects)) {
 133                 struct i915_vma *vma;
 134
 135                 obj = list_first_entry(&objects,
 136                                        struct drm_i915_gem_object,
 137                                        obj_exec_link);
 138
 139                 /*
 140                  * NOTE: We can leak any vmas created here when something fails
 141                  * later on. But that's no issue since vma_unbind can deal with
 142                  * vmas which are not actually bound. And since only
 143                  * lookup_or_create exists as an interface to get at the vma
 144                  * from the (obj, vm) we don't run the risk of creating
 145                  * duplicated vmas for the same vm.
 146                  */
 147                 vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
 148                 if (IS_ERR(vma)) {
 149                         DRM_DEBUG("Failed to lookup VMA\n");
 150                         ret = PTR_ERR(vma);
 151                         goto err;
 152                 }
 153
 154                 /* Transfer ownership from the objects list to the vmas list. */
 155                 list_add_tail(&vma->exec_list, &eb->vmas);
 156                 list_del_init(&obj->obj_exec_link);
 157
 158                 vma->exec_entry = &exec[i];
 159                 if (eb->and < 0) {
 160                         eb->lut[i] = vma;
 161                 } else {
 162                         uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
 163                         vma->exec_handle = handle;
 164                         hlist_add_head(&vma->exec_node,
 165                                        &eb->buckets[handle & eb->and]);
 166                 }
 167                 ++i;
 168         }
 169
 170         return 0;
 171
 172
 173 err:
 174         while (!list_empty(&objects)) {
 175                 obj = list_first_entry(&objects,
 176                                        struct drm_i915_gem_object,
 177                                        obj_exec_link);
 178                 list_del_init(&obj->obj_exec_link);
 179                 i915_gem_object_put(obj);
 180         }
 181         /*
 182          * Objects already transfered to the vmas list will be unreferenced by
 183          * eb_destroy.
 184          */
 185
 186         return ret;
 187 }
 188
 189 static inline struct i915_vma *
 190 eb_get_batch_vma(struct eb_vmas *eb)
 191 {
 192         /* The batch is always the LAST item in the VMA list */
 193         struct i915_vma *vma = list_last_entry(&eb->vmas, typeof(*vma), exec_list);
 194
 195         return vma;
 196 }
 197
 198 static struct drm_i915_gem_object *
 199 eb_get_batch(struct eb_vmas *eb)
 200 {
 201         struct i915_vma *vma = eb_get_batch_vma(eb);
 202
 203         /*
 204          * SNA is doing fancy tricks with compressing batch buffers, which leads
 205          * to negative relocation deltas. Usually that works out ok since the
 206          * relocate address is still positive, except when the batch is placed
 207          * very low in the GTT. Ensure this doesn't happen.
 208          *
 209          * Note that actual hangs have only been observed on gen7, but for
 210          * paranoia do it everywhere.
 211          */
 212         if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
 213                 vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
 214
 215         return vma->obj;
 216 }
 217
 218 static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
 219 {
 220         if (eb->and < 0) {
 221                 if (handle >= -eb->and)
 222                         return NULL;
 223                 return eb->lut[handle];
 224         } else {
 225                 struct hlist_head *head;
 226                 struct i915_vma *vma;
 227
 228                 head = &eb->buckets[handle & eb->and];
 229                 hlist_for_each_entry(vma, head, exec_node) {
 230                         if (vma->exec_handle == handle)
 231                                 return vma;
 232                 }
 233                 return NULL;
 234         }
 235 }
 236
 237 static void
 238 i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
 239 {
 240         struct drm_i915_gem_exec_object2 *entry;
 241         struct drm_i915_gem_object *obj = vma->obj;
 242
 243         if (!drm_mm_node_allocated(&vma->node))
 244                 return;
 245
 246         entry = vma->exec_entry;
 247
 248         if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
 249                 i915_gem_object_unpin_fence(obj);
 250
 251         if (entry->flags & __EXEC_OBJECT_HAS_PIN)
 252                 vma->pin_count--;
 253
 254         entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
 255 }
 256
 257 static void eb_destroy(struct eb_vmas *eb)
 258 {
 259         while (!list_empty(&eb->vmas)) {
 260                 struct i915_vma *vma;
 261
 262                 vma = list_first_entry(&eb->vmas,
 263                                        struct i915_vma,
 264                                        exec_list);
 265                 list_del_init(&vma->exec_list);
 266                 i915_gem_execbuffer_unreserve_vma(vma);
 267                 i915_gem_object_put(vma->obj);
 268         }
 269         kfree(eb);
 270 }
 271
 272 static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
 273 {
 274         return (HAS_LLC(obj->base.dev) ||
 275                 obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
 276                 obj->cache_level != I915_CACHE_NONE);
 277 }
 278
 279 /* Used to convert any address to canonical form.
 280  * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
 281  * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
 282  * addresses to be in a canonical form:
 283  * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
 284  * canonical form [63:48] == [47]."
 285  */
 286 #define GEN8_HIGH_ADDRESS_BIT 47
 287 static inline uint64_t gen8_canonical_addr(uint64_t address)
 288 {
 289         return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
 290 }
 291
 292 static inline uint64_t gen8_noncanonical_addr(uint64_t address)
 293 {
 294         return address & ((1ULL << (GEN8_HIGH_ADDRESS_BIT + 1)) - 1);
 295 }
 296
 297 static inline uint64_t
 298 relocation_target(struct drm_i915_gem_relocation_entry *reloc,
 299                   uint64_t target_offset)
 300 {
 301         return gen8_canonical_addr((int)reloc->delta + target_offset);
 302 }
 303
 304 static int
 305 relocate_entry_cpu(struct drm_i915_gem_object *obj,
 306                    struct drm_i915_gem_relocation_entry *reloc,
 307                    uint64_t target_offset)
 308 {
 309         struct drm_device *dev = obj->base.dev;
 310         uint32_t page_offset = offset_in_page(reloc->offset);
 311         uint64_t delta = relocation_target(reloc, target_offset);
 312         char *vaddr;
 313         int ret;
 314
 315         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 316         if (ret)
 317                 return ret;
 318
 319         vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 320                                 reloc->offset >> PAGE_SHIFT));
 321         *(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta);
 322
 323         if (INTEL_INFO(dev)->gen >= 8) {
 324                 page_offset = offset_in_page(page_offset + sizeof(uint32_t));
 325
 326                 if (page_offset == 0) {
 327                         kunmap_atomic(vaddr);
 328                         vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 329                             (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
 330                 }
 331
 332                 *(uint32_t *)(vaddr + page_offset) = upper_32_bits(delta);
 333         }
 334
 335         kunmap_atomic(vaddr);
 336
 337         return 0;
 338 }
 339
 340 static int
 341 relocate_entry_gtt(struct drm_i915_gem_object *obj,
 342                    struct drm_i915_gem_relocation_entry *reloc,
 343                    uint64_t target_offset)
 344 {
 345         struct drm_device *dev = obj->base.dev;
 346         struct drm_i915_private *dev_priv = to_i915(dev);
 347         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 348         uint64_t delta = relocation_target(reloc, target_offset);
 349         uint64_t offset;
 350         void __iomem *reloc_page;
 351         int ret;
 352
 353         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 354         if (ret)
 355                 return ret;
 356
 357         ret = i915_gem_object_put_fence(obj);
 358         if (ret)
 359                 return ret;
 360
 361         /* Map the page containing the relocation we're going to perform.  */
 362         offset = i915_gem_obj_ggtt_offset(obj);
 363         offset += reloc->offset;
 364         reloc_page = io_mapping_map_atomic_wc(ggtt->mappable,
 365                                               offset & PAGE_MASK);
 366         iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
 367
 368         if (INTEL_INFO(dev)->gen >= 8) {
 369                 offset += sizeof(uint32_t);
 370
 371                 if (offset_in_page(offset) == 0) {
 372                         io_mapping_unmap_atomic(reloc_page);
 373                         reloc_page =
 374                                 io_mapping_map_atomic_wc(ggtt->mappable,
 375                                                          offset);
 376                 }
 377
 378                 iowrite32(upper_32_bits(delta),
 379                           reloc_page + offset_in_page(offset));
 380         }
 381
 382         io_mapping_unmap_atomic(reloc_page);
 383
 384         return 0;
 385 }
 386
 387 static void
 388 clflush_write32(void *addr, uint32_t value)
 389 {
 390         /* This is not a fast path, so KISS. */
 391         drm_clflush_virt_range(addr, sizeof(uint32_t));
 392         *(uint32_t *)addr = value;
 393         drm_clflush_virt_range(addr, sizeof(uint32_t));
 394 }
 395
 396 static int
 397 relocate_entry_clflush(struct drm_i915_gem_object *obj,
 398                        struct drm_i915_gem_relocation_entry *reloc,
 399                        uint64_t target_offset)
 400 {
 401         struct drm_device *dev = obj->base.dev;
 402         uint32_t page_offset = offset_in_page(reloc->offset);
 403         uint64_t delta = relocation_target(reloc, target_offset);
 404         char *vaddr;
 405         int ret;
 406
 407         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 408         if (ret)
 409                 return ret;
 410
 411         vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 412                                 reloc->offset >> PAGE_SHIFT));
 413         clflush_write32(vaddr + page_offset, lower_32_bits(delta));
 414
 415         if (INTEL_INFO(dev)->gen >= 8) {
 416                 page_offset = offset_in_page(page_offset + sizeof(uint32_t));
 417
 418                 if (page_offset == 0) {
 419                         kunmap_atomic(vaddr);
 420                         vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
 421                             (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
 422                 }
 423
 424                 clflush_write32(vaddr + page_offset, upper_32_bits(delta));
 425         }
 426
 427         kunmap_atomic(vaddr);
 428
 429         return 0;
 430 }
 431
 432 static int
 433 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 434                                    struct eb_vmas *eb,
 435                                    struct drm_i915_gem_relocation_entry *reloc)
 436 {
 437         struct drm_device *dev = obj->base.dev;
 438         struct drm_gem_object *target_obj;
 439         struct drm_i915_gem_object *target_i915_obj;
 440         struct i915_vma *target_vma;
 441         uint64_t target_offset;
 442         int ret;
 443
 444         /* we've already hold a reference to all valid objects */
 445         target_vma = eb_get_vma(eb, reloc->target_handle);
 446         if (unlikely(target_vma == NULL))
 447                 return -ENOENT;
 448         target_i915_obj = target_vma->obj;
 449         target_obj = &target_vma->obj->base;
 450
 451         target_offset = gen8_canonical_addr(target_vma->node.start);
 452
 453         /* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
 454          * pipe_control writes because the gpu doesn't properly redirect them
 455          * through the ppgtt for non_secure batchbuffers. */
 456         if (unlikely(IS_GEN6(dev) &&
 457             reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
 458                 ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
 459                                     PIN_GLOBAL);
 460                 if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
 461                         return ret;
 462         }
 463
 464         /* Validate that the target is in a valid r/w GPU domain */
 465         if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
 466                 DRM_DEBUG("reloc with multiple write domains: "
 467                           "obj %p target %d offset %d "
 468                           "read %08x write %08x",
 469                           obj, reloc->target_handle,
 470                           (int) reloc->offset,
 471                           reloc->read_domains,
 472                           reloc->write_domain);
 473                 return -EINVAL;
 474         }
 475         if (unlikely((reloc->write_domain | reloc->read_domains)
 476                      & ~I915_GEM_GPU_DOMAINS)) {
 477                 DRM_DEBUG("reloc with read/write non-GPU domains: "
 478                           "obj %p target %d offset %d "
 479                           "read %08x write %08x",
 480                           obj, reloc->target_handle,
 481                           (int) reloc->offset,
 482                           reloc->read_domains,
 483                           reloc->write_domain);
 484                 return -EINVAL;
 485         }
 486
 487         target_obj->pending_read_domains |= reloc->read_domains;
 488         target_obj->pending_write_domain |= reloc->write_domain;
 489
 490         /* If the relocation already has the right value in it, no
 491          * more work needs to be done.
 492          */
 493         if (target_offset == reloc->presumed_offset)
 494                 return 0;
 495
 496         /* Check that the relocation address is valid... */
 497         if (unlikely(reloc->offset >
 498                 obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
 499                 DRM_DEBUG("Relocation beyond object bounds: "
 500                           "obj %p target %d offset %d size %d.\n",
 501                           obj, reloc->target_handle,
 502                           (int) reloc->offset,
 503                           (int) obj->base.size);
 504                 return -EINVAL;
 505         }
 506         if (unlikely(reloc->offset & 3)) {
 507                 DRM_DEBUG("Relocation not 4-byte aligned: "
 508                           "obj %p target %d offset %d.\n",
 509                           obj, reloc->target_handle,
 510                           (int) reloc->offset);
 511                 return -EINVAL;
 512         }
 513
 514         /* We can't wait for rendering with pagefaults disabled */
 515         if (obj->active && pagefault_disabled())
 516                 return -EFAULT;
 517
 518         if (use_cpu_reloc(obj))
 519                 ret = relocate_entry_cpu(obj, reloc, target_offset);
 520         else if (obj->map_and_fenceable)
 521                 ret = relocate_entry_gtt(obj, reloc, target_offset);
 522         else if (static_cpu_has(X86_FEATURE_CLFLUSH))
 523                 ret = relocate_entry_clflush(obj, reloc, target_offset);
 524         else {
 525                 WARN_ONCE(1, "Impossible case in relocation handling\n");
 526                 ret = -ENODEV;
 527         }
 528
 529         if (ret)
 530                 return ret;
 531
 532         /* and update the user's relocation entry */
 533         reloc->presumed_offset = target_offset;
 534
 535         return 0;
 536 }
 537
 538 static int
 539 i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
 540                                  struct eb_vmas *eb)
 541 {
 542 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
 543         struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
 544         struct drm_i915_gem_relocation_entry __user *user_relocs;
 545         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 546         int remain, ret;
 547
 548         user_relocs = u64_to_user_ptr(entry->relocs_ptr);
 549
 550         remain = entry->relocation_count;
 551         while (remain) {
 552                 struct drm_i915_gem_relocation_entry *r = stack_reloc;
 553                 int count = remain;
 554                 if (count > ARRAY_SIZE(stack_reloc))
 555                         count = ARRAY_SIZE(stack_reloc);
 556                 remain -= count;
 557
 558                 if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
 559                         return -EFAULT;
 560
 561                 do {
 562                         u64 offset = r->presumed_offset;
 563
 564                         ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
 565                         if (ret)
 566                                 return ret;
 567
 568                         if (r->presumed_offset != offset &&
 569                             __put_user(r->presumed_offset, &user_relocs->presumed_offset)) {
 570                                 return -EFAULT;
 571                         }
 572
 573                         user_relocs++;
 574                         r++;
 575                 } while (--count);
 576         }
 577
 578         return 0;
 579 #undef N_RELOC
 580 }
 581
 582 static int
 583 i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
 584                                       struct eb_vmas *eb,
 585                                       struct drm_i915_gem_relocation_entry *relocs)
 586 {
 587         const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 588         int i, ret;
 589
 590         for (i = 0; i < entry->relocation_count; i++) {
 591                 ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
 592                 if (ret)
 593                         return ret;
 594         }
 595
 596         return 0;
 597 }
 598
 599 static int
 600 i915_gem_execbuffer_relocate(struct eb_vmas *eb)
 601 {
 602         struct i915_vma *vma;
 603         int ret = 0;
 604
 605         /* This is the fast path and we cannot handle a pagefault whilst
 606          * holding the struct mutex lest the user pass in the relocations
 607          * contained within a mmaped bo. For in such a case we, the page
 608          * fault handler would call i915_gem_fault() and we would try to
 609          * acquire the struct mutex again. Obviously this is bad and so
 610          * lockdep complains vehemently.
 611          */
 612         pagefault_disable();
 613         list_for_each_entry(vma, &eb->vmas, exec_list) {
 614                 ret = i915_gem_execbuffer_relocate_vma(vma, eb);
 615                 if (ret)
 616                         break;
 617         }
 618         pagefault_enable();
 619
 620         return ret;
 621 }
 622
 623 static bool only_mappable_for_reloc(unsigned int flags)
 624 {
 625         return (flags & (EXEC_OBJECT_NEEDS_FENCE | __EXEC_OBJECT_NEEDS_MAP)) ==
 626                 __EXEC_OBJECT_NEEDS_MAP;
 627 }
 628
 629 static int
 630 i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
 631                                 struct intel_engine_cs *engine,
 632                                 bool *need_reloc)
 633 {
 634         struct drm_i915_gem_object *obj = vma->obj;
 635         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 636         uint64_t flags;
 637         int ret;
 638
 639         flags = PIN_USER;
 640         if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
 641                 flags |= PIN_GLOBAL;
 642
 643         if (!drm_mm_node_allocated(&vma->node)) {
 644                 /* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
 645                  * limit address to the first 4GBs for unflagged objects.
 646                  */
 647                 if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
 648                         flags |= PIN_ZONE_4G;
 649                 if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
 650                         flags |= PIN_GLOBAL | PIN_MAPPABLE;
 651                 if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
 652                         flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
 653                 if (entry->flags & EXEC_OBJECT_PINNED)
 654                         flags |= entry->offset | PIN_OFFSET_FIXED;
 655                 if ((flags & PIN_MAPPABLE) == 0)
 656                         flags |= PIN_HIGH;
 657         }
 658
 659         ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
 660         if ((ret == -ENOSPC  || ret == -E2BIG) &&
 661             only_mappable_for_reloc(entry->flags))
 662                 ret = i915_gem_object_pin(obj, vma->vm,
 663                                           entry->alignment,
 664                                           flags & ~PIN_MAPPABLE);
 665         if (ret)
 666                 return ret;
 667
 668         entry->flags |= __EXEC_OBJECT_HAS_PIN;
 669
 670         if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
 671                 ret = i915_gem_object_get_fence(obj);
 672                 if (ret)
 673                         return ret;
 674
 675                 if (i915_gem_object_pin_fence(obj))
 676                         entry->flags |= __EXEC_OBJECT_HAS_FENCE;
 677         }
 678
 679         if (entry->offset != vma->node.start) {
 680                 entry->offset = vma->node.start;
 681                 *need_reloc = true;
 682         }
 683
 684         if (entry->flags & EXEC_OBJECT_WRITE) {
 685                 obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
 686                 obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
 687         }
 688
 689         return 0;
 690 }
 691
 692 static bool
 693 need_reloc_mappable(struct i915_vma *vma)
 694 {
 695         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 696
 697         if (entry->relocation_count == 0)
 698                 return false;
 699
 700         if (!vma->is_ggtt)
 701                 return false;
 702
 703         /* See also use_cpu_reloc() */
 704         if (HAS_LLC(vma->obj->base.dev))
 705                 return false;
 706
 707         if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
 708                 return false;
 709
 710         return true;
 711 }
 712
 713 static bool
 714 eb_vma_misplaced(struct i915_vma *vma)
 715 {
 716         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 717         struct drm_i915_gem_object *obj = vma->obj;
 718
 719         WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP && !vma->is_ggtt);
 720
 721         if (entry->alignment &&
 722             vma->node.start & (entry->alignment - 1))
 723                 return true;
 724
 725         if (entry->flags & EXEC_OBJECT_PINNED &&
 726             vma->node.start != entry->offset)
 727                 return true;
 728
 729         if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
 730             vma->node.start < BATCH_OFFSET_BIAS)
 731                 return true;
 732
 733         /* avoid costly ping-pong once a batch bo ended up non-mappable */
 734         if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
 735                 return !only_mappable_for_reloc(entry->flags);
 736
 737         if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
 738             (vma->node.start + vma->node.size - 1) >> 32)
 739                 return true;
 740
 741         return false;
 742 }
 743
 744 static int
 745 i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
 746                             struct list_head *vmas,
 747                             struct i915_gem_context *ctx,
 748                             bool *need_relocs)
 749 {
 750         struct drm_i915_gem_object *obj;
 751         struct i915_vma *vma;
 752         struct i915_address_space *vm;
 753         struct list_head ordered_vmas;
 754         struct list_head pinned_vmas;
 755         bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
 756         int retry;
 757
 758         i915_gem_retire_requests_ring(engine);
 759
 760         vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
 761
 762         INIT_LIST_HEAD(&ordered_vmas);
 763         INIT_LIST_HEAD(&pinned_vmas);
 764         while (!list_empty(vmas)) {
 765                 struct drm_i915_gem_exec_object2 *entry;
 766                 bool need_fence, need_mappable;
 767
 768                 vma = list_first_entry(vmas, struct i915_vma, exec_list);
 769                 obj = vma->obj;
 770                 entry = vma->exec_entry;
 771
 772                 if (ctx->flags & CONTEXT_NO_ZEROMAP)
 773                         entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
 774
 775                 if (!has_fenced_gpu_access)
 776                         entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
 777                 need_fence =
 778                         entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 779                         obj->tiling_mode != I915_TILING_NONE;
 780                 need_mappable = need_fence || need_reloc_mappable(vma);
 781
 782                 if (entry->flags & EXEC_OBJECT_PINNED)
 783                         list_move_tail(&vma->exec_list, &pinned_vmas);
 784                 else if (need_mappable) {
 785                         entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
 786                         list_move(&vma->exec_list, &ordered_vmas);
 787                 } else
 788                         list_move_tail(&vma->exec_list, &ordered_vmas);
 789
 790                 obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
 791                 obj->base.pending_write_domain = 0;
 792         }
 793         list_splice(&ordered_vmas, vmas);
 794         list_splice(&pinned_vmas, vmas);
 795
 796         /* Attempt to pin all of the buffers into the GTT.
 797          * This is done in 3 phases:
 798          *
 799          * 1a. Unbind all objects that do not match the GTT constraints for
 800          *     the execbuffer (fenceable, mappable, alignment etc).
 801          * 1b. Increment pin count for already bound objects.
 802          * 2.  Bind new objects.
 803          * 3.  Decrement pin count.
 804          *
 805          * This avoid unnecessary unbinding of later objects in order to make
 806          * room for the earlier objects *unless* we need to defragment.
 807          */
 808         retry = 0;
 809         do {
 810                 int ret = 0;
 811
 812                 /* Unbind any ill-fitting objects or pin. */
 813                 list_for_each_entry(vma, vmas, exec_list) {
 814                         if (!drm_mm_node_allocated(&vma->node))
 815                                 continue;
 816
 817                         if (eb_vma_misplaced(vma))
 818                                 ret = i915_vma_unbind(vma);
 819                         else
 820                                 ret = i915_gem_execbuffer_reserve_vma(vma,
 821                                                                       engine,
 822                                                                       need_relocs);
 823                         if (ret)
 824                                 goto err;
 825                 }
 826
 827                 /* Bind fresh objects */
 828                 list_for_each_entry(vma, vmas, exec_list) {
 829                         if (drm_mm_node_allocated(&vma->node))
 830                                 continue;
 831
 832                         ret = i915_gem_execbuffer_reserve_vma(vma, engine,
 833                                                               need_relocs);
 834                         if (ret)
 835                                 goto err;
 836                 }
 837
 838 err:
 839                 if (ret != -ENOSPC || retry++)
 840                         return ret;
 841
 842                 /* Decrement pin count for bound objects */
 843                 list_for_each_entry(vma, vmas, exec_list)
 844                         i915_gem_execbuffer_unreserve_vma(vma);
 845
 846                 ret = i915_gem_evict_vm(vm, true);
 847                 if (ret)
 848                         return ret;
 849         } while (1);
 850 }
 851
 852 static int
 853 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
 854                                   struct drm_i915_gem_execbuffer2 *args,
 855                                   struct drm_file *file,
 856                                   struct intel_engine_cs *engine,
 857                                   struct eb_vmas *eb,
 858                                   struct drm_i915_gem_exec_object2 *exec,
 859                                   struct i915_gem_context *ctx)
 860 {
 861         struct drm_i915_gem_relocation_entry *reloc;
 862         struct i915_address_space *vm;
 863         struct i915_vma *vma;
 864         bool need_relocs;
 865         int *reloc_offset;
 866         int i, total, ret;
 867         unsigned count = args->buffer_count;
 868
 869         vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
 870
 871         /* We may process another execbuffer during the unlock... */
 872         while (!list_empty(&eb->vmas)) {
 873                 vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
 874                 list_del_init(&vma->exec_list);
 875                 i915_gem_execbuffer_unreserve_vma(vma);
 876                 i915_gem_object_put(vma->obj);
 877         }
 878
 879         mutex_unlock(&dev->struct_mutex);
 880
 881         total = 0;
 882         for (i = 0; i < count; i++)
 883                 total += exec[i].relocation_count;
 884
 885         reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
 886         reloc = drm_malloc_ab(total, sizeof(*reloc));
 887         if (reloc == NULL || reloc_offset == NULL) {
 888                 drm_free_large(reloc);
 889                 drm_free_large(reloc_offset);
 890                 mutex_lock(&dev->struct_mutex);
 891                 return -ENOMEM;
 892         }
 893
 894         total = 0;
 895         for (i = 0; i < count; i++) {
 896                 struct drm_i915_gem_relocation_entry __user *user_relocs;
 897                 u64 invalid_offset = (u64)-1;
 898                 int j;
 899
 900                 user_relocs = u64_to_user_ptr(exec[i].relocs_ptr);
 901
 902                 if (copy_from_user(reloc+total, user_relocs,
 903                                    exec[i].relocation_count * sizeof(*reloc))) {
 904                         ret = -EFAULT;
 905                         mutex_lock(&dev->struct_mutex);
 906                         goto err;
 907                 }
 908
 909                 /* As we do not update the known relocation offsets after
 910                  * relocating (due to the complexities in lock handling),
 911                  * we need to mark them as invalid now so that we force the
 912                  * relocation processing next time. Just in case the target
 913                  * object is evicted and then rebound into its old
 914                  * presumed_offset before the next execbuffer - if that
 915                  * happened we would make the mistake of assuming that the
 916                  * relocations were valid.
 917                  */
 918                 for (j = 0; j < exec[i].relocation_count; j++) {
 919                         if (__copy_to_user(&user_relocs[j].presumed_offset,
 920                                            &invalid_offset,
 921                                            sizeof(invalid_offset))) {
 922                                 ret = -EFAULT;
 923                                 mutex_lock(&dev->struct_mutex);
 924                                 goto err;
 925                         }
 926                 }
 927
 928                 reloc_offset[i] = total;
 929                 total += exec[i].relocation_count;
 930         }
 931
 932         ret = i915_mutex_lock_interruptible(dev);
 933         if (ret) {
 934                 mutex_lock(&dev->struct_mutex);
 935                 goto err;
 936         }
 937
 938         /* reacquire the objects */
 939         eb_reset(eb);
 940         ret = eb_lookup_vmas(eb, exec, args, vm, file);
 941         if (ret)
 942                 goto err;
 943
 944         need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
 945         ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
 946                                           &need_relocs);
 947         if (ret)
 948                 goto err;
 949
 950         list_for_each_entry(vma, &eb->vmas, exec_list) {
 951                 int offset = vma->exec_entry - exec;
 952                 ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
 953                                                             reloc + reloc_offset[offset]);
 954                 if (ret)
 955                         goto err;
 956         }
 957
 958         /* Leave the user relocations as are, this is the painfully slow path,
 959          * and we want to avoid the complication of dropping the lock whilst
 960          * having buffers reserved in the aperture and so causing spurious
 961          * ENOSPC for random operations.
 962          */
 963
 964 err:
 965         drm_free_large(reloc);
 966         drm_free_large(reloc_offset);
 967         return ret;
 968 }
 969
 970 static int
 971 i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
 972                                 struct list_head *vmas)
 973 {
 974         const unsigned other_rings = ~intel_engine_flag(req->engine);
 975         struct i915_vma *vma;
 976         uint32_t flush_domains = 0;
 977         bool flush_chipset = false;
 978         int ret;
 979
 980         list_for_each_entry(vma, vmas, exec_list) {
 981                 struct drm_i915_gem_object *obj = vma->obj;
 982
 983                 if (obj->active & other_rings) {
 984                         ret = i915_gem_object_sync(obj, req->engine, &req);
 985                         if (ret)
 986                                 return ret;
 987                 }
 988
 989                 if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
 990                         flush_chipset |= i915_gem_clflush_object(obj, false);
 991
 992                 flush_domains |= obj->base.write_domain;
 993         }
 994
 995         if (flush_chipset)
 996                 i915_gem_chipset_flush(req->engine->i915);
 997
 998         if (flush_domains & I915_GEM_DOMAIN_GTT)
 999                 wmb();
1000
1001         /* Unconditionally invalidate GPU caches and TLBs. */
1002         return req->engine->emit_flush(req, EMIT_INVALIDATE);
1003 }
1004
1005 static bool
1006 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
1007 {
1008         if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
1009                 return false;
1010
1011         /* Kernel clipping was a DRI1 misfeature */
1012         if (exec->num_cliprects || exec->cliprects_ptr)
1013                 return false;
1014
1015         if (exec->DR4 == 0xffffffff) {
1016                 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
1017                 exec->DR4 = 0;
1018         }
1019         if (exec->DR1 || exec->DR4)
1020                 return false;
1021
1022         if ((exec->batch_start_offset | exec->batch_len) & 0x7)
1023                 return false;
1024
1025         return true;
1026 }
1027
1028 static int
1029 validate_exec_list(struct drm_device *dev,
1030                    struct drm_i915_gem_exec_object2 *exec,
1031                    int count)
1032 {
1033         unsigned relocs_total = 0;
1034         unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
1035         unsigned invalid_flags;
1036         int i;
1037
1038         /* INTERNAL flags must not overlap with external ones */
1039         BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ~__EXEC_OBJECT_UNKNOWN_FLAGS);
1040
1041         invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
1042         if (USES_FULL_PPGTT(dev))
1043                 invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
1044
1045         for (i = 0; i < count; i++) {
1046                 char __user *ptr = u64_to_user_ptr(exec[i].relocs_ptr);
1047                 int length; /* limited by fault_in_pages_readable() */
1048
1049                 if (exec[i].flags & invalid_flags)
1050                         return -EINVAL;
1051
1052                 /* Offset can be used as input (EXEC_OBJECT_PINNED), reject
1053                  * any non-page-aligned or non-canonical addresses.
1054                  */
1055                 if (exec[i].flags & EXEC_OBJECT_PINNED) {
1056                         if (exec[i].offset !=
1057                             gen8_canonical_addr(exec[i].offset & PAGE_MASK))
1058                                 return -EINVAL;
1059
1060                         /* From drm_mm perspective address space is continuous,
1061                          * so from this point we're always using non-canonical
1062                          * form internally.
1063                          */
1064                         exec[i].offset = gen8_noncanonical_addr(exec[i].offset);
1065                 }
1066
1067                 if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
1068                         return -EINVAL;
1069
1070                 /* First check for malicious input causing overflow in
1071                  * the worst case where we need to allocate the entire
1072                  * relocation tree as a single array.
1073                  */
1074                 if (exec[i].relocation_count > relocs_max - relocs_total)
1075                         return -EINVAL;
1076                 relocs_total += exec[i].relocation_count;
1077
1078                 length = exec[i].relocation_count *
1079                         sizeof(struct drm_i915_gem_relocation_entry);
1080                 /*
1081                  * We must check that the entire relocation array is safe
1082                  * to read, but since we may need to update the presumed
1083                  * offsets during execution, check for full write access.
1084                  */
1085                 if (!access_ok(VERIFY_WRITE, ptr, length))
1086                         return -EFAULT;
1087
1088                 if (likely(!i915.prefault_disable)) {
1089                         if (fault_in_multipages_readable(ptr, length))
1090                                 return -EFAULT;
1091                 }
1092         }
1093
1094         return 0;
1095 }
1096
1097 static struct i915_gem_context *
1098 i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
1099                           struct intel_engine_cs *engine, const u32 ctx_id)
1100 {
1101         struct i915_gem_context *ctx = NULL;
1102         struct i915_ctx_hang_stats *hs;
1103
1104         if (engine->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
1105                 return ERR_PTR(-EINVAL);
1106
1107         ctx = i915_gem_context_lookup(file->driver_priv, ctx_id);
1108         if (IS_ERR(ctx))
1109                 return ctx;
1110
1111         hs = &ctx->hang_stats;
1112         if (hs->banned) {
1113                 DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
1114                 return ERR_PTR(-EIO);
1115         }
1116
1117         return ctx;
1118 }
1119
1120 void
1121 i915_gem_execbuffer_move_to_active(struct list_head *vmas,
1122                                    struct drm_i915_gem_request *req)
1123 {
1124         struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
1125         struct i915_vma *vma;
1126
1127         list_for_each_entry(vma, vmas, exec_list) {
1128                 struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
1129                 struct drm_i915_gem_object *obj = vma->obj;
1130                 u32 old_read = obj->base.read_domains;
1131                 u32 old_write = obj->base.write_domain;
1132
1133                 obj->dirty = 1; /* be paranoid  */
1134                 obj->base.write_domain = obj->base.pending_write_domain;
1135                 if (obj->base.write_domain == 0)
1136                         obj->base.pending_read_domains |= obj->base.read_domains;
1137                 obj->base.read_domains = obj->base.pending_read_domains;
1138
1139                 i915_vma_move_to_active(vma, req);
1140                 if (obj->base.write_domain) {
1141                         i915_gem_request_assign(&obj->last_write_req, req);
1142
1143                         intel_fb_obj_invalidate(obj, ORIGIN_CS);
1144
1145                         /* update for the implicit flush after a batch */
1146                         obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1147                 }
1148                 if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
1149                         i915_gem_request_assign(&obj->last_fenced_req, req);
1150                         if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
1151                                 struct drm_i915_private *dev_priv = engine->i915;
1152                                 list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
1153                                                &dev_priv->mm.fence_list);
1154                         }
1155                 }
1156
1157                 trace_i915_gem_object_change_domain(obj, old_read, old_write);
1158         }
1159 }
1160
1161 static void
1162 i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params)
1163 {
1164         /* Add a breadcrumb for the completion of the batch buffer */
1165         __i915_add_request(params->request, params->batch_obj, true);
1166 }
1167
1168 static int
1169 i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
1170 {
1171         struct intel_ring *ring = req->ring;
1172         int ret, i;
1173
1174         if (!IS_GEN7(req->i915) || req->engine->id != RCS) {
1175                 DRM_DEBUG("sol reset is gen7/rcs only\n");
1176                 return -EINVAL;
1177         }
1178
1179         ret = intel_ring_begin(req, 4 * 3);
1180         if (ret)
1181                 return ret;
1182
1183         for (i = 0; i < 4; i++) {
1184                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1185                 intel_ring_emit_reg(ring, GEN7_SO_WRITE_OFFSET(i));
1186                 intel_ring_emit(ring, 0);
1187         }
1188
1189         intel_ring_advance(ring);
1190
1191         return 0;
1192 }
1193
1194 static struct drm_i915_gem_object*
1195 i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
1196                           struct drm_i915_gem_exec_object2 *shadow_exec_entry,
1197                           struct eb_vmas *eb,
1198                           struct drm_i915_gem_object *batch_obj,
1199                           u32 batch_start_offset,
1200                           u32 batch_len,
1201                           bool is_master)
1202 {
1203         struct drm_i915_gem_object *shadow_batch_obj;
1204         struct i915_vma *vma;
1205         int ret;
1206
1207         shadow_batch_obj = i915_gem_batch_pool_get(&engine->batch_pool,
1208                                                    PAGE_ALIGN(batch_len));
1209         if (IS_ERR(shadow_batch_obj))
1210                 return shadow_batch_obj;
1211
1212         ret = intel_engine_cmd_parser(engine,
1213                                       batch_obj,
1214                                       shadow_batch_obj,
1215                                       batch_start_offset,
1216                                       batch_len,
1217                                       is_master);
1218         if (ret)
1219                 goto err;
1220
1221         ret = i915_gem_obj_ggtt_pin(shadow_batch_obj, 0, 0);
1222         if (ret)
1223                 goto err;
1224
1225         i915_gem_object_unpin_pages(shadow_batch_obj);
1226
1227         memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
1228
1229         vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
1230         vma->exec_entry = shadow_exec_entry;
1231         vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
1232         i915_gem_object_get(shadow_batch_obj);
1233         list_add_tail(&vma->exec_list, &eb->vmas);
1234
1235         shadow_batch_obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;
1236
1237         return shadow_batch_obj;
1238
1239 err:
1240         i915_gem_object_unpin_pages(shadow_batch_obj);
1241         if (ret == -EACCES) /* unhandled chained batch */
1242                 return batch_obj;
1243         else
1244                 return ERR_PTR(ret);
1245 }
1246
1247 int
1248 i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
1249                                struct drm_i915_gem_execbuffer2 *args,
1250                                struct list_head *vmas)
1251 {
1252         struct drm_i915_private *dev_priv = params->request->i915;
1253         u64 exec_start, exec_len;
1254         int instp_mode;
1255         u32 instp_mask;
1256         int ret;
1257
1258         ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
1259         if (ret)
1260                 return ret;
1261
1262         ret = i915_switch_context(params->request);
1263         if (ret)
1264                 return ret;
1265
1266         instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
1267         instp_mask = I915_EXEC_CONSTANTS_MASK;
1268         switch (instp_mode) {
1269         case I915_EXEC_CONSTANTS_REL_GENERAL:
1270         case I915_EXEC_CONSTANTS_ABSOLUTE:
1271         case I915_EXEC_CONSTANTS_REL_SURFACE:
1272                 if (instp_mode != 0 && params->engine->id != RCS) {
1273                         DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
1274                         return -EINVAL;
1275                 }
1276
1277                 if (instp_mode != dev_priv->relative_constants_mode) {
1278                         if (INTEL_INFO(dev_priv)->gen < 4) {
1279                                 DRM_DEBUG("no rel constants on pre-gen4\n");
1280                                 return -EINVAL;
1281                         }
1282
1283                         if (INTEL_INFO(dev_priv)->gen > 5 &&
1284                             instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
1285                                 DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
1286                                 return -EINVAL;
1287                         }
1288
1289                         /* The HW changed the meaning on this bit on gen6 */
1290                         if (INTEL_INFO(dev_priv)->gen >= 6)
1291                                 instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
1292                 }
1293                 break;
1294         default:
1295                 DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
1296                 return -EINVAL;
1297         }
1298
1299         if (params->engine->id == RCS &&
1300             instp_mode != dev_priv->relative_constants_mode) {
1301                 struct intel_ring *ring = params->request->ring;
1302
1303                 ret = intel_ring_begin(params->request, 4);
1304                 if (ret)
1305                         return ret;
1306
1307                 intel_ring_emit(ring, MI_NOOP);
1308                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
1309                 intel_ring_emit_reg(ring, INSTPM);
1310                 intel_ring_emit(ring, instp_mask << 16 | instp_mode);
1311                 intel_ring_advance(ring);
1312
1313                 dev_priv->relative_constants_mode = instp_mode;
1314         }
1315
1316         if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1317                 ret = i915_reset_gen7_sol_offsets(params->request);
1318                 if (ret)
1319                         return ret;
1320         }
1321
1322         exec_len   = args->batch_len;
1323         exec_start = params->batch_obj_vm_offset +
1324                      params->args_batch_start_offset;
1325
1326         if (exec_len == 0)
1327                 exec_len = params->batch_obj->base.size;
1328
1329         ret = params->engine->dispatch_execbuffer(params->request,
1330                                                   exec_start, exec_len,
1331                                                   params->dispatch_flags);
1332         if (ret)
1333                 return ret;
1334
1335         trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
1336
1337         i915_gem_execbuffer_move_to_active(vmas, params->request);
1338
1339         return 0;
1340 }
1341
1342 /**
1343  * Find one BSD ring to dispatch the corresponding BSD command.
1344  * The engine index is returned.
1345  */
1346 static unsigned int
1347 gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
1348                          struct drm_file *file)
1349 {
1350         struct drm_i915_file_private *file_priv = file->driver_priv;
1351
1352         /* Check whether the file_priv has already selected one ring. */
1353         if ((int)file_priv->bsd_engine < 0) {
1354                 /* If not, use the ping-pong mechanism to select one. */
1355                 mutex_lock(&dev_priv->drm.struct_mutex);
1356                 file_priv->bsd_engine = dev_priv->mm.bsd_engine_dispatch_index;
1357                 dev_priv->mm.bsd_engine_dispatch_index ^= 1;
1358                 mutex_unlock(&dev_priv->drm.struct_mutex);
1359         }
1360
1361         return file_priv->bsd_engine;
1362 }
1363
1364 #define I915_USER_RINGS (4)
1365
1366 static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
1367         [I915_EXEC_DEFAULT]     = RCS,
1368         [I915_EXEC_RENDER]      = RCS,
1369         [I915_EXEC_BLT]         = BCS,
1370         [I915_EXEC_BSD]         = VCS,
1371         [I915_EXEC_VEBOX]       = VECS
1372 };
1373
1374 static struct intel_engine_cs *
1375 eb_select_engine(struct drm_i915_private *dev_priv,
1376                  struct drm_file *file,
1377                  struct drm_i915_gem_execbuffer2 *args)
1378 {
1379         unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
1380         struct intel_engine_cs *engine;
1381
1382         if (user_ring_id > I915_USER_RINGS) {
1383                 DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
1384                 return NULL;
1385         }
1386
1387         if ((user_ring_id != I915_EXEC_BSD) &&
1388             ((args->flags & I915_EXEC_BSD_MASK) != 0)) {
1389                 DRM_DEBUG("execbuf with non bsd ring but with invalid "
1390                           "bsd dispatch flags: %d\n", (int)(args->flags));
1391                 return NULL;
1392         }
1393
1394         if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
1395                 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
1396
1397                 if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
1398                         bsd_idx = gen8_dispatch_bsd_engine(dev_priv, file);
1399                 } else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
1400                            bsd_idx <= I915_EXEC_BSD_RING2) {
1401                         bsd_idx >>= I915_EXEC_BSD_SHIFT;
1402                         bsd_idx--;
1403                 } else {
1404                         DRM_DEBUG("execbuf with unknown bsd ring: %u\n",
1405                                   bsd_idx);
1406                         return NULL;
1407                 }
1408
1409                 engine = &dev_priv->engine[_VCS(bsd_idx)];
1410         } else {
1411                 engine = &dev_priv->engine[user_ring_map[user_ring_id]];
1412         }
1413
1414         if (!intel_engine_initialized(engine)) {
1415                 DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id);
1416                 return NULL;
1417         }
1418
1419         return engine;
1420 }
1421
1422 static int
1423 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
1424                        struct drm_file *file,
1425                        struct drm_i915_gem_execbuffer2 *args,
1426                        struct drm_i915_gem_exec_object2 *exec)
1427 {
1428         struct drm_i915_private *dev_priv = to_i915(dev);
1429         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1430         struct drm_i915_gem_request *req = NULL;
1431         struct eb_vmas *eb;
1432         struct drm_i915_gem_object *batch_obj;
1433         struct drm_i915_gem_exec_object2 shadow_exec_entry;
1434         struct intel_engine_cs *engine;
1435         struct i915_gem_context *ctx;
1436         struct i915_address_space *vm;
1437         struct i915_execbuffer_params params_master; /* XXX: will be removed later */
1438         struct i915_execbuffer_params *params = &params_master;
1439         const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
1440         u32 dispatch_flags;
1441         int ret;
1442         bool need_relocs;
1443
1444         if (!i915_gem_check_execbuffer(args))
1445                 return -EINVAL;
1446
1447         ret = validate_exec_list(dev, exec, args->buffer_count);
1448         if (ret)
1449                 return ret;
1450
1451         dispatch_flags = 0;
1452         if (args->flags & I915_EXEC_SECURE) {
1453                 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
1454                     return -EPERM;
1455
1456                 dispatch_flags |= I915_DISPATCH_SECURE;
1457         }
1458         if (args->flags & I915_EXEC_IS_PINNED)
1459                 dispatch_flags |= I915_DISPATCH_PINNED;
1460
1461         engine = eb_select_engine(dev_priv, file, args);
1462         if (!engine)
1463                 return -EINVAL;
1464
1465         if (args->buffer_count < 1) {
1466                 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1467                 return -EINVAL;
1468         }
1469
1470         if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
1471                 if (!HAS_RESOURCE_STREAMER(dev)) {
1472                         DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
1473                         return -EINVAL;
1474                 }
1475                 if (engine->id != RCS) {
1476                         DRM_DEBUG("RS is not available on %s\n",
1477                                  engine->name);
1478                         return -EINVAL;
1479                 }
1480
1481                 dispatch_flags |= I915_DISPATCH_RS;
1482         }
1483
1484         /* Take a local wakeref for preparing to dispatch the execbuf as
1485          * we expect to access the hardware fairly frequently in the
1486          * process. Upon first dispatch, we acquire another prolonged
1487          * wakeref that we hold until the GPU has been idle for at least
1488          * 100ms.
1489          */
1490         intel_runtime_pm_get(dev_priv);
1491
1492         ret = i915_mutex_lock_interruptible(dev);
1493         if (ret)
1494                 goto pre_mutex_err;
1495
1496         ctx = i915_gem_validate_context(dev, file, engine, ctx_id);
1497         if (IS_ERR(ctx)) {
1498                 mutex_unlock(&dev->struct_mutex);
1499                 ret = PTR_ERR(ctx);
1500                 goto pre_mutex_err;
1501         }
1502
1503         i915_gem_context_get(ctx);
1504
1505         if (ctx->ppgtt)
1506                 vm = &ctx->ppgtt->base;
1507         else
1508                 vm = &ggtt->base;
1509
1510         memset(&params_master, 0x00, sizeof(params_master));
1511
1512         eb = eb_create(args);
1513         if (eb == NULL) {
1514                 i915_gem_context_put(ctx);
1515                 mutex_unlock(&dev->struct_mutex);
1516                 ret = -ENOMEM;
1517                 goto pre_mutex_err;
1518         }
1519
1520         /* Look up object handles */
1521         ret = eb_lookup_vmas(eb, exec, args, vm, file);
1522         if (ret)
1523                 goto err;
1524
1525         /* take note of the batch buffer before we might reorder the lists */
1526         batch_obj = eb_get_batch(eb);
1527
1528         /* Move the objects en-masse into the GTT, evicting if necessary. */
1529         need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
1530         ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
1531                                           &need_relocs);
1532         if (ret)
1533                 goto err;
1534
1535         /* The objects are in their final locations, apply the relocations. */
1536         if (need_relocs)
1537                 ret = i915_gem_execbuffer_relocate(eb);
1538         if (ret) {
1539                 if (ret == -EFAULT) {
1540                         ret = i915_gem_execbuffer_relocate_slow(dev, args, file,
1541                                                                 engine,
1542                                                                 eb, exec, ctx);
1543                         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1544                 }
1545                 if (ret)
1546                         goto err;
1547         }
1548
1549         /* Set the pending read domains for the batch buffer to COMMAND */
1550         if (batch_obj->base.pending_write_domain) {
1551                 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1552                 ret = -EINVAL;
1553                 goto err;
1554         }
1555
1556         params->args_batch_start_offset = args->batch_start_offset;
1557         if (intel_engine_needs_cmd_parser(engine) && args->batch_len) {
1558                 struct drm_i915_gem_object *parsed_batch_obj;
1559
1560                 parsed_batch_obj = i915_gem_execbuffer_parse(engine,
1561                                                              &shadow_exec_entry,
1562                                                              eb,
1563                                                              batch_obj,
1564                                                              args->batch_start_offset,
1565                                                              args->batch_len,
1566                                                              drm_is_current_master(file));
1567                 if (IS_ERR(parsed_batch_obj)) {
1568                         ret = PTR_ERR(parsed_batch_obj);
1569                         goto err;
1570                 }
1571
1572                 /*
1573                  * parsed_batch_obj == batch_obj means batch not fully parsed:
1574                  * Accept, but don't promote to secure.
1575                  */
1576
1577                 if (parsed_batch_obj != batch_obj) {
1578                         /*
1579                          * Batch parsed and accepted:
1580                          *
1581                          * Set the DISPATCH_SECURE bit to remove the NON_SECURE
1582                          * bit from MI_BATCH_BUFFER_START commands issued in
1583                          * the dispatch_execbuffer implementations. We
1584                          * specifically don't want that set on batches the
1585                          * command parser has accepted.
1586                          */
1587                         dispatch_flags |= I915_DISPATCH_SECURE;
1588                         params->args_batch_start_offset = 0;
1589                         batch_obj = parsed_batch_obj;
1590                 }
1591         }
1592
1593         batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
1594
1595         /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
1596          * batch" bit. Hence we need to pin secure batches into the global gtt.
1597          * hsw should have this fixed, but bdw mucks it up again. */
1598         if (dispatch_flags & I915_DISPATCH_SECURE) {
1599                 /*
1600                  * So on first glance it looks freaky that we pin the batch here
1601                  * outside of the reservation loop. But:
1602                  * - The batch is already pinned into the relevant ppgtt, so we
1603                  *   already have the backing storage fully allocated.
1604                  * - No other BO uses the global gtt (well contexts, but meh),
1605                  *   so we don't really have issues with multiple objects not
1606                  *   fitting due to fragmentation.
1607                  * So this is actually safe.
1608                  */
1609                 ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
1610                 if (ret)
1611                         goto err;
1612
1613                 params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj);
1614         } else
1615                 params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm);
1616
1617         /* Allocate a request for this batch buffer nice and early. */
1618         req = i915_gem_request_alloc(engine, ctx);
1619         if (IS_ERR(req)) {
1620                 ret = PTR_ERR(req);
1621                 goto err_batch_unpin;
1622         }
1623
1624         ret = i915_gem_request_add_to_client(req, file);
1625         if (ret)
1626                 goto err_request;
1627
1628         /*
1629          * Save assorted stuff away to pass through to *_submission().
1630          * NB: This data should be 'persistent' and not local as it will
1631          * kept around beyond the duration of the IOCTL once the GPU
1632          * scheduler arrives.
1633          */
1634         params->dev                     = dev;
1635         params->file                    = file;
1636         params->engine                    = engine;
1637         params->dispatch_flags          = dispatch_flags;
1638         params->batch_obj               = batch_obj;
1639         params->ctx                     = ctx;
1640         params->request                 = req;
1641
1642         ret = dev_priv->gt.execbuf_submit(params, args, &eb->vmas);
1643 err_request:
1644         i915_gem_execbuffer_retire_commands(params);
1645
1646 err_batch_unpin:
1647         /*
1648          * FIXME: We crucially rely upon the active tracking for the (ppgtt)
1649          * batch vma for correctness. For less ugly and less fragility this
1650          * needs to be adjusted to also track the ggtt batch vma properly as
1651          * active.
1652          */
1653         if (dispatch_flags & I915_DISPATCH_SECURE)
1654                 i915_gem_object_ggtt_unpin(batch_obj);
1655
1656 err:
1657         /* the request owns the ref now */
1658         i915_gem_context_put(ctx);
1659         eb_destroy(eb);
1660
1661         mutex_unlock(&dev->struct_mutex);
1662
1663 pre_mutex_err:
1664         /* intel_gpu_busy should also get a ref, so it will free when the device
1665          * is really idle. */
1666         intel_runtime_pm_put(dev_priv);
1667         return ret;
1668 }
1669
1670 /*
1671  * Legacy execbuffer just creates an exec2 list from the original exec object
1672  * list array and passes it to the real function.
1673  */
1674 int
1675 i915_gem_execbuffer(struct drm_device *dev, void *data,
1676                     struct drm_file *file)
1677 {
1678         struct drm_i915_gem_execbuffer *args = data;
1679         struct drm_i915_gem_execbuffer2 exec2;
1680         struct drm_i915_gem_exec_object *exec_list = NULL;
1681         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1682         int ret, i;
1683
1684         if (args->buffer_count < 1) {
1685                 DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1686                 return -EINVAL;
1687         }
1688
1689         /* Copy in the exec list from userland */
1690         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
1691         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
1692         if (exec_list == NULL || exec2_list == NULL) {
1693                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1694                           args->buffer_count);
1695                 drm_free_large(exec_list);
1696                 drm_free_large(exec2_list);
1697                 return -ENOMEM;
1698         }
1699         ret = copy_from_user(exec_list,
1700                              u64_to_user_ptr(args->buffers_ptr),
1701                              sizeof(*exec_list) * args->buffer_count);
1702         if (ret != 0) {
1703                 DRM_DEBUG("copy %d exec entries failed %d\n",
1704                           args->buffer_count, ret);
1705                 drm_free_large(exec_list);
1706                 drm_free_large(exec2_list);
1707                 return -EFAULT;
1708         }
1709
1710         for (i = 0; i < args->buffer_count; i++) {
1711                 exec2_list[i].handle = exec_list[i].handle;
1712                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
1713                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
1714                 exec2_list[i].alignment = exec_list[i].alignment;
1715                 exec2_list[i].offset = exec_list[i].offset;
1716                 if (INTEL_INFO(dev)->gen < 4)
1717                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
1718                 else
1719                         exec2_list[i].flags = 0;
1720         }
1721
1722         exec2.buffers_ptr = args->buffers_ptr;
1723         exec2.buffer_count = args->buffer_count;
1724         exec2.batch_start_offset = args->batch_start_offset;
1725         exec2.batch_len = args->batch_len;
1726         exec2.DR1 = args->DR1;
1727         exec2.DR4 = args->DR4;
1728         exec2.num_cliprects = args->num_cliprects;
1729         exec2.cliprects_ptr = args->cliprects_ptr;
1730         exec2.flags = I915_EXEC_RENDER;
1731         i915_execbuffer2_set_context_id(exec2, 0);
1732
1733         ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1734         if (!ret) {
1735                 struct drm_i915_gem_exec_object __user *user_exec_list =
1736                         u64_to_user_ptr(args->buffers_ptr);
1737
1738                 /* Copy the new buffer offsets back to the user's exec list. */
1739                 for (i = 0; i < args->buffer_count; i++) {
1740                         exec2_list[i].offset =
1741                                 gen8_canonical_addr(exec2_list[i].offset);
1742                         ret = __copy_to_user(&user_exec_list[i].offset,
1743                                              &exec2_list[i].offset,
1744                                              sizeof(user_exec_list[i].offset));
1745                         if (ret) {
1746                                 ret = -EFAULT;
1747                                 DRM_DEBUG("failed to copy %d exec entries "
1748                                           "back to user (%d)\n",
1749                                           args->buffer_count, ret);
1750                                 break;
1751                         }
1752                 }
1753         }
1754
1755         drm_free_large(exec_list);
1756         drm_free_large(exec2_list);
1757         return ret;
1758 }
1759
1760 int
1761 i915_gem_execbuffer2(struct drm_device *dev, void *data,
1762                      struct drm_file *file)
1763 {
1764         struct drm_i915_gem_execbuffer2 *args = data;
1765         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
1766         int ret;
1767
1768         if (args->buffer_count < 1 ||
1769             args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
1770                 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
1771                 return -EINVAL;
1772         }
1773
1774         if (args->rsvd2 != 0) {
1775                 DRM_DEBUG("dirty rvsd2 field\n");
1776                 return -EINVAL;
1777         }
1778
1779         exec2_list = drm_malloc_gfp(args->buffer_count,
1780                                     sizeof(*exec2_list),
1781                                     GFP_TEMPORARY);
1782         if (exec2_list == NULL) {
1783                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1784                           args->buffer_count);
1785                 return -ENOMEM;
1786         }
1787         ret = copy_from_user(exec2_list,
1788                              u64_to_user_ptr(args->buffers_ptr),
1789                              sizeof(*exec2_list) * args->buffer_count);
1790         if (ret != 0) {
1791                 DRM_DEBUG("copy %d exec entries failed %d\n",
1792                           args->buffer_count, ret);
1793                 drm_free_large(exec2_list);
1794                 return -EFAULT;
1795         }
1796
1797         ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1798         if (!ret) {
1799                 /* Copy the new buffer offsets back to the user's exec list. */
1800                 struct drm_i915_gem_exec_object2 __user *user_exec_list =
1801                                    u64_to_user_ptr(args->buffers_ptr);
1802                 int i;
1803
1804                 for (i = 0; i < args->buffer_count; i++) {
1805                         exec2_list[i].offset =
1806                                 gen8_canonical_addr(exec2_list[i].offset);
1807                         ret = __copy_to_user(&user_exec_list[i].offset,
1808                                              &exec2_list[i].offset,
1809                                              sizeof(user_exec_list[i].offset));
1810                         if (ret) {
1811                                 ret = -EFAULT;
1812                                 DRM_DEBUG("failed to copy %d exec entries "
1813                                           "back to user\n",
1814                                           args->buffer_count);
1815                                 break;
1816                         }
1817                 }
1818         }
1819
1820         drm_free_large(exec2_list);
1821         return ret;
1822 }