drivers/gpu/drm/i915/gt/intel_ring_submission.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2008-2021 Intel Corporation
   4  */
   5
   6 #include <drm/drm_cache.h>
   7
   8 #include "gem/i915_gem_internal.h"
   9
  10 #include "gen2_engine_cs.h"
  11 #include "gen6_engine_cs.h"
  12 #include "gen6_ppgtt.h"
  13 #include "gen7_renderclear.h"
  14 #include "i915_drv.h"
  15 #include "i915_mitigations.h"
  16 #include "intel_breadcrumbs.h"
  17 #include "intel_context.h"
  18 #include "intel_engine_regs.h"
  19 #include "intel_gt.h"
  20 #include "intel_gt_irq.h"
  21 #include "intel_gt_regs.h"
  22 #include "intel_reset.h"
  23 #include "intel_ring.h"
  24 #include "shmem_utils.h"
  25 #include "intel_engine_heartbeat.h"
  26 #include "intel_engine_pm.h"
  27
  28 /* Rough estimate of the typical request size, performing a flush,
  29  * set-context and then emitting the batch.
  30  */
  31 #define LEGACY_REQUEST_SIZE 200
  32
  33 static void set_hwstam(struct intel_engine_cs *engine, u32 mask)
  34 {
  35         /*
  36          * Keep the render interrupt unmasked as this papers over
  37          * lost interrupts following a reset.
  38          */
  39         if (engine->class == RENDER_CLASS) {
  40                 if (GRAPHICS_VER(engine->i915) >= 6)
  41                         mask &= ~BIT(0);
  42                 else
  43                         mask &= ~I915_USER_INTERRUPT;
  44         }
  45
  46         intel_engine_set_hwsp_writemask(engine, mask);
  47 }
  48
  49 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys)
  50 {
  51         u32 addr;
  52
  53         addr = lower_32_bits(phys);
  54         if (GRAPHICS_VER(engine->i915) >= 4)
  55                 addr |= (phys >> 28) & 0xf0;
  56
  57         intel_uncore_write(engine->uncore, HWS_PGA, addr);
  58 }
  59
  60 static struct page *status_page(struct intel_engine_cs *engine)
  61 {
  62         struct drm_i915_gem_object *obj = engine->status_page.vma->obj;
  63
  64         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
  65         return sg_page(obj->mm.pages->sgl);
  66 }
  67
  68 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
  69 {
  70         set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine))));
  71         set_hwstam(engine, ~0u);
  72 }
  73
  74 static void set_hwsp(struct intel_engine_cs *engine, u32 offset)
  75 {
  76         i915_reg_t hwsp;
  77
  78         /*
  79          * The ring status page addresses are no longer next to the rest of
  80          * the ring registers as of gen7.
  81          */
  82         if (GRAPHICS_VER(engine->i915) == 7) {
  83                 switch (engine->id) {
  84                 /*
  85                  * No more rings exist on Gen7. Default case is only to shut up
  86                  * gcc switch check warning.
  87                  */
  88                 default:
  89                         GEM_BUG_ON(engine->id);
  90                         fallthrough;
  91                 case RCS0:
  92                         hwsp = RENDER_HWS_PGA_GEN7;
  93                         break;
  94                 case BCS0:
  95                         hwsp = BLT_HWS_PGA_GEN7;
  96                         break;
  97                 case VCS0:
  98                         hwsp = BSD_HWS_PGA_GEN7;
  99                         break;
 100                 case VECS0:
 101                         hwsp = VEBOX_HWS_PGA_GEN7;
 102                         break;
 103                 }
 104         } else if (GRAPHICS_VER(engine->i915) == 6) {
 105                 hwsp = RING_HWS_PGA_GEN6(engine->mmio_base);
 106         } else {
 107                 hwsp = RING_HWS_PGA(engine->mmio_base);
 108         }
 109
 110         intel_uncore_write_fw(engine->uncore, hwsp, offset);
 111         intel_uncore_posting_read_fw(engine->uncore, hwsp);
 112 }
 113
 114 static void flush_cs_tlb(struct intel_engine_cs *engine)
 115 {
 116         if (!IS_GRAPHICS_VER(engine->i915, 6, 7))
 117                 return;
 118
 119         /* ring should be idle before issuing a sync flush*/
 120         if ((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0)
 121                 drm_warn(&engine->i915->drm, "%s not idle before sync flush!\n",
 122                          engine->name);
 123
 124         ENGINE_WRITE_FW(engine, RING_INSTPM,
 125                         _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
 126                                            INSTPM_SYNC_FLUSH));
 127         if (__intel_wait_for_register_fw(engine->uncore,
 128                                          RING_INSTPM(engine->mmio_base),
 129                                          INSTPM_SYNC_FLUSH, 0,
 130                                          2000, 0, NULL))
 131                 ENGINE_TRACE(engine,
 132                              "wait for SyncFlush to complete for TLB invalidation timed out\n");
 133 }
 134
 135 static void ring_setup_status_page(struct intel_engine_cs *engine)
 136 {
 137         set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma));
 138         set_hwstam(engine, ~0u);
 139
 140         flush_cs_tlb(engine);
 141 }
 142
 143 static struct i915_address_space *vm_alias(struct i915_address_space *vm)
 144 {
 145         if (i915_is_ggtt(vm))
 146                 vm = &i915_vm_to_ggtt(vm)->alias->vm;
 147
 148         return vm;
 149 }
 150
 151 static u32 pp_dir(struct i915_address_space *vm)
 152 {
 153         return to_gen6_ppgtt(i915_vm_to_ppgtt(vm))->pp_dir;
 154 }
 155
 156 static void set_pp_dir(struct intel_engine_cs *engine)
 157 {
 158         struct i915_address_space *vm = vm_alias(engine->gt->vm);
 159
 160         if (!vm)
 161                 return;
 162
 163         ENGINE_WRITE_FW(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G);
 164         ENGINE_WRITE_FW(engine, RING_PP_DIR_BASE, pp_dir(vm));
 165
 166         if (GRAPHICS_VER(engine->i915) >= 7) {
 167                 ENGINE_WRITE_FW(engine,
 168                                 RING_MODE_GEN7,
 169                                 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 170         }
 171 }
 172
 173 static bool stop_ring(struct intel_engine_cs *engine)
 174 {
 175         /* Empty the ring by skipping to the end */
 176         ENGINE_WRITE_FW(engine, RING_HEAD, ENGINE_READ_FW(engine, RING_TAIL));
 177         ENGINE_POSTING_READ(engine, RING_HEAD);
 178
 179         /* The ring must be empty before it is disabled */
 180         ENGINE_WRITE_FW(engine, RING_CTL, 0);
 181         ENGINE_POSTING_READ(engine, RING_CTL);
 182
 183         /* Then reset the disabled ring */
 184         ENGINE_WRITE_FW(engine, RING_HEAD, 0);
 185         ENGINE_WRITE_FW(engine, RING_TAIL, 0);
 186
 187         return (ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) == 0;
 188 }
 189
 190 static int xcs_resume(struct intel_engine_cs *engine)
 191 {
 192         struct intel_ring *ring = engine->legacy.ring;
 193
 194         ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n",
 195                      ring->head, ring->tail);
 196
 197         /*
 198          * Double check the ring is empty & disabled before we resume. Called
 199          * from atomic context during PCI probe, so _hardirq().
 200          */
 201         intel_synchronize_hardirq(engine->i915);
 202         if (!stop_ring(engine))
 203                 goto err;
 204
 205         if (HWS_NEEDS_PHYSICAL(engine->i915))
 206                 ring_setup_phys_status_page(engine);
 207         else
 208                 ring_setup_status_page(engine);
 209
 210         intel_breadcrumbs_reset(engine->breadcrumbs);
 211
 212         /* Enforce ordering by reading HEAD register back */
 213         ENGINE_POSTING_READ(engine, RING_HEAD);
 214
 215         /*
 216          * Initialize the ring. This must happen _after_ we've cleared the ring
 217          * registers with the above sequence (the readback of the HEAD registers
 218          * also enforces ordering), otherwise the hw might lose the new ring
 219          * register values.
 220          */
 221         ENGINE_WRITE_FW(engine, RING_START, i915_ggtt_offset(ring->vma));
 222
 223         /* Check that the ring offsets point within the ring! */
 224         GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head));
 225         GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail));
 226         intel_ring_update_space(ring);
 227
 228         set_pp_dir(engine);
 229
 230         /* First wake the ring up to an empty/idle ring */
 231         ENGINE_WRITE_FW(engine, RING_HEAD, ring->head);
 232         ENGINE_WRITE_FW(engine, RING_TAIL, ring->head);
 233         ENGINE_POSTING_READ(engine, RING_TAIL);
 234
 235         ENGINE_WRITE_FW(engine, RING_CTL,
 236                         RING_CTL_SIZE(ring->size) | RING_VALID);
 237
 238         /* If the head is still not zero, the ring is dead */
 239         if (__intel_wait_for_register_fw(engine->uncore,
 240                                          RING_CTL(engine->mmio_base),
 241                                          RING_VALID, RING_VALID,
 242                                          5000, 0, NULL))
 243                 goto err;
 244
 245         if (GRAPHICS_VER(engine->i915) > 2)
 246                 ENGINE_WRITE_FW(engine,
 247                                 RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
 248
 249         /* Now awake, let it get started */
 250         if (ring->tail != ring->head) {
 251                 ENGINE_WRITE_FW(engine, RING_TAIL, ring->tail);
 252                 ENGINE_POSTING_READ(engine, RING_TAIL);
 253         }
 254
 255         /* Papering over lost _interrupts_ immediately following the restart */
 256         intel_engine_signal_breadcrumbs(engine);
 257         return 0;
 258
 259 err:
 260         drm_err(&engine->i915->drm,
 261                 "%s initialization failed; "
 262                 "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
 263                 engine->name,
 264                 ENGINE_READ(engine, RING_CTL),
 265                 ENGINE_READ(engine, RING_CTL) & RING_VALID,
 266                 ENGINE_READ(engine, RING_HEAD), ring->head,
 267                 ENGINE_READ(engine, RING_TAIL), ring->tail,
 268                 ENGINE_READ(engine, RING_START),
 269                 i915_ggtt_offset(ring->vma));
 270         return -EIO;
 271 }
 272
 273 static void sanitize_hwsp(struct intel_engine_cs *engine)
 274 {
 275         struct intel_timeline *tl;
 276
 277         list_for_each_entry(tl, &engine->status_page.timelines, engine_link)
 278                 intel_timeline_reset_seqno(tl);
 279 }
 280
 281 static void xcs_sanitize(struct intel_engine_cs *engine)
 282 {
 283         /*
 284          * Poison residual state on resume, in case the suspend didn't!
 285          *
 286          * We have to assume that across suspend/resume (or other loss
 287          * of control) that the contents of our pinned buffers has been
 288          * lost, replaced by garbage. Since this doesn't always happen,
 289          * let's poison such state so that we more quickly spot when
 290          * we falsely assume it has been preserved.
 291          */
 292         if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 293                 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);
 294
 295         /*
 296          * The kernel_context HWSP is stored in the status_page. As above,
 297          * that may be lost on resume/initialisation, and so we need to
 298          * reset the value in the HWSP.
 299          */
 300         sanitize_hwsp(engine);
 301
 302         /* And scrub the dirty cachelines for the HWSP */
 303         drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE);
 304
 305         intel_engine_reset_pinned_contexts(engine);
 306 }
 307
 308 static void reset_prepare(struct intel_engine_cs *engine)
 309 {
 310         /*
 311          * We stop engines, otherwise we might get failed reset and a
 312          * dead gpu (on elk). Also as modern gpu as kbl can suffer
 313          * from system hang if batchbuffer is progressing when
 314          * the reset is issued, regardless of READY_TO_RESET ack.
 315          * Thus assume it is best to stop engines on all gens
 316          * where we have a gpu reset.
 317          *
 318          * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES)
 319          *
 320          * WaMediaResetMainRingCleanup:ctg,elk (presumably)
 321          * WaClearRingBufHeadRegAtInit:ctg,elk
 322          *
 323          * FIXME: Wa for more modern gens needs to be validated
 324          */
 325         ENGINE_TRACE(engine, "\n");
 326         intel_engine_stop_cs(engine);
 327
 328         if (!stop_ring(engine)) {
 329                 /* G45 ring initialization often fails to reset head to zero */
 330                 ENGINE_TRACE(engine,
 331                              "HEAD not reset to zero, "
 332                              "{ CTL:%08x, HEAD:%08x, TAIL:%08x, START:%08x }\n",
 333                              ENGINE_READ_FW(engine, RING_CTL),
 334                              ENGINE_READ_FW(engine, RING_HEAD),
 335                              ENGINE_READ_FW(engine, RING_TAIL),
 336                              ENGINE_READ_FW(engine, RING_START));
 337                 if (!stop_ring(engine)) {
 338                         drm_err(&engine->i915->drm,
 339                                 "failed to set %s head to zero "
 340                                 "ctl %08x head %08x tail %08x start %08x\n",
 341                                 engine->name,
 342                                 ENGINE_READ_FW(engine, RING_CTL),
 343                                 ENGINE_READ_FW(engine, RING_HEAD),
 344                                 ENGINE_READ_FW(engine, RING_TAIL),
 345                                 ENGINE_READ_FW(engine, RING_START));
 346                 }
 347         }
 348 }
 349
 350 static void reset_rewind(struct intel_engine_cs *engine, bool stalled)
 351 {
 352         struct i915_request *pos, *rq;
 353         unsigned long flags;
 354         u32 head;
 355
 356         rq = NULL;
 357         spin_lock_irqsave(&engine->sched_engine->lock, flags);
 358         rcu_read_lock();
 359         list_for_each_entry(pos, &engine->sched_engine->requests, sched.link) {
 360                 if (!__i915_request_is_complete(pos)) {
 361                         rq = pos;
 362                         break;
 363                 }
 364         }
 365         rcu_read_unlock();
 366
 367         /*
 368          * The guilty request will get skipped on a hung engine.
 369          *
 370          * Users of client default contexts do not rely on logical
 371          * state preserved between batches so it is safe to execute
 372          * queued requests following the hang. Non default contexts
 373          * rely on preserved state, so skipping a batch loses the
 374          * evolution of the state and it needs to be considered corrupted.
 375          * Executing more queued batches on top of corrupted state is
 376          * risky. But we take the risk by trying to advance through
 377          * the queued requests in order to make the client behaviour
 378          * more predictable around resets, by not throwing away random
 379          * amount of batches it has prepared for execution. Sophisticated
 380          * clients can use gem_reset_stats_ioctl and dma fence status
 381          * (exported via sync_file info ioctl on explicit fences) to observe
 382          * when it loses the context state and should rebuild accordingly.
 383          *
 384          * The context ban, and ultimately the client ban, mechanism are safety
 385          * valves if client submission ends up resulting in nothing more than
 386          * subsequent hangs.
 387          */
 388
 389         if (rq) {
 390                 /*
 391                  * Try to restore the logical GPU state to match the
 392                  * continuation of the request queue. If we skip the
 393                  * context/PD restore, then the next request may try to execute
 394                  * assuming that its context is valid and loaded on the GPU and
 395                  * so may try to access invalid memory, prompting repeated GPU
 396                  * hangs.
 397                  *
 398                  * If the request was guilty, we still restore the logical
 399                  * state in case the next request requires it (e.g. the
 400                  * aliasing ppgtt), but skip over the hung batch.
 401                  *
 402                  * If the request was innocent, we try to replay the request
 403                  * with the restored context.
 404                  */
 405                 __i915_request_reset(rq, stalled);
 406
 407                 GEM_BUG_ON(rq->ring != engine->legacy.ring);
 408                 head = rq->head;
 409         } else {
 410                 head = engine->legacy.ring->tail;
 411         }
 412         engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head);
 413
 414         spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
 415 }
 416
 417 static void reset_finish(struct intel_engine_cs *engine)
 418 {
 419 }
 420
 421 static void reset_cancel(struct intel_engine_cs *engine)
 422 {
 423         struct i915_request *request;
 424         unsigned long flags;
 425
 426         spin_lock_irqsave(&engine->sched_engine->lock, flags);
 427
 428         /* Mark all submitted requests as skipped. */
 429         list_for_each_entry(request, &engine->sched_engine->requests, sched.link)
 430                 i915_request_put(i915_request_mark_eio(request));
 431         intel_engine_signal_breadcrumbs(engine);
 432
 433         /* Remaining _unready_ requests will be nop'ed when submitted */
 434
 435         spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
 436 }
 437
 438 static void i9xx_submit_request(struct i915_request *request)
 439 {
 440         i915_request_submit(request);
 441         wmb(); /* paranoid flush writes out of the WCB before mmio */
 442
 443         ENGINE_WRITE(request->engine, RING_TAIL,
 444                      intel_ring_set_tail(request->ring, request->tail));
 445 }
 446
 447 static void __ring_context_fini(struct intel_context *ce)
 448 {
 449         i915_vma_put(ce->state);
 450 }
 451
 452 static void ring_context_destroy(struct kref *ref)
 453 {
 454         struct intel_context *ce = container_of(ref, typeof(*ce), ref);
 455
 456         GEM_BUG_ON(intel_context_is_pinned(ce));
 457
 458         if (ce->state)
 459                 __ring_context_fini(ce);
 460
 461         intel_context_fini(ce);
 462         intel_context_free(ce);
 463 }
 464
 465 static int ring_context_init_default_state(struct intel_context *ce,
 466                                            struct i915_gem_ww_ctx *ww)
 467 {
 468         struct drm_i915_gem_object *obj = ce->state->obj;
 469         void *vaddr;
 470
 471         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
 472         if (IS_ERR(vaddr))
 473                 return PTR_ERR(vaddr);
 474
 475         shmem_read(ce->engine->default_state, 0,
 476                    vaddr, ce->engine->context_size);
 477
 478         i915_gem_object_flush_map(obj);
 479         __i915_gem_object_release_map(obj);
 480
 481         __set_bit(CONTEXT_VALID_BIT, &ce->flags);
 482         return 0;
 483 }
 484
 485 static int ring_context_pre_pin(struct intel_context *ce,
 486                                 struct i915_gem_ww_ctx *ww,
 487                                 void **unused)
 488 {
 489         struct i915_address_space *vm;
 490         int err = 0;
 491
 492         if (ce->engine->default_state &&
 493             !test_bit(CONTEXT_VALID_BIT, &ce->flags)) {
 494                 err = ring_context_init_default_state(ce, ww);
 495                 if (err)
 496                         return err;
 497         }
 498
 499         vm = vm_alias(ce->vm);
 500         if (vm)
 501                 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww);
 502
 503         return err;
 504 }
 505
 506 static void __context_unpin_ppgtt(struct intel_context *ce)
 507 {
 508         struct i915_address_space *vm;
 509
 510         vm = vm_alias(ce->vm);
 511         if (vm)
 512                 gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm));
 513 }
 514
 515 static void ring_context_unpin(struct intel_context *ce)
 516 {
 517 }
 518
 519 static void ring_context_post_unpin(struct intel_context *ce)
 520 {
 521         __context_unpin_ppgtt(ce);
 522 }
 523
 524 static struct i915_vma *
 525 alloc_context_vma(struct intel_engine_cs *engine)
 526 {
 527         struct drm_i915_private *i915 = engine->i915;
 528         struct drm_i915_gem_object *obj;
 529         struct i915_vma *vma;
 530         int err;
 531
 532         obj = i915_gem_object_create_shmem(i915, engine->context_size);
 533         if (IS_ERR(obj))
 534                 return ERR_CAST(obj);
 535
 536         /*
 537          * Try to make the context utilize L3 as well as LLC.
 538          *
 539          * On VLV we don't have L3 controls in the PTEs so we
 540          * shouldn't touch the cache level, especially as that
 541          * would make the object snooped which might have a
 542          * negative performance impact.
 543          *
 544          * Snooping is required on non-llc platforms in execlist
 545          * mode, but since all GGTT accesses use PAT entry 0 we
 546          * get snooping anyway regardless of cache_level.
 547          *
 548          * This is only applicable for Ivy Bridge devices since
 549          * later platforms don't have L3 control bits in the PTE.
 550          */
 551         if (IS_IVYBRIDGE(i915))
 552                 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);
 553
 554         vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
 555         if (IS_ERR(vma)) {
 556                 err = PTR_ERR(vma);
 557                 goto err_obj;
 558         }
 559
 560         return vma;
 561
 562 err_obj:
 563         i915_gem_object_put(obj);
 564         return ERR_PTR(err);
 565 }
 566
 567 static int ring_context_alloc(struct intel_context *ce)
 568 {
 569         struct intel_engine_cs *engine = ce->engine;
 570
 571         /* One ringbuffer to rule them all */
 572         GEM_BUG_ON(!engine->legacy.ring);
 573         ce->ring = engine->legacy.ring;
 574         ce->timeline = intel_timeline_get(engine->legacy.timeline);
 575
 576         GEM_BUG_ON(ce->state);
 577         if (engine->context_size) {
 578                 struct i915_vma *vma;
 579
 580                 vma = alloc_context_vma(engine);
 581                 if (IS_ERR(vma))
 582                         return PTR_ERR(vma);
 583
 584                 ce->state = vma;
 585         }
 586
 587         return 0;
 588 }
 589
 590 static int ring_context_pin(struct intel_context *ce, void *unused)
 591 {
 592         return 0;
 593 }
 594
 595 static void ring_context_reset(struct intel_context *ce)
 596 {
 597         intel_ring_reset(ce->ring, ce->ring->emit);
 598         clear_bit(CONTEXT_VALID_BIT, &ce->flags);
 599 }
 600
 601 static void ring_context_revoke(struct intel_context *ce,
 602                                 struct i915_request *rq,
 603                                 unsigned int preempt_timeout_ms)
 604 {
 605         struct intel_engine_cs *engine;
 606
 607         if (!rq || !i915_request_is_active(rq))
 608                 return;
 609
 610         engine = rq->engine;
 611         lockdep_assert_held(&engine->sched_engine->lock);
 612         list_for_each_entry_continue(rq, &engine->sched_engine->requests,
 613                                      sched.link)
 614                 if (rq->context == ce) {
 615                         i915_request_set_error_once(rq, -EIO);
 616                         __i915_request_skip(rq);
 617                 }
 618 }
 619
 620 static void ring_context_cancel_request(struct intel_context *ce,
 621                                         struct i915_request *rq)
 622 {
 623         struct intel_engine_cs *engine = NULL;
 624
 625         i915_request_active_engine(rq, &engine);
 626
 627         if (engine && intel_engine_pulse(engine))
 628                 intel_gt_handle_error(engine->gt, engine->mask, 0,
 629                                       "request cancellation by %s",
 630                                       current->comm);
 631 }
 632
 633 static const struct intel_context_ops ring_context_ops = {
 634         .alloc = ring_context_alloc,
 635
 636         .cancel_request = ring_context_cancel_request,
 637
 638         .revoke = ring_context_revoke,
 639
 640         .pre_pin = ring_context_pre_pin,
 641         .pin = ring_context_pin,
 642         .unpin = ring_context_unpin,
 643         .post_unpin = ring_context_post_unpin,
 644
 645         .enter = intel_context_enter_engine,
 646         .exit = intel_context_exit_engine,
 647
 648         .reset = ring_context_reset,
 649         .destroy = ring_context_destroy,
 650 };
 651
 652 static int load_pd_dir(struct i915_request *rq,
 653                        struct i915_address_space *vm,
 654                        u32 valid)
 655 {
 656         const struct intel_engine_cs * const engine = rq->engine;
 657         u32 *cs;
 658
 659         cs = intel_ring_begin(rq, 12);
 660         if (IS_ERR(cs))
 661                 return PTR_ERR(cs);
 662
 663         *cs++ = MI_LOAD_REGISTER_IMM(1);
 664         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));
 665         *cs++ = valid;
 666
 667         *cs++ = MI_LOAD_REGISTER_IMM(1);
 668         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
 669         *cs++ = pp_dir(vm);
 670
 671         /* Stall until the page table load is complete? */
 672         *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
 673         *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
 674         *cs++ = intel_gt_scratch_offset(engine->gt,
 675                                         INTEL_GT_SCRATCH_FIELD_DEFAULT);
 676
 677         *cs++ = MI_LOAD_REGISTER_IMM(1);
 678         *cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base));
 679         *cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE);
 680
 681         intel_ring_advance(rq, cs);
 682
 683         return rq->engine->emit_flush(rq, EMIT_FLUSH);
 684 }
 685
 686 static int mi_set_context(struct i915_request *rq,
 687                           struct intel_context *ce,
 688                           u32 flags)
 689 {
 690         struct intel_engine_cs *engine = rq->engine;
 691         struct drm_i915_private *i915 = engine->i915;
 692         enum intel_engine_id id;
 693         const int num_engines =
 694                 IS_HASWELL(i915) ? engine->gt->info.num_engines - 1 : 0;
 695         bool force_restore = false;
 696         int len;
 697         u32 *cs;
 698
 699         len = 4;
 700         if (GRAPHICS_VER(i915) == 7)
 701                 len += 2 + (num_engines ? 4 * num_engines + 6 : 0);
 702         else if (GRAPHICS_VER(i915) == 5)
 703                 len += 2;
 704         if (flags & MI_FORCE_RESTORE) {
 705                 GEM_BUG_ON(flags & MI_RESTORE_INHIBIT);
 706                 flags &= ~MI_FORCE_RESTORE;
 707                 force_restore = true;
 708                 len += 2;
 709         }
 710
 711         cs = intel_ring_begin(rq, len);
 712         if (IS_ERR(cs))
 713                 return PTR_ERR(cs);
 714
 715         /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
 716         if (GRAPHICS_VER(i915) == 7) {
 717                 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
 718                 if (num_engines) {
 719                         struct intel_engine_cs *signaller;
 720
 721                         *cs++ = MI_LOAD_REGISTER_IMM(num_engines);
 722                         for_each_engine(signaller, engine->gt, id) {
 723                                 if (signaller == engine)
 724                                         continue;
 725
 726                                 *cs++ = i915_mmio_reg_offset(
 727                                            RING_PSMI_CTL(signaller->mmio_base));
 728                                 *cs++ = _MASKED_BIT_ENABLE(
 729                                                 GEN6_PSMI_SLEEP_MSG_DISABLE);
 730                         }
 731                 }
 732         } else if (GRAPHICS_VER(i915) == 5) {
 733                 /*
 734                  * This w/a is only listed for pre-production ilk a/b steppings,
 735                  * but is also mentioned for programming the powerctx. To be
 736                  * safe, just apply the workaround; we do not use SyncFlush so
 737                  * this should never take effect and so be a no-op!
 738                  */
 739                 *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN;
 740         }
 741
 742         if (force_restore) {
 743                 /*
 744                  * The HW doesn't handle being told to restore the current
 745                  * context very well. Quite often it likes goes to go off and
 746                  * sulk, especially when it is meant to be reloading PP_DIR.
 747                  * A very simple fix to force the reload is to simply switch
 748                  * away from the current context and back again.
 749                  *
 750                  * Note that the kernel_context will contain random state
 751                  * following the INHIBIT_RESTORE. We accept this since we
 752                  * never use the kernel_context state; it is merely a
 753                  * placeholder we use to flush other contexts.
 754                  */
 755                 *cs++ = MI_SET_CONTEXT;
 756                 *cs++ = i915_ggtt_offset(engine->kernel_context->state) |
 757                         MI_MM_SPACE_GTT |
 758                         MI_RESTORE_INHIBIT;
 759         }
 760
 761         *cs++ = MI_NOOP;
 762         *cs++ = MI_SET_CONTEXT;
 763         *cs++ = i915_ggtt_offset(ce->state) | flags;
 764         /*
 765          * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
 766          * WaMiSetContext_Hang:snb,ivb,vlv
 767          */
 768         *cs++ = MI_NOOP;
 769
 770         if (GRAPHICS_VER(i915) == 7) {
 771                 if (num_engines) {
 772                         struct intel_engine_cs *signaller;
 773                         i915_reg_t last_reg = INVALID_MMIO_REG; /* keep gcc quiet */
 774
 775                         *cs++ = MI_LOAD_REGISTER_IMM(num_engines);
 776                         for_each_engine(signaller, engine->gt, id) {
 777                                 if (signaller == engine)
 778                                         continue;
 779
 780                                 last_reg = RING_PSMI_CTL(signaller->mmio_base);
 781                                 *cs++ = i915_mmio_reg_offset(last_reg);
 782                                 *cs++ = _MASKED_BIT_DISABLE(
 783                                                 GEN6_PSMI_SLEEP_MSG_DISABLE);
 784                         }
 785
 786                         /* Insert a delay before the next switch! */
 787                         *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
 788                         *cs++ = i915_mmio_reg_offset(last_reg);
 789                         *cs++ = intel_gt_scratch_offset(engine->gt,
 790                                                         INTEL_GT_SCRATCH_FIELD_DEFAULT);
 791                         *cs++ = MI_NOOP;
 792                 }
 793                 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 794         } else if (GRAPHICS_VER(i915) == 5) {
 795                 *cs++ = MI_SUSPEND_FLUSH;
 796         }
 797
 798         intel_ring_advance(rq, cs);
 799
 800         return 0;
 801 }
 802
 803 static int remap_l3_slice(struct i915_request *rq, int slice)
 804 {
 805 #define L3LOG_DW (GEN7_L3LOG_SIZE / sizeof(u32))
 806         u32 *cs, *remap_info = rq->engine->i915->l3_parity.remap_info[slice];
 807         int i;
 808
 809         if (!remap_info)
 810                 return 0;
 811
 812         cs = intel_ring_begin(rq, L3LOG_DW * 2 + 2);
 813         if (IS_ERR(cs))
 814                 return PTR_ERR(cs);
 815
 816         /*
 817          * Note: We do not worry about the concurrent register cacheline hang
 818          * here because no other code should access these registers other than
 819          * at initialization time.
 820          */
 821         *cs++ = MI_LOAD_REGISTER_IMM(L3LOG_DW);
 822         for (i = 0; i < L3LOG_DW; i++) {
 823                 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));
 824                 *cs++ = remap_info[i];
 825         }
 826         *cs++ = MI_NOOP;
 827         intel_ring_advance(rq, cs);
 828
 829         return 0;
 830 #undef L3LOG_DW
 831 }
 832
 833 static int remap_l3(struct i915_request *rq)
 834 {
 835         struct i915_gem_context *ctx = i915_request_gem_context(rq);
 836         int i, err;
 837
 838         if (!ctx || !ctx->remap_slice)
 839                 return 0;
 840
 841         for (i = 0; i < MAX_L3_SLICES; i++) {
 842                 if (!(ctx->remap_slice & BIT(i)))
 843                         continue;
 844
 845                 err = remap_l3_slice(rq, i);
 846                 if (err)
 847                         return err;
 848         }
 849
 850         ctx->remap_slice = 0;
 851         return 0;
 852 }
 853
 854 static int switch_mm(struct i915_request *rq, struct i915_address_space *vm)
 855 {
 856         int ret;
 857
 858         if (!vm)
 859                 return 0;
 860
 861         ret = rq->engine->emit_flush(rq, EMIT_FLUSH);
 862         if (ret)
 863                 return ret;
 864
 865         /*
 866          * Not only do we need a full barrier (post-sync write) after
 867          * invalidating the TLBs, but we need to wait a little bit
 868          * longer. Whether this is merely delaying us, or the
 869          * subsequent flush is a key part of serialising with the
 870          * post-sync op, this extra pass appears vital before a
 871          * mm switch!
 872          */
 873         ret = load_pd_dir(rq, vm, PP_DIR_DCLV_2G);
 874         if (ret)
 875                 return ret;
 876
 877         return rq->engine->emit_flush(rq, EMIT_INVALIDATE);
 878 }
 879
 880 static int clear_residuals(struct i915_request *rq)
 881 {
 882         struct intel_engine_cs *engine = rq->engine;
 883         int ret;
 884
 885         ret = switch_mm(rq, vm_alias(engine->kernel_context->vm));
 886         if (ret)
 887                 return ret;
 888
 889         if (engine->kernel_context->state) {
 890                 ret = mi_set_context(rq,
 891                                      engine->kernel_context,
 892                                      MI_MM_SPACE_GTT | MI_RESTORE_INHIBIT);
 893                 if (ret)
 894                         return ret;
 895         }
 896
 897         ret = engine->emit_bb_start(rq,
 898                                     i915_vma_offset(engine->wa_ctx.vma), 0,
 899                                     0);
 900         if (ret)
 901                 return ret;
 902
 903         ret = engine->emit_flush(rq, EMIT_FLUSH);
 904         if (ret)
 905                 return ret;
 906
 907         /* Always invalidate before the next switch_mm() */
 908         return engine->emit_flush(rq, EMIT_INVALIDATE);
 909 }
 910
 911 static int switch_context(struct i915_request *rq)
 912 {
 913         struct intel_engine_cs *engine = rq->engine;
 914         struct intel_context *ce = rq->context;
 915         void **residuals = NULL;
 916         int ret;
 917
 918         GEM_BUG_ON(HAS_EXECLISTS(engine->i915));
 919
 920         if (engine->wa_ctx.vma && ce != engine->kernel_context) {
 921                 if (engine->wa_ctx.vma->private != ce &&
 922                     i915_mitigate_clear_residuals()) {
 923                         ret = clear_residuals(rq);
 924                         if (ret)
 925                                 return ret;
 926
 927                         residuals = &engine->wa_ctx.vma->private;
 928                 }
 929         }
 930
 931         ret = switch_mm(rq, vm_alias(ce->vm));
 932         if (ret)
 933                 return ret;
 934
 935         if (ce->state) {
 936                 u32 flags;
 937
 938                 GEM_BUG_ON(engine->id != RCS0);
 939
 940                 /* For resource streamer on HSW+ and power context elsewhere */
 941                 BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN);
 942                 BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN);
 943
 944                 flags = MI_SAVE_EXT_STATE_EN | MI_MM_SPACE_GTT;
 945                 if (test_bit(CONTEXT_VALID_BIT, &ce->flags))
 946                         flags |= MI_RESTORE_EXT_STATE_EN;
 947                 else
 948                         flags |= MI_RESTORE_INHIBIT;
 949
 950                 ret = mi_set_context(rq, ce, flags);
 951                 if (ret)
 952                         return ret;
 953         }
 954
 955         ret = remap_l3(rq);
 956         if (ret)
 957                 return ret;
 958
 959         /*
 960          * Now past the point of no return, this request _will_ be emitted.
 961          *
 962          * Or at least this preamble will be emitted, the request may be
 963          * interrupted prior to submitting the user payload. If so, we
 964          * still submit the "empty" request in order to preserve global
 965          * state tracking such as this, our tracking of the current
 966          * dirty context.
 967          */
 968         if (residuals) {
 969                 intel_context_put(*residuals);
 970                 *residuals = intel_context_get(ce);
 971         }
 972
 973         return 0;
 974 }
 975
 976 static int ring_request_alloc(struct i915_request *request)
 977 {
 978         int ret;
 979
 980         GEM_BUG_ON(!intel_context_is_pinned(request->context));
 981         GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb);
 982
 983         /*
 984          * Flush enough space to reduce the likelihood of waiting after
 985          * we start building the request - in which case we will just
 986          * have to repeat work.
 987          */
 988         request->reserved_space += LEGACY_REQUEST_SIZE;
 989
 990         /* Unconditionally invalidate GPU caches and TLBs. */
 991         ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
 992         if (ret)
 993                 return ret;
 994
 995         ret = switch_context(request);
 996         if (ret)
 997                 return ret;
 998
 999         request->reserved_space -= LEGACY_REQUEST_SIZE;
1000         return 0;
1001 }
1002
1003 static void gen6_bsd_submit_request(struct i915_request *request)
1004 {
1005         struct intel_uncore *uncore = request->engine->uncore;
1006
1007         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1008
1009        /* Every tail move must follow the sequence below */
1010
1011         /* Disable notification that the ring is IDLE. The GT
1012          * will then assume that it is busy and bring it out of rc6.
1013          */
1014         intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE),
1015                               _MASKED_BIT_ENABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
1016
1017         /* Clear the context id. Here be magic! */
1018         intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0);
1019
1020         /* Wait for the ring not to be idle, i.e. for it to wake up. */
1021         if (__intel_wait_for_register_fw(uncore,
1022                                          RING_PSMI_CTL(GEN6_BSD_RING_BASE),
1023                                          GEN6_BSD_SLEEP_INDICATOR,
1024                                          0,
1025                                          1000, 0, NULL))
1026                 drm_err(&uncore->i915->drm,
1027                         "timed out waiting for the BSD ring to wake up\n");
1028
1029         /* Now that the ring is fully powered up, update the tail */
1030         i9xx_submit_request(request);
1031
1032         /* Let the ring send IDLE messages to the GT again,
1033          * and so let it sleep to conserve power when idle.
1034          */
1035         intel_uncore_write_fw(uncore, RING_PSMI_CTL(GEN6_BSD_RING_BASE),
1036                               _MASKED_BIT_DISABLE(GEN6_PSMI_SLEEP_MSG_DISABLE));
1037
1038         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1039 }
1040
1041 static void i9xx_set_default_submission(struct intel_engine_cs *engine)
1042 {
1043         engine->submit_request = i9xx_submit_request;
1044 }
1045
1046 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
1047 {
1048         engine->submit_request = gen6_bsd_submit_request;
1049 }
1050
1051 static void ring_release(struct intel_engine_cs *engine)
1052 {
1053         struct drm_i915_private *dev_priv = engine->i915;
1054
1055         drm_WARN_ON(&dev_priv->drm, GRAPHICS_VER(dev_priv) > 2 &&
1056                     (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
1057
1058         intel_engine_cleanup_common(engine);
1059
1060         if (engine->wa_ctx.vma) {
1061                 intel_context_put(engine->wa_ctx.vma->private);
1062                 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);
1063         }
1064
1065         intel_ring_unpin(engine->legacy.ring);
1066         intel_ring_put(engine->legacy.ring);
1067
1068         intel_timeline_unpin(engine->legacy.timeline);
1069         intel_timeline_put(engine->legacy.timeline);
1070 }
1071
1072 static void irq_handler(struct intel_engine_cs *engine, u16 iir)
1073 {
1074         intel_engine_signal_breadcrumbs(engine);
1075 }
1076
1077 static void setup_irq(struct intel_engine_cs *engine)
1078 {
1079         struct drm_i915_private *i915 = engine->i915;
1080
1081         intel_engine_set_irq_handler(engine, irq_handler);
1082
1083         if (GRAPHICS_VER(i915) >= 6) {
1084                 engine->irq_enable = gen6_irq_enable;
1085                 engine->irq_disable = gen6_irq_disable;
1086         } else if (GRAPHICS_VER(i915) >= 5) {
1087                 engine->irq_enable = gen5_irq_enable;
1088                 engine->irq_disable = gen5_irq_disable;
1089         } else if (GRAPHICS_VER(i915) >= 3) {
1090                 engine->irq_enable = gen3_irq_enable;
1091                 engine->irq_disable = gen3_irq_disable;
1092         } else {
1093                 engine->irq_enable = gen2_irq_enable;
1094                 engine->irq_disable = gen2_irq_disable;
1095         }
1096 }
1097
1098 static void add_to_engine(struct i915_request *rq)
1099 {
1100         lockdep_assert_held(&rq->engine->sched_engine->lock);
1101         list_move_tail(&rq->sched.link, &rq->engine->sched_engine->requests);
1102 }
1103
1104 static void remove_from_engine(struct i915_request *rq)
1105 {
1106         spin_lock_irq(&rq->engine->sched_engine->lock);
1107         list_del_init(&rq->sched.link);
1108
1109         /* Prevent further __await_execution() registering a cb, then flush */
1110         set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
1111
1112         spin_unlock_irq(&rq->engine->sched_engine->lock);
1113
1114         i915_request_notify_execute_cb_imm(rq);
1115 }
1116
1117 static void setup_common(struct intel_engine_cs *engine)
1118 {
1119         struct drm_i915_private *i915 = engine->i915;
1120
1121         /* gen8+ are only supported with execlists */
1122         GEM_BUG_ON(GRAPHICS_VER(i915) >= 8);
1123
1124         setup_irq(engine);
1125
1126         engine->resume = xcs_resume;
1127         engine->sanitize = xcs_sanitize;
1128
1129         engine->reset.prepare = reset_prepare;
1130         engine->reset.rewind = reset_rewind;
1131         engine->reset.cancel = reset_cancel;
1132         engine->reset.finish = reset_finish;
1133
1134         engine->add_active_request = add_to_engine;
1135         engine->remove_active_request = remove_from_engine;
1136
1137         engine->cops = &ring_context_ops;
1138         engine->request_alloc = ring_request_alloc;
1139
1140         /*
1141          * Using a global execution timeline; the previous final breadcrumb is
1142          * equivalent to our next initial bread so we can elide
1143          * engine->emit_init_breadcrumb().
1144          */
1145         engine->emit_fini_breadcrumb = gen3_emit_breadcrumb;
1146         if (GRAPHICS_VER(i915) == 5)
1147                 engine->emit_fini_breadcrumb = gen5_emit_breadcrumb;
1148
1149         engine->set_default_submission = i9xx_set_default_submission;
1150
1151         if (GRAPHICS_VER(i915) >= 6)
1152                 engine->emit_bb_start = gen6_emit_bb_start;
1153         else if (GRAPHICS_VER(i915) >= 4)
1154                 engine->emit_bb_start = gen4_emit_bb_start;
1155         else if (IS_I830(i915) || IS_I845G(i915))
1156                 engine->emit_bb_start = i830_emit_bb_start;
1157         else
1158                 engine->emit_bb_start = gen3_emit_bb_start;
1159 }
1160
1161 static void setup_rcs(struct intel_engine_cs *engine)
1162 {
1163         struct drm_i915_private *i915 = engine->i915;
1164
1165         if (HAS_L3_DPF(i915))
1166                 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
1167
1168         engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
1169
1170         if (GRAPHICS_VER(i915) >= 7) {
1171                 engine->emit_flush = gen7_emit_flush_rcs;
1172                 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_rcs;
1173         } else if (GRAPHICS_VER(i915) == 6) {
1174                 engine->emit_flush = gen6_emit_flush_rcs;
1175                 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_rcs;
1176         } else if (GRAPHICS_VER(i915) == 5) {
1177                 engine->emit_flush = gen4_emit_flush_rcs;
1178         } else {
1179                 if (GRAPHICS_VER(i915) < 4)
1180                         engine->emit_flush = gen2_emit_flush;
1181                 else
1182                         engine->emit_flush = gen4_emit_flush_rcs;
1183                 engine->irq_enable_mask = I915_USER_INTERRUPT;
1184         }
1185
1186         if (IS_HASWELL(i915))
1187                 engine->emit_bb_start = hsw_emit_bb_start;
1188 }
1189
1190 static void setup_vcs(struct intel_engine_cs *engine)
1191 {
1192         struct drm_i915_private *i915 = engine->i915;
1193
1194         if (GRAPHICS_VER(i915) >= 6) {
1195                 /* gen6 bsd needs a special wa for tail updates */
1196                 if (GRAPHICS_VER(i915) == 6)
1197                         engine->set_default_submission = gen6_bsd_set_default_submission;
1198                 engine->emit_flush = gen6_emit_flush_vcs;
1199                 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1200
1201                 if (GRAPHICS_VER(i915) == 6)
1202                         engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs;
1203                 else
1204                         engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
1205         } else {
1206                 engine->emit_flush = gen4_emit_flush_vcs;
1207                 if (GRAPHICS_VER(i915) == 5)
1208                         engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
1209                 else
1210                         engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1211         }
1212 }
1213
1214 static void setup_bcs(struct intel_engine_cs *engine)
1215 {
1216         struct drm_i915_private *i915 = engine->i915;
1217
1218         engine->emit_flush = gen6_emit_flush_xcs;
1219         engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
1220
1221         if (GRAPHICS_VER(i915) == 6)
1222                 engine->emit_fini_breadcrumb = gen6_emit_breadcrumb_xcs;
1223         else
1224                 engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
1225 }
1226
1227 static void setup_vecs(struct intel_engine_cs *engine)
1228 {
1229         struct drm_i915_private *i915 = engine->i915;
1230
1231         GEM_BUG_ON(GRAPHICS_VER(i915) < 7);
1232
1233         engine->emit_flush = gen6_emit_flush_xcs;
1234         engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
1235         engine->irq_enable = hsw_irq_enable_vecs;
1236         engine->irq_disable = hsw_irq_disable_vecs;
1237
1238         engine->emit_fini_breadcrumb = gen7_emit_breadcrumb_xcs;
1239 }
1240
1241 static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine,
1242                                     struct i915_vma * const vma)
1243 {
1244         return gen7_setup_clear_gpr_bb(engine, vma);
1245 }
1246
1247 static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine,
1248                                    struct i915_gem_ww_ctx *ww,
1249                                    struct i915_vma *vma)
1250 {
1251         int err;
1252
1253         err = i915_vma_pin_ww(vma, ww, 0, 0, PIN_USER | PIN_HIGH);
1254         if (err)
1255                 return err;
1256
1257         err = i915_vma_sync(vma);
1258         if (err)
1259                 goto err_unpin;
1260
1261         err = gen7_ctx_switch_bb_setup(engine, vma);
1262         if (err)
1263                 goto err_unpin;
1264
1265         engine->wa_ctx.vma = vma;
1266         return 0;
1267
1268 err_unpin:
1269         i915_vma_unpin(vma);
1270         return err;
1271 }
1272
1273 static struct i915_vma *gen7_ctx_vma(struct intel_engine_cs *engine)
1274 {
1275         struct drm_i915_gem_object *obj;
1276         struct i915_vma *vma;
1277         int size, err;
1278
1279         if (GRAPHICS_VER(engine->i915) != 7 || engine->class != RENDER_CLASS)
1280                 return NULL;
1281
1282         err = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);
1283         if (err < 0)
1284                 return ERR_PTR(err);
1285         if (!err)
1286                 return NULL;
1287
1288         size = ALIGN(err, PAGE_SIZE);
1289
1290         obj = i915_gem_object_create_internal(engine->i915, size);
1291         if (IS_ERR(obj))
1292                 return ERR_CAST(obj);
1293
1294         vma = i915_vma_instance(obj, engine->gt->vm, NULL);
1295         if (IS_ERR(vma)) {
1296                 i915_gem_object_put(obj);
1297                 return ERR_CAST(vma);
1298         }
1299
1300         vma->private = intel_context_create(engine); /* dummy residuals */
1301         if (IS_ERR(vma->private)) {
1302                 err = PTR_ERR(vma->private);
1303                 vma->private = NULL;
1304                 i915_gem_object_put(obj);
1305                 return ERR_PTR(err);
1306         }
1307
1308         return vma;
1309 }
1310
1311 int intel_ring_submission_setup(struct intel_engine_cs *engine)
1312 {
1313         struct i915_gem_ww_ctx ww;
1314         struct intel_timeline *timeline;
1315         struct intel_ring *ring;
1316         struct i915_vma *gen7_wa_vma;
1317         int err;
1318
1319         setup_common(engine);
1320
1321         switch (engine->class) {
1322         case RENDER_CLASS:
1323                 setup_rcs(engine);
1324                 break;
1325         case VIDEO_DECODE_CLASS:
1326                 setup_vcs(engine);
1327                 break;
1328         case COPY_ENGINE_CLASS:
1329                 setup_bcs(engine);
1330                 break;
1331         case VIDEO_ENHANCEMENT_CLASS:
1332                 setup_vecs(engine);
1333                 break;
1334         default:
1335                 MISSING_CASE(engine->class);
1336                 return -ENODEV;
1337         }
1338
1339         timeline = intel_timeline_create_from_engine(engine,
1340                                                      I915_GEM_HWS_SEQNO_ADDR);
1341         if (IS_ERR(timeline)) {
1342                 err = PTR_ERR(timeline);
1343                 goto err;
1344         }
1345         GEM_BUG_ON(timeline->has_initial_breadcrumb);
1346
1347         ring = intel_engine_create_ring(engine, SZ_16K);
1348         if (IS_ERR(ring)) {
1349                 err = PTR_ERR(ring);
1350                 goto err_timeline;
1351         }
1352
1353         GEM_BUG_ON(engine->legacy.ring);
1354         engine->legacy.ring = ring;
1355         engine->legacy.timeline = timeline;
1356
1357         gen7_wa_vma = gen7_ctx_vma(engine);
1358         if (IS_ERR(gen7_wa_vma)) {
1359                 err = PTR_ERR(gen7_wa_vma);
1360                 goto err_ring;
1361         }
1362
1363         i915_gem_ww_ctx_init(&ww, false);
1364
1365 retry:
1366         err = i915_gem_object_lock(timeline->hwsp_ggtt->obj, &ww);
1367         if (!err && gen7_wa_vma)
1368                 err = i915_gem_object_lock(gen7_wa_vma->obj, &ww);
1369         if (!err)
1370                 err = i915_gem_object_lock(engine->legacy.ring->vma->obj, &ww);
1371         if (!err)
1372                 err = intel_timeline_pin(timeline, &ww);
1373         if (!err) {
1374                 err = intel_ring_pin(ring, &ww);
1375                 if (err)
1376                         intel_timeline_unpin(timeline);
1377         }
1378         if (err)
1379                 goto out;
1380
1381         GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
1382
1383         if (gen7_wa_vma) {
1384                 err = gen7_ctx_switch_bb_init(engine, &ww, gen7_wa_vma);
1385                 if (err) {
1386                         intel_ring_unpin(ring);
1387                         intel_timeline_unpin(timeline);
1388                 }
1389         }
1390
1391 out:
1392         if (err == -EDEADLK) {
1393                 err = i915_gem_ww_ctx_backoff(&ww);
1394                 if (!err)
1395                         goto retry;
1396         }
1397         i915_gem_ww_ctx_fini(&ww);
1398         if (err)
1399                 goto err_gen7_put;
1400
1401         /* Finally, take ownership and responsibility for cleanup! */
1402         engine->release = ring_release;
1403
1404         return 0;
1405
1406 err_gen7_put:
1407         if (gen7_wa_vma) {
1408                 intel_context_put(gen7_wa_vma->private);
1409                 i915_gem_object_put(gen7_wa_vma->obj);
1410         }
1411 err_ring:
1412         intel_ring_put(ring);
1413 err_timeline:
1414         intel_timeline_put(timeline);
1415 err:
1416         intel_engine_cleanup_common(engine);
1417         return err;
1418 }
1419
1420 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1421 #include "selftest_ring_submission.c"
1422 #endif