drivers/gpu/drm/i915/gvt/scheduler.c

   1 /*
   2  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Zhi Wang <zhi.a.wang@intel.com>
  25  *
  26  * Contributors:
  27  *    Ping Gao <ping.a.gao@intel.com>
  28  *    Tina Zhang <tina.zhang@intel.com>
  29  *    Chanbin Du <changbin.du@intel.com>
  30  *    Min He <min.he@intel.com>
  31  *    Bing Niu <bing.niu@intel.com>
  32  *    Zhenyu Wang <zhenyuw@linux.intel.com>
  33  *
  34  */
  35
  36 #include <linux/kthread.h>
  37
  38 #include "gem/i915_gem_pm.h"
  39 #include "gt/intel_context.h"
  40 #include "gt/intel_ring.h"
  41
  42 #include "i915_drv.h"
  43 #include "i915_gem_gtt.h"
  44 #include "gvt.h"
  45
  46 #define RING_CTX_OFF(x) \
  47         offsetof(struct execlist_ring_context, x)
  48
  49 static void set_context_pdp_root_pointer(
  50                 struct execlist_ring_context *ring_context,
  51                 u32 pdp[8])
  52 {
  53         int i;
  54
  55         for (i = 0; i < 8; i++)
  56                 ring_context->pdps[i].val = pdp[7 - i];
  57 }
  58
  59 static void update_shadow_pdps(struct intel_vgpu_workload *workload)
  60 {
  61         struct execlist_ring_context *shadow_ring_context;
  62         struct intel_context *ctx = workload->req->context;
  63
  64         if (WARN_ON(!workload->shadow_mm))
  65                 return;
  66
  67         if (WARN_ON(!atomic_read(&workload->shadow_mm->pincount)))
  68                 return;
  69
  70         shadow_ring_context = (struct execlist_ring_context *)ctx->lrc_reg_state;
  71         set_context_pdp_root_pointer(shadow_ring_context,
  72                         (void *)workload->shadow_mm->ppgtt_mm.shadow_pdps);
  73 }
  74
  75 /*
  76  * when populating shadow ctx from guest, we should not overrride oa related
  77  * registers, so that they will not be overlapped by guest oa configs. Thus
  78  * made it possible to capture oa data from host for both host and guests.
  79  */
  80 static void sr_oa_regs(struct intel_vgpu_workload *workload,
  81                 u32 *reg_state, bool save)
  82 {
  83         struct drm_i915_private *dev_priv = workload->vgpu->gvt->gt->i915;
  84         u32 ctx_oactxctrl = dev_priv->perf.ctx_oactxctrl_offset;
  85         u32 ctx_flexeu0 = dev_priv->perf.ctx_flexeu0_offset;
  86         int i = 0;
  87         u32 flex_mmio[] = {
  88                 i915_mmio_reg_offset(EU_PERF_CNTL0),
  89                 i915_mmio_reg_offset(EU_PERF_CNTL1),
  90                 i915_mmio_reg_offset(EU_PERF_CNTL2),
  91                 i915_mmio_reg_offset(EU_PERF_CNTL3),
  92                 i915_mmio_reg_offset(EU_PERF_CNTL4),
  93                 i915_mmio_reg_offset(EU_PERF_CNTL5),
  94                 i915_mmio_reg_offset(EU_PERF_CNTL6),
  95         };
  96
  97         if (workload->engine->id != RCS0)
  98                 return;
  99
 100         if (save) {
 101                 workload->oactxctrl = reg_state[ctx_oactxctrl + 1];
 102
 103                 for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
 104                         u32 state_offset = ctx_flexeu0 + i * 2;
 105
 106                         workload->flex_mmio[i] = reg_state[state_offset + 1];
 107                 }
 108         } else {
 109                 reg_state[ctx_oactxctrl] =
 110                         i915_mmio_reg_offset(GEN8_OACTXCONTROL);
 111                 reg_state[ctx_oactxctrl + 1] = workload->oactxctrl;
 112
 113                 for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
 114                         u32 state_offset = ctx_flexeu0 + i * 2;
 115                         u32 mmio = flex_mmio[i];
 116
 117                         reg_state[state_offset] = mmio;
 118                         reg_state[state_offset + 1] = workload->flex_mmio[i];
 119                 }
 120         }
 121 }
 122
 123 static int populate_shadow_context(struct intel_vgpu_workload *workload)
 124 {
 125         struct intel_vgpu *vgpu = workload->vgpu;
 126         struct intel_gvt *gvt = vgpu->gvt;
 127         struct intel_context *ctx = workload->req->context;
 128         struct execlist_ring_context *shadow_ring_context;
 129         void *dst;
 130         void *context_base;
 131         unsigned long context_gpa, context_page_num;
 132         unsigned long gpa_base; /* first gpa of consecutive GPAs */
 133         unsigned long gpa_size; /* size of consecutive GPAs */
 134         struct intel_vgpu_submission *s = &vgpu->submission;
 135         int i;
 136         bool skip = false;
 137         int ring_id = workload->engine->id;
 138
 139         GEM_BUG_ON(!intel_context_is_pinned(ctx));
 140
 141         context_base = (void *) ctx->lrc_reg_state -
 142                                 (LRC_STATE_PN << I915_GTT_PAGE_SHIFT);
 143
 144         shadow_ring_context = (void *) ctx->lrc_reg_state;
 145
 146         sr_oa_regs(workload, (u32 *)shadow_ring_context, true);
 147 #define COPY_REG(name) \
 148         intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
 149                 + RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
 150 #define COPY_REG_MASKED(name) {\
 151                 intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
 152                                               + RING_CTX_OFF(name.val),\
 153                                               &shadow_ring_context->name.val, 4);\
 154                 shadow_ring_context->name.val |= 0xffff << 16;\
 155         }
 156
 157         COPY_REG_MASKED(ctx_ctrl);
 158         COPY_REG(ctx_timestamp);
 159
 160         if (workload->engine->id == RCS0) {
 161                 COPY_REG(bb_per_ctx_ptr);
 162                 COPY_REG(rcs_indirect_ctx);
 163                 COPY_REG(rcs_indirect_ctx_offset);
 164         }
 165 #undef COPY_REG
 166 #undef COPY_REG_MASKED
 167
 168         intel_gvt_hypervisor_read_gpa(vgpu,
 169                         workload->ring_context_gpa +
 170                         sizeof(*shadow_ring_context),
 171                         (void *)shadow_ring_context +
 172                         sizeof(*shadow_ring_context),
 173                         I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
 174
 175         sr_oa_regs(workload, (u32 *)shadow_ring_context, false);
 176
 177         gvt_dbg_sched("ring %s workload lrca %x, ctx_id %x, ctx gpa %llx",
 178                         workload->engine->name, workload->ctx_desc.lrca,
 179                         workload->ctx_desc.context_id,
 180                         workload->ring_context_gpa);
 181
 182         /* only need to ensure this context is not pinned/unpinned during the
 183          * period from last submission to this this submission.
 184          * Upon reaching this function, the currently submitted context is not
 185          * supposed to get unpinned. If a misbehaving guest driver ever does
 186          * this, it would corrupt itself.
 187          */
 188         if (s->last_ctx[ring_id].valid &&
 189                         (s->last_ctx[ring_id].lrca ==
 190                                 workload->ctx_desc.lrca) &&
 191                         (s->last_ctx[ring_id].ring_context_gpa ==
 192                                 workload->ring_context_gpa))
 193                 skip = true;
 194
 195         s->last_ctx[ring_id].lrca = workload->ctx_desc.lrca;
 196         s->last_ctx[ring_id].ring_context_gpa = workload->ring_context_gpa;
 197
 198         if (IS_RESTORE_INHIBIT(shadow_ring_context->ctx_ctrl.val) || skip)
 199                 return 0;
 200
 201         s->last_ctx[ring_id].valid = false;
 202         context_page_num = workload->engine->context_size;
 203         context_page_num = context_page_num >> PAGE_SHIFT;
 204
 205         if (IS_BROADWELL(gvt->gt->i915) && workload->engine->id == RCS0)
 206                 context_page_num = 19;
 207
 208         /* find consecutive GPAs from gma until the first inconsecutive GPA.
 209          * read from the continuous GPAs into dst virtual address
 210          */
 211         gpa_size = 0;
 212         for (i = 2; i < context_page_num; i++) {
 213                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
 214                                 (u32)((workload->ctx_desc.lrca + i) <<
 215                                 I915_GTT_PAGE_SHIFT));
 216                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
 217                         gvt_vgpu_err("Invalid guest context descriptor\n");
 218                         return -EFAULT;
 219                 }
 220
 221                 if (gpa_size == 0) {
 222                         gpa_base = context_gpa;
 223                         dst = context_base + (i << I915_GTT_PAGE_SHIFT);
 224                 } else if (context_gpa != gpa_base + gpa_size)
 225                         goto read;
 226
 227                 gpa_size += I915_GTT_PAGE_SIZE;
 228
 229                 if (i == context_page_num - 1)
 230                         goto read;
 231
 232                 continue;
 233
 234 read:
 235                 intel_gvt_hypervisor_read_gpa(vgpu, gpa_base, dst, gpa_size);
 236                 gpa_base = context_gpa;
 237                 gpa_size = I915_GTT_PAGE_SIZE;
 238                 dst = context_base + (i << I915_GTT_PAGE_SHIFT);
 239         }
 240         s->last_ctx[ring_id].valid = true;
 241         return 0;
 242 }
 243
 244 static inline bool is_gvt_request(struct i915_request *rq)
 245 {
 246         return intel_context_force_single_submission(rq->context);
 247 }
 248
 249 static void save_ring_hw_state(struct intel_vgpu *vgpu,
 250                                const struct intel_engine_cs *engine)
 251 {
 252         struct intel_uncore *uncore = engine->uncore;
 253         i915_reg_t reg;
 254
 255         reg = RING_INSTDONE(engine->mmio_base);
 256         vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) =
 257                 intel_uncore_read(uncore, reg);
 258
 259         reg = RING_ACTHD(engine->mmio_base);
 260         vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) =
 261                 intel_uncore_read(uncore, reg);
 262
 263         reg = RING_ACTHD_UDW(engine->mmio_base);
 264         vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) =
 265                 intel_uncore_read(uncore, reg);
 266 }
 267
 268 static int shadow_context_status_change(struct notifier_block *nb,
 269                 unsigned long action, void *data)
 270 {
 271         struct i915_request *rq = data;
 272         struct intel_gvt *gvt = container_of(nb, struct intel_gvt,
 273                                 shadow_ctx_notifier_block[rq->engine->id]);
 274         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 275         enum intel_engine_id ring_id = rq->engine->id;
 276         struct intel_vgpu_workload *workload;
 277         unsigned long flags;
 278
 279         if (!is_gvt_request(rq)) {
 280                 spin_lock_irqsave(&scheduler->mmio_context_lock, flags);
 281                 if (action == INTEL_CONTEXT_SCHEDULE_IN &&
 282                     scheduler->engine_owner[ring_id]) {
 283                         /* Switch ring from vGPU to host. */
 284                         intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
 285                                               NULL, rq->engine);
 286                         scheduler->engine_owner[ring_id] = NULL;
 287                 }
 288                 spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags);
 289
 290                 return NOTIFY_OK;
 291         }
 292
 293         workload = scheduler->current_workload[ring_id];
 294         if (unlikely(!workload))
 295                 return NOTIFY_OK;
 296
 297         switch (action) {
 298         case INTEL_CONTEXT_SCHEDULE_IN:
 299                 spin_lock_irqsave(&scheduler->mmio_context_lock, flags);
 300                 if (workload->vgpu != scheduler->engine_owner[ring_id]) {
 301                         /* Switch ring from host to vGPU or vGPU to vGPU. */
 302                         intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
 303                                               workload->vgpu, rq->engine);
 304                         scheduler->engine_owner[ring_id] = workload->vgpu;
 305                 } else
 306                         gvt_dbg_sched("skip ring %d mmio switch for vgpu%d\n",
 307                                       ring_id, workload->vgpu->id);
 308                 spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags);
 309                 atomic_set(&workload->shadow_ctx_active, 1);
 310                 break;
 311         case INTEL_CONTEXT_SCHEDULE_OUT:
 312                 save_ring_hw_state(workload->vgpu, rq->engine);
 313                 atomic_set(&workload->shadow_ctx_active, 0);
 314                 break;
 315         case INTEL_CONTEXT_SCHEDULE_PREEMPTED:
 316                 save_ring_hw_state(workload->vgpu, rq->engine);
 317                 break;
 318         default:
 319                 WARN_ON(1);
 320                 return NOTIFY_OK;
 321         }
 322         wake_up(&workload->shadow_ctx_status_wq);
 323         return NOTIFY_OK;
 324 }
 325
 326 static void
 327 shadow_context_descriptor_update(struct intel_context *ce,
 328                                  struct intel_vgpu_workload *workload)
 329 {
 330         u64 desc = ce->lrc.desc;
 331
 332         /*
 333          * Update bits 0-11 of the context descriptor which includes flags
 334          * like GEN8_CTX_* cached in desc_template
 335          */
 336         desc &= ~(0x3ull << GEN8_CTX_ADDRESSING_MODE_SHIFT);
 337         desc |= (u64)workload->ctx_desc.addressing_mode <<
 338                 GEN8_CTX_ADDRESSING_MODE_SHIFT;
 339
 340         ce->lrc.desc = desc;
 341 }
 342
 343 static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload)
 344 {
 345         struct intel_vgpu *vgpu = workload->vgpu;
 346         struct i915_request *req = workload->req;
 347         void *shadow_ring_buffer_va;
 348         u32 *cs;
 349         int err;
 350
 351         if (IS_GEN(req->i915, 9) && is_inhibit_context(req->context))
 352                 intel_vgpu_restore_inhibit_context(vgpu, req);
 353
 354         /*
 355          * To track whether a request has started on HW, we can emit a
 356          * breadcrumb at the beginning of the request and check its
 357          * timeline's HWSP to see if the breadcrumb has advanced past the
 358          * start of this request. Actually, the request must have the
 359          * init_breadcrumb if its timeline set has_init_bread_crumb, or the
 360          * scheduler might get a wrong state of it during reset. Since the
 361          * requests from gvt always set the has_init_breadcrumb flag, here
 362          * need to do the emit_init_breadcrumb for all the requests.
 363          */
 364         if (req->engine->emit_init_breadcrumb) {
 365                 err = req->engine->emit_init_breadcrumb(req);
 366                 if (err) {
 367                         gvt_vgpu_err("fail to emit init breadcrumb\n");
 368                         return err;
 369                 }
 370         }
 371
 372         /* allocate shadow ring buffer */
 373         cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32));
 374         if (IS_ERR(cs)) {
 375                 gvt_vgpu_err("fail to alloc size =%ld shadow  ring buffer\n",
 376                         workload->rb_len);
 377                 return PTR_ERR(cs);
 378         }
 379
 380         shadow_ring_buffer_va = workload->shadow_ring_buffer_va;
 381
 382         /* get shadow ring buffer va */
 383         workload->shadow_ring_buffer_va = cs;
 384
 385         memcpy(cs, shadow_ring_buffer_va,
 386                         workload->rb_len);
 387
 388         cs += workload->rb_len / sizeof(u32);
 389         intel_ring_advance(workload->req, cs);
 390
 391         return 0;
 392 }
 393
 394 static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 395 {
 396         if (!wa_ctx->indirect_ctx.obj)
 397                 return;
 398
 399         i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj);
 400         i915_gem_object_put(wa_ctx->indirect_ctx.obj);
 401
 402         wa_ctx->indirect_ctx.obj = NULL;
 403         wa_ctx->indirect_ctx.shadow_va = NULL;
 404 }
 405
 406 static void set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload,
 407                                           struct intel_context *ce)
 408 {
 409         struct intel_vgpu_mm *mm = workload->shadow_mm;
 410         struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(ce->vm);
 411         int i = 0;
 412
 413         if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
 414                 px_dma(ppgtt->pd) = mm->ppgtt_mm.shadow_pdps[0];
 415         } else {
 416                 for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) {
 417                         struct i915_page_directory * const pd =
 418                                 i915_pd_entry(ppgtt->pd, i);
 419                         /* skip now as current i915 ppgtt alloc won't allocate
 420                            top level pdp for non 4-level table, won't impact
 421                            shadow ppgtt. */
 422                         if (!pd)
 423                                 break;
 424                         px_dma(pd) = mm->ppgtt_mm.shadow_pdps[i];
 425                 }
 426         }
 427 }
 428
 429 static int
 430 intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload)
 431 {
 432         struct intel_vgpu *vgpu = workload->vgpu;
 433         struct intel_vgpu_submission *s = &vgpu->submission;
 434         struct i915_request *rq;
 435
 436         if (workload->req)
 437                 return 0;
 438
 439         rq = i915_request_create(s->shadow[workload->engine->id]);
 440         if (IS_ERR(rq)) {
 441                 gvt_vgpu_err("fail to allocate gem request\n");
 442                 return PTR_ERR(rq);
 443         }
 444
 445         workload->req = i915_request_get(rq);
 446         return 0;
 447 }
 448
 449 /**
 450  * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and
 451  * shadow it as well, include ringbuffer,wa_ctx and ctx.
 452  * @workload: an abstract entity for each execlist submission.
 453  *
 454  * This function is called before the workload submitting to i915, to make
 455  * sure the content of the workload is valid.
 456  */
 457 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 458 {
 459         struct intel_vgpu *vgpu = workload->vgpu;
 460         struct intel_vgpu_submission *s = &vgpu->submission;
 461         int ret;
 462
 463         lockdep_assert_held(&vgpu->vgpu_lock);
 464
 465         if (workload->shadow)
 466                 return 0;
 467
 468         if (!test_and_set_bit(workload->engine->id, s->shadow_ctx_desc_updated))
 469                 shadow_context_descriptor_update(s->shadow[workload->engine->id],
 470                                                  workload);
 471
 472         ret = intel_gvt_scan_and_shadow_ringbuffer(workload);
 473         if (ret)
 474                 return ret;
 475
 476         if (workload->engine->id == RCS0 &&
 477             workload->wa_ctx.indirect_ctx.size) {
 478                 ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
 479                 if (ret)
 480                         goto err_shadow;
 481         }
 482
 483         workload->shadow = true;
 484         return 0;
 485
 486 err_shadow:
 487         release_shadow_wa_ctx(&workload->wa_ctx);
 488         return ret;
 489 }
 490
 491 static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload);
 492
 493 static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
 494 {
 495         struct intel_gvt *gvt = workload->vgpu->gvt;
 496         const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
 497         struct intel_vgpu_shadow_bb *bb;
 498         int ret;
 499
 500         list_for_each_entry(bb, &workload->shadow_bb, list) {
 501                 /* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va
 502                  * is only updated into ring_scan_buffer, not real ring address
 503                  * allocated in later copy_workload_to_ring_buffer. pls be noted
 504                  * shadow_ring_buffer_va is now pointed to real ring buffer va
 505                  * in copy_workload_to_ring_buffer.
 506                  */
 507
 508                 if (bb->bb_offset)
 509                         bb->bb_start_cmd_va = workload->shadow_ring_buffer_va
 510                                 + bb->bb_offset;
 511
 512                 if (bb->ppgtt) {
 513                         /* for non-priv bb, scan&shadow is only for
 514                          * debugging purpose, so the content of shadow bb
 515                          * is the same as original bb. Therefore,
 516                          * here, rather than switch to shadow bb's gma
 517                          * address, we directly use original batch buffer's
 518                          * gma address, and send original bb to hardware
 519                          * directly
 520                          */
 521                         if (bb->clflush & CLFLUSH_AFTER) {
 522                                 drm_clflush_virt_range(bb->va,
 523                                                 bb->obj->base.size);
 524                                 bb->clflush &= ~CLFLUSH_AFTER;
 525                         }
 526                         i915_gem_object_finish_access(bb->obj);
 527                         bb->accessing = false;
 528
 529                 } else {
 530                         bb->vma = i915_gem_object_ggtt_pin(bb->obj,
 531                                         NULL, 0, 0, 0);
 532                         if (IS_ERR(bb->vma)) {
 533                                 ret = PTR_ERR(bb->vma);
 534                                 goto err;
 535                         }
 536
 537                         /* relocate shadow batch buffer */
 538                         bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
 539                         if (gmadr_bytes == 8)
 540                                 bb->bb_start_cmd_va[2] = 0;
 541
 542                         /* No one is going to touch shadow bb from now on. */
 543                         if (bb->clflush & CLFLUSH_AFTER) {
 544                                 drm_clflush_virt_range(bb->va,
 545                                                 bb->obj->base.size);
 546                                 bb->clflush &= ~CLFLUSH_AFTER;
 547                         }
 548
 549                         ret = i915_gem_object_set_to_gtt_domain(bb->obj,
 550                                                                 false);
 551                         if (ret)
 552                                 goto err;
 553
 554                         ret = i915_vma_move_to_active(bb->vma,
 555                                                       workload->req,
 556                                                       0);
 557                         if (ret)
 558                                 goto err;
 559
 560                         i915_gem_object_finish_access(bb->obj);
 561                         bb->accessing = false;
 562                 }
 563         }
 564         return 0;
 565 err:
 566         release_shadow_batch_buffer(workload);
 567         return ret;
 568 }
 569
 570 static void update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 571 {
 572         struct intel_vgpu_workload *workload =
 573                 container_of(wa_ctx, struct intel_vgpu_workload, wa_ctx);
 574         struct i915_request *rq = workload->req;
 575         struct execlist_ring_context *shadow_ring_context =
 576                 (struct execlist_ring_context *)rq->context->lrc_reg_state;
 577
 578         shadow_ring_context->bb_per_ctx_ptr.val =
 579                 (shadow_ring_context->bb_per_ctx_ptr.val &
 580                 (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
 581         shadow_ring_context->rcs_indirect_ctx.val =
 582                 (shadow_ring_context->rcs_indirect_ctx.val &
 583                 (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
 584 }
 585
 586 static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 587 {
 588         struct i915_vma *vma;
 589         unsigned char *per_ctx_va =
 590                 (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
 591                 wa_ctx->indirect_ctx.size;
 592
 593         if (wa_ctx->indirect_ctx.size == 0)
 594                 return 0;
 595
 596         vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
 597                                        0, CACHELINE_BYTES, 0);
 598         if (IS_ERR(vma))
 599                 return PTR_ERR(vma);
 600
 601         /* FIXME: we are not tracking our pinned VMA leaving it
 602          * up to the core to fix up the stray pin_count upon
 603          * free.
 604          */
 605
 606         wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);
 607
 608         wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
 609         memset(per_ctx_va, 0, CACHELINE_BYTES);
 610
 611         update_wa_ctx_2_shadow_ctx(wa_ctx);
 612         return 0;
 613 }
 614
 615 static void update_vreg_in_ctx(struct intel_vgpu_workload *workload)
 616 {
 617         vgpu_vreg_t(workload->vgpu, RING_START(workload->engine->mmio_base)) =
 618                 workload->rb_start;
 619 }
 620
 621 static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
 622 {
 623         struct intel_vgpu_shadow_bb *bb, *pos;
 624
 625         if (list_empty(&workload->shadow_bb))
 626                 return;
 627
 628         bb = list_first_entry(&workload->shadow_bb,
 629                         struct intel_vgpu_shadow_bb, list);
 630
 631         list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) {
 632                 if (bb->obj) {
 633                         if (bb->accessing)
 634                                 i915_gem_object_finish_access(bb->obj);
 635
 636                         if (bb->va && !IS_ERR(bb->va))
 637                                 i915_gem_object_unpin_map(bb->obj);
 638
 639                         if (bb->vma && !IS_ERR(bb->vma))
 640                                 i915_vma_unpin(bb->vma);
 641
 642                         i915_gem_object_put(bb->obj);
 643                 }
 644                 list_del(&bb->list);
 645                 kfree(bb);
 646         }
 647 }
 648
 649 static int
 650 intel_vgpu_shadow_mm_pin(struct intel_vgpu_workload *workload)
 651 {
 652         struct intel_vgpu *vgpu = workload->vgpu;
 653         struct intel_vgpu_mm *m;
 654         int ret = 0;
 655
 656         ret = intel_vgpu_pin_mm(workload->shadow_mm);
 657         if (ret) {
 658                 gvt_vgpu_err("fail to vgpu pin mm\n");
 659                 return ret;
 660         }
 661
 662         if (workload->shadow_mm->type != INTEL_GVT_MM_PPGTT ||
 663             !workload->shadow_mm->ppgtt_mm.shadowed) {
 664                 gvt_vgpu_err("workload shadow ppgtt isn't ready\n");
 665                 return -EINVAL;
 666         }
 667
 668         if (!list_empty(&workload->lri_shadow_mm)) {
 669                 list_for_each_entry(m, &workload->lri_shadow_mm,
 670                                     ppgtt_mm.link) {
 671                         ret = intel_vgpu_pin_mm(m);
 672                         if (ret) {
 673                                 list_for_each_entry_from_reverse(m,
 674                                                                  &workload->lri_shadow_mm,
 675                                                                  ppgtt_mm.link)
 676                                         intel_vgpu_unpin_mm(m);
 677                                 gvt_vgpu_err("LRI shadow ppgtt fail to pin\n");
 678                                 break;
 679                         }
 680                 }
 681         }
 682
 683         if (ret)
 684                 intel_vgpu_unpin_mm(workload->shadow_mm);
 685
 686         return ret;
 687 }
 688
 689 static void
 690 intel_vgpu_shadow_mm_unpin(struct intel_vgpu_workload *workload)
 691 {
 692         struct intel_vgpu_mm *m;
 693
 694         if (!list_empty(&workload->lri_shadow_mm)) {
 695                 list_for_each_entry(m, &workload->lri_shadow_mm,
 696                                     ppgtt_mm.link)
 697                         intel_vgpu_unpin_mm(m);
 698         }
 699         intel_vgpu_unpin_mm(workload->shadow_mm);
 700 }
 701
 702 static int prepare_workload(struct intel_vgpu_workload *workload)
 703 {
 704         struct intel_vgpu *vgpu = workload->vgpu;
 705         struct intel_vgpu_submission *s = &vgpu->submission;
 706         int ret = 0;
 707
 708         ret = intel_vgpu_shadow_mm_pin(workload);
 709         if (ret) {
 710                 gvt_vgpu_err("fail to pin shadow mm\n");
 711                 return ret;
 712         }
 713
 714         update_shadow_pdps(workload);
 715
 716         set_context_ppgtt_from_shadow(workload, s->shadow[workload->engine->id]);
 717
 718         ret = intel_vgpu_sync_oos_pages(workload->vgpu);
 719         if (ret) {
 720                 gvt_vgpu_err("fail to vgpu sync oos pages\n");
 721                 goto err_unpin_mm;
 722         }
 723
 724         ret = intel_vgpu_flush_post_shadow(workload->vgpu);
 725         if (ret) {
 726                 gvt_vgpu_err("fail to flush post shadow\n");
 727                 goto err_unpin_mm;
 728         }
 729
 730         ret = copy_workload_to_ring_buffer(workload);
 731         if (ret) {
 732                 gvt_vgpu_err("fail to generate request\n");
 733                 goto err_unpin_mm;
 734         }
 735
 736         ret = prepare_shadow_batch_buffer(workload);
 737         if (ret) {
 738                 gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n");
 739                 goto err_unpin_mm;
 740         }
 741
 742         ret = prepare_shadow_wa_ctx(&workload->wa_ctx);
 743         if (ret) {
 744                 gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n");
 745                 goto err_shadow_batch;
 746         }
 747
 748         if (workload->prepare) {
 749                 ret = workload->prepare(workload);
 750                 if (ret)
 751                         goto err_shadow_wa_ctx;
 752         }
 753
 754         return 0;
 755 err_shadow_wa_ctx:
 756         release_shadow_wa_ctx(&workload->wa_ctx);
 757 err_shadow_batch:
 758         release_shadow_batch_buffer(workload);
 759 err_unpin_mm:
 760         intel_vgpu_shadow_mm_unpin(workload);
 761         return ret;
 762 }
 763
 764 static int dispatch_workload(struct intel_vgpu_workload *workload)
 765 {
 766         struct intel_vgpu *vgpu = workload->vgpu;
 767         struct i915_request *rq;
 768         int ret;
 769
 770         gvt_dbg_sched("ring id %s prepare to dispatch workload %p\n",
 771                       workload->engine->name, workload);
 772
 773         mutex_lock(&vgpu->vgpu_lock);
 774
 775         ret = intel_gvt_workload_req_alloc(workload);
 776         if (ret)
 777                 goto err_req;
 778
 779         ret = intel_gvt_scan_and_shadow_workload(workload);
 780         if (ret)
 781                 goto out;
 782
 783         ret = populate_shadow_context(workload);
 784         if (ret) {
 785                 release_shadow_wa_ctx(&workload->wa_ctx);
 786                 goto out;
 787         }
 788
 789         ret = prepare_workload(workload);
 790 out:
 791         if (ret) {
 792                 /* We might still need to add request with
 793                  * clean ctx to retire it properly..
 794                  */
 795                 rq = fetch_and_zero(&workload->req);
 796                 i915_request_put(rq);
 797         }
 798
 799         if (!IS_ERR_OR_NULL(workload->req)) {
 800                 gvt_dbg_sched("ring id %s submit workload to i915 %p\n",
 801                               workload->engine->name, workload->req);
 802                 i915_request_add(workload->req);
 803                 workload->dispatched = true;
 804         }
 805 err_req:
 806         if (ret)
 807                 workload->status = ret;
 808         mutex_unlock(&vgpu->vgpu_lock);
 809         return ret;
 810 }
 811
 812 static struct intel_vgpu_workload *
 813 pick_next_workload(struct intel_gvt *gvt, struct intel_engine_cs *engine)
 814 {
 815         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 816         struct intel_vgpu_workload *workload = NULL;
 817
 818         mutex_lock(&gvt->sched_lock);
 819
 820         /*
 821          * no current vgpu / will be scheduled out / no workload
 822          * bail out
 823          */
 824         if (!scheduler->current_vgpu) {
 825                 gvt_dbg_sched("ring %s stop - no current vgpu\n", engine->name);
 826                 goto out;
 827         }
 828
 829         if (scheduler->need_reschedule) {
 830                 gvt_dbg_sched("ring %s stop - will reschedule\n", engine->name);
 831                 goto out;
 832         }
 833
 834         if (!scheduler->current_vgpu->active ||
 835             list_empty(workload_q_head(scheduler->current_vgpu, engine)))
 836                 goto out;
 837
 838         /*
 839          * still have current workload, maybe the workload disptacher
 840          * fail to submit it for some reason, resubmit it.
 841          */
 842         if (scheduler->current_workload[engine->id]) {
 843                 workload = scheduler->current_workload[engine->id];
 844                 gvt_dbg_sched("ring %s still have current workload %p\n",
 845                               engine->name, workload);
 846                 goto out;
 847         }
 848
 849         /*
 850          * pick a workload as current workload
 851          * once current workload is set, schedule policy routines
 852          * will wait the current workload is finished when trying to
 853          * schedule out a vgpu.
 854          */
 855         scheduler->current_workload[engine->id] =
 856                 list_first_entry(workload_q_head(scheduler->current_vgpu,
 857                                                  engine),
 858                                  struct intel_vgpu_workload, list);
 859
 860         workload = scheduler->current_workload[engine->id];
 861
 862         gvt_dbg_sched("ring %s pick new workload %p\n", engine->name, workload);
 863
 864         atomic_inc(&workload->vgpu->submission.running_workload_num);
 865 out:
 866         mutex_unlock(&gvt->sched_lock);
 867         return workload;
 868 }
 869
 870 static void update_guest_pdps(struct intel_vgpu *vgpu,
 871                               u64 ring_context_gpa, u32 pdp[8])
 872 {
 873         u64 gpa;
 874         int i;
 875
 876         gpa = ring_context_gpa + RING_CTX_OFF(pdps[0].val);
 877
 878         for (i = 0; i < 8; i++)
 879                 intel_gvt_hypervisor_write_gpa(vgpu,
 880                                 gpa + i * 8, &pdp[7 - i], 4);
 881 }
 882
 883 static __maybe_unused bool
 884 check_shadow_context_ppgtt(struct execlist_ring_context *c, struct intel_vgpu_mm *m)
 885 {
 886         if (m->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
 887                 u64 shadow_pdp = c->pdps[7].val | (u64) c->pdps[6].val << 32;
 888
 889                 if (shadow_pdp != m->ppgtt_mm.shadow_pdps[0]) {
 890                         gvt_dbg_mm("4-level context ppgtt not match LRI command\n");
 891                         return false;
 892                 }
 893                 return true;
 894         } else {
 895                 /* see comment in LRI handler in cmd_parser.c */
 896                 gvt_dbg_mm("invalid shadow mm type\n");
 897                 return false;
 898         }
 899 }
 900
 901 static void update_guest_context(struct intel_vgpu_workload *workload)
 902 {
 903         struct i915_request *rq = workload->req;
 904         struct intel_vgpu *vgpu = workload->vgpu;
 905         struct execlist_ring_context *shadow_ring_context;
 906         struct intel_context *ctx = workload->req->context;
 907         void *context_base;
 908         void *src;
 909         unsigned long context_gpa, context_page_num;
 910         unsigned long gpa_base; /* first gpa of consecutive GPAs */
 911         unsigned long gpa_size; /* size of consecutive GPAs*/
 912         int i;
 913         u32 ring_base;
 914         u32 head, tail;
 915         u16 wrap_count;
 916
 917         gvt_dbg_sched("ring id %d workload lrca %x\n", rq->engine->id,
 918                       workload->ctx_desc.lrca);
 919
 920         GEM_BUG_ON(!intel_context_is_pinned(ctx));
 921
 922         head = workload->rb_head;
 923         tail = workload->rb_tail;
 924         wrap_count = workload->guest_rb_head >> RB_HEAD_WRAP_CNT_OFF;
 925
 926         if (tail < head) {
 927                 if (wrap_count == RB_HEAD_WRAP_CNT_MAX)
 928                         wrap_count = 0;
 929                 else
 930                         wrap_count += 1;
 931         }
 932
 933         head = (wrap_count << RB_HEAD_WRAP_CNT_OFF) | tail;
 934
 935         ring_base = rq->engine->mmio_base;
 936         vgpu_vreg_t(vgpu, RING_TAIL(ring_base)) = tail;
 937         vgpu_vreg_t(vgpu, RING_HEAD(ring_base)) = head;
 938
 939         context_page_num = rq->engine->context_size;
 940         context_page_num = context_page_num >> PAGE_SHIFT;
 941
 942         if (IS_BROADWELL(rq->i915) && rq->engine->id == RCS0)
 943                 context_page_num = 19;
 944
 945         context_base = (void *) ctx->lrc_reg_state -
 946                         (LRC_STATE_PN << I915_GTT_PAGE_SHIFT);
 947
 948         /* find consecutive GPAs from gma until the first inconsecutive GPA.
 949          * write to the consecutive GPAs from src virtual address
 950          */
 951         gpa_size = 0;
 952         for (i = 2; i < context_page_num; i++) {
 953                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
 954                                 (u32)((workload->ctx_desc.lrca + i) <<
 955                                         I915_GTT_PAGE_SHIFT));
 956                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
 957                         gvt_vgpu_err("invalid guest context descriptor\n");
 958                         return;
 959                 }
 960
 961                 if (gpa_size == 0) {
 962                         gpa_base = context_gpa;
 963                         src = context_base + (i << I915_GTT_PAGE_SHIFT);
 964                 } else if (context_gpa != gpa_base + gpa_size)
 965                         goto write;
 966
 967                 gpa_size += I915_GTT_PAGE_SIZE;
 968
 969                 if (i == context_page_num - 1)
 970                         goto write;
 971
 972                 continue;
 973
 974 write:
 975                 intel_gvt_hypervisor_write_gpa(vgpu, gpa_base, src, gpa_size);
 976                 gpa_base = context_gpa;
 977                 gpa_size = I915_GTT_PAGE_SIZE;
 978                 src = context_base + (i << I915_GTT_PAGE_SHIFT);
 979         }
 980
 981         intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa +
 982                 RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4);
 983
 984         shadow_ring_context = (void *) ctx->lrc_reg_state;
 985
 986         if (!list_empty(&workload->lri_shadow_mm)) {
 987                 struct intel_vgpu_mm *m = list_last_entry(&workload->lri_shadow_mm,
 988                                                           struct intel_vgpu_mm,
 989                                                           ppgtt_mm.link);
 990                 GEM_BUG_ON(!check_shadow_context_ppgtt(shadow_ring_context, m));
 991                 update_guest_pdps(vgpu, workload->ring_context_gpa,
 992                                   (void *)m->ppgtt_mm.guest_pdps);
 993         }
 994
 995 #define COPY_REG(name) \
 996         intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \
 997                 RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
 998
 999         COPY_REG(ctx_ctrl);
1000         COPY_REG(ctx_timestamp);
1001
1002 #undef COPY_REG
1003
1004         intel_gvt_hypervisor_write_gpa(vgpu,
1005                         workload->ring_context_gpa +
1006                         sizeof(*shadow_ring_context),
1007                         (void *)shadow_ring_context +
1008                         sizeof(*shadow_ring_context),
1009                         I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
1010 }
1011
1012 void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu,
1013                                 intel_engine_mask_t engine_mask)
1014 {
1015         struct intel_vgpu_submission *s = &vgpu->submission;
1016         struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915;
1017         struct intel_engine_cs *engine;
1018         struct intel_vgpu_workload *pos, *n;
1019         intel_engine_mask_t tmp;
1020
1021         /* free the unsubmited workloads in the queues. */
1022         for_each_engine_masked(engine, &dev_priv->gt, engine_mask, tmp) {
1023                 list_for_each_entry_safe(pos, n,
1024                         &s->workload_q_head[engine->id], list) {
1025                         list_del_init(&pos->list);
1026                         intel_vgpu_destroy_workload(pos);
1027                 }
1028                 clear_bit(engine->id, s->shadow_ctx_desc_updated);
1029         }
1030 }
1031
1032 static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
1033 {
1034         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1035         struct intel_vgpu_workload *workload =
1036                 scheduler->current_workload[ring_id];
1037         struct intel_vgpu *vgpu = workload->vgpu;
1038         struct intel_vgpu_submission *s = &vgpu->submission;
1039         struct i915_request *rq = workload->req;
1040         int event;
1041
1042         mutex_lock(&vgpu->vgpu_lock);
1043         mutex_lock(&gvt->sched_lock);
1044
1045         /* For the workload w/ request, needs to wait for the context
1046          * switch to make sure request is completed.
1047          * For the workload w/o request, directly complete the workload.
1048          */
1049         if (rq) {
1050                 wait_event(workload->shadow_ctx_status_wq,
1051                            !atomic_read(&workload->shadow_ctx_active));
1052
1053                 /* If this request caused GPU hang, req->fence.error will
1054                  * be set to -EIO. Use -EIO to set workload status so
1055                  * that when this request caused GPU hang, didn't trigger
1056                  * context switch interrupt to guest.
1057                  */
1058                 if (likely(workload->status == -EINPROGRESS)) {
1059                         if (workload->req->fence.error == -EIO)
1060                                 workload->status = -EIO;
1061                         else
1062                                 workload->status = 0;
1063                 }
1064
1065                 if (!workload->status &&
1066                     !(vgpu->resetting_eng & BIT(ring_id))) {
1067                         update_guest_context(workload);
1068
1069                         for_each_set_bit(event, workload->pending_events,
1070                                          INTEL_GVT_EVENT_MAX)
1071                                 intel_vgpu_trigger_virtual_event(vgpu, event);
1072                 }
1073
1074                 i915_request_put(fetch_and_zero(&workload->req));
1075         }
1076
1077         gvt_dbg_sched("ring id %d complete workload %p status %d\n",
1078                         ring_id, workload, workload->status);
1079
1080         scheduler->current_workload[ring_id] = NULL;
1081
1082         list_del_init(&workload->list);
1083
1084         if (workload->status || vgpu->resetting_eng & BIT(ring_id)) {
1085                 /* if workload->status is not successful means HW GPU
1086                  * has occurred GPU hang or something wrong with i915/GVT,
1087                  * and GVT won't inject context switch interrupt to guest.
1088                  * So this error is a vGPU hang actually to the guest.
1089                  * According to this we should emunlate a vGPU hang. If
1090                  * there are pending workloads which are already submitted
1091                  * from guest, we should clean them up like HW GPU does.
1092                  *
1093                  * if it is in middle of engine resetting, the pending
1094                  * workloads won't be submitted to HW GPU and will be
1095                  * cleaned up during the resetting process later, so doing
1096                  * the workload clean up here doesn't have any impact.
1097                  **/
1098                 intel_vgpu_clean_workloads(vgpu, BIT(ring_id));
1099         }
1100
1101         workload->complete(workload);
1102
1103         intel_vgpu_shadow_mm_unpin(workload);
1104         intel_vgpu_destroy_workload(workload);
1105
1106         atomic_dec(&s->running_workload_num);
1107         wake_up(&scheduler->workload_complete_wq);
1108
1109         if (gvt->scheduler.need_reschedule)
1110                 intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED);
1111
1112         mutex_unlock(&gvt->sched_lock);
1113         mutex_unlock(&vgpu->vgpu_lock);
1114 }
1115
1116 static int workload_thread(void *arg)
1117 {
1118         struct intel_engine_cs *engine = arg;
1119         const bool need_force_wake = INTEL_GEN(engine->i915) >= 9;
1120         struct intel_gvt *gvt = engine->i915->gvt;
1121         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1122         struct intel_vgpu_workload *workload = NULL;
1123         struct intel_vgpu *vgpu = NULL;
1124         int ret;
1125         DEFINE_WAIT_FUNC(wait, woken_wake_function);
1126
1127         gvt_dbg_core("workload thread for ring %s started\n", engine->name);
1128
1129         while (!kthread_should_stop()) {
1130                 intel_wakeref_t wakeref;
1131
1132                 add_wait_queue(&scheduler->waitq[engine->id], &wait);
1133                 do {
1134                         workload = pick_next_workload(gvt, engine);
1135                         if (workload)
1136                                 break;
1137                         wait_woken(&wait, TASK_INTERRUPTIBLE,
1138                                    MAX_SCHEDULE_TIMEOUT);
1139                 } while (!kthread_should_stop());
1140                 remove_wait_queue(&scheduler->waitq[engine->id], &wait);
1141
1142                 if (!workload)
1143                         break;
1144
1145                 gvt_dbg_sched("ring %s next workload %p vgpu %d\n",
1146                               engine->name, workload,
1147                               workload->vgpu->id);
1148
1149                 wakeref = intel_runtime_pm_get(engine->uncore->rpm);
1150
1151                 gvt_dbg_sched("ring %s will dispatch workload %p\n",
1152                               engine->name, workload);
1153
1154                 if (need_force_wake)
1155                         intel_uncore_forcewake_get(engine->uncore,
1156                                                    FORCEWAKE_ALL);
1157                 /*
1158                  * Update the vReg of the vGPU which submitted this
1159                  * workload. The vGPU may use these registers for checking
1160                  * the context state. The value comes from GPU commands
1161                  * in this workload.
1162                  */
1163                 update_vreg_in_ctx(workload);
1164
1165                 ret = dispatch_workload(workload);
1166
1167                 if (ret) {
1168                         vgpu = workload->vgpu;
1169                         gvt_vgpu_err("fail to dispatch workload, skip\n");
1170                         goto complete;
1171                 }
1172
1173                 gvt_dbg_sched("ring %s wait workload %p\n",
1174                               engine->name, workload);
1175                 i915_request_wait(workload->req, 0, MAX_SCHEDULE_TIMEOUT);
1176
1177 complete:
1178                 gvt_dbg_sched("will complete workload %p, status: %d\n",
1179                               workload, workload->status);
1180
1181                 complete_current_workload(gvt, engine->id);
1182
1183                 if (need_force_wake)
1184                         intel_uncore_forcewake_put(engine->uncore,
1185                                                    FORCEWAKE_ALL);
1186
1187                 intel_runtime_pm_put(engine->uncore->rpm, wakeref);
1188                 if (ret && (vgpu_is_vm_unhealthy(ret)))
1189                         enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
1190         }
1191         return 0;
1192 }
1193
1194 void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu)
1195 {
1196         struct intel_vgpu_submission *s = &vgpu->submission;
1197         struct intel_gvt *gvt = vgpu->gvt;
1198         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1199
1200         if (atomic_read(&s->running_workload_num)) {
1201                 gvt_dbg_sched("wait vgpu idle\n");
1202
1203                 wait_event(scheduler->workload_complete_wq,
1204                                 !atomic_read(&s->running_workload_num));
1205         }
1206 }
1207
1208 void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt)
1209 {
1210         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1211         struct intel_engine_cs *engine;
1212         enum intel_engine_id i;
1213
1214         gvt_dbg_core("clean workload scheduler\n");
1215
1216         for_each_engine(engine, gvt->gt, i) {
1217                 atomic_notifier_chain_unregister(
1218                                         &engine->context_status_notifier,
1219                                         &gvt->shadow_ctx_notifier_block[i]);
1220                 kthread_stop(scheduler->thread[i]);
1221         }
1222 }
1223
1224 int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt)
1225 {
1226         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1227         struct intel_engine_cs *engine;
1228         enum intel_engine_id i;
1229         int ret;
1230
1231         gvt_dbg_core("init workload scheduler\n");
1232
1233         init_waitqueue_head(&scheduler->workload_complete_wq);
1234
1235         for_each_engine(engine, gvt->gt, i) {
1236                 init_waitqueue_head(&scheduler->waitq[i]);
1237
1238                 scheduler->thread[i] = kthread_run(workload_thread, engine,
1239                                                    "gvt:%s", engine->name);
1240                 if (IS_ERR(scheduler->thread[i])) {
1241                         gvt_err("fail to create workload thread\n");
1242                         ret = PTR_ERR(scheduler->thread[i]);
1243                         goto err;
1244                 }
1245
1246                 gvt->shadow_ctx_notifier_block[i].notifier_call =
1247                                         shadow_context_status_change;
1248                 atomic_notifier_chain_register(&engine->context_status_notifier,
1249                                         &gvt->shadow_ctx_notifier_block[i]);
1250         }
1251
1252         return 0;
1253
1254 err:
1255         intel_gvt_clean_workload_scheduler(gvt);
1256         return ret;
1257 }
1258
1259 static void
1260 i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s,
1261                                 struct i915_ppgtt *ppgtt)
1262 {
1263         int i;
1264
1265         if (i915_vm_is_4lvl(&ppgtt->vm)) {
1266                 px_dma(ppgtt->pd) = s->i915_context_pml4;
1267         } else {
1268                 for (i = 0; i < GEN8_3LVL_PDPES; i++) {
1269                         struct i915_page_directory * const pd =
1270                                 i915_pd_entry(ppgtt->pd, i);
1271
1272                         px_dma(pd) = s->i915_context_pdps[i];
1273                 }
1274         }
1275 }
1276
1277 /**
1278  * intel_vgpu_clean_submission - free submission-related resource for vGPU
1279  * @vgpu: a vGPU
1280  *
1281  * This function is called when a vGPU is being destroyed.
1282  *
1283  */
1284 void intel_vgpu_clean_submission(struct intel_vgpu *vgpu)
1285 {
1286         struct intel_vgpu_submission *s = &vgpu->submission;
1287         struct intel_engine_cs *engine;
1288         enum intel_engine_id id;
1289
1290         intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0);
1291
1292         i915_context_ppgtt_root_restore(s, i915_vm_to_ppgtt(s->shadow[0]->vm));
1293         for_each_engine(engine, vgpu->gvt->gt, id)
1294                 intel_context_unpin(s->shadow[id]);
1295
1296         kmem_cache_destroy(s->workloads);
1297 }
1298
1299
1300 /**
1301  * intel_vgpu_reset_submission - reset submission-related resource for vGPU
1302  * @vgpu: a vGPU
1303  * @engine_mask: engines expected to be reset
1304  *
1305  * This function is called when a vGPU is being destroyed.
1306  *
1307  */
1308 void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
1309                                  intel_engine_mask_t engine_mask)
1310 {
1311         struct intel_vgpu_submission *s = &vgpu->submission;
1312
1313         if (!s->active)
1314                 return;
1315
1316         intel_vgpu_clean_workloads(vgpu, engine_mask);
1317         s->ops->reset(vgpu, engine_mask);
1318 }
1319
1320 static void
1321 i915_context_ppgtt_root_save(struct intel_vgpu_submission *s,
1322                              struct i915_ppgtt *ppgtt)
1323 {
1324         int i;
1325
1326         if (i915_vm_is_4lvl(&ppgtt->vm)) {
1327                 s->i915_context_pml4 = px_dma(ppgtt->pd);
1328         } else {
1329                 for (i = 0; i < GEN8_3LVL_PDPES; i++) {
1330                         struct i915_page_directory * const pd =
1331                                 i915_pd_entry(ppgtt->pd, i);
1332
1333                         s->i915_context_pdps[i] = px_dma(pd);
1334                 }
1335         }
1336 }
1337
1338 /**
1339  * intel_vgpu_setup_submission - setup submission-related resource for vGPU
1340  * @vgpu: a vGPU
1341  *
1342  * This function is called when a vGPU is being created.
1343  *
1344  * Returns:
1345  * Zero on success, negative error code if failed.
1346  *
1347  */
1348 int intel_vgpu_setup_submission(struct intel_vgpu *vgpu)
1349 {
1350         struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
1351         struct intel_vgpu_submission *s = &vgpu->submission;
1352         struct intel_engine_cs *engine;
1353         struct i915_ppgtt *ppgtt;
1354         enum intel_engine_id i;
1355         int ret;
1356
1357         ppgtt = i915_ppgtt_create(&i915->gt);
1358         if (IS_ERR(ppgtt))
1359                 return PTR_ERR(ppgtt);
1360
1361         i915_context_ppgtt_root_save(s, ppgtt);
1362
1363         for_each_engine(engine, vgpu->gvt->gt, i) {
1364                 struct intel_context *ce;
1365
1366                 INIT_LIST_HEAD(&s->workload_q_head[i]);
1367                 s->shadow[i] = ERR_PTR(-EINVAL);
1368
1369                 ce = intel_context_create(engine);
1370                 if (IS_ERR(ce)) {
1371                         ret = PTR_ERR(ce);
1372                         goto out_shadow_ctx;
1373                 }
1374
1375                 i915_vm_put(ce->vm);
1376                 ce->vm = i915_vm_get(&ppgtt->vm);
1377                 intel_context_set_single_submission(ce);
1378
1379                 /* Max ring buffer size */
1380                 if (!intel_uc_wants_guc_submission(&engine->gt->uc)) {
1381                         const unsigned int ring_size = 512 * SZ_4K;
1382
1383                         ce->ring = __intel_context_ring_size(ring_size);
1384                 }
1385
1386                 ret = intel_context_pin(ce);
1387                 intel_context_put(ce);
1388                 if (ret)
1389                         goto out_shadow_ctx;
1390
1391                 s->shadow[i] = ce;
1392         }
1393
1394         bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES);
1395
1396         s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload",
1397                                                   sizeof(struct intel_vgpu_workload), 0,
1398                                                   SLAB_HWCACHE_ALIGN,
1399                                                   offsetof(struct intel_vgpu_workload, rb_tail),
1400                                                   sizeof_field(struct intel_vgpu_workload, rb_tail),
1401                                                   NULL);
1402
1403         if (!s->workloads) {
1404                 ret = -ENOMEM;
1405                 goto out_shadow_ctx;
1406         }
1407
1408         atomic_set(&s->running_workload_num, 0);
1409         bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES);
1410
1411         memset(s->last_ctx, 0, sizeof(s->last_ctx));
1412
1413         i915_vm_put(&ppgtt->vm);
1414         return 0;
1415
1416 out_shadow_ctx:
1417         i915_context_ppgtt_root_restore(s, ppgtt);
1418         for_each_engine(engine, vgpu->gvt->gt, i) {
1419                 if (IS_ERR(s->shadow[i]))
1420                         break;
1421
1422                 intel_context_unpin(s->shadow[i]);
1423                 intel_context_put(s->shadow[i]);
1424         }
1425         i915_vm_put(&ppgtt->vm);
1426         return ret;
1427 }
1428
1429 /**
1430  * intel_vgpu_select_submission_ops - select virtual submission interface
1431  * @vgpu: a vGPU
1432  * @engine_mask: either ALL_ENGINES or target engine mask
1433  * @interface: expected vGPU virtual submission interface
1434  *
1435  * This function is called when guest configures submission interface.
1436  *
1437  * Returns:
1438  * Zero on success, negative error code if failed.
1439  *
1440  */
1441 int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
1442                                      intel_engine_mask_t engine_mask,
1443                                      unsigned int interface)
1444 {
1445         struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
1446         struct intel_vgpu_submission *s = &vgpu->submission;
1447         const struct intel_vgpu_submission_ops *ops[] = {
1448                 [INTEL_VGPU_EXECLIST_SUBMISSION] =
1449                         &intel_vgpu_execlist_submission_ops,
1450         };
1451         int ret;
1452
1453         if (drm_WARN_ON(&i915->drm, interface >= ARRAY_SIZE(ops)))
1454                 return -EINVAL;
1455
1456         if (drm_WARN_ON(&i915->drm,
1457                         interface == 0 && engine_mask != ALL_ENGINES))
1458                 return -EINVAL;
1459
1460         if (s->active)
1461                 s->ops->clean(vgpu, engine_mask);
1462
1463         if (interface == 0) {
1464                 s->ops = NULL;
1465                 s->virtual_submission_interface = 0;
1466                 s->active = false;
1467                 gvt_dbg_core("vgpu%d: remove submission ops\n", vgpu->id);
1468                 return 0;
1469         }
1470
1471         ret = ops[interface]->init(vgpu, engine_mask);
1472         if (ret)
1473                 return ret;
1474
1475         s->ops = ops[interface];
1476         s->virtual_submission_interface = interface;
1477         s->active = true;
1478
1479         gvt_dbg_core("vgpu%d: activate ops [ %s ]\n",
1480                         vgpu->id, s->ops->name);
1481
1482         return 0;
1483 }
1484
1485 /**
1486  * intel_vgpu_destroy_workload - destroy a vGPU workload
1487  * @workload: workload to destroy
1488  *
1489  * This function is called when destroy a vGPU workload.
1490  *
1491  */
1492 void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload)
1493 {
1494         struct intel_vgpu_submission *s = &workload->vgpu->submission;
1495
1496         release_shadow_batch_buffer(workload);
1497         release_shadow_wa_ctx(&workload->wa_ctx);
1498
1499         if (!list_empty(&workload->lri_shadow_mm)) {
1500                 struct intel_vgpu_mm *m, *mm;
1501                 list_for_each_entry_safe(m, mm, &workload->lri_shadow_mm,
1502                                          ppgtt_mm.link) {
1503                         list_del(&m->ppgtt_mm.link);
1504                         intel_vgpu_mm_put(m);
1505                 }
1506         }
1507
1508         GEM_BUG_ON(!list_empty(&workload->lri_shadow_mm));
1509         if (workload->shadow_mm)
1510                 intel_vgpu_mm_put(workload->shadow_mm);
1511
1512         kmem_cache_free(s->workloads, workload);
1513 }
1514
1515 static struct intel_vgpu_workload *
1516 alloc_workload(struct intel_vgpu *vgpu)
1517 {
1518         struct intel_vgpu_submission *s = &vgpu->submission;
1519         struct intel_vgpu_workload *workload;
1520
1521         workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL);
1522         if (!workload)
1523                 return ERR_PTR(-ENOMEM);
1524
1525         INIT_LIST_HEAD(&workload->list);
1526         INIT_LIST_HEAD(&workload->shadow_bb);
1527         INIT_LIST_HEAD(&workload->lri_shadow_mm);
1528
1529         init_waitqueue_head(&workload->shadow_ctx_status_wq);
1530         atomic_set(&workload->shadow_ctx_active, 0);
1531
1532         workload->status = -EINPROGRESS;
1533         workload->vgpu = vgpu;
1534
1535         return workload;
1536 }
1537
1538 #define RING_CTX_OFF(x) \
1539         offsetof(struct execlist_ring_context, x)
1540
1541 static void read_guest_pdps(struct intel_vgpu *vgpu,
1542                 u64 ring_context_gpa, u32 pdp[8])
1543 {
1544         u64 gpa;
1545         int i;
1546
1547         gpa = ring_context_gpa + RING_CTX_OFF(pdps[0].val);
1548
1549         for (i = 0; i < 8; i++)
1550                 intel_gvt_hypervisor_read_gpa(vgpu,
1551                                 gpa + i * 8, &pdp[7 - i], 4);
1552 }
1553
1554 static int prepare_mm(struct intel_vgpu_workload *workload)
1555 {
1556         struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
1557         struct intel_vgpu_mm *mm;
1558         struct intel_vgpu *vgpu = workload->vgpu;
1559         enum intel_gvt_gtt_type root_entry_type;
1560         u64 pdps[GVT_RING_CTX_NR_PDPS];
1561
1562         switch (desc->addressing_mode) {
1563         case 1: /* legacy 32-bit */
1564                 root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
1565                 break;
1566         case 3: /* legacy 64-bit */
1567                 root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
1568                 break;
1569         default:
1570                 gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n");
1571                 return -EINVAL;
1572         }
1573
1574         read_guest_pdps(workload->vgpu, workload->ring_context_gpa, (void *)pdps);
1575
1576         mm = intel_vgpu_get_ppgtt_mm(workload->vgpu, root_entry_type, pdps);
1577         if (IS_ERR(mm))
1578                 return PTR_ERR(mm);
1579
1580         workload->shadow_mm = mm;
1581         return 0;
1582 }
1583
1584 #define same_context(a, b) (((a)->context_id == (b)->context_id) && \
1585                 ((a)->lrca == (b)->lrca))
1586
1587 /**
1588  * intel_vgpu_create_workload - create a vGPU workload
1589  * @vgpu: a vGPU
1590  * @engine: the engine
1591  * @desc: a guest context descriptor
1592  *
1593  * This function is called when creating a vGPU workload.
1594  *
1595  * Returns:
1596  * struct intel_vgpu_workload * on success, negative error code in
1597  * pointer if failed.
1598  *
1599  */
1600 struct intel_vgpu_workload *
1601 intel_vgpu_create_workload(struct intel_vgpu *vgpu,
1602                            const struct intel_engine_cs *engine,
1603                            struct execlist_ctx_descriptor_format *desc)
1604 {
1605         struct intel_vgpu_submission *s = &vgpu->submission;
1606         struct list_head *q = workload_q_head(vgpu, engine);
1607         struct intel_vgpu_workload *last_workload = NULL;
1608         struct intel_vgpu_workload *workload = NULL;
1609         u64 ring_context_gpa;
1610         u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
1611         u32 guest_head;
1612         int ret;
1613
1614         ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
1615                         (u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT));
1616         if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
1617                 gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca);
1618                 return ERR_PTR(-EINVAL);
1619         }
1620
1621         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1622                         RING_CTX_OFF(ring_header.val), &head, 4);
1623
1624         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1625                         RING_CTX_OFF(ring_tail.val), &tail, 4);
1626
1627         guest_head = head;
1628
1629         head &= RB_HEAD_OFF_MASK;
1630         tail &= RB_TAIL_OFF_MASK;
1631
1632         list_for_each_entry_reverse(last_workload, q, list) {
1633
1634                 if (same_context(&last_workload->ctx_desc, desc)) {
1635                         gvt_dbg_el("ring %s cur workload == last\n",
1636                                    engine->name);
1637                         gvt_dbg_el("ctx head %x real head %lx\n", head,
1638                                    last_workload->rb_tail);
1639                         /*
1640                          * cannot use guest context head pointer here,
1641                          * as it might not be updated at this time
1642                          */
1643                         head = last_workload->rb_tail;
1644                         break;
1645                 }
1646         }
1647
1648         gvt_dbg_el("ring %s begin a new workload\n", engine->name);
1649
1650         /* record some ring buffer register values for scan and shadow */
1651         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1652                         RING_CTX_OFF(rb_start.val), &start, 4);
1653         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1654                         RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
1655         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1656                         RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);
1657
1658         if (!intel_gvt_ggtt_validate_range(vgpu, start,
1659                                 _RING_CTL_BUF_SIZE(ctl))) {
1660                 gvt_vgpu_err("context contain invalid rb at: 0x%x\n", start);
1661                 return ERR_PTR(-EINVAL);
1662         }
1663
1664         workload = alloc_workload(vgpu);
1665         if (IS_ERR(workload))
1666                 return workload;
1667
1668         workload->engine = engine;
1669         workload->ctx_desc = *desc;
1670         workload->ring_context_gpa = ring_context_gpa;
1671         workload->rb_head = head;
1672         workload->guest_rb_head = guest_head;
1673         workload->rb_tail = tail;
1674         workload->rb_start = start;
1675         workload->rb_ctl = ctl;
1676
1677         if (engine->id == RCS0) {
1678                 intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1679                         RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
1680                 intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1681                         RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);
1682
1683                 workload->wa_ctx.indirect_ctx.guest_gma =
1684                         indirect_ctx & INDIRECT_CTX_ADDR_MASK;
1685                 workload->wa_ctx.indirect_ctx.size =
1686                         (indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
1687                         CACHELINE_BYTES;
1688
1689                 if (workload->wa_ctx.indirect_ctx.size != 0) {
1690                         if (!intel_gvt_ggtt_validate_range(vgpu,
1691                                 workload->wa_ctx.indirect_ctx.guest_gma,
1692                                 workload->wa_ctx.indirect_ctx.size)) {
1693                                 gvt_vgpu_err("invalid wa_ctx at: 0x%lx\n",
1694                                     workload->wa_ctx.indirect_ctx.guest_gma);
1695                                 kmem_cache_free(s->workloads, workload);
1696                                 return ERR_PTR(-EINVAL);
1697                         }
1698                 }
1699
1700                 workload->wa_ctx.per_ctx.guest_gma =
1701                         per_ctx & PER_CTX_ADDR_MASK;
1702                 workload->wa_ctx.per_ctx.valid = per_ctx & 1;
1703                 if (workload->wa_ctx.per_ctx.valid) {
1704                         if (!intel_gvt_ggtt_validate_range(vgpu,
1705                                 workload->wa_ctx.per_ctx.guest_gma,
1706                                 CACHELINE_BYTES)) {
1707                                 gvt_vgpu_err("invalid per_ctx at: 0x%lx\n",
1708                                         workload->wa_ctx.per_ctx.guest_gma);
1709                                 kmem_cache_free(s->workloads, workload);
1710                                 return ERR_PTR(-EINVAL);
1711                         }
1712                 }
1713         }
1714
1715         gvt_dbg_el("workload %p ring %s head %x tail %x start %x ctl %x\n",
1716                    workload, engine->name, head, tail, start, ctl);
1717
1718         ret = prepare_mm(workload);
1719         if (ret) {
1720                 kmem_cache_free(s->workloads, workload);
1721                 return ERR_PTR(ret);
1722         }
1723
1724         /* Only scan and shadow the first workload in the queue
1725          * as there is only one pre-allocated buf-obj for shadow.
1726          */
1727         if (list_empty(q)) {
1728                 intel_wakeref_t wakeref;
1729
1730                 with_intel_runtime_pm(engine->gt->uncore->rpm, wakeref)
1731                         ret = intel_gvt_scan_and_shadow_workload(workload);
1732         }
1733
1734         if (ret) {
1735                 if (vgpu_is_vm_unhealthy(ret))
1736                         enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
1737                 intel_vgpu_destroy_workload(workload);
1738                 return ERR_PTR(ret);
1739         }
1740
1741         return workload;
1742 }
1743
1744 /**
1745  * intel_vgpu_queue_workload - Qeue a vGPU workload
1746  * @workload: the workload to queue in
1747  */
1748 void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload)
1749 {
1750         list_add_tail(&workload->list,
1751                       workload_q_head(workload->vgpu, workload->engine));
1752         intel_gvt_kick_schedule(workload->vgpu->gvt);
1753         wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->engine->id]);
1754 }