drivers/accel/ivpu/ivpu_job.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2023 Intel Corporation
   4  */
   5
   6 #include <drm/drm_file.h>
   7
   8 #include <linux/bitfield.h>
   9 #include <linux/highmem.h>
  10 #include <linux/pci.h>
  11 #include <linux/module.h>
  12 #include <uapi/drm/ivpu_accel.h>
  13
  14 #include "ivpu_drv.h"
  15 #include "ivpu_hw.h"
  16 #include "ivpu_ipc.h"
  17 #include "ivpu_job.h"
  18 #include "ivpu_jsm_msg.h"
  19 #include "ivpu_pm.h"
  20
  21 #define CMD_BUF_IDX          0
  22 #define JOB_ID_JOB_MASK      GENMASK(7, 0)
  23 #define JOB_ID_CONTEXT_MASK  GENMASK(31, 8)
  24 #define JOB_MAX_BUFFER_COUNT 65535
  25
  26 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
  27 {
  28         ivpu_hw_reg_db_set(vdev, cmdq->db_id);
  29 }
  30
  31 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv, u16 engine)
  32 {
  33         struct xa_limit db_xa_limit = {.max = IVPU_MAX_DB, .min = IVPU_MIN_DB};
  34         struct ivpu_device *vdev = file_priv->vdev;
  35         struct vpu_job_queue_header *jobq_header;
  36         struct ivpu_cmdq *cmdq;
  37         int ret;
  38
  39         cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);
  40         if (!cmdq)
  41                 return NULL;
  42
  43         ret = xa_alloc(&vdev->db_xa, &cmdq->db_id, NULL, db_xa_limit, GFP_KERNEL);
  44         if (ret) {
  45                 ivpu_err(vdev, "Failed to allocate doorbell id: %d\n", ret);
  46                 goto err_free_cmdq;
  47         }
  48
  49         cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
  50         if (!cmdq->mem)
  51                 goto err_erase_xa;
  52
  53         cmdq->entry_count = (u32)((ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header)) /
  54                                   sizeof(struct vpu_job_queue_entry));
  55
  56         cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);
  57         jobq_header = &cmdq->jobq->header;
  58         jobq_header->engine_idx = engine;
  59         jobq_header->head = 0;
  60         jobq_header->tail = 0;
  61         wmb(); /* Flush WC buffer for jobq->header */
  62
  63         return cmdq;
  64
  65 err_erase_xa:
  66         xa_erase(&vdev->db_xa, cmdq->db_id);
  67 err_free_cmdq:
  68         kfree(cmdq);
  69         return NULL;
  70 }
  71
  72 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
  73 {
  74         if (!cmdq)
  75                 return;
  76
  77         ivpu_bo_free(cmdq->mem);
  78         xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
  79         kfree(cmdq);
  80 }
  81
  82 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u16 engine)
  83 {
  84         struct ivpu_device *vdev = file_priv->vdev;
  85         struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
  86         int ret;
  87
  88         lockdep_assert_held(&file_priv->lock);
  89
  90         if (!cmdq) {
  91                 cmdq = ivpu_cmdq_alloc(file_priv, engine);
  92                 if (!cmdq)
  93                         return NULL;
  94                 file_priv->cmdq[engine] = cmdq;
  95         }
  96
  97         if (cmdq->db_registered)
  98                 return cmdq;
  99
 100         ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
 101                                    cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
 102         if (ret)
 103                 return NULL;
 104
 105         cmdq->db_registered = true;
 106
 107         return cmdq;
 108 }
 109
 110 static void ivpu_cmdq_release_locked(struct ivpu_file_priv *file_priv, u16 engine)
 111 {
 112         struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
 113
 114         lockdep_assert_held(&file_priv->lock);
 115
 116         if (cmdq) {
 117                 file_priv->cmdq[engine] = NULL;
 118                 if (cmdq->db_registered)
 119                         ivpu_jsm_unregister_db(file_priv->vdev, cmdq->db_id);
 120
 121                 ivpu_cmdq_free(file_priv, cmdq);
 122         }
 123 }
 124
 125 void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv)
 126 {
 127         int i;
 128
 129         lockdep_assert_held(&file_priv->lock);
 130
 131         for (i = 0; i < IVPU_NUM_ENGINES; i++)
 132                 ivpu_cmdq_release_locked(file_priv, i);
 133 }
 134
 135 /*
 136  * Mark the doorbell as unregistered and reset job queue pointers.
 137  * This function needs to be called when the VPU hardware is restarted
 138  * and FW loses job queue state. The next time job queue is used it
 139  * will be registered again.
 140  */
 141 static void ivpu_cmdq_reset_locked(struct ivpu_file_priv *file_priv, u16 engine)
 142 {
 143         struct ivpu_cmdq *cmdq = file_priv->cmdq[engine];
 144
 145         lockdep_assert_held(&file_priv->lock);
 146
 147         if (cmdq) {
 148                 cmdq->db_registered = false;
 149                 cmdq->jobq->header.head = 0;
 150                 cmdq->jobq->header.tail = 0;
 151                 wmb(); /* Flush WC buffer for jobq header */
 152         }
 153 }
 154
 155 static void ivpu_cmdq_reset_all(struct ivpu_file_priv *file_priv)
 156 {
 157         int i;
 158
 159         mutex_lock(&file_priv->lock);
 160
 161         for (i = 0; i < IVPU_NUM_ENGINES; i++)
 162                 ivpu_cmdq_reset_locked(file_priv, i);
 163
 164         mutex_unlock(&file_priv->lock);
 165 }
 166
 167 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev)
 168 {
 169         struct ivpu_file_priv *file_priv;
 170         unsigned long ctx_id;
 171
 172         mutex_lock(&vdev->context_list_lock);
 173
 174         xa_for_each(&vdev->context_xa, ctx_id, file_priv)
 175                 ivpu_cmdq_reset_all(file_priv);
 176
 177         mutex_unlock(&vdev->context_list_lock);
 178
 179 }
 180
 181 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
 182 {
 183         struct ivpu_device *vdev = job->vdev;
 184         struct vpu_job_queue_header *header = &cmdq->jobq->header;
 185         struct vpu_job_queue_entry *entry;
 186         u32 tail = READ_ONCE(header->tail);
 187         u32 next_entry = (tail + 1) % cmdq->entry_count;
 188
 189         /* Check if there is space left in job queue */
 190         if (next_entry == header->head) {
 191                 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d engine %d db %d head %d tail %d\n",
 192                          job->file_priv->ctx.id, job->engine_idx, cmdq->db_id, header->head, tail);
 193                 return -EBUSY;
 194         }
 195
 196         entry = &cmdq->jobq->job[tail];
 197         entry->batch_buf_addr = job->cmd_buf_vpu_addr;
 198         entry->job_id = job->job_id;
 199         entry->flags = 0;
 200         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))
 201                 entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;
 202         wmb(); /* Ensure that tail is updated after filling entry */
 203         header->tail = next_entry;
 204         wmb(); /* Flush WC buffer for jobq header */
 205
 206         return 0;
 207 }
 208
 209 struct ivpu_fence {
 210         struct dma_fence base;
 211         spinlock_t lock; /* protects base */
 212         struct ivpu_device *vdev;
 213 };
 214
 215 static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence)
 216 {
 217         return container_of(fence, struct ivpu_fence, base);
 218 }
 219
 220 static const char *ivpu_fence_get_driver_name(struct dma_fence *fence)
 221 {
 222         return DRIVER_NAME;
 223 }
 224
 225 static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence)
 226 {
 227         struct ivpu_fence *ivpu_fence = to_vpu_fence(fence);
 228
 229         return dev_name(ivpu_fence->vdev->drm.dev);
 230 }
 231
 232 static const struct dma_fence_ops ivpu_fence_ops = {
 233         .get_driver_name = ivpu_fence_get_driver_name,
 234         .get_timeline_name = ivpu_fence_get_timeline_name,
 235 };
 236
 237 static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev)
 238 {
 239         struct ivpu_fence *fence;
 240
 241         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 242         if (!fence)
 243                 return NULL;
 244
 245         fence->vdev = vdev;
 246         spin_lock_init(&fence->lock);
 247         dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1);
 248
 249         return &fence->base;
 250 }
 251
 252 static void ivpu_job_destroy(struct ivpu_job *job)
 253 {
 254         struct ivpu_device *vdev = job->vdev;
 255         u32 i;
 256
 257         ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d",
 258                  job->job_id, job->file_priv->ctx.id, job->engine_idx);
 259
 260         for (i = 0; i < job->bo_count; i++)
 261                 if (job->bos[i])
 262                         drm_gem_object_put(&job->bos[i]->base.base);
 263
 264         dma_fence_put(job->done_fence);
 265         ivpu_file_priv_put(&job->file_priv);
 266         kfree(job);
 267 }
 268
 269 static struct ivpu_job *
 270 ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
 271 {
 272         struct ivpu_device *vdev = file_priv->vdev;
 273         struct ivpu_job *job;
 274
 275         job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);
 276         if (!job)
 277                 return NULL;
 278
 279         job->vdev = vdev;
 280         job->engine_idx = engine_idx;
 281         job->bo_count = bo_count;
 282         job->done_fence = ivpu_fence_create(vdev);
 283         if (!job->done_fence) {
 284                 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n");
 285                 goto err_free_job;
 286         }
 287
 288         job->file_priv = ivpu_file_priv_get(file_priv);
 289
 290         ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);
 291         return job;
 292
 293 err_free_job:
 294         kfree(job);
 295         return NULL;
 296 }
 297
 298 static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)
 299 {
 300         struct ivpu_job *job;
 301
 302         job = xa_erase(&vdev->submitted_jobs_xa, job_id);
 303         if (!job)
 304                 return -ENOENT;
 305
 306         if (job->file_priv->has_mmu_faults)
 307                 job_status = DRM_IVPU_JOB_STATUS_ABORTED;
 308
 309         job->bos[CMD_BUF_IDX]->job_status = job_status;
 310         dma_fence_signal(job->done_fence);
 311
 312         ivpu_dbg(vdev, JOB, "Job complete:  id %3u ctx %2d engine %d status 0x%x\n",
 313                  job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);
 314
 315         ivpu_job_destroy(job);
 316         ivpu_stop_job_timeout_detection(vdev);
 317
 318         ivpu_rpm_put(vdev);
 319         return 0;
 320 }
 321
 322 void ivpu_jobs_abort_all(struct ivpu_device *vdev)
 323 {
 324         struct ivpu_job *job;
 325         unsigned long id;
 326
 327         xa_for_each(&vdev->submitted_jobs_xa, id, job)
 328                 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED);
 329 }
 330
 331 static int ivpu_job_submit(struct ivpu_job *job)
 332 {
 333         struct ivpu_file_priv *file_priv = job->file_priv;
 334         struct ivpu_device *vdev = job->vdev;
 335         struct xa_limit job_id_range;
 336         struct ivpu_cmdq *cmdq;
 337         int ret;
 338
 339         ret = ivpu_rpm_get(vdev);
 340         if (ret < 0)
 341                 return ret;
 342
 343         mutex_lock(&file_priv->lock);
 344
 345         cmdq = ivpu_cmdq_acquire(job->file_priv, job->engine_idx);
 346         if (!cmdq) {
 347                 ivpu_warn_ratelimited(vdev, "Failed get job queue, ctx %d engine %d\n",
 348                                       file_priv->ctx.id, job->engine_idx);
 349                 ret = -EINVAL;
 350                 goto err_unlock_file_priv;
 351         }
 352
 353         job_id_range.min = FIELD_PREP(JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1));
 354         job_id_range.max = job_id_range.min | JOB_ID_JOB_MASK;
 355
 356         xa_lock(&vdev->submitted_jobs_xa);
 357         ret = __xa_alloc(&vdev->submitted_jobs_xa, &job->job_id, job, job_id_range, GFP_KERNEL);
 358         if (ret) {
 359                 ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n",
 360                          file_priv->ctx.id);
 361                 ret = -EBUSY;
 362                 goto err_unlock_submitted_jobs_xa;
 363         }
 364
 365         ret = ivpu_cmdq_push_job(cmdq, job);
 366         if (ret)
 367                 goto err_erase_xa;
 368
 369         ivpu_start_job_timeout_detection(vdev);
 370
 371         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) {
 372                 cmdq->jobq->header.head = cmdq->jobq->header.tail;
 373                 wmb(); /* Flush WC buffer for jobq header */
 374         } else {
 375                 ivpu_cmdq_ring_db(vdev, cmdq);
 376         }
 377
 378         ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d addr 0x%llx next %d\n",
 379                  job->job_id, file_priv->ctx.id, job->engine_idx,
 380                  job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);
 381
 382         xa_unlock(&vdev->submitted_jobs_xa);
 383
 384         mutex_unlock(&file_priv->lock);
 385
 386         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW))
 387                 ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);
 388
 389         return 0;
 390
 391 err_erase_xa:
 392         __xa_erase(&vdev->submitted_jobs_xa, job->job_id);
 393 err_unlock_submitted_jobs_xa:
 394         xa_unlock(&vdev->submitted_jobs_xa);
 395 err_unlock_file_priv:
 396         mutex_unlock(&file_priv->lock);
 397         ivpu_rpm_put(vdev);
 398         return ret;
 399 }
 400
 401 static int
 402 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,
 403                                 u32 buf_count, u32 commands_offset)
 404 {
 405         struct ivpu_file_priv *file_priv = file->driver_priv;
 406         struct ivpu_device *vdev = file_priv->vdev;
 407         struct ww_acquire_ctx acquire_ctx;
 408         enum dma_resv_usage usage;
 409         struct ivpu_bo *bo;
 410         int ret;
 411         u32 i;
 412
 413         for (i = 0; i < buf_count; i++) {
 414                 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]);
 415
 416                 if (!obj)
 417                         return -ENOENT;
 418
 419                 job->bos[i] = to_ivpu_bo(obj);
 420
 421                 ret = ivpu_bo_pin(job->bos[i]);
 422                 if (ret)
 423                         return ret;
 424         }
 425
 426         bo = job->bos[CMD_BUF_IDX];
 427         if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) {
 428                 ivpu_warn(vdev, "Buffer is already in use\n");
 429                 return -EBUSY;
 430         }
 431
 432         if (commands_offset >= ivpu_bo_size(bo)) {
 433                 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset);
 434                 return -EINVAL;
 435         }
 436
 437         job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;
 438
 439         ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,
 440                                         &acquire_ctx);
 441         if (ret) {
 442                 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret);
 443                 return ret;
 444         }
 445
 446         for (i = 0; i < buf_count; i++) {
 447                 ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);
 448                 if (ret) {
 449                         ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
 450                         goto unlock_reservations;
 451                 }
 452         }
 453
 454         for (i = 0; i < buf_count; i++) {
 455                 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;
 456                 dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);
 457         }
 458
 459 unlock_reservations:
 460         drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);
 461
 462         wmb(); /* Flush write combining buffers */
 463
 464         return ret;
 465 }
 466
 467 int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 468 {
 469         struct ivpu_file_priv *file_priv = file->driver_priv;
 470         struct ivpu_device *vdev = file_priv->vdev;
 471         struct drm_ivpu_submit *params = data;
 472         struct ivpu_job *job;
 473         u32 *buf_handles;
 474         int idx, ret;
 475
 476         if (params->engine > DRM_IVPU_ENGINE_COPY)
 477                 return -EINVAL;
 478
 479         if (params->priority > DRM_IVPU_JOB_PRIORITY_REALTIME)
 480                 return -EINVAL;
 481
 482         if (params->buffer_count == 0 || params->buffer_count > JOB_MAX_BUFFER_COUNT)
 483                 return -EINVAL;
 484
 485         if (!IS_ALIGNED(params->commands_offset, 8))
 486                 return -EINVAL;
 487
 488         if (!file_priv->ctx.id)
 489                 return -EINVAL;
 490
 491         if (file_priv->has_mmu_faults)
 492                 return -EBADFD;
 493
 494         buf_handles = kcalloc(params->buffer_count, sizeof(u32), GFP_KERNEL);
 495         if (!buf_handles)
 496                 return -ENOMEM;
 497
 498         ret = copy_from_user(buf_handles,
 499                              (void __user *)params->buffers_ptr,
 500                              params->buffer_count * sizeof(u32));
 501         if (ret) {
 502                 ret = -EFAULT;
 503                 goto err_free_handles;
 504         }
 505
 506         if (!drm_dev_enter(&vdev->drm, &idx)) {
 507                 ret = -ENODEV;
 508                 goto err_free_handles;
 509         }
 510
 511         ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n",
 512                  file_priv->ctx.id, params->buffer_count);
 513
 514         job = ivpu_job_create(file_priv, params->engine, params->buffer_count);
 515         if (!job) {
 516                 ivpu_err(vdev, "Failed to create job\n");
 517                 ret = -ENOMEM;
 518                 goto err_exit_dev;
 519         }
 520
 521         ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, params->buffer_count,
 522                                               params->commands_offset);
 523         if (ret) {
 524                 ivpu_err(vdev, "Failed to prepare job: %d\n", ret);
 525                 goto err_destroy_job;
 526         }
 527
 528         down_read(&vdev->pm->reset_lock);
 529         ret = ivpu_job_submit(job);
 530         up_read(&vdev->pm->reset_lock);
 531         if (ret)
 532                 goto err_signal_fence;
 533
 534         drm_dev_exit(idx);
 535         kfree(buf_handles);
 536         return ret;
 537
 538 err_signal_fence:
 539         dma_fence_signal(job->done_fence);
 540 err_destroy_job:
 541         ivpu_job_destroy(job);
 542 err_exit_dev:
 543         drm_dev_exit(idx);
 544 err_free_handles:
 545         kfree(buf_handles);
 546         return ret;
 547 }
 548
 549 static void
 550 ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
 551                        struct vpu_jsm_msg *jsm_msg)
 552 {
 553         struct vpu_ipc_msg_payload_job_done *payload;
 554         int ret;
 555
 556         if (!jsm_msg) {
 557                 ivpu_err(vdev, "IPC message has no JSM payload\n");
 558                 return;
 559         }
 560
 561         if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
 562                 ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result);
 563                 return;
 564         }
 565
 566         payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload;
 567         ret = ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status);
 568         if (!ret && !xa_empty(&vdev->submitted_jobs_xa))
 569                 ivpu_start_job_timeout_detection(vdev);
 570 }
 571
 572 void ivpu_job_done_consumer_init(struct ivpu_device *vdev)
 573 {
 574         ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer,
 575                               VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback);
 576 }
 577
 578 void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)
 579 {
 580         ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);
 581 }