drivers/accel/ivpu/ivpu_drv.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2023 Intel Corporation
   4  */
   5
   6 #include <linux/firmware.h>
   7 #include <linux/module.h>
   8 #include <linux/pci.h>
   9
  10 #include <drm/drm_accel.h>
  11 #include <drm/drm_drv.h>
  12 #include <drm/drm_file.h>
  13 #include <drm/drm_gem.h>
  14 #include <drm/drm_ioctl.h>
  15 #include <drm/drm_prime.h>
  16
  17 #include "vpu_boot_api.h"
  18 #include "ivpu_drv.h"
  19 #include "ivpu_fw.h"
  20 #include "ivpu_gem.h"
  21 #include "ivpu_hw.h"
  22 #include "ivpu_ipc.h"
  23 #include "ivpu_job.h"
  24 #include "ivpu_jsm_msg.h"
  25 #include "ivpu_mmu.h"
  26 #include "ivpu_mmu_context.h"
  27 #include "ivpu_pm.h"
  28
  29 #ifndef DRIVER_VERSION_STR
  30 #define DRIVER_VERSION_STR __stringify(DRM_IVPU_DRIVER_MAJOR) "." \
  31                            __stringify(DRM_IVPU_DRIVER_MINOR) "."
  32 #endif
  33
  34 static const struct drm_driver driver;
  35
  36 static struct lock_class_key submitted_jobs_xa_lock_class_key;
  37
  38 int ivpu_dbg_mask;
  39 module_param_named(dbg_mask, ivpu_dbg_mask, int, 0644);
  40 MODULE_PARM_DESC(dbg_mask, "Driver debug mask. See IVPU_DBG_* macros.");
  41
  42 int ivpu_test_mode;
  43 module_param_named_unsafe(test_mode, ivpu_test_mode, int, 0644);
  44 MODULE_PARM_DESC(test_mode, "Test mode: 0 - normal operation, 1 - fw unit test, 2 - null hw");
  45
  46 u8 ivpu_pll_min_ratio;
  47 module_param_named(pll_min_ratio, ivpu_pll_min_ratio, byte, 0644);
  48 MODULE_PARM_DESC(pll_min_ratio, "Minimum PLL ratio used to set VPU frequency");
  49
  50 u8 ivpu_pll_max_ratio = U8_MAX;
  51 module_param_named(pll_max_ratio, ivpu_pll_max_ratio, byte, 0644);
  52 MODULE_PARM_DESC(pll_max_ratio, "Maximum PLL ratio used to set VPU frequency");
  53
  54 struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv)
  55 {
  56         struct ivpu_device *vdev = file_priv->vdev;
  57
  58         kref_get(&file_priv->ref);
  59
  60         ivpu_dbg(vdev, KREF, "file_priv get: ctx %u refcount %u\n",
  61                  file_priv->ctx.id, kref_read(&file_priv->ref));
  62
  63         return file_priv;
  64 }
  65
  66 struct ivpu_file_priv *ivpu_file_priv_get_by_ctx_id(struct ivpu_device *vdev, unsigned long id)
  67 {
  68         struct ivpu_file_priv *file_priv;
  69
  70         xa_lock_irq(&vdev->context_xa);
  71         file_priv = xa_load(&vdev->context_xa, id);
  72         /* file_priv may still be in context_xa during file_priv_release() */
  73         if (file_priv && !kref_get_unless_zero(&file_priv->ref))
  74                 file_priv = NULL;
  75         xa_unlock_irq(&vdev->context_xa);
  76
  77         if (file_priv)
  78                 ivpu_dbg(vdev, KREF, "file_priv get by id: ctx %u refcount %u\n",
  79                          file_priv->ctx.id, kref_read(&file_priv->ref));
  80
  81         return file_priv;
  82 }
  83
  84 static void file_priv_release(struct kref *ref)
  85 {
  86         struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref);
  87         struct ivpu_device *vdev = file_priv->vdev;
  88
  89         ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id);
  90
  91         ivpu_cmdq_release_all(file_priv);
  92         ivpu_bo_remove_all_bos_from_context(&file_priv->ctx);
  93         ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
  94         drm_WARN_ON(&vdev->drm, xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv);
  95         mutex_destroy(&file_priv->lock);
  96         kfree(file_priv);
  97 }
  98
  99 void ivpu_file_priv_put(struct ivpu_file_priv **link)
 100 {
 101         struct ivpu_file_priv *file_priv = *link;
 102         struct ivpu_device *vdev = file_priv->vdev;
 103
 104         drm_WARN_ON(&vdev->drm, !file_priv);
 105
 106         ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n",
 107                  file_priv->ctx.id, kref_read(&file_priv->ref));
 108
 109         *link = NULL;
 110         kref_put(&file_priv->ref, file_priv_release);
 111 }
 112
 113 static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 114 {
 115         struct ivpu_file_priv *file_priv = file->driver_priv;
 116         struct ivpu_device *vdev = file_priv->vdev;
 117         struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
 118         struct drm_ivpu_param *args = data;
 119         int ret = 0;
 120
 121         switch (args->param) {
 122         case DRM_IVPU_PARAM_DEVICE_ID:
 123                 args->value = pdev->device;
 124                 break;
 125         case DRM_IVPU_PARAM_DEVICE_REVISION:
 126                 args->value = pdev->revision;
 127                 break;
 128         case DRM_IVPU_PARAM_PLATFORM_TYPE:
 129                 args->value = vdev->platform;
 130                 break;
 131         case DRM_IVPU_PARAM_CORE_CLOCK_RATE:
 132                 args->value = ivpu_hw_reg_pll_freq_get(vdev);
 133                 break;
 134         case DRM_IVPU_PARAM_NUM_CONTEXTS:
 135                 args->value = ivpu_get_context_count(vdev);
 136                 break;
 137         case DRM_IVPU_PARAM_CONTEXT_BASE_ADDRESS:
 138                 args->value = vdev->hw->ranges.user_low.start;
 139                 break;
 140         case DRM_IVPU_PARAM_CONTEXT_PRIORITY:
 141                 args->value = file_priv->priority;
 142                 break;
 143         case DRM_IVPU_PARAM_CONTEXT_ID:
 144                 args->value = file_priv->ctx.id;
 145                 break;
 146         case DRM_IVPU_PARAM_FW_API_VERSION:
 147                 if (args->index < VPU_FW_API_VER_NUM) {
 148                         struct vpu_firmware_header *fw_hdr;
 149
 150                         fw_hdr = (struct vpu_firmware_header *)vdev->fw->file->data;
 151                         args->value = fw_hdr->api_version[args->index];
 152                 } else {
 153                         ret = -EINVAL;
 154                 }
 155                 break;
 156         case DRM_IVPU_PARAM_ENGINE_HEARTBEAT:
 157                 ret = ivpu_jsm_get_heartbeat(vdev, args->index, &args->value);
 158                 break;
 159         case DRM_IVPU_PARAM_UNIQUE_INFERENCE_ID:
 160                 args->value = (u64)atomic64_inc_return(&vdev->unique_id_counter);
 161                 break;
 162         case DRM_IVPU_PARAM_TILE_CONFIG:
 163                 args->value = vdev->hw->tile_fuse;
 164                 break;
 165         case DRM_IVPU_PARAM_SKU:
 166                 args->value = vdev->hw->sku;
 167                 break;
 168         default:
 169                 ret = -EINVAL;
 170                 break;
 171         }
 172
 173         return ret;
 174 }
 175
 176 static int ivpu_set_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 177 {
 178         struct ivpu_file_priv *file_priv = file->driver_priv;
 179         struct drm_ivpu_param *args = data;
 180         int ret = 0;
 181
 182         switch (args->param) {
 183         case DRM_IVPU_PARAM_CONTEXT_PRIORITY:
 184                 if (args->value <= DRM_IVPU_CONTEXT_PRIORITY_REALTIME)
 185                         file_priv->priority = args->value;
 186                 else
 187                         ret = -EINVAL;
 188                 break;
 189         default:
 190                 ret = -EINVAL;
 191         }
 192
 193         return ret;
 194 }
 195
 196 static int ivpu_open(struct drm_device *dev, struct drm_file *file)
 197 {
 198         struct ivpu_device *vdev = to_ivpu_device(dev);
 199         struct ivpu_file_priv *file_priv;
 200         u32 ctx_id;
 201         void *old;
 202         int ret;
 203
 204         ret = xa_alloc_irq(&vdev->context_xa, &ctx_id, NULL, vdev->context_xa_limit, GFP_KERNEL);
 205         if (ret) {
 206                 ivpu_err(vdev, "Failed to allocate context id: %d\n", ret);
 207                 return ret;
 208         }
 209
 210         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
 211         if (!file_priv) {
 212                 ret = -ENOMEM;
 213                 goto err_xa_erase;
 214         }
 215
 216         file_priv->vdev = vdev;
 217         file_priv->priority = DRM_IVPU_CONTEXT_PRIORITY_NORMAL;
 218         kref_init(&file_priv->ref);
 219         mutex_init(&file_priv->lock);
 220
 221         ret = ivpu_mmu_user_context_init(vdev, &file_priv->ctx, ctx_id);
 222         if (ret)
 223                 goto err_mutex_destroy;
 224
 225         old = xa_store_irq(&vdev->context_xa, ctx_id, file_priv, GFP_KERNEL);
 226         if (xa_is_err(old)) {
 227                 ret = xa_err(old);
 228                 ivpu_err(vdev, "Failed to store context %u: %d\n", ctx_id, ret);
 229                 goto err_ctx_fini;
 230         }
 231
 232         ivpu_dbg(vdev, FILE, "file_priv create: ctx %u process %s pid %d\n",
 233                  ctx_id, current->comm, task_pid_nr(current));
 234
 235         file->driver_priv = file_priv;
 236         return 0;
 237
 238 err_ctx_fini:
 239         ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
 240 err_mutex_destroy:
 241         mutex_destroy(&file_priv->lock);
 242         kfree(file_priv);
 243 err_xa_erase:
 244         xa_erase_irq(&vdev->context_xa, ctx_id);
 245         return ret;
 246 }
 247
 248 static void ivpu_postclose(struct drm_device *dev, struct drm_file *file)
 249 {
 250         struct ivpu_file_priv *file_priv = file->driver_priv;
 251         struct ivpu_device *vdev = to_ivpu_device(dev);
 252
 253         ivpu_dbg(vdev, FILE, "file_priv close: ctx %u process %s pid %d\n",
 254                  file_priv->ctx.id, current->comm, task_pid_nr(current));
 255
 256         ivpu_file_priv_put(&file_priv);
 257 }
 258
 259 static const struct drm_ioctl_desc ivpu_drm_ioctls[] = {
 260         DRM_IOCTL_DEF_DRV(IVPU_GET_PARAM, ivpu_get_param_ioctl, 0),
 261         DRM_IOCTL_DEF_DRV(IVPU_SET_PARAM, ivpu_set_param_ioctl, 0),
 262         DRM_IOCTL_DEF_DRV(IVPU_BO_CREATE, ivpu_bo_create_ioctl, 0),
 263         DRM_IOCTL_DEF_DRV(IVPU_BO_INFO, ivpu_bo_info_ioctl, 0),
 264         DRM_IOCTL_DEF_DRV(IVPU_SUBMIT, ivpu_submit_ioctl, 0),
 265         DRM_IOCTL_DEF_DRV(IVPU_BO_WAIT, ivpu_bo_wait_ioctl, 0),
 266 };
 267
 268 static int ivpu_wait_for_ready(struct ivpu_device *vdev)
 269 {
 270         struct ivpu_ipc_consumer cons;
 271         struct ivpu_ipc_hdr ipc_hdr;
 272         unsigned long timeout;
 273         int ret;
 274
 275         if (ivpu_test_mode == IVPU_TEST_MODE_FW_TEST)
 276                 return 0;
 277
 278         ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG);
 279
 280         timeout = jiffies + msecs_to_jiffies(vdev->timeout.boot);
 281         while (1) {
 282                 ret = ivpu_ipc_irq_handler(vdev);
 283                 if (ret)
 284                         break;
 285                 ret = ivpu_ipc_receive(vdev, &cons, &ipc_hdr, NULL, 0);
 286                 if (ret != -ETIMEDOUT || time_after_eq(jiffies, timeout))
 287                         break;
 288
 289                 cond_resched();
 290         }
 291
 292         ivpu_ipc_consumer_del(vdev, &cons);
 293
 294         if (!ret && ipc_hdr.data_addr != IVPU_IPC_BOOT_MSG_DATA_ADDR) {
 295                 ivpu_err(vdev, "Invalid VPU ready message: 0x%x\n",
 296                          ipc_hdr.data_addr);
 297                 return -EIO;
 298         }
 299
 300         if (!ret)
 301                 ivpu_info(vdev, "VPU ready message received successfully\n");
 302         else
 303                 ivpu_hw_diagnose_failure(vdev);
 304
 305         return ret;
 306 }
 307
 308 /**
 309  * ivpu_boot() - Start VPU firmware
 310  * @vdev: VPU device
 311  *
 312  * This function is paired with ivpu_shutdown() but it doesn't power up the
 313  * VPU because power up has to be called very early in ivpu_probe().
 314  */
 315 int ivpu_boot(struct ivpu_device *vdev)
 316 {
 317         int ret;
 318
 319         /* Update boot params located at first 4KB of FW memory */
 320         ivpu_fw_boot_params_setup(vdev, vdev->fw->mem->kvaddr);
 321
 322         ret = ivpu_hw_boot_fw(vdev);
 323         if (ret) {
 324                 ivpu_err(vdev, "Failed to start the firmware: %d\n", ret);
 325                 return ret;
 326         }
 327
 328         ret = ivpu_wait_for_ready(vdev);
 329         if (ret) {
 330                 ivpu_err(vdev, "Failed to boot the firmware: %d\n", ret);
 331                 return ret;
 332         }
 333
 334         ivpu_hw_irq_clear(vdev);
 335         enable_irq(vdev->irq);
 336         ivpu_hw_irq_enable(vdev);
 337         ivpu_ipc_enable(vdev);
 338         return 0;
 339 }
 340
 341 int ivpu_shutdown(struct ivpu_device *vdev)
 342 {
 343         int ret;
 344
 345         ivpu_hw_irq_disable(vdev);
 346         disable_irq(vdev->irq);
 347         ivpu_ipc_disable(vdev);
 348         ivpu_mmu_disable(vdev);
 349
 350         ret = ivpu_hw_power_down(vdev);
 351         if (ret)
 352                 ivpu_warn(vdev, "Failed to power down HW: %d\n", ret);
 353
 354         return ret;
 355 }
 356
 357 static const struct file_operations ivpu_fops = {
 358         .owner          = THIS_MODULE,
 359         DRM_ACCEL_FOPS,
 360 };
 361
 362 static const struct drm_driver driver = {
 363         .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
 364
 365         .open = ivpu_open,
 366         .postclose = ivpu_postclose,
 367         .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 368         .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 369         .gem_prime_import = ivpu_gem_prime_import,
 370         .gem_prime_mmap = drm_gem_prime_mmap,
 371
 372         .ioctls = ivpu_drm_ioctls,
 373         .num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls),
 374         .fops = &ivpu_fops,
 375
 376         .name = DRIVER_NAME,
 377         .desc = DRIVER_DESC,
 378         .date = DRIVER_DATE,
 379         .major = DRM_IVPU_DRIVER_MAJOR,
 380         .minor = DRM_IVPU_DRIVER_MINOR,
 381 };
 382
 383 static int ivpu_irq_init(struct ivpu_device *vdev)
 384 {
 385         struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
 386         int ret;
 387
 388         ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX);
 389         if (ret < 0) {
 390                 ivpu_err(vdev, "Failed to allocate a MSI IRQ: %d\n", ret);
 391                 return ret;
 392         }
 393
 394         vdev->irq = pci_irq_vector(pdev, 0);
 395
 396         ret = devm_request_irq(vdev->drm.dev, vdev->irq, vdev->hw->ops->irq_handler,
 397                                IRQF_NO_AUTOEN, DRIVER_NAME, vdev);
 398         if (ret)
 399                 ivpu_err(vdev, "Failed to request an IRQ %d\n", ret);
 400
 401         return ret;
 402 }
 403
 404 static int ivpu_pci_init(struct ivpu_device *vdev)
 405 {
 406         struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
 407         struct resource *bar0 = &pdev->resource[0];
 408         struct resource *bar4 = &pdev->resource[4];
 409         int ret;
 410
 411         ivpu_dbg(vdev, MISC, "Mapping BAR0 (RegV) %pR\n", bar0);
 412         vdev->regv = devm_ioremap_resource(vdev->drm.dev, bar0);
 413         if (IS_ERR(vdev->regv)) {
 414                 ivpu_err(vdev, "Failed to map bar 0: %pe\n", vdev->regv);
 415                 return PTR_ERR(vdev->regv);
 416         }
 417
 418         ivpu_dbg(vdev, MISC, "Mapping BAR4 (RegB) %pR\n", bar4);
 419         vdev->regb = devm_ioremap_resource(vdev->drm.dev, bar4);
 420         if (IS_ERR(vdev->regb)) {
 421                 ivpu_err(vdev, "Failed to map bar 4: %pe\n", vdev->regb);
 422                 return PTR_ERR(vdev->regb);
 423         }
 424
 425         ret = dma_set_mask_and_coherent(vdev->drm.dev, DMA_BIT_MASK(38));
 426         if (ret) {
 427                 ivpu_err(vdev, "Failed to set DMA mask: %d\n", ret);
 428                 return ret;
 429         }
 430
 431         /* Clear any pending errors */
 432         pcie_capability_clear_word(pdev, PCI_EXP_DEVSTA, 0x3f);
 433
 434         ret = pcim_enable_device(pdev);
 435         if (ret) {
 436                 ivpu_err(vdev, "Failed to enable PCI device: %d\n", ret);
 437                 return ret;
 438         }
 439
 440         pci_set_master(pdev);
 441
 442         return 0;
 443 }
 444
 445 static int ivpu_dev_init(struct ivpu_device *vdev)
 446 {
 447         int ret;
 448
 449         vdev->hw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->hw), GFP_KERNEL);
 450         if (!vdev->hw)
 451                 return -ENOMEM;
 452
 453         vdev->mmu = drmm_kzalloc(&vdev->drm, sizeof(*vdev->mmu), GFP_KERNEL);
 454         if (!vdev->mmu)
 455                 return -ENOMEM;
 456
 457         vdev->fw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->fw), GFP_KERNEL);
 458         if (!vdev->fw)
 459                 return -ENOMEM;
 460
 461         vdev->ipc = drmm_kzalloc(&vdev->drm, sizeof(*vdev->ipc), GFP_KERNEL);
 462         if (!vdev->ipc)
 463                 return -ENOMEM;
 464
 465         vdev->pm = drmm_kzalloc(&vdev->drm, sizeof(*vdev->pm), GFP_KERNEL);
 466         if (!vdev->pm)
 467                 return -ENOMEM;
 468
 469         vdev->hw->ops = &ivpu_hw_mtl_ops;
 470         vdev->platform = IVPU_PLATFORM_INVALID;
 471         vdev->context_xa_limit.min = IVPU_GLOBAL_CONTEXT_MMU_SSID + 1;
 472         vdev->context_xa_limit.max = IVPU_CONTEXT_LIMIT;
 473         atomic64_set(&vdev->unique_id_counter, 0);
 474         xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC);
 475         xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1);
 476         lockdep_set_class(&vdev->submitted_jobs_xa.xa_lock, &submitted_jobs_xa_lock_class_key);
 477
 478         ret = ivpu_pci_init(vdev);
 479         if (ret) {
 480                 ivpu_err(vdev, "Failed to initialize PCI device: %d\n", ret);
 481                 goto err_xa_destroy;
 482         }
 483
 484         ret = ivpu_irq_init(vdev);
 485         if (ret) {
 486                 ivpu_err(vdev, "Failed to initialize IRQs: %d\n", ret);
 487                 goto err_xa_destroy;
 488         }
 489
 490         /* Init basic HW info based on buttress registers which are accessible before power up */
 491         ret = ivpu_hw_info_init(vdev);
 492         if (ret) {
 493                 ivpu_err(vdev, "Failed to initialize HW info: %d\n", ret);
 494                 goto err_xa_destroy;
 495         }
 496
 497         /* Power up early so the rest of init code can access VPU registers */
 498         ret = ivpu_hw_power_up(vdev);
 499         if (ret) {
 500                 ivpu_err(vdev, "Failed to power up HW: %d\n", ret);
 501                 goto err_xa_destroy;
 502         }
 503
 504         ret = ivpu_mmu_global_context_init(vdev);
 505         if (ret) {
 506                 ivpu_err(vdev, "Failed to initialize global MMU context: %d\n", ret);
 507                 goto err_power_down;
 508         }
 509
 510         ret = ivpu_mmu_init(vdev);
 511         if (ret) {
 512                 ivpu_err(vdev, "Failed to initialize MMU device: %d\n", ret);
 513                 goto err_mmu_gctx_fini;
 514         }
 515
 516         ret = ivpu_fw_init(vdev);
 517         if (ret) {
 518                 ivpu_err(vdev, "Failed to initialize firmware: %d\n", ret);
 519                 goto err_mmu_gctx_fini;
 520         }
 521
 522         ret = ivpu_ipc_init(vdev);
 523         if (ret) {
 524                 ivpu_err(vdev, "Failed to initialize IPC: %d\n", ret);
 525                 goto err_fw_fini;
 526         }
 527
 528         ret = ivpu_pm_init(vdev);
 529         if (ret) {
 530                 ivpu_err(vdev, "Failed to initialize PM: %d\n", ret);
 531                 goto err_ipc_fini;
 532         }
 533
 534         ret = ivpu_job_done_thread_init(vdev);
 535         if (ret) {
 536                 ivpu_err(vdev, "Failed to initialize job done thread: %d\n", ret);
 537                 goto err_ipc_fini;
 538         }
 539
 540         ret = ivpu_fw_load(vdev);
 541         if (ret) {
 542                 ivpu_err(vdev, "Failed to load firmware: %d\n", ret);
 543                 goto err_job_done_thread_fini;
 544         }
 545
 546         ret = ivpu_boot(vdev);
 547         if (ret) {
 548                 ivpu_err(vdev, "Failed to boot: %d\n", ret);
 549                 goto err_job_done_thread_fini;
 550         }
 551
 552         ivpu_pm_enable(vdev);
 553
 554         return 0;
 555
 556 err_job_done_thread_fini:
 557         ivpu_job_done_thread_fini(vdev);
 558 err_ipc_fini:
 559         ivpu_ipc_fini(vdev);
 560 err_fw_fini:
 561         ivpu_fw_fini(vdev);
 562 err_mmu_gctx_fini:
 563         ivpu_mmu_global_context_fini(vdev);
 564 err_power_down:
 565         ivpu_hw_power_down(vdev);
 566 err_xa_destroy:
 567         xa_destroy(&vdev->submitted_jobs_xa);
 568         xa_destroy(&vdev->context_xa);
 569         return ret;
 570 }
 571
 572 static void ivpu_dev_fini(struct ivpu_device *vdev)
 573 {
 574         ivpu_pm_disable(vdev);
 575         ivpu_shutdown(vdev);
 576         ivpu_job_done_thread_fini(vdev);
 577         ivpu_ipc_fini(vdev);
 578         ivpu_fw_fini(vdev);
 579         ivpu_mmu_global_context_fini(vdev);
 580
 581         drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa));
 582         xa_destroy(&vdev->submitted_jobs_xa);
 583         drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa));
 584         xa_destroy(&vdev->context_xa);
 585 }
 586
 587 static struct pci_device_id ivpu_pci_ids[] = {
 588         { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_MTL) },
 589         { }
 590 };
 591 MODULE_DEVICE_TABLE(pci, ivpu_pci_ids);
 592
 593 static int ivpu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 594 {
 595         struct ivpu_device *vdev;
 596         int ret;
 597
 598         vdev = devm_drm_dev_alloc(&pdev->dev, &driver, struct ivpu_device, drm);
 599         if (IS_ERR(vdev))
 600                 return PTR_ERR(vdev);
 601
 602         pci_set_drvdata(pdev, vdev);
 603
 604         ret = ivpu_dev_init(vdev);
 605         if (ret) {
 606                 dev_err(&pdev->dev, "Failed to initialize VPU device: %d\n", ret);
 607                 return ret;
 608         }
 609
 610         ret = drm_dev_register(&vdev->drm, 0);
 611         if (ret) {
 612                 dev_err(&pdev->dev, "Failed to register DRM device: %d\n", ret);
 613                 ivpu_dev_fini(vdev);
 614         }
 615
 616         return ret;
 617 }
 618
 619 static void ivpu_remove(struct pci_dev *pdev)
 620 {
 621         struct ivpu_device *vdev = pci_get_drvdata(pdev);
 622
 623         drm_dev_unregister(&vdev->drm);
 624         ivpu_dev_fini(vdev);
 625 }
 626
 627 static const struct dev_pm_ops ivpu_drv_pci_pm = {
 628         SET_SYSTEM_SLEEP_PM_OPS(ivpu_pm_suspend_cb, ivpu_pm_resume_cb)
 629         SET_RUNTIME_PM_OPS(ivpu_pm_runtime_suspend_cb, ivpu_pm_runtime_resume_cb, NULL)
 630 };
 631
 632 static const struct pci_error_handlers ivpu_drv_pci_err = {
 633         .reset_prepare = ivpu_pm_reset_prepare_cb,
 634         .reset_done = ivpu_pm_reset_done_cb,
 635 };
 636
 637 static struct pci_driver ivpu_pci_driver = {
 638         .name = KBUILD_MODNAME,
 639         .id_table = ivpu_pci_ids,
 640         .probe = ivpu_probe,
 641         .remove = ivpu_remove,
 642         .driver = {
 643                 .pm = &ivpu_drv_pci_pm,
 644         },
 645         .err_handler = &ivpu_drv_pci_err,
 646 };
 647
 648 module_pci_driver(ivpu_pci_driver);
 649
 650 MODULE_AUTHOR("Intel Corporation");
 651 MODULE_DESCRIPTION(DRIVER_DESC);
 652 MODULE_LICENSE("GPL and additional rights");
 653 MODULE_VERSION(DRIVER_VERSION_STR);