Commit | Line | Data |
---|---|---|
35b13763 JL |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Copyright (C) 2020-2023 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include <linux/firmware.h> | |
7 | #include <linux/module.h> | |
8 | #include <linux/pci.h> | |
9 | ||
10 | #include <drm/drm_accel.h> | |
11 | #include <drm/drm_drv.h> | |
12 | #include <drm/drm_file.h> | |
13 | #include <drm/drm_gem.h> | |
14 | #include <drm/drm_ioctl.h> | |
647371a6 | 15 | #include <drm/drm_prime.h> |
35b13763 | 16 | |
02d5b0aa | 17 | #include "vpu_boot_api.h" |
35b13763 | 18 | #include "ivpu_drv.h" |
02d5b0aa | 19 | #include "ivpu_fw.h" |
647371a6 | 20 | #include "ivpu_gem.h" |
35b13763 | 21 | #include "ivpu_hw.h" |
5d7422cf | 22 | #include "ivpu_ipc.h" |
cd727221 | 23 | #include "ivpu_job.h" |
02d5b0aa | 24 | #include "ivpu_jsm_msg.h" |
263b2ba5 JL |
25 | #include "ivpu_mmu.h" |
26 | #include "ivpu_mmu_context.h" | |
852be13f | 27 | #include "ivpu_pm.h" |
35b13763 JL |
28 | |
29 | #ifndef DRIVER_VERSION_STR | |
30 | #define DRIVER_VERSION_STR __stringify(DRM_IVPU_DRIVER_MAJOR) "." \ | |
31 | __stringify(DRM_IVPU_DRIVER_MINOR) "." | |
32 | #endif | |
33 | ||
34 | static const struct drm_driver driver; | |
35 | ||
cd727221 JL |
36 | static struct lock_class_key submitted_jobs_xa_lock_class_key; |
37 | ||
35b13763 JL |
38 | int ivpu_dbg_mask; |
39 | module_param_named(dbg_mask, ivpu_dbg_mask, int, 0644); | |
40 | MODULE_PARM_DESC(dbg_mask, "Driver debug mask. See IVPU_DBG_* macros."); | |
41 | ||
02d5b0aa JL |
42 | int ivpu_test_mode; |
43 | module_param_named_unsafe(test_mode, ivpu_test_mode, int, 0644); | |
44 | MODULE_PARM_DESC(test_mode, "Test mode: 0 - normal operation, 1 - fw unit test, 2 - null hw"); | |
45 | ||
35b13763 JL |
46 | u8 ivpu_pll_min_ratio; |
47 | module_param_named(pll_min_ratio, ivpu_pll_min_ratio, byte, 0644); | |
48 | MODULE_PARM_DESC(pll_min_ratio, "Minimum PLL ratio used to set VPU frequency"); | |
49 | ||
50 | u8 ivpu_pll_max_ratio = U8_MAX; | |
51 | module_param_named(pll_max_ratio, ivpu_pll_max_ratio, byte, 0644); | |
52 | MODULE_PARM_DESC(pll_max_ratio, "Maximum PLL ratio used to set VPU frequency"); | |
53 | ||
54 | struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv) | |
55 | { | |
263b2ba5 JL |
56 | struct ivpu_device *vdev = file_priv->vdev; |
57 | ||
35b13763 | 58 | kref_get(&file_priv->ref); |
263b2ba5 JL |
59 | |
60 | ivpu_dbg(vdev, KREF, "file_priv get: ctx %u refcount %u\n", | |
61 | file_priv->ctx.id, kref_read(&file_priv->ref)); | |
62 | ||
35b13763 JL |
63 | return file_priv; |
64 | } | |
65 | ||
647371a6 JL |
66 | struct ivpu_file_priv *ivpu_file_priv_get_by_ctx_id(struct ivpu_device *vdev, unsigned long id) |
67 | { | |
68 | struct ivpu_file_priv *file_priv; | |
69 | ||
70 | xa_lock_irq(&vdev->context_xa); | |
71 | file_priv = xa_load(&vdev->context_xa, id); | |
72 | /* file_priv may still be in context_xa during file_priv_release() */ | |
73 | if (file_priv && !kref_get_unless_zero(&file_priv->ref)) | |
74 | file_priv = NULL; | |
75 | xa_unlock_irq(&vdev->context_xa); | |
76 | ||
77 | if (file_priv) | |
78 | ivpu_dbg(vdev, KREF, "file_priv get by id: ctx %u refcount %u\n", | |
79 | file_priv->ctx.id, kref_read(&file_priv->ref)); | |
80 | ||
81 | return file_priv; | |
82 | } | |
83 | ||
35b13763 JL |
84 | static void file_priv_release(struct kref *ref) |
85 | { | |
86 | struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref); | |
263b2ba5 | 87 | struct ivpu_device *vdev = file_priv->vdev; |
35b13763 | 88 | |
263b2ba5 JL |
89 | ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id); |
90 | ||
cd727221 JL |
91 | ivpu_cmdq_release_all(file_priv); |
92 | ivpu_bo_remove_all_bos_from_context(&file_priv->ctx); | |
dffaa98c | 93 | ivpu_jsm_context_release(vdev, file_priv->ctx.id); |
263b2ba5 | 94 | ivpu_mmu_user_context_fini(vdev, &file_priv->ctx); |
647371a6 | 95 | drm_WARN_ON(&vdev->drm, xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv); |
cd727221 | 96 | mutex_destroy(&file_priv->lock); |
35b13763 JL |
97 | kfree(file_priv); |
98 | } | |
99 | ||
100 | void ivpu_file_priv_put(struct ivpu_file_priv **link) | |
101 | { | |
102 | struct ivpu_file_priv *file_priv = *link; | |
263b2ba5 | 103 | struct ivpu_device *vdev = file_priv->vdev; |
35b13763 | 104 | |
647371a6 | 105 | drm_WARN_ON(&vdev->drm, !file_priv); |
35b13763 | 106 | |
263b2ba5 JL |
107 | ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n", |
108 | file_priv->ctx.id, kref_read(&file_priv->ref)); | |
109 | ||
35b13763 JL |
110 | *link = NULL; |
111 | kref_put(&file_priv->ref, file_priv_release); | |
112 | } | |
113 | ||
114 | static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file) | |
115 | { | |
116 | struct ivpu_file_priv *file_priv = file->driver_priv; | |
117 | struct ivpu_device *vdev = file_priv->vdev; | |
118 | struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); | |
119 | struct drm_ivpu_param *args = data; | |
120 | int ret = 0; | |
121 | ||
122 | switch (args->param) { | |
123 | case DRM_IVPU_PARAM_DEVICE_ID: | |
124 | args->value = pdev->device; | |
125 | break; | |
126 | case DRM_IVPU_PARAM_DEVICE_REVISION: | |
127 | args->value = pdev->revision; | |
128 | break; | |
129 | case DRM_IVPU_PARAM_PLATFORM_TYPE: | |
130 | args->value = vdev->platform; | |
131 | break; | |
132 | case DRM_IVPU_PARAM_CORE_CLOCK_RATE: | |
133 | args->value = ivpu_hw_reg_pll_freq_get(vdev); | |
134 | break; | |
135 | case DRM_IVPU_PARAM_NUM_CONTEXTS: | |
136 | args->value = ivpu_get_context_count(vdev); | |
137 | break; | |
138 | case DRM_IVPU_PARAM_CONTEXT_BASE_ADDRESS: | |
139 | args->value = vdev->hw->ranges.user_low.start; | |
140 | break; | |
141 | case DRM_IVPU_PARAM_CONTEXT_PRIORITY: | |
142 | args->value = file_priv->priority; | |
143 | break; | |
263b2ba5 JL |
144 | case DRM_IVPU_PARAM_CONTEXT_ID: |
145 | args->value = file_priv->ctx.id; | |
146 | break; | |
02d5b0aa JL |
147 | case DRM_IVPU_PARAM_FW_API_VERSION: |
148 | if (args->index < VPU_FW_API_VER_NUM) { | |
149 | struct vpu_firmware_header *fw_hdr; | |
150 | ||
151 | fw_hdr = (struct vpu_firmware_header *)vdev->fw->file->data; | |
152 | args->value = fw_hdr->api_version[args->index]; | |
153 | } else { | |
154 | ret = -EINVAL; | |
155 | } | |
156 | break; | |
157 | case DRM_IVPU_PARAM_ENGINE_HEARTBEAT: | |
158 | ret = ivpu_jsm_get_heartbeat(vdev, args->index, &args->value); | |
159 | break; | |
160 | case DRM_IVPU_PARAM_UNIQUE_INFERENCE_ID: | |
161 | args->value = (u64)atomic64_inc_return(&vdev->unique_id_counter); | |
162 | break; | |
163 | case DRM_IVPU_PARAM_TILE_CONFIG: | |
164 | args->value = vdev->hw->tile_fuse; | |
165 | break; | |
166 | case DRM_IVPU_PARAM_SKU: | |
167 | args->value = vdev->hw->sku; | |
168 | break; | |
35b13763 JL |
169 | default: |
170 | ret = -EINVAL; | |
171 | break; | |
172 | } | |
173 | ||
174 | return ret; | |
175 | } | |
176 | ||
177 | static int ivpu_set_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file) | |
178 | { | |
179 | struct ivpu_file_priv *file_priv = file->driver_priv; | |
180 | struct drm_ivpu_param *args = data; | |
181 | int ret = 0; | |
182 | ||
183 | switch (args->param) { | |
184 | case DRM_IVPU_PARAM_CONTEXT_PRIORITY: | |
185 | if (args->value <= DRM_IVPU_CONTEXT_PRIORITY_REALTIME) | |
186 | file_priv->priority = args->value; | |
187 | else | |
188 | ret = -EINVAL; | |
189 | break; | |
190 | default: | |
191 | ret = -EINVAL; | |
192 | } | |
193 | ||
194 | return ret; | |
195 | } | |
196 | ||
197 | static int ivpu_open(struct drm_device *dev, struct drm_file *file) | |
198 | { | |
199 | struct ivpu_device *vdev = to_ivpu_device(dev); | |
200 | struct ivpu_file_priv *file_priv; | |
263b2ba5 JL |
201 | u32 ctx_id; |
202 | void *old; | |
203 | int ret; | |
204 | ||
205 | ret = xa_alloc_irq(&vdev->context_xa, &ctx_id, NULL, vdev->context_xa_limit, GFP_KERNEL); | |
206 | if (ret) { | |
207 | ivpu_err(vdev, "Failed to allocate context id: %d\n", ret); | |
208 | return ret; | |
209 | } | |
35b13763 JL |
210 | |
211 | file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); | |
263b2ba5 JL |
212 | if (!file_priv) { |
213 | ret = -ENOMEM; | |
214 | goto err_xa_erase; | |
215 | } | |
35b13763 JL |
216 | |
217 | file_priv->vdev = vdev; | |
218 | file_priv->priority = DRM_IVPU_CONTEXT_PRIORITY_NORMAL; | |
219 | kref_init(&file_priv->ref); | |
cd727221 | 220 | mutex_init(&file_priv->lock); |
35b13763 | 221 | |
263b2ba5 JL |
222 | ret = ivpu_mmu_user_context_init(vdev, &file_priv->ctx, ctx_id); |
223 | if (ret) | |
cd727221 | 224 | goto err_mutex_destroy; |
263b2ba5 JL |
225 | |
226 | old = xa_store_irq(&vdev->context_xa, ctx_id, file_priv, GFP_KERNEL); | |
227 | if (xa_is_err(old)) { | |
228 | ret = xa_err(old); | |
229 | ivpu_err(vdev, "Failed to store context %u: %d\n", ctx_id, ret); | |
230 | goto err_ctx_fini; | |
231 | } | |
232 | ||
233 | ivpu_dbg(vdev, FILE, "file_priv create: ctx %u process %s pid %d\n", | |
234 | ctx_id, current->comm, task_pid_nr(current)); | |
235 | ||
35b13763 JL |
236 | file->driver_priv = file_priv; |
237 | return 0; | |
263b2ba5 JL |
238 | |
239 | err_ctx_fini: | |
240 | ivpu_mmu_user_context_fini(vdev, &file_priv->ctx); | |
cd727221 JL |
241 | err_mutex_destroy: |
242 | mutex_destroy(&file_priv->lock); | |
263b2ba5 JL |
243 | kfree(file_priv); |
244 | err_xa_erase: | |
245 | xa_erase_irq(&vdev->context_xa, ctx_id); | |
246 | return ret; | |
35b13763 JL |
247 | } |
248 | ||
249 | static void ivpu_postclose(struct drm_device *dev, struct drm_file *file) | |
250 | { | |
251 | struct ivpu_file_priv *file_priv = file->driver_priv; | |
263b2ba5 JL |
252 | struct ivpu_device *vdev = to_ivpu_device(dev); |
253 | ||
254 | ivpu_dbg(vdev, FILE, "file_priv close: ctx %u process %s pid %d\n", | |
255 | file_priv->ctx.id, current->comm, task_pid_nr(current)); | |
35b13763 JL |
256 | |
257 | ivpu_file_priv_put(&file_priv); | |
258 | } | |
259 | ||
260 | static const struct drm_ioctl_desc ivpu_drm_ioctls[] = { | |
261 | DRM_IOCTL_DEF_DRV(IVPU_GET_PARAM, ivpu_get_param_ioctl, 0), | |
262 | DRM_IOCTL_DEF_DRV(IVPU_SET_PARAM, ivpu_set_param_ioctl, 0), | |
647371a6 JL |
263 | DRM_IOCTL_DEF_DRV(IVPU_BO_CREATE, ivpu_bo_create_ioctl, 0), |
264 | DRM_IOCTL_DEF_DRV(IVPU_BO_INFO, ivpu_bo_info_ioctl, 0), | |
cd727221 JL |
265 | DRM_IOCTL_DEF_DRV(IVPU_SUBMIT, ivpu_submit_ioctl, 0), |
266 | DRM_IOCTL_DEF_DRV(IVPU_BO_WAIT, ivpu_bo_wait_ioctl, 0), | |
35b13763 JL |
267 | }; |
268 | ||
02d5b0aa JL |
269 | static int ivpu_wait_for_ready(struct ivpu_device *vdev) |
270 | { | |
271 | struct ivpu_ipc_consumer cons; | |
272 | struct ivpu_ipc_hdr ipc_hdr; | |
273 | unsigned long timeout; | |
274 | int ret; | |
275 | ||
276 | if (ivpu_test_mode == IVPU_TEST_MODE_FW_TEST) | |
277 | return 0; | |
278 | ||
279 | ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG); | |
280 | ||
281 | timeout = jiffies + msecs_to_jiffies(vdev->timeout.boot); | |
282 | while (1) { | |
283 | ret = ivpu_ipc_irq_handler(vdev); | |
284 | if (ret) | |
285 | break; | |
286 | ret = ivpu_ipc_receive(vdev, &cons, &ipc_hdr, NULL, 0); | |
287 | if (ret != -ETIMEDOUT || time_after_eq(jiffies, timeout)) | |
288 | break; | |
289 | ||
290 | cond_resched(); | |
291 | } | |
292 | ||
293 | ivpu_ipc_consumer_del(vdev, &cons); | |
294 | ||
295 | if (!ret && ipc_hdr.data_addr != IVPU_IPC_BOOT_MSG_DATA_ADDR) { | |
296 | ivpu_err(vdev, "Invalid VPU ready message: 0x%x\n", | |
297 | ipc_hdr.data_addr); | |
298 | return -EIO; | |
299 | } | |
300 | ||
301 | if (!ret) | |
302 | ivpu_info(vdev, "VPU ready message received successfully\n"); | |
303 | else | |
304 | ivpu_hw_diagnose_failure(vdev); | |
305 | ||
306 | return ret; | |
307 | } | |
308 | ||
309 | /** | |
310 | * ivpu_boot() - Start VPU firmware | |
311 | * @vdev: VPU device | |
312 | * | |
313 | * This function is paired with ivpu_shutdown() but it doesn't power up the | |
314 | * VPU because power up has to be called very early in ivpu_probe(). | |
315 | */ | |
316 | int ivpu_boot(struct ivpu_device *vdev) | |
317 | { | |
318 | int ret; | |
319 | ||
320 | /* Update boot params located at first 4KB of FW memory */ | |
321 | ivpu_fw_boot_params_setup(vdev, vdev->fw->mem->kvaddr); | |
322 | ||
323 | ret = ivpu_hw_boot_fw(vdev); | |
324 | if (ret) { | |
325 | ivpu_err(vdev, "Failed to start the firmware: %d\n", ret); | |
326 | return ret; | |
327 | } | |
328 | ||
329 | ret = ivpu_wait_for_ready(vdev); | |
330 | if (ret) { | |
331 | ivpu_err(vdev, "Failed to boot the firmware: %d\n", ret); | |
332 | return ret; | |
333 | } | |
334 | ||
335 | ivpu_hw_irq_clear(vdev); | |
336 | enable_irq(vdev->irq); | |
337 | ivpu_hw_irq_enable(vdev); | |
338 | ivpu_ipc_enable(vdev); | |
339 | return 0; | |
340 | } | |
341 | ||
35b13763 JL |
342 | int ivpu_shutdown(struct ivpu_device *vdev) |
343 | { | |
344 | int ret; | |
345 | ||
346 | ivpu_hw_irq_disable(vdev); | |
02d5b0aa | 347 | disable_irq(vdev->irq); |
5d7422cf | 348 | ivpu_ipc_disable(vdev); |
263b2ba5 | 349 | ivpu_mmu_disable(vdev); |
35b13763 JL |
350 | |
351 | ret = ivpu_hw_power_down(vdev); | |
352 | if (ret) | |
353 | ivpu_warn(vdev, "Failed to power down HW: %d\n", ret); | |
354 | ||
355 | return ret; | |
356 | } | |
357 | ||
358 | static const struct file_operations ivpu_fops = { | |
359 | .owner = THIS_MODULE, | |
35b13763 JL |
360 | DRM_ACCEL_FOPS, |
361 | }; | |
362 | ||
363 | static const struct drm_driver driver = { | |
364 | .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, | |
365 | ||
366 | .open = ivpu_open, | |
367 | .postclose = ivpu_postclose, | |
647371a6 JL |
368 | .prime_handle_to_fd = drm_gem_prime_handle_to_fd, |
369 | .prime_fd_to_handle = drm_gem_prime_fd_to_handle, | |
370 | .gem_prime_import = ivpu_gem_prime_import, | |
371 | .gem_prime_mmap = drm_gem_prime_mmap, | |
35b13763 JL |
372 | |
373 | .ioctls = ivpu_drm_ioctls, | |
374 | .num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls), | |
375 | .fops = &ivpu_fops, | |
376 | ||
377 | .name = DRIVER_NAME, | |
378 | .desc = DRIVER_DESC, | |
379 | .date = DRIVER_DATE, | |
380 | .major = DRM_IVPU_DRIVER_MAJOR, | |
381 | .minor = DRM_IVPU_DRIVER_MINOR, | |
382 | }; | |
383 | ||
384 | static int ivpu_irq_init(struct ivpu_device *vdev) | |
385 | { | |
386 | struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); | |
387 | int ret; | |
388 | ||
389 | ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX); | |
390 | if (ret < 0) { | |
391 | ivpu_err(vdev, "Failed to allocate a MSI IRQ: %d\n", ret); | |
392 | return ret; | |
393 | } | |
394 | ||
395 | vdev->irq = pci_irq_vector(pdev, 0); | |
396 | ||
397 | ret = devm_request_irq(vdev->drm.dev, vdev->irq, vdev->hw->ops->irq_handler, | |
398 | IRQF_NO_AUTOEN, DRIVER_NAME, vdev); | |
399 | if (ret) | |
400 | ivpu_err(vdev, "Failed to request an IRQ %d\n", ret); | |
401 | ||
402 | return ret; | |
403 | } | |
404 | ||
405 | static int ivpu_pci_init(struct ivpu_device *vdev) | |
406 | { | |
407 | struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); | |
408 | struct resource *bar0 = &pdev->resource[0]; | |
409 | struct resource *bar4 = &pdev->resource[4]; | |
410 | int ret; | |
411 | ||
412 | ivpu_dbg(vdev, MISC, "Mapping BAR0 (RegV) %pR\n", bar0); | |
413 | vdev->regv = devm_ioremap_resource(vdev->drm.dev, bar0); | |
414 | if (IS_ERR(vdev->regv)) { | |
415 | ivpu_err(vdev, "Failed to map bar 0: %pe\n", vdev->regv); | |
416 | return PTR_ERR(vdev->regv); | |
417 | } | |
418 | ||
419 | ivpu_dbg(vdev, MISC, "Mapping BAR4 (RegB) %pR\n", bar4); | |
420 | vdev->regb = devm_ioremap_resource(vdev->drm.dev, bar4); | |
421 | if (IS_ERR(vdev->regb)) { | |
422 | ivpu_err(vdev, "Failed to map bar 4: %pe\n", vdev->regb); | |
423 | return PTR_ERR(vdev->regb); | |
424 | } | |
425 | ||
426 | ret = dma_set_mask_and_coherent(vdev->drm.dev, DMA_BIT_MASK(38)); | |
427 | if (ret) { | |
428 | ivpu_err(vdev, "Failed to set DMA mask: %d\n", ret); | |
429 | return ret; | |
430 | } | |
431 | ||
432 | /* Clear any pending errors */ | |
433 | pcie_capability_clear_word(pdev, PCI_EXP_DEVSTA, 0x3f); | |
434 | ||
435 | ret = pcim_enable_device(pdev); | |
436 | if (ret) { | |
437 | ivpu_err(vdev, "Failed to enable PCI device: %d\n", ret); | |
438 | return ret; | |
439 | } | |
440 | ||
441 | pci_set_master(pdev); | |
442 | ||
443 | return 0; | |
444 | } | |
445 | ||
446 | static int ivpu_dev_init(struct ivpu_device *vdev) | |
447 | { | |
448 | int ret; | |
449 | ||
450 | vdev->hw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->hw), GFP_KERNEL); | |
451 | if (!vdev->hw) | |
452 | return -ENOMEM; | |
453 | ||
263b2ba5 JL |
454 | vdev->mmu = drmm_kzalloc(&vdev->drm, sizeof(*vdev->mmu), GFP_KERNEL); |
455 | if (!vdev->mmu) | |
456 | return -ENOMEM; | |
457 | ||
02d5b0aa JL |
458 | vdev->fw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->fw), GFP_KERNEL); |
459 | if (!vdev->fw) | |
460 | return -ENOMEM; | |
461 | ||
5d7422cf JL |
462 | vdev->ipc = drmm_kzalloc(&vdev->drm, sizeof(*vdev->ipc), GFP_KERNEL); |
463 | if (!vdev->ipc) | |
464 | return -ENOMEM; | |
465 | ||
852be13f JL |
466 | vdev->pm = drmm_kzalloc(&vdev->drm, sizeof(*vdev->pm), GFP_KERNEL); |
467 | if (!vdev->pm) | |
468 | return -ENOMEM; | |
469 | ||
35b13763 JL |
470 | vdev->hw->ops = &ivpu_hw_mtl_ops; |
471 | vdev->platform = IVPU_PLATFORM_INVALID; | |
472 | vdev->context_xa_limit.min = IVPU_GLOBAL_CONTEXT_MMU_SSID + 1; | |
473 | vdev->context_xa_limit.max = IVPU_CONTEXT_LIMIT; | |
02d5b0aa | 474 | atomic64_set(&vdev->unique_id_counter, 0); |
35b13763 | 475 | xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC); |
cd727221 JL |
476 | xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1); |
477 | lockdep_set_class(&vdev->submitted_jobs_xa.xa_lock, &submitted_jobs_xa_lock_class_key); | |
35b13763 JL |
478 | |
479 | ret = ivpu_pci_init(vdev); | |
480 | if (ret) { | |
481 | ivpu_err(vdev, "Failed to initialize PCI device: %d\n", ret); | |
482 | goto err_xa_destroy; | |
483 | } | |
484 | ||
485 | ret = ivpu_irq_init(vdev); | |
486 | if (ret) { | |
487 | ivpu_err(vdev, "Failed to initialize IRQs: %d\n", ret); | |
488 | goto err_xa_destroy; | |
489 | } | |
490 | ||
491 | /* Init basic HW info based on buttress registers which are accessible before power up */ | |
492 | ret = ivpu_hw_info_init(vdev); | |
493 | if (ret) { | |
494 | ivpu_err(vdev, "Failed to initialize HW info: %d\n", ret); | |
495 | goto err_xa_destroy; | |
496 | } | |
497 | ||
498 | /* Power up early so the rest of init code can access VPU registers */ | |
499 | ret = ivpu_hw_power_up(vdev); | |
500 | if (ret) { | |
501 | ivpu_err(vdev, "Failed to power up HW: %d\n", ret); | |
502 | goto err_xa_destroy; | |
503 | } | |
504 | ||
263b2ba5 JL |
505 | ret = ivpu_mmu_global_context_init(vdev); |
506 | if (ret) { | |
507 | ivpu_err(vdev, "Failed to initialize global MMU context: %d\n", ret); | |
508 | goto err_power_down; | |
509 | } | |
510 | ||
511 | ret = ivpu_mmu_init(vdev); | |
512 | if (ret) { | |
513 | ivpu_err(vdev, "Failed to initialize MMU device: %d\n", ret); | |
514 | goto err_mmu_gctx_fini; | |
515 | } | |
516 | ||
02d5b0aa JL |
517 | ret = ivpu_fw_init(vdev); |
518 | if (ret) { | |
519 | ivpu_err(vdev, "Failed to initialize firmware: %d\n", ret); | |
520 | goto err_mmu_gctx_fini; | |
521 | } | |
522 | ||
5d7422cf JL |
523 | ret = ivpu_ipc_init(vdev); |
524 | if (ret) { | |
525 | ivpu_err(vdev, "Failed to initialize IPC: %d\n", ret); | |
02d5b0aa JL |
526 | goto err_fw_fini; |
527 | } | |
528 | ||
852be13f JL |
529 | ret = ivpu_pm_init(vdev); |
530 | if (ret) { | |
531 | ivpu_err(vdev, "Failed to initialize PM: %d\n", ret); | |
532 | goto err_ipc_fini; | |
533 | } | |
534 | ||
cd727221 JL |
535 | ret = ivpu_job_done_thread_init(vdev); |
536 | if (ret) { | |
537 | ivpu_err(vdev, "Failed to initialize job done thread: %d\n", ret); | |
538 | goto err_ipc_fini; | |
539 | } | |
540 | ||
02d5b0aa JL |
541 | ret = ivpu_fw_load(vdev); |
542 | if (ret) { | |
543 | ivpu_err(vdev, "Failed to load firmware: %d\n", ret); | |
cd727221 | 544 | goto err_job_done_thread_fini; |
02d5b0aa JL |
545 | } |
546 | ||
547 | ret = ivpu_boot(vdev); | |
548 | if (ret) { | |
549 | ivpu_err(vdev, "Failed to boot: %d\n", ret); | |
cd727221 | 550 | goto err_job_done_thread_fini; |
5d7422cf JL |
551 | } |
552 | ||
852be13f JL |
553 | ivpu_pm_enable(vdev); |
554 | ||
35b13763 JL |
555 | return 0; |
556 | ||
cd727221 JL |
557 | err_job_done_thread_fini: |
558 | ivpu_job_done_thread_fini(vdev); | |
559 | err_ipc_fini: | |
560 | ivpu_ipc_fini(vdev); | |
02d5b0aa JL |
561 | err_fw_fini: |
562 | ivpu_fw_fini(vdev); | |
263b2ba5 JL |
563 | err_mmu_gctx_fini: |
564 | ivpu_mmu_global_context_fini(vdev); | |
565 | err_power_down: | |
566 | ivpu_hw_power_down(vdev); | |
35b13763 | 567 | err_xa_destroy: |
cd727221 | 568 | xa_destroy(&vdev->submitted_jobs_xa); |
35b13763 JL |
569 | xa_destroy(&vdev->context_xa); |
570 | return ret; | |
571 | } | |
572 | ||
573 | static void ivpu_dev_fini(struct ivpu_device *vdev) | |
574 | { | |
852be13f | 575 | ivpu_pm_disable(vdev); |
35b13763 | 576 | ivpu_shutdown(vdev); |
cd727221 | 577 | ivpu_job_done_thread_fini(vdev); |
5d7422cf | 578 | ivpu_ipc_fini(vdev); |
02d5b0aa | 579 | ivpu_fw_fini(vdev); |
263b2ba5 | 580 | ivpu_mmu_global_context_fini(vdev); |
35b13763 | 581 | |
cd727221 JL |
582 | drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa)); |
583 | xa_destroy(&vdev->submitted_jobs_xa); | |
35b13763 JL |
584 | drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa)); |
585 | xa_destroy(&vdev->context_xa); | |
586 | } | |
587 | ||
588 | static struct pci_device_id ivpu_pci_ids[] = { | |
589 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_MTL) }, | |
590 | { } | |
591 | }; | |
592 | MODULE_DEVICE_TABLE(pci, ivpu_pci_ids); | |
593 | ||
594 | static int ivpu_probe(struct pci_dev *pdev, const struct pci_device_id *id) | |
595 | { | |
596 | struct ivpu_device *vdev; | |
597 | int ret; | |
598 | ||
599 | vdev = devm_drm_dev_alloc(&pdev->dev, &driver, struct ivpu_device, drm); | |
600 | if (IS_ERR(vdev)) | |
601 | return PTR_ERR(vdev); | |
602 | ||
603 | pci_set_drvdata(pdev, vdev); | |
604 | ||
605 | ret = ivpu_dev_init(vdev); | |
606 | if (ret) { | |
607 | dev_err(&pdev->dev, "Failed to initialize VPU device: %d\n", ret); | |
608 | return ret; | |
609 | } | |
610 | ||
611 | ret = drm_dev_register(&vdev->drm, 0); | |
612 | if (ret) { | |
613 | dev_err(&pdev->dev, "Failed to register DRM device: %d\n", ret); | |
614 | ivpu_dev_fini(vdev); | |
615 | } | |
616 | ||
617 | return ret; | |
618 | } | |
619 | ||
620 | static void ivpu_remove(struct pci_dev *pdev) | |
621 | { | |
622 | struct ivpu_device *vdev = pci_get_drvdata(pdev); | |
623 | ||
624 | drm_dev_unregister(&vdev->drm); | |
625 | ivpu_dev_fini(vdev); | |
626 | } | |
627 | ||
852be13f JL |
628 | static const struct dev_pm_ops ivpu_drv_pci_pm = { |
629 | SET_SYSTEM_SLEEP_PM_OPS(ivpu_pm_suspend_cb, ivpu_pm_resume_cb) | |
630 | SET_RUNTIME_PM_OPS(ivpu_pm_runtime_suspend_cb, ivpu_pm_runtime_resume_cb, NULL) | |
631 | }; | |
632 | ||
633 | static const struct pci_error_handlers ivpu_drv_pci_err = { | |
634 | .reset_prepare = ivpu_pm_reset_prepare_cb, | |
635 | .reset_done = ivpu_pm_reset_done_cb, | |
636 | }; | |
637 | ||
35b13763 JL |
638 | static struct pci_driver ivpu_pci_driver = { |
639 | .name = KBUILD_MODNAME, | |
640 | .id_table = ivpu_pci_ids, | |
641 | .probe = ivpu_probe, | |
642 | .remove = ivpu_remove, | |
852be13f JL |
643 | .driver = { |
644 | .pm = &ivpu_drv_pci_pm, | |
645 | }, | |
646 | .err_handler = &ivpu_drv_pci_err, | |
35b13763 JL |
647 | }; |
648 | ||
649 | module_pci_driver(ivpu_pci_driver); | |
650 | ||
651 | MODULE_AUTHOR("Intel Corporation"); | |
652 | MODULE_DESCRIPTION(DRIVER_DESC); | |
653 | MODULE_LICENSE("GPL and additional rights"); | |
654 | MODULE_VERSION(DRIVER_VERSION_STR); |