Commit | Line | Data |
---|---|---|
35b13763 JL |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Copyright (C) 2020-2023 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include <linux/firmware.h> | |
7 | #include <linux/module.h> | |
8 | #include <linux/pci.h> | |
9 | ||
10 | #include <drm/drm_accel.h> | |
35b13763 JL |
11 | #include <drm/drm_file.h> |
12 | #include <drm/drm_gem.h> | |
13 | #include <drm/drm_ioctl.h> | |
647371a6 | 14 | #include <drm/drm_prime.h> |
35b13763 | 15 | |
02d5b0aa | 16 | #include "vpu_boot_api.h" |
edde4cae | 17 | #include "ivpu_debugfs.h" |
35b13763 | 18 | #include "ivpu_drv.h" |
02d5b0aa | 19 | #include "ivpu_fw.h" |
647371a6 | 20 | #include "ivpu_gem.h" |
35b13763 | 21 | #include "ivpu_hw.h" |
5d7422cf | 22 | #include "ivpu_ipc.h" |
cd727221 | 23 | #include "ivpu_job.h" |
02d5b0aa | 24 | #include "ivpu_jsm_msg.h" |
263b2ba5 JL |
25 | #include "ivpu_mmu.h" |
26 | #include "ivpu_mmu_context.h" | |
852be13f | 27 | #include "ivpu_pm.h" |
35b13763 JL |
28 | |
29 | #ifndef DRIVER_VERSION_STR | |
30 | #define DRIVER_VERSION_STR __stringify(DRM_IVPU_DRIVER_MAJOR) "." \ | |
31 | __stringify(DRM_IVPU_DRIVER_MINOR) "." | |
32 | #endif | |
33 | ||
34 | static const struct drm_driver driver; | |
35 | ||
cd727221 JL |
36 | static struct lock_class_key submitted_jobs_xa_lock_class_key; |
37 | ||
35b13763 JL |
38 | int ivpu_dbg_mask; |
39 | module_param_named(dbg_mask, ivpu_dbg_mask, int, 0644); | |
40 | MODULE_PARM_DESC(dbg_mask, "Driver debug mask. See IVPU_DBG_* macros."); | |
41 | ||
02d5b0aa JL |
42 | int ivpu_test_mode; |
43 | module_param_named_unsafe(test_mode, ivpu_test_mode, int, 0644); | |
44 | MODULE_PARM_DESC(test_mode, "Test mode: 0 - normal operation, 1 - fw unit test, 2 - null hw"); | |
45 | ||
35b13763 JL |
46 | u8 ivpu_pll_min_ratio; |
47 | module_param_named(pll_min_ratio, ivpu_pll_min_ratio, byte, 0644); | |
48 | MODULE_PARM_DESC(pll_min_ratio, "Minimum PLL ratio used to set VPU frequency"); | |
49 | ||
50 | u8 ivpu_pll_max_ratio = U8_MAX; | |
51 | module_param_named(pll_max_ratio, ivpu_pll_max_ratio, byte, 0644); | |
52 | MODULE_PARM_DESC(pll_max_ratio, "Maximum PLL ratio used to set VPU frequency"); | |
53 | ||
95d44018 KW |
54 | bool ivpu_disable_mmu_cont_pages; |
55 | module_param_named(disable_mmu_cont_pages, ivpu_disable_mmu_cont_pages, bool, 0644); | |
56 | MODULE_PARM_DESC(disable_mmu_cont_pages, "Disable MMU contiguous pages optimization"); | |
57 | ||
35b13763 JL |
58 | struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv) |
59 | { | |
263b2ba5 JL |
60 | struct ivpu_device *vdev = file_priv->vdev; |
61 | ||
35b13763 | 62 | kref_get(&file_priv->ref); |
263b2ba5 JL |
63 | |
64 | ivpu_dbg(vdev, KREF, "file_priv get: ctx %u refcount %u\n", | |
65 | file_priv->ctx.id, kref_read(&file_priv->ref)); | |
66 | ||
35b13763 JL |
67 | return file_priv; |
68 | } | |
69 | ||
647371a6 JL |
70 | struct ivpu_file_priv *ivpu_file_priv_get_by_ctx_id(struct ivpu_device *vdev, unsigned long id) |
71 | { | |
72 | struct ivpu_file_priv *file_priv; | |
73 | ||
74 | xa_lock_irq(&vdev->context_xa); | |
75 | file_priv = xa_load(&vdev->context_xa, id); | |
76 | /* file_priv may still be in context_xa during file_priv_release() */ | |
77 | if (file_priv && !kref_get_unless_zero(&file_priv->ref)) | |
78 | file_priv = NULL; | |
79 | xa_unlock_irq(&vdev->context_xa); | |
80 | ||
81 | if (file_priv) | |
82 | ivpu_dbg(vdev, KREF, "file_priv get by id: ctx %u refcount %u\n", | |
83 | file_priv->ctx.id, kref_read(&file_priv->ref)); | |
84 | ||
85 | return file_priv; | |
86 | } | |
87 | ||
35b13763 JL |
88 | static void file_priv_release(struct kref *ref) |
89 | { | |
90 | struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref); | |
263b2ba5 | 91 | struct ivpu_device *vdev = file_priv->vdev; |
35b13763 | 92 | |
263b2ba5 JL |
93 | ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id); |
94 | ||
cd727221 JL |
95 | ivpu_cmdq_release_all(file_priv); |
96 | ivpu_bo_remove_all_bos_from_context(&file_priv->ctx); | |
dffaa98c | 97 | ivpu_jsm_context_release(vdev, file_priv->ctx.id); |
263b2ba5 | 98 | ivpu_mmu_user_context_fini(vdev, &file_priv->ctx); |
647371a6 | 99 | drm_WARN_ON(&vdev->drm, xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv); |
cd727221 | 100 | mutex_destroy(&file_priv->lock); |
35b13763 JL |
101 | kfree(file_priv); |
102 | } | |
103 | ||
104 | void ivpu_file_priv_put(struct ivpu_file_priv **link) | |
105 | { | |
106 | struct ivpu_file_priv *file_priv = *link; | |
263b2ba5 | 107 | struct ivpu_device *vdev = file_priv->vdev; |
35b13763 | 108 | |
647371a6 | 109 | drm_WARN_ON(&vdev->drm, !file_priv); |
35b13763 | 110 | |
263b2ba5 JL |
111 | ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n", |
112 | file_priv->ctx.id, kref_read(&file_priv->ref)); | |
113 | ||
35b13763 JL |
114 | *link = NULL; |
115 | kref_put(&file_priv->ref, file_priv_release); | |
116 | } | |
117 | ||
aa5f04d2 SG |
118 | static int ivpu_get_capabilities(struct ivpu_device *vdev, struct drm_ivpu_param *args) |
119 | { | |
120 | switch (args->index) { | |
121 | case DRM_IVPU_CAP_METRIC_STREAMER: | |
122 | args->value = 0; | |
123 | break; | |
124 | case DRM_IVPU_CAP_DMA_MEMORY_RANGE: | |
162f17b2 | 125 | args->value = 1; |
aa5f04d2 SG |
126 | break; |
127 | default: | |
128 | return -EINVAL; | |
129 | } | |
130 | ||
131 | return 0; | |
132 | } | |
133 | ||
35b13763 JL |
134 | static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file) |
135 | { | |
136 | struct ivpu_file_priv *file_priv = file->driver_priv; | |
137 | struct ivpu_device *vdev = file_priv->vdev; | |
138 | struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); | |
139 | struct drm_ivpu_param *args = data; | |
140 | int ret = 0; | |
4522ad76 SG |
141 | int idx; |
142 | ||
143 | if (!drm_dev_enter(dev, &idx)) | |
144 | return -ENODEV; | |
35b13763 JL |
145 | |
146 | switch (args->param) { | |
147 | case DRM_IVPU_PARAM_DEVICE_ID: | |
148 | args->value = pdev->device; | |
149 | break; | |
150 | case DRM_IVPU_PARAM_DEVICE_REVISION: | |
151 | args->value = pdev->revision; | |
152 | break; | |
153 | case DRM_IVPU_PARAM_PLATFORM_TYPE: | |
154 | args->value = vdev->platform; | |
155 | break; | |
156 | case DRM_IVPU_PARAM_CORE_CLOCK_RATE: | |
157 | args->value = ivpu_hw_reg_pll_freq_get(vdev); | |
158 | break; | |
159 | case DRM_IVPU_PARAM_NUM_CONTEXTS: | |
160 | args->value = ivpu_get_context_count(vdev); | |
161 | break; | |
162 | case DRM_IVPU_PARAM_CONTEXT_BASE_ADDRESS: | |
162f17b2 | 163 | args->value = vdev->hw->ranges.user.start; |
35b13763 JL |
164 | break; |
165 | case DRM_IVPU_PARAM_CONTEXT_PRIORITY: | |
166 | args->value = file_priv->priority; | |
167 | break; | |
263b2ba5 JL |
168 | case DRM_IVPU_PARAM_CONTEXT_ID: |
169 | args->value = file_priv->ctx.id; | |
170 | break; | |
02d5b0aa JL |
171 | case DRM_IVPU_PARAM_FW_API_VERSION: |
172 | if (args->index < VPU_FW_API_VER_NUM) { | |
173 | struct vpu_firmware_header *fw_hdr; | |
174 | ||
175 | fw_hdr = (struct vpu_firmware_header *)vdev->fw->file->data; | |
176 | args->value = fw_hdr->api_version[args->index]; | |
177 | } else { | |
178 | ret = -EINVAL; | |
179 | } | |
180 | break; | |
181 | case DRM_IVPU_PARAM_ENGINE_HEARTBEAT: | |
182 | ret = ivpu_jsm_get_heartbeat(vdev, args->index, &args->value); | |
183 | break; | |
184 | case DRM_IVPU_PARAM_UNIQUE_INFERENCE_ID: | |
185 | args->value = (u64)atomic64_inc_return(&vdev->unique_id_counter); | |
186 | break; | |
187 | case DRM_IVPU_PARAM_TILE_CONFIG: | |
188 | args->value = vdev->hw->tile_fuse; | |
189 | break; | |
190 | case DRM_IVPU_PARAM_SKU: | |
191 | args->value = vdev->hw->sku; | |
192 | break; | |
aa5f04d2 SG |
193 | case DRM_IVPU_PARAM_CAPABILITIES: |
194 | ret = ivpu_get_capabilities(vdev, args); | |
195 | break; | |
35b13763 JL |
196 | default: |
197 | ret = -EINVAL; | |
198 | break; | |
199 | } | |
200 | ||
4522ad76 | 201 | drm_dev_exit(idx); |
35b13763 JL |
202 | return ret; |
203 | } | |
204 | ||
205 | static int ivpu_set_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file) | |
206 | { | |
207 | struct ivpu_file_priv *file_priv = file->driver_priv; | |
208 | struct drm_ivpu_param *args = data; | |
209 | int ret = 0; | |
210 | ||
211 | switch (args->param) { | |
212 | case DRM_IVPU_PARAM_CONTEXT_PRIORITY: | |
213 | if (args->value <= DRM_IVPU_CONTEXT_PRIORITY_REALTIME) | |
214 | file_priv->priority = args->value; | |
215 | else | |
216 | ret = -EINVAL; | |
217 | break; | |
218 | default: | |
219 | ret = -EINVAL; | |
220 | } | |
221 | ||
222 | return ret; | |
223 | } | |
224 | ||
225 | static int ivpu_open(struct drm_device *dev, struct drm_file *file) | |
226 | { | |
227 | struct ivpu_device *vdev = to_ivpu_device(dev); | |
228 | struct ivpu_file_priv *file_priv; | |
263b2ba5 JL |
229 | u32 ctx_id; |
230 | void *old; | |
231 | int ret; | |
232 | ||
233 | ret = xa_alloc_irq(&vdev->context_xa, &ctx_id, NULL, vdev->context_xa_limit, GFP_KERNEL); | |
234 | if (ret) { | |
235 | ivpu_err(vdev, "Failed to allocate context id: %d\n", ret); | |
236 | return ret; | |
237 | } | |
35b13763 JL |
238 | |
239 | file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); | |
263b2ba5 JL |
240 | if (!file_priv) { |
241 | ret = -ENOMEM; | |
242 | goto err_xa_erase; | |
243 | } | |
35b13763 JL |
244 | |
245 | file_priv->vdev = vdev; | |
246 | file_priv->priority = DRM_IVPU_CONTEXT_PRIORITY_NORMAL; | |
247 | kref_init(&file_priv->ref); | |
cd727221 | 248 | mutex_init(&file_priv->lock); |
35b13763 | 249 | |
263b2ba5 JL |
250 | ret = ivpu_mmu_user_context_init(vdev, &file_priv->ctx, ctx_id); |
251 | if (ret) | |
cd727221 | 252 | goto err_mutex_destroy; |
263b2ba5 JL |
253 | |
254 | old = xa_store_irq(&vdev->context_xa, ctx_id, file_priv, GFP_KERNEL); | |
255 | if (xa_is_err(old)) { | |
256 | ret = xa_err(old); | |
257 | ivpu_err(vdev, "Failed to store context %u: %d\n", ctx_id, ret); | |
258 | goto err_ctx_fini; | |
259 | } | |
260 | ||
261 | ivpu_dbg(vdev, FILE, "file_priv create: ctx %u process %s pid %d\n", | |
262 | ctx_id, current->comm, task_pid_nr(current)); | |
263 | ||
35b13763 JL |
264 | file->driver_priv = file_priv; |
265 | return 0; | |
263b2ba5 JL |
266 | |
267 | err_ctx_fini: | |
268 | ivpu_mmu_user_context_fini(vdev, &file_priv->ctx); | |
cd727221 JL |
269 | err_mutex_destroy: |
270 | mutex_destroy(&file_priv->lock); | |
263b2ba5 JL |
271 | kfree(file_priv); |
272 | err_xa_erase: | |
273 | xa_erase_irq(&vdev->context_xa, ctx_id); | |
274 | return ret; | |
35b13763 JL |
275 | } |
276 | ||
277 | static void ivpu_postclose(struct drm_device *dev, struct drm_file *file) | |
278 | { | |
279 | struct ivpu_file_priv *file_priv = file->driver_priv; | |
263b2ba5 JL |
280 | struct ivpu_device *vdev = to_ivpu_device(dev); |
281 | ||
282 | ivpu_dbg(vdev, FILE, "file_priv close: ctx %u process %s pid %d\n", | |
283 | file_priv->ctx.id, current->comm, task_pid_nr(current)); | |
35b13763 JL |
284 | |
285 | ivpu_file_priv_put(&file_priv); | |
286 | } | |
287 | ||
288 | static const struct drm_ioctl_desc ivpu_drm_ioctls[] = { | |
289 | DRM_IOCTL_DEF_DRV(IVPU_GET_PARAM, ivpu_get_param_ioctl, 0), | |
290 | DRM_IOCTL_DEF_DRV(IVPU_SET_PARAM, ivpu_set_param_ioctl, 0), | |
647371a6 JL |
291 | DRM_IOCTL_DEF_DRV(IVPU_BO_CREATE, ivpu_bo_create_ioctl, 0), |
292 | DRM_IOCTL_DEF_DRV(IVPU_BO_INFO, ivpu_bo_info_ioctl, 0), | |
cd727221 JL |
293 | DRM_IOCTL_DEF_DRV(IVPU_SUBMIT, ivpu_submit_ioctl, 0), |
294 | DRM_IOCTL_DEF_DRV(IVPU_BO_WAIT, ivpu_bo_wait_ioctl, 0), | |
35b13763 JL |
295 | }; |
296 | ||
02d5b0aa JL |
297 | static int ivpu_wait_for_ready(struct ivpu_device *vdev) |
298 | { | |
299 | struct ivpu_ipc_consumer cons; | |
300 | struct ivpu_ipc_hdr ipc_hdr; | |
301 | unsigned long timeout; | |
302 | int ret; | |
303 | ||
304 | if (ivpu_test_mode == IVPU_TEST_MODE_FW_TEST) | |
305 | return 0; | |
306 | ||
307 | ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG); | |
308 | ||
309 | timeout = jiffies + msecs_to_jiffies(vdev->timeout.boot); | |
310 | while (1) { | |
311 | ret = ivpu_ipc_irq_handler(vdev); | |
312 | if (ret) | |
313 | break; | |
314 | ret = ivpu_ipc_receive(vdev, &cons, &ipc_hdr, NULL, 0); | |
315 | if (ret != -ETIMEDOUT || time_after_eq(jiffies, timeout)) | |
316 | break; | |
317 | ||
318 | cond_resched(); | |
319 | } | |
320 | ||
321 | ivpu_ipc_consumer_del(vdev, &cons); | |
322 | ||
323 | if (!ret && ipc_hdr.data_addr != IVPU_IPC_BOOT_MSG_DATA_ADDR) { | |
324 | ivpu_err(vdev, "Invalid VPU ready message: 0x%x\n", | |
325 | ipc_hdr.data_addr); | |
326 | return -EIO; | |
327 | } | |
328 | ||
329 | if (!ret) | |
00265255 | 330 | ivpu_dbg(vdev, PM, "VPU ready message received successfully\n"); |
02d5b0aa JL |
331 | else |
332 | ivpu_hw_diagnose_failure(vdev); | |
333 | ||
334 | return ret; | |
335 | } | |
336 | ||
337 | /** | |
338 | * ivpu_boot() - Start VPU firmware | |
339 | * @vdev: VPU device | |
340 | * | |
341 | * This function is paired with ivpu_shutdown() but it doesn't power up the | |
342 | * VPU because power up has to be called very early in ivpu_probe(). | |
343 | */ | |
344 | int ivpu_boot(struct ivpu_device *vdev) | |
345 | { | |
346 | int ret; | |
347 | ||
348 | /* Update boot params located at first 4KB of FW memory */ | |
349 | ivpu_fw_boot_params_setup(vdev, vdev->fw->mem->kvaddr); | |
350 | ||
351 | ret = ivpu_hw_boot_fw(vdev); | |
352 | if (ret) { | |
353 | ivpu_err(vdev, "Failed to start the firmware: %d\n", ret); | |
354 | return ret; | |
355 | } | |
356 | ||
357 | ret = ivpu_wait_for_ready(vdev); | |
358 | if (ret) { | |
359 | ivpu_err(vdev, "Failed to boot the firmware: %d\n", ret); | |
360 | return ret; | |
361 | } | |
362 | ||
363 | ivpu_hw_irq_clear(vdev); | |
364 | enable_irq(vdev->irq); | |
365 | ivpu_hw_irq_enable(vdev); | |
366 | ivpu_ipc_enable(vdev); | |
367 | return 0; | |
368 | } | |
369 | ||
828d6304 | 370 | void ivpu_prepare_for_reset(struct ivpu_device *vdev) |
35b13763 | 371 | { |
35b13763 | 372 | ivpu_hw_irq_disable(vdev); |
02d5b0aa | 373 | disable_irq(vdev->irq); |
5d7422cf | 374 | ivpu_ipc_disable(vdev); |
263b2ba5 | 375 | ivpu_mmu_disable(vdev); |
828d6304 JL |
376 | } |
377 | ||
378 | int ivpu_shutdown(struct ivpu_device *vdev) | |
379 | { | |
380 | int ret; | |
381 | ||
382 | ivpu_prepare_for_reset(vdev); | |
35b13763 JL |
383 | |
384 | ret = ivpu_hw_power_down(vdev); | |
385 | if (ret) | |
386 | ivpu_warn(vdev, "Failed to power down HW: %d\n", ret); | |
387 | ||
388 | return ret; | |
389 | } | |
390 | ||
391 | static const struct file_operations ivpu_fops = { | |
392 | .owner = THIS_MODULE, | |
35b13763 JL |
393 | DRM_ACCEL_FOPS, |
394 | }; | |
395 | ||
396 | static const struct drm_driver driver = { | |
397 | .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, | |
398 | ||
399 | .open = ivpu_open, | |
400 | .postclose = ivpu_postclose, | |
647371a6 | 401 | .gem_prime_import = ivpu_gem_prime_import, |
35b13763 | 402 | |
edde4cae SG |
403 | #if defined(CONFIG_DEBUG_FS) |
404 | .debugfs_init = ivpu_debugfs_init, | |
405 | #endif | |
406 | ||
35b13763 JL |
407 | .ioctls = ivpu_drm_ioctls, |
408 | .num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls), | |
409 | .fops = &ivpu_fops, | |
410 | ||
411 | .name = DRIVER_NAME, | |
412 | .desc = DRIVER_DESC, | |
413 | .date = DRIVER_DATE, | |
414 | .major = DRM_IVPU_DRIVER_MAJOR, | |
415 | .minor = DRM_IVPU_DRIVER_MINOR, | |
416 | }; | |
417 | ||
418 | static int ivpu_irq_init(struct ivpu_device *vdev) | |
419 | { | |
420 | struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); | |
421 | int ret; | |
422 | ||
423 | ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX); | |
424 | if (ret < 0) { | |
425 | ivpu_err(vdev, "Failed to allocate a MSI IRQ: %d\n", ret); | |
426 | return ret; | |
427 | } | |
428 | ||
429 | vdev->irq = pci_irq_vector(pdev, 0); | |
430 | ||
431 | ret = devm_request_irq(vdev->drm.dev, vdev->irq, vdev->hw->ops->irq_handler, | |
432 | IRQF_NO_AUTOEN, DRIVER_NAME, vdev); | |
433 | if (ret) | |
434 | ivpu_err(vdev, "Failed to request an IRQ %d\n", ret); | |
435 | ||
436 | return ret; | |
437 | } | |
438 | ||
439 | static int ivpu_pci_init(struct ivpu_device *vdev) | |
440 | { | |
441 | struct pci_dev *pdev = to_pci_dev(vdev->drm.dev); | |
442 | struct resource *bar0 = &pdev->resource[0]; | |
443 | struct resource *bar4 = &pdev->resource[4]; | |
444 | int ret; | |
445 | ||
446 | ivpu_dbg(vdev, MISC, "Mapping BAR0 (RegV) %pR\n", bar0); | |
447 | vdev->regv = devm_ioremap_resource(vdev->drm.dev, bar0); | |
448 | if (IS_ERR(vdev->regv)) { | |
449 | ivpu_err(vdev, "Failed to map bar 0: %pe\n", vdev->regv); | |
450 | return PTR_ERR(vdev->regv); | |
451 | } | |
452 | ||
453 | ivpu_dbg(vdev, MISC, "Mapping BAR4 (RegB) %pR\n", bar4); | |
454 | vdev->regb = devm_ioremap_resource(vdev->drm.dev, bar4); | |
455 | if (IS_ERR(vdev->regb)) { | |
456 | ivpu_err(vdev, "Failed to map bar 4: %pe\n", vdev->regb); | |
457 | return PTR_ERR(vdev->regb); | |
458 | } | |
459 | ||
a4172d6c | 460 | ret = dma_set_mask_and_coherent(vdev->drm.dev, DMA_BIT_MASK(vdev->hw->dma_bits)); |
35b13763 JL |
461 | if (ret) { |
462 | ivpu_err(vdev, "Failed to set DMA mask: %d\n", ret); | |
463 | return ret; | |
464 | } | |
62079b6f | 465 | dma_set_max_seg_size(vdev->drm.dev, UINT_MAX); |
35b13763 JL |
466 | |
467 | /* Clear any pending errors */ | |
468 | pcie_capability_clear_word(pdev, PCI_EXP_DEVSTA, 0x3f); | |
469 | ||
51d66a7b JL |
470 | /* VPU 37XX does not require 10m D3hot delay */ |
471 | if (ivpu_hw_gen(vdev) == IVPU_HW_37XX) | |
cb949ce5 KW |
472 | pdev->d3hot_delay = 0; |
473 | ||
35b13763 JL |
474 | ret = pcim_enable_device(pdev); |
475 | if (ret) { | |
476 | ivpu_err(vdev, "Failed to enable PCI device: %d\n", ret); | |
477 | return ret; | |
478 | } | |
479 | ||
480 | pci_set_master(pdev); | |
481 | ||
482 | return 0; | |
483 | } | |
484 | ||
485 | static int ivpu_dev_init(struct ivpu_device *vdev) | |
486 | { | |
487 | int ret; | |
488 | ||
489 | vdev->hw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->hw), GFP_KERNEL); | |
490 | if (!vdev->hw) | |
491 | return -ENOMEM; | |
492 | ||
263b2ba5 JL |
493 | vdev->mmu = drmm_kzalloc(&vdev->drm, sizeof(*vdev->mmu), GFP_KERNEL); |
494 | if (!vdev->mmu) | |
495 | return -ENOMEM; | |
496 | ||
02d5b0aa JL |
497 | vdev->fw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->fw), GFP_KERNEL); |
498 | if (!vdev->fw) | |
499 | return -ENOMEM; | |
500 | ||
5d7422cf JL |
501 | vdev->ipc = drmm_kzalloc(&vdev->drm, sizeof(*vdev->ipc), GFP_KERNEL); |
502 | if (!vdev->ipc) | |
503 | return -ENOMEM; | |
504 | ||
852be13f JL |
505 | vdev->pm = drmm_kzalloc(&vdev->drm, sizeof(*vdev->pm), GFP_KERNEL); |
506 | if (!vdev->pm) | |
507 | return -ENOMEM; | |
508 | ||
79cdc56c SG |
509 | if (ivpu_hw_gen(vdev) >= IVPU_HW_40XX) { |
510 | vdev->hw->ops = &ivpu_hw_40xx_ops; | |
511 | vdev->hw->dma_bits = 48; | |
512 | } else { | |
513 | vdev->hw->ops = &ivpu_hw_37xx_ops; | |
514 | vdev->hw->dma_bits = 38; | |
515 | } | |
a4172d6c | 516 | |
35b13763 | 517 | vdev->platform = IVPU_PLATFORM_INVALID; |
3ff6edbc SG |
518 | vdev->context_xa_limit.min = IVPU_USER_CONTEXT_MIN_SSID; |
519 | vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID; | |
02d5b0aa | 520 | atomic64_set(&vdev->unique_id_counter, 0); |
35b13763 | 521 | xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC); |
cd727221 JL |
522 | xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1); |
523 | lockdep_set_class(&vdev->submitted_jobs_xa.xa_lock, &submitted_jobs_xa_lock_class_key); | |
35b13763 JL |
524 | |
525 | ret = ivpu_pci_init(vdev); | |
526 | if (ret) { | |
527 | ivpu_err(vdev, "Failed to initialize PCI device: %d\n", ret); | |
528 | goto err_xa_destroy; | |
529 | } | |
530 | ||
531 | ret = ivpu_irq_init(vdev); | |
532 | if (ret) { | |
533 | ivpu_err(vdev, "Failed to initialize IRQs: %d\n", ret); | |
534 | goto err_xa_destroy; | |
535 | } | |
536 | ||
537 | /* Init basic HW info based on buttress registers which are accessible before power up */ | |
538 | ret = ivpu_hw_info_init(vdev); | |
539 | if (ret) { | |
540 | ivpu_err(vdev, "Failed to initialize HW info: %d\n", ret); | |
541 | goto err_xa_destroy; | |
542 | } | |
543 | ||
544 | /* Power up early so the rest of init code can access VPU registers */ | |
545 | ret = ivpu_hw_power_up(vdev); | |
546 | if (ret) { | |
547 | ivpu_err(vdev, "Failed to power up HW: %d\n", ret); | |
548 | goto err_xa_destroy; | |
549 | } | |
550 | ||
263b2ba5 JL |
551 | ret = ivpu_mmu_global_context_init(vdev); |
552 | if (ret) { | |
553 | ivpu_err(vdev, "Failed to initialize global MMU context: %d\n", ret); | |
554 | goto err_power_down; | |
555 | } | |
556 | ||
557 | ret = ivpu_mmu_init(vdev); | |
558 | if (ret) { | |
559 | ivpu_err(vdev, "Failed to initialize MMU device: %d\n", ret); | |
560 | goto err_mmu_gctx_fini; | |
561 | } | |
562 | ||
02d5b0aa JL |
563 | ret = ivpu_fw_init(vdev); |
564 | if (ret) { | |
565 | ivpu_err(vdev, "Failed to initialize firmware: %d\n", ret); | |
566 | goto err_mmu_gctx_fini; | |
567 | } | |
568 | ||
5d7422cf JL |
569 | ret = ivpu_ipc_init(vdev); |
570 | if (ret) { | |
571 | ivpu_err(vdev, "Failed to initialize IPC: %d\n", ret); | |
02d5b0aa JL |
572 | goto err_fw_fini; |
573 | } | |
574 | ||
852be13f JL |
575 | ret = ivpu_pm_init(vdev); |
576 | if (ret) { | |
577 | ivpu_err(vdev, "Failed to initialize PM: %d\n", ret); | |
578 | goto err_ipc_fini; | |
579 | } | |
580 | ||
cd727221 JL |
581 | ret = ivpu_job_done_thread_init(vdev); |
582 | if (ret) { | |
583 | ivpu_err(vdev, "Failed to initialize job done thread: %d\n", ret); | |
584 | goto err_ipc_fini; | |
585 | } | |
586 | ||
02d5b0aa JL |
587 | ret = ivpu_fw_load(vdev); |
588 | if (ret) { | |
589 | ivpu_err(vdev, "Failed to load firmware: %d\n", ret); | |
cd727221 | 590 | goto err_job_done_thread_fini; |
02d5b0aa JL |
591 | } |
592 | ||
593 | ret = ivpu_boot(vdev); | |
594 | if (ret) { | |
595 | ivpu_err(vdev, "Failed to boot: %d\n", ret); | |
cd727221 | 596 | goto err_job_done_thread_fini; |
5d7422cf JL |
597 | } |
598 | ||
852be13f JL |
599 | ivpu_pm_enable(vdev); |
600 | ||
35b13763 JL |
601 | return 0; |
602 | ||
cd727221 JL |
603 | err_job_done_thread_fini: |
604 | ivpu_job_done_thread_fini(vdev); | |
605 | err_ipc_fini: | |
606 | ivpu_ipc_fini(vdev); | |
02d5b0aa JL |
607 | err_fw_fini: |
608 | ivpu_fw_fini(vdev); | |
263b2ba5 JL |
609 | err_mmu_gctx_fini: |
610 | ivpu_mmu_global_context_fini(vdev); | |
611 | err_power_down: | |
612 | ivpu_hw_power_down(vdev); | |
3d8b2727 SG |
613 | if (IVPU_WA(d3hot_after_power_off)) |
614 | pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot); | |
35b13763 | 615 | err_xa_destroy: |
cd727221 | 616 | xa_destroy(&vdev->submitted_jobs_xa); |
35b13763 JL |
617 | xa_destroy(&vdev->context_xa); |
618 | return ret; | |
619 | } | |
620 | ||
621 | static void ivpu_dev_fini(struct ivpu_device *vdev) | |
622 | { | |
852be13f | 623 | ivpu_pm_disable(vdev); |
35b13763 | 624 | ivpu_shutdown(vdev); |
3d8b2727 SG |
625 | if (IVPU_WA(d3hot_after_power_off)) |
626 | pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot); | |
cd727221 | 627 | ivpu_job_done_thread_fini(vdev); |
6013aa84 SG |
628 | ivpu_pm_cancel_recovery(vdev); |
629 | ||
5d7422cf | 630 | ivpu_ipc_fini(vdev); |
02d5b0aa | 631 | ivpu_fw_fini(vdev); |
263b2ba5 | 632 | ivpu_mmu_global_context_fini(vdev); |
35b13763 | 633 | |
cd727221 JL |
634 | drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa)); |
635 | xa_destroy(&vdev->submitted_jobs_xa); | |
35b13763 JL |
636 | drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa)); |
637 | xa_destroy(&vdev->context_xa); | |
638 | } | |
639 | ||
640 | static struct pci_device_id ivpu_pci_ids[] = { | |
641 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_MTL) }, | |
9c1b2429 | 642 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_ARL) }, |
79cdc56c | 643 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_LNL) }, |
35b13763 JL |
644 | { } |
645 | }; | |
646 | MODULE_DEVICE_TABLE(pci, ivpu_pci_ids); | |
647 | ||
648 | static int ivpu_probe(struct pci_dev *pdev, const struct pci_device_id *id) | |
649 | { | |
650 | struct ivpu_device *vdev; | |
651 | int ret; | |
652 | ||
653 | vdev = devm_drm_dev_alloc(&pdev->dev, &driver, struct ivpu_device, drm); | |
654 | if (IS_ERR(vdev)) | |
655 | return PTR_ERR(vdev); | |
656 | ||
657 | pci_set_drvdata(pdev, vdev); | |
658 | ||
659 | ret = ivpu_dev_init(vdev); | |
660 | if (ret) { | |
661 | dev_err(&pdev->dev, "Failed to initialize VPU device: %d\n", ret); | |
662 | return ret; | |
663 | } | |
664 | ||
665 | ret = drm_dev_register(&vdev->drm, 0); | |
666 | if (ret) { | |
667 | dev_err(&pdev->dev, "Failed to register DRM device: %d\n", ret); | |
668 | ivpu_dev_fini(vdev); | |
669 | } | |
670 | ||
671 | return ret; | |
672 | } | |
673 | ||
674 | static void ivpu_remove(struct pci_dev *pdev) | |
675 | { | |
676 | struct ivpu_device *vdev = pci_get_drvdata(pdev); | |
677 | ||
4522ad76 | 678 | drm_dev_unplug(&vdev->drm); |
35b13763 JL |
679 | ivpu_dev_fini(vdev); |
680 | } | |
681 | ||
852be13f JL |
682 | static const struct dev_pm_ops ivpu_drv_pci_pm = { |
683 | SET_SYSTEM_SLEEP_PM_OPS(ivpu_pm_suspend_cb, ivpu_pm_resume_cb) | |
684 | SET_RUNTIME_PM_OPS(ivpu_pm_runtime_suspend_cb, ivpu_pm_runtime_resume_cb, NULL) | |
685 | }; | |
686 | ||
687 | static const struct pci_error_handlers ivpu_drv_pci_err = { | |
688 | .reset_prepare = ivpu_pm_reset_prepare_cb, | |
689 | .reset_done = ivpu_pm_reset_done_cb, | |
690 | }; | |
691 | ||
35b13763 JL |
692 | static struct pci_driver ivpu_pci_driver = { |
693 | .name = KBUILD_MODNAME, | |
694 | .id_table = ivpu_pci_ids, | |
695 | .probe = ivpu_probe, | |
696 | .remove = ivpu_remove, | |
852be13f JL |
697 | .driver = { |
698 | .pm = &ivpu_drv_pci_pm, | |
699 | }, | |
700 | .err_handler = &ivpu_drv_pci_err, | |
35b13763 JL |
701 | }; |
702 | ||
703 | module_pci_driver(ivpu_pci_driver); | |
704 | ||
705 | MODULE_AUTHOR("Intel Corporation"); | |
706 | MODULE_DESCRIPTION(DRIVER_DESC); | |
707 | MODULE_LICENSE("GPL and additional rights"); | |
708 | MODULE_VERSION(DRIVER_VERSION_STR); |