Commit | Line | Data |
---|---|---|
5d7422cf JL |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
f0cf7ffc | 3 | * Copyright (C) 2020-2024 Intel Corporation |
5d7422cf JL |
4 | */ |
5 | ||
6 | #include <linux/genalloc.h> | |
7 | #include <linux/highmem.h> | |
c015fb6d | 8 | #include <linux/pm_runtime.h> |
5d7422cf JL |
9 | #include <linux/wait.h> |
10 | ||
11 | #include "ivpu_drv.h" | |
12 | #include "ivpu_gem.h" | |
13 | #include "ivpu_hw.h" | |
14 | #include "ivpu_hw_reg_io.h" | |
15 | #include "ivpu_ipc.h" | |
16 | #include "ivpu_jsm_msg.h" | |
852be13f | 17 | #include "ivpu_pm.h" |
5d7422cf JL |
18 | |
19 | #define IPC_MAX_RX_MSG 128 | |
5d7422cf JL |
20 | |
21 | struct ivpu_ipc_tx_buf { | |
22 | struct ivpu_ipc_hdr ipc; | |
23 | struct vpu_jsm_msg jsm; | |
24 | }; | |
25 | ||
5d7422cf JL |
26 | static void ivpu_ipc_msg_dump(struct ivpu_device *vdev, char *c, |
27 | struct ivpu_ipc_hdr *ipc_hdr, u32 vpu_addr) | |
28 | { | |
29 | ivpu_dbg(vdev, IPC, | |
30 | "%s: vpu:0x%x (data_addr:0x%08x, data_size:0x%x, channel:0x%x, src_node:0x%x, dst_node:0x%x, status:0x%x)", | |
31 | c, vpu_addr, ipc_hdr->data_addr, ipc_hdr->data_size, ipc_hdr->channel, | |
32 | ipc_hdr->src_node, ipc_hdr->dst_node, ipc_hdr->status); | |
33 | } | |
34 | ||
35 | static void ivpu_jsm_msg_dump(struct ivpu_device *vdev, char *c, | |
36 | struct vpu_jsm_msg *jsm_msg, u32 vpu_addr) | |
37 | { | |
38 | u32 *payload = (u32 *)&jsm_msg->payload; | |
39 | ||
40 | ivpu_dbg(vdev, JSM, | |
a3cd664e KP |
41 | "%s: vpu:0x%08x (type:%s, status:0x%x, id: 0x%x, result: 0x%x, payload:0x%x 0x%x 0x%x 0x%x 0x%x)\n", |
42 | c, vpu_addr, ivpu_jsm_msg_type_to_str(jsm_msg->type), | |
43 | jsm_msg->status, jsm_msg->request_id, jsm_msg->result, | |
5d7422cf JL |
44 | payload[0], payload[1], payload[2], payload[3], payload[4]); |
45 | } | |
46 | ||
47 | static void | |
48 | ivpu_ipc_rx_mark_free(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr, | |
49 | struct vpu_jsm_msg *jsm_msg) | |
50 | { | |
51 | ipc_hdr->status = IVPU_IPC_HDR_FREE; | |
52 | if (jsm_msg) | |
53 | jsm_msg->status = VPU_JSM_MSG_FREE; | |
54 | wmb(); /* Flush WC buffers for message statuses */ | |
55 | } | |
56 | ||
57 | static void ivpu_ipc_mem_fini(struct ivpu_device *vdev) | |
58 | { | |
59 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
60 | ||
42328003 WK |
61 | ivpu_bo_free(ipc->mem_rx); |
62 | ivpu_bo_free(ipc->mem_tx); | |
5d7422cf JL |
63 | } |
64 | ||
65 | static int | |
66 | ivpu_ipc_tx_prepare(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, | |
67 | struct vpu_jsm_msg *req) | |
68 | { | |
69 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
70 | struct ivpu_ipc_tx_buf *tx_buf; | |
71 | u32 tx_buf_vpu_addr; | |
72 | u32 jsm_vpu_addr; | |
73 | ||
74 | tx_buf_vpu_addr = gen_pool_alloc(ipc->mm_tx, sizeof(*tx_buf)); | |
75 | if (!tx_buf_vpu_addr) { | |
276e4834 KP |
76 | ivpu_err_ratelimited(vdev, "Failed to reserve IPC buffer, size %ld\n", |
77 | sizeof(*tx_buf)); | |
5d7422cf JL |
78 | return -ENOMEM; |
79 | } | |
80 | ||
81 | tx_buf = ivpu_to_cpu_addr(ipc->mem_tx, tx_buf_vpu_addr); | |
82 | if (drm_WARN_ON(&vdev->drm, !tx_buf)) { | |
83 | gen_pool_free(ipc->mm_tx, tx_buf_vpu_addr, sizeof(*tx_buf)); | |
84 | return -EIO; | |
85 | } | |
86 | ||
87 | jsm_vpu_addr = tx_buf_vpu_addr + offsetof(struct ivpu_ipc_tx_buf, jsm); | |
88 | ||
89 | if (tx_buf->ipc.status != IVPU_IPC_HDR_FREE) | |
276e4834 KP |
90 | ivpu_warn_ratelimited(vdev, "IPC message vpu:0x%x not released by firmware\n", |
91 | tx_buf_vpu_addr); | |
5d7422cf JL |
92 | |
93 | if (tx_buf->jsm.status != VPU_JSM_MSG_FREE) | |
276e4834 KP |
94 | ivpu_warn_ratelimited(vdev, "JSM message vpu:0x%x not released by firmware\n", |
95 | jsm_vpu_addr); | |
5d7422cf JL |
96 | |
97 | memset(tx_buf, 0, sizeof(*tx_buf)); | |
98 | tx_buf->ipc.data_addr = jsm_vpu_addr; | |
99 | /* TODO: Set data_size to actual JSM message size, not union of all messages */ | |
100 | tx_buf->ipc.data_size = sizeof(*req); | |
101 | tx_buf->ipc.channel = cons->channel; | |
102 | tx_buf->ipc.src_node = 0; | |
103 | tx_buf->ipc.dst_node = 1; | |
104 | tx_buf->ipc.status = IVPU_IPC_HDR_ALLOCATED; | |
105 | tx_buf->jsm.type = req->type; | |
106 | tx_buf->jsm.status = VPU_JSM_MSG_ALLOCATED; | |
107 | tx_buf->jsm.payload = req->payload; | |
108 | ||
109 | req->request_id = atomic_inc_return(&ipc->request_id); | |
110 | tx_buf->jsm.request_id = req->request_id; | |
111 | cons->request_id = req->request_id; | |
112 | wmb(); /* Flush WC buffers for IPC, JSM msgs */ | |
113 | ||
114 | cons->tx_vpu_addr = tx_buf_vpu_addr; | |
115 | ||
116 | ivpu_jsm_msg_dump(vdev, "TX", &tx_buf->jsm, jsm_vpu_addr); | |
117 | ivpu_ipc_msg_dump(vdev, "TX", &tx_buf->ipc, tx_buf_vpu_addr); | |
118 | ||
119 | return 0; | |
120 | } | |
121 | ||
122 | static void ivpu_ipc_tx_release(struct ivpu_device *vdev, u32 vpu_addr) | |
123 | { | |
124 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
125 | ||
126 | if (vpu_addr) | |
127 | gen_pool_free(ipc->mm_tx, vpu_addr, sizeof(struct ivpu_ipc_tx_buf)); | |
128 | } | |
129 | ||
130 | static void ivpu_ipc_tx(struct ivpu_device *vdev, u32 vpu_addr) | |
131 | { | |
132 | ivpu_hw_reg_ipc_tx_set(vdev, vpu_addr); | |
133 | } | |
134 | ||
3b434a34 JL |
135 | static void |
136 | ivpu_ipc_rx_msg_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, | |
137 | struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg) | |
138 | { | |
139 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
140 | struct ivpu_ipc_rx_msg *rx_msg; | |
141 | ||
142 | lockdep_assert_held(&ipc->cons_lock); | |
143 | lockdep_assert_irqs_disabled(); | |
144 | ||
145 | rx_msg = kzalloc(sizeof(*rx_msg), GFP_ATOMIC); | |
146 | if (!rx_msg) { | |
147 | ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg); | |
148 | return; | |
149 | } | |
150 | ||
151 | atomic_inc(&ipc->rx_msg_count); | |
152 | ||
153 | rx_msg->ipc_hdr = ipc_hdr; | |
154 | rx_msg->jsm_msg = jsm_msg; | |
155 | rx_msg->callback = cons->rx_callback; | |
156 | ||
157 | if (rx_msg->callback) { | |
158 | list_add_tail(&rx_msg->link, &ipc->cb_msg_list); | |
159 | } else { | |
160 | spin_lock(&cons->rx_lock); | |
161 | list_add_tail(&rx_msg->link, &cons->rx_msg_list); | |
162 | spin_unlock(&cons->rx_lock); | |
163 | wake_up(&cons->rx_msg_wq); | |
164 | } | |
165 | } | |
166 | ||
167 | static void | |
168 | ivpu_ipc_rx_msg_del(struct ivpu_device *vdev, struct ivpu_ipc_rx_msg *rx_msg) | |
169 | { | |
170 | list_del(&rx_msg->link); | |
171 | ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg); | |
172 | atomic_dec(&vdev->ipc->rx_msg_count); | |
173 | kfree(rx_msg); | |
174 | } | |
175 | ||
176 | void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, | |
177 | u32 channel, ivpu_ipc_rx_callback_t rx_callback) | |
5d7422cf JL |
178 | { |
179 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
180 | ||
181 | INIT_LIST_HEAD(&cons->link); | |
182 | cons->channel = channel; | |
183 | cons->tx_vpu_addr = 0; | |
184 | cons->request_id = 0; | |
ba6b035d | 185 | cons->aborted = false; |
3b434a34 | 186 | cons->rx_callback = rx_callback; |
043a2d5d | 187 | spin_lock_init(&cons->rx_lock); |
5d7422cf JL |
188 | INIT_LIST_HEAD(&cons->rx_msg_list); |
189 | init_waitqueue_head(&cons->rx_msg_wq); | |
190 | ||
3b434a34 | 191 | spin_lock_irq(&ipc->cons_lock); |
5d7422cf | 192 | list_add_tail(&cons->link, &ipc->cons_list); |
3b434a34 | 193 | spin_unlock_irq(&ipc->cons_lock); |
5d7422cf JL |
194 | } |
195 | ||
196 | void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons) | |
197 | { | |
198 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
199 | struct ivpu_ipc_rx_msg *rx_msg, *r; | |
200 | ||
3b434a34 | 201 | spin_lock_irq(&ipc->cons_lock); |
5d7422cf | 202 | list_del(&cons->link); |
3b434a34 | 203 | spin_unlock_irq(&ipc->cons_lock); |
5d7422cf | 204 | |
043a2d5d | 205 | spin_lock_irq(&cons->rx_lock); |
3b434a34 JL |
206 | list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link) |
207 | ivpu_ipc_rx_msg_del(vdev, rx_msg); | |
043a2d5d | 208 | spin_unlock_irq(&cons->rx_lock); |
5d7422cf JL |
209 | |
210 | ivpu_ipc_tx_release(vdev, cons->tx_vpu_addr); | |
211 | } | |
212 | ||
213 | static int | |
214 | ivpu_ipc_send(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, struct vpu_jsm_msg *req) | |
215 | { | |
216 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
217 | int ret; | |
218 | ||
b563e479 | 219 | mutex_lock(&ipc->lock); |
5d7422cf JL |
220 | |
221 | if (!ipc->on) { | |
222 | ret = -EAGAIN; | |
223 | goto unlock; | |
224 | } | |
225 | ||
226 | ret = ivpu_ipc_tx_prepare(vdev, cons, req); | |
227 | if (ret) | |
228 | goto unlock; | |
229 | ||
230 | ivpu_ipc_tx(vdev, cons->tx_vpu_addr); | |
231 | ||
232 | unlock: | |
233 | mutex_unlock(&ipc->lock); | |
234 | return ret; | |
235 | } | |
236 | ||
3b434a34 | 237 | static bool ivpu_ipc_rx_need_wakeup(struct ivpu_ipc_consumer *cons) |
57c7e3e4 | 238 | { |
3b434a34 | 239 | bool ret; |
57c7e3e4 | 240 | |
043a2d5d | 241 | spin_lock_irq(&cons->rx_lock); |
3b434a34 | 242 | ret = !list_empty(&cons->rx_msg_list) || cons->aborted; |
043a2d5d | 243 | spin_unlock_irq(&cons->rx_lock); |
57c7e3e4 SG |
244 | |
245 | return ret; | |
246 | } | |
247 | ||
5d7422cf JL |
248 | int ivpu_ipc_receive(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, |
249 | struct ivpu_ipc_hdr *ipc_buf, | |
3b434a34 | 250 | struct vpu_jsm_msg *jsm_msg, unsigned long timeout_ms) |
5d7422cf | 251 | { |
5d7422cf JL |
252 | struct ivpu_ipc_rx_msg *rx_msg; |
253 | int wait_ret, ret = 0; | |
254 | ||
3b434a34 JL |
255 | if (drm_WARN_ONCE(&vdev->drm, cons->rx_callback, "Consumer works only in async mode\n")) |
256 | return -EINVAL; | |
257 | ||
b0873eea | 258 | wait_ret = wait_event_timeout(cons->rx_msg_wq, |
3bf3e21c | 259 | ivpu_ipc_rx_need_wakeup(cons), |
b0873eea | 260 | msecs_to_jiffies(timeout_ms)); |
5d7422cf | 261 | |
5d7422cf JL |
262 | if (wait_ret == 0) |
263 | return -ETIMEDOUT; | |
264 | ||
043a2d5d | 265 | spin_lock_irq(&cons->rx_lock); |
b3c10b71 SG |
266 | if (cons->aborted) { |
267 | spin_unlock_irq(&cons->rx_lock); | |
268 | return -ECANCELED; | |
269 | } | |
5d7422cf JL |
270 | rx_msg = list_first_entry_or_null(&cons->rx_msg_list, struct ivpu_ipc_rx_msg, link); |
271 | if (!rx_msg) { | |
043a2d5d | 272 | spin_unlock_irq(&cons->rx_lock); |
5d7422cf JL |
273 | return -EAGAIN; |
274 | } | |
5d7422cf JL |
275 | |
276 | if (ipc_buf) | |
277 | memcpy(ipc_buf, rx_msg->ipc_hdr, sizeof(*ipc_buf)); | |
278 | if (rx_msg->jsm_msg) { | |
3b434a34 | 279 | u32 size = min_t(int, rx_msg->ipc_hdr->data_size, sizeof(*jsm_msg)); |
5d7422cf JL |
280 | |
281 | if (rx_msg->jsm_msg->result != VPU_JSM_STATUS_SUCCESS) { | |
282 | ivpu_dbg(vdev, IPC, "IPC resp result error: %d\n", rx_msg->jsm_msg->result); | |
283 | ret = -EBADMSG; | |
284 | } | |
285 | ||
3b434a34 JL |
286 | if (jsm_msg) |
287 | memcpy(jsm_msg, rx_msg->jsm_msg, size); | |
5d7422cf JL |
288 | } |
289 | ||
3b434a34 JL |
290 | ivpu_ipc_rx_msg_del(vdev, rx_msg); |
291 | spin_unlock_irq(&cons->rx_lock); | |
5d7422cf JL |
292 | return ret; |
293 | } | |
294 | ||
295 | static int | |
296 | ivpu_ipc_send_receive_internal(struct ivpu_device *vdev, struct vpu_jsm_msg *req, | |
297 | enum vpu_ipc_msg_type expected_resp_type, | |
298 | struct vpu_jsm_msg *resp, u32 channel, | |
299 | unsigned long timeout_ms) | |
300 | { | |
301 | struct ivpu_ipc_consumer cons; | |
302 | int ret; | |
303 | ||
3b434a34 | 304 | ivpu_ipc_consumer_add(vdev, &cons, channel, NULL); |
5d7422cf JL |
305 | |
306 | ret = ivpu_ipc_send(vdev, &cons, req); | |
307 | if (ret) { | |
276e4834 | 308 | ivpu_warn_ratelimited(vdev, "IPC send failed: %d\n", ret); |
5d7422cf JL |
309 | goto consumer_del; |
310 | } | |
311 | ||
312 | ret = ivpu_ipc_receive(vdev, &cons, NULL, resp, timeout_ms); | |
313 | if (ret) { | |
a3cd664e KP |
314 | ivpu_warn_ratelimited(vdev, "IPC receive failed: type %s, ret %d\n", |
315 | ivpu_jsm_msg_type_to_str(req->type), ret); | |
5d7422cf JL |
316 | goto consumer_del; |
317 | } | |
318 | ||
319 | if (resp->type != expected_resp_type) { | |
276e4834 | 320 | ivpu_warn_ratelimited(vdev, "Invalid JSM response type: 0x%x\n", resp->type); |
5d7422cf JL |
321 | ret = -EBADE; |
322 | } | |
323 | ||
324 | consumer_del: | |
325 | ivpu_ipc_consumer_del(vdev, &cons); | |
326 | return ret; | |
327 | } | |
328 | ||
45e45362 KW |
329 | int ivpu_ipc_send_receive_active(struct ivpu_device *vdev, struct vpu_jsm_msg *req, |
330 | enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp, | |
331 | u32 channel, unsigned long timeout_ms) | |
5d7422cf JL |
332 | { |
333 | struct vpu_jsm_msg hb_req = { .type = VPU_JSM_MSG_QUERY_ENGINE_HB }; | |
334 | struct vpu_jsm_msg hb_resp; | |
852be13f JL |
335 | int ret, hb_ret; |
336 | ||
c015fb6d | 337 | drm_WARN_ON(&vdev->drm, pm_runtime_status_suspended(vdev->drm.dev)); |
5d7422cf | 338 | |
45e45362 | 339 | ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp, resp, channel, timeout_ms); |
5d7422cf | 340 | if (ret != -ETIMEDOUT) |
45e45362 | 341 | return ret; |
5d7422cf | 342 | |
852be13f JL |
343 | hb_ret = ivpu_ipc_send_receive_internal(vdev, &hb_req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE, |
344 | &hb_resp, VPU_IPC_CHAN_ASYNC_CMD, | |
345 | vdev->timeout.jsm); | |
27d19268 JL |
346 | if (hb_ret == -ETIMEDOUT) |
347 | ivpu_pm_trigger_recovery(vdev, "IPC timeout"); | |
5d7422cf | 348 | |
45e45362 KW |
349 | return ret; |
350 | } | |
351 | ||
352 | int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req, | |
353 | enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp, | |
354 | u32 channel, unsigned long timeout_ms) | |
355 | { | |
356 | int ret; | |
357 | ||
358 | ret = ivpu_rpm_get(vdev); | |
359 | if (ret < 0) | |
360 | return ret; | |
361 | ||
362 | ret = ivpu_ipc_send_receive_active(vdev, req, expected_resp, resp, channel, timeout_ms); | |
363 | ||
852be13f | 364 | ivpu_rpm_put(vdev); |
5d7422cf JL |
365 | return ret; |
366 | } | |
367 | ||
368 | static bool | |
369 | ivpu_ipc_match_consumer(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, | |
370 | struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg) | |
371 | { | |
372 | if (cons->channel != ipc_hdr->channel) | |
373 | return false; | |
374 | ||
375 | if (!jsm_msg || jsm_msg->request_id == cons->request_id) | |
376 | return true; | |
377 | ||
378 | return false; | |
379 | } | |
380 | ||
3b434a34 | 381 | void ivpu_ipc_irq_handler(struct ivpu_device *vdev, bool *wake_thread) |
5d7422cf JL |
382 | { |
383 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
384 | struct ivpu_ipc_consumer *cons; | |
385 | struct ivpu_ipc_hdr *ipc_hdr; | |
386 | struct vpu_jsm_msg *jsm_msg; | |
387 | unsigned long flags; | |
388 | bool dispatched; | |
389 | u32 vpu_addr; | |
390 | ||
391 | /* | |
392 | * Driver needs to purge all messages from IPC FIFO to clear IPC interrupt. | |
393 | * Without purge IPC FIFO to 0 next IPC interrupts won't be generated. | |
394 | */ | |
395 | while (ivpu_hw_reg_ipc_rx_count_get(vdev)) { | |
396 | vpu_addr = ivpu_hw_reg_ipc_rx_addr_get(vdev); | |
397 | if (vpu_addr == REG_IO_ERROR) { | |
276e4834 | 398 | ivpu_err_ratelimited(vdev, "Failed to read IPC rx addr register\n"); |
3b434a34 | 399 | return; |
5d7422cf JL |
400 | } |
401 | ||
402 | ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr); | |
403 | if (!ipc_hdr) { | |
276e4834 | 404 | ivpu_warn_ratelimited(vdev, "IPC msg 0x%x out of range\n", vpu_addr); |
5d7422cf JL |
405 | continue; |
406 | } | |
407 | ivpu_ipc_msg_dump(vdev, "RX", ipc_hdr, vpu_addr); | |
408 | ||
409 | jsm_msg = NULL; | |
410 | if (ipc_hdr->channel != IVPU_IPC_CHAN_BOOT_MSG) { | |
411 | jsm_msg = ivpu_to_cpu_addr(ipc->mem_rx, ipc_hdr->data_addr); | |
412 | if (!jsm_msg) { | |
276e4834 KP |
413 | ivpu_warn_ratelimited(vdev, "JSM msg 0x%x out of range\n", |
414 | ipc_hdr->data_addr); | |
5d7422cf JL |
415 | ivpu_ipc_rx_mark_free(vdev, ipc_hdr, NULL); |
416 | continue; | |
417 | } | |
418 | ivpu_jsm_msg_dump(vdev, "RX", jsm_msg, ipc_hdr->data_addr); | |
419 | } | |
420 | ||
421 | if (atomic_read(&ipc->rx_msg_count) > IPC_MAX_RX_MSG) { | |
276e4834 KP |
422 | ivpu_warn_ratelimited(vdev, "IPC RX msg dropped, msg count %d\n", |
423 | IPC_MAX_RX_MSG); | |
5d7422cf JL |
424 | ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg); |
425 | continue; | |
426 | } | |
427 | ||
428 | dispatched = false; | |
3b434a34 | 429 | spin_lock_irqsave(&ipc->cons_lock, flags); |
5d7422cf JL |
430 | list_for_each_entry(cons, &ipc->cons_list, link) { |
431 | if (ivpu_ipc_match_consumer(vdev, cons, ipc_hdr, jsm_msg)) { | |
3b434a34 | 432 | ivpu_ipc_rx_msg_add(vdev, cons, ipc_hdr, jsm_msg); |
5d7422cf JL |
433 | dispatched = true; |
434 | break; | |
435 | } | |
436 | } | |
3b434a34 | 437 | spin_unlock_irqrestore(&ipc->cons_lock, flags); |
5d7422cf JL |
438 | |
439 | if (!dispatched) { | |
440 | ivpu_dbg(vdev, IPC, "IPC RX msg 0x%x dropped (no consumer)\n", vpu_addr); | |
441 | ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg); | |
442 | } | |
443 | } | |
444 | ||
3b434a34 JL |
445 | if (wake_thread) |
446 | *wake_thread = !list_empty(&ipc->cb_msg_list); | |
447 | } | |
448 | ||
449 | irqreturn_t ivpu_ipc_irq_thread_handler(struct ivpu_device *vdev) | |
450 | { | |
451 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
452 | struct ivpu_ipc_rx_msg *rx_msg, *r; | |
453 | struct list_head cb_msg_list; | |
454 | ||
455 | INIT_LIST_HEAD(&cb_msg_list); | |
456 | ||
457 | spin_lock_irq(&ipc->cons_lock); | |
458 | list_splice_tail_init(&ipc->cb_msg_list, &cb_msg_list); | |
459 | spin_unlock_irq(&ipc->cons_lock); | |
460 | ||
461 | list_for_each_entry_safe(rx_msg, r, &cb_msg_list, link) { | |
462 | rx_msg->callback(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg); | |
463 | ivpu_ipc_rx_msg_del(vdev, rx_msg); | |
464 | } | |
465 | ||
466 | return IRQ_HANDLED; | |
5d7422cf JL |
467 | } |
468 | ||
469 | int ivpu_ipc_init(struct ivpu_device *vdev) | |
470 | { | |
471 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
0a9cd792 | 472 | int ret; |
5d7422cf | 473 | |
42328003 | 474 | ipc->mem_tx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); |
0a9cd792 JL |
475 | if (!ipc->mem_tx) { |
476 | ivpu_err(vdev, "Failed to allocate mem_tx\n"); | |
477 | return -ENOMEM; | |
478 | } | |
5d7422cf | 479 | |
42328003 | 480 | ipc->mem_rx = ivpu_bo_create_global(vdev, SZ_16K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE); |
0a9cd792 JL |
481 | if (!ipc->mem_rx) { |
482 | ivpu_err(vdev, "Failed to allocate mem_rx\n"); | |
483 | ret = -ENOMEM; | |
5d7422cf | 484 | goto err_free_tx; |
0a9cd792 | 485 | } |
5d7422cf JL |
486 | |
487 | ipc->mm_tx = devm_gen_pool_create(vdev->drm.dev, __ffs(IVPU_IPC_ALIGNMENT), | |
488 | -1, "TX_IPC_JSM"); | |
489 | if (IS_ERR(ipc->mm_tx)) { | |
490 | ret = PTR_ERR(ipc->mm_tx); | |
491 | ivpu_err(vdev, "Failed to create gen pool, %pe\n", ipc->mm_tx); | |
492 | goto err_free_rx; | |
493 | } | |
494 | ||
a18f1724 | 495 | ret = gen_pool_add(ipc->mm_tx, ipc->mem_tx->vpu_addr, ivpu_bo_size(ipc->mem_tx), -1); |
5d7422cf JL |
496 | if (ret) { |
497 | ivpu_err(vdev, "gen_pool_add failed, ret %d\n", ret); | |
498 | goto err_free_rx; | |
499 | } | |
500 | ||
3b434a34 | 501 | spin_lock_init(&ipc->cons_lock); |
5d7422cf | 502 | INIT_LIST_HEAD(&ipc->cons_list); |
3b434a34 | 503 | INIT_LIST_HEAD(&ipc->cb_msg_list); |
f0cf7ffc WK |
504 | ret = drmm_mutex_init(&vdev->drm, &ipc->lock); |
505 | if (ret) { | |
506 | ivpu_err(vdev, "Failed to initialize ipc->lock, ret %d\n", ret); | |
507 | goto err_free_rx; | |
508 | } | |
5d7422cf JL |
509 | ivpu_ipc_reset(vdev); |
510 | return 0; | |
511 | ||
512 | err_free_rx: | |
42328003 | 513 | ivpu_bo_free(ipc->mem_rx); |
5d7422cf | 514 | err_free_tx: |
42328003 | 515 | ivpu_bo_free(ipc->mem_tx); |
5d7422cf JL |
516 | return ret; |
517 | } | |
518 | ||
519 | void ivpu_ipc_fini(struct ivpu_device *vdev) | |
520 | { | |
3b434a34 JL |
521 | struct ivpu_ipc_info *ipc = vdev->ipc; |
522 | ||
523 | drm_WARN_ON(&vdev->drm, ipc->on); | |
524 | drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cons_list)); | |
525 | drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list)); | |
526 | drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0); | |
527 | ||
5d7422cf JL |
528 | ivpu_ipc_mem_fini(vdev); |
529 | } | |
530 | ||
531 | void ivpu_ipc_enable(struct ivpu_device *vdev) | |
532 | { | |
533 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
534 | ||
535 | mutex_lock(&ipc->lock); | |
536 | ipc->on = true; | |
537 | mutex_unlock(&ipc->lock); | |
538 | } | |
539 | ||
540 | void ivpu_ipc_disable(struct ivpu_device *vdev) | |
541 | { | |
542 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
543 | struct ivpu_ipc_consumer *cons, *c; | |
3b434a34 JL |
544 | struct ivpu_ipc_rx_msg *rx_msg, *r; |
545 | ||
546 | drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list)); | |
5d7422cf JL |
547 | |
548 | mutex_lock(&ipc->lock); | |
549 | ipc->on = false; | |
550 | mutex_unlock(&ipc->lock); | |
551 | ||
3b434a34 | 552 | spin_lock_irq(&ipc->cons_lock); |
ba6b035d | 553 | list_for_each_entry_safe(cons, c, &ipc->cons_list, link) { |
3b434a34 JL |
554 | spin_lock(&cons->rx_lock); |
555 | if (!cons->rx_callback) | |
b3c10b71 | 556 | cons->aborted = true; |
3b434a34 JL |
557 | list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link) |
558 | ivpu_ipc_rx_msg_del(vdev, rx_msg); | |
559 | spin_unlock(&cons->rx_lock); | |
5d7422cf | 560 | wake_up(&cons->rx_msg_wq); |
ba6b035d | 561 | } |
3b434a34 JL |
562 | spin_unlock_irq(&ipc->cons_lock); |
563 | ||
564 | drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0); | |
5d7422cf JL |
565 | } |
566 | ||
567 | void ivpu_ipc_reset(struct ivpu_device *vdev) | |
568 | { | |
569 | struct ivpu_ipc_info *ipc = vdev->ipc; | |
570 | ||
571 | mutex_lock(&ipc->lock); | |
ba6b035d | 572 | drm_WARN_ON(&vdev->drm, ipc->on); |
5d7422cf | 573 | |
a18f1724 JL |
574 | memset(ivpu_bo_vaddr(ipc->mem_tx), 0, ivpu_bo_size(ipc->mem_tx)); |
575 | memset(ivpu_bo_vaddr(ipc->mem_rx), 0, ivpu_bo_size(ipc->mem_rx)); | |
5d7422cf JL |
576 | wmb(); /* Flush WC buffers for TX and RX rings */ |
577 | ||
578 | mutex_unlock(&ipc->lock); | |
579 | } |