Commit | Line | Data |
---|---|---|
811fe9f5 MW |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2023 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include <linux/bitfield.h> | |
7 | #include <linux/delay.h> | |
8 | ||
9 | #include <drm/drm_managed.h> | |
10 | ||
927b042a | 11 | #include <kunit/static_stub.h> |
2b35ae10 | 12 | #include <kunit/test-bug.h> |
927b042a | 13 | |
811fe9f5 MW |
14 | #include "abi/guc_actions_sriov_abi.h" |
15 | #include "abi/guc_relay_actions_abi.h" | |
16 | #include "abi/guc_relay_communication_abi.h" | |
17 | ||
18 | #include "xe_assert.h" | |
19 | #include "xe_device.h" | |
20 | #include "xe_gt.h" | |
21 | #include "xe_gt_sriov_printk.h" | |
98e62805 | 22 | #include "xe_gt_sriov_pf_service.h" |
811fe9f5 MW |
23 | #include "xe_guc.h" |
24 | #include "xe_guc_ct.h" | |
25 | #include "xe_guc_hxg_helpers.h" | |
26 | #include "xe_guc_relay.h" | |
27 | #include "xe_guc_relay_types.h" | |
28 | #include "xe_sriov.h" | |
29 | ||
30 | /* | |
31 | * How long should we wait for the response? | |
32 | * XXX this value is subject for the profiling. | |
33 | */ | |
34 | #define RELAY_TIMEOUT_MSEC (2500) | |
35 | ||
36 | static void relays_worker_fn(struct work_struct *w); | |
37 | ||
38 | static struct xe_guc *relay_to_guc(struct xe_guc_relay *relay) | |
39 | { | |
40 | return container_of(relay, struct xe_guc, relay); | |
41 | } | |
42 | ||
43 | static struct xe_guc_ct *relay_to_ct(struct xe_guc_relay *relay) | |
44 | { | |
45 | return &relay_to_guc(relay)->ct; | |
46 | } | |
47 | ||
48 | static struct xe_gt *relay_to_gt(struct xe_guc_relay *relay) | |
49 | { | |
50 | return guc_to_gt(relay_to_guc(relay)); | |
51 | } | |
52 | ||
53 | static struct xe_device *relay_to_xe(struct xe_guc_relay *relay) | |
54 | { | |
55 | return gt_to_xe(relay_to_gt(relay)); | |
56 | } | |
57 | ||
58 | #define relay_assert(relay, condition) xe_gt_assert(relay_to_gt(relay), condition) | |
59 | #define relay_notice(relay, msg...) xe_gt_sriov_notice(relay_to_gt(relay), "relay: " msg) | |
60 | #define relay_debug(relay, msg...) xe_gt_sriov_dbg_verbose(relay_to_gt(relay), "relay: " msg) | |
61 | ||
62 | static int relay_get_totalvfs(struct xe_guc_relay *relay) | |
63 | { | |
64 | struct xe_device *xe = relay_to_xe(relay); | |
65 | struct pci_dev *pdev = to_pci_dev(xe->drm.dev); | |
66 | ||
927b042a | 67 | KUNIT_STATIC_STUB_REDIRECT(relay_get_totalvfs, relay); |
811fe9f5 MW |
68 | return IS_SRIOV_VF(xe) ? 0 : pci_sriov_get_totalvfs(pdev); |
69 | } | |
70 | ||
71 | static bool relay_is_ready(struct xe_guc_relay *relay) | |
72 | { | |
73 | return mempool_initialized(&relay->pool); | |
74 | } | |
75 | ||
76 | static u32 relay_get_next_rid(struct xe_guc_relay *relay) | |
77 | { | |
78 | u32 rid; | |
79 | ||
80 | spin_lock(&relay->lock); | |
81 | rid = ++relay->last_rid; | |
82 | spin_unlock(&relay->lock); | |
83 | ||
84 | return rid; | |
85 | } | |
86 | ||
87 | /** | |
88 | * struct relay_transaction - internal data used to handle transactions | |
89 | * | |
90 | * Relation between struct relay_transaction members:: | |
91 | * | |
92 | * <-------------------- GUC_CTB_MAX_DWORDS --------------> | |
93 | * <-------- GUC_RELAY_MSG_MAX_LEN ---> | |
94 | * <--- offset ---> <--- request_len -------> | |
95 | * +----------------+-------------------------+----------+--+ | |
96 | * | | | | | | |
97 | * +----------------+-------------------------+----------+--+ | |
98 | * ^ ^ | |
99 | * / / | |
100 | * request_buf request | |
101 | * | |
102 | * <-------------------- GUC_CTB_MAX_DWORDS --------------> | |
103 | * <-------- GUC_RELAY_MSG_MAX_LEN ---> | |
104 | * <--- offset ---> <--- response_len ---> | |
105 | * +----------------+----------------------+-------------+--+ | |
106 | * | | | | | | |
107 | * +----------------+----------------------+-------------+--+ | |
108 | * ^ ^ | |
109 | * / / | |
110 | * response_buf response | |
111 | */ | |
112 | struct relay_transaction { | |
113 | /** | |
114 | * @incoming: indicates whether this transaction represents an incoming | |
115 | * request from the remote VF/PF or this transaction | |
116 | * represents outgoing request to the remote VF/PF. | |
117 | */ | |
118 | bool incoming; | |
119 | ||
120 | /** | |
121 | * @remote: PF/VF identifier of the origin (or target) of the relay | |
122 | * request message. | |
123 | */ | |
124 | u32 remote; | |
125 | ||
126 | /** @rid: identifier of the VF/PF relay message. */ | |
127 | u32 rid; | |
128 | ||
129 | /** | |
130 | * @request: points to the inner VF/PF request message, copied to the | |
131 | * #response_buf starting at #offset. | |
132 | */ | |
133 | u32 *request; | |
134 | ||
135 | /** @request_len: length of the inner VF/PF request message. */ | |
136 | u32 request_len; | |
137 | ||
138 | /** | |
139 | * @response: points to the placeholder buffer where inner VF/PF | |
140 | * response will be located, for outgoing transaction | |
141 | * this could be caller's buffer (if provided) otherwise | |
142 | * it points to the #response_buf starting at #offset. | |
143 | */ | |
144 | u32 *response; | |
145 | ||
146 | /** | |
147 | * @response_len: length of the inner VF/PF response message (only | |
148 | * if #status is 0), initially set to the size of the | |
149 | * placeholder buffer where response message will be | |
150 | * copied. | |
151 | */ | |
152 | u32 response_len; | |
153 | ||
154 | /** | |
155 | * @offset: offset to the start of the inner VF/PF relay message inside | |
156 | * buffers; this offset is equal the length of the outer GuC | |
157 | * relay header message. | |
158 | */ | |
159 | u32 offset; | |
160 | ||
161 | /** | |
162 | * @request_buf: buffer with VF/PF request message including outer | |
163 | * transport message. | |
164 | */ | |
165 | u32 request_buf[GUC_CTB_MAX_DWORDS]; | |
166 | ||
167 | /** | |
168 | * @response_buf: buffer with VF/PF response message including outer | |
169 | * transport message. | |
170 | */ | |
171 | u32 response_buf[GUC_CTB_MAX_DWORDS]; | |
172 | ||
173 | /** | |
174 | * @reply: status of the reply, 0 means that data pointed by the | |
175 | * #response is valid. | |
176 | */ | |
177 | int reply; | |
178 | ||
179 | /** @done: completion of the outgoing transaction. */ | |
180 | struct completion done; | |
181 | ||
182 | /** @link: transaction list link */ | |
183 | struct list_head link; | |
184 | }; | |
185 | ||
186 | static u32 prepare_pf2guc(u32 *msg, u32 target, u32 rid) | |
187 | { | |
188 | msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | | |
189 | FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | | |
190 | FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_PF2GUC_RELAY_TO_VF); | |
191 | msg[1] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_1_VFID, target); | |
192 | msg[2] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_2_RELAY_ID, rid); | |
193 | ||
194 | return PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN; | |
195 | } | |
196 | ||
197 | static u32 prepare_vf2guc(u32 *msg, u32 rid) | |
198 | { | |
199 | msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | | |
200 | FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | | |
201 | FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_VF2GUC_RELAY_TO_PF); | |
202 | msg[1] = FIELD_PREP(VF2GUC_RELAY_TO_PF_REQUEST_MSG_1_RELAY_ID, rid); | |
203 | ||
204 | return VF2GUC_RELAY_TO_PF_REQUEST_MSG_MIN_LEN; | |
205 | } | |
206 | ||
207 | static struct relay_transaction * | |
208 | __relay_get_transaction(struct xe_guc_relay *relay, bool incoming, u32 remote, u32 rid, | |
209 | const u32 *action, u32 action_len, u32 *resp, u32 resp_size) | |
210 | { | |
211 | struct relay_transaction *txn; | |
212 | ||
213 | relay_assert(relay, action_len >= GUC_RELAY_MSG_MIN_LEN); | |
214 | relay_assert(relay, action_len <= GUC_RELAY_MSG_MAX_LEN); | |
215 | relay_assert(relay, !(!!resp ^ !!resp_size)); | |
216 | relay_assert(relay, resp_size <= GUC_RELAY_MSG_MAX_LEN); | |
217 | relay_assert(relay, resp_size == 0 || resp_size >= GUC_RELAY_MSG_MIN_LEN); | |
218 | ||
219 | if (unlikely(!relay_is_ready(relay))) | |
220 | return ERR_PTR(-ENODEV); | |
221 | ||
222 | /* | |
223 | * For incoming requests we can't use GFP_KERNEL as those are delivered | |
224 | * with CTB lock held which is marked as used in the reclaim path. | |
225 | * Btw, that's one of the reason why we use mempool here! | |
226 | */ | |
227 | txn = mempool_alloc(&relay->pool, incoming ? GFP_ATOMIC : GFP_KERNEL); | |
228 | if (!txn) | |
229 | return ERR_PTR(-ENOMEM); | |
230 | ||
231 | txn->incoming = incoming; | |
232 | txn->remote = remote; | |
233 | txn->rid = rid; | |
234 | txn->offset = remote ? | |
235 | prepare_pf2guc(incoming ? txn->response_buf : txn->request_buf, remote, rid) : | |
236 | prepare_vf2guc(incoming ? txn->response_buf : txn->request_buf, rid); | |
237 | ||
238 | relay_assert(relay, txn->offset); | |
239 | relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->request_buf)); | |
240 | relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->response_buf)); | |
241 | ||
242 | txn->request = txn->request_buf + txn->offset; | |
243 | memcpy(&txn->request_buf[txn->offset], action, sizeof(u32) * action_len); | |
244 | txn->request_len = action_len; | |
245 | ||
246 | txn->response = resp ?: txn->response_buf + txn->offset; | |
247 | txn->response_len = resp_size ?: GUC_RELAY_MSG_MAX_LEN; | |
248 | txn->reply = -ENOMSG; | |
249 | INIT_LIST_HEAD(&txn->link); | |
250 | init_completion(&txn->done); | |
251 | ||
252 | return txn; | |
253 | } | |
254 | ||
255 | static struct relay_transaction * | |
256 | relay_new_transaction(struct xe_guc_relay *relay, u32 target, const u32 *action, u32 len, | |
257 | u32 *resp, u32 resp_size) | |
258 | { | |
259 | u32 rid = relay_get_next_rid(relay); | |
260 | ||
261 | return __relay_get_transaction(relay, false, target, rid, action, len, resp, resp_size); | |
262 | } | |
263 | ||
264 | static struct relay_transaction * | |
265 | relay_new_incoming_transaction(struct xe_guc_relay *relay, u32 origin, u32 rid, | |
266 | const u32 *action, u32 len) | |
267 | { | |
268 | return __relay_get_transaction(relay, true, origin, rid, action, len, NULL, 0); | |
269 | } | |
270 | ||
271 | static void relay_release_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn) | |
272 | { | |
273 | relay_assert(relay, list_empty(&txn->link)); | |
274 | ||
275 | txn->offset = 0; | |
276 | txn->response = NULL; | |
277 | txn->reply = -ESTALE; | |
278 | mempool_free(txn, &relay->pool); | |
279 | } | |
280 | ||
281 | static int relay_send_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn) | |
282 | { | |
283 | u32 len = txn->incoming ? txn->response_len : txn->request_len; | |
284 | u32 *buf = txn->incoming ? txn->response_buf : txn->request_buf; | |
285 | u32 *msg = buf + txn->offset; | |
286 | int ret; | |
287 | ||
288 | relay_assert(relay, txn->offset); | |
289 | relay_assert(relay, txn->offset + len <= GUC_CTB_MAX_DWORDS); | |
290 | relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN); | |
291 | relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN); | |
292 | ||
293 | relay_debug(relay, "sending %s.%u to %u = %*ph\n", | |
294 | guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])), | |
295 | txn->rid, txn->remote, (int)sizeof(u32) * len, msg); | |
296 | ||
297 | ret = xe_guc_ct_send_block(relay_to_ct(relay), buf, len + txn->offset); | |
298 | ||
299 | if (unlikely(ret > 0)) { | |
300 | relay_notice(relay, "Unexpected data=%d from GuC, wrong ABI?\n", ret); | |
301 | ret = -EPROTO; | |
302 | } | |
303 | if (unlikely(ret < 0)) { | |
304 | relay_notice(relay, "Failed to send %s.%x to GuC (%pe) %*ph ...\n", | |
305 | guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, buf[0])), | |
306 | FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, buf[0]), | |
307 | ERR_PTR(ret), (int)sizeof(u32) * txn->offset, buf); | |
308 | relay_notice(relay, "Failed to send %s.%u to %u (%pe) %*ph\n", | |
309 | guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])), | |
310 | txn->rid, txn->remote, ERR_PTR(ret), (int)sizeof(u32) * len, msg); | |
311 | } | |
312 | ||
313 | return ret; | |
314 | } | |
315 | ||
316 | static void __fini_relay(struct drm_device *drm, void *arg) | |
317 | { | |
318 | struct xe_guc_relay *relay = arg; | |
319 | ||
320 | mempool_exit(&relay->pool); | |
321 | } | |
322 | ||
323 | /** | |
324 | * xe_guc_relay_init - Initialize a &xe_guc_relay | |
325 | * @relay: the &xe_guc_relay to initialize | |
326 | * | |
327 | * Initialize remaining members of &xe_guc_relay that may depend | |
328 | * on the SR-IOV mode. | |
329 | * | |
330 | * Return: 0 on success or a negative error code on failure. | |
331 | */ | |
332 | int xe_guc_relay_init(struct xe_guc_relay *relay) | |
333 | { | |
334 | const int XE_RELAY_MEMPOOL_MIN_NUM = 1; | |
335 | struct xe_device *xe = relay_to_xe(relay); | |
336 | int err; | |
337 | ||
338 | relay_assert(relay, !relay_is_ready(relay)); | |
339 | ||
340 | if (!IS_SRIOV(xe)) | |
341 | return 0; | |
342 | ||
343 | spin_lock_init(&relay->lock); | |
344 | INIT_WORK(&relay->worker, relays_worker_fn); | |
345 | INIT_LIST_HEAD(&relay->pending_relays); | |
346 | INIT_LIST_HEAD(&relay->incoming_actions); | |
347 | ||
348 | err = mempool_init_kmalloc_pool(&relay->pool, XE_RELAY_MEMPOOL_MIN_NUM + | |
349 | relay_get_totalvfs(relay), | |
350 | sizeof(struct relay_transaction)); | |
351 | if (err) | |
352 | return err; | |
353 | ||
354 | relay_debug(relay, "using mempool with %d elements\n", relay->pool.min_nr); | |
355 | ||
356 | return drmm_add_action_or_reset(&xe->drm, __fini_relay, relay); | |
357 | } | |
358 | ||
359 | static u32 to_relay_error(int err) | |
360 | { | |
361 | /* XXX: assume that relay errors match errno codes */ | |
362 | return err < 0 ? -err : GUC_RELAY_ERROR_UNDISCLOSED; | |
363 | } | |
364 | ||
365 | static int from_relay_error(u32 error) | |
366 | { | |
367 | /* XXX: assume that relay errors match errno codes */ | |
368 | return error ? -error : -ENODATA; | |
369 | } | |
370 | ||
371 | static u32 sanitize_relay_error(u32 error) | |
372 | { | |
373 | /* XXX TBD if generic error codes will be allowed */ | |
374 | if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG)) | |
375 | error = GUC_RELAY_ERROR_UNDISCLOSED; | |
376 | return error; | |
377 | } | |
378 | ||
379 | static u32 sanitize_relay_error_hint(u32 hint) | |
380 | { | |
381 | /* XXX TBD if generic error codes will be allowed */ | |
382 | if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG)) | |
383 | hint = 0; | |
384 | return hint; | |
385 | } | |
386 | ||
387 | static u32 prepare_error_reply(u32 *msg, u32 error, u32 hint) | |
388 | { | |
389 | msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | | |
390 | FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_FAILURE) | | |
391 | FIELD_PREP(GUC_HXG_FAILURE_MSG_0_HINT, hint) | | |
392 | FIELD_PREP(GUC_HXG_FAILURE_MSG_0_ERROR, error); | |
393 | ||
394 | XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_ERROR, error)); | |
395 | XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_HINT, hint)); | |
396 | ||
397 | return GUC_HXG_FAILURE_MSG_LEN; | |
398 | } | |
399 | ||
927b042a MW |
400 | static void relay_testonly_nop(struct xe_guc_relay *relay) |
401 | { | |
402 | KUNIT_STATIC_STUB_REDIRECT(relay_testonly_nop, relay); | |
403 | } | |
404 | ||
811fe9f5 MW |
405 | static int relay_send_message_and_wait(struct xe_guc_relay *relay, |
406 | struct relay_transaction *txn, | |
407 | u32 *buf, u32 buf_size) | |
408 | { | |
409 | unsigned long timeout = msecs_to_jiffies(RELAY_TIMEOUT_MSEC); | |
410 | u32 *msg = &txn->request_buf[txn->offset]; | |
411 | u32 len = txn->request_len; | |
412 | u32 type, action, data0; | |
413 | int ret; | |
414 | long n; | |
415 | ||
416 | type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); | |
417 | action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]); | |
418 | data0 = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]); | |
419 | ||
420 | relay_debug(relay, "%s.%u to %u action %#x:%u\n", | |
421 | guc_hxg_type_to_string(type), | |
422 | txn->rid, txn->remote, action, data0); | |
423 | ||
424 | /* list ordering does not need to match RID ordering */ | |
425 | spin_lock(&relay->lock); | |
426 | list_add_tail(&txn->link, &relay->pending_relays); | |
427 | spin_unlock(&relay->lock); | |
428 | ||
429 | resend: | |
430 | ret = relay_send_transaction(relay, txn); | |
431 | if (unlikely(ret < 0)) | |
432 | goto unlink; | |
433 | ||
434 | wait: | |
435 | n = wait_for_completion_timeout(&txn->done, timeout); | |
436 | if (unlikely(n == 0 && txn->reply)) { | |
437 | ret = -ETIME; | |
438 | goto unlink; | |
439 | } | |
440 | ||
441 | relay_debug(relay, "%u.%u reply %d after %u msec\n", | |
442 | txn->remote, txn->rid, txn->reply, jiffies_to_msecs(timeout - n)); | |
443 | if (unlikely(txn->reply)) { | |
444 | reinit_completion(&txn->done); | |
445 | if (txn->reply == -EAGAIN) | |
446 | goto resend; | |
927b042a MW |
447 | if (txn->reply == -EBUSY) { |
448 | relay_testonly_nop(relay); | |
811fe9f5 | 449 | goto wait; |
927b042a | 450 | } |
811fe9f5 MW |
451 | if (txn->reply > 0) |
452 | ret = from_relay_error(txn->reply); | |
453 | else | |
454 | ret = txn->reply; | |
455 | goto unlink; | |
456 | } | |
457 | ||
458 | relay_debug(relay, "%u.%u response %*ph\n", txn->remote, txn->rid, | |
459 | (int)sizeof(u32) * txn->response_len, txn->response); | |
460 | relay_assert(relay, txn->response_len >= GUC_RELAY_MSG_MIN_LEN); | |
461 | ret = txn->response_len; | |
462 | ||
463 | unlink: | |
464 | spin_lock(&relay->lock); | |
465 | list_del_init(&txn->link); | |
466 | spin_unlock(&relay->lock); | |
467 | ||
468 | if (unlikely(ret < 0)) { | |
469 | relay_notice(relay, "Unsuccessful %s.%u %#x:%u to %u (%pe) %*ph\n", | |
470 | guc_hxg_type_to_string(type), txn->rid, | |
471 | action, data0, txn->remote, ERR_PTR(ret), | |
472 | (int)sizeof(u32) * len, msg); | |
473 | } | |
474 | ||
475 | return ret; | |
476 | } | |
477 | ||
478 | static int relay_send_to(struct xe_guc_relay *relay, u32 target, | |
479 | const u32 *msg, u32 len, u32 *buf, u32 buf_size) | |
480 | { | |
481 | struct relay_transaction *txn; | |
482 | int ret; | |
483 | ||
484 | relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN); | |
485 | relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN); | |
486 | relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_HOST); | |
487 | relay_assert(relay, guc_hxg_type_is_action(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]))); | |
488 | ||
489 | if (unlikely(!relay_is_ready(relay))) | |
490 | return -ENODEV; | |
491 | ||
492 | txn = relay_new_transaction(relay, target, msg, len, buf, buf_size); | |
493 | if (IS_ERR(txn)) | |
494 | return PTR_ERR(txn); | |
495 | ||
496 | switch (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])) { | |
497 | case GUC_HXG_TYPE_REQUEST: | |
498 | ret = relay_send_message_and_wait(relay, txn, buf, buf_size); | |
499 | break; | |
500 | case GUC_HXG_TYPE_FAST_REQUEST: | |
501 | relay_assert(relay, !GUC_HXG_TYPE_FAST_REQUEST); | |
502 | fallthrough; | |
503 | case GUC_HXG_TYPE_EVENT: | |
504 | ret = relay_send_transaction(relay, txn); | |
505 | break; | |
506 | default: | |
507 | ret = -EINVAL; | |
508 | break; | |
509 | } | |
510 | ||
511 | relay_release_transaction(relay, txn); | |
512 | return ret; | |
513 | } | |
514 | ||
515 | #ifdef CONFIG_PCI_IOV | |
516 | /** | |
517 | * xe_guc_relay_send_to_vf - Send a message to the VF. | |
518 | * @relay: the &xe_guc_relay which will send the message | |
519 | * @target: target VF number | |
520 | * @msg: request message to be sent | |
521 | * @len: length of the request message (in dwords, can't be 0) | |
522 | * @buf: placeholder for the response message | |
523 | * @buf_size: size of the response message placeholder (in dwords) | |
524 | * | |
525 | * This function can only be used by the driver running in the SR-IOV PF mode. | |
526 | * | |
527 | * Return: Non-negative response length (in dwords) or | |
528 | * a negative error code on failure. | |
529 | */ | |
530 | int xe_guc_relay_send_to_vf(struct xe_guc_relay *relay, u32 target, | |
531 | const u32 *msg, u32 len, u32 *buf, u32 buf_size) | |
532 | { | |
533 | relay_assert(relay, IS_SRIOV_PF(relay_to_xe(relay))); | |
534 | ||
535 | return relay_send_to(relay, target, msg, len, buf, buf_size); | |
536 | } | |
537 | #endif | |
538 | ||
539 | /** | |
540 | * xe_guc_relay_send_to_pf - Send a message to the PF. | |
541 | * @relay: the &xe_guc_relay which will send the message | |
542 | * @msg: request message to be sent | |
543 | * @len: length of the message (in dwords, can't be 0) | |
544 | * @buf: placeholder for the response message | |
545 | * @buf_size: size of the response message placeholder (in dwords) | |
546 | * | |
547 | * This function can only be used by driver running in SR-IOV VF mode. | |
548 | * | |
549 | * Return: Non-negative response length (in dwords) or | |
550 | * a negative error code on failure. | |
551 | */ | |
552 | int xe_guc_relay_send_to_pf(struct xe_guc_relay *relay, | |
553 | const u32 *msg, u32 len, u32 *buf, u32 buf_size) | |
554 | { | |
555 | relay_assert(relay, IS_SRIOV_VF(relay_to_xe(relay))); | |
556 | ||
557 | return relay_send_to(relay, PFID, msg, len, buf, buf_size); | |
558 | } | |
559 | ||
560 | static int relay_handle_reply(struct xe_guc_relay *relay, u32 origin, | |
561 | u32 rid, int reply, const u32 *msg, u32 len) | |
562 | { | |
563 | struct relay_transaction *pending; | |
564 | int err = -ESRCH; | |
565 | ||
566 | spin_lock(&relay->lock); | |
567 | list_for_each_entry(pending, &relay->pending_relays, link) { | |
568 | if (pending->remote != origin || pending->rid != rid) { | |
569 | relay_debug(relay, "%u.%u still awaits response\n", | |
570 | pending->remote, pending->rid); | |
571 | continue; | |
572 | } | |
573 | err = 0; /* found! */ | |
574 | if (reply == 0) { | |
575 | if (len > pending->response_len) { | |
576 | reply = -ENOBUFS; | |
577 | err = -ENOBUFS; | |
578 | } else { | |
579 | memcpy(pending->response, msg, 4 * len); | |
580 | pending->response_len = len; | |
581 | } | |
582 | } | |
583 | pending->reply = reply; | |
584 | complete_all(&pending->done); | |
585 | break; | |
586 | } | |
587 | spin_unlock(&relay->lock); | |
588 | ||
589 | return err; | |
590 | } | |
591 | ||
592 | static int relay_handle_failure(struct xe_guc_relay *relay, u32 origin, | |
593 | u32 rid, const u32 *msg, u32 len) | |
594 | { | |
595 | int error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, msg[0]); | |
596 | u32 hint __maybe_unused = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, msg[0]); | |
597 | ||
598 | relay_assert(relay, len); | |
599 | relay_debug(relay, "%u.%u error %#x (%pe) hint %u debug %*ph\n", | |
600 | origin, rid, error, ERR_PTR(-error), hint, 4 * (len - 1), msg + 1); | |
601 | ||
602 | return relay_handle_reply(relay, origin, rid, error ?: -EREMOTEIO, NULL, 0); | |
603 | } | |
604 | ||
605 | static int relay_testloop_action_handler(struct xe_guc_relay *relay, u32 origin, | |
606 | const u32 *msg, u32 len, u32 *response, u32 size) | |
607 | { | |
608 | static ktime_t last_reply = 0; | |
609 | u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); | |
610 | u32 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]); | |
611 | u32 opcode = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]); | |
612 | ktime_t now = ktime_get(); | |
613 | bool busy; | |
614 | int ret; | |
615 | ||
616 | relay_assert(relay, guc_hxg_type_is_action(type)); | |
617 | relay_assert(relay, action == GUC_RELAY_ACTION_VFXPF_TESTLOOP); | |
618 | ||
619 | if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) | |
620 | return -ECONNREFUSED; | |
621 | ||
622 | if (!last_reply) | |
623 | last_reply = now; | |
624 | busy = ktime_before(now, ktime_add_ms(last_reply, 2 * RELAY_TIMEOUT_MSEC)); | |
625 | if (!busy) | |
626 | last_reply = now; | |
627 | ||
628 | switch (opcode) { | |
629 | case VFXPF_TESTLOOP_OPCODE_NOP: | |
630 | if (type == GUC_HXG_TYPE_EVENT) | |
631 | return 0; | |
632 | return guc_hxg_msg_encode_success(response, 0); | |
633 | case VFXPF_TESTLOOP_OPCODE_BUSY: | |
634 | if (type == GUC_HXG_TYPE_EVENT) | |
635 | return -EPROTO; | |
636 | msleep(RELAY_TIMEOUT_MSEC / 8); | |
637 | if (busy) | |
638 | return -EINPROGRESS; | |
639 | return guc_hxg_msg_encode_success(response, 0); | |
640 | case VFXPF_TESTLOOP_OPCODE_RETRY: | |
641 | if (type == GUC_HXG_TYPE_EVENT) | |
642 | return -EPROTO; | |
643 | msleep(RELAY_TIMEOUT_MSEC / 8); | |
644 | if (busy) | |
645 | return guc_hxg_msg_encode_retry(response, 0); | |
646 | return guc_hxg_msg_encode_success(response, 0); | |
647 | case VFXPF_TESTLOOP_OPCODE_ECHO: | |
648 | if (type == GUC_HXG_TYPE_EVENT) | |
649 | return -EPROTO; | |
650 | if (size < len) | |
651 | return -ENOBUFS; | |
652 | ret = guc_hxg_msg_encode_success(response, len); | |
653 | memcpy(response + ret, msg + ret, (len - ret) * sizeof(u32)); | |
654 | return len; | |
655 | case VFXPF_TESTLOOP_OPCODE_FAIL: | |
656 | return -EHWPOISON; | |
657 | default: | |
658 | break; | |
659 | } | |
660 | ||
661 | relay_notice(relay, "Unexpected action %#x opcode %#x\n", action, opcode); | |
662 | return -EBADRQC; | |
663 | } | |
664 | ||
665 | static int relay_action_handler(struct xe_guc_relay *relay, u32 origin, | |
666 | const u32 *msg, u32 len, u32 *response, u32 size) | |
667 | { | |
98e62805 | 668 | struct xe_gt *gt = relay_to_gt(relay); |
811fe9f5 MW |
669 | u32 type; |
670 | int ret; | |
671 | ||
672 | relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN); | |
673 | ||
674 | if (FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]) == GUC_RELAY_ACTION_VFXPF_TESTLOOP) | |
675 | return relay_testloop_action_handler(relay, origin, msg, len, response, size); | |
676 | ||
677 | type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); | |
678 | ||
98e62805 MW |
679 | if (IS_SRIOV_PF(relay_to_xe(relay))) |
680 | ret = xe_gt_sriov_pf_service_process_request(gt, origin, msg, len, response, size); | |
681 | else | |
682 | ret = -EOPNOTSUPP; | |
811fe9f5 MW |
683 | |
684 | if (type == GUC_HXG_TYPE_EVENT) | |
685 | relay_assert(relay, ret <= 0); | |
686 | ||
687 | return ret; | |
688 | } | |
689 | ||
690 | static struct relay_transaction *relay_dequeue_transaction(struct xe_guc_relay *relay) | |
691 | { | |
692 | struct relay_transaction *txn; | |
693 | ||
694 | spin_lock(&relay->lock); | |
695 | txn = list_first_entry_or_null(&relay->incoming_actions, struct relay_transaction, link); | |
696 | if (txn) | |
697 | list_del_init(&txn->link); | |
698 | spin_unlock(&relay->lock); | |
699 | ||
700 | return txn; | |
701 | } | |
702 | ||
703 | static void relay_process_incoming_action(struct xe_guc_relay *relay) | |
704 | { | |
705 | struct relay_transaction *txn; | |
706 | bool again = false; | |
707 | u32 type; | |
708 | int ret; | |
709 | ||
710 | txn = relay_dequeue_transaction(relay); | |
711 | if (!txn) | |
712 | return; | |
713 | ||
714 | type = FIELD_GET(GUC_HXG_MSG_0_TYPE, txn->request_buf[txn->offset]); | |
715 | ||
716 | ret = relay_action_handler(relay, txn->remote, | |
717 | txn->request_buf + txn->offset, txn->request_len, | |
718 | txn->response_buf + txn->offset, | |
719 | ARRAY_SIZE(txn->response_buf) - txn->offset); | |
720 | ||
721 | if (ret == -EINPROGRESS) { | |
722 | again = true; | |
723 | ret = guc_hxg_msg_encode_busy(txn->response_buf + txn->offset, 0); | |
724 | } | |
725 | ||
726 | if (ret > 0) { | |
727 | txn->response_len = ret; | |
728 | ret = relay_send_transaction(relay, txn); | |
729 | } | |
730 | ||
731 | if (ret < 0) { | |
732 | u32 error = to_relay_error(ret); | |
733 | ||
734 | relay_notice(relay, "Failed to handle %s.%u from %u (%pe) %*ph\n", | |
735 | guc_hxg_type_to_string(type), txn->rid, txn->remote, | |
736 | ERR_PTR(ret), 4 * txn->request_len, txn->request_buf + txn->offset); | |
737 | ||
738 | txn->response_len = prepare_error_reply(txn->response_buf + txn->offset, | |
739 | txn->remote ? | |
740 | sanitize_relay_error(error) : error, | |
741 | txn->remote ? | |
742 | sanitize_relay_error_hint(-ret) : -ret); | |
743 | ret = relay_send_transaction(relay, txn); | |
744 | again = false; | |
745 | } | |
746 | ||
747 | if (again) { | |
748 | spin_lock(&relay->lock); | |
749 | list_add(&txn->link, &relay->incoming_actions); | |
750 | spin_unlock(&relay->lock); | |
751 | return; | |
752 | } | |
753 | ||
754 | if (unlikely(ret < 0)) | |
755 | relay_notice(relay, "Failed to process action.%u (%pe) %*ph\n", | |
756 | txn->rid, ERR_PTR(ret), 4 * txn->request_len, | |
757 | txn->request_buf + txn->offset); | |
758 | ||
759 | relay_release_transaction(relay, txn); | |
760 | } | |
761 | ||
762 | static bool relay_needs_worker(struct xe_guc_relay *relay) | |
763 | { | |
8b01f970 ND |
764 | bool is_empty; |
765 | ||
766 | spin_lock(&relay->lock); | |
767 | is_empty = list_empty(&relay->incoming_actions); | |
768 | spin_unlock(&relay->lock); | |
769 | ||
770 | return !is_empty; | |
771 | ||
811fe9f5 MW |
772 | } |
773 | ||
774 | static void relay_kick_worker(struct xe_guc_relay *relay) | |
775 | { | |
927b042a | 776 | KUNIT_STATIC_STUB_REDIRECT(relay_kick_worker, relay); |
811fe9f5 MW |
777 | queue_work(relay_to_xe(relay)->sriov.wq, &relay->worker); |
778 | } | |
779 | ||
780 | static void relays_worker_fn(struct work_struct *w) | |
781 | { | |
782 | struct xe_guc_relay *relay = container_of(w, struct xe_guc_relay, worker); | |
783 | ||
784 | relay_process_incoming_action(relay); | |
785 | ||
786 | if (relay_needs_worker(relay)) | |
787 | relay_kick_worker(relay); | |
788 | } | |
789 | ||
790 | static int relay_queue_action_msg(struct xe_guc_relay *relay, u32 origin, u32 rid, | |
791 | const u32 *msg, u32 len) | |
792 | { | |
793 | struct relay_transaction *txn; | |
794 | ||
795 | txn = relay_new_incoming_transaction(relay, origin, rid, msg, len); | |
796 | if (IS_ERR(txn)) | |
797 | return PTR_ERR(txn); | |
798 | ||
799 | spin_lock(&relay->lock); | |
800 | list_add_tail(&txn->link, &relay->incoming_actions); | |
801 | spin_unlock(&relay->lock); | |
802 | ||
803 | relay_kick_worker(relay); | |
804 | return 0; | |
805 | } | |
806 | ||
807 | static int relay_process_msg(struct xe_guc_relay *relay, u32 origin, u32 rid, | |
808 | const u32 *msg, u32 len) | |
809 | { | |
810 | u32 type; | |
811 | int err; | |
812 | ||
813 | if (unlikely(len < GUC_HXG_MSG_MIN_LEN)) | |
814 | return -EPROTO; | |
815 | ||
816 | if (FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) != GUC_HXG_ORIGIN_HOST) | |
817 | return -EPROTO; | |
818 | ||
819 | type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]); | |
820 | relay_debug(relay, "received %s.%u from %u = %*ph\n", | |
821 | guc_hxg_type_to_string(type), rid, origin, 4 * len, msg); | |
822 | ||
823 | switch (type) { | |
824 | case GUC_HXG_TYPE_REQUEST: | |
825 | case GUC_HXG_TYPE_FAST_REQUEST: | |
826 | case GUC_HXG_TYPE_EVENT: | |
827 | err = relay_queue_action_msg(relay, origin, rid, msg, len); | |
828 | break; | |
829 | case GUC_HXG_TYPE_RESPONSE_SUCCESS: | |
830 | err = relay_handle_reply(relay, origin, rid, 0, msg, len); | |
831 | break; | |
832 | case GUC_HXG_TYPE_NO_RESPONSE_BUSY: | |
833 | err = relay_handle_reply(relay, origin, rid, -EBUSY, NULL, 0); | |
834 | break; | |
835 | case GUC_HXG_TYPE_NO_RESPONSE_RETRY: | |
836 | err = relay_handle_reply(relay, origin, rid, -EAGAIN, NULL, 0); | |
837 | break; | |
838 | case GUC_HXG_TYPE_RESPONSE_FAILURE: | |
839 | err = relay_handle_failure(relay, origin, rid, msg, len); | |
840 | break; | |
841 | default: | |
842 | err = -EBADRQC; | |
843 | } | |
844 | ||
845 | if (unlikely(err)) | |
846 | relay_notice(relay, "Failed to process %s.%u from %u (%pe) %*ph\n", | |
847 | guc_hxg_type_to_string(type), rid, origin, | |
848 | ERR_PTR(err), 4 * len, msg); | |
849 | ||
850 | return err; | |
851 | } | |
852 | ||
853 | /** | |
854 | * xe_guc_relay_process_guc2vf - Handle relay notification message from the GuC. | |
855 | * @relay: the &xe_guc_relay which will handle the message | |
856 | * @msg: message to be handled | |
857 | * @len: length of the message (in dwords) | |
858 | * | |
859 | * This function will handle relay messages received from the GuC. | |
860 | * | |
861 | * This function is can only be used if driver is running in SR-IOV mode. | |
862 | * | |
863 | * Return: 0 on success or a negative error code on failure. | |
864 | */ | |
865 | int xe_guc_relay_process_guc2vf(struct xe_guc_relay *relay, const u32 *msg, u32 len) | |
866 | { | |
867 | u32 rid; | |
868 | ||
869 | relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN); | |
870 | relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC); | |
871 | relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT); | |
872 | relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) == | |
873 | XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF); | |
874 | ||
927b042a | 875 | if (unlikely(!IS_SRIOV_VF(relay_to_xe(relay)) && !kunit_get_current_test())) |
811fe9f5 MW |
876 | return -EPERM; |
877 | ||
878 | if (unlikely(!relay_is_ready(relay))) | |
879 | return -ENODEV; | |
880 | ||
881 | if (unlikely(len < GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN)) | |
882 | return -EPROTO; | |
883 | ||
884 | if (unlikely(len > GUC2VF_RELAY_FROM_PF_EVENT_MSG_MAX_LEN)) | |
885 | return -EMSGSIZE; | |
886 | ||
887 | if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0]))) | |
888 | return -EPFNOSUPPORT; | |
889 | ||
890 | rid = FIELD_GET(GUC2VF_RELAY_FROM_PF_EVENT_MSG_1_RELAY_ID, msg[1]); | |
891 | ||
892 | return relay_process_msg(relay, PFID, rid, | |
893 | msg + GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN, | |
894 | len - GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN); | |
895 | } | |
896 | ||
897 | #ifdef CONFIG_PCI_IOV | |
898 | /** | |
899 | * xe_guc_relay_process_guc2pf - Handle relay notification message from the GuC. | |
900 | * @relay: the &xe_guc_relay which will handle the message | |
901 | * @msg: message to be handled | |
902 | * @len: length of the message (in dwords) | |
903 | * | |
904 | * This function will handle relay messages received from the GuC. | |
905 | * | |
906 | * This function can only be used if driver is running in SR-IOV PF mode. | |
907 | * | |
908 | * Return: 0 on success or a negative error code on failure. | |
909 | */ | |
910 | int xe_guc_relay_process_guc2pf(struct xe_guc_relay *relay, const u32 *msg, u32 len) | |
911 | { | |
912 | u32 origin, rid; | |
913 | int err; | |
914 | ||
915 | relay_assert(relay, len >= GUC_HXG_EVENT_MSG_MIN_LEN); | |
916 | relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC); | |
917 | relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT); | |
918 | relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) == | |
919 | XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF); | |
920 | ||
927b042a | 921 | if (unlikely(!IS_SRIOV_PF(relay_to_xe(relay)) && !kunit_get_current_test())) |
811fe9f5 MW |
922 | return -EPERM; |
923 | ||
924 | if (unlikely(!relay_is_ready(relay))) | |
925 | return -ENODEV; | |
926 | ||
927 | if (unlikely(len < GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN)) | |
928 | return -EPROTO; | |
929 | ||
930 | if (unlikely(len > GUC2PF_RELAY_FROM_VF_EVENT_MSG_MAX_LEN)) | |
931 | return -EMSGSIZE; | |
932 | ||
933 | if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0]))) | |
934 | return -EPFNOSUPPORT; | |
935 | ||
936 | origin = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_1_VFID, msg[1]); | |
937 | rid = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_2_RELAY_ID, msg[2]); | |
938 | ||
939 | if (unlikely(origin > relay_get_totalvfs(relay))) | |
940 | return -ENOENT; | |
941 | ||
942 | err = relay_process_msg(relay, origin, rid, | |
943 | msg + GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN, | |
944 | len - GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN); | |
945 | ||
946 | return err; | |
947 | } | |
948 | #endif | |
927b042a MW |
949 | |
950 | #if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST) | |
951 | #include "tests/xe_guc_relay_test.c" | |
952 | #endif |