1 // SPDX-License-Identifier: MIT
3 * Copyright © 2014 Intel Corporation
6 #include <linux/circ_buf.h>
8 #include "gem/i915_gem_context.h"
9 #include "gem/i915_gem_lmem.h"
10 #include "gt/gen8_engine_cs.h"
11 #include "gt/intel_breadcrumbs.h"
12 #include "gt/intel_context.h"
13 #include "gt/intel_engine_heartbeat.h"
14 #include "gt/intel_engine_pm.h"
15 #include "gt/intel_engine_regs.h"
16 #include "gt/intel_gpu_commands.h"
17 #include "gt/intel_gt.h"
18 #include "gt/intel_gt_clock_utils.h"
19 #include "gt/intel_gt_irq.h"
20 #include "gt/intel_gt_pm.h"
21 #include "gt/intel_gt_regs.h"
22 #include "gt/intel_gt_requests.h"
23 #include "gt/intel_lrc.h"
24 #include "gt/intel_lrc_reg.h"
25 #include "gt/intel_mocs.h"
26 #include "gt/intel_ring.h"
28 #include "intel_guc_ads.h"
29 #include "intel_guc_capture.h"
30 #include "intel_guc_print.h"
31 #include "intel_guc_submission.h"
36 #include "i915_trace.h"
39 * DOC: GuC-based command submission
41 * The Scratch registers:
42 * There are 16 MMIO-based registers start from 0xC180. The kernel driver writes
43 * a value to the action register (SOFT_SCRATCH_0) along with any data. It then
44 * triggers an interrupt on the GuC via another register write (0xC4C8).
45 * Firmware writes a success/fail code back to the action register after
46 * processes the request. The kernel driver polls waiting for this update and
49 * Command Transport buffers (CTBs):
50 * Covered in detail in other sections but CTBs (Host to GuC - H2G, GuC to Host
51 * - G2H) are a message interface between the i915 and GuC.
53 * Context registration:
54 * Before a context can be submitted it must be registered with the GuC via a
55 * H2G. A unique guc_id is associated with each context. The context is either
56 * registered at request creation time (normal operation) or at submission time
57 * (abnormal operation, e.g. after a reset).
60 * The i915 updates the LRC tail value in memory. The i915 must enable the
61 * scheduling of the context within the GuC for the GuC to actually consider it.
62 * Therefore, the first time a disabled context is submitted we use a schedule
63 * enable H2G, while follow up submissions are done via the context submit H2G,
64 * which informs the GuC that a previously enabled context has new work
68 * To unpin a context a H2G is used to disable scheduling. When the
69 * corresponding G2H returns indicating the scheduling disable operation has
70 * completed it is safe to unpin the context. While a disable is in flight it
71 * isn't safe to resubmit the context so a fence is used to stall all future
72 * requests of that context until the G2H is returned. Because this interaction
73 * with the GuC takes a non-zero amount of time we delay the disabling of
74 * scheduling after the pin count goes to zero by a configurable period of time
75 * (see SCHED_DISABLE_DELAY_MS). The thought is this gives the user a window of
76 * time to resubmit something on the context before doing this costly operation.
77 * This delay is only done if the context isn't closed and the guc_id usage is
78 * less than a threshold (see NUM_SCHED_DISABLE_GUC_IDS_THRESHOLD).
80 * Context deregistration:
81 * Before a context can be destroyed or if we steal its guc_id we must
82 * deregister the context with the GuC via H2G. If stealing the guc_id it isn't
83 * safe to submit anything to this guc_id until the deregister completes so a
84 * fence is used to stall all requests associated with this guc_id until the
85 * corresponding G2H returns indicating the guc_id has been deregistered.
87 * submission_state.guc_ids:
88 * Unique number associated with private GuC context data passed in during
89 * context registration / submission / deregistration. 64k available. Simple ida
90 * is used for allocation.
93 * If no guc_ids are available they can be stolen from another context at
94 * request creation time if that context is unpinned. If a guc_id can't be found
95 * we punt this problem to the user as we believe this is near impossible to hit
96 * during normal use cases.
99 * In the GuC submission code we have 3 basic spin locks which protect
100 * everything. Details about each below.
103 * This is the submission lock for all contexts that share an i915 schedule
104 * engine (sched_engine), thus only one of the contexts which share a
105 * sched_engine can be submitting at a time. Currently only one sched_engine is
106 * used for all of GuC submission but that could change in the future.
108 * guc->submission_state.lock
109 * Global lock for GuC submission state. Protects guc_ids and destroyed contexts
113 * Protects everything under ce->guc_state. Ensures that a context is in the
114 * correct state before issuing a H2G. e.g. We don't issue a schedule disable
115 * on a disabled context (bad idea), we don't issue a schedule enable when a
116 * schedule disable is in flight, etc... Also protects list of inflight requests
117 * on the context and the priority management state. Lock is individual to each
120 * Lock ordering rules:
121 * sched_engine->lock -> ce->guc_state.lock
122 * guc->submission_state.lock -> ce->guc_state.lock
125 * When a full GT reset is triggered it is assumed that some G2H responses to
126 * H2Gs can be lost as the GuC is also reset. Losing these G2H can prove to be
127 * fatal as we do certain operations upon receiving a G2H (e.g. destroy
128 * contexts, release guc_ids, etc...). When this occurs we can scrub the
129 * context state and cleanup appropriately, however this is quite racey.
130 * To avoid races, the reset code must disable submission before scrubbing for
131 * the missing G2H, while the submission code must check for submission being
132 * disabled and skip sending H2Gs and updating context states when it is. Both
133 * sides must also make sure to hold the relevant locks.
136 /* GuC Virtual Engine */
137 struct guc_virtual_engine {
138 struct intel_engine_cs base;
139 struct intel_context context;
142 static struct intel_context *
143 guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
144 unsigned long flags);
146 static struct intel_context *
147 guc_create_parallel(struct intel_engine_cs **engines,
148 unsigned int num_siblings,
151 #define GUC_REQUEST_SIZE 64 /* bytes */
154 * We reserve 1/16 of the guc_ids for multi-lrc as these need to be contiguous
155 * per the GuC submission interface. A different allocation algorithm is used
156 * (bitmap vs. ida) between multi-lrc and single-lrc hence the reason to
157 * partition the guc_id space. We believe the number of multi-lrc contexts in
158 * use should be low and 1/16 should be sufficient. Minimum of 32 guc_ids for
161 #define NUMBER_MULTI_LRC_GUC_ID(guc) \
162 ((guc)->submission_state.num_guc_ids / 16)
165 * Below is a set of functions which control the GuC scheduling state which
168 #define SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER BIT(0)
169 #define SCHED_STATE_DESTROYED BIT(1)
170 #define SCHED_STATE_PENDING_DISABLE BIT(2)
171 #define SCHED_STATE_BANNED BIT(3)
172 #define SCHED_STATE_ENABLED BIT(4)
173 #define SCHED_STATE_PENDING_ENABLE BIT(5)
174 #define SCHED_STATE_REGISTERED BIT(6)
175 #define SCHED_STATE_POLICY_REQUIRED BIT(7)
176 #define SCHED_STATE_CLOSED BIT(8)
177 #define SCHED_STATE_BLOCKED_SHIFT 9
178 #define SCHED_STATE_BLOCKED BIT(SCHED_STATE_BLOCKED_SHIFT)
179 #define SCHED_STATE_BLOCKED_MASK (0xfff << SCHED_STATE_BLOCKED_SHIFT)
181 static inline void init_sched_state(struct intel_context *ce)
183 lockdep_assert_held(&ce->guc_state.lock);
184 ce->guc_state.sched_state &= SCHED_STATE_BLOCKED_MASK;
188 * Kernel contexts can have SCHED_STATE_REGISTERED after suspend.
189 * A context close can race with the submission path, so SCHED_STATE_CLOSED
190 * can be set immediately before we try to register.
192 #define SCHED_STATE_VALID_INIT \
193 (SCHED_STATE_BLOCKED_MASK | \
194 SCHED_STATE_CLOSED | \
195 SCHED_STATE_REGISTERED)
198 static bool sched_state_is_init(struct intel_context *ce)
200 return !(ce->guc_state.sched_state & ~SCHED_STATE_VALID_INIT);
204 context_wait_for_deregister_to_register(struct intel_context *ce)
206 return ce->guc_state.sched_state &
207 SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;
211 set_context_wait_for_deregister_to_register(struct intel_context *ce)
213 lockdep_assert_held(&ce->guc_state.lock);
214 ce->guc_state.sched_state |=
215 SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;
219 clr_context_wait_for_deregister_to_register(struct intel_context *ce)
221 lockdep_assert_held(&ce->guc_state.lock);
222 ce->guc_state.sched_state &=
223 ~SCHED_STATE_WAIT_FOR_DEREGISTER_TO_REGISTER;
227 context_destroyed(struct intel_context *ce)
229 return ce->guc_state.sched_state & SCHED_STATE_DESTROYED;
233 set_context_destroyed(struct intel_context *ce)
235 lockdep_assert_held(&ce->guc_state.lock);
236 ce->guc_state.sched_state |= SCHED_STATE_DESTROYED;
239 static inline bool context_pending_disable(struct intel_context *ce)
241 return ce->guc_state.sched_state & SCHED_STATE_PENDING_DISABLE;
244 static inline void set_context_pending_disable(struct intel_context *ce)
246 lockdep_assert_held(&ce->guc_state.lock);
247 ce->guc_state.sched_state |= SCHED_STATE_PENDING_DISABLE;
250 static inline void clr_context_pending_disable(struct intel_context *ce)
252 lockdep_assert_held(&ce->guc_state.lock);
253 ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_DISABLE;
256 static inline bool context_banned(struct intel_context *ce)
258 return ce->guc_state.sched_state & SCHED_STATE_BANNED;
261 static inline void set_context_banned(struct intel_context *ce)
263 lockdep_assert_held(&ce->guc_state.lock);
264 ce->guc_state.sched_state |= SCHED_STATE_BANNED;
267 static inline void clr_context_banned(struct intel_context *ce)
269 lockdep_assert_held(&ce->guc_state.lock);
270 ce->guc_state.sched_state &= ~SCHED_STATE_BANNED;
273 static inline bool context_enabled(struct intel_context *ce)
275 return ce->guc_state.sched_state & SCHED_STATE_ENABLED;
278 static inline void set_context_enabled(struct intel_context *ce)
280 lockdep_assert_held(&ce->guc_state.lock);
281 ce->guc_state.sched_state |= SCHED_STATE_ENABLED;
284 static inline void clr_context_enabled(struct intel_context *ce)
286 lockdep_assert_held(&ce->guc_state.lock);
287 ce->guc_state.sched_state &= ~SCHED_STATE_ENABLED;
290 static inline bool context_pending_enable(struct intel_context *ce)
292 return ce->guc_state.sched_state & SCHED_STATE_PENDING_ENABLE;
295 static inline void set_context_pending_enable(struct intel_context *ce)
297 lockdep_assert_held(&ce->guc_state.lock);
298 ce->guc_state.sched_state |= SCHED_STATE_PENDING_ENABLE;
301 static inline void clr_context_pending_enable(struct intel_context *ce)
303 lockdep_assert_held(&ce->guc_state.lock);
304 ce->guc_state.sched_state &= ~SCHED_STATE_PENDING_ENABLE;
307 static inline bool context_registered(struct intel_context *ce)
309 return ce->guc_state.sched_state & SCHED_STATE_REGISTERED;
312 static inline void set_context_registered(struct intel_context *ce)
314 lockdep_assert_held(&ce->guc_state.lock);
315 ce->guc_state.sched_state |= SCHED_STATE_REGISTERED;
318 static inline void clr_context_registered(struct intel_context *ce)
320 lockdep_assert_held(&ce->guc_state.lock);
321 ce->guc_state.sched_state &= ~SCHED_STATE_REGISTERED;
324 static inline bool context_policy_required(struct intel_context *ce)
326 return ce->guc_state.sched_state & SCHED_STATE_POLICY_REQUIRED;
329 static inline void set_context_policy_required(struct intel_context *ce)
331 lockdep_assert_held(&ce->guc_state.lock);
332 ce->guc_state.sched_state |= SCHED_STATE_POLICY_REQUIRED;
335 static inline void clr_context_policy_required(struct intel_context *ce)
337 lockdep_assert_held(&ce->guc_state.lock);
338 ce->guc_state.sched_state &= ~SCHED_STATE_POLICY_REQUIRED;
341 static inline bool context_close_done(struct intel_context *ce)
343 return ce->guc_state.sched_state & SCHED_STATE_CLOSED;
346 static inline void set_context_close_done(struct intel_context *ce)
348 lockdep_assert_held(&ce->guc_state.lock);
349 ce->guc_state.sched_state |= SCHED_STATE_CLOSED;
352 static inline u32 context_blocked(struct intel_context *ce)
354 return (ce->guc_state.sched_state & SCHED_STATE_BLOCKED_MASK) >>
355 SCHED_STATE_BLOCKED_SHIFT;
358 static inline void incr_context_blocked(struct intel_context *ce)
360 lockdep_assert_held(&ce->guc_state.lock);
362 ce->guc_state.sched_state += SCHED_STATE_BLOCKED;
364 GEM_BUG_ON(!context_blocked(ce)); /* Overflow check */
367 static inline void decr_context_blocked(struct intel_context *ce)
369 lockdep_assert_held(&ce->guc_state.lock);
371 GEM_BUG_ON(!context_blocked(ce)); /* Underflow check */
373 ce->guc_state.sched_state -= SCHED_STATE_BLOCKED;
376 static struct intel_context *
377 request_to_scheduling_context(struct i915_request *rq)
379 return intel_context_to_parent(rq->context);
382 static inline bool context_guc_id_invalid(struct intel_context *ce)
384 return ce->guc_id.id == GUC_INVALID_CONTEXT_ID;
387 static inline void set_context_guc_id_invalid(struct intel_context *ce)
389 ce->guc_id.id = GUC_INVALID_CONTEXT_ID;
392 static inline struct intel_guc *ce_to_guc(struct intel_context *ce)
394 return &ce->engine->gt->uc.guc;
397 static inline struct i915_priolist *to_priolist(struct rb_node *rb)
399 return rb_entry(rb, struct i915_priolist, node);
403 * When using multi-lrc submission a scratch memory area is reserved in the
404 * parent's context state for the process descriptor, work queue, and handshake
405 * between the parent + children contexts to insert safe preemption points
406 * between each of the BBs. Currently the scratch area is sized to a page.
408 * The layout of this scratch area is below:
410 * + sizeof(struct guc_process_desc) child go
411 * + CACHELINE_BYTES child join[0]
413 * + CACHELINE_BYTES child join[n - 1]
415 * PARENT_SCRATCH_SIZE / 2 work queue start
417 * PARENT_SCRATCH_SIZE - 1 work queue end
419 #define WQ_SIZE (PARENT_SCRATCH_SIZE / 2)
420 #define WQ_OFFSET (PARENT_SCRATCH_SIZE - WQ_SIZE)
422 struct sync_semaphore {
424 u8 unused[CACHELINE_BYTES - sizeof(u32)];
427 struct parent_scratch {
429 struct guc_sched_wq_desc wq_desc;
430 struct guc_process_desc_v69 pdesc;
433 struct sync_semaphore go;
434 struct sync_semaphore join[MAX_ENGINE_INSTANCE + 1];
436 u8 unused[WQ_OFFSET - sizeof(union guc_descs) -
437 sizeof(struct sync_semaphore) * (MAX_ENGINE_INSTANCE + 2)];
439 u32 wq[WQ_SIZE / sizeof(u32)];
442 static u32 __get_parent_scratch_offset(struct intel_context *ce)
444 GEM_BUG_ON(!ce->parallel.guc.parent_page);
446 return ce->parallel.guc.parent_page * PAGE_SIZE;
449 static u32 __get_wq_offset(struct intel_context *ce)
451 BUILD_BUG_ON(offsetof(struct parent_scratch, wq) != WQ_OFFSET);
453 return __get_parent_scratch_offset(ce) + WQ_OFFSET;
456 static struct parent_scratch *
457 __get_parent_scratch(struct intel_context *ce)
459 BUILD_BUG_ON(sizeof(struct parent_scratch) != PARENT_SCRATCH_SIZE);
460 BUILD_BUG_ON(sizeof(struct sync_semaphore) != CACHELINE_BYTES);
463 * Need to subtract LRC_STATE_OFFSET here as the
464 * parallel.guc.parent_page is the offset into ce->state while
465 * ce->lrc_reg_reg is ce->state + LRC_STATE_OFFSET.
467 return (struct parent_scratch *)
469 ((__get_parent_scratch_offset(ce) -
470 LRC_STATE_OFFSET) / sizeof(u32)));
473 static struct guc_process_desc_v69 *
474 __get_process_desc_v69(struct intel_context *ce)
476 struct parent_scratch *ps = __get_parent_scratch(ce);
478 return &ps->descs.pdesc;
481 static struct guc_sched_wq_desc *
482 __get_wq_desc_v70(struct intel_context *ce)
484 struct parent_scratch *ps = __get_parent_scratch(ce);
486 return &ps->descs.wq_desc;
489 static u32 *get_wq_pointer(struct intel_context *ce, u32 wqi_size)
492 * Check for space in work queue. Caching a value of head pointer in
493 * intel_context structure in order reduce the number accesses to shared
494 * GPU memory which may be across a PCIe bus.
496 #define AVAILABLE_SPACE \
497 CIRC_SPACE(ce->parallel.guc.wqi_tail, ce->parallel.guc.wqi_head, WQ_SIZE)
498 if (wqi_size > AVAILABLE_SPACE) {
499 ce->parallel.guc.wqi_head = READ_ONCE(*ce->parallel.guc.wq_head);
501 if (wqi_size > AVAILABLE_SPACE)
504 #undef AVAILABLE_SPACE
506 return &__get_parent_scratch(ce)->wq[ce->parallel.guc.wqi_tail / sizeof(u32)];
509 static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id)
511 struct intel_context *ce = xa_load(&guc->context_lookup, id);
513 GEM_BUG_ON(id >= GUC_MAX_CONTEXT_ID);
518 static struct guc_lrc_desc_v69 *__get_lrc_desc_v69(struct intel_guc *guc, u32 index)
520 struct guc_lrc_desc_v69 *base = guc->lrc_desc_pool_vaddr_v69;
525 GEM_BUG_ON(index >= GUC_MAX_CONTEXT_ID);
530 static int guc_lrc_desc_pool_create_v69(struct intel_guc *guc)
535 size = PAGE_ALIGN(sizeof(struct guc_lrc_desc_v69) *
537 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->lrc_desc_pool_v69,
538 (void **)&guc->lrc_desc_pool_vaddr_v69);
545 static void guc_lrc_desc_pool_destroy_v69(struct intel_guc *guc)
547 if (!guc->lrc_desc_pool_vaddr_v69)
550 guc->lrc_desc_pool_vaddr_v69 = NULL;
551 i915_vma_unpin_and_release(&guc->lrc_desc_pool_v69, I915_VMA_RELEASE_MAP);
554 static inline bool guc_submission_initialized(struct intel_guc *guc)
556 return guc->submission_initialized;
559 static inline void _reset_lrc_desc_v69(struct intel_guc *guc, u32 id)
561 struct guc_lrc_desc_v69 *desc = __get_lrc_desc_v69(guc, id);
564 memset(desc, 0, sizeof(*desc));
567 static inline bool ctx_id_mapped(struct intel_guc *guc, u32 id)
569 return __get_context(guc, id);
572 static inline void set_ctx_id_mapping(struct intel_guc *guc, u32 id,
573 struct intel_context *ce)
578 * xarray API doesn't have xa_save_irqsave wrapper, so calling the
579 * lower level functions directly.
581 xa_lock_irqsave(&guc->context_lookup, flags);
582 __xa_store(&guc->context_lookup, id, ce, GFP_ATOMIC);
583 xa_unlock_irqrestore(&guc->context_lookup, flags);
586 static inline void clr_ctx_id_mapping(struct intel_guc *guc, u32 id)
590 if (unlikely(!guc_submission_initialized(guc)))
593 _reset_lrc_desc_v69(guc, id);
596 * xarray API doesn't have xa_erase_irqsave wrapper, so calling
597 * the lower level functions directly.
599 xa_lock_irqsave(&guc->context_lookup, flags);
600 __xa_erase(&guc->context_lookup, id);
601 xa_unlock_irqrestore(&guc->context_lookup, flags);
604 static void decr_outstanding_submission_g2h(struct intel_guc *guc)
606 if (atomic_dec_and_test(&guc->outstanding_submission_g2h))
607 wake_up_all(&guc->ct.wq);
610 static int guc_submission_send_busy_loop(struct intel_guc *guc,
617 * We always loop when a send requires a reply (i.e. g2h_len_dw > 0),
618 * so we don't handle the case where we don't get a reply because we
619 * aborted the send due to the channel being busy.
621 GEM_BUG_ON(g2h_len_dw && !loop);
624 atomic_inc(&guc->outstanding_submission_g2h);
626 return intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop);
629 int intel_guc_wait_for_pending_msg(struct intel_guc *guc,
634 const int state = interruptible ?
635 TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
639 GEM_BUG_ON(timeout < 0);
641 if (!atomic_read(wait_var))
648 prepare_to_wait(&guc->ct.wq, &wait, state);
650 if (!atomic_read(wait_var))
653 if (signal_pending_state(state, current)) {
663 timeout = io_schedule_timeout(timeout);
665 finish_wait(&guc->ct.wq, &wait);
667 return (timeout < 0) ? timeout : 0;
670 int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout)
672 if (!intel_uc_uses_guc_submission(&guc_to_gt(guc)->uc))
675 return intel_guc_wait_for_pending_msg(guc,
676 &guc->outstanding_submission_g2h,
680 static int guc_context_policy_init_v70(struct intel_context *ce, bool loop);
681 static int try_context_registration(struct intel_context *ce, bool loop);
683 static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq)
686 struct intel_context *ce = request_to_scheduling_context(rq);
692 lockdep_assert_held(&rq->engine->sched_engine->lock);
695 * Corner case where requests were sitting in the priority list or a
696 * request resubmitted after the context was banned.
698 if (unlikely(!intel_context_is_schedulable(ce))) {
699 i915_request_put(i915_request_mark_eio(rq));
700 intel_engine_signal_breadcrumbs(ce->engine);
704 GEM_BUG_ON(!atomic_read(&ce->guc_id.ref));
705 GEM_BUG_ON(context_guc_id_invalid(ce));
707 if (context_policy_required(ce)) {
708 err = guc_context_policy_init_v70(ce, false);
713 spin_lock(&ce->guc_state.lock);
716 * The request / context will be run on the hardware when scheduling
717 * gets enabled in the unblock. For multi-lrc we still submit the
718 * context to move the LRC tails.
720 if (unlikely(context_blocked(ce) && !intel_context_is_parent(ce)))
723 enabled = context_enabled(ce) || context_blocked(ce);
726 action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET;
727 action[len++] = ce->guc_id.id;
728 action[len++] = GUC_CONTEXT_ENABLE;
729 set_context_pending_enable(ce);
730 intel_context_get(ce);
731 g2h_len_dw = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET;
733 action[len++] = INTEL_GUC_ACTION_SCHED_CONTEXT;
734 action[len++] = ce->guc_id.id;
737 err = intel_guc_send_nb(guc, action, len, g2h_len_dw);
738 if (!enabled && !err) {
739 trace_intel_context_sched_enable(ce);
740 atomic_inc(&guc->outstanding_submission_g2h);
741 set_context_enabled(ce);
744 * Without multi-lrc KMD does the submission step (moving the
745 * lrc tail) so enabling scheduling is sufficient to submit the
746 * context. This isn't the case in multi-lrc submission as the
747 * GuC needs to move the tails, hence the need for another H2G
748 * to submit a multi-lrc context after enabling scheduling.
750 if (intel_context_is_parent(ce)) {
751 action[0] = INTEL_GUC_ACTION_SCHED_CONTEXT;
752 err = intel_guc_send_nb(guc, action, len - 1, 0);
754 } else if (!enabled) {
755 clr_context_pending_enable(ce);
756 intel_context_put(ce);
759 trace_i915_request_guc_submit(rq);
762 spin_unlock(&ce->guc_state.lock);
766 static int guc_add_request(struct intel_guc *guc, struct i915_request *rq)
768 int ret = __guc_add_request(guc, rq);
770 if (unlikely(ret == -EBUSY)) {
771 guc->stalled_request = rq;
772 guc->submission_stall_reason = STALL_ADD_REQUEST;
778 static inline void guc_set_lrc_tail(struct i915_request *rq)
780 rq->context->lrc_reg_state[CTX_RING_TAIL] =
781 intel_ring_set_tail(rq->ring, rq->tail);
784 static inline int rq_prio(const struct i915_request *rq)
786 return rq->sched.attr.priority;
789 static bool is_multi_lrc_rq(struct i915_request *rq)
791 return intel_context_is_parallel(rq->context);
794 static bool can_merge_rq(struct i915_request *rq,
795 struct i915_request *last)
797 return request_to_scheduling_context(rq) ==
798 request_to_scheduling_context(last);
801 static u32 wq_space_until_wrap(struct intel_context *ce)
803 return (WQ_SIZE - ce->parallel.guc.wqi_tail);
806 static void write_wqi(struct intel_context *ce, u32 wqi_size)
808 BUILD_BUG_ON(!is_power_of_2(WQ_SIZE));
811 * Ensure WQI are visible before updating tail
813 intel_guc_write_barrier(ce_to_guc(ce));
815 ce->parallel.guc.wqi_tail = (ce->parallel.guc.wqi_tail + wqi_size) &
817 WRITE_ONCE(*ce->parallel.guc.wq_tail, ce->parallel.guc.wqi_tail);
820 static int guc_wq_noop_append(struct intel_context *ce)
822 u32 *wqi = get_wq_pointer(ce, wq_space_until_wrap(ce));
823 u32 len_dw = wq_space_until_wrap(ce) / sizeof(u32) - 1;
828 GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw));
830 *wqi = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) |
831 FIELD_PREP(WQ_LEN_MASK, len_dw);
832 ce->parallel.guc.wqi_tail = 0;
837 static int __guc_wq_item_append(struct i915_request *rq)
839 struct intel_context *ce = request_to_scheduling_context(rq);
840 struct intel_context *child;
841 unsigned int wqi_size = (ce->parallel.number_children + 4) *
844 u32 len_dw = (wqi_size / sizeof(u32)) - 1;
847 /* Ensure context is in correct state updating work queue */
848 GEM_BUG_ON(!atomic_read(&ce->guc_id.ref));
849 GEM_BUG_ON(context_guc_id_invalid(ce));
850 GEM_BUG_ON(context_wait_for_deregister_to_register(ce));
851 GEM_BUG_ON(!ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id));
853 /* Insert NOOP if this work queue item will wrap the tail pointer. */
854 if (wqi_size > wq_space_until_wrap(ce)) {
855 ret = guc_wq_noop_append(ce);
860 wqi = get_wq_pointer(ce, wqi_size);
864 GEM_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw));
866 *wqi++ = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) |
867 FIELD_PREP(WQ_LEN_MASK, len_dw);
868 *wqi++ = ce->lrc.lrca;
869 *wqi++ = FIELD_PREP(WQ_GUC_ID_MASK, ce->guc_id.id) |
870 FIELD_PREP(WQ_RING_TAIL_MASK, ce->ring->tail / sizeof(u64));
871 *wqi++ = 0; /* fence_id */
872 for_each_child(ce, child)
873 *wqi++ = child->ring->tail / sizeof(u64);
875 write_wqi(ce, wqi_size);
880 static int guc_wq_item_append(struct intel_guc *guc,
881 struct i915_request *rq)
883 struct intel_context *ce = request_to_scheduling_context(rq);
886 if (unlikely(!intel_context_is_schedulable(ce)))
889 ret = __guc_wq_item_append(rq);
890 if (unlikely(ret == -EBUSY)) {
891 guc->stalled_request = rq;
892 guc->submission_stall_reason = STALL_MOVE_LRC_TAIL;
898 static bool multi_lrc_submit(struct i915_request *rq)
900 struct intel_context *ce = request_to_scheduling_context(rq);
902 intel_ring_set_tail(rq->ring, rq->tail);
905 * We expect the front end (execbuf IOCTL) to set this flag on the last
906 * request generated from a multi-BB submission. This indicates to the
907 * backend (GuC interface) that we should submit this context thus
908 * submitting all the requests generated in parallel.
910 return test_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, &rq->fence.flags) ||
911 !intel_context_is_schedulable(ce);
914 static int guc_dequeue_one_context(struct intel_guc *guc)
916 struct i915_sched_engine * const sched_engine = guc->sched_engine;
917 struct i915_request *last = NULL;
922 lockdep_assert_held(&sched_engine->lock);
924 if (guc->stalled_request) {
926 last = guc->stalled_request;
928 switch (guc->submission_stall_reason) {
929 case STALL_REGISTER_CONTEXT:
930 goto register_context;
931 case STALL_MOVE_LRC_TAIL:
933 case STALL_ADD_REQUEST:
936 MISSING_CASE(guc->submission_stall_reason);
940 while ((rb = rb_first_cached(&sched_engine->queue))) {
941 struct i915_priolist *p = to_priolist(rb);
942 struct i915_request *rq, *rn;
944 priolist_for_each_request_consume(rq, rn, p) {
945 if (last && !can_merge_rq(rq, last))
946 goto register_context;
948 list_del_init(&rq->sched.link);
950 __i915_request_submit(rq);
952 trace_i915_request_in(rq, 0);
955 if (is_multi_lrc_rq(rq)) {
957 * We need to coalesce all multi-lrc requests in
958 * a relationship into a single H2G. We are
959 * guaranteed that all of these requests will be
960 * submitted sequentially.
962 if (multi_lrc_submit(rq)) {
964 goto register_context;
971 rb_erase_cached(&p->node, &sched_engine->queue);
972 i915_priolist_free(p);
977 struct intel_context *ce = request_to_scheduling_context(last);
979 if (unlikely(!ctx_id_mapped(guc, ce->guc_id.id) &&
980 intel_context_is_schedulable(ce))) {
981 ret = try_context_registration(ce, false);
982 if (unlikely(ret == -EPIPE)) {
984 } else if (ret == -EBUSY) {
985 guc->stalled_request = last;
986 guc->submission_stall_reason =
987 STALL_REGISTER_CONTEXT;
988 goto schedule_tasklet;
989 } else if (ret != 0) {
990 GEM_WARN_ON(ret); /* Unexpected */
996 if (is_multi_lrc_rq(last)) {
997 ret = guc_wq_item_append(guc, last);
999 goto schedule_tasklet;
1000 } else if (ret != 0) {
1001 GEM_WARN_ON(ret); /* Unexpected */
1005 guc_set_lrc_tail(last);
1009 ret = guc_add_request(guc, last);
1010 if (unlikely(ret == -EPIPE)) {
1012 } else if (ret == -EBUSY) {
1013 goto schedule_tasklet;
1014 } else if (ret != 0) {
1015 GEM_WARN_ON(ret); /* Unexpected */
1020 guc->stalled_request = NULL;
1021 guc->submission_stall_reason = STALL_NONE;
1025 sched_engine->tasklet.callback = NULL;
1026 tasklet_disable_nosync(&sched_engine->tasklet);
1030 tasklet_schedule(&sched_engine->tasklet);
1034 static void guc_submission_tasklet(struct tasklet_struct *t)
1036 struct i915_sched_engine *sched_engine =
1037 from_tasklet(sched_engine, t, tasklet);
1038 unsigned long flags;
1041 spin_lock_irqsave(&sched_engine->lock, flags);
1044 loop = guc_dequeue_one_context(sched_engine->private_data);
1047 i915_sched_engine_reset_on_empty(sched_engine);
1049 spin_unlock_irqrestore(&sched_engine->lock, flags);
1052 static void cs_irq_handler(struct intel_engine_cs *engine, u16 iir)
1054 if (iir & GT_RENDER_USER_INTERRUPT)
1055 intel_engine_signal_breadcrumbs(engine);
1058 static void __guc_context_destroy(struct intel_context *ce);
1059 static void release_guc_id(struct intel_guc *guc, struct intel_context *ce);
1060 static void guc_signal_context_fence(struct intel_context *ce);
1061 static void guc_cancel_context_requests(struct intel_context *ce);
1062 static void guc_blocked_fence_complete(struct intel_context *ce);
1064 static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)
1066 struct intel_context *ce;
1067 unsigned long index, flags;
1068 bool pending_disable, pending_enable, deregister, destroyed, banned;
1070 xa_lock_irqsave(&guc->context_lookup, flags);
1071 xa_for_each(&guc->context_lookup, index, ce) {
1073 * Corner case where the ref count on the object is zero but and
1074 * deregister G2H was lost. In this case we don't touch the ref
1075 * count and finish the destroy of the context.
1077 bool do_put = kref_get_unless_zero(&ce->ref);
1079 xa_unlock(&guc->context_lookup);
1081 if (test_bit(CONTEXT_GUC_INIT, &ce->flags) &&
1082 (cancel_delayed_work(&ce->guc_state.sched_disable_delay_work))) {
1083 /* successful cancel so jump straight to close it */
1084 intel_context_sched_disable_unpin(ce);
1087 spin_lock(&ce->guc_state.lock);
1090 * Once we are at this point submission_disabled() is guaranteed
1091 * to be visible to all callers who set the below flags (see above
1092 * flush and flushes in reset_prepare). If submission_disabled()
1093 * is set, the caller shouldn't set these flags.
1096 destroyed = context_destroyed(ce);
1097 pending_enable = context_pending_enable(ce);
1098 pending_disable = context_pending_disable(ce);
1099 deregister = context_wait_for_deregister_to_register(ce);
1100 banned = context_banned(ce);
1101 init_sched_state(ce);
1103 spin_unlock(&ce->guc_state.lock);
1105 if (pending_enable || destroyed || deregister) {
1106 decr_outstanding_submission_g2h(guc);
1108 guc_signal_context_fence(ce);
1110 intel_gt_pm_put_async_untracked(guc_to_gt(guc));
1111 release_guc_id(guc, ce);
1112 __guc_context_destroy(ce);
1114 if (pending_enable || deregister)
1115 intel_context_put(ce);
1118 /* Not mutualy exclusive with above if statement. */
1119 if (pending_disable) {
1120 guc_signal_context_fence(ce);
1122 guc_cancel_context_requests(ce);
1123 intel_engine_signal_breadcrumbs(ce->engine);
1125 intel_context_sched_disable_unpin(ce);
1126 decr_outstanding_submission_g2h(guc);
1128 spin_lock(&ce->guc_state.lock);
1129 guc_blocked_fence_complete(ce);
1130 spin_unlock(&ce->guc_state.lock);
1132 intel_context_put(ce);
1136 intel_context_put(ce);
1137 xa_lock(&guc->context_lookup);
1139 xa_unlock_irqrestore(&guc->context_lookup, flags);
1143 * GuC stores busyness stats for each engine at context in/out boundaries. A
1144 * context 'in' logs execution start time, 'out' adds in -> out delta to total.
1145 * i915/kmd accesses 'start', 'total' and 'context id' from memory shared with
1148 * __i915_pmu_event_read samples engine busyness. When sampling, if context id
1149 * is valid (!= ~0) and start is non-zero, the engine is considered to be
1150 * active. For an active engine total busyness = total + (now - start), where
1151 * 'now' is the time at which the busyness is sampled. For inactive engine,
1152 * total busyness = total.
1154 * All times are captured from GUCPMTIMESTAMP reg and are in gt clock domain.
1156 * The start and total values provided by GuC are 32 bits and wrap around in a
1157 * few minutes. Since perf pmu provides busyness as 64 bit monotonically
1158 * increasing ns values, there is a need for this implementation to account for
1159 * overflows and extend the GuC provided values to 64 bits before returning
1160 * busyness to the user. In order to do that, a worker runs periodically at
1161 * frequency = 1/8th the time it takes for the timestamp to wrap (i.e. once in
1162 * 27 seconds for a gt clock frequency of 19.2 MHz).
1165 #define WRAP_TIME_CLKS U32_MAX
1166 #define POLL_TIME_CLKS (WRAP_TIME_CLKS >> 3)
1169 __extend_last_switch(struct intel_guc *guc, u64 *prev_start, u32 new_start)
1171 u32 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);
1172 u32 gt_stamp_last = lower_32_bits(guc->timestamp.gt_stamp);
1174 if (new_start == lower_32_bits(*prev_start))
1178 * When gt is unparked, we update the gt timestamp and start the ping
1179 * worker that updates the gt_stamp every POLL_TIME_CLKS. As long as gt
1180 * is unparked, all switched in contexts will have a start time that is
1181 * within +/- POLL_TIME_CLKS of the most recent gt_stamp.
1183 * If neither gt_stamp nor new_start has rolled over, then the
1184 * gt_stamp_hi does not need to be adjusted, however if one of them has
1185 * rolled over, we need to adjust gt_stamp_hi accordingly.
1187 * The below conditions address the cases of new_start rollover and
1188 * gt_stamp_last rollover respectively.
1190 if (new_start < gt_stamp_last &&
1191 (new_start - gt_stamp_last) <= POLL_TIME_CLKS)
1194 if (new_start > gt_stamp_last &&
1195 (gt_stamp_last - new_start) <= POLL_TIME_CLKS && gt_stamp_hi)
1198 *prev_start = ((u64)gt_stamp_hi << 32) | new_start;
1201 #define record_read(map_, field_) \
1202 iosys_map_rd_field(map_, 0, struct guc_engine_usage_record, field_)
1205 * GuC updates shared memory and KMD reads it. Since this is not synchronized,
1206 * we run into a race where the value read is inconsistent. Sometimes the
1207 * inconsistency is in reading the upper MSB bytes of the last_in value when
1208 * this race occurs. 2 types of cases are seen - upper 8 bits are zero and upper
1209 * 24 bits are zero. Since these are non-zero values, it is non-trivial to
1210 * determine validity of these values. Instead we read the values multiple times
1211 * until they are consistent. In test runs, 3 attempts results in consistent
1212 * values. The upper bound is set to 6 attempts and may need to be tuned as per
1213 * any new occurences.
1215 static void __get_engine_usage_record(struct intel_engine_cs *engine,
1216 u32 *last_in, u32 *id, u32 *total)
1218 struct iosys_map rec_map = intel_guc_engine_usage_record_map(engine);
1222 *last_in = record_read(&rec_map, last_switch_in_stamp);
1223 *id = record_read(&rec_map, current_context_index);
1224 *total = record_read(&rec_map, total_runtime);
1226 if (record_read(&rec_map, last_switch_in_stamp) == *last_in &&
1227 record_read(&rec_map, current_context_index) == *id &&
1228 record_read(&rec_map, total_runtime) == *total)
1233 static void guc_update_engine_gt_clks(struct intel_engine_cs *engine)
1235 struct intel_engine_guc_stats *stats = &engine->stats.guc;
1236 struct intel_guc *guc = &engine->gt->uc.guc;
1237 u32 last_switch, ctx_id, total;
1239 lockdep_assert_held(&guc->timestamp.lock);
1241 __get_engine_usage_record(engine, &last_switch, &ctx_id, &total);
1243 stats->running = ctx_id != ~0U && last_switch;
1245 __extend_last_switch(guc, &stats->start_gt_clk, last_switch);
1248 * Instead of adjusting the total for overflow, just add the
1249 * difference from previous sample stats->total_gt_clks
1251 if (total && total != ~0U) {
1252 stats->total_gt_clks += (u32)(total - stats->prev_total);
1253 stats->prev_total = total;
1257 static u32 gpm_timestamp_shift(struct intel_gt *gt)
1259 intel_wakeref_t wakeref;
1262 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
1263 reg = intel_uncore_read(gt->uncore, RPM_CONFIG0);
1265 shift = (reg & GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >>
1266 GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT;
1271 static void guc_update_pm_timestamp(struct intel_guc *guc, ktime_t *now)
1273 struct intel_gt *gt = guc_to_gt(guc);
1274 u32 gt_stamp_lo, gt_stamp_hi;
1277 lockdep_assert_held(&guc->timestamp.lock);
1279 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);
1280 gpm_ts = intel_uncore_read64_2x32(gt->uncore, MISC_STATUS0,
1281 MISC_STATUS1) >> guc->timestamp.shift;
1282 gt_stamp_lo = lower_32_bits(gpm_ts);
1285 if (gt_stamp_lo < lower_32_bits(guc->timestamp.gt_stamp))
1288 guc->timestamp.gt_stamp = ((u64)gt_stamp_hi << 32) | gt_stamp_lo;
1292 * Unlike the execlist mode of submission total and active times are in terms of
1293 * gt clocks. The *now parameter is retained to return the cpu time at which the
1294 * busyness was sampled.
1296 static ktime_t guc_engine_busyness(struct intel_engine_cs *engine, ktime_t *now)
1298 struct intel_engine_guc_stats stats_saved, *stats = &engine->stats.guc;
1299 struct i915_gpu_error *gpu_error = &engine->i915->gpu_error;
1300 struct intel_gt *gt = engine->gt;
1301 struct intel_guc *guc = >->uc.guc;
1302 u64 total, gt_stamp_saved;
1303 unsigned long flags;
1306 intel_wakeref_t wakeref;
1308 spin_lock_irqsave(&guc->timestamp.lock, flags);
1311 * If a reset happened, we risk reading partially updated engine
1312 * busyness from GuC, so we just use the driver stored copy of busyness.
1313 * Synchronize with gt reset using reset_count and the
1314 * I915_RESET_BACKOFF flag. Note that reset flow updates the reset_count
1315 * after I915_RESET_BACKOFF flag, so ensure that the reset_count is
1316 * usable by checking the flag afterwards.
1318 reset_count = i915_reset_count(gpu_error);
1319 in_reset = test_bit(I915_RESET_BACKOFF, >->reset.flags);
1324 * The active busyness depends on start_gt_clk and gt_stamp.
1325 * gt_stamp is updated by i915 only when gt is awake and the
1326 * start_gt_clk is derived from GuC state. To get a consistent
1327 * view of activity, we query the GuC state only if gt is awake.
1329 wakeref = in_reset ? 0 : intel_gt_pm_get_if_awake(gt);
1331 stats_saved = *stats;
1332 gt_stamp_saved = guc->timestamp.gt_stamp;
1334 * Update gt_clks, then gt timestamp to simplify the 'gt_stamp -
1335 * start_gt_clk' calculation below for active engines.
1337 guc_update_engine_gt_clks(engine);
1338 guc_update_pm_timestamp(guc, now);
1339 intel_gt_pm_put_async(gt, wakeref);
1340 if (i915_reset_count(gpu_error) != reset_count) {
1341 *stats = stats_saved;
1342 guc->timestamp.gt_stamp = gt_stamp_saved;
1346 total = intel_gt_clock_interval_to_ns(gt, stats->total_gt_clks);
1347 if (stats->running) {
1348 u64 clk = guc->timestamp.gt_stamp - stats->start_gt_clk;
1350 total += intel_gt_clock_interval_to_ns(gt, clk);
1353 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1355 return ns_to_ktime(total);
1358 static void guc_enable_busyness_worker(struct intel_guc *guc)
1360 mod_delayed_work(system_highpri_wq, &guc->timestamp.work, guc->timestamp.ping_delay);
1363 static void guc_cancel_busyness_worker(struct intel_guc *guc)
1365 cancel_delayed_work_sync(&guc->timestamp.work);
1368 static void __reset_guc_busyness_stats(struct intel_guc *guc)
1370 struct intel_gt *gt = guc_to_gt(guc);
1371 struct intel_engine_cs *engine;
1372 enum intel_engine_id id;
1373 unsigned long flags;
1376 guc_cancel_busyness_worker(guc);
1378 spin_lock_irqsave(&guc->timestamp.lock, flags);
1380 guc_update_pm_timestamp(guc, &unused);
1381 for_each_engine(engine, gt, id) {
1382 guc_update_engine_gt_clks(engine);
1383 engine->stats.guc.prev_total = 0;
1386 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1389 static void __update_guc_busyness_stats(struct intel_guc *guc)
1391 struct intel_gt *gt = guc_to_gt(guc);
1392 struct intel_engine_cs *engine;
1393 enum intel_engine_id id;
1394 unsigned long flags;
1397 guc->timestamp.last_stat_jiffies = jiffies;
1399 spin_lock_irqsave(&guc->timestamp.lock, flags);
1401 guc_update_pm_timestamp(guc, &unused);
1402 for_each_engine(engine, gt, id)
1403 guc_update_engine_gt_clks(engine);
1405 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1408 static void __guc_context_update_stats(struct intel_context *ce)
1410 struct intel_guc *guc = ce_to_guc(ce);
1411 unsigned long flags;
1413 spin_lock_irqsave(&guc->timestamp.lock, flags);
1414 lrc_update_runtime(ce);
1415 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1418 static void guc_context_update_stats(struct intel_context *ce)
1420 if (!intel_context_pin_if_active(ce))
1423 __guc_context_update_stats(ce);
1424 intel_context_unpin(ce);
1427 static void guc_timestamp_ping(struct work_struct *wrk)
1429 struct intel_guc *guc = container_of(wrk, typeof(*guc),
1430 timestamp.work.work);
1431 struct intel_uc *uc = container_of(guc, typeof(*uc), guc);
1432 struct intel_gt *gt = guc_to_gt(guc);
1433 struct intel_context *ce;
1434 intel_wakeref_t wakeref;
1435 unsigned long index;
1439 * Ideally the busyness worker should take a gt pm wakeref because the
1440 * worker only needs to be active while gt is awake. However, the
1441 * gt_park path cancels the worker synchronously and this complicates
1442 * the flow if the worker is also running at the same time. The cancel
1443 * waits for the worker and when the worker releases the wakeref, that
1444 * would call gt_park and would lead to a deadlock.
1446 * The resolution is to take the global pm wakeref if runtime pm is
1447 * already active. If not, we don't need to update the busyness stats as
1448 * the stats would already be updated when the gt was parked.
1451 * - We do not requeue the worker if we cannot take a reference to runtime
1452 * pm since intel_guc_busyness_unpark would requeue the worker in the
1455 * - If the gt was parked longer than time taken for GT timestamp to roll
1456 * over, we ignore those rollovers since we don't care about tracking
1457 * the exact GT time. We only care about roll overs when the gt is
1458 * active and running workloads.
1460 * - There is a window of time between gt_park and runtime suspend,
1461 * where the worker may run. This is acceptable since the worker will
1462 * not find any new data to update busyness.
1464 wakeref = intel_runtime_pm_get_if_active(>->i915->runtime_pm);
1469 * Synchronize with gt reset to make sure the worker does not
1470 * corrupt the engine/guc stats. NB: can't actually block waiting
1471 * for a reset to complete as the reset requires flushing out
1472 * this worker thread if started. So waiting would deadlock.
1474 ret = intel_gt_reset_trylock(gt, &srcu);
1478 __update_guc_busyness_stats(guc);
1480 /* adjust context stats for overflow */
1481 xa_for_each(&guc->context_lookup, index, ce)
1482 guc_context_update_stats(ce);
1484 intel_gt_reset_unlock(gt, srcu);
1486 guc_enable_busyness_worker(guc);
1489 intel_runtime_pm_put(>->i915->runtime_pm, wakeref);
1492 static int guc_action_enable_usage_stats(struct intel_guc *guc)
1494 u32 offset = intel_guc_engine_usage_offset(guc);
1496 INTEL_GUC_ACTION_SET_ENG_UTIL_BUFF,
1501 return intel_guc_send(guc, action, ARRAY_SIZE(action));
1504 static int guc_init_engine_stats(struct intel_guc *guc)
1506 struct intel_gt *gt = guc_to_gt(guc);
1507 intel_wakeref_t wakeref;
1510 with_intel_runtime_pm(>->i915->runtime_pm, wakeref)
1511 ret = guc_action_enable_usage_stats(guc);
1514 guc_err(guc, "Failed to enable usage stats: %pe\n", ERR_PTR(ret));
1516 guc_enable_busyness_worker(guc);
1521 static void guc_fini_engine_stats(struct intel_guc *guc)
1523 guc_cancel_busyness_worker(guc);
1526 void intel_guc_busyness_park(struct intel_gt *gt)
1528 struct intel_guc *guc = >->uc.guc;
1530 if (!guc_submission_initialized(guc))
1534 * There is a race with suspend flow where the worker runs after suspend
1535 * and causes an unclaimed register access warning. Cancel the worker
1536 * synchronously here.
1538 guc_cancel_busyness_worker(guc);
1541 * Before parking, we should sample engine busyness stats if we need to.
1542 * We can skip it if we are less than half a ping from the last time we
1543 * sampled the busyness stats.
1545 if (guc->timestamp.last_stat_jiffies &&
1546 !time_after(jiffies, guc->timestamp.last_stat_jiffies +
1547 (guc->timestamp.ping_delay / 2)))
1550 __update_guc_busyness_stats(guc);
1553 void intel_guc_busyness_unpark(struct intel_gt *gt)
1555 struct intel_guc *guc = >->uc.guc;
1556 unsigned long flags;
1559 if (!guc_submission_initialized(guc))
1562 spin_lock_irqsave(&guc->timestamp.lock, flags);
1563 guc_update_pm_timestamp(guc, &unused);
1564 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1565 guc_enable_busyness_worker(guc);
1569 submission_disabled(struct intel_guc *guc)
1571 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1573 return unlikely(!sched_engine ||
1574 !__tasklet_is_enabled(&sched_engine->tasklet) ||
1575 intel_gt_is_wedged(guc_to_gt(guc)));
1578 static void disable_submission(struct intel_guc *guc)
1580 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1582 if (__tasklet_is_enabled(&sched_engine->tasklet)) {
1583 GEM_BUG_ON(!guc->ct.enabled);
1584 __tasklet_disable_sync_once(&sched_engine->tasklet);
1585 sched_engine->tasklet.callback = NULL;
1589 static void enable_submission(struct intel_guc *guc)
1591 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1592 unsigned long flags;
1594 spin_lock_irqsave(&guc->sched_engine->lock, flags);
1595 sched_engine->tasklet.callback = guc_submission_tasklet;
1596 wmb(); /* Make sure callback visible */
1597 if (!__tasklet_is_enabled(&sched_engine->tasklet) &&
1598 __tasklet_enable(&sched_engine->tasklet)) {
1599 GEM_BUG_ON(!guc->ct.enabled);
1601 /* And kick in case we missed a new request submission. */
1602 tasklet_hi_schedule(&sched_engine->tasklet);
1604 spin_unlock_irqrestore(&guc->sched_engine->lock, flags);
1607 static void guc_flush_submissions(struct intel_guc *guc)
1609 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1610 unsigned long flags;
1612 spin_lock_irqsave(&sched_engine->lock, flags);
1613 spin_unlock_irqrestore(&sched_engine->lock, flags);
1616 static void guc_flush_destroyed_contexts(struct intel_guc *guc);
1618 void intel_guc_submission_reset_prepare(struct intel_guc *guc)
1620 if (unlikely(!guc_submission_initialized(guc))) {
1621 /* Reset called during driver load? GuC not yet initialised! */
1625 intel_gt_park_heartbeats(guc_to_gt(guc));
1626 disable_submission(guc);
1627 guc->interrupts.disable(guc);
1628 __reset_guc_busyness_stats(guc);
1630 /* Flush IRQ handler */
1631 spin_lock_irq(guc_to_gt(guc)->irq_lock);
1632 spin_unlock_irq(guc_to_gt(guc)->irq_lock);
1634 guc_flush_submissions(guc);
1635 guc_flush_destroyed_contexts(guc);
1636 flush_work(&guc->ct.requests.worker);
1638 scrub_guc_desc_for_outstanding_g2h(guc);
1641 static struct intel_engine_cs *
1642 guc_virtual_get_sibling(struct intel_engine_cs *ve, unsigned int sibling)
1644 struct intel_engine_cs *engine;
1645 intel_engine_mask_t tmp, mask = ve->mask;
1646 unsigned int num_siblings = 0;
1648 for_each_engine_masked(engine, ve->gt, mask, tmp)
1649 if (num_siblings++ == sibling)
1655 static inline struct intel_engine_cs *
1656 __context_to_physical_engine(struct intel_context *ce)
1658 struct intel_engine_cs *engine = ce->engine;
1660 if (intel_engine_is_virtual(engine))
1661 engine = guc_virtual_get_sibling(engine, 0);
1666 static void guc_reset_state(struct intel_context *ce, u32 head, bool scrub)
1668 struct intel_engine_cs *engine = __context_to_physical_engine(ce);
1670 if (!intel_context_is_schedulable(ce))
1673 GEM_BUG_ON(!intel_context_is_pinned(ce));
1676 * We want a simple context + ring to execute the breadcrumb update.
1677 * We cannot rely on the context being intact across the GPU hang,
1678 * so clear it and rebuild just what we need for the breadcrumb.
1679 * All pending requests for this context will be zapped, and any
1680 * future request will be after userspace has had the opportunity
1681 * to recreate its own state.
1684 lrc_init_regs(ce, engine, true);
1686 /* Rerun the request; its payload has been neutered (if guilty). */
1687 lrc_update_regs(ce, engine, head);
1690 static void guc_engine_reset_prepare(struct intel_engine_cs *engine)
1693 * Wa_22011802037: In addition to stopping the cs, we need
1694 * to wait for any pending mi force wakeups
1696 if (intel_engine_reset_needs_wa_22011802037(engine->gt)) {
1697 intel_engine_stop_cs(engine);
1698 intel_engine_wait_for_pending_mi_fw(engine);
1702 static void guc_reset_nop(struct intel_engine_cs *engine)
1706 static void guc_rewind_nop(struct intel_engine_cs *engine, bool stalled)
1711 __unwind_incomplete_requests(struct intel_context *ce)
1713 struct i915_request *rq, *rn;
1714 struct list_head *pl;
1715 int prio = I915_PRIORITY_INVALID;
1716 struct i915_sched_engine * const sched_engine =
1717 ce->engine->sched_engine;
1718 unsigned long flags;
1720 spin_lock_irqsave(&sched_engine->lock, flags);
1721 spin_lock(&ce->guc_state.lock);
1722 list_for_each_entry_safe_reverse(rq, rn,
1723 &ce->guc_state.requests,
1725 if (i915_request_completed(rq))
1728 list_del_init(&rq->sched.link);
1729 __i915_request_unsubmit(rq);
1731 /* Push the request back into the queue for later resubmission. */
1732 GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
1733 if (rq_prio(rq) != prio) {
1735 pl = i915_sched_lookup_priolist(sched_engine, prio);
1737 GEM_BUG_ON(i915_sched_engine_is_empty(sched_engine));
1739 list_add(&rq->sched.link, pl);
1740 set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
1742 spin_unlock(&ce->guc_state.lock);
1743 spin_unlock_irqrestore(&sched_engine->lock, flags);
1746 static void __guc_reset_context(struct intel_context *ce, intel_engine_mask_t stalled)
1749 struct i915_request *rq;
1750 unsigned long flags;
1752 int i, number_children = ce->parallel.number_children;
1753 struct intel_context *parent = ce;
1755 GEM_BUG_ON(intel_context_is_child(ce));
1757 intel_context_get(ce);
1760 * GuC will implicitly mark the context as non-schedulable when it sends
1761 * the reset notification. Make sure our state reflects this change. The
1762 * context will be marked enabled on resubmission.
1764 spin_lock_irqsave(&ce->guc_state.lock, flags);
1765 clr_context_enabled(ce);
1766 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
1769 * For each context in the relationship find the hanging request
1770 * resetting each context / request as needed
1772 for (i = 0; i < number_children + 1; ++i) {
1773 if (!intel_context_is_pinned(ce))
1777 rq = intel_context_get_active_request(ce);
1779 head = ce->ring->tail;
1783 if (i915_request_started(rq))
1784 guilty = stalled & ce->engine->mask;
1786 GEM_BUG_ON(i915_active_is_idle(&ce->active));
1787 head = intel_ring_wrap(ce->ring, rq->head);
1789 __i915_request_reset(rq, guilty);
1790 i915_request_put(rq);
1792 guc_reset_state(ce, head, guilty);
1794 if (i != number_children)
1795 ce = list_next_entry(ce, parallel.child_link);
1798 __unwind_incomplete_requests(parent);
1799 intel_context_put(parent);
1802 void wake_up_all_tlb_invalidate(struct intel_guc *guc)
1804 struct intel_guc_tlb_wait *wait;
1807 if (!intel_guc_tlb_invalidation_is_available(guc))
1810 xa_lock_irq(&guc->tlb_lookup);
1811 xa_for_each(&guc->tlb_lookup, i, wait)
1813 xa_unlock_irq(&guc->tlb_lookup);
1816 void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled)
1818 struct intel_context *ce;
1819 unsigned long index;
1820 unsigned long flags;
1822 if (unlikely(!guc_submission_initialized(guc))) {
1823 /* Reset called during driver load? GuC not yet initialised! */
1827 xa_lock_irqsave(&guc->context_lookup, flags);
1828 xa_for_each(&guc->context_lookup, index, ce) {
1829 if (!kref_get_unless_zero(&ce->ref))
1832 xa_unlock(&guc->context_lookup);
1834 if (intel_context_is_pinned(ce) &&
1835 !intel_context_is_child(ce))
1836 __guc_reset_context(ce, stalled);
1838 intel_context_put(ce);
1840 xa_lock(&guc->context_lookup);
1842 xa_unlock_irqrestore(&guc->context_lookup, flags);
1844 /* GuC is blown away, drop all references to contexts */
1845 xa_destroy(&guc->context_lookup);
1848 static void guc_cancel_context_requests(struct intel_context *ce)
1850 struct i915_sched_engine *sched_engine = ce_to_guc(ce)->sched_engine;
1851 struct i915_request *rq;
1852 unsigned long flags;
1854 /* Mark all executing requests as skipped. */
1855 spin_lock_irqsave(&sched_engine->lock, flags);
1856 spin_lock(&ce->guc_state.lock);
1857 list_for_each_entry(rq, &ce->guc_state.requests, sched.link)
1858 i915_request_put(i915_request_mark_eio(rq));
1859 spin_unlock(&ce->guc_state.lock);
1860 spin_unlock_irqrestore(&sched_engine->lock, flags);
1864 guc_cancel_sched_engine_requests(struct i915_sched_engine *sched_engine)
1866 struct i915_request *rq, *rn;
1868 unsigned long flags;
1870 /* Can be called during boot if GuC fails to load */
1875 * Before we call engine->cancel_requests(), we should have exclusive
1876 * access to the submission state. This is arranged for us by the
1877 * caller disabling the interrupt generation, the tasklet and other
1878 * threads that may then access the same state, giving us a free hand
1879 * to reset state. However, we still need to let lockdep be aware that
1880 * we know this state may be accessed in hardirq context, so we
1881 * disable the irq around this manipulation and we want to keep
1882 * the spinlock focused on its duties and not accidentally conflate
1883 * coverage to the submission's irq state. (Similarly, although we
1884 * shouldn't need to disable irq around the manipulation of the
1885 * submission's irq state, we also wish to remind ourselves that
1888 spin_lock_irqsave(&sched_engine->lock, flags);
1890 /* Flush the queued requests to the timeline list (for retiring). */
1891 while ((rb = rb_first_cached(&sched_engine->queue))) {
1892 struct i915_priolist *p = to_priolist(rb);
1894 priolist_for_each_request_consume(rq, rn, p) {
1895 list_del_init(&rq->sched.link);
1897 __i915_request_submit(rq);
1899 i915_request_put(i915_request_mark_eio(rq));
1902 rb_erase_cached(&p->node, &sched_engine->queue);
1903 i915_priolist_free(p);
1906 /* Remaining _unready_ requests will be nop'ed when submitted */
1908 sched_engine->queue_priority_hint = INT_MIN;
1909 sched_engine->queue = RB_ROOT_CACHED;
1911 spin_unlock_irqrestore(&sched_engine->lock, flags);
1914 void intel_guc_submission_cancel_requests(struct intel_guc *guc)
1916 struct intel_context *ce;
1917 unsigned long index;
1918 unsigned long flags;
1920 xa_lock_irqsave(&guc->context_lookup, flags);
1921 xa_for_each(&guc->context_lookup, index, ce) {
1922 if (!kref_get_unless_zero(&ce->ref))
1925 xa_unlock(&guc->context_lookup);
1927 if (intel_context_is_pinned(ce) &&
1928 !intel_context_is_child(ce))
1929 guc_cancel_context_requests(ce);
1931 intel_context_put(ce);
1933 xa_lock(&guc->context_lookup);
1935 xa_unlock_irqrestore(&guc->context_lookup, flags);
1937 guc_cancel_sched_engine_requests(guc->sched_engine);
1939 /* GuC is blown away, drop all references to contexts */
1940 xa_destroy(&guc->context_lookup);
1943 * Wedged GT won't respond to any TLB invalidation request. Simply
1944 * release all the blocked waiters.
1946 wake_up_all_tlb_invalidate(guc);
1949 void intel_guc_submission_reset_finish(struct intel_guc *guc)
1951 /* Reset called during driver load or during wedge? */
1952 if (unlikely(!guc_submission_initialized(guc) ||
1953 intel_gt_is_wedged(guc_to_gt(guc)))) {
1958 * Technically possible for either of these values to be non-zero here,
1959 * but very unlikely + harmless. Regardless let's add a warn so we can
1960 * see in CI if this happens frequently / a precursor to taking down the
1963 GEM_WARN_ON(atomic_read(&guc->outstanding_submission_g2h));
1964 atomic_set(&guc->outstanding_submission_g2h, 0);
1966 intel_guc_global_policies_update(guc);
1967 enable_submission(guc);
1968 intel_gt_unpark_heartbeats(guc_to_gt(guc));
1971 * The full GT reset will have cleared the TLB caches and flushed the
1972 * G2H message queue; we can release all the blocked waiters.
1974 wake_up_all_tlb_invalidate(guc);
1977 static void destroyed_worker_func(struct work_struct *w);
1978 static void reset_fail_worker_func(struct work_struct *w);
1980 bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc)
1982 return HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915) &&
1983 intel_guc_is_ready(guc);
1986 static int init_tlb_lookup(struct intel_guc *guc)
1988 struct intel_guc_tlb_wait *wait;
1991 if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))
1994 xa_init_flags(&guc->tlb_lookup, XA_FLAGS_ALLOC);
1996 wait = kzalloc(sizeof(*wait), GFP_KERNEL);
2000 init_waitqueue_head(&wait->wq);
2002 /* Preallocate a shared id for use under memory pressure. */
2003 err = xa_alloc_cyclic_irq(&guc->tlb_lookup, &guc->serial_slot, wait,
2004 xa_limit_32b, &guc->next_seqno, GFP_KERNEL);
2013 static void fini_tlb_lookup(struct intel_guc *guc)
2015 struct intel_guc_tlb_wait *wait;
2017 if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))
2020 wait = xa_load(&guc->tlb_lookup, guc->serial_slot);
2021 if (wait && wait->busy)
2022 guc_err(guc, "Unexpected busy item in tlb_lookup on fini\n");
2025 xa_destroy(&guc->tlb_lookup);
2029 * Set up the memory resources to be shared with the GuC (via the GGTT)
2030 * at firmware loading time.
2032 int intel_guc_submission_init(struct intel_guc *guc)
2034 struct intel_gt *gt = guc_to_gt(guc);
2037 if (guc->submission_initialized)
2040 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 0, 0)) {
2041 ret = guc_lrc_desc_pool_create_v69(guc);
2046 ret = init_tlb_lookup(guc);
2050 guc->submission_state.guc_ids_bitmap =
2051 bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc), GFP_KERNEL);
2052 if (!guc->submission_state.guc_ids_bitmap) {
2057 guc->timestamp.ping_delay = (POLL_TIME_CLKS / gt->clock_frequency + 1) * HZ;
2058 guc->timestamp.shift = gpm_timestamp_shift(gt);
2059 guc->submission_initialized = true;
2064 fini_tlb_lookup(guc);
2066 guc_lrc_desc_pool_destroy_v69(guc);
2070 void intel_guc_submission_fini(struct intel_guc *guc)
2072 if (!guc->submission_initialized)
2075 guc_flush_destroyed_contexts(guc);
2076 guc_lrc_desc_pool_destroy_v69(guc);
2077 i915_sched_engine_put(guc->sched_engine);
2078 bitmap_free(guc->submission_state.guc_ids_bitmap);
2079 fini_tlb_lookup(guc);
2080 guc->submission_initialized = false;
2083 static inline void queue_request(struct i915_sched_engine *sched_engine,
2084 struct i915_request *rq,
2087 GEM_BUG_ON(!list_empty(&rq->sched.link));
2088 list_add_tail(&rq->sched.link,
2089 i915_sched_lookup_priolist(sched_engine, prio));
2090 set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
2091 tasklet_hi_schedule(&sched_engine->tasklet);
2094 static int guc_bypass_tasklet_submit(struct intel_guc *guc,
2095 struct i915_request *rq)
2099 __i915_request_submit(rq);
2101 trace_i915_request_in(rq, 0);
2103 if (is_multi_lrc_rq(rq)) {
2104 if (multi_lrc_submit(rq)) {
2105 ret = guc_wq_item_append(guc, rq);
2107 ret = guc_add_request(guc, rq);
2110 guc_set_lrc_tail(rq);
2111 ret = guc_add_request(guc, rq);
2114 if (unlikely(ret == -EPIPE))
2115 disable_submission(guc);
2120 static bool need_tasklet(struct intel_guc *guc, struct i915_request *rq)
2122 struct i915_sched_engine *sched_engine = rq->engine->sched_engine;
2123 struct intel_context *ce = request_to_scheduling_context(rq);
2125 return submission_disabled(guc) || guc->stalled_request ||
2126 !i915_sched_engine_is_empty(sched_engine) ||
2127 !ctx_id_mapped(guc, ce->guc_id.id);
2130 static void guc_submit_request(struct i915_request *rq)
2132 struct i915_sched_engine *sched_engine = rq->engine->sched_engine;
2133 struct intel_guc *guc = &rq->engine->gt->uc.guc;
2134 unsigned long flags;
2136 /* Will be called from irq-context when using foreign fences. */
2137 spin_lock_irqsave(&sched_engine->lock, flags);
2139 if (need_tasklet(guc, rq))
2140 queue_request(sched_engine, rq, rq_prio(rq));
2141 else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY)
2142 tasklet_hi_schedule(&sched_engine->tasklet);
2144 spin_unlock_irqrestore(&sched_engine->lock, flags);
2147 static int new_guc_id(struct intel_guc *guc, struct intel_context *ce)
2151 GEM_BUG_ON(intel_context_is_child(ce));
2153 if (intel_context_is_parent(ce))
2154 ret = bitmap_find_free_region(guc->submission_state.guc_ids_bitmap,
2155 NUMBER_MULTI_LRC_GUC_ID(guc),
2156 order_base_2(ce->parallel.number_children
2159 ret = ida_simple_get(&guc->submission_state.guc_ids,
2160 NUMBER_MULTI_LRC_GUC_ID(guc),
2161 guc->submission_state.num_guc_ids,
2162 GFP_KERNEL | __GFP_RETRY_MAYFAIL |
2164 if (unlikely(ret < 0))
2167 if (!intel_context_is_parent(ce))
2168 ++guc->submission_state.guc_ids_in_use;
2170 ce->guc_id.id = ret;
2174 static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce)
2176 GEM_BUG_ON(intel_context_is_child(ce));
2178 if (!context_guc_id_invalid(ce)) {
2179 if (intel_context_is_parent(ce)) {
2180 bitmap_release_region(guc->submission_state.guc_ids_bitmap,
2182 order_base_2(ce->parallel.number_children
2185 --guc->submission_state.guc_ids_in_use;
2186 ida_simple_remove(&guc->submission_state.guc_ids,
2189 clr_ctx_id_mapping(guc, ce->guc_id.id);
2190 set_context_guc_id_invalid(ce);
2192 if (!list_empty(&ce->guc_id.link))
2193 list_del_init(&ce->guc_id.link);
2196 static void release_guc_id(struct intel_guc *guc, struct intel_context *ce)
2198 unsigned long flags;
2200 spin_lock_irqsave(&guc->submission_state.lock, flags);
2201 __release_guc_id(guc, ce);
2202 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
2205 static int steal_guc_id(struct intel_guc *guc, struct intel_context *ce)
2207 struct intel_context *cn;
2209 lockdep_assert_held(&guc->submission_state.lock);
2210 GEM_BUG_ON(intel_context_is_child(ce));
2211 GEM_BUG_ON(intel_context_is_parent(ce));
2213 if (!list_empty(&guc->submission_state.guc_id_list)) {
2214 cn = list_first_entry(&guc->submission_state.guc_id_list,
2215 struct intel_context,
2218 GEM_BUG_ON(atomic_read(&cn->guc_id.ref));
2219 GEM_BUG_ON(context_guc_id_invalid(cn));
2220 GEM_BUG_ON(intel_context_is_child(cn));
2221 GEM_BUG_ON(intel_context_is_parent(cn));
2223 list_del_init(&cn->guc_id.link);
2224 ce->guc_id.id = cn->guc_id.id;
2226 spin_lock(&cn->guc_state.lock);
2227 clr_context_registered(cn);
2228 spin_unlock(&cn->guc_state.lock);
2230 set_context_guc_id_invalid(cn);
2232 #ifdef CONFIG_DRM_I915_SELFTEST
2233 guc->number_guc_id_stolen++;
2242 static int assign_guc_id(struct intel_guc *guc, struct intel_context *ce)
2246 lockdep_assert_held(&guc->submission_state.lock);
2247 GEM_BUG_ON(intel_context_is_child(ce));
2249 ret = new_guc_id(guc, ce);
2250 if (unlikely(ret < 0)) {
2251 if (intel_context_is_parent(ce))
2254 ret = steal_guc_id(guc, ce);
2259 if (intel_context_is_parent(ce)) {
2260 struct intel_context *child;
2263 for_each_child(ce, child)
2264 child->guc_id.id = ce->guc_id.id + i++;
2270 #define PIN_GUC_ID_TRIES 4
2271 static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce)
2274 unsigned long flags, tries = PIN_GUC_ID_TRIES;
2276 GEM_BUG_ON(atomic_read(&ce->guc_id.ref));
2279 spin_lock_irqsave(&guc->submission_state.lock, flags);
2281 might_lock(&ce->guc_state.lock);
2283 if (context_guc_id_invalid(ce)) {
2284 ret = assign_guc_id(guc, ce);
2287 ret = 1; /* Indidcates newly assigned guc_id */
2289 if (!list_empty(&ce->guc_id.link))
2290 list_del_init(&ce->guc_id.link);
2291 atomic_inc(&ce->guc_id.ref);
2294 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
2297 * -EAGAIN indicates no guc_id are available, let's retire any
2298 * outstanding requests to see if that frees up a guc_id. If the first
2299 * retire didn't help, insert a sleep with the timeslice duration before
2300 * attempting to retire more requests. Double the sleep period each
2301 * subsequent pass before finally giving up. The sleep period has max of
2302 * 100ms and minimum of 1ms.
2304 if (ret == -EAGAIN && --tries) {
2305 if (PIN_GUC_ID_TRIES - tries > 1) {
2306 unsigned int timeslice_shifted =
2307 ce->engine->props.timeslice_duration_ms <<
2308 (PIN_GUC_ID_TRIES - tries - 2);
2309 unsigned int max = min_t(unsigned int, 100,
2312 msleep(max_t(unsigned int, max, 1));
2314 intel_gt_retire_requests(guc_to_gt(guc));
2321 static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce)
2323 unsigned long flags;
2325 GEM_BUG_ON(atomic_read(&ce->guc_id.ref) < 0);
2326 GEM_BUG_ON(intel_context_is_child(ce));
2328 if (unlikely(context_guc_id_invalid(ce) ||
2329 intel_context_is_parent(ce)))
2332 spin_lock_irqsave(&guc->submission_state.lock, flags);
2333 if (!context_guc_id_invalid(ce) && list_empty(&ce->guc_id.link) &&
2334 !atomic_read(&ce->guc_id.ref))
2335 list_add_tail(&ce->guc_id.link,
2336 &guc->submission_state.guc_id_list);
2337 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
2340 static int __guc_action_register_multi_lrc_v69(struct intel_guc *guc,
2341 struct intel_context *ce,
2346 struct intel_context *child;
2347 u32 action[4 + MAX_ENGINE_INSTANCE];
2350 GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE);
2352 action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
2353 action[len++] = guc_id;
2354 action[len++] = ce->parallel.number_children + 1;
2355 action[len++] = offset;
2356 for_each_child(ce, child) {
2357 offset += sizeof(struct guc_lrc_desc_v69);
2358 action[len++] = offset;
2361 return guc_submission_send_busy_loop(guc, action, len, 0, loop);
2364 static int __guc_action_register_multi_lrc_v70(struct intel_guc *guc,
2365 struct intel_context *ce,
2366 struct guc_ctxt_registration_info *info,
2369 struct intel_context *child;
2370 u32 action[13 + (MAX_ENGINE_INSTANCE * 2)];
2374 GEM_BUG_ON(ce->parallel.number_children > MAX_ENGINE_INSTANCE);
2376 action[len++] = INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC;
2377 action[len++] = info->flags;
2378 action[len++] = info->context_idx;
2379 action[len++] = info->engine_class;
2380 action[len++] = info->engine_submit_mask;
2381 action[len++] = info->wq_desc_lo;
2382 action[len++] = info->wq_desc_hi;
2383 action[len++] = info->wq_base_lo;
2384 action[len++] = info->wq_base_hi;
2385 action[len++] = info->wq_size;
2386 action[len++] = ce->parallel.number_children + 1;
2387 action[len++] = info->hwlrca_lo;
2388 action[len++] = info->hwlrca_hi;
2390 next_id = info->context_idx + 1;
2391 for_each_child(ce, child) {
2392 GEM_BUG_ON(next_id++ != child->guc_id.id);
2395 * NB: GuC interface supports 64 bit LRCA even though i915/HW
2396 * only supports 32 bit currently.
2398 action[len++] = lower_32_bits(child->lrc.lrca);
2399 action[len++] = upper_32_bits(child->lrc.lrca);
2402 GEM_BUG_ON(len > ARRAY_SIZE(action));
2404 return guc_submission_send_busy_loop(guc, action, len, 0, loop);
2407 static int __guc_action_register_context_v69(struct intel_guc *guc,
2413 INTEL_GUC_ACTION_REGISTER_CONTEXT,
2418 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2422 static int __guc_action_register_context_v70(struct intel_guc *guc,
2423 struct guc_ctxt_registration_info *info,
2427 INTEL_GUC_ACTION_REGISTER_CONTEXT,
2431 info->engine_submit_mask,
2441 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2445 static void prepare_context_registration_info_v69(struct intel_context *ce);
2446 static void prepare_context_registration_info_v70(struct intel_context *ce,
2447 struct guc_ctxt_registration_info *info);
2450 register_context_v69(struct intel_guc *guc, struct intel_context *ce, bool loop)
2452 u32 offset = intel_guc_ggtt_offset(guc, guc->lrc_desc_pool_v69) +
2453 ce->guc_id.id * sizeof(struct guc_lrc_desc_v69);
2455 prepare_context_registration_info_v69(ce);
2457 if (intel_context_is_parent(ce))
2458 return __guc_action_register_multi_lrc_v69(guc, ce, ce->guc_id.id,
2461 return __guc_action_register_context_v69(guc, ce->guc_id.id,
2466 register_context_v70(struct intel_guc *guc, struct intel_context *ce, bool loop)
2468 struct guc_ctxt_registration_info info;
2470 prepare_context_registration_info_v70(ce, &info);
2472 if (intel_context_is_parent(ce))
2473 return __guc_action_register_multi_lrc_v70(guc, ce, &info, loop);
2475 return __guc_action_register_context_v70(guc, &info, loop);
2478 static int register_context(struct intel_context *ce, bool loop)
2480 struct intel_guc *guc = ce_to_guc(ce);
2483 GEM_BUG_ON(intel_context_is_child(ce));
2484 trace_intel_context_register(ce);
2486 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))
2487 ret = register_context_v70(guc, ce, loop);
2489 ret = register_context_v69(guc, ce, loop);
2492 unsigned long flags;
2494 spin_lock_irqsave(&ce->guc_state.lock, flags);
2495 set_context_registered(ce);
2496 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
2498 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))
2499 guc_context_policy_init_v70(ce, loop);
2505 static int __guc_action_deregister_context(struct intel_guc *guc,
2509 INTEL_GUC_ACTION_DEREGISTER_CONTEXT,
2513 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2514 G2H_LEN_DW_DEREGISTER_CONTEXT,
2518 static int deregister_context(struct intel_context *ce, u32 guc_id)
2520 struct intel_guc *guc = ce_to_guc(ce);
2522 GEM_BUG_ON(intel_context_is_child(ce));
2523 trace_intel_context_deregister(ce);
2525 return __guc_action_deregister_context(guc, guc_id);
2528 static inline void clear_children_join_go_memory(struct intel_context *ce)
2530 struct parent_scratch *ps = __get_parent_scratch(ce);
2533 ps->go.semaphore = 0;
2534 for (i = 0; i < ce->parallel.number_children + 1; ++i)
2535 ps->join[i].semaphore = 0;
2538 static inline u32 get_children_go_value(struct intel_context *ce)
2540 return __get_parent_scratch(ce)->go.semaphore;
2543 static inline u32 get_children_join_value(struct intel_context *ce,
2546 return __get_parent_scratch(ce)->join[child_index].semaphore;
2549 struct context_policy {
2551 struct guc_update_context_policy h2g;
2554 static u32 __guc_context_policy_action_size(struct context_policy *policy)
2556 size_t bytes = sizeof(policy->h2g.header) +
2557 (sizeof(policy->h2g.klv[0]) * policy->count);
2559 return bytes / sizeof(u32);
2562 static void __guc_context_policy_start_klv(struct context_policy *policy, u16 guc_id)
2564 policy->h2g.header.action = INTEL_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES;
2565 policy->h2g.header.ctx_id = guc_id;
2569 #define MAKE_CONTEXT_POLICY_ADD(func, id) \
2570 static void __guc_context_policy_add_##func(struct context_policy *policy, u32 data) \
2572 GEM_BUG_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \
2573 policy->h2g.klv[policy->count].kl = \
2574 FIELD_PREP(GUC_KLV_0_KEY, GUC_CONTEXT_POLICIES_KLV_ID_##id) | \
2575 FIELD_PREP(GUC_KLV_0_LEN, 1); \
2576 policy->h2g.klv[policy->count].value = data; \
2580 MAKE_CONTEXT_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM)
2581 MAKE_CONTEXT_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT)
2582 MAKE_CONTEXT_POLICY_ADD(priority, SCHEDULING_PRIORITY)
2583 MAKE_CONTEXT_POLICY_ADD(preempt_to_idle, PREEMPT_TO_IDLE_ON_QUANTUM_EXPIRY)
2585 #undef MAKE_CONTEXT_POLICY_ADD
2587 static int __guc_context_set_context_policies(struct intel_guc *guc,
2588 struct context_policy *policy,
2591 return guc_submission_send_busy_loop(guc, (u32 *)&policy->h2g,
2592 __guc_context_policy_action_size(policy),
2596 static int guc_context_policy_init_v70(struct intel_context *ce, bool loop)
2598 struct intel_engine_cs *engine = ce->engine;
2599 struct intel_guc *guc = &engine->gt->uc.guc;
2600 struct context_policy policy;
2601 u32 execution_quantum;
2602 u32 preemption_timeout;
2603 unsigned long flags;
2606 /* NB: For both of these, zero means disabled. */
2607 GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,
2608 execution_quantum));
2609 GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,
2610 preemption_timeout));
2611 execution_quantum = engine->props.timeslice_duration_ms * 1000;
2612 preemption_timeout = engine->props.preempt_timeout_ms * 1000;
2614 __guc_context_policy_start_klv(&policy, ce->guc_id.id);
2616 __guc_context_policy_add_priority(&policy, ce->guc_state.prio);
2617 __guc_context_policy_add_execution_quantum(&policy, execution_quantum);
2618 __guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);
2620 if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)
2621 __guc_context_policy_add_preempt_to_idle(&policy, 1);
2623 ret = __guc_context_set_context_policies(guc, &policy, loop);
2625 spin_lock_irqsave(&ce->guc_state.lock, flags);
2627 set_context_policy_required(ce);
2629 clr_context_policy_required(ce);
2630 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
2635 static void guc_context_policy_init_v69(struct intel_engine_cs *engine,
2636 struct guc_lrc_desc_v69 *desc)
2638 desc->policy_flags = 0;
2640 if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)
2641 desc->policy_flags |= CONTEXT_POLICY_FLAG_PREEMPT_TO_IDLE_V69;
2643 /* NB: For both of these, zero means disabled. */
2644 GEM_BUG_ON(overflows_type(engine->props.timeslice_duration_ms * 1000,
2645 desc->execution_quantum));
2646 GEM_BUG_ON(overflows_type(engine->props.preempt_timeout_ms * 1000,
2647 desc->preemption_timeout));
2648 desc->execution_quantum = engine->props.timeslice_duration_ms * 1000;
2649 desc->preemption_timeout = engine->props.preempt_timeout_ms * 1000;
2652 static u32 map_guc_prio_to_lrc_desc_prio(u8 prio)
2655 * this matches the mapping we do in map_i915_prio_to_guc_prio()
2656 * (e.g. prio < I915_PRIORITY_NORMAL maps to GUC_CLIENT_PRIORITY_NORMAL)
2662 case GUC_CLIENT_PRIORITY_KMD_NORMAL:
2663 return GEN12_CTX_PRIORITY_NORMAL;
2664 case GUC_CLIENT_PRIORITY_NORMAL:
2665 return GEN12_CTX_PRIORITY_LOW;
2666 case GUC_CLIENT_PRIORITY_HIGH:
2667 case GUC_CLIENT_PRIORITY_KMD_HIGH:
2668 return GEN12_CTX_PRIORITY_HIGH;
2672 static void prepare_context_registration_info_v69(struct intel_context *ce)
2674 struct intel_engine_cs *engine = ce->engine;
2675 struct intel_guc *guc = &engine->gt->uc.guc;
2676 u32 ctx_id = ce->guc_id.id;
2677 struct guc_lrc_desc_v69 *desc;
2678 struct intel_context *child;
2680 GEM_BUG_ON(!engine->mask);
2683 * Ensure LRC + CT vmas are is same region as write barrier is done
2684 * based on CT vma region.
2686 GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=
2687 i915_gem_object_is_lmem(ce->ring->vma->obj));
2689 desc = __get_lrc_desc_v69(guc, ctx_id);
2691 desc->engine_class = engine_class_to_guc_class(engine->class);
2692 desc->engine_submit_mask = engine->logical_mask;
2693 desc->hw_context_desc = ce->lrc.lrca;
2694 desc->priority = ce->guc_state.prio;
2695 desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD;
2696 guc_context_policy_init_v69(engine, desc);
2699 * If context is a parent, we need to register a process descriptor
2700 * describing a work queue and register all child contexts.
2702 if (intel_context_is_parent(ce)) {
2703 struct guc_process_desc_v69 *pdesc;
2705 ce->parallel.guc.wqi_tail = 0;
2706 ce->parallel.guc.wqi_head = 0;
2708 desc->process_desc = i915_ggtt_offset(ce->state) +
2709 __get_parent_scratch_offset(ce);
2710 desc->wq_addr = i915_ggtt_offset(ce->state) +
2711 __get_wq_offset(ce);
2712 desc->wq_size = WQ_SIZE;
2714 pdesc = __get_process_desc_v69(ce);
2715 memset(pdesc, 0, sizeof(*(pdesc)));
2716 pdesc->stage_id = ce->guc_id.id;
2717 pdesc->wq_base_addr = desc->wq_addr;
2718 pdesc->wq_size_bytes = desc->wq_size;
2719 pdesc->wq_status = WQ_STATUS_ACTIVE;
2721 ce->parallel.guc.wq_head = &pdesc->head;
2722 ce->parallel.guc.wq_tail = &pdesc->tail;
2723 ce->parallel.guc.wq_status = &pdesc->wq_status;
2725 for_each_child(ce, child) {
2726 desc = __get_lrc_desc_v69(guc, child->guc_id.id);
2728 desc->engine_class =
2729 engine_class_to_guc_class(engine->class);
2730 desc->hw_context_desc = child->lrc.lrca;
2731 desc->priority = ce->guc_state.prio;
2732 desc->context_flags = CONTEXT_REGISTRATION_FLAG_KMD;
2733 guc_context_policy_init_v69(engine, desc);
2736 clear_children_join_go_memory(ce);
2740 static void prepare_context_registration_info_v70(struct intel_context *ce,
2741 struct guc_ctxt_registration_info *info)
2743 struct intel_engine_cs *engine = ce->engine;
2744 struct intel_guc *guc = &engine->gt->uc.guc;
2745 u32 ctx_id = ce->guc_id.id;
2747 GEM_BUG_ON(!engine->mask);
2750 * Ensure LRC + CT vmas are is same region as write barrier is done
2751 * based on CT vma region.
2753 GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=
2754 i915_gem_object_is_lmem(ce->ring->vma->obj));
2756 memset(info, 0, sizeof(*info));
2757 info->context_idx = ctx_id;
2758 info->engine_class = engine_class_to_guc_class(engine->class);
2759 info->engine_submit_mask = engine->logical_mask;
2761 * NB: GuC interface supports 64 bit LRCA even though i915/HW
2762 * only supports 32 bit currently.
2764 info->hwlrca_lo = lower_32_bits(ce->lrc.lrca);
2765 info->hwlrca_hi = upper_32_bits(ce->lrc.lrca);
2766 if (engine->flags & I915_ENGINE_HAS_EU_PRIORITY)
2767 info->hwlrca_lo |= map_guc_prio_to_lrc_desc_prio(ce->guc_state.prio);
2768 info->flags = CONTEXT_REGISTRATION_FLAG_KMD;
2771 * If context is a parent, we need to register a process descriptor
2772 * describing a work queue and register all child contexts.
2774 if (intel_context_is_parent(ce)) {
2775 struct guc_sched_wq_desc *wq_desc;
2776 u64 wq_desc_offset, wq_base_offset;
2778 ce->parallel.guc.wqi_tail = 0;
2779 ce->parallel.guc.wqi_head = 0;
2781 wq_desc_offset = i915_ggtt_offset(ce->state) +
2782 __get_parent_scratch_offset(ce);
2783 wq_base_offset = i915_ggtt_offset(ce->state) +
2784 __get_wq_offset(ce);
2785 info->wq_desc_lo = lower_32_bits(wq_desc_offset);
2786 info->wq_desc_hi = upper_32_bits(wq_desc_offset);
2787 info->wq_base_lo = lower_32_bits(wq_base_offset);
2788 info->wq_base_hi = upper_32_bits(wq_base_offset);
2789 info->wq_size = WQ_SIZE;
2791 wq_desc = __get_wq_desc_v70(ce);
2792 memset(wq_desc, 0, sizeof(*wq_desc));
2793 wq_desc->wq_status = WQ_STATUS_ACTIVE;
2795 ce->parallel.guc.wq_head = &wq_desc->head;
2796 ce->parallel.guc.wq_tail = &wq_desc->tail;
2797 ce->parallel.guc.wq_status = &wq_desc->wq_status;
2799 clear_children_join_go_memory(ce);
2803 static int try_context_registration(struct intel_context *ce, bool loop)
2805 struct intel_engine_cs *engine = ce->engine;
2806 struct intel_runtime_pm *runtime_pm = engine->uncore->rpm;
2807 struct intel_guc *guc = &engine->gt->uc.guc;
2808 intel_wakeref_t wakeref;
2809 u32 ctx_id = ce->guc_id.id;
2810 bool context_registered;
2813 GEM_BUG_ON(!sched_state_is_init(ce));
2815 context_registered = ctx_id_mapped(guc, ctx_id);
2817 clr_ctx_id_mapping(guc, ctx_id);
2818 set_ctx_id_mapping(guc, ctx_id, ce);
2821 * The context_lookup xarray is used to determine if the hardware
2822 * context is currently registered. There are two cases in which it
2823 * could be registered either the guc_id has been stolen from another
2824 * context or the lrc descriptor address of this context has changed. In
2825 * either case the context needs to be deregistered with the GuC before
2826 * registering this context.
2828 if (context_registered) {
2830 unsigned long flags;
2832 trace_intel_context_steal_guc_id(ce);
2835 /* Seal race with Reset */
2836 spin_lock_irqsave(&ce->guc_state.lock, flags);
2837 disabled = submission_disabled(guc);
2838 if (likely(!disabled)) {
2839 set_context_wait_for_deregister_to_register(ce);
2840 intel_context_get(ce);
2842 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
2843 if (unlikely(disabled)) {
2844 clr_ctx_id_mapping(guc, ctx_id);
2845 return 0; /* Will get registered later */
2849 * If stealing the guc_id, this ce has the same guc_id as the
2850 * context whose guc_id was stolen.
2852 with_intel_runtime_pm(runtime_pm, wakeref)
2853 ret = deregister_context(ce, ce->guc_id.id);
2854 if (unlikely(ret == -ENODEV))
2855 ret = 0; /* Will get registered later */
2857 with_intel_runtime_pm(runtime_pm, wakeref)
2858 ret = register_context(ce, loop);
2859 if (unlikely(ret == -EBUSY)) {
2860 clr_ctx_id_mapping(guc, ctx_id);
2861 } else if (unlikely(ret == -ENODEV)) {
2862 clr_ctx_id_mapping(guc, ctx_id);
2863 ret = 0; /* Will get registered later */
2870 static int __guc_context_pre_pin(struct intel_context *ce,
2871 struct intel_engine_cs *engine,
2872 struct i915_gem_ww_ctx *ww,
2875 return lrc_pre_pin(ce, engine, ww, vaddr);
2878 static int __guc_context_pin(struct intel_context *ce,
2879 struct intel_engine_cs *engine,
2882 if (i915_ggtt_offset(ce->state) !=
2883 (ce->lrc.lrca & CTX_GTT_ADDRESS_MASK))
2884 set_bit(CONTEXT_LRCA_DIRTY, &ce->flags);
2887 * GuC context gets pinned in guc_request_alloc. See that function for
2888 * explaination of why.
2891 return lrc_pin(ce, engine, vaddr);
2894 static int guc_context_pre_pin(struct intel_context *ce,
2895 struct i915_gem_ww_ctx *ww,
2898 return __guc_context_pre_pin(ce, ce->engine, ww, vaddr);
2901 static int guc_context_pin(struct intel_context *ce, void *vaddr)
2903 int ret = __guc_context_pin(ce, ce->engine, vaddr);
2905 if (likely(!ret && !intel_context_is_barrier(ce)))
2906 intel_engine_pm_get(ce->engine);
2911 static void guc_context_unpin(struct intel_context *ce)
2913 struct intel_guc *guc = ce_to_guc(ce);
2915 __guc_context_update_stats(ce);
2916 unpin_guc_id(guc, ce);
2919 if (likely(!intel_context_is_barrier(ce)))
2920 intel_engine_pm_put_async(ce->engine);
2923 static void guc_context_post_unpin(struct intel_context *ce)
2928 static void __guc_context_sched_enable(struct intel_guc *guc,
2929 struct intel_context *ce)
2932 INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,
2937 trace_intel_context_sched_enable(ce);
2939 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2940 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true);
2943 static void __guc_context_sched_disable(struct intel_guc *guc,
2944 struct intel_context *ce,
2948 INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_SET,
2949 guc_id, /* ce->guc_id.id not stable */
2953 GEM_BUG_ON(guc_id == GUC_INVALID_CONTEXT_ID);
2955 GEM_BUG_ON(intel_context_is_child(ce));
2956 trace_intel_context_sched_disable(ce);
2958 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2959 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, true);
2962 static void guc_blocked_fence_complete(struct intel_context *ce)
2964 lockdep_assert_held(&ce->guc_state.lock);
2966 if (!i915_sw_fence_done(&ce->guc_state.blocked))
2967 i915_sw_fence_complete(&ce->guc_state.blocked);
2970 static void guc_blocked_fence_reinit(struct intel_context *ce)
2972 lockdep_assert_held(&ce->guc_state.lock);
2973 GEM_BUG_ON(!i915_sw_fence_done(&ce->guc_state.blocked));
2976 * This fence is always complete unless a pending schedule disable is
2977 * outstanding. We arm the fence here and complete it when we receive
2978 * the pending schedule disable complete message.
2980 i915_sw_fence_fini(&ce->guc_state.blocked);
2981 i915_sw_fence_reinit(&ce->guc_state.blocked);
2982 i915_sw_fence_await(&ce->guc_state.blocked);
2983 i915_sw_fence_commit(&ce->guc_state.blocked);
2986 static u16 prep_context_pending_disable(struct intel_context *ce)
2988 lockdep_assert_held(&ce->guc_state.lock);
2990 set_context_pending_disable(ce);
2991 clr_context_enabled(ce);
2992 guc_blocked_fence_reinit(ce);
2993 intel_context_get(ce);
2995 return ce->guc_id.id;
2998 static struct i915_sw_fence *guc_context_block(struct intel_context *ce)
3000 struct intel_guc *guc = ce_to_guc(ce);
3001 unsigned long flags;
3002 struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;
3003 intel_wakeref_t wakeref;
3007 GEM_BUG_ON(intel_context_is_child(ce));
3009 spin_lock_irqsave(&ce->guc_state.lock, flags);
3011 incr_context_blocked(ce);
3013 enabled = context_enabled(ce);
3014 if (unlikely(!enabled || submission_disabled(guc))) {
3016 clr_context_enabled(ce);
3017 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3018 return &ce->guc_state.blocked;
3022 * We add +2 here as the schedule disable complete CTB handler calls
3023 * intel_context_sched_disable_unpin (-2 to pin_count).
3025 atomic_add(2, &ce->pin_count);
3027 guc_id = prep_context_pending_disable(ce);
3029 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3031 with_intel_runtime_pm(runtime_pm, wakeref)
3032 __guc_context_sched_disable(guc, ce, guc_id);
3034 return &ce->guc_state.blocked;
3037 #define SCHED_STATE_MULTI_BLOCKED_MASK \
3038 (SCHED_STATE_BLOCKED_MASK & ~SCHED_STATE_BLOCKED)
3039 #define SCHED_STATE_NO_UNBLOCK \
3040 (SCHED_STATE_MULTI_BLOCKED_MASK | \
3041 SCHED_STATE_PENDING_DISABLE | \
3044 static bool context_cant_unblock(struct intel_context *ce)
3046 lockdep_assert_held(&ce->guc_state.lock);
3048 return (ce->guc_state.sched_state & SCHED_STATE_NO_UNBLOCK) ||
3049 context_guc_id_invalid(ce) ||
3050 !ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id) ||
3051 !intel_context_is_pinned(ce);
3054 static void guc_context_unblock(struct intel_context *ce)
3056 struct intel_guc *guc = ce_to_guc(ce);
3057 unsigned long flags;
3058 struct intel_runtime_pm *runtime_pm = ce->engine->uncore->rpm;
3059 intel_wakeref_t wakeref;
3062 GEM_BUG_ON(context_enabled(ce));
3063 GEM_BUG_ON(intel_context_is_child(ce));
3065 spin_lock_irqsave(&ce->guc_state.lock, flags);
3067 if (unlikely(submission_disabled(guc) ||
3068 context_cant_unblock(ce))) {
3072 set_context_pending_enable(ce);
3073 set_context_enabled(ce);
3074 intel_context_get(ce);
3077 decr_context_blocked(ce);
3079 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3082 with_intel_runtime_pm(runtime_pm, wakeref)
3083 __guc_context_sched_enable(guc, ce);
3087 static void guc_context_cancel_request(struct intel_context *ce,
3088 struct i915_request *rq)
3090 struct intel_context *block_context =
3091 request_to_scheduling_context(rq);
3093 if (i915_sw_fence_signaled(&rq->submit)) {
3094 struct i915_sw_fence *fence;
3096 intel_context_get(ce);
3097 fence = guc_context_block(block_context);
3098 i915_sw_fence_wait(fence);
3099 if (!i915_request_completed(rq)) {
3100 __i915_request_skip(rq);
3101 guc_reset_state(ce, intel_ring_wrap(ce->ring, rq->head),
3105 guc_context_unblock(block_context);
3106 intel_context_put(ce);
3110 static void __guc_context_set_preemption_timeout(struct intel_guc *guc,
3112 u32 preemption_timeout)
3114 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {
3115 struct context_policy policy;
3117 __guc_context_policy_start_klv(&policy, guc_id);
3118 __guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);
3119 __guc_context_set_context_policies(guc, &policy, true);
3122 INTEL_GUC_ACTION_V69_SET_CONTEXT_PREEMPTION_TIMEOUT,
3127 intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
3132 guc_context_revoke(struct intel_context *ce, struct i915_request *rq,
3133 unsigned int preempt_timeout_ms)
3135 struct intel_guc *guc = ce_to_guc(ce);
3136 struct intel_runtime_pm *runtime_pm =
3137 &ce->engine->gt->i915->runtime_pm;
3138 intel_wakeref_t wakeref;
3139 unsigned long flags;
3141 GEM_BUG_ON(intel_context_is_child(ce));
3143 guc_flush_submissions(guc);
3145 spin_lock_irqsave(&ce->guc_state.lock, flags);
3146 set_context_banned(ce);
3148 if (submission_disabled(guc) ||
3149 (!context_enabled(ce) && !context_pending_disable(ce))) {
3150 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3152 guc_cancel_context_requests(ce);
3153 intel_engine_signal_breadcrumbs(ce->engine);
3154 } else if (!context_pending_disable(ce)) {
3158 * We add +2 here as the schedule disable complete CTB handler
3159 * calls intel_context_sched_disable_unpin (-2 to pin_count).
3161 atomic_add(2, &ce->pin_count);
3163 guc_id = prep_context_pending_disable(ce);
3164 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3167 * In addition to disabling scheduling, set the preemption
3168 * timeout to the minimum value (1 us) so the banned context
3169 * gets kicked off the HW ASAP.
3171 with_intel_runtime_pm(runtime_pm, wakeref) {
3172 __guc_context_set_preemption_timeout(guc, guc_id,
3173 preempt_timeout_ms);
3174 __guc_context_sched_disable(guc, ce, guc_id);
3177 if (!context_guc_id_invalid(ce))
3178 with_intel_runtime_pm(runtime_pm, wakeref)
3179 __guc_context_set_preemption_timeout(guc,
3181 preempt_timeout_ms);
3182 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3186 static void do_sched_disable(struct intel_guc *guc, struct intel_context *ce,
3187 unsigned long flags)
3188 __releases(ce->guc_state.lock)
3190 struct intel_runtime_pm *runtime_pm = &ce->engine->gt->i915->runtime_pm;
3191 intel_wakeref_t wakeref;
3194 lockdep_assert_held(&ce->guc_state.lock);
3195 guc_id = prep_context_pending_disable(ce);
3197 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3199 with_intel_runtime_pm(runtime_pm, wakeref)
3200 __guc_context_sched_disable(guc, ce, guc_id);
3203 static bool bypass_sched_disable(struct intel_guc *guc,
3204 struct intel_context *ce)
3206 lockdep_assert_held(&ce->guc_state.lock);
3207 GEM_BUG_ON(intel_context_is_child(ce));
3209 if (submission_disabled(guc) || context_guc_id_invalid(ce) ||
3210 !ctx_id_mapped(guc, ce->guc_id.id)) {
3211 clr_context_enabled(ce);
3215 return !context_enabled(ce);
3218 static void __delay_sched_disable(struct work_struct *wrk)
3220 struct intel_context *ce =
3221 container_of(wrk, typeof(*ce), guc_state.sched_disable_delay_work.work);
3222 struct intel_guc *guc = ce_to_guc(ce);
3223 unsigned long flags;
3225 spin_lock_irqsave(&ce->guc_state.lock, flags);
3227 if (bypass_sched_disable(guc, ce)) {
3228 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3229 intel_context_sched_disable_unpin(ce);
3231 do_sched_disable(guc, ce, flags);
3235 static bool guc_id_pressure(struct intel_guc *guc, struct intel_context *ce)
3238 * parent contexts are perma-pinned, if we are unpinning do schedule
3239 * disable immediately.
3241 if (intel_context_is_parent(ce))
3245 * If we are beyond the threshold for avail guc_ids, do schedule disable immediately.
3247 return guc->submission_state.guc_ids_in_use >
3248 guc->submission_state.sched_disable_gucid_threshold;
3251 static void guc_context_sched_disable(struct intel_context *ce)
3253 struct intel_guc *guc = ce_to_guc(ce);
3254 u64 delay = guc->submission_state.sched_disable_delay_ms;
3255 unsigned long flags;
3257 spin_lock_irqsave(&ce->guc_state.lock, flags);
3259 if (bypass_sched_disable(guc, ce)) {
3260 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3261 intel_context_sched_disable_unpin(ce);
3262 } else if (!intel_context_is_closed(ce) && !guc_id_pressure(guc, ce) &&
3264 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3265 mod_delayed_work(system_unbound_wq,
3266 &ce->guc_state.sched_disable_delay_work,
3267 msecs_to_jiffies(delay));
3269 do_sched_disable(guc, ce, flags);
3273 static void guc_context_close(struct intel_context *ce)
3275 unsigned long flags;
3277 if (test_bit(CONTEXT_GUC_INIT, &ce->flags) &&
3278 cancel_delayed_work(&ce->guc_state.sched_disable_delay_work))
3279 __delay_sched_disable(&ce->guc_state.sched_disable_delay_work.work);
3281 spin_lock_irqsave(&ce->guc_state.lock, flags);
3282 set_context_close_done(ce);
3283 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3286 static inline void guc_lrc_desc_unpin(struct intel_context *ce)
3288 struct intel_guc *guc = ce_to_guc(ce);
3289 struct intel_gt *gt = guc_to_gt(guc);
3290 unsigned long flags;
3293 GEM_BUG_ON(!intel_gt_pm_is_awake(gt));
3294 GEM_BUG_ON(!ctx_id_mapped(guc, ce->guc_id.id));
3295 GEM_BUG_ON(ce != __get_context(guc, ce->guc_id.id));
3296 GEM_BUG_ON(context_enabled(ce));
3298 /* Seal race with Reset */
3299 spin_lock_irqsave(&ce->guc_state.lock, flags);
3300 disabled = submission_disabled(guc);
3301 if (likely(!disabled)) {
3302 __intel_gt_pm_get(gt);
3303 set_context_destroyed(ce);
3304 clr_context_registered(ce);
3306 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3307 if (unlikely(disabled)) {
3308 release_guc_id(guc, ce);
3309 __guc_context_destroy(ce);
3313 deregister_context(ce, ce->guc_id.id);
3316 static void __guc_context_destroy(struct intel_context *ce)
3318 GEM_BUG_ON(ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_HIGH] ||
3319 ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_HIGH] ||
3320 ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_KMD_NORMAL] ||
3321 ce->guc_state.prio_count[GUC_CLIENT_PRIORITY_NORMAL]);
3324 intel_context_fini(ce);
3326 if (intel_engine_is_virtual(ce->engine)) {
3327 struct guc_virtual_engine *ve =
3328 container_of(ce, typeof(*ve), context);
3330 if (ve->base.breadcrumbs)
3331 intel_breadcrumbs_put(ve->base.breadcrumbs);
3335 intel_context_free(ce);
3339 static void guc_flush_destroyed_contexts(struct intel_guc *guc)
3341 struct intel_context *ce;
3342 unsigned long flags;
3344 GEM_BUG_ON(!submission_disabled(guc) &&
3345 guc_submission_initialized(guc));
3347 while (!list_empty(&guc->submission_state.destroyed_contexts)) {
3348 spin_lock_irqsave(&guc->submission_state.lock, flags);
3349 ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,
3350 struct intel_context,
3353 list_del_init(&ce->destroyed_link);
3354 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
3359 release_guc_id(guc, ce);
3360 __guc_context_destroy(ce);
3364 static void deregister_destroyed_contexts(struct intel_guc *guc)
3366 struct intel_context *ce;
3367 unsigned long flags;
3369 while (!list_empty(&guc->submission_state.destroyed_contexts)) {
3370 spin_lock_irqsave(&guc->submission_state.lock, flags);
3371 ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,
3372 struct intel_context,
3375 list_del_init(&ce->destroyed_link);
3376 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
3381 guc_lrc_desc_unpin(ce);
3385 static void destroyed_worker_func(struct work_struct *w)
3387 struct intel_guc *guc = container_of(w, struct intel_guc,
3388 submission_state.destroyed_worker);
3389 struct intel_gt *gt = guc_to_gt(guc);
3390 intel_wakeref_t wakeref;
3392 with_intel_gt_pm(gt, wakeref)
3393 deregister_destroyed_contexts(guc);
3396 static void guc_context_destroy(struct kref *kref)
3398 struct intel_context *ce = container_of(kref, typeof(*ce), ref);
3399 struct intel_guc *guc = ce_to_guc(ce);
3400 unsigned long flags;
3404 * If the guc_id is invalid this context has been stolen and we can free
3405 * it immediately. Also can be freed immediately if the context is not
3406 * registered with the GuC or the GuC is in the middle of a reset.
3408 spin_lock_irqsave(&guc->submission_state.lock, flags);
3409 destroy = submission_disabled(guc) || context_guc_id_invalid(ce) ||
3410 !ctx_id_mapped(guc, ce->guc_id.id);
3411 if (likely(!destroy)) {
3412 if (!list_empty(&ce->guc_id.link))
3413 list_del_init(&ce->guc_id.link);
3414 list_add_tail(&ce->destroyed_link,
3415 &guc->submission_state.destroyed_contexts);
3417 __release_guc_id(guc, ce);
3419 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
3420 if (unlikely(destroy)) {
3421 __guc_context_destroy(ce);
3426 * We use a worker to issue the H2G to deregister the context as we can
3427 * take the GT PM for the first time which isn't allowed from an atomic
3430 queue_work(system_unbound_wq, &guc->submission_state.destroyed_worker);
3433 static int guc_context_alloc(struct intel_context *ce)
3435 return lrc_alloc(ce, ce->engine);
3438 static void __guc_context_set_prio(struct intel_guc *guc,
3439 struct intel_context *ce)
3441 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {
3442 struct context_policy policy;
3444 __guc_context_policy_start_klv(&policy, ce->guc_id.id);
3445 __guc_context_policy_add_priority(&policy, ce->guc_state.prio);
3446 __guc_context_set_context_policies(guc, &policy, true);
3449 INTEL_GUC_ACTION_V69_SET_CONTEXT_PRIORITY,
3454 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
3458 static void guc_context_set_prio(struct intel_guc *guc,
3459 struct intel_context *ce,
3462 GEM_BUG_ON(prio < GUC_CLIENT_PRIORITY_KMD_HIGH ||
3463 prio > GUC_CLIENT_PRIORITY_NORMAL);
3464 lockdep_assert_held(&ce->guc_state.lock);
3466 if (ce->guc_state.prio == prio || submission_disabled(guc) ||
3467 !context_registered(ce)) {
3468 ce->guc_state.prio = prio;
3472 ce->guc_state.prio = prio;
3473 __guc_context_set_prio(guc, ce);
3475 trace_intel_context_set_prio(ce);
3478 static inline u8 map_i915_prio_to_guc_prio(int prio)
3480 if (prio == I915_PRIORITY_NORMAL)
3481 return GUC_CLIENT_PRIORITY_KMD_NORMAL;
3482 else if (prio < I915_PRIORITY_NORMAL)
3483 return GUC_CLIENT_PRIORITY_NORMAL;
3484 else if (prio < I915_PRIORITY_DISPLAY)
3485 return GUC_CLIENT_PRIORITY_HIGH;
3487 return GUC_CLIENT_PRIORITY_KMD_HIGH;
3490 static inline void add_context_inflight_prio(struct intel_context *ce,
3493 lockdep_assert_held(&ce->guc_state.lock);
3494 GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count));
3496 ++ce->guc_state.prio_count[guc_prio];
3498 /* Overflow protection */
3499 GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]);
3502 static inline void sub_context_inflight_prio(struct intel_context *ce,
3505 lockdep_assert_held(&ce->guc_state.lock);
3506 GEM_BUG_ON(guc_prio >= ARRAY_SIZE(ce->guc_state.prio_count));
3508 /* Underflow protection */
3509 GEM_WARN_ON(!ce->guc_state.prio_count[guc_prio]);
3511 --ce->guc_state.prio_count[guc_prio];
3514 static inline void update_context_prio(struct intel_context *ce)
3516 struct intel_guc *guc = &ce->engine->gt->uc.guc;
3519 BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH != 0);
3520 BUILD_BUG_ON(GUC_CLIENT_PRIORITY_KMD_HIGH > GUC_CLIENT_PRIORITY_NORMAL);
3522 lockdep_assert_held(&ce->guc_state.lock);
3524 for (i = 0; i < ARRAY_SIZE(ce->guc_state.prio_count); ++i) {
3525 if (ce->guc_state.prio_count[i]) {
3526 guc_context_set_prio(guc, ce, i);
3532 static inline bool new_guc_prio_higher(u8 old_guc_prio, u8 new_guc_prio)
3534 /* Lower value is higher priority */
3535 return new_guc_prio < old_guc_prio;
3538 static void add_to_context(struct i915_request *rq)
3540 struct intel_context *ce = request_to_scheduling_context(rq);
3541 u8 new_guc_prio = map_i915_prio_to_guc_prio(rq_prio(rq));
3543 GEM_BUG_ON(intel_context_is_child(ce));
3544 GEM_BUG_ON(rq->guc_prio == GUC_PRIO_FINI);
3546 spin_lock(&ce->guc_state.lock);
3547 list_move_tail(&rq->sched.link, &ce->guc_state.requests);
3549 if (rq->guc_prio == GUC_PRIO_INIT) {
3550 rq->guc_prio = new_guc_prio;
3551 add_context_inflight_prio(ce, rq->guc_prio);
3552 } else if (new_guc_prio_higher(rq->guc_prio, new_guc_prio)) {
3553 sub_context_inflight_prio(ce, rq->guc_prio);
3554 rq->guc_prio = new_guc_prio;
3555 add_context_inflight_prio(ce, rq->guc_prio);
3557 update_context_prio(ce);
3559 spin_unlock(&ce->guc_state.lock);
3562 static void guc_prio_fini(struct i915_request *rq, struct intel_context *ce)
3564 lockdep_assert_held(&ce->guc_state.lock);
3566 if (rq->guc_prio != GUC_PRIO_INIT &&
3567 rq->guc_prio != GUC_PRIO_FINI) {
3568 sub_context_inflight_prio(ce, rq->guc_prio);
3569 update_context_prio(ce);
3571 rq->guc_prio = GUC_PRIO_FINI;
3574 static void remove_from_context(struct i915_request *rq)
3576 struct intel_context *ce = request_to_scheduling_context(rq);
3578 GEM_BUG_ON(intel_context_is_child(ce));
3580 spin_lock_irq(&ce->guc_state.lock);
3582 list_del_init(&rq->sched.link);
3583 clear_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags);
3585 /* Prevent further __await_execution() registering a cb, then flush */
3586 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
3588 guc_prio_fini(rq, ce);
3590 spin_unlock_irq(&ce->guc_state.lock);
3592 atomic_dec(&ce->guc_id.ref);
3593 i915_request_notify_execute_cb_imm(rq);
3596 static const struct intel_context_ops guc_context_ops = {
3597 .flags = COPS_RUNTIME_CYCLES,
3598 .alloc = guc_context_alloc,
3600 .close = guc_context_close,
3602 .pre_pin = guc_context_pre_pin,
3603 .pin = guc_context_pin,
3604 .unpin = guc_context_unpin,
3605 .post_unpin = guc_context_post_unpin,
3607 .revoke = guc_context_revoke,
3609 .cancel_request = guc_context_cancel_request,
3611 .enter = intel_context_enter_engine,
3612 .exit = intel_context_exit_engine,
3614 .sched_disable = guc_context_sched_disable,
3616 .update_stats = guc_context_update_stats,
3619 .destroy = guc_context_destroy,
3621 .create_virtual = guc_create_virtual,
3622 .create_parallel = guc_create_parallel,
3625 static void submit_work_cb(struct irq_work *wrk)
3627 struct i915_request *rq = container_of(wrk, typeof(*rq), submit_work);
3629 might_lock(&rq->engine->sched_engine->lock);
3630 i915_sw_fence_complete(&rq->submit);
3633 static void __guc_signal_context_fence(struct intel_context *ce)
3635 struct i915_request *rq, *rn;
3637 lockdep_assert_held(&ce->guc_state.lock);
3639 if (!list_empty(&ce->guc_state.fences))
3640 trace_intel_context_fence_release(ce);
3643 * Use an IRQ to ensure locking order of sched_engine->lock ->
3644 * ce->guc_state.lock is preserved.
3646 list_for_each_entry_safe(rq, rn, &ce->guc_state.fences,
3648 list_del(&rq->guc_fence_link);
3649 irq_work_queue(&rq->submit_work);
3652 INIT_LIST_HEAD(&ce->guc_state.fences);
3655 static void guc_signal_context_fence(struct intel_context *ce)
3657 unsigned long flags;
3659 GEM_BUG_ON(intel_context_is_child(ce));
3661 spin_lock_irqsave(&ce->guc_state.lock, flags);
3662 clr_context_wait_for_deregister_to_register(ce);
3663 __guc_signal_context_fence(ce);
3664 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3667 static bool context_needs_register(struct intel_context *ce, bool new_guc_id)
3669 return (new_guc_id || test_bit(CONTEXT_LRCA_DIRTY, &ce->flags) ||
3670 !ctx_id_mapped(ce_to_guc(ce), ce->guc_id.id)) &&
3671 !submission_disabled(ce_to_guc(ce));
3674 static void guc_context_init(struct intel_context *ce)
3676 const struct i915_gem_context *ctx;
3677 int prio = I915_CONTEXT_DEFAULT_PRIORITY;
3680 ctx = rcu_dereference(ce->gem_context);
3682 prio = ctx->sched.priority;
3685 ce->guc_state.prio = map_i915_prio_to_guc_prio(prio);
3687 INIT_DELAYED_WORK(&ce->guc_state.sched_disable_delay_work,
3688 __delay_sched_disable);
3690 set_bit(CONTEXT_GUC_INIT, &ce->flags);
3693 static int guc_request_alloc(struct i915_request *rq)
3695 struct intel_context *ce = request_to_scheduling_context(rq);
3696 struct intel_guc *guc = ce_to_guc(ce);
3697 unsigned long flags;
3700 GEM_BUG_ON(!intel_context_is_pinned(rq->context));
3703 * Flush enough space to reduce the likelihood of waiting after
3704 * we start building the request - in which case we will just
3705 * have to repeat work.
3707 rq->reserved_space += GUC_REQUEST_SIZE;
3710 * Note that after this point, we have committed to using
3711 * this request as it is being used to both track the
3712 * state of engine initialisation and liveness of the
3713 * golden renderstate above. Think twice before you try
3714 * to cancel/unwind this request now.
3717 /* Unconditionally invalidate GPU caches and TLBs. */
3718 ret = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
3722 rq->reserved_space -= GUC_REQUEST_SIZE;
3724 if (unlikely(!test_bit(CONTEXT_GUC_INIT, &ce->flags)))
3725 guc_context_init(ce);
3728 * If the context gets closed while the execbuf is ongoing, the context
3729 * close code will race with the below code to cancel the delayed work.
3730 * If the context close wins the race and cancels the work, it will
3731 * immediately call the sched disable (see guc_context_close), so there
3732 * is a chance we can get past this check while the sched_disable code
3733 * is being executed. To make sure that code completes before we check
3734 * the status further down, we wait for the close process to complete.
3735 * Else, this code path could send a request down thinking that the
3736 * context is still in a schedule-enable mode while the GuC ends up
3737 * dropping the request completely because the disable did go from the
3738 * context_close path right to GuC just prior. In the event the CT is
3739 * full, we could potentially need to wait up to 1.5 seconds.
3741 if (cancel_delayed_work_sync(&ce->guc_state.sched_disable_delay_work))
3742 intel_context_sched_disable_unpin(ce);
3743 else if (intel_context_is_closed(ce))
3744 if (wait_for(context_close_done(ce), 1500))
3745 guc_warn(guc, "timed out waiting on context sched close before realloc\n");
3747 * Call pin_guc_id here rather than in the pinning step as with
3748 * dma_resv, contexts can be repeatedly pinned / unpinned trashing the
3749 * guc_id and creating horrible race conditions. This is especially bad
3750 * when guc_id are being stolen due to over subscription. By the time
3751 * this function is reached, it is guaranteed that the guc_id will be
3752 * persistent until the generated request is retired. Thus, sealing these
3753 * race conditions. It is still safe to fail here if guc_id are
3754 * exhausted and return -EAGAIN to the user indicating that they can try
3755 * again in the future.
3757 * There is no need for a lock here as the timeline mutex ensures at
3758 * most one context can be executing this code path at once. The
3759 * guc_id_ref is incremented once for every request in flight and
3760 * decremented on each retire. When it is zero, a lock around the
3761 * increment (in pin_guc_id) is needed to seal a race with unpin_guc_id.
3763 if (atomic_add_unless(&ce->guc_id.ref, 1, 0))
3766 ret = pin_guc_id(guc, ce); /* returns 1 if new guc_id assigned */
3767 if (unlikely(ret < 0))
3769 if (context_needs_register(ce, !!ret)) {
3770 ret = try_context_registration(ce, true);
3771 if (unlikely(ret)) { /* unwind */
3772 if (ret == -EPIPE) {
3773 disable_submission(guc);
3774 goto out; /* GPU will be reset */
3776 atomic_dec(&ce->guc_id.ref);
3777 unpin_guc_id(guc, ce);
3782 clear_bit(CONTEXT_LRCA_DIRTY, &ce->flags);
3786 * We block all requests on this context if a G2H is pending for a
3787 * schedule disable or context deregistration as the GuC will fail a
3788 * schedule enable or context registration if either G2H is pending
3789 * respectfully. Once a G2H returns, the fence is released that is
3790 * blocking these requests (see guc_signal_context_fence).
3792 spin_lock_irqsave(&ce->guc_state.lock, flags);
3793 if (context_wait_for_deregister_to_register(ce) ||
3794 context_pending_disable(ce)) {
3795 init_irq_work(&rq->submit_work, submit_work_cb);
3796 i915_sw_fence_await(&rq->submit);
3798 list_add_tail(&rq->guc_fence_link, &ce->guc_state.fences);
3800 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
3805 static int guc_virtual_context_pre_pin(struct intel_context *ce,
3806 struct i915_gem_ww_ctx *ww,
3809 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
3811 return __guc_context_pre_pin(ce, engine, ww, vaddr);
3814 static int guc_virtual_context_pin(struct intel_context *ce, void *vaddr)
3816 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
3817 int ret = __guc_context_pin(ce, engine, vaddr);
3818 intel_engine_mask_t tmp, mask = ce->engine->mask;
3821 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
3822 intel_engine_pm_get(engine);
3827 static void guc_virtual_context_unpin(struct intel_context *ce)
3829 intel_engine_mask_t tmp, mask = ce->engine->mask;
3830 struct intel_engine_cs *engine;
3831 struct intel_guc *guc = ce_to_guc(ce);
3833 GEM_BUG_ON(context_enabled(ce));
3834 GEM_BUG_ON(intel_context_is_barrier(ce));
3836 unpin_guc_id(guc, ce);
3839 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
3840 intel_engine_pm_put_async(engine);
3843 static void guc_virtual_context_enter(struct intel_context *ce)
3845 intel_engine_mask_t tmp, mask = ce->engine->mask;
3846 struct intel_engine_cs *engine;
3848 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
3849 intel_engine_pm_get(engine);
3851 intel_timeline_enter(ce->timeline);
3854 static void guc_virtual_context_exit(struct intel_context *ce)
3856 intel_engine_mask_t tmp, mask = ce->engine->mask;
3857 struct intel_engine_cs *engine;
3859 for_each_engine_masked(engine, ce->engine->gt, mask, tmp)
3860 intel_engine_pm_put(engine);
3862 intel_timeline_exit(ce->timeline);
3865 static int guc_virtual_context_alloc(struct intel_context *ce)
3867 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
3869 return lrc_alloc(ce, engine);
3872 static const struct intel_context_ops virtual_guc_context_ops = {
3873 .flags = COPS_RUNTIME_CYCLES,
3874 .alloc = guc_virtual_context_alloc,
3876 .close = guc_context_close,
3878 .pre_pin = guc_virtual_context_pre_pin,
3879 .pin = guc_virtual_context_pin,
3880 .unpin = guc_virtual_context_unpin,
3881 .post_unpin = guc_context_post_unpin,
3883 .revoke = guc_context_revoke,
3885 .cancel_request = guc_context_cancel_request,
3887 .enter = guc_virtual_context_enter,
3888 .exit = guc_virtual_context_exit,
3890 .sched_disable = guc_context_sched_disable,
3891 .update_stats = guc_context_update_stats,
3893 .destroy = guc_context_destroy,
3895 .get_sibling = guc_virtual_get_sibling,
3898 static int guc_parent_context_pin(struct intel_context *ce, void *vaddr)
3900 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
3901 struct intel_guc *guc = ce_to_guc(ce);
3904 GEM_BUG_ON(!intel_context_is_parent(ce));
3905 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
3907 ret = pin_guc_id(guc, ce);
3908 if (unlikely(ret < 0))
3911 return __guc_context_pin(ce, engine, vaddr);
3914 static int guc_child_context_pin(struct intel_context *ce, void *vaddr)
3916 struct intel_engine_cs *engine = guc_virtual_get_sibling(ce->engine, 0);
3918 GEM_BUG_ON(!intel_context_is_child(ce));
3919 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
3921 __intel_context_pin(ce->parallel.parent);
3922 return __guc_context_pin(ce, engine, vaddr);
3925 static void guc_parent_context_unpin(struct intel_context *ce)
3927 struct intel_guc *guc = ce_to_guc(ce);
3929 GEM_BUG_ON(context_enabled(ce));
3930 GEM_BUG_ON(intel_context_is_barrier(ce));
3931 GEM_BUG_ON(!intel_context_is_parent(ce));
3932 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
3934 unpin_guc_id(guc, ce);
3938 static void guc_child_context_unpin(struct intel_context *ce)
3940 GEM_BUG_ON(context_enabled(ce));
3941 GEM_BUG_ON(intel_context_is_barrier(ce));
3942 GEM_BUG_ON(!intel_context_is_child(ce));
3943 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
3948 static void guc_child_context_post_unpin(struct intel_context *ce)
3950 GEM_BUG_ON(!intel_context_is_child(ce));
3951 GEM_BUG_ON(!intel_context_is_pinned(ce->parallel.parent));
3952 GEM_BUG_ON(!intel_engine_is_virtual(ce->engine));
3955 intel_context_unpin(ce->parallel.parent);
3958 static void guc_child_context_destroy(struct kref *kref)
3960 struct intel_context *ce = container_of(kref, typeof(*ce), ref);
3962 __guc_context_destroy(ce);
3965 static const struct intel_context_ops virtual_parent_context_ops = {
3966 .alloc = guc_virtual_context_alloc,
3968 .close = guc_context_close,
3970 .pre_pin = guc_context_pre_pin,
3971 .pin = guc_parent_context_pin,
3972 .unpin = guc_parent_context_unpin,
3973 .post_unpin = guc_context_post_unpin,
3975 .revoke = guc_context_revoke,
3977 .cancel_request = guc_context_cancel_request,
3979 .enter = guc_virtual_context_enter,
3980 .exit = guc_virtual_context_exit,
3982 .sched_disable = guc_context_sched_disable,
3984 .destroy = guc_context_destroy,
3986 .get_sibling = guc_virtual_get_sibling,
3989 static const struct intel_context_ops virtual_child_context_ops = {
3990 .alloc = guc_virtual_context_alloc,
3992 .pre_pin = guc_context_pre_pin,
3993 .pin = guc_child_context_pin,
3994 .unpin = guc_child_context_unpin,
3995 .post_unpin = guc_child_context_post_unpin,
3997 .cancel_request = guc_context_cancel_request,
3999 .enter = guc_virtual_context_enter,
4000 .exit = guc_virtual_context_exit,
4002 .destroy = guc_child_context_destroy,
4004 .get_sibling = guc_virtual_get_sibling,
4008 * The below override of the breadcrumbs is enabled when the user configures a
4009 * context for parallel submission (multi-lrc, parent-child).
4011 * The overridden breadcrumbs implements an algorithm which allows the GuC to
4012 * safely preempt all the hw contexts configured for parallel submission
4013 * between each BB. The contract between the i915 and GuC is if the parent
4014 * context can be preempted, all the children can be preempted, and the GuC will
4015 * always try to preempt the parent before the children. A handshake between the
4016 * parent / children breadcrumbs ensures the i915 holds up its end of the deal
4017 * creating a window to preempt between each set of BBs.
4019 static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,
4020 u64 offset, u32 len,
4021 const unsigned int flags);
4022 static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,
4023 u64 offset, u32 len,
4024 const unsigned int flags);
4026 emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
4029 emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
4032 static struct intel_context *
4033 guc_create_parallel(struct intel_engine_cs **engines,
4034 unsigned int num_siblings,
4037 struct intel_engine_cs **siblings = NULL;
4038 struct intel_context *parent = NULL, *ce, *err;
4041 siblings = kmalloc_array(num_siblings,
4045 return ERR_PTR(-ENOMEM);
4047 for (i = 0; i < width; ++i) {
4048 for (j = 0; j < num_siblings; ++j)
4049 siblings[j] = engines[i * num_siblings + j];
4051 ce = intel_engine_create_virtual(siblings, num_siblings,
4060 parent->ops = &virtual_parent_context_ops;
4062 ce->ops = &virtual_child_context_ops;
4063 intel_context_bind_parent_child(parent, ce);
4067 parent->parallel.fence_context = dma_fence_context_alloc(1);
4069 parent->engine->emit_bb_start =
4070 emit_bb_start_parent_no_preempt_mid_batch;
4071 parent->engine->emit_fini_breadcrumb =
4072 emit_fini_breadcrumb_parent_no_preempt_mid_batch;
4073 parent->engine->emit_fini_breadcrumb_dw =
4074 12 + 4 * parent->parallel.number_children;
4075 for_each_child(parent, ce) {
4076 ce->engine->emit_bb_start =
4077 emit_bb_start_child_no_preempt_mid_batch;
4078 ce->engine->emit_fini_breadcrumb =
4079 emit_fini_breadcrumb_child_no_preempt_mid_batch;
4080 ce->engine->emit_fini_breadcrumb_dw = 16;
4088 intel_context_put(parent);
4094 guc_irq_enable_breadcrumbs(struct intel_breadcrumbs *b)
4096 struct intel_engine_cs *sibling;
4097 intel_engine_mask_t tmp, mask = b->engine_mask;
4098 bool result = false;
4100 for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)
4101 result |= intel_engine_irq_enable(sibling);
4107 guc_irq_disable_breadcrumbs(struct intel_breadcrumbs *b)
4109 struct intel_engine_cs *sibling;
4110 intel_engine_mask_t tmp, mask = b->engine_mask;
4112 for_each_engine_masked(sibling, b->irq_engine->gt, mask, tmp)
4113 intel_engine_irq_disable(sibling);
4116 static void guc_init_breadcrumbs(struct intel_engine_cs *engine)
4121 * In GuC submission mode we do not know which physical engine a request
4122 * will be scheduled on, this creates a problem because the breadcrumb
4123 * interrupt is per physical engine. To work around this we attach
4124 * requests and direct all breadcrumb interrupts to the first instance
4125 * of an engine per class. In addition all breadcrumb interrupts are
4126 * enabled / disabled across an engine class in unison.
4128 for (i = 0; i < MAX_ENGINE_INSTANCE; ++i) {
4129 struct intel_engine_cs *sibling =
4130 engine->gt->engine_class[engine->class][i];
4133 if (engine->breadcrumbs != sibling->breadcrumbs) {
4134 intel_breadcrumbs_put(engine->breadcrumbs);
4135 engine->breadcrumbs =
4136 intel_breadcrumbs_get(sibling->breadcrumbs);
4142 if (engine->breadcrumbs) {
4143 engine->breadcrumbs->engine_mask |= engine->mask;
4144 engine->breadcrumbs->irq_enable = guc_irq_enable_breadcrumbs;
4145 engine->breadcrumbs->irq_disable = guc_irq_disable_breadcrumbs;
4149 static void guc_bump_inflight_request_prio(struct i915_request *rq,
4152 struct intel_context *ce = request_to_scheduling_context(rq);
4153 u8 new_guc_prio = map_i915_prio_to_guc_prio(prio);
4155 /* Short circuit function */
4156 if (prio < I915_PRIORITY_NORMAL ||
4157 rq->guc_prio == GUC_PRIO_FINI ||
4158 (rq->guc_prio != GUC_PRIO_INIT &&
4159 !new_guc_prio_higher(rq->guc_prio, new_guc_prio)))
4162 spin_lock(&ce->guc_state.lock);
4163 if (rq->guc_prio != GUC_PRIO_FINI) {
4164 if (rq->guc_prio != GUC_PRIO_INIT)
4165 sub_context_inflight_prio(ce, rq->guc_prio);
4166 rq->guc_prio = new_guc_prio;
4167 add_context_inflight_prio(ce, rq->guc_prio);
4168 update_context_prio(ce);
4170 spin_unlock(&ce->guc_state.lock);
4173 static void guc_retire_inflight_request_prio(struct i915_request *rq)
4175 struct intel_context *ce = request_to_scheduling_context(rq);
4177 spin_lock(&ce->guc_state.lock);
4178 guc_prio_fini(rq, ce);
4179 spin_unlock(&ce->guc_state.lock);
4182 static void sanitize_hwsp(struct intel_engine_cs *engine)
4184 struct intel_timeline *tl;
4186 list_for_each_entry(tl, &engine->status_page.timelines, engine_link)
4187 intel_timeline_reset_seqno(tl);
4190 static void guc_sanitize(struct intel_engine_cs *engine)
4193 * Poison residual state on resume, in case the suspend didn't!
4195 * We have to assume that across suspend/resume (or other loss
4196 * of control) that the contents of our pinned buffers has been
4197 * lost, replaced by garbage. Since this doesn't always happen,
4198 * let's poison such state so that we more quickly spot when
4199 * we falsely assume it has been preserved.
4201 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
4202 memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE);
4205 * The kernel_context HWSP is stored in the status_page. As above,
4206 * that may be lost on resume/initialisation, and so we need to
4207 * reset the value in the HWSP.
4209 sanitize_hwsp(engine);
4211 /* And scrub the dirty cachelines for the HWSP */
4212 drm_clflush_virt_range(engine->status_page.addr, PAGE_SIZE);
4214 intel_engine_reset_pinned_contexts(engine);
4217 static void setup_hwsp(struct intel_engine_cs *engine)
4219 intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */
4221 ENGINE_WRITE_FW(engine,
4223 i915_ggtt_offset(engine->status_page.vma));
4226 static void start_engine(struct intel_engine_cs *engine)
4228 ENGINE_WRITE_FW(engine,
4230 _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
4232 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
4233 ENGINE_POSTING_READ(engine, RING_MI_MODE);
4236 static int guc_resume(struct intel_engine_cs *engine)
4238 assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
4240 intel_mocs_init_engine(engine);
4242 intel_breadcrumbs_reset(engine->breadcrumbs);
4245 start_engine(engine);
4247 if (engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE)
4248 xehp_enable_ccs_engines(engine);
4253 static bool guc_sched_engine_disabled(struct i915_sched_engine *sched_engine)
4255 return !sched_engine->tasklet.callback;
4258 static void guc_set_default_submission(struct intel_engine_cs *engine)
4260 engine->submit_request = guc_submit_request;
4263 static inline int guc_kernel_context_pin(struct intel_guc *guc,
4264 struct intel_context *ce)
4269 * Note: we purposefully do not check the returns below because
4270 * the registration can only fail if a reset is just starting.
4271 * This is called at the end of reset so presumably another reset
4272 * isn't happening and even it did this code would be run again.
4275 if (context_guc_id_invalid(ce)) {
4276 ret = pin_guc_id(guc, ce);
4282 if (!test_bit(CONTEXT_GUC_INIT, &ce->flags))
4283 guc_context_init(ce);
4285 ret = try_context_registration(ce, true);
4287 unpin_guc_id(guc, ce);
4292 static inline int guc_init_submission(struct intel_guc *guc)
4294 struct intel_gt *gt = guc_to_gt(guc);
4295 struct intel_engine_cs *engine;
4296 enum intel_engine_id id;
4298 /* make sure all descriptors are clean... */
4299 xa_destroy(&guc->context_lookup);
4302 * A reset might have occurred while we had a pending stalled request,
4303 * so make sure we clean that up.
4305 guc->stalled_request = NULL;
4306 guc->submission_stall_reason = STALL_NONE;
4309 * Some contexts might have been pinned before we enabled GuC
4310 * submission, so we need to add them to the GuC bookeeping.
4311 * Also, after a reset the of the GuC we want to make sure that the
4312 * information shared with GuC is properly reset. The kernel LRCs are
4313 * not attached to the gem_context, so they need to be added separately.
4315 for_each_engine(engine, gt, id) {
4316 struct intel_context *ce;
4318 list_for_each_entry(ce, &engine->pinned_contexts_list,
4319 pinned_contexts_link) {
4320 int ret = guc_kernel_context_pin(guc, ce);
4323 /* No point in trying to clean up as i915 will wedge on failure */
4332 static void guc_release(struct intel_engine_cs *engine)
4334 engine->sanitize = NULL; /* no longer in control, nothing to sanitize */
4336 intel_engine_cleanup_common(engine);
4337 lrc_fini_wa_ctx(engine);
4340 static void virtual_guc_bump_serial(struct intel_engine_cs *engine)
4342 struct intel_engine_cs *e;
4343 intel_engine_mask_t tmp, mask = engine->mask;
4345 for_each_engine_masked(e, engine->gt, mask, tmp)
4349 static void guc_default_vfuncs(struct intel_engine_cs *engine)
4351 /* Default vfuncs which can be overridden by each engine. */
4353 engine->resume = guc_resume;
4355 engine->cops = &guc_context_ops;
4356 engine->request_alloc = guc_request_alloc;
4357 engine->add_active_request = add_to_context;
4358 engine->remove_active_request = remove_from_context;
4360 engine->sched_engine->schedule = i915_schedule;
4362 engine->reset.prepare = guc_engine_reset_prepare;
4363 engine->reset.rewind = guc_rewind_nop;
4364 engine->reset.cancel = guc_reset_nop;
4365 engine->reset.finish = guc_reset_nop;
4367 engine->emit_flush = gen8_emit_flush_xcs;
4368 engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb;
4369 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_xcs;
4370 if (GRAPHICS_VER(engine->i915) >= 12) {
4371 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_xcs;
4372 engine->emit_flush = gen12_emit_flush_xcs;
4374 engine->set_default_submission = guc_set_default_submission;
4375 engine->busyness = guc_engine_busyness;
4377 engine->flags |= I915_ENGINE_SUPPORTS_STATS;
4378 engine->flags |= I915_ENGINE_HAS_PREEMPTION;
4379 engine->flags |= I915_ENGINE_HAS_TIMESLICES;
4381 /* Wa_14014475959:dg2 */
4382 if (engine->class == COMPUTE_CLASS)
4383 if (IS_GFX_GT_IP_STEP(engine->gt, IP_VER(12, 70), STEP_A0, STEP_B0) ||
4384 IS_DG2(engine->i915))
4385 engine->flags |= I915_ENGINE_USES_WA_HOLD_CCS_SWITCHOUT;
4388 * TODO: GuC supports timeslicing and semaphores as well, but they're
4389 * handled by the firmware so some minor tweaks are required before
4392 * engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
4395 engine->emit_bb_start = gen8_emit_bb_start;
4396 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
4397 engine->emit_bb_start = xehp_emit_bb_start;
4400 static void rcs_submission_override(struct intel_engine_cs *engine)
4402 switch (GRAPHICS_VER(engine->i915)) {
4404 engine->emit_flush = gen12_emit_flush_rcs;
4405 engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb_rcs;
4408 engine->emit_flush = gen11_emit_flush_rcs;
4409 engine->emit_fini_breadcrumb = gen11_emit_fini_breadcrumb_rcs;
4412 engine->emit_flush = gen8_emit_flush_rcs;
4413 engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb_rcs;
4418 static inline void guc_default_irqs(struct intel_engine_cs *engine)
4420 engine->irq_keep_mask = GT_RENDER_USER_INTERRUPT;
4421 intel_engine_set_irq_handler(engine, cs_irq_handler);
4424 static void guc_sched_engine_destroy(struct kref *kref)
4426 struct i915_sched_engine *sched_engine =
4427 container_of(kref, typeof(*sched_engine), ref);
4428 struct intel_guc *guc = sched_engine->private_data;
4430 guc->sched_engine = NULL;
4431 tasklet_kill(&sched_engine->tasklet); /* flush the callback */
4432 kfree(sched_engine);
4435 int intel_guc_submission_setup(struct intel_engine_cs *engine)
4437 struct drm_i915_private *i915 = engine->i915;
4438 struct intel_guc *guc = &engine->gt->uc.guc;
4441 * The setup relies on several assumptions (e.g. irqs always enabled)
4442 * that are only valid on gen11+
4444 GEM_BUG_ON(GRAPHICS_VER(i915) < 11);
4446 if (!guc->sched_engine) {
4447 guc->sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL);
4448 if (!guc->sched_engine)
4451 guc->sched_engine->schedule = i915_schedule;
4452 guc->sched_engine->disabled = guc_sched_engine_disabled;
4453 guc->sched_engine->private_data = guc;
4454 guc->sched_engine->destroy = guc_sched_engine_destroy;
4455 guc->sched_engine->bump_inflight_request_prio =
4456 guc_bump_inflight_request_prio;
4457 guc->sched_engine->retire_inflight_request_prio =
4458 guc_retire_inflight_request_prio;
4459 tasklet_setup(&guc->sched_engine->tasklet,
4460 guc_submission_tasklet);
4462 i915_sched_engine_put(engine->sched_engine);
4463 engine->sched_engine = i915_sched_engine_get(guc->sched_engine);
4465 guc_default_vfuncs(engine);
4466 guc_default_irqs(engine);
4467 guc_init_breadcrumbs(engine);
4469 if (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)
4470 rcs_submission_override(engine);
4472 lrc_init_wa_ctx(engine);
4474 /* Finally, take ownership and responsibility for cleanup! */
4475 engine->sanitize = guc_sanitize;
4476 engine->release = guc_release;
4481 struct scheduling_policy {
4483 u32 max_words, num_words;
4486 struct guc_update_scheduling_policy h2g;
4489 static u32 __guc_scheduling_policy_action_size(struct scheduling_policy *policy)
4491 u32 *start = (void *)&policy->h2g;
4492 u32 *end = policy->h2g.data + policy->num_words;
4493 size_t delta = end - start;
4498 static struct scheduling_policy *__guc_scheduling_policy_start_klv(struct scheduling_policy *policy)
4500 policy->h2g.header.action = INTEL_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV;
4501 policy->max_words = ARRAY_SIZE(policy->h2g.data);
4502 policy->num_words = 0;
4508 static void __guc_scheduling_policy_add_klv(struct scheduling_policy *policy,
4509 u32 action, u32 *data, u32 len)
4511 u32 *klv_ptr = policy->h2g.data + policy->num_words;
4513 GEM_BUG_ON((policy->num_words + 1 + len) > policy->max_words);
4514 *(klv_ptr++) = FIELD_PREP(GUC_KLV_0_KEY, action) |
4515 FIELD_PREP(GUC_KLV_0_LEN, len);
4516 memcpy(klv_ptr, data, sizeof(u32) * len);
4517 policy->num_words += 1 + len;
4521 static int __guc_action_set_scheduling_policies(struct intel_guc *guc,
4522 struct scheduling_policy *policy)
4526 ret = intel_guc_send(guc, (u32 *)&policy->h2g,
4527 __guc_scheduling_policy_action_size(policy));
4529 guc_probe_error(guc, "Failed to configure global scheduling policies: %pe!\n",
4534 if (ret != policy->count) {
4535 guc_warn(guc, "global scheduler policy processed %d of %d KLVs!",
4536 ret, policy->count);
4537 if (ret > policy->count)
4544 static int guc_init_global_schedule_policy(struct intel_guc *guc)
4546 struct scheduling_policy policy;
4547 struct intel_gt *gt = guc_to_gt(guc);
4548 intel_wakeref_t wakeref;
4551 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0))
4554 __guc_scheduling_policy_start_klv(&policy);
4556 with_intel_runtime_pm(>->i915->runtime_pm, wakeref) {
4558 GLOBAL_SCHEDULE_POLICY_RC_YIELD_DURATION,
4559 GLOBAL_SCHEDULE_POLICY_RC_YIELD_RATIO,
4562 __guc_scheduling_policy_add_klv(&policy,
4563 GUC_SCHEDULING_POLICIES_KLV_ID_RENDER_COMPUTE_YIELD,
4564 yield, ARRAY_SIZE(yield));
4566 ret = __guc_action_set_scheduling_policies(guc, &policy);
4572 static void guc_route_semaphores(struct intel_guc *guc, bool to_guc)
4574 struct intel_gt *gt = guc_to_gt(guc);
4577 if (GRAPHICS_VER(gt->i915) < 12)
4581 val = GUC_SEM_INTR_ROUTE_TO_GUC | GUC_SEM_INTR_ENABLE_ALL;
4585 intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES, val);
4588 int intel_guc_submission_enable(struct intel_guc *guc)
4592 /* Semaphore interrupt enable and route to GuC */
4593 guc_route_semaphores(guc, true);
4595 ret = guc_init_submission(guc);
4599 ret = guc_init_engine_stats(guc);
4603 ret = guc_init_global_schedule_policy(guc);
4610 guc_fini_engine_stats(guc);
4612 guc_route_semaphores(guc, false);
4616 /* Note: By the time we're here, GuC may have already been reset */
4617 void intel_guc_submission_disable(struct intel_guc *guc)
4619 guc_cancel_busyness_worker(guc);
4621 /* Semaphore interrupt disable and route to host */
4622 guc_route_semaphores(guc, false);
4625 static bool __guc_submission_supported(struct intel_guc *guc)
4627 /* GuC submission is unavailable for pre-Gen11 */
4628 return intel_guc_is_supported(guc) &&
4629 GRAPHICS_VER(guc_to_i915(guc)) >= 11;
4632 static bool __guc_submission_selected(struct intel_guc *guc)
4634 struct drm_i915_private *i915 = guc_to_i915(guc);
4636 if (!intel_guc_submission_is_supported(guc))
4639 return i915->params.enable_guc & ENABLE_GUC_SUBMISSION;
4642 int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc)
4644 return guc->submission_state.num_guc_ids - NUMBER_MULTI_LRC_GUC_ID(guc);
4648 * This default value of 33 milisecs (+1 milisec round up) ensures 30fps or higher
4649 * workloads are able to enjoy the latency reduction when delaying the schedule-disable
4650 * operation. This matches the 30fps game-render + encode (real world) workload this
4651 * knob was tested against.
4653 #define SCHED_DISABLE_DELAY_MS 34
4656 * A threshold of 75% is a reasonable starting point considering that real world apps
4657 * generally don't get anywhere near this.
4659 #define NUM_SCHED_DISABLE_GUCIDS_DEFAULT_THRESHOLD(__guc) \
4660 (((intel_guc_sched_disable_gucid_threshold_max(guc)) * 3) / 4)
4662 void intel_guc_submission_init_early(struct intel_guc *guc)
4664 xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ);
4666 spin_lock_init(&guc->submission_state.lock);
4667 INIT_LIST_HEAD(&guc->submission_state.guc_id_list);
4668 ida_init(&guc->submission_state.guc_ids);
4669 INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts);
4670 INIT_WORK(&guc->submission_state.destroyed_worker,
4671 destroyed_worker_func);
4672 INIT_WORK(&guc->submission_state.reset_fail_worker,
4673 reset_fail_worker_func);
4675 spin_lock_init(&guc->timestamp.lock);
4676 INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping);
4678 guc->submission_state.sched_disable_delay_ms = SCHED_DISABLE_DELAY_MS;
4679 guc->submission_state.num_guc_ids = GUC_MAX_CONTEXT_ID;
4680 guc->submission_state.sched_disable_gucid_threshold =
4681 NUM_SCHED_DISABLE_GUCIDS_DEFAULT_THRESHOLD(guc);
4682 guc->submission_supported = __guc_submission_supported(guc);
4683 guc->submission_selected = __guc_submission_selected(guc);
4686 static inline struct intel_context *
4687 g2h_context_lookup(struct intel_guc *guc, u32 ctx_id)
4689 struct intel_context *ce;
4691 if (unlikely(ctx_id >= GUC_MAX_CONTEXT_ID)) {
4692 guc_err(guc, "Invalid ctx_id %u\n", ctx_id);
4696 ce = __get_context(guc, ctx_id);
4697 if (unlikely(!ce)) {
4698 guc_err(guc, "Context is NULL, ctx_id %u\n", ctx_id);
4702 if (unlikely(intel_context_is_child(ce))) {
4703 guc_err(guc, "Context is child, ctx_id %u\n", ctx_id);
4710 static void wait_wake_outstanding_tlb_g2h(struct intel_guc *guc, u32 seqno)
4712 struct intel_guc_tlb_wait *wait;
4713 unsigned long flags;
4715 xa_lock_irqsave(&guc->tlb_lookup, flags);
4716 wait = xa_load(&guc->tlb_lookup, seqno);
4722 "Stale TLB invalidation response with seqno %d\n", seqno);
4724 xa_unlock_irqrestore(&guc->tlb_lookup, flags);
4727 int intel_guc_tlb_invalidation_done(struct intel_guc *guc,
4728 const u32 *payload, u32 len)
4733 wait_wake_outstanding_tlb_g2h(guc, payload[0]);
4737 static long must_wait_woken(struct wait_queue_entry *wq_entry, long timeout)
4740 * This is equivalent to wait_woken() with the exception that
4741 * we do not wake up early if the kthread task has been completed.
4742 * As we are called from page reclaim in any task context,
4743 * we may be invoked from stopped kthreads, but we *must*
4744 * complete the wait from the HW.
4747 set_current_state(TASK_UNINTERRUPTIBLE);
4748 if (wq_entry->flags & WQ_FLAG_WOKEN)
4751 timeout = schedule_timeout(timeout);
4754 /* See wait_woken() and woken_wake_function() */
4755 __set_current_state(TASK_RUNNING);
4756 smp_store_mb(wq_entry->flags, wq_entry->flags & ~WQ_FLAG_WOKEN);
4761 static bool intel_gt_is_enabled(const struct intel_gt *gt)
4763 /* Check if GT is wedged or suspended */
4764 if (intel_gt_is_wedged(gt) || !intel_irqs_enabled(gt->i915))
4769 static int guc_send_invalidate_tlb(struct intel_guc *guc,
4770 enum intel_guc_tlb_invalidation_type type)
4772 struct intel_guc_tlb_wait _wq, *wq = &_wq;
4773 struct intel_gt *gt = guc_to_gt(guc);
4774 DEFINE_WAIT_FUNC(wait, woken_wake_function);
4778 INTEL_GUC_ACTION_TLB_INVALIDATION,
4780 REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_TYPE_MASK, type) |
4781 REG_FIELD_PREP(INTEL_GUC_TLB_INVAL_MODE_MASK,
4782 INTEL_GUC_TLB_INVAL_MODE_HEAVY) |
4783 INTEL_GUC_TLB_INVAL_FLUSH_CACHE,
4785 u32 size = ARRAY_SIZE(action);
4788 * Early guard against GT enablement. TLB invalidation should not be
4789 * attempted if the GT is disabled due to suspend/wedge.
4791 if (!intel_gt_is_enabled(gt))
4794 init_waitqueue_head(&_wq.wq);
4796 if (xa_alloc_cyclic_irq(&guc->tlb_lookup, &seqno, wq,
4797 xa_limit_32b, &guc->next_seqno,
4798 GFP_ATOMIC | __GFP_NOWARN) < 0) {
4799 /* Under severe memory pressure? Serialise TLB allocations */
4800 xa_lock_irq(&guc->tlb_lookup);
4801 wq = xa_load(&guc->tlb_lookup, guc->serial_slot);
4802 wait_event_lock_irq(wq->wq,
4803 !READ_ONCE(wq->busy),
4804 guc->tlb_lookup.xa_lock);
4806 * Update wq->busy under lock to ensure only one waiter can
4807 * issue the TLB invalidation command using the serial slot at a
4808 * time. The condition is set to true before releasing the lock
4809 * so that other caller continue to wait until woken up again.
4812 xa_unlock_irq(&guc->tlb_lookup);
4814 seqno = guc->serial_slot;
4819 add_wait_queue(&wq->wq, &wait);
4821 /* This is a critical reclaim path and thus we must loop here. */
4822 err = intel_guc_send_busy_loop(guc, action, size, G2H_LEN_DW_INVALIDATE_TLB, true);
4827 * Late guard against GT enablement. It is not an error for the TLB
4828 * invalidation to time out if the GT is disabled during the process
4829 * due to suspend/wedge. In fact, the TLB invalidation is cancelled
4832 if (!must_wait_woken(&wait, intel_guc_ct_max_queue_time_jiffies()) &&
4833 intel_gt_is_enabled(gt)) {
4835 "TLB invalidation response timed out for seqno %u\n", seqno);
4839 remove_wait_queue(&wq->wq, &wait);
4840 if (seqno != guc->serial_slot)
4841 xa_erase_irq(&guc->tlb_lookup, seqno);
4846 /* Send a H2G command to invalidate the TLBs at engine level and beyond. */
4847 int intel_guc_invalidate_tlb_engines(struct intel_guc *guc)
4849 return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_ENGINES);
4852 /* Send a H2G command to invalidate the GuC's internal TLB. */
4853 int intel_guc_invalidate_tlb_guc(struct intel_guc *guc)
4855 return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_GUC);
4858 int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
4862 struct intel_context *ce;
4865 if (unlikely(len < 1)) {
4866 guc_err(guc, "Invalid length %u\n", len);
4871 ce = g2h_context_lookup(guc, ctx_id);
4875 trace_intel_context_deregister_done(ce);
4877 #ifdef CONFIG_DRM_I915_SELFTEST
4878 if (unlikely(ce->drop_deregister)) {
4879 ce->drop_deregister = false;
4884 if (context_wait_for_deregister_to_register(ce)) {
4885 struct intel_runtime_pm *runtime_pm =
4886 &ce->engine->gt->i915->runtime_pm;
4887 intel_wakeref_t wakeref;
4890 * Previous owner of this guc_id has been deregistered, now safe
4891 * register this context.
4893 with_intel_runtime_pm(runtime_pm, wakeref)
4894 register_context(ce, true);
4895 guc_signal_context_fence(ce);
4896 intel_context_put(ce);
4897 } else if (context_destroyed(ce)) {
4898 /* Context has been destroyed */
4899 intel_gt_pm_put_async_untracked(guc_to_gt(guc));
4900 release_guc_id(guc, ce);
4901 __guc_context_destroy(ce);
4904 decr_outstanding_submission_g2h(guc);
4909 int intel_guc_sched_done_process_msg(struct intel_guc *guc,
4913 struct intel_context *ce;
4914 unsigned long flags;
4917 if (unlikely(len < 2)) {
4918 guc_err(guc, "Invalid length %u\n", len);
4923 ce = g2h_context_lookup(guc, ctx_id);
4927 if (unlikely(context_destroyed(ce) ||
4928 (!context_pending_enable(ce) &&
4929 !context_pending_disable(ce)))) {
4930 guc_err(guc, "Bad context sched_state 0x%x, ctx_id %u\n",
4931 ce->guc_state.sched_state, ctx_id);
4935 trace_intel_context_sched_done(ce);
4937 if (context_pending_enable(ce)) {
4938 #ifdef CONFIG_DRM_I915_SELFTEST
4939 if (unlikely(ce->drop_schedule_enable)) {
4940 ce->drop_schedule_enable = false;
4945 spin_lock_irqsave(&ce->guc_state.lock, flags);
4946 clr_context_pending_enable(ce);
4947 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
4948 } else if (context_pending_disable(ce)) {
4951 #ifdef CONFIG_DRM_I915_SELFTEST
4952 if (unlikely(ce->drop_schedule_disable)) {
4953 ce->drop_schedule_disable = false;
4959 * Unpin must be done before __guc_signal_context_fence,
4960 * otherwise a race exists between the requests getting
4961 * submitted + retired before this unpin completes resulting in
4962 * the pin_count going to zero and the context still being
4965 intel_context_sched_disable_unpin(ce);
4967 spin_lock_irqsave(&ce->guc_state.lock, flags);
4968 banned = context_banned(ce);
4969 clr_context_banned(ce);
4970 clr_context_pending_disable(ce);
4971 __guc_signal_context_fence(ce);
4972 guc_blocked_fence_complete(ce);
4973 spin_unlock_irqrestore(&ce->guc_state.lock, flags);
4976 guc_cancel_context_requests(ce);
4977 intel_engine_signal_breadcrumbs(ce->engine);
4981 decr_outstanding_submission_g2h(guc);
4982 intel_context_put(ce);
4987 static void capture_error_state(struct intel_guc *guc,
4988 struct intel_context *ce)
4990 struct intel_gt *gt = guc_to_gt(guc);
4991 struct drm_i915_private *i915 = gt->i915;
4992 intel_wakeref_t wakeref;
4993 intel_engine_mask_t engine_mask;
4995 if (intel_engine_is_virtual(ce->engine)) {
4996 struct intel_engine_cs *e;
4997 intel_engine_mask_t tmp, virtual_mask = ce->engine->mask;
5000 for_each_engine_masked(e, ce->engine->gt, virtual_mask, tmp) {
5001 bool match = intel_guc_capture_is_matching_engine(gt, ce, e);
5004 intel_engine_set_hung_context(e, ce);
5005 engine_mask |= e->mask;
5006 i915_increase_reset_engine_count(&i915->gpu_error,
5012 guc_warn(guc, "No matching physical engine capture for virtual engine context 0x%04X / %s",
5013 ce->guc_id.id, ce->engine->name);
5017 intel_engine_set_hung_context(ce->engine, ce);
5018 engine_mask = ce->engine->mask;
5019 i915_increase_reset_engine_count(&i915->gpu_error, ce->engine);
5022 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
5023 i915_capture_error_state(gt, engine_mask, CORE_DUMP_FLAG_IS_GUC_CAPTURE);
5026 static void guc_context_replay(struct intel_context *ce)
5028 struct i915_sched_engine *sched_engine = ce->engine->sched_engine;
5030 __guc_reset_context(ce, ce->engine->mask);
5031 tasklet_hi_schedule(&sched_engine->tasklet);
5034 static void guc_handle_context_reset(struct intel_guc *guc,
5035 struct intel_context *ce)
5037 bool capture = intel_context_is_schedulable(ce);
5039 trace_intel_context_reset(ce);
5041 guc_dbg(guc, "%s context reset notification: 0x%04X on %s, exiting = %s, banned = %s\n",
5042 capture ? "Got" : "Ignoring",
5043 ce->guc_id.id, ce->engine->name,
5044 str_yes_no(intel_context_is_exiting(ce)),
5045 str_yes_no(intel_context_is_banned(ce)));
5048 capture_error_state(guc, ce);
5049 guc_context_replay(ce);
5053 int intel_guc_context_reset_process_msg(struct intel_guc *guc,
5054 const u32 *msg, u32 len)
5056 struct intel_context *ce;
5057 unsigned long flags;
5060 if (unlikely(len != 1)) {
5061 guc_err(guc, "Invalid length %u", len);
5068 * The context lookup uses the xarray but lookups only require an RCU lock
5069 * not the full spinlock. So take the lock explicitly and keep it until the
5070 * context has been reference count locked to ensure it can't be destroyed
5071 * asynchronously until the reset is done.
5073 xa_lock_irqsave(&guc->context_lookup, flags);
5074 ce = g2h_context_lookup(guc, ctx_id);
5076 intel_context_get(ce);
5077 xa_unlock_irqrestore(&guc->context_lookup, flags);
5082 guc_handle_context_reset(guc, ce);
5083 intel_context_put(ce);
5088 int intel_guc_error_capture_process_msg(struct intel_guc *guc,
5089 const u32 *msg, u32 len)
5093 if (unlikely(len != 1)) {
5094 guc_dbg(guc, "Invalid length %u", len);
5098 status = msg[0] & INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_MASK;
5099 if (status == INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE)
5100 guc_warn(guc, "No space for error capture");
5102 intel_guc_capture_process(guc);
5107 struct intel_engine_cs *
5108 intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance)
5110 struct intel_gt *gt = guc_to_gt(guc);
5111 u8 engine_class = guc_class_to_engine_class(guc_class);
5113 /* Class index is checked in class converter */
5114 GEM_BUG_ON(instance > MAX_ENGINE_INSTANCE);
5116 return gt->engine_class[engine_class][instance];
5119 static void reset_fail_worker_func(struct work_struct *w)
5121 struct intel_guc *guc = container_of(w, struct intel_guc,
5122 submission_state.reset_fail_worker);
5123 struct intel_gt *gt = guc_to_gt(guc);
5124 intel_engine_mask_t reset_fail_mask;
5125 unsigned long flags;
5127 spin_lock_irqsave(&guc->submission_state.lock, flags);
5128 reset_fail_mask = guc->submission_state.reset_fail_mask;
5129 guc->submission_state.reset_fail_mask = 0;
5130 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
5132 if (likely(reset_fail_mask)) {
5133 struct intel_engine_cs *engine;
5134 enum intel_engine_id id;
5137 * GuC is toast at this point - it dead loops after sending the failed
5138 * reset notification. So need to manually determine the guilty context.
5139 * Note that it should be reliable to do this here because the GuC is
5140 * toast and will not be scheduling behind the KMD's back.
5142 for_each_engine_masked(engine, gt, reset_fail_mask, id)
5143 intel_guc_find_hung_context(engine);
5145 intel_gt_handle_error(gt, reset_fail_mask,
5147 "GuC failed to reset engine mask=0x%x",
5152 int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
5153 const u32 *msg, u32 len)
5155 struct intel_engine_cs *engine;
5156 u8 guc_class, instance;
5158 unsigned long flags;
5160 if (unlikely(len != 3)) {
5161 guc_err(guc, "Invalid length %u", len);
5169 engine = intel_guc_lookup_engine(guc, guc_class, instance);
5170 if (unlikely(!engine)) {
5171 guc_err(guc, "Invalid engine %d:%d", guc_class, instance);
5176 * This is an unexpected failure of a hardware feature. So, log a real
5177 * error message not just the informational that comes with the reset.
5179 guc_err(guc, "Engine reset failed on %d:%d (%s) because 0x%08X",
5180 guc_class, instance, engine->name, reason);
5182 spin_lock_irqsave(&guc->submission_state.lock, flags);
5183 guc->submission_state.reset_fail_mask |= engine->mask;
5184 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
5187 * A GT reset flushes this worker queue (G2H handler) so we must use
5188 * another worker to trigger a GT reset.
5190 queue_work(system_unbound_wq, &guc->submission_state.reset_fail_worker);
5195 void intel_guc_find_hung_context(struct intel_engine_cs *engine)
5197 struct intel_guc *guc = &engine->gt->uc.guc;
5198 struct intel_context *ce;
5199 struct i915_request *rq;
5200 unsigned long index;
5201 unsigned long flags;
5203 /* Reset called during driver load? GuC not yet initialised! */
5204 if (unlikely(!guc_submission_initialized(guc)))
5207 xa_lock_irqsave(&guc->context_lookup, flags);
5208 xa_for_each(&guc->context_lookup, index, ce) {
5211 if (!kref_get_unless_zero(&ce->ref))
5214 xa_unlock(&guc->context_lookup);
5216 if (!intel_context_is_pinned(ce))
5219 if (intel_engine_is_virtual(ce->engine)) {
5220 if (!(ce->engine->mask & engine->mask))
5223 if (ce->engine != engine)
5228 spin_lock(&ce->guc_state.lock);
5229 list_for_each_entry(rq, &ce->guc_state.requests, sched.link) {
5230 if (i915_test_request_state(rq) != I915_REQUEST_ACTIVE)
5236 spin_unlock(&ce->guc_state.lock);
5239 intel_engine_set_hung_context(engine, ce);
5241 /* Can only cope with one hang at a time... */
5242 intel_context_put(ce);
5243 xa_lock(&guc->context_lookup);
5248 intel_context_put(ce);
5249 xa_lock(&guc->context_lookup);
5252 xa_unlock_irqrestore(&guc->context_lookup, flags);
5255 void intel_guc_dump_active_requests(struct intel_engine_cs *engine,
5256 struct i915_request *hung_rq,
5257 struct drm_printer *m)
5259 struct intel_guc *guc = &engine->gt->uc.guc;
5260 struct intel_context *ce;
5261 unsigned long index;
5262 unsigned long flags;
5264 /* Reset called during driver load? GuC not yet initialised! */
5265 if (unlikely(!guc_submission_initialized(guc)))
5268 xa_lock_irqsave(&guc->context_lookup, flags);
5269 xa_for_each(&guc->context_lookup, index, ce) {
5270 if (!kref_get_unless_zero(&ce->ref))
5273 xa_unlock(&guc->context_lookup);
5275 if (!intel_context_is_pinned(ce))
5278 if (intel_engine_is_virtual(ce->engine)) {
5279 if (!(ce->engine->mask & engine->mask))
5282 if (ce->engine != engine)
5286 spin_lock(&ce->guc_state.lock);
5287 intel_engine_dump_active_requests(&ce->guc_state.requests,
5289 spin_unlock(&ce->guc_state.lock);
5292 intel_context_put(ce);
5293 xa_lock(&guc->context_lookup);
5295 xa_unlock_irqrestore(&guc->context_lookup, flags);
5298 void intel_guc_submission_print_info(struct intel_guc *guc,
5299 struct drm_printer *p)
5301 struct i915_sched_engine *sched_engine = guc->sched_engine;
5303 unsigned long flags;
5308 drm_printf(p, "GuC Submission API Version: %d.%d.%d\n",
5309 guc->submission_version.major, guc->submission_version.minor,
5310 guc->submission_version.patch);
5311 drm_printf(p, "GuC Number Outstanding Submission G2H: %u\n",
5312 atomic_read(&guc->outstanding_submission_g2h));
5313 drm_printf(p, "GuC tasklet count: %u\n",
5314 atomic_read(&sched_engine->tasklet.count));
5316 spin_lock_irqsave(&sched_engine->lock, flags);
5317 drm_printf(p, "Requests in GuC submit tasklet:\n");
5318 for (rb = rb_first_cached(&sched_engine->queue); rb; rb = rb_next(rb)) {
5319 struct i915_priolist *pl = to_priolist(rb);
5320 struct i915_request *rq;
5322 priolist_for_each_request(rq, pl)
5323 drm_printf(p, "guc_id=%u, seqno=%llu\n",
5324 rq->context->guc_id.id,
5327 spin_unlock_irqrestore(&sched_engine->lock, flags);
5328 drm_printf(p, "\n");
5331 static inline void guc_log_context_priority(struct drm_printer *p,
5332 struct intel_context *ce)
5336 drm_printf(p, "\t\tPriority: %d\n", ce->guc_state.prio);
5337 drm_printf(p, "\t\tNumber Requests (lower index == higher priority)\n");
5338 for (i = GUC_CLIENT_PRIORITY_KMD_HIGH;
5339 i < GUC_CLIENT_PRIORITY_NUM; ++i) {
5340 drm_printf(p, "\t\tNumber requests in priority band[%d]: %d\n",
5341 i, ce->guc_state.prio_count[i]);
5343 drm_printf(p, "\n");
5346 static inline void guc_log_context(struct drm_printer *p,
5347 struct intel_context *ce)
5349 drm_printf(p, "GuC lrc descriptor %u:\n", ce->guc_id.id);
5350 drm_printf(p, "\tHW Context Desc: 0x%08x\n", ce->lrc.lrca);
5351 drm_printf(p, "\t\tLRC Head: Internal %u, Memory %u\n",
5353 ce->lrc_reg_state[CTX_RING_HEAD]);
5354 drm_printf(p, "\t\tLRC Tail: Internal %u, Memory %u\n",
5356 ce->lrc_reg_state[CTX_RING_TAIL]);
5357 drm_printf(p, "\t\tContext Pin Count: %u\n",
5358 atomic_read(&ce->pin_count));
5359 drm_printf(p, "\t\tGuC ID Ref Count: %u\n",
5360 atomic_read(&ce->guc_id.ref));
5361 drm_printf(p, "\t\tSchedule State: 0x%x\n",
5362 ce->guc_state.sched_state);
5365 void intel_guc_submission_print_context_info(struct intel_guc *guc,
5366 struct drm_printer *p)
5368 struct intel_context *ce;
5369 unsigned long index;
5370 unsigned long flags;
5372 xa_lock_irqsave(&guc->context_lookup, flags);
5373 xa_for_each(&guc->context_lookup, index, ce) {
5374 GEM_BUG_ON(intel_context_is_child(ce));
5376 guc_log_context(p, ce);
5377 guc_log_context_priority(p, ce);
5379 if (intel_context_is_parent(ce)) {
5380 struct intel_context *child;
5382 drm_printf(p, "\t\tNumber children: %u\n",
5383 ce->parallel.number_children);
5385 if (ce->parallel.guc.wq_status) {
5386 drm_printf(p, "\t\tWQI Head: %u\n",
5387 READ_ONCE(*ce->parallel.guc.wq_head));
5388 drm_printf(p, "\t\tWQI Tail: %u\n",
5389 READ_ONCE(*ce->parallel.guc.wq_tail));
5390 drm_printf(p, "\t\tWQI Status: %u\n",
5391 READ_ONCE(*ce->parallel.guc.wq_status));
5394 if (ce->engine->emit_bb_start ==
5395 emit_bb_start_parent_no_preempt_mid_batch) {
5398 drm_printf(p, "\t\tChildren Go: %u\n",
5399 get_children_go_value(ce));
5400 for (i = 0; i < ce->parallel.number_children; ++i)
5401 drm_printf(p, "\t\tChildren Join: %u\n",
5402 get_children_join_value(ce, i));
5405 for_each_child(ce, child)
5406 guc_log_context(p, child);
5409 xa_unlock_irqrestore(&guc->context_lookup, flags);
5412 static inline u32 get_children_go_addr(struct intel_context *ce)
5414 GEM_BUG_ON(!intel_context_is_parent(ce));
5416 return i915_ggtt_offset(ce->state) +
5417 __get_parent_scratch_offset(ce) +
5418 offsetof(struct parent_scratch, go.semaphore);
5421 static inline u32 get_children_join_addr(struct intel_context *ce,
5424 GEM_BUG_ON(!intel_context_is_parent(ce));
5426 return i915_ggtt_offset(ce->state) +
5427 __get_parent_scratch_offset(ce) +
5428 offsetof(struct parent_scratch, join[child_index].semaphore);
5431 #define PARENT_GO_BB 1
5432 #define PARENT_GO_FINI_BREADCRUMB 0
5433 #define CHILD_GO_BB 1
5434 #define CHILD_GO_FINI_BREADCRUMB 0
5435 static int emit_bb_start_parent_no_preempt_mid_batch(struct i915_request *rq,
5436 u64 offset, u32 len,
5437 const unsigned int flags)
5439 struct intel_context *ce = rq->context;
5443 GEM_BUG_ON(!intel_context_is_parent(ce));
5445 cs = intel_ring_begin(rq, 10 + 4 * ce->parallel.number_children);
5449 /* Wait on children */
5450 for (i = 0; i < ce->parallel.number_children; ++i) {
5451 *cs++ = (MI_SEMAPHORE_WAIT |
5452 MI_SEMAPHORE_GLOBAL_GTT |
5454 MI_SEMAPHORE_SAD_EQ_SDD);
5455 *cs++ = PARENT_GO_BB;
5456 *cs++ = get_children_join_addr(ce, i);
5460 /* Turn off preemption */
5461 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5464 /* Tell children go */
5465 cs = gen8_emit_ggtt_write(cs,
5467 get_children_go_addr(ce),
5471 *cs++ = MI_BATCH_BUFFER_START_GEN8 |
5472 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
5473 *cs++ = lower_32_bits(offset);
5474 *cs++ = upper_32_bits(offset);
5477 intel_ring_advance(rq, cs);
5482 static int emit_bb_start_child_no_preempt_mid_batch(struct i915_request *rq,
5483 u64 offset, u32 len,
5484 const unsigned int flags)
5486 struct intel_context *ce = rq->context;
5487 struct intel_context *parent = intel_context_to_parent(ce);
5490 GEM_BUG_ON(!intel_context_is_child(ce));
5492 cs = intel_ring_begin(rq, 12);
5497 cs = gen8_emit_ggtt_write(cs,
5499 get_children_join_addr(parent,
5500 ce->parallel.child_index),
5503 /* Wait on parent for go */
5504 *cs++ = (MI_SEMAPHORE_WAIT |
5505 MI_SEMAPHORE_GLOBAL_GTT |
5507 MI_SEMAPHORE_SAD_EQ_SDD);
5508 *cs++ = CHILD_GO_BB;
5509 *cs++ = get_children_go_addr(parent);
5512 /* Turn off preemption */
5513 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5516 *cs++ = MI_BATCH_BUFFER_START_GEN8 |
5517 (flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
5518 *cs++ = lower_32_bits(offset);
5519 *cs++ = upper_32_bits(offset);
5521 intel_ring_advance(rq, cs);
5527 __emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
5530 struct intel_context *ce = rq->context;
5533 GEM_BUG_ON(!intel_context_is_parent(ce));
5535 /* Wait on children */
5536 for (i = 0; i < ce->parallel.number_children; ++i) {
5537 *cs++ = (MI_SEMAPHORE_WAIT |
5538 MI_SEMAPHORE_GLOBAL_GTT |
5540 MI_SEMAPHORE_SAD_EQ_SDD);
5541 *cs++ = PARENT_GO_FINI_BREADCRUMB;
5542 *cs++ = get_children_join_addr(ce, i);
5546 /* Turn on preemption */
5547 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
5550 /* Tell children go */
5551 cs = gen8_emit_ggtt_write(cs,
5552 CHILD_GO_FINI_BREADCRUMB,
5553 get_children_go_addr(ce),
5560 * If this true, a submission of multi-lrc requests had an error and the
5561 * requests need to be skipped. The front end (execuf IOCTL) should've called
5562 * i915_request_skip which squashes the BB but we still need to emit the fini
5563 * breadrcrumbs seqno write. At this point we don't know how many of the
5564 * requests in the multi-lrc submission were generated so we can't do the
5565 * handshake between the parent and children (e.g. if 4 requests should be
5566 * generated but 2nd hit an error only 1 would be seen by the GuC backend).
5567 * Simply skip the handshake, but still emit the breadcrumbd seqno, if an error
5568 * has occurred on any of the requests in submission / relationship.
5570 static inline bool skip_handshake(struct i915_request *rq)
5572 return test_bit(I915_FENCE_FLAG_SKIP_PARALLEL, &rq->fence.flags);
5575 #define NON_SKIP_LEN 6
5577 emit_fini_breadcrumb_parent_no_preempt_mid_batch(struct i915_request *rq,
5580 struct intel_context *ce = rq->context;
5581 __maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs;
5582 __maybe_unused u32 *start_fini_breadcrumb_cs = cs;
5584 GEM_BUG_ON(!intel_context_is_parent(ce));
5586 if (unlikely(skip_handshake(rq))) {
5588 * NOP everything in __emit_fini_breadcrumb_parent_no_preempt_mid_batch,
5589 * the NON_SKIP_LEN comes from the length of the emits below.
5591 memset(cs, 0, sizeof(u32) *
5592 (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN));
5593 cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;
5595 cs = __emit_fini_breadcrumb_parent_no_preempt_mid_batch(rq, cs);
5598 /* Emit fini breadcrumb */
5599 before_fini_breadcrumb_user_interrupt_cs = cs;
5600 cs = gen8_emit_ggtt_write(cs,
5602 i915_request_active_timeline(rq)->hwsp_offset,
5605 /* User interrupt */
5606 *cs++ = MI_USER_INTERRUPT;
5609 /* Ensure our math for skip + emit is correct */
5610 GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=
5612 GEM_BUG_ON(start_fini_breadcrumb_cs +
5613 ce->engine->emit_fini_breadcrumb_dw != cs);
5615 rq->tail = intel_ring_offset(rq, cs);
5621 __emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
5624 struct intel_context *ce = rq->context;
5625 struct intel_context *parent = intel_context_to_parent(ce);
5627 GEM_BUG_ON(!intel_context_is_child(ce));
5629 /* Turn on preemption */
5630 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
5634 cs = gen8_emit_ggtt_write(cs,
5635 PARENT_GO_FINI_BREADCRUMB,
5636 get_children_join_addr(parent,
5637 ce->parallel.child_index),
5640 /* Wait parent on for go */
5641 *cs++ = (MI_SEMAPHORE_WAIT |
5642 MI_SEMAPHORE_GLOBAL_GTT |
5644 MI_SEMAPHORE_SAD_EQ_SDD);
5645 *cs++ = CHILD_GO_FINI_BREADCRUMB;
5646 *cs++ = get_children_go_addr(parent);
5653 emit_fini_breadcrumb_child_no_preempt_mid_batch(struct i915_request *rq,
5656 struct intel_context *ce = rq->context;
5657 __maybe_unused u32 *before_fini_breadcrumb_user_interrupt_cs;
5658 __maybe_unused u32 *start_fini_breadcrumb_cs = cs;
5660 GEM_BUG_ON(!intel_context_is_child(ce));
5662 if (unlikely(skip_handshake(rq))) {
5664 * NOP everything in __emit_fini_breadcrumb_child_no_preempt_mid_batch,
5665 * the NON_SKIP_LEN comes from the length of the emits below.
5667 memset(cs, 0, sizeof(u32) *
5668 (ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN));
5669 cs += ce->engine->emit_fini_breadcrumb_dw - NON_SKIP_LEN;
5671 cs = __emit_fini_breadcrumb_child_no_preempt_mid_batch(rq, cs);
5674 /* Emit fini breadcrumb */
5675 before_fini_breadcrumb_user_interrupt_cs = cs;
5676 cs = gen8_emit_ggtt_write(cs,
5678 i915_request_active_timeline(rq)->hwsp_offset,
5681 /* User interrupt */
5682 *cs++ = MI_USER_INTERRUPT;
5685 /* Ensure our math for skip + emit is correct */
5686 GEM_BUG_ON(before_fini_breadcrumb_user_interrupt_cs + NON_SKIP_LEN !=
5688 GEM_BUG_ON(start_fini_breadcrumb_cs +
5689 ce->engine->emit_fini_breadcrumb_dw != cs);
5691 rq->tail = intel_ring_offset(rq, cs);
5698 static struct intel_context *
5699 guc_create_virtual(struct intel_engine_cs **siblings, unsigned int count,
5700 unsigned long flags)
5702 struct guc_virtual_engine *ve;
5703 struct intel_guc *guc;
5707 ve = kzalloc(sizeof(*ve), GFP_KERNEL);
5709 return ERR_PTR(-ENOMEM);
5711 guc = &siblings[0]->gt->uc.guc;
5713 ve->base.i915 = siblings[0]->i915;
5714 ve->base.gt = siblings[0]->gt;
5715 ve->base.uncore = siblings[0]->uncore;
5718 ve->base.uabi_class = I915_ENGINE_CLASS_INVALID;
5719 ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
5720 ve->base.uabi_instance = I915_ENGINE_CLASS_INVALID_VIRTUAL;
5721 ve->base.saturated = ALL_ENGINES;
5723 snprintf(ve->base.name, sizeof(ve->base.name), "virtual");
5725 ve->base.sched_engine = i915_sched_engine_get(guc->sched_engine);
5727 ve->base.cops = &virtual_guc_context_ops;
5728 ve->base.request_alloc = guc_request_alloc;
5729 ve->base.bump_serial = virtual_guc_bump_serial;
5731 ve->base.submit_request = guc_submit_request;
5733 ve->base.flags = I915_ENGINE_IS_VIRTUAL;
5735 BUILD_BUG_ON(ilog2(VIRTUAL_ENGINES) < I915_NUM_ENGINES);
5736 ve->base.mask = VIRTUAL_ENGINES;
5738 intel_context_init(&ve->context, &ve->base);
5740 for (n = 0; n < count; n++) {
5741 struct intel_engine_cs *sibling = siblings[n];
5743 GEM_BUG_ON(!is_power_of_2(sibling->mask));
5744 if (sibling->mask & ve->base.mask) {
5745 guc_dbg(guc, "duplicate %s entry in load balancer\n",
5751 ve->base.mask |= sibling->mask;
5752 ve->base.logical_mask |= sibling->logical_mask;
5754 if (n != 0 && ve->base.class != sibling->class) {
5755 guc_dbg(guc, "invalid mixing of engine class, sibling %d, already %d\n",
5756 sibling->class, ve->base.class);
5759 } else if (n == 0) {
5760 ve->base.class = sibling->class;
5761 ve->base.uabi_class = sibling->uabi_class;
5762 snprintf(ve->base.name, sizeof(ve->base.name),
5763 "v%dx%d", ve->base.class, count);
5764 ve->base.context_size = sibling->context_size;
5766 ve->base.add_active_request =
5767 sibling->add_active_request;
5768 ve->base.remove_active_request =
5769 sibling->remove_active_request;
5770 ve->base.emit_bb_start = sibling->emit_bb_start;
5771 ve->base.emit_flush = sibling->emit_flush;
5772 ve->base.emit_init_breadcrumb =
5773 sibling->emit_init_breadcrumb;
5774 ve->base.emit_fini_breadcrumb =
5775 sibling->emit_fini_breadcrumb;
5776 ve->base.emit_fini_breadcrumb_dw =
5777 sibling->emit_fini_breadcrumb_dw;
5778 ve->base.breadcrumbs =
5779 intel_breadcrumbs_get(sibling->breadcrumbs);
5781 ve->base.flags |= sibling->flags;
5783 ve->base.props.timeslice_duration_ms =
5784 sibling->props.timeslice_duration_ms;
5785 ve->base.props.preempt_timeout_ms =
5786 sibling->props.preempt_timeout_ms;
5790 return &ve->context;
5793 intel_context_put(&ve->context);
5794 return ERR_PTR(err);
5797 bool intel_guc_virtual_engine_has_heartbeat(const struct intel_engine_cs *ve)
5799 struct intel_engine_cs *engine;
5800 intel_engine_mask_t tmp, mask = ve->mask;
5802 for_each_engine_masked(engine, ve->gt, mask, tmp)
5803 if (READ_ONCE(engine->props.heartbeat_interval_ms))
5809 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
5810 #include "selftest_guc.c"
5811 #include "selftest_guc_multi_lrc.c"
5812 #include "selftest_guc_hangcheck.c"