2 * Copyright 2022 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/slab.h>
24 #include <drm/drm_print.h>
26 #include "amdgpu_ring_mux.h"
27 #include "amdgpu_ring.h"
30 #define AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT (HZ / 2)
31 #define AMDGPU_MAX_LAST_UNSIGNALED_THRESHOLD_US 10000
33 static const struct ring_info {
35 const char *ring_name;
37 { AMDGPU_RING_PRIO_DEFAULT, "gfx_low"},
38 { AMDGPU_RING_PRIO_2, "gfx_high"},
41 static struct kmem_cache *amdgpu_mux_chunk_slab;
43 static inline struct amdgpu_mux_entry *amdgpu_ring_mux_sw_entry(struct amdgpu_ring_mux *mux,
44 struct amdgpu_ring *ring)
46 return ring->entry_index < mux->ring_entry_size ?
47 &mux->ring_entry[ring->entry_index] : NULL;
50 /* copy packages on sw ring range[begin, end) */
51 static void amdgpu_ring_mux_copy_pkt_from_sw_ring(struct amdgpu_ring_mux *mux,
52 struct amdgpu_ring *ring,
53 u64 s_start, u64 s_end)
56 struct amdgpu_ring *real_ring = mux->real_ring;
58 start = s_start & ring->buf_mask;
59 end = s_end & ring->buf_mask;
62 DRM_ERROR("no more data copied from sw ring\n");
66 amdgpu_ring_alloc(real_ring, (ring->ring_size >> 2) + end - start);
67 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start],
68 (ring->ring_size >> 2) - start);
69 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[0], end);
71 amdgpu_ring_alloc(real_ring, end - start);
72 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], end - start);
76 static void amdgpu_mux_resubmit_chunks(struct amdgpu_ring_mux *mux)
78 struct amdgpu_mux_entry *e = NULL;
79 struct amdgpu_mux_chunk *chunk;
80 uint32_t seq, last_seq;
83 /*find low priority entries:*/
87 for (i = 0; i < mux->num_ring_entries; i++) {
88 if (mux->ring_entry[i].ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) {
89 e = &mux->ring_entry[i];
95 DRM_ERROR("%s no low priority ring found\n", __func__);
99 last_seq = atomic_read(&e->ring->fence_drv.last_seq);
100 seq = mux->seqno_to_resubmit;
101 if (last_seq < seq) {
102 /*resubmit all the fences between (last_seq, seq]*/
103 list_for_each_entry(chunk, &e->list, entry) {
104 if (chunk->sync_seq > last_seq && chunk->sync_seq <= seq) {
105 amdgpu_fence_update_start_timestamp(e->ring,
108 amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, e->ring,
111 mux->wptr_resubmit = chunk->end;
112 amdgpu_ring_commit(mux->real_ring);
117 del_timer(&mux->resubmit_timer);
118 mux->s_resubmit = false;
121 static void amdgpu_ring_mux_schedule_resubmit(struct amdgpu_ring_mux *mux)
123 mod_timer(&mux->resubmit_timer, jiffies + AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT);
126 static void amdgpu_mux_resubmit_fallback(struct timer_list *t)
128 struct amdgpu_ring_mux *mux = from_timer(mux, t, resubmit_timer);
130 if (!spin_trylock(&mux->lock)) {
131 amdgpu_ring_mux_schedule_resubmit(mux);
132 DRM_ERROR("reschedule resubmit\n");
135 amdgpu_mux_resubmit_chunks(mux);
136 spin_unlock(&mux->lock);
139 int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring,
140 unsigned int entry_size)
142 mux->real_ring = ring;
143 mux->num_ring_entries = 0;
145 mux->ring_entry = kcalloc(entry_size, sizeof(struct amdgpu_mux_entry), GFP_KERNEL);
146 if (!mux->ring_entry)
149 mux->ring_entry_size = entry_size;
150 mux->s_resubmit = false;
152 amdgpu_mux_chunk_slab = kmem_cache_create("amdgpu_mux_chunk",
153 sizeof(struct amdgpu_mux_chunk), 0,
154 SLAB_HWCACHE_ALIGN, NULL);
155 if (!amdgpu_mux_chunk_slab) {
156 DRM_ERROR("create amdgpu_mux_chunk cache failed\n");
160 spin_lock_init(&mux->lock);
161 timer_setup(&mux->resubmit_timer, amdgpu_mux_resubmit_fallback, 0);
166 void amdgpu_ring_mux_fini(struct amdgpu_ring_mux *mux)
168 struct amdgpu_mux_entry *e;
169 struct amdgpu_mux_chunk *chunk, *chunk2;
172 for (i = 0; i < mux->num_ring_entries; i++) {
173 e = &mux->ring_entry[i];
174 list_for_each_entry_safe(chunk, chunk2, &e->list, entry) {
175 list_del(&chunk->entry);
176 kmem_cache_free(amdgpu_mux_chunk_slab, chunk);
179 kmem_cache_destroy(amdgpu_mux_chunk_slab);
180 kfree(mux->ring_entry);
181 mux->ring_entry = NULL;
182 mux->num_ring_entries = 0;
183 mux->ring_entry_size = 0;
186 int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
188 struct amdgpu_mux_entry *e;
190 if (mux->num_ring_entries >= mux->ring_entry_size) {
191 DRM_ERROR("add sw ring exceeding max entry size\n");
195 e = &mux->ring_entry[mux->num_ring_entries];
196 ring->entry_index = mux->num_ring_entries;
199 INIT_LIST_HEAD(&e->list);
200 mux->num_ring_entries += 1;
204 void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr)
206 struct amdgpu_mux_entry *e;
208 spin_lock(&mux->lock);
210 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT)
211 amdgpu_mux_resubmit_chunks(mux);
213 e = amdgpu_ring_mux_sw_entry(mux, ring);
215 DRM_ERROR("cannot find entry for sw ring\n");
216 spin_unlock(&mux->lock);
220 /* We could skip this set wptr as preemption in process. */
221 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && mux->pending_trailing_fence_signaled) {
222 spin_unlock(&mux->lock);
226 e->sw_cptr = e->sw_wptr;
227 /* Update cptr if the package already copied in resubmit functions */
228 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && e->sw_cptr < mux->wptr_resubmit)
229 e->sw_cptr = mux->wptr_resubmit;
231 e->start_ptr_in_hw_ring = mux->real_ring->wptr;
233 /* Skip copying for the packages already resubmitted.*/
234 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT || mux->wptr_resubmit < wptr) {
235 amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, ring, e->sw_cptr, wptr);
236 e->end_ptr_in_hw_ring = mux->real_ring->wptr;
237 amdgpu_ring_commit(mux->real_ring);
239 e->end_ptr_in_hw_ring = mux->real_ring->wptr;
241 spin_unlock(&mux->lock);
244 u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
246 struct amdgpu_mux_entry *e;
248 e = amdgpu_ring_mux_sw_entry(mux, ring);
250 DRM_ERROR("cannot find entry for sw ring\n");
258 * amdgpu_ring_mux_get_rptr - get the readptr of the software ring
259 * @mux: the multiplexer the software rings attach to
260 * @ring: the software ring of which we calculate the readptr
262 * The return value of the readptr is not precise while the other rings could
263 * write data onto the real ring buffer.After overwriting on the real ring, we
264 * can not decide if our packages have been excuted or not read yet. However,
265 * this function is only called by the tools such as umr to collect the latest
266 * packages for the hang analysis. We assume the hang happens near our latest
267 * submit. Thus we could use the following logic to give the clue:
268 * If the readptr is between start and end, then we return the copy pointer
269 * plus the distance from start to readptr. If the readptr is before start, we
270 * return the copy pointer. Lastly, if the readptr is past end, we return the
273 u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
275 struct amdgpu_mux_entry *e;
276 u64 readp, offset, start, end;
278 e = amdgpu_ring_mux_sw_entry(mux, ring);
280 DRM_ERROR("no sw entry found!\n");
284 readp = amdgpu_ring_get_rptr(mux->real_ring);
286 start = e->start_ptr_in_hw_ring & mux->real_ring->buf_mask;
287 end = e->end_ptr_in_hw_ring & mux->real_ring->buf_mask;
290 readp += mux->real_ring->ring_size >> 2;
291 end += mux->real_ring->ring_size >> 2;
294 if (start <= readp && readp <= end) {
295 offset = readp - start;
296 e->sw_rptr = (e->sw_cptr + offset) & ring->buf_mask;
297 } else if (readp < start) {
298 e->sw_rptr = e->sw_cptr;
301 e->sw_rptr = e->sw_wptr;
307 u64 amdgpu_sw_ring_get_rptr_gfx(struct amdgpu_ring *ring)
309 struct amdgpu_device *adev = ring->adev;
310 struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
312 WARN_ON(!ring->is_sw_ring);
313 return amdgpu_ring_mux_get_rptr(mux, ring);
316 u64 amdgpu_sw_ring_get_wptr_gfx(struct amdgpu_ring *ring)
318 struct amdgpu_device *adev = ring->adev;
319 struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
321 WARN_ON(!ring->is_sw_ring);
322 return amdgpu_ring_mux_get_wptr(mux, ring);
325 void amdgpu_sw_ring_set_wptr_gfx(struct amdgpu_ring *ring)
327 struct amdgpu_device *adev = ring->adev;
328 struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
330 WARN_ON(!ring->is_sw_ring);
331 amdgpu_ring_mux_set_wptr(mux, ring, ring->wptr);
334 /* Override insert_nop to prevent emitting nops to the software rings */
335 void amdgpu_sw_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
337 WARN_ON(!ring->is_sw_ring);
340 const char *amdgpu_sw_ring_name(int idx)
342 return idx < ARRAY_SIZE(sw_ring_info) ?
343 sw_ring_info[idx].ring_name : NULL;
346 unsigned int amdgpu_sw_ring_priority(int idx)
348 return idx < ARRAY_SIZE(sw_ring_info) ?
349 sw_ring_info[idx].hw_pio : AMDGPU_RING_PRIO_DEFAULT;
352 /*Scan on low prio rings to have unsignaled fence and high ring has no fence.*/
353 static int amdgpu_mcbp_scan(struct amdgpu_ring_mux *mux)
355 struct amdgpu_ring *ring;
359 for (i = 0; i < mux->num_ring_entries; i++) {
360 ring = mux->ring_entry[i].ring;
361 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT &&
362 amdgpu_fence_count_emitted(ring) > 0)
364 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT &&
365 amdgpu_fence_last_unsignaled_time_us(ring) >
366 AMDGPU_MAX_LAST_UNSIGNALED_THRESHOLD_US)
369 return need_preempt && !mux->s_resubmit;
372 /* Trigger Mid-Command Buffer Preemption (MCBP) and find if we need to resubmit. */
373 static int amdgpu_mcbp_trigger_preempt(struct amdgpu_ring_mux *mux)
377 spin_lock(&mux->lock);
378 mux->pending_trailing_fence_signaled = true;
379 r = amdgpu_ring_preempt_ib(mux->real_ring);
380 spin_unlock(&mux->lock);
384 void amdgpu_sw_ring_ib_begin(struct amdgpu_ring *ring)
386 struct amdgpu_device *adev = ring->adev;
387 struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
389 WARN_ON(!ring->is_sw_ring);
390 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) {
391 if (amdgpu_mcbp_scan(mux) > 0)
392 amdgpu_mcbp_trigger_preempt(mux);
396 amdgpu_ring_mux_start_ib(mux, ring);
399 void amdgpu_sw_ring_ib_end(struct amdgpu_ring *ring)
401 struct amdgpu_device *adev = ring->adev;
402 struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
404 WARN_ON(!ring->is_sw_ring);
405 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT)
407 amdgpu_ring_mux_end_ib(mux, ring);
410 void amdgpu_ring_mux_start_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
412 struct amdgpu_mux_entry *e;
413 struct amdgpu_mux_chunk *chunk;
415 spin_lock(&mux->lock);
416 amdgpu_mux_resubmit_chunks(mux);
417 spin_unlock(&mux->lock);
419 e = amdgpu_ring_mux_sw_entry(mux, ring);
421 DRM_ERROR("cannot find entry!\n");
425 chunk = kmem_cache_alloc(amdgpu_mux_chunk_slab, GFP_KERNEL);
427 DRM_ERROR("alloc amdgpu_mux_chunk_slab failed\n");
431 chunk->start = ring->wptr;
432 list_add_tail(&chunk->entry, &e->list);
435 static void scan_and_remove_signaled_chunk(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
437 uint32_t last_seq = 0;
438 struct amdgpu_mux_entry *e;
439 struct amdgpu_mux_chunk *chunk, *tmp;
441 e = amdgpu_ring_mux_sw_entry(mux, ring);
443 DRM_ERROR("cannot find entry!\n");
447 last_seq = atomic_read(&ring->fence_drv.last_seq);
449 list_for_each_entry_safe(chunk, tmp, &e->list, entry) {
450 if (chunk->sync_seq <= last_seq) {
451 list_del(&chunk->entry);
452 kmem_cache_free(amdgpu_mux_chunk_slab, chunk);
457 void amdgpu_ring_mux_end_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
459 struct amdgpu_mux_entry *e;
460 struct amdgpu_mux_chunk *chunk;
462 e = amdgpu_ring_mux_sw_entry(mux, ring);
464 DRM_ERROR("cannot find entry!\n");
468 chunk = list_last_entry(&e->list, struct amdgpu_mux_chunk, entry);
470 DRM_ERROR("cannot find chunk!\n");
474 chunk->end = ring->wptr;
475 chunk->sync_seq = READ_ONCE(ring->fence_drv.sync_seq);
477 scan_and_remove_signaled_chunk(mux, ring);
480 bool amdgpu_mcbp_handle_trailing_fence_irq(struct amdgpu_ring_mux *mux)
482 struct amdgpu_mux_entry *e;
483 struct amdgpu_ring *ring = NULL;
486 if (!mux->pending_trailing_fence_signaled)
489 if (mux->real_ring->trail_seq != le32_to_cpu(*mux->real_ring->trail_fence_cpu_addr))
492 for (i = 0; i < mux->num_ring_entries; i++) {
493 e = &mux->ring_entry[i];
494 if (e->ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) {
501 DRM_ERROR("cannot find low priority ring\n");
505 amdgpu_fence_process(ring);
506 if (amdgpu_fence_count_emitted(ring) > 0) {
507 mux->s_resubmit = true;
508 mux->seqno_to_resubmit = ring->fence_drv.sync_seq;
509 amdgpu_ring_mux_schedule_resubmit(mux);
512 mux->pending_trailing_fence_signaled = false;