[linux-block.git] / drivers / gpu / drm / msm / adreno / a5xx_preempt.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2017 The Linux Foundation. All rights reserved.
 */

#include "msm_gem.h"
#include "a5xx_gpu.h"

/*
 * Try to transition the preemption state from old to new. Return
 * true on success or false if the original state wasn't 'old'
 */
static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
		enum preempt_state old, enum preempt_state new)
{
	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
		old, new);

	return (cur == old);
}

/*
 * Force the preemption state to the specified state.  This is used in cases
 * where the current state is known and won't change
 */
static inline void set_preempt_state(struct a5xx_gpu *gpu,
		enum preempt_state new)
{
	/*
	 * preempt_state may be read by other cores trying to trigger a
	 * preemption or in the interrupt handler so barriers are needed
	 * before...
	 */
	smp_mb__before_atomic();
	atomic_set(&gpu->preempt_state, new);
	/* ... and after*/
	smp_mb__after_atomic();
}

/* Write the most recent wptr for the given ring into the hardware */
static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
{
	unsigned long flags;
	uint32_t wptr;

	if (!ring)
		return;

	spin_lock_irqsave(&ring->lock, flags);
	wptr = get_wptr(ring);
	spin_unlock_irqrestore(&ring->lock, flags);

	gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
}

/* Return the highest priority ringbuffer with something in it */
static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
{
	unsigned long flags;
	int i;

	for (i = 0; i < gpu->nr_rings; i++) {
		bool empty;
		struct msm_ringbuffer *ring = gpu->rb[i];

		spin_lock_irqsave(&ring->lock, flags);
		empty = (get_wptr(ring) == ring->memptrs->rptr);
		spin_unlock_irqrestore(&ring->lock, flags);

		if (!empty)
			return ring;
	}

	return NULL;
}

static void a5xx_preempt_timer(struct timer_list *t)
{
	struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
	struct msm_gpu *gpu = &a5xx_gpu->base.base;
	struct drm_device *dev = gpu->dev;
	struct msm_drm_private *priv = dev->dev_private;

	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
		return;

	DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
	queue_work(priv->wq, &gpu->recover_work);
}

/* Try to trigger a preemption switch */
void a5xx_preempt_trigger(struct msm_gpu *gpu)
{
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	unsigned long flags;
	struct msm_ringbuffer *ring;

	if (gpu->nr_rings == 1)
		return;

	/*
	 * Try to start preemption by moving from NONE to START. If
	 * unsuccessful, a preemption is already in flight
	 */
	if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
		return;

	/* Get the next ring to preempt to */
	ring = get_next_ring(gpu);

	/*
	 * If no ring is populated or the highest priority ring is the current
	 * one do nothing except to update the wptr to the latest and greatest
	 */
	if (!ring || (a5xx_gpu->cur_ring == ring)) {
		/*
		 * Its possible that while a preemption request is in progress
		 * from an irq context, a user context trying to submit might
		 * fail to update the write pointer, because it determines
		 * that the preempt state is not PREEMPT_NONE.
		 *
		 * Close the race by introducing an intermediate
		 * state PREEMPT_ABORT to let the submit path
		 * know that the ringbuffer is not going to change
		 * and can safely update the write pointer.
		 */

		set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
		update_wptr(gpu, a5xx_gpu->cur_ring);
		set_preempt_state(a5xx_gpu, PREEMPT_NONE);
		return;
	}

	/* Make sure the wptr doesn't update while we're in motion */
	spin_lock_irqsave(&ring->lock, flags);
	a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
	spin_unlock_irqrestore(&ring->lock, flags);

	/* Set the address of the incoming preemption record */
	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
		REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
		a5xx_gpu->preempt_iova[ring->id]);

	a5xx_gpu->next_ring = ring;

	/* Start a timer to catch a stuck preemption */
	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));

	/* Set the preemption state to triggered */
	set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);

	/* Make sure everything is written before hitting the button */
	wmb();

	/* And actually start the preemption */
	gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
}

void a5xx_preempt_irq(struct msm_gpu *gpu)
{
	uint32_t status;
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	struct drm_device *dev = gpu->dev;
	struct msm_drm_private *priv = dev->dev_private;

	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
		return;

	/* Delete the preemption watchdog timer */
	del_timer(&a5xx_gpu->preempt_timer);

	/*
	 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
	 * firing the interrupt, but there is a non zero chance of a hardware
	 * condition or a software race that could set it again before we have a
	 * chance to finish. If that happens, log and go for recovery
	 */
	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
	if (unlikely(status)) {
		set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
		DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
			gpu->name);
		queue_work(priv->wq, &gpu->recover_work);
		return;
	}

	a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
	a5xx_gpu->next_ring = NULL;

	update_wptr(gpu, a5xx_gpu->cur_ring);

	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
}

void a5xx_preempt_hw_init(struct msm_gpu *gpu)
{
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	int i;

	/* Always come up on rb 0 */
	a5xx_gpu->cur_ring = gpu->rb[0];

	/* No preemption if we only have one ring */
	if (gpu->nr_rings == 1)
		return;

	for (i = 0; i < gpu->nr_rings; i++) {
		a5xx_gpu->preempt[i]->wptr = 0;
		a5xx_gpu->preempt[i]->rptr = 0;
		a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
	}

	/* Write a 0 to signal that we aren't switching pagetables */
	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
		REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);

	/* Reset the preemption state */
	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
}

static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
		struct msm_ringbuffer *ring)
{
	struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
	struct msm_gpu *gpu = &adreno_gpu->base;
	struct a5xx_preempt_record *ptr;
	struct drm_gem_object *bo = NULL;
	u64 iova = 0;

	ptr = msm_gem_kernel_new(gpu->dev,
		A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
		MSM_BO_UNCACHED, gpu->aspace, &bo, &iova);

	if (IS_ERR(ptr))
		return PTR_ERR(ptr);

	msm_gem_object_set_name(bo, "preempt");

	a5xx_gpu->preempt_bo[ring->id] = bo;
	a5xx_gpu->preempt_iova[ring->id] = iova;
	a5xx_gpu->preempt[ring->id] = ptr;

	/* Set up the defaults on the preemption record */

	ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
	ptr->info = 0;
	ptr->data = 0;
	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT;
	ptr->rptr_addr = rbmemptr(ring, rptr);
	ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE;

	return 0;
}

void a5xx_preempt_fini(struct msm_gpu *gpu)
{
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	int i;

	for (i = 0; i < gpu->nr_rings; i++)
		msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace, true);
}

void a5xx_preempt_init(struct msm_gpu *gpu)
{
	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	int i;

	/* No preemption if we only have one ring */
	if (gpu->nr_rings <= 1)
		return;

	for (i = 0; i < gpu->nr_rings; i++) {
		if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
			/*
			 * On any failure our adventure is over. Clean up and
			 * set nr_rings to 1 to force preemption off
			 */
			a5xx_preempt_fini(gpu);
			gpu->nr_rings = 1;

			return;
		}
	}

	timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
}
Commit	Line	Data
97fb5e8d	1	// SPDX-License-Identifier: GPL-2.0-only
b1fc2839	2	/* Copyright (c) 2017 The Linux Foundation. All rights reserved.
b1fc2839 JC	3	*/
	4
	5	#include "msm_gem.h"
	6	#include "a5xx_gpu.h"
	7
	8	/*
	9	* Try to transition the preemption state from old to new. Return
	10	* true on success or false if the original state wasn't 'old'
	11	*/
	12	static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
	13	enum preempt_state old, enum preempt_state new)
	14	{
	15	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
	16	old, new);
	17
	18	return (cur == old);
	19	}
	20
	21	/*
	22	* Force the preemption state to the specified state. This is used in cases
	23	* where the current state is known and won't change
	24	*/
	25	static inline void set_preempt_state(struct a5xx_gpu *gpu,
	26	enum preempt_state new)
	27	{
	28	/*
	29	* preempt_state may be read by other cores trying to trigger a
	30	* preemption or in the interrupt handler so barriers are needed
	31	* before...
	32	*/
	33	smp_mb__before_atomic();
	34	atomic_set(&gpu->preempt_state, new);
	35	/* ... and after*/
	36	smp_mb__after_atomic();
	37	}
	38
	39	/* Write the most recent wptr for the given ring into the hardware */
	40	static inline void update_wptr(struct msm_gpu gpu, struct msm_ringbuffer ring)
	41	{
	42	unsigned long flags;
	43	uint32_t wptr;
	44
	45	if (!ring)
	46	return;
	47
	48	spin_lock_irqsave(&ring->lock, flags);
	49	wptr = get_wptr(ring);
	50	spin_unlock_irqrestore(&ring->lock, flags);
	51
	52	gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
	53	}
	54
	55	/* Return the highest priority ringbuffer with something in it */
	56	static struct msm_ringbuffer get_next_ring(struct msm_gpu gpu)
	57	{
	58	unsigned long flags;
	59	int i;
	60
	61	for (i = 0; i < gpu->nr_rings; i++) {
	62	bool empty;
	63	struct msm_ringbuffer *ring = gpu->rb[i];
	64
	65	spin_lock_irqsave(&ring->lock, flags);
	66	empty = (get_wptr(ring) == ring->memptrs->rptr);
67	spin_unlock_irqrestore(&ring->lock, flags);
68
69	if (!empty)
70	return ring;
71	}
72
73	return NULL;
74	}
75
e99e88a9	76	static void a5xx_preempt_timer(struct timer_list *t)
b1fc2839	77	{
e99e88a9	78	struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
b1fc2839 JC	79	struct msm_gpu *gpu = &a5xx_gpu->base.base;
	80	struct drm_device *dev = gpu->dev;
	81	struct msm_drm_private *priv = dev->dev_private;
	82
	83	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
	84	return;
	85
6a41da17	86	DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
b1fc2839 JC	87	queue_work(priv->wq, &gpu->recover_work);
	88	}
	89
	90	/* Try to trigger a preemption switch */
	91	void a5xx_preempt_trigger(struct msm_gpu *gpu)
	92	{
	93	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	94	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	95	unsigned long flags;
	96	struct msm_ringbuffer *ring;
	97
	98	if (gpu->nr_rings == 1)
	99	return;
	100
	101	/*
	102	* Try to start preemption by moving from NONE to START. If
	103	* unsuccessful, a preemption is already in flight
	104	*/
	105	if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
	106	return;
	107
	108	/* Get the next ring to preempt to */
	109	ring = get_next_ring(gpu);
	110
	111	/*
	112	* If no ring is populated or the highest priority ring is the current
	113	* one do nothing except to update the wptr to the latest and greatest
	114	*/
	115	if (!ring \|\| (a5xx_gpu->cur_ring == ring)) {
	116	/*
	117	* Its possible that while a preemption request is in progress
	118	* from an irq context, a user context trying to submit might
	119	* fail to update the write pointer, because it determines
	120	* that the preempt state is not PREEMPT_NONE.
	121	*
	122	* Close the race by introducing an intermediate
	123	* state PREEMPT_ABORT to let the submit path
	124	* know that the ringbuffer is not going to change
	125	* and can safely update the write pointer.
	126	*/
	127
	128	set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
	129	update_wptr(gpu, a5xx_gpu->cur_ring);
	130	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
	131	return;
	132	}
	133
	134	/* Make sure the wptr doesn't update while we're in motion */
	135	spin_lock_irqsave(&ring->lock, flags);
	136	a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
	137	spin_unlock_irqrestore(&ring->lock, flags);
	138
	139	/* Set the address of the incoming preemption record */
	140	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
	141	REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
	142	a5xx_gpu->preempt_iova[ring->id]);
	143
	144	a5xx_gpu->next_ring = ring;
	145
	146	/* Start a timer to catch a stuck preemption */
	147	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
	148
	149	/* Set the preemption state to triggered */
	150	set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
151
152	/* Make sure everything is written before hitting the button */
153	wmb();
154
155	/* And actually start the preemption */
156	gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
157	}
158
159	void a5xx_preempt_irq(struct msm_gpu *gpu)
160	{
161	uint32_t status;
162	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
163	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
164	struct drm_device *dev = gpu->dev;
165	struct msm_drm_private *priv = dev->dev_private;
166
167	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
168	return;
169
170	/* Delete the preemption watchdog timer */
171	del_timer(&a5xx_gpu->preempt_timer);
172
173	/*
174	* The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
175	* firing the interrupt, but there is a non zero chance of a hardware
176	* condition or a software race that could set it again before we have a
177	* chance to finish. If that happens, log and go for recovery
178	*/
179	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
180	if (unlikely(status)) {
181	set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
6a41da17	182	DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
b1fc2839 JC	183	gpu->name);
	184	queue_work(priv->wq, &gpu->recover_work);
	185	return;
	186	}
	187
	188	a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
	189	a5xx_gpu->next_ring = NULL;
	190
	191	update_wptr(gpu, a5xx_gpu->cur_ring);
	192
	193	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
	194	}
	195
	196	void a5xx_preempt_hw_init(struct msm_gpu *gpu)
	197	{
	198	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	199	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	200	int i;
	201
fc6510ac SM	202	/* Always come up on rb 0 */
	203	a5xx_gpu->cur_ring = gpu->rb[0];
	204
	205	/* No preemption if we only have one ring */
	206	if (gpu->nr_rings == 1)
	207	return;
	208
b1fc2839 JC	209	for (i = 0; i < gpu->nr_rings; i++) {
	210	a5xx_gpu->preempt[i]->wptr = 0;
	211	a5xx_gpu->preempt[i]->rptr = 0;
	212	a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
	213	}
	214
	215	/* Write a 0 to signal that we aren't switching pagetables */
	216	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
	217	REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);
	218
	219	/* Reset the preemption state */
	220	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
b1fc2839 JC	221	}
	222
	223	static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
	224	struct msm_ringbuffer *ring)
	225	{
	226	struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
	227	struct msm_gpu *gpu = &adreno_gpu->base;
	228	struct a5xx_preempt_record *ptr;
	229	struct drm_gem_object *bo = NULL;
	230	u64 iova = 0;
	231
	232	ptr = msm_gem_kernel_new(gpu->dev,
	233	A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
	234	MSM_BO_UNCACHED, gpu->aspace, &bo, &iova);
	235
	236	if (IS_ERR(ptr))
	237	return PTR_ERR(ptr);
	238
0815d774 JC	239	msm_gem_object_set_name(bo, "preempt");
0815d774 JC	240
b1fc2839 JC	241	a5xx_gpu->preempt_bo[ring->id] = bo;
	242	a5xx_gpu->preempt_iova[ring->id] = iova;
	243	a5xx_gpu->preempt[ring->id] = ptr;
	244
	245	/* Set up the defaults on the preemption record */
	246
	247	ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
	248	ptr->info = 0;
	249	ptr->data = 0;
	250	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT;
	251	ptr->rptr_addr = rbmemptr(ring, rptr);
	252	ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE;
	253
	254	return 0;
	255	}
	256
	257	void a5xx_preempt_fini(struct msm_gpu *gpu)
	258	{
	259	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	260	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	261	int i;
	262
1e29dff0 JC	263	for (i = 0; i < gpu->nr_rings; i++)
1e29dff0 JC	264	msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace, true);
b1fc2839 JC	265	}
	266
	267	void a5xx_preempt_init(struct msm_gpu *gpu)
	268	{
	269	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	270	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
	271	int i;
	272
	273	/* No preemption if we only have one ring */
	274	if (gpu->nr_rings <= 1)
	275	return;
	276
	277	for (i = 0; i < gpu->nr_rings; i++) {
	278	if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
	279	/*
	280	* On any failure our adventure is over. Clean up and
	281	* set nr_rings to 1 to force preemption off
	282	*/
	283	a5xx_preempt_fini(gpu);
	284	gpu->nr_rings = 1;
	285
	286	return;
	287	}
	288	}
	289
e99e88a9	290	timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
b1fc2839	291	}