[linux-2.6-block.git] / drivers / gpu / drm / i915 / gt / intel_timeline.c

/*
 * SPDX-License-Identifier: MIT
 *
 * Copyright © 2016-2018 Intel Corporation
 */

#include "i915_drv.h"

#include "i915_active.h"
#include "i915_syncmap.h"
#include "intel_gt.h"
#include "intel_ring.h"
#include "intel_timeline.h"

#define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) | BIT(bit)))
#define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))

struct intel_timeline_hwsp {
	struct intel_gt *gt;
	struct intel_gt_timelines *gt_timelines;
	struct list_head free_link;
	struct i915_vma *vma;
	u64 free_bitmap;
};

struct intel_timeline_cacheline {
	struct i915_active active;
	struct intel_timeline_hwsp *hwsp;
	void *vaddr;
#define CACHELINE_BITS 6
#define CACHELINE_FREE CACHELINE_BITS
};

static struct i915_vma *__hwsp_alloc(struct intel_gt *gt)
{
	struct drm_i915_private *i915 = gt->i915;
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;

	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
	if (IS_ERR(obj))
		return ERR_CAST(obj);

	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);

	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
	if (IS_ERR(vma))
		i915_gem_object_put(obj);

	return vma;
}

static struct i915_vma *
hwsp_alloc(struct intel_timeline *timeline, unsigned int *cacheline)
{
	struct intel_gt_timelines *gt = &timeline->gt->timelines;
	struct intel_timeline_hwsp *hwsp;

	BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);

	spin_lock_irq(&gt->hwsp_lock);

	/* hwsp_free_list only contains HWSP that have available cachelines */
	hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
					typeof(*hwsp), free_link);
	if (!hwsp) {
		struct i915_vma *vma;

		spin_unlock_irq(&gt->hwsp_lock);

		hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
		if (!hwsp)
			return ERR_PTR(-ENOMEM);

		vma = __hwsp_alloc(timeline->gt);
		if (IS_ERR(vma)) {
			kfree(hwsp);
			return vma;
		}

		vma->private = hwsp;
		hwsp->gt = timeline->gt;
		hwsp->vma = vma;
		hwsp->free_bitmap = ~0ull;
		hwsp->gt_timelines = gt;

		spin_lock_irq(&gt->hwsp_lock);
		list_add(&hwsp->free_link, &gt->hwsp_free_list);
	}

	GEM_BUG_ON(!hwsp->free_bitmap);
	*cacheline = __ffs64(hwsp->free_bitmap);
	hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
	if (!hwsp->free_bitmap)
		list_del(&hwsp->free_link);

	spin_unlock_irq(&gt->hwsp_lock);

	GEM_BUG_ON(hwsp->vma->private != hwsp);
	return hwsp->vma;
}

static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline)
{
	struct intel_gt_timelines *gt = hwsp->gt_timelines;
	unsigned long flags;

	spin_lock_irqsave(&gt->hwsp_lock, flags);

	/* As a cacheline becomes available, publish the HWSP on the freelist */
	if (!hwsp->free_bitmap)
		list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);

	GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
	hwsp->free_bitmap |= BIT_ULL(cacheline);

	/* And if no one is left using it, give the page back to the system */
	if (hwsp->free_bitmap == ~0ull) {
		i915_vma_put(hwsp->vma);
		list_del(&hwsp->free_link);
		kfree(hwsp);
	}

	spin_unlock_irqrestore(&gt->hwsp_lock, flags);
}

static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
{
	GEM_BUG_ON(!i915_active_is_idle(&cl->active));

	i915_gem_object_unpin_map(cl->hwsp->vma->obj);
	i915_vma_put(cl->hwsp->vma);
	__idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));

	i915_active_fini(&cl->active);
	kfree(cl);
}

__i915_active_call
static void __cacheline_retire(struct i915_active *active)
{
	struct intel_timeline_cacheline *cl =
		container_of(active, typeof(*cl), active);

	i915_vma_unpin(cl->hwsp->vma);
	if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))
		__idle_cacheline_free(cl);
}

static int __cacheline_active(struct i915_active *active)
{
	struct intel_timeline_cacheline *cl =
		container_of(active, typeof(*cl), active);

	__i915_vma_pin(cl->hwsp->vma);
	return 0;
}

static struct intel_timeline_cacheline *
cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
{
	struct intel_timeline_cacheline *cl;
	void *vaddr;

	GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));

	cl = kmalloc(sizeof(*cl), GFP_KERNEL);
	if (!cl)
		return ERR_PTR(-ENOMEM);

	vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
	if (IS_ERR(vaddr)) {
		kfree(cl);
		return ERR_CAST(vaddr);
	}

	i915_vma_get(hwsp->vma);
	cl->hwsp = hwsp;
	cl->vaddr = page_pack_bits(vaddr, cacheline);

	i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);

	return cl;
}

static void cacheline_acquire(struct intel_timeline_cacheline *cl)
{
	if (cl)
		i915_active_acquire(&cl->active);
}

static void cacheline_release(struct intel_timeline_cacheline *cl)
{
	if (cl)
		i915_active_release(&cl->active);
}

static void cacheline_free(struct intel_timeline_cacheline *cl)
{
	GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));
	cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);

	if (i915_active_is_idle(&cl->active))
		__idle_cacheline_free(cl);
}

int intel_timeline_init(struct intel_timeline *timeline,
			struct intel_gt *gt,
			struct i915_vma *hwsp)
{
	void *vaddr;

	kref_init(&timeline->kref);
	atomic_set(&timeline->pin_count, 0);

	timeline->gt = gt;

	timeline->has_initial_breadcrumb = !hwsp;
	timeline->hwsp_cacheline = NULL;

	if (!hwsp) {
		struct intel_timeline_cacheline *cl;
		unsigned int cacheline;

		hwsp = hwsp_alloc(timeline, &cacheline);
		if (IS_ERR(hwsp))
			return PTR_ERR(hwsp);

		cl = cacheline_alloc(hwsp->private, cacheline);
		if (IS_ERR(cl)) {
			__idle_hwsp_free(hwsp->private, cacheline);
			return PTR_ERR(cl);
		}

		timeline->hwsp_cacheline = cl;
		timeline->hwsp_offset = cacheline * CACHELINE_BYTES;

		vaddr = page_mask_bits(cl->vaddr);
	} else {
		timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;

		vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
		if (IS_ERR(vaddr))
			return PTR_ERR(vaddr);
	}

	timeline->hwsp_seqno =
		memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);

	timeline->hwsp_ggtt = i915_vma_get(hwsp);
	GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);

	timeline->fence_context = dma_fence_context_alloc(1);

	mutex_init(&timeline->mutex);

	INIT_ACTIVE_FENCE(&timeline->last_request, &timeline->mutex);
	INIT_LIST_HEAD(&timeline->requests);

	i915_syncmap_init(&timeline->sync);

	return 0;
}

static void timelines_init(struct intel_gt *gt)
{
	struct intel_gt_timelines *timelines = &gt->timelines;

	spin_lock_init(&timelines->lock);
	INIT_LIST_HEAD(&timelines->active_list);

	spin_lock_init(&timelines->hwsp_lock);
	INIT_LIST_HEAD(&timelines->hwsp_free_list);
}

void intel_timelines_init(struct drm_i915_private *i915)
{
	timelines_init(&i915->gt);
}

void intel_timeline_fini(struct intel_timeline *timeline)
{
	GEM_BUG_ON(atomic_read(&timeline->pin_count));
	GEM_BUG_ON(!list_empty(&timeline->requests));

	if (timeline->hwsp_cacheline)
		cacheline_free(timeline->hwsp_cacheline);
	else
		i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);

	i915_vma_put(timeline->hwsp_ggtt);
}

struct intel_timeline *
intel_timeline_create(struct intel_gt *gt, struct i915_vma *global_hwsp)
{
	struct intel_timeline *timeline;
	int err;

	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
	if (!timeline)
		return ERR_PTR(-ENOMEM);

	err = intel_timeline_init(timeline, gt, global_hwsp);
	if (err) {
		kfree(timeline);
		return ERR_PTR(err);
	}

	return timeline;
}

int intel_timeline_pin(struct intel_timeline *tl)
{
	int err;

	if (atomic_add_unless(&tl->pin_count, 1, 0))
		return 0;

	err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL | PIN_HIGH);
	if (err)
		return err;

	tl->hwsp_offset =
		i915_ggtt_offset(tl->hwsp_ggtt) +
		offset_in_page(tl->hwsp_offset);

	cacheline_acquire(tl->hwsp_cacheline);
	if (atomic_fetch_inc(&tl->pin_count)) {
		cacheline_release(tl->hwsp_cacheline);
		__i915_vma_unpin(tl->hwsp_ggtt);
	}

	return 0;
}

void intel_timeline_enter(struct intel_timeline *tl)
{
	struct intel_gt_timelines *timelines = &tl->gt->timelines;
	unsigned long flags;

	lockdep_assert_held(&tl->mutex);

	GEM_BUG_ON(!atomic_read(&tl->pin_count));
	if (tl->active_count++)
		return;
	GEM_BUG_ON(!tl->active_count); /* overflow? */

	spin_lock_irqsave(&timelines->lock, flags);
	list_add(&tl->link, &timelines->active_list);
	spin_unlock_irqrestore(&timelines->lock, flags);
}

void intel_timeline_exit(struct intel_timeline *tl)
{
	struct intel_gt_timelines *timelines = &tl->gt->timelines;
	unsigned long flags;

	lockdep_assert_held(&tl->mutex);

	GEM_BUG_ON(!tl->active_count);
	if (--tl->active_count)
		return;

	spin_lock_irqsave(&timelines->lock, flags);
	list_del(&tl->link);
	spin_unlock_irqrestore(&timelines->lock, flags);

	/*
	 * Since this timeline is idle, all bariers upon which we were waiting
	 * must also be complete and so we can discard the last used barriers
	 * without loss of information.
	 */
	i915_syncmap_free(&tl->sync);
}

static u32 timeline_advance(struct intel_timeline *tl)
{
	GEM_BUG_ON(!atomic_read(&tl->pin_count));
	GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);

	return tl->seqno += 1 + tl->has_initial_breadcrumb;
}

static void timeline_rollback(struct intel_timeline *tl)
{
	tl->seqno -= 1 + tl->has_initial_breadcrumb;
}

static noinline int
__intel_timeline_get_seqno(struct intel_timeline *tl,
			   struct i915_request *rq,
			   u32 *seqno)
{
	struct intel_timeline_cacheline *cl;
	unsigned int cacheline;
	struct i915_vma *vma;
	void *vaddr;
	int err;

	/*
	 * If there is an outstanding GPU reference to this cacheline,
	 * such as it being sampled by a HW semaphore on another timeline,
	 * we cannot wraparound our seqno value (the HW semaphore does
	 * a strict greater-than-or-equals compare, not i915_seqno_passed).
	 * So if the cacheline is still busy, we must detach ourselves
	 * from it and leave it inflight alongside its users.
	 *
	 * However, if nobody is watching and we can guarantee that nobody
	 * will, we could simply reuse the same cacheline.
	 *
	 * if (i915_active_request_is_signaled(&tl->last_request) &&
	 *     i915_active_is_signaled(&tl->hwsp_cacheline->active))
	 *	return 0;
	 *
	 * That seems unlikely for a busy timeline that needed to wrap in
	 * the first place, so just replace the cacheline.
	 */

	vma = hwsp_alloc(tl, &cacheline);
	if (IS_ERR(vma)) {
		err = PTR_ERR(vma);
		goto err_rollback;
	}

	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
	if (err) {
		__idle_hwsp_free(vma->private, cacheline);
		goto err_rollback;
	}

	cl = cacheline_alloc(vma->private, cacheline);
	if (IS_ERR(cl)) {
		err = PTR_ERR(cl);
		__idle_hwsp_free(vma->private, cacheline);
		goto err_unpin;
	}
	GEM_BUG_ON(cl->hwsp->vma != vma);

	/*
	 * Attach the old cacheline to the current request, so that we only
	 * free it after the current request is retired, which ensures that
	 * all writes into the cacheline from previous requests are complete.
	 */
	err = i915_active_ref(&tl->hwsp_cacheline->active, tl, &rq->fence);
	if (err)
		goto err_cacheline;

	cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
	cacheline_free(tl->hwsp_cacheline);

	i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
	i915_vma_put(tl->hwsp_ggtt);

	tl->hwsp_ggtt = i915_vma_get(vma);

	vaddr = page_mask_bits(cl->vaddr);
	tl->hwsp_offset = cacheline * CACHELINE_BYTES;
	tl->hwsp_seqno =
		memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);

	tl->hwsp_offset += i915_ggtt_offset(vma);

	cacheline_acquire(cl);
	tl->hwsp_cacheline = cl;

	*seqno = timeline_advance(tl);
	GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
	return 0;

err_cacheline:
	cacheline_free(cl);
err_unpin:
	i915_vma_unpin(vma);
err_rollback:
	timeline_rollback(tl);
	return err;
}

int intel_timeline_get_seqno(struct intel_timeline *tl,
			     struct i915_request *rq,
			     u32 *seqno)
{
	*seqno = timeline_advance(tl);

	/* Replace the HWSP on wraparound for HW semaphores */
	if (unlikely(!*seqno && tl->hwsp_cacheline))
		return __intel_timeline_get_seqno(tl, rq, seqno);

	return 0;
}

static int cacheline_ref(struct intel_timeline_cacheline *cl,
			 struct i915_request *rq)
{
	return i915_active_add_request(&cl->active, rq);
}

int intel_timeline_read_hwsp(struct i915_request *from,
			     struct i915_request *to,
			     u32 *hwsp)
{
	struct intel_timeline *tl;
	int err;

	rcu_read_lock();
	tl = rcu_dereference(from->timeline);
	if (i915_request_completed(from) || !kref_get_unless_zero(&tl->kref))
		tl = NULL;
	rcu_read_unlock();
	if (!tl) /* already completed */
		return 1;

	GEM_BUG_ON(rcu_access_pointer(to->timeline) == tl);

	err = -EBUSY;
	if (mutex_trylock(&tl->mutex)) {
		struct intel_timeline_cacheline *cl = from->hwsp_cacheline;

		if (i915_request_completed(from)) {
			err = 1;
			goto unlock;
		}

		err = cacheline_ref(cl, to);
		if (err)
			goto unlock;

		if (likely(cl == tl->hwsp_cacheline)) {
			*hwsp = tl->hwsp_offset;
		} else { /* across a seqno wrap, recover the original offset */
			*hwsp = i915_ggtt_offset(cl->hwsp->vma) +
				ptr_unmask_bits(cl->vaddr, CACHELINE_BITS) *
				CACHELINE_BYTES;
		}

unlock:
		mutex_unlock(&tl->mutex);
	}
	intel_timeline_put(tl);

	return err;
}

void intel_timeline_unpin(struct intel_timeline *tl)
{
	GEM_BUG_ON(!atomic_read(&tl->pin_count));
	if (!atomic_dec_and_test(&tl->pin_count))
		return;

	cacheline_release(tl->hwsp_cacheline);

	__i915_vma_unpin(tl->hwsp_ggtt);
}

void __intel_timeline_free(struct kref *kref)
{
	struct intel_timeline *timeline =
		container_of(kref, typeof(*timeline), kref);

	intel_timeline_fini(timeline);
	kfree_rcu(timeline, rcu);
}

static void timelines_fini(struct intel_gt *gt)
{
	struct intel_gt_timelines *timelines = &gt->timelines;

	GEM_BUG_ON(!list_empty(&timelines->active_list));
	GEM_BUG_ON(!list_empty(&timelines->hwsp_free_list));
}

void intel_timelines_fini(struct drm_i915_private *i915)
{
	timelines_fini(&i915->gt);
}

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "gt/selftests/mock_timeline.c"
#include "gt/selftest_timeline.c"
#endif
Commit	Line	Data
a89d1f92 CW	1	/*
	2	* SPDX-License-Identifier: MIT
	3	*
	4	* Copyright © 2016-2018 Intel Corporation
	5	*/
	6
	7	#include "i915_drv.h"
	8
ebece753	9	#include "i915_active.h"
a89d1f92	10	#include "i915_syncmap.h"
2871ea85 CW	11	#include "intel_gt.h"
	12	#include "intel_ring.h"
	13	#include "intel_timeline.h"
ebece753 CW	14
	15	#define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) \| BIT(bit)))
	16	#define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))
a89d1f92	17
f0c02c1b	18	struct intel_timeline_hwsp {
4c6d51ea	19	struct intel_gt *gt;
c6fe28b0	20	struct intel_gt_timelines *gt_timelines;
8ba306a6	21	struct list_head free_link;
ebece753	22	struct i915_vma *vma;
8ba306a6 CW	23	u64 free_bitmap;
	24	};
	25
f0c02c1b	26	struct intel_timeline_cacheline {
ebece753	27	struct i915_active active;
f0c02c1b	28	struct intel_timeline_hwsp *hwsp;
ebece753 CW	29	void *vaddr;
	30	#define CACHELINE_BITS 6
	31	#define CACHELINE_FREE CACHELINE_BITS
	32	};
	33
4c6d51ea	34	static struct i915_vma __hwsp_alloc(struct intel_gt gt)
52954edd	35	{
4c6d51ea	36	struct drm_i915_private *i915 = gt->i915;
52954edd CW	37	struct drm_i915_gem_object *obj;
	38	struct i915_vma *vma;
	39
	40	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
	41	if (IS_ERR(obj))
	42	return ERR_CAST(obj);
	43
	44	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
	45
4c6d51ea	46	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
52954edd CW	47	if (IS_ERR(vma))
	48	i915_gem_object_put(obj);
	49
	50	return vma;
	51	}
	52
8ba306a6	53	static struct i915_vma *
f0c02c1b	54	hwsp_alloc(struct intel_timeline timeline, unsigned int cacheline)
52954edd	55	{
c6fe28b0	56	struct intel_gt_timelines *gt = &timeline->gt->timelines;
f0c02c1b	57	struct intel_timeline_hwsp *hwsp;
52954edd	58
8ba306a6	59	BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
52954edd	60
155ab883	61	spin_lock_irq(&gt->hwsp_lock);
52954edd	62
8ba306a6 CW	63	/* hwsp_free_list only contains HWSP that have available cachelines */
	64	hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
	65	typeof(*hwsp), free_link);
	66	if (!hwsp) {
	67	struct i915_vma *vma;
	68
155ab883	69	spin_unlock_irq(&gt->hwsp_lock);
8ba306a6 CW	70
	71	hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
	72	if (!hwsp)
	73	return ERR_PTR(-ENOMEM);
	74
4c6d51ea	75	vma = __hwsp_alloc(timeline->gt);
8ba306a6 CW	76	if (IS_ERR(vma)) {
	77	kfree(hwsp);
	78	return vma;
	79	}
	80
	81	vma->private = hwsp;
4c6d51ea	82	hwsp->gt = timeline->gt;
8ba306a6 CW	83	hwsp->vma = vma;
8ba306a6 CW	84	hwsp->free_bitmap = ~0ull;
4c6d51ea	85	hwsp->gt_timelines = gt;
8ba306a6	86
155ab883	87	spin_lock_irq(&gt->hwsp_lock);
8ba306a6 CW	88	list_add(&hwsp->free_link, &gt->hwsp_free_list);
	89	}
	90
	91	GEM_BUG_ON(!hwsp->free_bitmap);
	92	*cacheline = __ffs64(hwsp->free_bitmap);
	93	hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
	94	if (!hwsp->free_bitmap)
	95	list_del(&hwsp->free_link);
	96
155ab883	97	spin_unlock_irq(&gt->hwsp_lock);
8ba306a6 CW	98
	99	GEM_BUG_ON(hwsp->vma->private != hwsp);
	100	return hwsp->vma;
	101	}
	102
f0c02c1b	103	static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline)
8ba306a6	104	{
c6fe28b0	105	struct intel_gt_timelines *gt = hwsp->gt_timelines;
155ab883	106	unsigned long flags;
8ba306a6	107
155ab883	108	spin_lock_irqsave(&gt->hwsp_lock, flags);
8ba306a6 CW	109
	110	/* As a cacheline becomes available, publish the HWSP on the freelist */
	111	if (!hwsp->free_bitmap)
	112	list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);
	113
ebece753 CW	114	GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
ebece753 CW	115	hwsp->free_bitmap \|= BIT_ULL(cacheline);
8ba306a6 CW	116
	117	/* And if no one is left using it, give the page back to the system */
	118	if (hwsp->free_bitmap == ~0ull) {
	119	i915_vma_put(hwsp->vma);
	120	list_del(&hwsp->free_link);
	121	kfree(hwsp);
	122	}
	123
155ab883	124	spin_unlock_irqrestore(&gt->hwsp_lock, flags);
52954edd CW	125	}
52954edd CW	126
f0c02c1b	127	static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
ebece753 CW	128	{
	129	GEM_BUG_ON(!i915_active_is_idle(&cl->active));
	130
	131	i915_gem_object_unpin_map(cl->hwsp->vma->obj);
	132	i915_vma_put(cl->hwsp->vma);
	133	__idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));
	134
	135	i915_active_fini(&cl->active);
	136	kfree(cl);
	137	}
	138
274cbf20	139	__i915_active_call
ebece753 CW	140	static void __cacheline_retire(struct i915_active *active)
ebece753 CW	141	{
f0c02c1b	142	struct intel_timeline_cacheline *cl =
ebece753 CW	143	container_of(active, typeof(*cl), active);
	144
	145	i915_vma_unpin(cl->hwsp->vma);
	146	if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))
	147	__idle_cacheline_free(cl);
	148	}
	149
12c255b5 CW	150	static int __cacheline_active(struct i915_active *active)
	151	{
	152	struct intel_timeline_cacheline *cl =
	153	container_of(active, typeof(*cl), active);
	154
	155	__i915_vma_pin(cl->hwsp->vma);
	156	return 0;
	157	}
	158
f0c02c1b TU	159	static struct intel_timeline_cacheline *
f0c02c1b TU	160	cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
ebece753	161	{
f0c02c1b	162	struct intel_timeline_cacheline *cl;
ebece753 CW	163	void *vaddr;
	164
	165	GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));
	166
	167	cl = kmalloc(sizeof(*cl), GFP_KERNEL);
	168	if (!cl)
	169	return ERR_PTR(-ENOMEM);
	170
	171	vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
	172	if (IS_ERR(vaddr)) {
	173	kfree(cl);
	174	return ERR_CAST(vaddr);
	175	}
	176
	177	i915_vma_get(hwsp->vma);
	178	cl->hwsp = hwsp;
	179	cl->vaddr = page_pack_bits(vaddr, cacheline);
	180
b1e3177b	181	i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
ebece753 CW	182
	183	return cl;
	184	}
	185
f0c02c1b	186	static void cacheline_acquire(struct intel_timeline_cacheline *cl)
ebece753	187	{
12c255b5 CW	188	if (cl)
12c255b5 CW	189	i915_active_acquire(&cl->active);
ebece753 CW	190	}
ebece753 CW	191
f0c02c1b	192	static void cacheline_release(struct intel_timeline_cacheline *cl)
ebece753 CW	193	{
	194	if (cl)
	195	i915_active_release(&cl->active);
	196	}
	197
f0c02c1b	198	static void cacheline_free(struct intel_timeline_cacheline *cl)
ebece753 CW	199	{
	200	GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));
	201	cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);
	202
	203	if (i915_active_is_idle(&cl->active))
	204	__idle_cacheline_free(cl);
	205	}
	206
f0c02c1b TU	207	int intel_timeline_init(struct intel_timeline *timeline,
	208	struct intel_gt *gt,
	209	struct i915_vma *hwsp)
a89d1f92	210	{
52954edd	211	void *vaddr;
a89d1f92	212
f0ca820c	213	kref_init(&timeline->kref);
ccb23d2d	214	atomic_set(&timeline->pin_count, 0);
f0ca820c	215
4c6d51ea	216	timeline->gt = gt;
f0ca820c	217
85474441	218	timeline->has_initial_breadcrumb = !hwsp;
ebece753	219	timeline->hwsp_cacheline = NULL;
52954edd	220
8ba306a6	221	if (!hwsp) {
f0c02c1b	222	struct intel_timeline_cacheline *cl;
8ba306a6 CW	223	unsigned int cacheline;
	224
	225	hwsp = hwsp_alloc(timeline, &cacheline);
	226	if (IS_ERR(hwsp))
	227	return PTR_ERR(hwsp);
	228
ebece753 CW	229	cl = cacheline_alloc(hwsp->private, cacheline);
	230	if (IS_ERR(cl)) {
	231	__idle_hwsp_free(hwsp->private, cacheline);
	232	return PTR_ERR(cl);
	233	}
	234
	235	timeline->hwsp_cacheline = cl;
8ba306a6	236	timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
a89d1f92	237
ebece753 CW	238	vaddr = page_mask_bits(cl->vaddr);
	239	} else {
	240	timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
	241
	242	vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
	243	if (IS_ERR(vaddr))
	244	return PTR_ERR(vaddr);
52954edd	245	}
a89d1f92	246
52954edd CW	247	timeline->hwsp_seqno =
52954edd CW	248	memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
a89d1f92	249
ebece753 CW	250	timeline->hwsp_ggtt = i915_vma_get(hwsp);
	251	GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
	252
a89d1f92 CW	253	timeline->fence_context = dma_fence_context_alloc(1);
a89d1f92 CW	254
3ef71149	255	mutex_init(&timeline->mutex);
a89d1f92	256
b1e3177b	257	INIT_ACTIVE_FENCE(&timeline->last_request, &timeline->mutex);
a89d1f92 CW	258	INIT_LIST_HEAD(&timeline->requests);
	259
	260	i915_syncmap_init(&timeline->sync);
52954edd	261
52954edd	262	return 0;
a89d1f92 CW	263	}
a89d1f92 CW	264
390c8205	265	static void timelines_init(struct intel_gt *gt)
1e345568	266	{
c6fe28b0	267	struct intel_gt_timelines *timelines = &gt->timelines;
1e345568	268
338aade9	269	spin_lock_init(&timelines->lock);
390c8205	270	INIT_LIST_HEAD(&timelines->active_list);
1e345568	271
390c8205 TU	272	spin_lock_init(&timelines->hwsp_lock);
390c8205 TU	273	INIT_LIST_HEAD(&timelines->hwsp_free_list);
390c8205 TU	274	}
390c8205 TU	275
f0c02c1b	276	void intel_timelines_init(struct drm_i915_private *i915)
390c8205 TU	277	{
390c8205 TU	278	timelines_init(&i915->gt);
1e345568 CW	279	}
1e345568 CW	280
f0c02c1b	281	void intel_timeline_fini(struct intel_timeline *timeline)
a89d1f92	282	{
ccb23d2d	283	GEM_BUG_ON(atomic_read(&timeline->pin_count));
a89d1f92 CW	284	GEM_BUG_ON(!list_empty(&timeline->requests));
a89d1f92 CW	285
ebece753 CW	286	if (timeline->hwsp_cacheline)
	287	cacheline_free(timeline->hwsp_cacheline);
	288	else
	289	i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
	290
52954edd	291	i915_vma_put(timeline->hwsp_ggtt);
a89d1f92 CW	292	}
a89d1f92 CW	293
f0c02c1b TU	294	struct intel_timeline *
f0c02c1b TU	295	intel_timeline_create(struct intel_gt gt, struct i915_vma global_hwsp)
a89d1f92	296	{
f0c02c1b	297	struct intel_timeline *timeline;
52954edd	298	int err;
a89d1f92 CW	299
	300	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
	301	if (!timeline)
	302	return ERR_PTR(-ENOMEM);
	303
f0c02c1b	304	err = intel_timeline_init(timeline, gt, global_hwsp);
52954edd CW	305	if (err) {
	306	kfree(timeline);
	307	return ERR_PTR(err);
	308	}
	309
a89d1f92 CW	310	return timeline;
	311	}
	312
f0c02c1b	313	int intel_timeline_pin(struct intel_timeline *tl)
52954edd CW	314	{
	315	int err;
	316
ccb23d2d	317	if (atomic_add_unless(&tl->pin_count, 1, 0))
52954edd	318	return 0;
52954edd CW	319
	320	err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL \| PIN_HIGH);
	321	if (err)
ccb23d2d	322	return err;
52954edd	323
5013eb8c CW	324	tl->hwsp_offset =
	325	i915_ggtt_offset(tl->hwsp_ggtt) +
	326	offset_in_page(tl->hwsp_offset);
	327
ebece753	328	cacheline_acquire(tl->hwsp_cacheline);
ccb23d2d CW	329	if (atomic_fetch_inc(&tl->pin_count)) {
	330	cacheline_release(tl->hwsp_cacheline);
	331	__i915_vma_unpin(tl->hwsp_ggtt);
	332	}
9407d3bd	333
52954edd	334	return 0;
52954edd CW	335	}
52954edd CW	336
531958f6 CW	337	void intel_timeline_enter(struct intel_timeline *tl)
	338	{
	339	struct intel_gt_timelines *timelines = &tl->gt->timelines;
6dcb85a0	340	unsigned long flags;
531958f6	341
6c69a454 CW	342	lockdep_assert_held(&tl->mutex);
6c69a454 CW	343
ccb23d2d	344	GEM_BUG_ON(!atomic_read(&tl->pin_count));
531958f6 CW	345	if (tl->active_count++)
	346	return;
	347	GEM_BUG_ON(!tl->active_count); /* overflow? */
	348
6dcb85a0	349	spin_lock_irqsave(&timelines->lock, flags);
531958f6	350	list_add(&tl->link, &timelines->active_list);
6dcb85a0	351	spin_unlock_irqrestore(&timelines->lock, flags);
531958f6 CW	352	}
	353
	354	void intel_timeline_exit(struct intel_timeline *tl)
	355	{
	356	struct intel_gt_timelines *timelines = &tl->gt->timelines;
6dcb85a0	357	unsigned long flags;
531958f6	358
6c69a454 CW	359	lockdep_assert_held(&tl->mutex);
6c69a454 CW	360
531958f6 CW	361	GEM_BUG_ON(!tl->active_count);
	362	if (--tl->active_count)
	363	return;
	364
6dcb85a0	365	spin_lock_irqsave(&timelines->lock, flags);
531958f6	366	list_del(&tl->link);
6dcb85a0	367	spin_unlock_irqrestore(&timelines->lock, flags);
531958f6 CW	368
	369	/*
	370	* Since this timeline is idle, all bariers upon which we were waiting
	371	* must also be complete and so we can discard the last used barriers
	372	* without loss of information.
	373	*/
	374	i915_syncmap_free(&tl->sync);
	375	}
	376
f0c02c1b	377	static u32 timeline_advance(struct intel_timeline *tl)
ebece753	378	{
ccb23d2d	379	GEM_BUG_ON(!atomic_read(&tl->pin_count));
ebece753 CW	380	GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);
	381
	382	return tl->seqno += 1 + tl->has_initial_breadcrumb;
	383	}
	384
f0c02c1b	385	static void timeline_rollback(struct intel_timeline *tl)
ebece753 CW	386	{
	387	tl->seqno -= 1 + tl->has_initial_breadcrumb;
	388	}
	389
	390	static noinline int
f0c02c1b TU	391	__intel_timeline_get_seqno(struct intel_timeline *tl,
	392	struct i915_request *rq,
	393	u32 *seqno)
ebece753	394	{
f0c02c1b	395	struct intel_timeline_cacheline *cl;
ebece753 CW	396	unsigned int cacheline;
	397	struct i915_vma *vma;
	398	void *vaddr;
	399	int err;
	400
	401	/*
	402	* If there is an outstanding GPU reference to this cacheline,
	403	* such as it being sampled by a HW semaphore on another timeline,
	404	* we cannot wraparound our seqno value (the HW semaphore does
	405	* a strict greater-than-or-equals compare, not i915_seqno_passed).
	406	* So if the cacheline is still busy, we must detach ourselves
	407	* from it and leave it inflight alongside its users.
	408	*
	409	* However, if nobody is watching and we can guarantee that nobody
	410	* will, we could simply reuse the same cacheline.
	411	*
	412	* if (i915_active_request_is_signaled(&tl->last_request) &&
	413	* i915_active_is_signaled(&tl->hwsp_cacheline->active))
	414	* return 0;
	415	*
	416	* That seems unlikely for a busy timeline that needed to wrap in
	417	* the first place, so just replace the cacheline.
	418	*/
	419
	420	vma = hwsp_alloc(tl, &cacheline);
	421	if (IS_ERR(vma)) {
	422	err = PTR_ERR(vma);
	423	goto err_rollback;
	424	}
	425
	426	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL \| PIN_HIGH);
	427	if (err) {
	428	__idle_hwsp_free(vma->private, cacheline);
	429	goto err_rollback;
	430	}
	431
	432	cl = cacheline_alloc(vma->private, cacheline);
	433	if (IS_ERR(cl)) {
	434	err = PTR_ERR(cl);
	435	__idle_hwsp_free(vma->private, cacheline);
	436	goto err_unpin;
	437	}
	438	GEM_BUG_ON(cl->hwsp->vma != vma);
	439
	440	/*
	441	* Attach the old cacheline to the current request, so that we only
	442	* free it after the current request is retired, which ensures that
	443	* all writes into the cacheline from previous requests are complete.
	444	*/
b1e3177b	445	err = i915_active_ref(&tl->hwsp_cacheline->active, tl, &rq->fence);
ebece753 CW	446	if (err)
	447	goto err_cacheline;
	448
	449	cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
	450	cacheline_free(tl->hwsp_cacheline);
	451
	452	i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
	453	i915_vma_put(tl->hwsp_ggtt);
	454
	455	tl->hwsp_ggtt = i915_vma_get(vma);
	456
	457	vaddr = page_mask_bits(cl->vaddr);
	458	tl->hwsp_offset = cacheline * CACHELINE_BYTES;
	459	tl->hwsp_seqno =
	460	memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);
	461
	462	tl->hwsp_offset += i915_ggtt_offset(vma);
	463
	464	cacheline_acquire(cl);
	465	tl->hwsp_cacheline = cl;
	466
	467	*seqno = timeline_advance(tl);
	468	GEM_BUG_ON(i915_seqno_passed(tl->hwsp_seqno, seqno));
	469	return 0;
	470
	471	err_cacheline:
	472	cacheline_free(cl);
	473	err_unpin:
	474	i915_vma_unpin(vma);
	475	err_rollback:
	476	timeline_rollback(tl);
	477	return err;
	478	}
	479
f0c02c1b TU	480	int intel_timeline_get_seqno(struct intel_timeline *tl,
	481	struct i915_request *rq,
	482	u32 *seqno)
ebece753 CW	483	{
	484	*seqno = timeline_advance(tl);
	485
	486	/* Replace the HWSP on wraparound for HW semaphores */
	487	if (unlikely(!*seqno && tl->hwsp_cacheline))
f0c02c1b	488	return __intel_timeline_get_seqno(tl, rq, seqno);
ebece753 CW	489
	490	return 0;
	491	}
	492
f0c02c1b	493	static int cacheline_ref(struct intel_timeline_cacheline *cl,
ebece753 CW	494	struct i915_request *rq)
ebece753 CW	495	{
d19d71fc	496	return i915_active_add_request(&cl->active, rq);
ebece753 CW	497	}
ebece753 CW	498
f0c02c1b TU	499	int intel_timeline_read_hwsp(struct i915_request *from,
	500	struct i915_request *to,
	501	u32 *hwsp)
ebece753	502	{
9eee0dd7	503	struct intel_timeline *tl;
ebece753 CW	504	int err;
ebece753 CW	505
9eee0dd7 CW	506	rcu_read_lock();
	507	tl = rcu_dereference(from->timeline);
	508	if (i915_request_completed(from) \|\| !kref_get_unless_zero(&tl->kref))
	509	tl = NULL;
	510	rcu_read_unlock();
	511	if (!tl) /* already completed */
	512	return 1;
	513
d19d71fc	514	GEM_BUG_ON(rcu_access_pointer(to->timeline) == tl);
ebece753	515
9eee0dd7 CW	516	err = -EBUSY;
	517	if (mutex_trylock(&tl->mutex)) {
	518	struct intel_timeline_cacheline *cl = from->hwsp_cacheline;
	519
	520	if (i915_request_completed(from)) {
	521	err = 1;
	522	goto unlock;
	523	}
	524
ebece753	525	err = cacheline_ref(cl, to);
9eee0dd7 CW	526	if (err)
	527	goto unlock;
	528
ebece753 CW	529	if (likely(cl == tl->hwsp_cacheline)) {
	530	*hwsp = tl->hwsp_offset;
	531	} else { /* across a seqno wrap, recover the original offset */
	532	*hwsp = i915_ggtt_offset(cl->hwsp->vma) +
	533	ptr_unmask_bits(cl->vaddr, CACHELINE_BITS) *
	534	CACHELINE_BYTES;
	535	}
9eee0dd7 CW	536
	537	unlock:
	538	mutex_unlock(&tl->mutex);
ebece753	539	}
9eee0dd7	540	intel_timeline_put(tl);
ebece753 CW	541
	542	return err;
	543	}
	544
f0c02c1b	545	void intel_timeline_unpin(struct intel_timeline *tl)
52954edd	546	{
ccb23d2d CW	547	GEM_BUG_ON(!atomic_read(&tl->pin_count));
ccb23d2d CW	548	if (!atomic_dec_and_test(&tl->pin_count))
52954edd CW	549	return;
52954edd CW	550
ebece753	551	cacheline_release(tl->hwsp_cacheline);
9407d3bd	552
52954edd CW	553	__i915_vma_unpin(tl->hwsp_ggtt);
	554	}
	555
f0c02c1b	556	void __intel_timeline_free(struct kref *kref)
a89d1f92	557	{
f0c02c1b	558	struct intel_timeline *timeline =
a89d1f92 CW	559	container_of(kref, typeof(*timeline), kref);
a89d1f92 CW	560
f0c02c1b	561	intel_timeline_fini(timeline);
d19d71fc	562	kfree_rcu(timeline, rcu);
a89d1f92 CW	563	}
a89d1f92 CW	564
390c8205	565	static void timelines_fini(struct intel_gt *gt)
1e345568	566	{
c6fe28b0	567	struct intel_gt_timelines *timelines = &gt->timelines;
1e345568	568
390c8205 TU	569	GEM_BUG_ON(!list_empty(&timelines->active_list));
390c8205 TU	570	GEM_BUG_ON(!list_empty(&timelines->hwsp_free_list));
390c8205 TU	571	}
390c8205 TU	572
f0c02c1b	573	void intel_timelines_fini(struct drm_i915_private *i915)
390c8205 TU	574	{
390c8205 TU	575	timelines_fini(&i915->gt);
1e345568 CW	576	}
1e345568 CW	577
a89d1f92	578	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
f0c02c1b TU	579	#include "gt/selftests/mock_timeline.c"
f0c02c1b TU	580	#include "gt/selftest_timeline.c"
a89d1f92	581	#endif