[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_timeline.c

/*
 * SPDX-License-Identifier: MIT
 *
 * Copyright © 2016-2018 Intel Corporation
 */

#include "i915_drv.h"

#include "i915_timeline.h"
#include "i915_syncmap.h"

struct i915_timeline_hwsp {
	struct i915_vma *vma;
	struct list_head free_link;
	u64 free_bitmap;
};

static inline struct i915_timeline_hwsp *
i915_timeline_hwsp(const struct i915_timeline *tl)
{
	return tl->hwsp_ggtt->private;
}

static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915)
{
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;

	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
	if (IS_ERR(obj))
		return ERR_CAST(obj);

	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);

	vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
	if (IS_ERR(vma))
		i915_gem_object_put(obj);

	return vma;
}

static struct i915_vma *
hwsp_alloc(struct i915_timeline *timeline, unsigned int *cacheline)
{
	struct drm_i915_private *i915 = timeline->i915;
	struct i915_gt_timelines *gt = &i915->gt.timelines;
	struct i915_timeline_hwsp *hwsp;

	BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);

	spin_lock(&gt->hwsp_lock);

	/* hwsp_free_list only contains HWSP that have available cachelines */
	hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
					typeof(*hwsp), free_link);
	if (!hwsp) {
		struct i915_vma *vma;

		spin_unlock(&gt->hwsp_lock);

		hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
		if (!hwsp)
			return ERR_PTR(-ENOMEM);

		vma = __hwsp_alloc(i915);
		if (IS_ERR(vma)) {
			kfree(hwsp);
			return vma;
		}

		vma->private = hwsp;
		hwsp->vma = vma;
		hwsp->free_bitmap = ~0ull;

		spin_lock(&gt->hwsp_lock);
		list_add(&hwsp->free_link, &gt->hwsp_free_list);
	}

	GEM_BUG_ON(!hwsp->free_bitmap);
	*cacheline = __ffs64(hwsp->free_bitmap);
	hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
	if (!hwsp->free_bitmap)
		list_del(&hwsp->free_link);

	spin_unlock(&gt->hwsp_lock);

	GEM_BUG_ON(hwsp->vma->private != hwsp);
	return hwsp->vma;
}

static void hwsp_free(struct i915_timeline *timeline)
{
	struct i915_gt_timelines *gt = &timeline->i915->gt.timelines;
	struct i915_timeline_hwsp *hwsp;

	hwsp = i915_timeline_hwsp(timeline);
	if (!hwsp) /* leave global HWSP alone! */
		return;

	spin_lock(&gt->hwsp_lock);

	/* As a cacheline becomes available, publish the HWSP on the freelist */
	if (!hwsp->free_bitmap)
		list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);

	hwsp->free_bitmap |= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);

	/* And if no one is left using it, give the page back to the system */
	if (hwsp->free_bitmap == ~0ull) {
		i915_vma_put(hwsp->vma);
		list_del(&hwsp->free_link);
		kfree(hwsp);
	}

	spin_unlock(&gt->hwsp_lock);
}

int i915_timeline_init(struct drm_i915_private *i915,
		       struct i915_timeline *timeline,
		       const char *name,
		       struct i915_vma *hwsp)
{
	void *vaddr;

	/*
	 * Ideally we want a set of engines on a single leaf as we expect
	 * to mostly be tracking synchronisation between engines. It is not
	 * a huge issue if this is not the case, but we may want to mitigate
	 * any page crossing penalties if they become an issue.
	 *
	 * Called during early_init before we know how many engines there are.
	 */
	BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES);

	timeline->i915 = i915;
	timeline->name = name;
	timeline->pin_count = 0;
	timeline->has_initial_breadcrumb = !hwsp;

	timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
	if (!hwsp) {
		unsigned int cacheline;

		hwsp = hwsp_alloc(timeline, &cacheline);
		if (IS_ERR(hwsp))
			return PTR_ERR(hwsp);

		timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
	}
	timeline->hwsp_ggtt = i915_vma_get(hwsp);

	vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
	if (IS_ERR(vaddr)) {
		hwsp_free(timeline);
		i915_vma_put(hwsp);
		return PTR_ERR(vaddr);
	}

	timeline->hwsp_seqno =
		memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);

	timeline->fence_context = dma_fence_context_alloc(1);

	spin_lock_init(&timeline->lock);

	init_request_active(&timeline->last_request, NULL);
	INIT_LIST_HEAD(&timeline->requests);

	i915_syncmap_init(&timeline->sync);

	return 0;
}

void i915_timelines_init(struct drm_i915_private *i915)
{
	struct i915_gt_timelines *gt = &i915->gt.timelines;

	mutex_init(&gt->mutex);
	INIT_LIST_HEAD(&gt->active_list);

	spin_lock_init(&gt->hwsp_lock);
	INIT_LIST_HEAD(&gt->hwsp_free_list);

	/* via i915_gem_wait_for_idle() */
	i915_gem_shrinker_taints_mutex(i915, &gt->mutex);
}

static void timeline_add_to_active(struct i915_timeline *tl)
{
	struct i915_gt_timelines *gt = &tl->i915->gt.timelines;

	mutex_lock(&gt->mutex);
	list_add(&tl->link, &gt->active_list);
	mutex_unlock(&gt->mutex);
}

static void timeline_remove_from_active(struct i915_timeline *tl)
{
	struct i915_gt_timelines *gt = &tl->i915->gt.timelines;

	mutex_lock(&gt->mutex);
	list_del(&tl->link);
	mutex_unlock(&gt->mutex);
}

/**
 * i915_timelines_park - called when the driver idles
 * @i915: the drm_i915_private device
 *
 * When the driver is completely idle, we know that all of our sync points
 * have been signaled and our tracking is then entirely redundant. Any request
 * to wait upon an older sync point will be completed instantly as we know
 * the fence is signaled and therefore we will not even look them up in the
 * sync point map.
 */
void i915_timelines_park(struct drm_i915_private *i915)
{
	struct i915_gt_timelines *gt = &i915->gt.timelines;
	struct i915_timeline *timeline;

	mutex_lock(&gt->mutex);
	list_for_each_entry(timeline, &gt->active_list, link) {
		/*
		 * All known fences are completed so we can scrap
		 * the current sync point tracking and start afresh,
		 * any attempt to wait upon a previous sync point
		 * will be skipped as the fence was signaled.
		 */
		i915_syncmap_free(&timeline->sync);
	}
	mutex_unlock(&gt->mutex);
}

void i915_timeline_fini(struct i915_timeline *timeline)
{
	GEM_BUG_ON(timeline->pin_count);
	GEM_BUG_ON(!list_empty(&timeline->requests));

	i915_syncmap_free(&timeline->sync);
	hwsp_free(timeline);

	i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
	i915_vma_put(timeline->hwsp_ggtt);
}

struct i915_timeline *
i915_timeline_create(struct drm_i915_private *i915,
		     const char *name,
		     struct i915_vma *global_hwsp)
{
	struct i915_timeline *timeline;
	int err;

	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
	if (!timeline)
		return ERR_PTR(-ENOMEM);

	err = i915_timeline_init(i915, timeline, name, global_hwsp);
	if (err) {
		kfree(timeline);
		return ERR_PTR(err);
	}

	kref_init(&timeline->kref);

	return timeline;
}

int i915_timeline_pin(struct i915_timeline *tl)
{
	int err;

	if (tl->pin_count++)
		return 0;
	GEM_BUG_ON(!tl->pin_count);

	err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL | PIN_HIGH);
	if (err)
		goto unpin;

	tl->hwsp_offset =
		i915_ggtt_offset(tl->hwsp_ggtt) +
		offset_in_page(tl->hwsp_offset);

	timeline_add_to_active(tl);

	return 0;

unpin:
	tl->pin_count = 0;
	return err;
}

void i915_timeline_unpin(struct i915_timeline *tl)
{
	GEM_BUG_ON(!tl->pin_count);
	if (--tl->pin_count)
		return;

	timeline_remove_from_active(tl);

	/*
	 * Since this timeline is idle, all bariers upon which we were waiting
	 * must also be complete and so we can discard the last used barriers
	 * without loss of information.
	 */
	i915_syncmap_free(&tl->sync);

	__i915_vma_unpin(tl->hwsp_ggtt);
}

void __i915_timeline_free(struct kref *kref)
{
	struct i915_timeline *timeline =
		container_of(kref, typeof(*timeline), kref);

	i915_timeline_fini(timeline);
	kfree(timeline);
}

void i915_timelines_fini(struct drm_i915_private *i915)
{
	struct i915_gt_timelines *gt = &i915->gt.timelines;

	GEM_BUG_ON(!list_empty(&gt->active_list));
	GEM_BUG_ON(!list_empty(&gt->hwsp_free_list));

	mutex_destroy(&gt->mutex);
}

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/mock_timeline.c"
#include "selftests/i915_timeline.c"
#endif
Commit	Line	Data
a89d1f92 CW	1	/*
	2	* SPDX-License-Identifier: MIT
	3	*
	4	* Copyright © 2016-2018 Intel Corporation
	5	*/
	6
	7	#include "i915_drv.h"
	8
	9	#include "i915_timeline.h"
	10	#include "i915_syncmap.h"
	11
8ba306a6 CW	12	struct i915_timeline_hwsp {
	13	struct i915_vma *vma;
	14	struct list_head free_link;
	15	u64 free_bitmap;
	16	};
	17
	18	static inline struct i915_timeline_hwsp *
	19	i915_timeline_hwsp(const struct i915_timeline *tl)
	20	{
	21	return tl->hwsp_ggtt->private;
	22	}
	23
52954edd CW	24	static struct i915_vma __hwsp_alloc(struct drm_i915_private i915)
	25	{
	26	struct drm_i915_gem_object *obj;
	27	struct i915_vma *vma;
	28
	29	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
	30	if (IS_ERR(obj))
	31	return ERR_CAST(obj);
	32
	33	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
	34
	35	vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
	36	if (IS_ERR(vma))
	37	i915_gem_object_put(obj);
	38
	39	return vma;
	40	}
	41
8ba306a6 CW	42	static struct i915_vma *
8ba306a6 CW	43	hwsp_alloc(struct i915_timeline timeline, unsigned int cacheline)
52954edd	44	{
8ba306a6 CW	45	struct drm_i915_private *i915 = timeline->i915;
	46	struct i915_gt_timelines *gt = &i915->gt.timelines;
	47	struct i915_timeline_hwsp *hwsp;
52954edd	48
8ba306a6	49	BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
52954edd	50
8ba306a6	51	spin_lock(&gt->hwsp_lock);
52954edd	52
8ba306a6 CW	53	/* hwsp_free_list only contains HWSP that have available cachelines */
	54	hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
	55	typeof(*hwsp), free_link);
	56	if (!hwsp) {
	57	struct i915_vma *vma;
	58
	59	spin_unlock(&gt->hwsp_lock);
	60
	61	hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
	62	if (!hwsp)
	63	return ERR_PTR(-ENOMEM);
	64
	65	vma = __hwsp_alloc(i915);
	66	if (IS_ERR(vma)) {
	67	kfree(hwsp);
	68	return vma;
	69	}
	70
	71	vma->private = hwsp;
	72	hwsp->vma = vma;
	73	hwsp->free_bitmap = ~0ull;
	74
	75	spin_lock(&gt->hwsp_lock);
	76	list_add(&hwsp->free_link, &gt->hwsp_free_list);
	77	}
	78
	79	GEM_BUG_ON(!hwsp->free_bitmap);
	80	*cacheline = __ffs64(hwsp->free_bitmap);
	81	hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
	82	if (!hwsp->free_bitmap)
	83	list_del(&hwsp->free_link);
	84
	85	spin_unlock(&gt->hwsp_lock);
	86
	87	GEM_BUG_ON(hwsp->vma->private != hwsp);
	88	return hwsp->vma;
	89	}
	90
	91	static void hwsp_free(struct i915_timeline *timeline)
	92	{
	93	struct i915_gt_timelines *gt = &timeline->i915->gt.timelines;
	94	struct i915_timeline_hwsp *hwsp;
	95
	96	hwsp = i915_timeline_hwsp(timeline);
	97	if (!hwsp) /* leave global HWSP alone! */
	98	return;
	99
	100	spin_lock(&gt->hwsp_lock);
	101
	102	/* As a cacheline becomes available, publish the HWSP on the freelist */
	103	if (!hwsp->free_bitmap)
	104	list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);
	105
	106	hwsp->free_bitmap \|= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);
	107
	108	/* And if no one is left using it, give the page back to the system */
	109	if (hwsp->free_bitmap == ~0ull) {
	110	i915_vma_put(hwsp->vma);
	111	list_del(&hwsp->free_link);
	112	kfree(hwsp);
	113	}
	114
	115	spin_unlock(&gt->hwsp_lock);
52954edd CW	116	}
	117
	118	int i915_timeline_init(struct drm_i915_private *i915,
	119	struct i915_timeline *timeline,
	120	const char *name,
8ba306a6	121	struct i915_vma *hwsp)
a89d1f92	122	{
52954edd	123	void *vaddr;
a89d1f92 CW	124
	125	/*
	126	* Ideally we want a set of engines on a single leaf as we expect
	127	* to mostly be tracking synchronisation between engines. It is not
	128	* a huge issue if this is not the case, but we may want to mitigate
	129	* any page crossing penalties if they become an issue.
52954edd CW	130	*
52954edd CW	131	* Called during early_init before we know how many engines there are.
a89d1f92 CW	132	*/
	133	BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES);
	134
1e345568	135	timeline->i915 = i915;
a89d1f92	136	timeline->name = name;
52954edd	137	timeline->pin_count = 0;
85474441	138	timeline->has_initial_breadcrumb = !hwsp;
52954edd	139
8ba306a6 CW	140	timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
	141	if (!hwsp) {
	142	unsigned int cacheline;
	143
	144	hwsp = hwsp_alloc(timeline, &cacheline);
	145	if (IS_ERR(hwsp))
	146	return PTR_ERR(hwsp);
	147
	148	timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
52954edd	149	}
8ba306a6	150	timeline->hwsp_ggtt = i915_vma_get(hwsp);
a89d1f92	151
8ba306a6	152	vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
52954edd	153	if (IS_ERR(vaddr)) {
8ba306a6 CW	154	hwsp_free(timeline);
8ba306a6 CW	155	i915_vma_put(hwsp);
52954edd CW	156	return PTR_ERR(vaddr);
52954edd CW	157	}
a89d1f92	158
52954edd CW	159	timeline->hwsp_seqno =
52954edd CW	160	memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
a89d1f92 CW	161
	162	timeline->fence_context = dma_fence_context_alloc(1);
	163
	164	spin_lock_init(&timeline->lock);
	165
	166	init_request_active(&timeline->last_request, NULL);
	167	INIT_LIST_HEAD(&timeline->requests);
	168
	169	i915_syncmap_init(&timeline->sync);
52954edd	170
52954edd	171	return 0;
a89d1f92 CW	172	}
a89d1f92 CW	173
1e345568 CW	174	void i915_timelines_init(struct drm_i915_private *i915)
	175	{
	176	struct i915_gt_timelines *gt = &i915->gt.timelines;
	177
	178	mutex_init(&gt->mutex);
9407d3bd	179	INIT_LIST_HEAD(&gt->active_list);
1e345568	180
8ba306a6 CW	181	spin_lock_init(&gt->hwsp_lock);
	182	INIT_LIST_HEAD(&gt->hwsp_free_list);
	183
1e345568 CW	184	/* via i915_gem_wait_for_idle() */
	185	i915_gem_shrinker_taints_mutex(i915, &gt->mutex);
	186	}
	187
9407d3bd CW	188	static void timeline_add_to_active(struct i915_timeline *tl)
	189	{
	190	struct i915_gt_timelines *gt = &tl->i915->gt.timelines;
	191
	192	mutex_lock(&gt->mutex);
	193	list_add(&tl->link, &gt->active_list);
	194	mutex_unlock(&gt->mutex);
	195	}
	196
	197	static void timeline_remove_from_active(struct i915_timeline *tl)
	198	{
	199	struct i915_gt_timelines *gt = &tl->i915->gt.timelines;
	200
	201	mutex_lock(&gt->mutex);
	202	list_del(&tl->link);
	203	mutex_unlock(&gt->mutex);
	204	}
	205
a89d1f92 CW	206	/**
	207	* i915_timelines_park - called when the driver idles
	208	* @i915: the drm_i915_private device
	209	*
	210	* When the driver is completely idle, we know that all of our sync points
	211	* have been signaled and our tracking is then entirely redundant. Any request
	212	* to wait upon an older sync point will be completed instantly as we know
	213	* the fence is signaled and therefore we will not even look them up in the
	214	* sync point map.
	215	*/
	216	void i915_timelines_park(struct drm_i915_private *i915)
	217	{
1e345568	218	struct i915_gt_timelines *gt = &i915->gt.timelines;
a89d1f92 CW	219	struct i915_timeline *timeline;
a89d1f92 CW	220
1e345568	221	mutex_lock(&gt->mutex);
9407d3bd	222	list_for_each_entry(timeline, &gt->active_list, link) {
a89d1f92 CW	223	/*
	224	* All known fences are completed so we can scrap
	225	* the current sync point tracking and start afresh,
	226	* any attempt to wait upon a previous sync point
	227	* will be skipped as the fence was signaled.
	228	*/
	229	i915_syncmap_free(&timeline->sync);
	230	}
1e345568	231	mutex_unlock(&gt->mutex);
a89d1f92 CW	232	}
	233
	234	void i915_timeline_fini(struct i915_timeline *timeline)
	235	{
52954edd	236	GEM_BUG_ON(timeline->pin_count);
a89d1f92 CW	237	GEM_BUG_ON(!list_empty(&timeline->requests));
a89d1f92 CW	238
8ba306a6 CW	239	i915_syncmap_free(&timeline->sync);
	240	hwsp_free(timeline);
	241
52954edd CW	242	i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
52954edd CW	243	i915_vma_put(timeline->hwsp_ggtt);
a89d1f92 CW	244	}
	245
	246	struct i915_timeline *
52954edd CW	247	i915_timeline_create(struct drm_i915_private *i915,
	248	const char *name,
	249	struct i915_vma *global_hwsp)
a89d1f92 CW	250	{
a89d1f92 CW	251	struct i915_timeline *timeline;
52954edd	252	int err;
a89d1f92 CW	253
	254	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
	255	if (!timeline)
	256	return ERR_PTR(-ENOMEM);
	257
52954edd CW	258	err = i915_timeline_init(i915, timeline, name, global_hwsp);
	259	if (err) {
	260	kfree(timeline);
	261	return ERR_PTR(err);
	262	}
	263
a89d1f92 CW	264	kref_init(&timeline->kref);
	265
	266	return timeline;
	267	}
	268
52954edd CW	269	int i915_timeline_pin(struct i915_timeline *tl)
	270	{
	271	int err;
	272
	273	if (tl->pin_count++)
	274	return 0;
	275	GEM_BUG_ON(!tl->pin_count);
	276
	277	err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL \| PIN_HIGH);
	278	if (err)
	279	goto unpin;
	280
5013eb8c CW	281	tl->hwsp_offset =
	282	i915_ggtt_offset(tl->hwsp_ggtt) +
	283	offset_in_page(tl->hwsp_offset);
	284
9407d3bd CW	285	timeline_add_to_active(tl);
9407d3bd CW	286
52954edd CW	287	return 0;
	288
	289	unpin:
	290	tl->pin_count = 0;
	291	return err;
	292	}
	293
	294	void i915_timeline_unpin(struct i915_timeline *tl)
	295	{
	296	GEM_BUG_ON(!tl->pin_count);
	297	if (--tl->pin_count)
	298	return;
	299
9407d3bd CW	300	timeline_remove_from_active(tl);
9407d3bd CW	301
52954edd CW	302	/*
	303	* Since this timeline is idle, all bariers upon which we were waiting
	304	* must also be complete and so we can discard the last used barriers
	305	* without loss of information.
	306	*/
	307	i915_syncmap_free(&tl->sync);
	308
	309	__i915_vma_unpin(tl->hwsp_ggtt);
	310	}
	311
a89d1f92 CW	312	void __i915_timeline_free(struct kref *kref)
	313	{
	314	struct i915_timeline *timeline =
	315	container_of(kref, typeof(*timeline), kref);
	316
	317	i915_timeline_fini(timeline);
	318	kfree(timeline);
	319	}
	320
1e345568 CW	321	void i915_timelines_fini(struct drm_i915_private *i915)
	322	{
	323	struct i915_gt_timelines *gt = &i915->gt.timelines;
	324
9407d3bd	325	GEM_BUG_ON(!list_empty(&gt->active_list));
8ba306a6	326	GEM_BUG_ON(!list_empty(&gt->hwsp_free_list));
1e345568 CW	327
	328	mutex_destroy(&gt->mutex);
	329	}
	330
a89d1f92 CW	331	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
	332	#include "selftests/mock_timeline.c"
	333	#include "selftests/i915_timeline.c"
	334	#endif