[linux-block.git] / drivers / gpu / drm / radeon / radeon_mn.c

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
/*
 * Authors:
 *    Christian König <christian.koenig@amd.com>
 */

#include <linux/firmware.h>
#include <linux/module.h>
#include <linux/mmu_notifier.h>

#include <drm/drm.h>

#include "radeon.h"

struct radeon_mn {
	/* constant after initialisation */
	struct radeon_device	*rdev;
	struct mm_struct	*mm;
	struct mmu_notifier	mn;

	/* only used on destruction */
	struct work_struct	work;

	/* protected by rdev->mn_lock */
	struct hlist_node	node;

	/* objects protected by lock */
	struct mutex		lock;
	struct rb_root_cached	objects;
};

struct radeon_mn_node {
	struct interval_tree_node	it;
	struct list_head		bos;
};

/**
 * radeon_mn_destroy - destroy the rmn
 *
 * @work: previously sheduled work item
 *
 * Lazy destroys the notifier from a work item
 */
static void radeon_mn_destroy(struct work_struct *work)
{
	struct radeon_mn *rmn = container_of(work, struct radeon_mn, work);
	struct radeon_device *rdev = rmn->rdev;
	struct radeon_mn_node *node, *next_node;
	struct radeon_bo *bo, *next_bo;

	mutex_lock(&rdev->mn_lock);
	mutex_lock(&rmn->lock);
	hash_del(&rmn->node);
	rbtree_postorder_for_each_entry_safe(node, next_node,
					     &rmn->objects.rb_root, it.rb) {

		interval_tree_remove(&node->it, &rmn->objects);
		list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
			bo->mn = NULL;
			list_del_init(&bo->mn_list);
		}
		kfree(node);
	}
	mutex_unlock(&rmn->lock);
	mutex_unlock(&rdev->mn_lock);
	mmu_notifier_unregister(&rmn->mn, rmn->mm);
	kfree(rmn);
}

/**
 * radeon_mn_release - callback to notify about mm destruction
 *
 * @mn: our notifier
 * @mn: the mm this callback is about
 *
 * Shedule a work item to lazy destroy our notifier.
 */
static void radeon_mn_release(struct mmu_notifier *mn,
			      struct mm_struct *mm)
{
	struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn);
	INIT_WORK(&rmn->work, radeon_mn_destroy);
	schedule_work(&rmn->work);
}

/**
 * radeon_mn_invalidate_range_start - callback to notify about mm change
 *
 * @mn: our notifier
 * @mn: the mm this callback is about
 * @start: start of updated range
 * @end: end of updated range
 *
 * We block for all BOs between start and end to be idle and
 * unmap them by move them into system domain again.
 */
static int radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
				const struct mmu_notifier_range *range)
{
	struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn);
	struct ttm_operation_ctx ctx = { false, false };
	struct interval_tree_node *it;
	unsigned long end;
	int ret = 0;

	/* notification is exclusive, but interval is inclusive */
	end = range->end - 1;

	/* TODO we should be able to split locking for interval tree and
	 * the tear down.
	 */
	if (mmu_notifier_range_blockable(range))
		mutex_lock(&rmn->lock);
	else if (!mutex_trylock(&rmn->lock))
		return -EAGAIN;

	it = interval_tree_iter_first(&rmn->objects, range->start, end);
	while (it) {
		struct radeon_mn_node *node;
		struct radeon_bo *bo;
		long r;

		if (!mmu_notifier_range_blockable(range)) {
			ret = -EAGAIN;
			goto out_unlock;
		}

		node = container_of(it, struct radeon_mn_node, it);
		it = interval_tree_iter_next(it, range->start, end);

		list_for_each_entry(bo, &node->bos, mn_list) {

			if (!bo->tbo.ttm || bo->tbo.ttm->state != tt_bound)
				continue;

			r = radeon_bo_reserve(bo, true);
			if (r) {
				DRM_ERROR("(%ld) failed to reserve user bo\n", r);
				continue;
			}

			r = dma_resv_wait_timeout_rcu(bo->tbo.base.resv,
				true, false, MAX_SCHEDULE_TIMEOUT);
			if (r <= 0)
				DRM_ERROR("(%ld) failed to wait for user bo\n", r);

			radeon_ttm_placement_from_domain(bo, RADEON_GEM_DOMAIN_CPU);
			r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
			if (r)
				DRM_ERROR("(%ld) failed to validate user bo\n", r);

			radeon_bo_unreserve(bo);
		}
	}
	
out_unlock:
	mutex_unlock(&rmn->lock);

	return ret;
}

static const struct mmu_notifier_ops radeon_mn_ops = {
	.release = radeon_mn_release,
	.invalidate_range_start = radeon_mn_invalidate_range_start,
};

/**
 * radeon_mn_get - create notifier context
 *
 * @rdev: radeon device pointer
 *
 * Creates a notifier context for current->mm.
 */
static struct radeon_mn *radeon_mn_get(struct radeon_device *rdev)
{
	struct mm_struct *mm = current->mm;
	struct radeon_mn *rmn;
	int r;

	if (down_write_killable(&mm->mmap_sem))
		return ERR_PTR(-EINTR);

	mutex_lock(&rdev->mn_lock);

	hash_for_each_possible(rdev->mn_hash, rmn, node, (unsigned long)mm)
		if (rmn->mm == mm)
			goto release_locks;

	rmn = kzalloc(sizeof(*rmn), GFP_KERNEL);
	if (!rmn) {
		rmn = ERR_PTR(-ENOMEM);
		goto release_locks;
	}

	rmn->rdev = rdev;
	rmn->mm = mm;
	rmn->mn.ops = &radeon_mn_ops;
	mutex_init(&rmn->lock);
	rmn->objects = RB_ROOT_CACHED;
	
	r = __mmu_notifier_register(&rmn->mn, mm);
	if (r)
		goto free_rmn;

	hash_add(rdev->mn_hash, &rmn->node, (unsigned long)mm);

release_locks:
	mutex_unlock(&rdev->mn_lock);
	up_write(&mm->mmap_sem);

	return rmn;

free_rmn:
	mutex_unlock(&rdev->mn_lock);
	up_write(&mm->mmap_sem);
	kfree(rmn);

	return ERR_PTR(r);
}

/**
 * radeon_mn_register - register a BO for notifier updates
 *
 * @bo: radeon buffer object
 * @addr: userptr addr we should monitor
 *
 * Registers an MMU notifier for the given BO at the specified address.
 * Returns 0 on success, -ERRNO if anything goes wrong.
 */
int radeon_mn_register(struct radeon_bo *bo, unsigned long addr)
{
	unsigned long end = addr + radeon_bo_size(bo) - 1;
	struct radeon_device *rdev = bo->rdev;
	struct radeon_mn *rmn;
	struct radeon_mn_node *node = NULL;
	struct list_head bos;
	struct interval_tree_node *it;

	rmn = radeon_mn_get(rdev);
	if (IS_ERR(rmn))
		return PTR_ERR(rmn);

	INIT_LIST_HEAD(&bos);

	mutex_lock(&rmn->lock);

	while ((it = interval_tree_iter_first(&rmn->objects, addr, end))) {
		kfree(node);
		node = container_of(it, struct radeon_mn_node, it);
		interval_tree_remove(&node->it, &rmn->objects);
		addr = min(it->start, addr);
		end = max(it->last, end);
		list_splice(&node->bos, &bos);
	}

	if (!node) {
		node = kmalloc(sizeof(struct radeon_mn_node), GFP_KERNEL);
		if (!node) {
			mutex_unlock(&rmn->lock);
			return -ENOMEM;
		}
	}

	bo->mn = rmn;

	node->it.start = addr;
	node->it.last = end;
	INIT_LIST_HEAD(&node->bos);
	list_splice(&bos, &node->bos);
	list_add(&bo->mn_list, &node->bos);

	interval_tree_insert(&node->it, &rmn->objects);

	mutex_unlock(&rmn->lock);

	return 0;
}

/**
 * radeon_mn_unregister - unregister a BO for notifier updates
 *
 * @bo: radeon buffer object
 *
 * Remove any registration of MMU notifier updates from the buffer object.
 */
void radeon_mn_unregister(struct radeon_bo *bo)
{
	struct radeon_device *rdev = bo->rdev;
	struct radeon_mn *rmn;
	struct list_head *head;

	mutex_lock(&rdev->mn_lock);
	rmn = bo->mn;
	if (rmn == NULL) {
		mutex_unlock(&rdev->mn_lock);
		return;
	}

	mutex_lock(&rmn->lock);
	/* save the next list entry for later */
	head = bo->mn_list.next;

	bo->mn = NULL;
	list_del(&bo->mn_list);

	if (list_empty(head)) {
		struct radeon_mn_node *node;
		node = container_of(head, struct radeon_mn_node, bos);
		interval_tree_remove(&node->it, &rmn->objects);
		kfree(node);
	}

	mutex_unlock(&rmn->lock);
	mutex_unlock(&rdev->mn_lock);
}
Commit	Line	Data
341cb9e4 CK	1	/*
	2	* Copyright 2014 Advanced Micro Devices, Inc.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sub license, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	14	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	15	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
	16	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
	17	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
	18	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
	19	* USE OR OTHER DEALINGS IN THE SOFTWARE.
	20	*
	21	* The above copyright notice and this permission notice (including the
	22	* next paragraph) shall be included in all copies or substantial portions
	23	* of the Software.
	24	*
	25	*/
	26	/*
	27	* Authors:
	28	* Christian König <christian.koenig@amd.com>
	29	*/
	30
	31	#include <linux/firmware.h>
	32	#include <linux/module.h>
	33	#include <linux/mmu_notifier.h>
f9183127	34
341cb9e4 CK	35	#include <drm/drm.h>
	36
	37	#include "radeon.h"
	38
	39	struct radeon_mn {
	40	/* constant after initialisation */
	41	struct radeon_device *rdev;
	42	struct mm_struct *mm;
	43	struct mmu_notifier mn;
	44
	45	/* only used on destruction */
	46	struct work_struct work;
	47
	48	/* protected by rdev->mn_lock */
	49	struct hlist_node node;
	50
	51	/* objects protected by lock */
	52	struct mutex lock;
f808c13f	53	struct rb_root_cached objects;
341cb9e4 CK	54	};
341cb9e4 CK	55
49ecb10e CK	56	struct radeon_mn_node {
	57	struct interval_tree_node it;
	58	struct list_head bos;
	59	};
	60
341cb9e4 CK	61	/**
	62	* radeon_mn_destroy - destroy the rmn
	63	*
	64	* @work: previously sheduled work item
	65	*
	66	* Lazy destroys the notifier from a work item
	67	*/
	68	static void radeon_mn_destroy(struct work_struct *work)
	69	{
	70	struct radeon_mn *rmn = container_of(work, struct radeon_mn, work);
	71	struct radeon_device *rdev = rmn->rdev;
49ecb10e CK	72	struct radeon_mn_node node, next_node;
49ecb10e CK	73	struct radeon_bo bo, next_bo;
341cb9e4 CK	74
	75	mutex_lock(&rdev->mn_lock);
	76	mutex_lock(&rmn->lock);
	77	hash_del(&rmn->node);
f808c13f DB	78	rbtree_postorder_for_each_entry_safe(node, next_node,
f808c13f DB	79	&rmn->objects.rb_root, it.rb) {
49ecb10e CK	80
	81	interval_tree_remove(&node->it, &rmn->objects);
	82	list_for_each_entry_safe(bo, next_bo, &node->bos, mn_list) {
	83	bo->mn = NULL;
	84	list_del_init(&bo->mn_list);
	85	}
	86	kfree(node);
341cb9e4 CK	87	}
	88	mutex_unlock(&rmn->lock);
	89	mutex_unlock(&rdev->mn_lock);
	90	mmu_notifier_unregister(&rmn->mn, rmn->mm);
	91	kfree(rmn);
	92	}
	93
	94	/**
	95	* radeon_mn_release - callback to notify about mm destruction
	96	*
	97	* @mn: our notifier
	98	* @mn: the mm this callback is about
	99	*
	100	* Shedule a work item to lazy destroy our notifier.
	101	*/
	102	static void radeon_mn_release(struct mmu_notifier *mn,
	103	struct mm_struct *mm)
	104	{
	105	struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn);
	106	INIT_WORK(&rmn->work, radeon_mn_destroy);
	107	schedule_work(&rmn->work);
	108	}
	109
	110	/**
	111	* radeon_mn_invalidate_range_start - callback to notify about mm change
	112	*
	113	* @mn: our notifier
	114	* @mn: the mm this callback is about
	115	* @start: start of updated range
	116	* @end: end of updated range
	117	*
	118	* We block for all BOs between start and end to be idle and
	119	* unmap them by move them into system domain again.
	120	*/
93065ac7	121	static int radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
5d6527a7	122	const struct mmu_notifier_range *range)
341cb9e4 CK	123	{
341cb9e4 CK	124	struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn);
19be5570	125	struct ttm_operation_ctx ctx = { false, false };
341cb9e4	126	struct interval_tree_node *it;
5d6527a7	127	unsigned long end;
93065ac7	128	int ret = 0;
341cb9e4 CK	129
341cb9e4 CK	130	/* notification is exclusive, but interval is inclusive */
5d6527a7	131	end = range->end - 1;
341cb9e4	132
93065ac7 MH	133	/* TODO we should be able to split locking for interval tree and
	134	* the tear down.
	135	*/
dfcd6660	136	if (mmu_notifier_range_blockable(range))
93065ac7 MH	137	mutex_lock(&rmn->lock);
	138	else if (!mutex_trylock(&rmn->lock))
	139	return -EAGAIN;
341cb9e4	140
5d6527a7	141	it = interval_tree_iter_first(&rmn->objects, range->start, end);
341cb9e4	142	while (it) {
49ecb10e	143	struct radeon_mn_node *node;
341cb9e4	144	struct radeon_bo *bo;
9fb2bcf9	145	long r;
341cb9e4	146
dfcd6660	147	if (!mmu_notifier_range_blockable(range)) {
93065ac7 MH	148	ret = -EAGAIN;
	149	goto out_unlock;
	150	}
	151
49ecb10e	152	node = container_of(it, struct radeon_mn_node, it);
5d6527a7	153	it = interval_tree_iter_next(it, range->start, end);
341cb9e4	154
49ecb10e	155	list_for_each_entry(bo, &node->bos, mn_list) {
341cb9e4	156
247c4050 CK	157	if (!bo->tbo.ttm \|\| bo->tbo.ttm->state != tt_bound)
	158	continue;
	159
49ecb10e CK	160	r = radeon_bo_reserve(bo, true);
49ecb10e CK	161	if (r) {
9fb2bcf9	162	DRM_ERROR("(%ld) failed to reserve user bo\n", r);
49ecb10e CK	163	continue;
49ecb10e CK	164	}
341cb9e4	165
52791eee	166	r = dma_resv_wait_timeout_rcu(bo->tbo.base.resv,
49ecb10e	167	true, false, MAX_SCHEDULE_TIMEOUT);
9fb2bcf9 CK	168	if (r <= 0)
9fb2bcf9 CK	169	DRM_ERROR("(%ld) failed to wait for user bo\n", r);
341cb9e4	170
49ecb10e	171	radeon_ttm_placement_from_domain(bo, RADEON_GEM_DOMAIN_CPU);
19be5570	172	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
49ecb10e	173	if (r)
9fb2bcf9	174	DRM_ERROR("(%ld) failed to validate user bo\n", r);
49ecb10e CK	175
	176	radeon_bo_unreserve(bo);
	177	}
341cb9e4 CK	178	}
341cb9e4 CK	179
93065ac7	180	out_unlock:
341cb9e4	181	mutex_unlock(&rmn->lock);
93065ac7 MH	182
93065ac7 MH	183	return ret;
341cb9e4 CK	184	}
	185
	186	static const struct mmu_notifier_ops radeon_mn_ops = {
	187	.release = radeon_mn_release,
	188	.invalidate_range_start = radeon_mn_invalidate_range_start,
	189	};
	190
	191	/**
	192	* radeon_mn_get - create notifier context
	193	*
	194	* @rdev: radeon device pointer
	195	*
	196	* Creates a notifier context for current->mm.
	197	*/
	198	static struct radeon_mn radeon_mn_get(struct radeon_device rdev)
	199	{
	200	struct mm_struct *mm = current->mm;
	201	struct radeon_mn *rmn;
	202	int r;
	203
2267c299 MH	204	if (down_write_killable(&mm->mmap_sem))
	205	return ERR_PTR(-EINTR);
	206
341cb9e4 CK	207	mutex_lock(&rdev->mn_lock);
	208
	209	hash_for_each_possible(rdev->mn_hash, rmn, node, (unsigned long)mm)
	210	if (rmn->mm == mm)
	211	goto release_locks;
	212
	213	rmn = kzalloc(sizeof(*rmn), GFP_KERNEL);
	214	if (!rmn) {
	215	rmn = ERR_PTR(-ENOMEM);
	216	goto release_locks;
	217	}
	218
	219	rmn->rdev = rdev;
	220	rmn->mm = mm;
	221	rmn->mn.ops = &radeon_mn_ops;
	222	mutex_init(&rmn->lock);
f808c13f	223	rmn->objects = RB_ROOT_CACHED;
341cb9e4 CK	224
	225	r = __mmu_notifier_register(&rmn->mn, mm);
	226	if (r)
	227	goto free_rmn;
	228
	229	hash_add(rdev->mn_hash, &rmn->node, (unsigned long)mm);
	230
	231	release_locks:
	232	mutex_unlock(&rdev->mn_lock);
	233	up_write(&mm->mmap_sem);
	234
	235	return rmn;
	236
	237	free_rmn:
	238	mutex_unlock(&rdev->mn_lock);
	239	up_write(&mm->mmap_sem);
	240	kfree(rmn);
	241
	242	return ERR_PTR(r);
	243	}
	244
	245	/**
	246	* radeon_mn_register - register a BO for notifier updates
	247	*
	248	* @bo: radeon buffer object
	249	* @addr: userptr addr we should monitor
	250	*
	251	* Registers an MMU notifier for the given BO at the specified address.
	252	* Returns 0 on success, -ERRNO if anything goes wrong.
	253	*/
	254	int radeon_mn_register(struct radeon_bo *bo, unsigned long addr)
	255	{
	256	unsigned long end = addr + radeon_bo_size(bo) - 1;
	257	struct radeon_device *rdev = bo->rdev;
	258	struct radeon_mn *rmn;
49ecb10e CK	259	struct radeon_mn_node *node = NULL;
49ecb10e CK	260	struct list_head bos;
341cb9e4 CK	261	struct interval_tree_node *it;
	262
	263	rmn = radeon_mn_get(rdev);
	264	if (IS_ERR(rmn))
	265	return PTR_ERR(rmn);
	266
49ecb10e CK	267	INIT_LIST_HEAD(&bos);
49ecb10e CK	268
341cb9e4 CK	269	mutex_lock(&rmn->lock);
341cb9e4 CK	270
49ecb10e CK	271	while ((it = interval_tree_iter_first(&rmn->objects, addr, end))) {
	272	kfree(node);
	273	node = container_of(it, struct radeon_mn_node, it);
	274	interval_tree_remove(&node->it, &rmn->objects);
	275	addr = min(it->start, addr);
	276	end = max(it->last, end);
	277	list_splice(&node->bos, &bos);
	278	}
	279
	280	if (!node) {
	281	node = kmalloc(sizeof(struct radeon_mn_node), GFP_KERNEL);
	282	if (!node) {
	283	mutex_unlock(&rmn->lock);
	284	return -ENOMEM;
	285	}
341cb9e4 CK	286	}
	287
	288	bo->mn = rmn;
49ecb10e CK	289
	290	node->it.start = addr;
	291	node->it.last = end;
	292	INIT_LIST_HEAD(&node->bos);
	293	list_splice(&bos, &node->bos);
	294	list_add(&bo->mn_list, &node->bos);
	295
	296	interval_tree_insert(&node->it, &rmn->objects);
341cb9e4 CK	297
	298	mutex_unlock(&rmn->lock);
	299
	300	return 0;
	301	}
	302
	303	/**
	304	* radeon_mn_unregister - unregister a BO for notifier updates
	305	*
	306	* @bo: radeon buffer object
	307	*
	308	* Remove any registration of MMU notifier updates from the buffer object.
	309	*/
	310	void radeon_mn_unregister(struct radeon_bo *bo)
	311	{
	312	struct radeon_device *rdev = bo->rdev;
	313	struct radeon_mn *rmn;
49ecb10e	314	struct list_head *head;
341cb9e4 CK	315
	316	mutex_lock(&rdev->mn_lock);
	317	rmn = bo->mn;
	318	if (rmn == NULL) {
	319	mutex_unlock(&rdev->mn_lock);
	320	return;
	321	}
	322
	323	mutex_lock(&rmn->lock);
49ecb10e CK	324	/* save the next list entry for later */
	325	head = bo->mn_list.next;
	326
341cb9e4	327	bo->mn = NULL;
49ecb10e CK	328	list_del(&bo->mn_list);
	329
	330	if (list_empty(head)) {
	331	struct radeon_mn_node *node;
	332	node = container_of(head, struct radeon_mn_node, bos);
	333	interval_tree_remove(&node->it, &rmn->objects);
	334	kfree(node);
	335	}
	336
341cb9e4 CK	337	mutex_unlock(&rmn->lock);
	338	mutex_unlock(&rdev->mn_lock);
	339	}