[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_vma_resource.h

/* SPDX-License-Identifier: MIT */
/*
 * Copyright © 2021 Intel Corporation
 */

#ifndef __I915_VMA_RESOURCE_H__
#define __I915_VMA_RESOURCE_H__

#include <linux/dma-fence.h>
#include <linux/refcount.h>

#include "i915_gem.h"
#include "i915_scatterlist.h"
#include "i915_sw_fence.h"
#include "intel_runtime_pm.h"

struct intel_memory_region;

struct i915_page_sizes {
	/**
	 * The sg mask of the pages sg_table. i.e the mask of
	 * the lengths for each sg entry.
	 */
	unsigned int phys;

	/**
	 * The gtt page sizes we are allowed to use given the
	 * sg mask and the supported page sizes. This will
	 * express the smallest unit we can use for the whole
	 * object, as well as the larger sizes we may be able
	 * to use opportunistically.
	 */
	unsigned int sg;
};

/**
 * struct i915_vma_bindinfo - Information needed for async bind
 * only but that can be dropped after the bind has taken place.
 * Consider making this a separate argument to the bind_vma
 * op, coalescing with other arguments like vm, stash, cache_level
 * and flags
 * @pages: The pages sg-table.
 * @page_sizes: Page sizes of the pages.
 * @pages_rsgt: Refcounted sg-table when delayed object destruction
 * is supported. May be NULL.
 * @readonly: Whether the vma should be bound read-only.
 * @lmem: Whether the vma points to lmem.
 */
struct i915_vma_bindinfo {
	struct sg_table *pages;
	struct i915_page_sizes page_sizes;
	struct i915_refct_sgt *pages_rsgt;
	bool readonly:1;
	bool lmem:1;
};

/**
 * struct i915_vma_resource - Snapshotted unbind information.
 * @unbind_fence: Fence to mark unbinding complete. Note that this fence
 * is not considered published until unbind is scheduled, and as such it
 * is illegal to access this fence before scheduled unbind other than
 * for refcounting.
 * @lock: The @unbind_fence lock.
 * @hold_count: Number of holders blocking the fence from finishing.
 * The vma itself is keeping a hold, which is released when unbind
 * is scheduled.
 * @work: Work struct for deferred unbind work.
 * @chain: Pointer to struct i915_sw_fence used to await dependencies.
 * @rb: Rb node for the vm's pending unbind interval tree.
 * @__subtree_last: Interval tree private member.
 * @wakeref: wakeref.
 * @vm: non-refcounted pointer to the vm. This is for internal use only and
 * this member is cleared after vm_resource unbind.
 * @mr: The memory region of the object pointed to by the vma.
 * @ops: Pointer to the backend i915_vma_ops.
 * @private: Bind backend private info.
 * @start: Offset into the address space of bind range start. Note that
 * this is after any padding that might have been allocated.
 * @node_size: Size of the allocated range manager node with padding
 * subtracted.
 * @vma_size: Bind size.
 * @guard: The size of guard area preceding and trailing the bind.
 * @page_sizes_gtt: Resulting page sizes from the bind operation.
 * @bound_flags: Flags indicating binding status.
 * @allocated: Backend private data. TODO: Should move into @private.
 * @immediate_unbind: Unbind can be done immediately and doesn't need to be
 * deferred to a work item awaiting unsignaled fences. This is a hack.
 * (dma_fence_work uses a fence flag for this, but this seems slightly
 * cleaner).
 * @needs_wakeref: Whether a wakeref is needed during unbind. Since we can't
 * take a wakeref in the dma-fence signalling critical path, it needs to be
 * taken when the unbind is scheduled.
 * @skip_pte_rewrite: During ggtt suspend and vm takedown pte rewriting
 * needs to be skipped for unbind.
 * @tlb: pointer for obj->mm.tlb, if async unbind. Otherwise, NULL
 *
 * The lifetime of a struct i915_vma_resource is from a binding request to
 * the actual possible asynchronous unbind has completed.
 */
struct i915_vma_resource {
	struct dma_fence unbind_fence;
	/* See above for description of the lock. */
	spinlock_t lock;
	refcount_t hold_count;
	struct work_struct work;
	struct i915_sw_fence chain;
	struct rb_node rb;
	u64 __subtree_last;
	struct i915_address_space *vm;
	intel_wakeref_t wakeref;

	/**
	 * @bi: Information needed for async bind only but that can be dropped
	 * after the bind has taken place.
	 *
	 * Consider making this a separate argument to the bind_vma op,
	 * coalescing with other arguments like vm, stash, cache_level and flags
	 */
	struct i915_vma_bindinfo bi;

#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
	struct intel_memory_region *mr;
#endif
	const struct i915_vma_ops *ops;
	void *private;
	u64 start;
	u64 node_size;
	u64 vma_size;
	u32 guard;
	u32 page_sizes_gtt;

	u32 bound_flags;
	bool allocated:1;
	bool immediate_unbind:1;
	bool needs_wakeref:1;
	bool skip_pte_rewrite:1;

	u32 *tlb;
};

bool i915_vma_resource_hold(struct i915_vma_resource *vma_res,
			    bool *lockdep_cookie);

void i915_vma_resource_unhold(struct i915_vma_resource *vma_res,
			      bool lockdep_cookie);

struct i915_vma_resource *i915_vma_resource_alloc(void);

void i915_vma_resource_free(struct i915_vma_resource *vma_res);

struct dma_fence *i915_vma_resource_unbind(struct i915_vma_resource *vma_res,
					   u32 *tlb);

void __i915_vma_resource_init(struct i915_vma_resource *vma_res);

/**
 * i915_vma_resource_get - Take a reference on a vma resource
 * @vma_res: The vma resource on which to take a reference.
 *
 * Return: The @vma_res pointer
 */
static inline struct i915_vma_resource
*i915_vma_resource_get(struct i915_vma_resource *vma_res)
{
	dma_fence_get(&vma_res->unbind_fence);
	return vma_res;
}

/**
 * i915_vma_resource_put - Release a reference to a struct i915_vma_resource
 * @vma_res: The resource
 */
static inline void i915_vma_resource_put(struct i915_vma_resource *vma_res)
{
	dma_fence_put(&vma_res->unbind_fence);
}

/**
 * i915_vma_resource_init - Initialize a vma resource.
 * @vma_res: The vma resource to initialize
 * @vm: Pointer to the vm.
 * @pages: The pages sg-table.
 * @page_sizes: Page sizes of the pages.
 * @pages_rsgt: Pointer to a struct i915_refct_sgt of an object with
 * delayed destruction.
 * @readonly: Whether the vma should be bound read-only.
 * @lmem: Whether the vma points to lmem.
 * @mr: The memory region of the object the vma points to.
 * @ops: The backend ops.
 * @private: Bind backend private info.
 * @start: Offset into the address space of bind range start after padding.
 * @node_size: Size of the allocated range manager node minus padding.
 * @size: Bind size.
 * @guard: The size of the guard area preceding and trailing the bind.
 *
 * Initializes a vma resource allocated using i915_vma_resource_alloc().
 * The reason for having separate allocate and initialize function is that
 * initialization may need to be performed from under a lock where
 * allocation is not allowed.
 */
static inline void i915_vma_resource_init(struct i915_vma_resource *vma_res,
					  struct i915_address_space *vm,
					  struct sg_table *pages,
					  const struct i915_page_sizes *page_sizes,
					  struct i915_refct_sgt *pages_rsgt,
					  bool readonly,
					  bool lmem,
					  struct intel_memory_region *mr,
					  const struct i915_vma_ops *ops,
					  void *private,
					  u64 start,
					  u64 node_size,
					  u64 size,
					  u32 guard)
{
	__i915_vma_resource_init(vma_res);
	vma_res->vm = vm;
	vma_res->bi.pages = pages;
	vma_res->bi.page_sizes = *page_sizes;
	if (pages_rsgt)
		vma_res->bi.pages_rsgt = i915_refct_sgt_get(pages_rsgt);
	vma_res->bi.readonly = readonly;
	vma_res->bi.lmem = lmem;
#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
	vma_res->mr = mr;
#endif
	vma_res->ops = ops;
	vma_res->private = private;
	vma_res->start = start;
	vma_res->node_size = node_size;
	vma_res->vma_size = size;
	vma_res->guard = guard;
}

static inline void i915_vma_resource_fini(struct i915_vma_resource *vma_res)
{
	GEM_BUG_ON(refcount_read(&vma_res->hold_count) != 1);
	if (vma_res->bi.pages_rsgt)
		i915_refct_sgt_put(vma_res->bi.pages_rsgt);
	i915_sw_fence_fini(&vma_res->chain);
}

int i915_vma_resource_bind_dep_sync(struct i915_address_space *vm,
				    u64 first,
				    u64 last,
				    bool intr);

int i915_vma_resource_bind_dep_await(struct i915_address_space *vm,
				     struct i915_sw_fence *sw_fence,
				     u64 first,
				     u64 last,
				     bool intr,
				     gfp_t gfp);

void i915_vma_resource_bind_dep_sync_all(struct i915_address_space *vm);

void i915_vma_resource_module_exit(void);

int i915_vma_resource_module_init(void);

#endif
Commit	Line	Data
e1a4bbb6 TH	1	/* SPDX-License-Identifier: MIT */
	2	/*
	3	* Copyright © 2021 Intel Corporation
	4	*/
	5
	6	#ifndef __I915_VMA_RESOURCE_H__
	7	#define __I915_VMA_RESOURCE_H__
	8
	9	#include <linux/dma-fence.h>
	10	#include <linux/refcount.h>
	11
39a2bd34	12	#include "i915_gem.h"
60dc43d1	13	#include "i915_scatterlist.h"
2f6b90da TH	14	#include "i915_sw_fence.h"
2f6b90da TH	15	#include "intel_runtime_pm.h"
39a2bd34	16
60dc43d1 TH	17	struct intel_memory_region;
60dc43d1 TH	18
39a2bd34 TH	19	struct i915_page_sizes {
	20	/**
	21	* The sg mask of the pages sg_table. i.e the mask of
	22	* the lengths for each sg entry.
	23	*/
	24	unsigned int phys;
	25
	26	/**
	27	* The gtt page sizes we are allowed to use given the
	28	* sg mask and the supported page sizes. This will
	29	* express the smallest unit we can use for the whole
	30	* object, as well as the larger sizes we may be able
	31	* to use opportunistically.
	32	*/
	33	unsigned int sg;
	34	};
	35
e9711213 JN	36	/**
	37	* struct i915_vma_bindinfo - Information needed for async bind
	38	* only but that can be dropped after the bind has taken place.
	39	* Consider making this a separate argument to the bind_vma
	40	* op, coalescing with other arguments like vm, stash, cache_level
	41	* and flags
	42	* @pages: The pages sg-table.
	43	* @page_sizes: Page sizes of the pages.
	44	* @pages_rsgt: Refcounted sg-table when delayed object destruction
	45	* is supported. May be NULL.
	46	* @readonly: Whether the vma should be bound read-only.
	47	* @lmem: Whether the vma points to lmem.
	48	*/
	49	struct i915_vma_bindinfo {
	50	struct sg_table *pages;
	51	struct i915_page_sizes page_sizes;
	52	struct i915_refct_sgt *pages_rsgt;
	53	bool readonly:1;
	54	bool lmem:1;
	55	};
	56
e1a4bbb6 TH	57	/**
	58	* struct i915_vma_resource - Snapshotted unbind information.
	59	* @unbind_fence: Fence to mark unbinding complete. Note that this fence
	60	* is not considered published until unbind is scheduled, and as such it
	61	* is illegal to access this fence before scheduled unbind other than
	62	* for refcounting.
	63	* @lock: The @unbind_fence lock.
	64	* @hold_count: Number of holders blocking the fence from finishing.
	65	* The vma itself is keeping a hold, which is released when unbind
	66	* is scheduled.
2f6b90da TH	67	* @work: Work struct for deferred unbind work.
	68	* @chain: Pointer to struct i915_sw_fence used to await dependencies.
	69	* @rb: Rb node for the vm's pending unbind interval tree.
	70	* @__subtree_last: Interval tree private member.
28487ecb	71	* @wakeref: wakeref.
2f6b90da TH	72	* @vm: non-refcounted pointer to the vm. This is for internal use only and
2f6b90da TH	73	* this member is cleared after vm_resource unbind.
60dc43d1	74	* @mr: The memory region of the object pointed to by the vma.
2f6b90da	75	* @ops: Pointer to the backend i915_vma_ops.
39a2bd34	76	* @private: Bind backend private info.
8e4ee5e8 CW	77	* @start: Offset into the address space of bind range start. Note that
	78	* this is after any padding that might have been allocated.
	79	* @node_size: Size of the allocated range manager node with padding
	80	* subtracted.
39a2bd34	81	* @vma_size: Bind size.
61102251	82	* @guard: The size of guard area preceding and trailing the bind.
39a2bd34 TH	83	* @page_sizes_gtt: Resulting page sizes from the bind operation.
	84	* @bound_flags: Flags indicating binding status.
	85	* @allocated: Backend private data. TODO: Should move into @private.
60dc43d1 TH	86	* @immediate_unbind: Unbind can be done immediately and doesn't need to be
	87	* deferred to a work item awaiting unsignaled fences. This is a hack.
	88	* (dma_fence_work uses a fence flag for this, but this seems slightly
	89	* cleaner).
e1a7ab4f TH	90	* @needs_wakeref: Whether a wakeref is needed during unbind. Since we can't
	91	* take a wakeref in the dma-fence signalling critical path, it needs to be
	92	* taken when the unbind is scheduled.
	93	* @skip_pte_rewrite: During ggtt suspend and vm takedown pte rewriting
	94	* needs to be skipped for unbind.
5d36acb7	95	* @tlb: pointer for obj->mm.tlb, if async unbind. Otherwise, NULL
e1a4bbb6 TH	96	*
	97	* The lifetime of a struct i915_vma_resource is from a binding request to
	98	* the actual possible asynchronous unbind has completed.
	99	*/
	100	struct i915_vma_resource {
	101	struct dma_fence unbind_fence;
	102	/* See above for description of the lock. */
	103	spinlock_t lock;
	104	refcount_t hold_count;
2f6b90da TH	105	struct work_struct work;
	106	struct i915_sw_fence chain;
	107	struct rb_node rb;
	108	u64 __subtree_last;
	109	struct i915_address_space *vm;
	110	intel_wakeref_t wakeref;
39a2bd34 TH	111
39a2bd34 TH	112	/**
e9711213 JN	113	* @bi: Information needed for async bind only but that can be dropped
	114	* after the bind has taken place.
	115	*
	116	* Consider making this a separate argument to the bind_vma op,
	117	* coalescing with other arguments like vm, stash, cache_level and flags
39a2bd34	118	*/
e9711213	119	struct i915_vma_bindinfo bi;
39a2bd34	120
60dc43d1 TH	121	#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
	122	struct intel_memory_region *mr;
	123	#endif
2f6b90da	124	const struct i915_vma_ops *ops;
39a2bd34 TH	125	void *private;
	126	u64 start;
	127	u64 node_size;
	128	u64 vma_size;
61102251	129	u32 guard;
39a2bd34	130	u32 page_sizes_gtt;
2f6b90da	131
39a2bd34 TH	132	u32 bound_flags;
39a2bd34 TH	133	bool allocated:1;
2f6b90da TH	134	bool immediate_unbind:1;
2f6b90da TH	135	bool needs_wakeref:1;
e1a7ab4f	136	bool skip_pte_rewrite:1;
5d36acb7 CW	137
5d36acb7 CW	138	u32 *tlb;
e1a4bbb6 TH	139	};
	140
	141	bool i915_vma_resource_hold(struct i915_vma_resource *vma_res,
	142	bool *lockdep_cookie);
	143
	144	void i915_vma_resource_unhold(struct i915_vma_resource *vma_res,
	145	bool lockdep_cookie);
	146
	147	struct i915_vma_resource *i915_vma_resource_alloc(void);
	148
2f6b90da TH	149	void i915_vma_resource_free(struct i915_vma_resource *vma_res);
2f6b90da TH	150
5d36acb7 CW	151	struct dma_fence i915_vma_resource_unbind(struct i915_vma_resource vma_res,
5d36acb7 CW	152	u32 *tlb);
e1a4bbb6	153
39a2bd34 TH	154	void __i915_vma_resource_init(struct i915_vma_resource *vma_res);
39a2bd34 TH	155
e1a4bbb6 TH	156	/**
	157	* i915_vma_resource_get - Take a reference on a vma resource
	158	* @vma_res: The vma resource on which to take a reference.
	159	*
	160	* Return: The @vma_res pointer
	161	*/
	162	static inline struct i915_vma_resource
	163	i915_vma_resource_get(struct i915_vma_resource vma_res)
	164	{
	165	dma_fence_get(&vma_res->unbind_fence);
	166	return vma_res;
	167	}
	168
	169	/**
	170	* i915_vma_resource_put - Release a reference to a struct i915_vma_resource
	171	* @vma_res: The resource
	172	*/
	173	static inline void i915_vma_resource_put(struct i915_vma_resource *vma_res)
	174	{
	175	dma_fence_put(&vma_res->unbind_fence);
	176	}
	177
39a2bd34 TH	178	/**
	179	* i915_vma_resource_init - Initialize a vma resource.
	180	* @vma_res: The vma resource to initialize
2f6b90da	181	* @vm: Pointer to the vm.
39a2bd34 TH	182	* @pages: The pages sg-table.
39a2bd34 TH	183	* @page_sizes: Page sizes of the pages.
60dc43d1 TH	184	* @pages_rsgt: Pointer to a struct i915_refct_sgt of an object with
60dc43d1 TH	185	* delayed destruction.
39a2bd34 TH	186	* @readonly: Whether the vma should be bound read-only.
39a2bd34 TH	187	* @lmem: Whether the vma points to lmem.
60dc43d1	188	* @mr: The memory region of the object the vma points to.
2f6b90da	189	* @ops: The backend ops.
39a2bd34	190	* @private: Bind backend private info.
8e4ee5e8 CW	191	* @start: Offset into the address space of bind range start after padding.
8e4ee5e8 CW	192	* @node_size: Size of the allocated range manager node minus padding.
39a2bd34	193	* @size: Bind size.
61102251	194	* @guard: The size of the guard area preceding and trailing the bind.
39a2bd34 TH	195	*
	196	* Initializes a vma resource allocated using i915_vma_resource_alloc().
	197	* The reason for having separate allocate and initialize function is that
	198	* initialization may need to be performed from under a lock where
	199	* allocation is not allowed.
	200	*/
	201	static inline void i915_vma_resource_init(struct i915_vma_resource *vma_res,
2f6b90da	202	struct i915_address_space *vm,
39a2bd34 TH	203	struct sg_table *pages,
39a2bd34 TH	204	const struct i915_page_sizes *page_sizes,
60dc43d1	205	struct i915_refct_sgt *pages_rsgt,
39a2bd34 TH	206	bool readonly,
39a2bd34 TH	207	bool lmem,
60dc43d1	208	struct intel_memory_region *mr,
2f6b90da	209	const struct i915_vma_ops *ops,
39a2bd34 TH	210	void *private,
	211	u64 start,
	212	u64 node_size,
61102251 CW	213	u64 size,
61102251 CW	214	u32 guard)
39a2bd34 TH	215	{
39a2bd34 TH	216	__i915_vma_resource_init(vma_res);
2f6b90da	217	vma_res->vm = vm;
39a2bd34 TH	218	vma_res->bi.pages = pages;
39a2bd34 TH	219	vma_res->bi.page_sizes = *page_sizes;
60dc43d1 TH	220	if (pages_rsgt)
60dc43d1 TH	221	vma_res->bi.pages_rsgt = i915_refct_sgt_get(pages_rsgt);
39a2bd34 TH	222	vma_res->bi.readonly = readonly;
39a2bd34 TH	223	vma_res->bi.lmem = lmem;
60dc43d1 TH	224	#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
	225	vma_res->mr = mr;
	226	#endif
2f6b90da	227	vma_res->ops = ops;
39a2bd34 TH	228	vma_res->private = private;
	229	vma_res->start = start;
	230	vma_res->node_size = node_size;
	231	vma_res->vma_size = size;
61102251	232	vma_res->guard = guard;
39a2bd34 TH	233	}
	234
	235	static inline void i915_vma_resource_fini(struct i915_vma_resource *vma_res)
	236	{
	237	GEM_BUG_ON(refcount_read(&vma_res->hold_count) != 1);
60dc43d1 TH	238	if (vma_res->bi.pages_rsgt)
60dc43d1 TH	239	i915_refct_sgt_put(vma_res->bi.pages_rsgt);
2f6b90da	240	i915_sw_fence_fini(&vma_res->chain);
39a2bd34	241	}
e1a4bbb6	242
2f6b90da TH	243	int i915_vma_resource_bind_dep_sync(struct i915_address_space *vm,
	244	u64 first,
	245	u64 last,
	246	bool intr);
	247
	248	int i915_vma_resource_bind_dep_await(struct i915_address_space *vm,
	249	struct i915_sw_fence *sw_fence,
	250	u64 first,
	251	u64 last,
	252	bool intr,
	253	gfp_t gfp);
	254
	255	void i915_vma_resource_bind_dep_sync_all(struct i915_address_space *vm);
	256
	257	void i915_vma_resource_module_exit(void);
	258
	259	int i915_vma_resource_module_init(void);
	260
e1a4bbb6	261	#endif