[linux-2.6-block.git] / drivers / gpu / drm / i915 / i915_vma.h

/*
 * Copyright © 2016 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

#ifndef __I915_VMA_H__
#define __I915_VMA_H__

#include <linux/io-mapping.h>
#include <linux/rbtree.h>

#include <drm/drm_mm.h>

#include "i915_gem_gtt.h"
#include "i915_gem_fence_reg.h"
#include "i915_gem_object.h"

#include "i915_active.h"
#include "i915_request.h"

enum i915_cache_level;

/**
 * A VMA represents a GEM BO that is bound into an address space. Therefore, a
 * VMA's presence cannot be guaranteed before binding, or after unbinding the
 * object into/from the address space.
 *
 * To make things as simple as possible (ie. no refcounting), a VMA's lifetime
 * will always be <= an objects lifetime. So object refcounting should cover us.
 */
struct i915_vma {
	struct drm_mm_node node;
	struct drm_i915_gem_object *obj;
	struct i915_address_space *vm;
	const struct i915_vma_ops *ops;
	struct drm_i915_fence_reg *fence;
	struct reservation_object *resv; /** Alias of obj->resv */
	struct sg_table *pages;
	void __iomem *iomap;
	void *private; /* owned by creator */
	u64 size;
	u64 display_alignment;
	struct i915_page_sizes page_sizes;

	u32 fence_size;
	u32 fence_alignment;

	/**
	 * Count of the number of times this vma has been opened by different
	 * handles (but same file) for execbuf, i.e. the number of aliases
	 * that exist in the ctx->handle_vmas LUT for this vma.
	 */
	unsigned int open_count;
	unsigned long flags;
	/**
	 * How many users have pinned this object in GTT space.
	 *
	 * This is a tightly bound, fairly small number of users, so we
	 * stuff inside the flags field so that we can both check for overflow
	 * and detect a no-op i915_vma_pin() in a single check, while also
	 * pinning the vma.
	 *
	 * The worst case display setup would have the same vma pinned for
	 * use on each plane on each crtc, while also building the next atomic
	 * state and holding a pin for the length of the cleanup queue. In the
	 * future, the flip queue may be increased from 1.
	 * Estimated worst case: 3 [qlen] * 4 [max crtcs] * 7 [max planes] = 84
	 *
	 * For GEM, the number of concurrent users for pwrite/pread is
	 * unbounded. For execbuffer, it is currently one but will in future
	 * be extended to allow multiple clients to pin vma concurrently.
	 *
	 * We also use suballocated pages, with each suballocation claiming
	 * its own pin on the shared vma. At present, this is limited to
	 * exclusive cachelines of a single page, so a maximum of 64 possible
	 * users.
	 */
#define I915_VMA_PIN_MASK 0xff
#define I915_VMA_PIN_OVERFLOW	BIT(8)

	/** Flags and address space this VMA is bound to */
#define I915_VMA_GLOBAL_BIND	BIT(9)
#define I915_VMA_LOCAL_BIND	BIT(10)
#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)

#define I915_VMA_GGTT		BIT(11)
#define I915_VMA_CAN_FENCE	BIT(12)
#define I915_VMA_CLOSED		BIT(13)
#define I915_VMA_USERFAULT_BIT	14
#define I915_VMA_USERFAULT	BIT(I915_VMA_USERFAULT_BIT)
#define I915_VMA_GGTT_WRITE	BIT(15)

	struct i915_active active;
	struct i915_active_request last_fence;

	/**
	 * Support different GGTT views into the same object.
	 * This means there can be multiple VMA mappings per object and per VM.
	 * i915_ggtt_view_type is used to distinguish between those entries.
	 * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also
	 * assumed in GEM functions which take no ggtt view parameter.
	 */
	struct i915_ggtt_view ggtt_view;

	/** This object's place on the active/inactive lists */
	struct list_head vm_link;

	struct list_head obj_link; /* Link in the object's VMA list */
	struct rb_node obj_node;
	struct hlist_node obj_hash;

	/** This vma's place in the execbuf reservation list */
	struct list_head exec_link;
	struct list_head reloc_link;

	/** This vma's place in the eviction list */
	struct list_head evict_link;

	struct list_head closed_link;

	/**
	 * Used for performing relocations during execbuffer insertion.
	 */
	unsigned int *exec_flags;
	struct hlist_node exec_node;
	u32 exec_handle;
};

struct i915_vma *
i915_vma_instance(struct drm_i915_gem_object *obj,
		  struct i915_address_space *vm,
		  const struct i915_ggtt_view *view);

void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags);
#define I915_VMA_RELEASE_MAP BIT(0)

static inline bool i915_vma_is_active(const struct i915_vma *vma)
{
	return !i915_active_is_idle(&vma->active);
}

int __must_check i915_vma_move_to_active(struct i915_vma *vma,
					 struct i915_request *rq,
					 unsigned int flags);

static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
{
	return vma->flags & I915_VMA_GGTT;
}

static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma)
{
	return vma->flags & I915_VMA_GGTT_WRITE;
}

static inline void i915_vma_set_ggtt_write(struct i915_vma *vma)
{
	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
	vma->flags |= I915_VMA_GGTT_WRITE;
}

static inline void i915_vma_unset_ggtt_write(struct i915_vma *vma)
{
	vma->flags &= ~I915_VMA_GGTT_WRITE;
}

void i915_vma_flush_writes(struct i915_vma *vma);

static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
{
	return vma->flags & I915_VMA_CAN_FENCE;
}

static inline bool i915_vma_is_closed(const struct i915_vma *vma)
{
	return vma->flags & I915_VMA_CLOSED;
}

static inline bool i915_vma_set_userfault(struct i915_vma *vma)
{
	GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
	return __test_and_set_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
}

static inline void i915_vma_unset_userfault(struct i915_vma *vma)
{
	return __clear_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
}

static inline bool i915_vma_has_userfault(const struct i915_vma *vma)
{
	return test_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
}

static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
{
	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
	GEM_BUG_ON(!vma->node.allocated);
	GEM_BUG_ON(upper_32_bits(vma->node.start));
	GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1));
	return lower_32_bits(vma->node.start);
}

static inline u32 i915_ggtt_pin_bias(struct i915_vma *vma)
{
	return i915_vm_to_ggtt(vma->vm)->pin_bias;
}

static inline struct i915_vma *i915_vma_get(struct i915_vma *vma)
{
	i915_gem_object_get(vma->obj);
	return vma;
}

static inline void i915_vma_put(struct i915_vma *vma)
{
	i915_gem_object_put(vma->obj);
}

static __always_inline ptrdiff_t ptrdiff(const void *a, const void *b)
{
	return a - b;
}

static inline long
i915_vma_compare(struct i915_vma *vma,
		 struct i915_address_space *vm,
		 const struct i915_ggtt_view *view)
{
	ptrdiff_t cmp;

	GEM_BUG_ON(view && !i915_is_ggtt(vm));

	cmp = ptrdiff(vma->vm, vm);
	if (cmp)
		return cmp;

	BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL != 0);
	cmp = vma->ggtt_view.type;
	if (!view)
		return cmp;

	cmp -= view->type;
	if (cmp)
		return cmp;

	assert_i915_gem_gtt_types();

	/* ggtt_view.type also encodes its size so that we both distinguish
	 * different views using it as a "type" and also use a compact (no
	 * accessing of uninitialised padding bytes) memcmp without storing
	 * an extra parameter or adding more code.
	 *
	 * To ensure that the memcmp is valid for all branches of the union,
	 * even though the code looks like it is just comparing one branch,
	 * we assert above that all branches have the same address, and that
	 * each branch has a unique type/size.
	 */
	BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL >= I915_GGTT_VIEW_PARTIAL);
	BUILD_BUG_ON(I915_GGTT_VIEW_PARTIAL >= I915_GGTT_VIEW_ROTATED);
	BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=
		     offsetof(typeof(*view), partial));
	return memcmp(&vma->ggtt_view.partial, &view->partial, view->type);
}

int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
		  u32 flags);
bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level);
bool i915_vma_misplaced(const struct i915_vma *vma,
			u64 size, u64 alignment, u64 flags);
void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
void i915_vma_revoke_mmap(struct i915_vma *vma);
int __must_check i915_vma_unbind(struct i915_vma *vma);
void i915_vma_unlink_ctx(struct i915_vma *vma);
void i915_vma_close(struct i915_vma *vma);
void i915_vma_reopen(struct i915_vma *vma);
void i915_vma_destroy(struct i915_vma *vma);

int __i915_vma_do_pin(struct i915_vma *vma,
		      u64 size, u64 alignment, u64 flags);
static inline int __must_check
i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
{
	BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
	BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
	BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);

	/* Pin early to prevent the shrinker/eviction logic from destroying
	 * our vma as we insert and bind.
	 */
	if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) {
		GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
		GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
		return 0;
	}

	return __i915_vma_do_pin(vma, size, alignment, flags);
}

static inline int i915_vma_pin_count(const struct i915_vma *vma)
{
	return vma->flags & I915_VMA_PIN_MASK;
}

static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
{
	return i915_vma_pin_count(vma);
}

static inline void __i915_vma_pin(struct i915_vma *vma)
{
	vma->flags++;
	GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
}

static inline void __i915_vma_unpin(struct i915_vma *vma)
{
	vma->flags--;
}

static inline void i915_vma_unpin(struct i915_vma *vma)
{
	GEM_BUG_ON(!i915_vma_is_pinned(vma));
	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
	__i915_vma_unpin(vma);
}

static inline bool i915_vma_is_bound(const struct i915_vma *vma,
				     unsigned int where)
{
	return vma->flags & where;
}

/**
 * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
 * @vma: VMA to iomap
 *
 * The passed in VMA has to be pinned in the global GTT mappable region.
 * An extra pinning of the VMA is acquired for the return iomapping,
 * the caller must call i915_vma_unpin_iomap to relinquish the pinning
 * after the iomapping is no longer required.
 *
 * Callers must hold the struct_mutex.
 *
 * Returns a valid iomapped pointer or ERR_PTR.
 */
void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))

/**
 * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
 * @vma: VMA to unpin
 *
 * Unpins the previously iomapped VMA from i915_vma_pin_iomap().
 *
 * Callers must hold the struct_mutex. This function is only valid to be
 * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap().
 */
void i915_vma_unpin_iomap(struct i915_vma *vma);

static inline struct page *i915_vma_first_page(struct i915_vma *vma)
{
	GEM_BUG_ON(!vma->pages);
	return sg_page(vma->pages->sgl);
}

/**
 * i915_vma_pin_fence - pin fencing state
 * @vma: vma to pin fencing for
 *
 * This pins the fencing state (whether tiled or untiled) to make sure the
 * vma (and its object) is ready to be used as a scanout target. Fencing
 * status must be synchronize first by calling i915_vma_get_fence():
 *
 * The resulting fence pin reference must be released again with
 * i915_vma_unpin_fence().
 *
 * Returns:
 *
 * True if the vma has a fence, false otherwise.
 */
int i915_vma_pin_fence(struct i915_vma *vma);
int __must_check i915_vma_put_fence(struct i915_vma *vma);

static inline void __i915_vma_unpin_fence(struct i915_vma *vma)
{
	GEM_BUG_ON(vma->fence->pin_count <= 0);
	vma->fence->pin_count--;
}

/**
 * i915_vma_unpin_fence - unpin fencing state
 * @vma: vma to unpin fencing for
 *
 * This releases the fence pin reference acquired through
 * i915_vma_pin_fence. It will handle both objects with and without an
 * attached fence correctly, callers do not need to distinguish this.
 */
static inline void
i915_vma_unpin_fence(struct i915_vma *vma)
{
	/* lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); */
	if (vma->fence)
		__i915_vma_unpin_fence(vma);
}

void i915_vma_parked(struct drm_i915_private *i915);

#define for_each_until(cond) if (cond) break; else

/**
 * for_each_ggtt_vma - Iterate over the GGTT VMA belonging to an object.
 * @V: the #i915_vma iterator
 * @OBJ: the #drm_i915_gem_object
 *
 * GGTT VMA are placed at the being of the object's vma_list, see
 * vma_create(), so we can stop our walk as soon as we see a ppgtt VMA,
 * or the list is empty ofc.
 */
#define for_each_ggtt_vma(V, OBJ) \
	list_for_each_entry(V, &(OBJ)->vma.list, obj_link)		\
		for_each_until(!i915_vma_is_ggtt(V))

struct i915_vma *i915_vma_alloc(void);
void i915_vma_free(struct i915_vma *vma);

#endif
Commit	Line	Data
b42fe9ca JL	1	/*
	2	* Copyright © 2016 Intel Corporation
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice (including the next
	12	* paragraph) shall be included in all copies or substantial portions of the
	13	* Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
	21	* IN THE SOFTWARE.
	22	*
	23	*/
	24
	25	#ifndef __I915_VMA_H__
	26	#define __I915_VMA_H__
	27
	28	#include <linux/io-mapping.h>
5c3f8c22	29	#include <linux/rbtree.h>
b42fe9ca JL	30
	31	#include <drm/drm_mm.h>
	32
	33	#include "i915_gem_gtt.h"
	34	#include "i915_gem_fence_reg.h"
	35	#include "i915_gem_object.h"
b42fe9ca	36
64d6c500	37	#include "i915_active.h"
e61e0f51	38	#include "i915_request.h"
b42fe9ca JL	39
	40	enum i915_cache_level;
	41
	42	/**
	43	* A VMA represents a GEM BO that is bound into an address space. Therefore, a
	44	* VMA's presence cannot be guaranteed before binding, or after unbinding the
	45	* object into/from the address space.
	46	*
	47	* To make things as simple as possible (ie. no refcounting), a VMA's lifetime
	48	* will always be <= an objects lifetime. So object refcounting should cover us.
	49	*/
	50	struct i915_vma {
	51	struct drm_mm_node node;
	52	struct drm_i915_gem_object *obj;
	53	struct i915_address_space *vm;
93f2cde2	54	const struct i915_vma_ops *ops;
b42fe9ca	55	struct drm_i915_fence_reg *fence;
95ff7c7d	56	struct reservation_object resv; /* Alias of obj->resv */
b42fe9ca JL	57	struct sg_table *pages;
b42fe9ca JL	58	void __iomem *iomap;
e9e7dc41	59	void private; / owned by creator */
b42fe9ca JL	60	u64 size;
b42fe9ca JL	61	u64 display_alignment;
7464284b	62	struct i915_page_sizes page_sizes;
b42fe9ca	63
944397f0 CW	64	u32 fence_size;
	65	u32 fence_alignment;
	66
3ffff017 CW	67	/**
	68	* Count of the number of times this vma has been opened by different
	69	* handles (but same file) for execbuf, i.e. the number of aliases
	70	* that exist in the ctx->handle_vmas LUT for this vma.
	71	*/
	72	unsigned int open_count;
a65adaf8	73	unsigned long flags;
b42fe9ca	74	/**
b18fe4be	75	* How many users have pinned this object in GTT space.
b42fe9ca	76	*
b18fe4be CW	77	* This is a tightly bound, fairly small number of users, so we
	78	* stuff inside the flags field so that we can both check for overflow
	79	* and detect a no-op i915_vma_pin() in a single check, while also
	80	* pinning the vma.
	81	*
	82	* The worst case display setup would have the same vma pinned for
	83	* use on each plane on each crtc, while also building the next atomic
	84	* state and holding a pin for the length of the cleanup queue. In the
	85	* future, the flip queue may be increased from 1.
	86	* Estimated worst case: 3 [qlen] * 4 [max crtcs] * 7 [max planes] = 84
	87	*
	88	* For GEM, the number of concurrent users for pwrite/pread is
	89	* unbounded. For execbuffer, it is currently one but will in future
	90	* be extended to allow multiple clients to pin vma concurrently.
	91	*
	92	* We also use suballocated pages, with each suballocation claiming
	93	* its own pin on the shared vma. At present, this is limited to
	94	* exclusive cachelines of a single page, so a maximum of 64 possible
	95	* users.
b42fe9ca	96	*/
b18fe4be CW	97	#define I915_VMA_PIN_MASK 0xff
b18fe4be CW	98	#define I915_VMA_PIN_OVERFLOW BIT(8)
b42fe9ca JL	99
b42fe9ca JL	100	/** Flags and address space this VMA is bound to */
b18fe4be CW	101	#define I915_VMA_GLOBAL_BIND BIT(9)
b18fe4be CW	102	#define I915_VMA_LOCAL_BIND BIT(10)
b42fe9ca JL	103	#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND \| I915_VMA_LOCAL_BIND \| I915_VMA_PIN_OVERFLOW)
b42fe9ca JL	104
b18fe4be CW	105	#define I915_VMA_GGTT BIT(11)
	106	#define I915_VMA_CAN_FENCE BIT(12)
	107	#define I915_VMA_CLOSED BIT(13)
	108	#define I915_VMA_USERFAULT_BIT 14
a65adaf8	109	#define I915_VMA_USERFAULT BIT(I915_VMA_USERFAULT_BIT)
b18fe4be	110	#define I915_VMA_GGTT_WRITE BIT(15)
b42fe9ca	111
64d6c500	112	struct i915_active active;
21950ee7	113	struct i915_active_request last_fence;
b42fe9ca JL	114
	115	/**
	116	* Support different GGTT views into the same object.
	117	* This means there can be multiple VMA mappings per object and per VM.
	118	* i915_ggtt_view_type is used to distinguish between those entries.
	119	* The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also
	120	* assumed in GEM functions which take no ggtt view parameter.
	121	*/
	122	struct i915_ggtt_view ggtt_view;
	123
	124	/** This object's place on the active/inactive lists */
	125	struct list_head vm_link;
	126
	127	struct list_head obj_link; /* Link in the object's VMA list */
	128	struct rb_node obj_node;
4ff4b44c	129	struct hlist_node obj_hash;
b42fe9ca	130
8c45cec4 CW	131	/** This vma's place in the execbuf reservation list */
8c45cec4 CW	132	struct list_head exec_link;
2889caa9	133	struct list_head reloc_link;
8c45cec4 CW	134
	135	/** This vma's place in the eviction list */
	136	struct list_head evict_link;
b42fe9ca	137
3365e226 CW	138	struct list_head closed_link;
3365e226 CW	139
b42fe9ca JL	140	/**
	141	* Used for performing relocations during execbuffer insertion.
	142	*/
c7c6e46f	143	unsigned int *exec_flags;
95ff7c7d	144	struct hlist_node exec_node;
4ff4b44c	145	u32 exec_handle;
b42fe9ca JL	146	};
b42fe9ca JL	147
718659a6 CW	148	struct i915_vma *
	149	i915_vma_instance(struct drm_i915_gem_object *obj,
	150	struct i915_address_space *vm,
	151	const struct i915_ggtt_view *view);
	152
6a2f59e4 CW	153	void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags);
6a2f59e4 CW	154	#define I915_VMA_RELEASE_MAP BIT(0)
b42fe9ca	155
64d6c500	156	static inline bool i915_vma_is_active(const struct i915_vma *vma)
5c3f8c22	157	{
64d6c500	158	return !i915_active_is_idle(&vma->active);
5c3f8c22 CW	159	}
	160
	161	int __must_check i915_vma_move_to_active(struct i915_vma *vma,
	162	struct i915_request *rq,
	163	unsigned int flags);
	164
b42fe9ca JL	165	static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
	166	{
	167	return vma->flags & I915_VMA_GGTT;
	168	}
	169
7125397b CW	170	static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma)
	171	{
	172	return vma->flags & I915_VMA_GGTT_WRITE;
	173	}
	174
	175	static inline void i915_vma_set_ggtt_write(struct i915_vma *vma)
	176	{
	177	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
	178	vma->flags \|= I915_VMA_GGTT_WRITE;
	179	}
	180
	181	static inline void i915_vma_unset_ggtt_write(struct i915_vma *vma)
	182	{
	183	vma->flags &= ~I915_VMA_GGTT_WRITE;
	184	}
	185
	186	void i915_vma_flush_writes(struct i915_vma *vma);
	187
b42fe9ca JL	188	static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
	189	{
	190	return vma->flags & I915_VMA_CAN_FENCE;
	191	}
	192
	193	static inline bool i915_vma_is_closed(const struct i915_vma *vma)
	194	{
	195	return vma->flags & I915_VMA_CLOSED;
	196	}
	197
a65adaf8 CW	198	static inline bool i915_vma_set_userfault(struct i915_vma *vma)
	199	{
	200	GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
	201	return __test_and_set_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
	202	}
	203
	204	static inline void i915_vma_unset_userfault(struct i915_vma *vma)
	205	{
	206	return __clear_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
	207	}
	208
	209	static inline bool i915_vma_has_userfault(const struct i915_vma *vma)
	210	{
	211	return test_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
	212	}
	213
b42fe9ca JL	214	static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
	215	{
	216	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
	217	GEM_BUG_ON(!vma->node.allocated);
	218	GEM_BUG_ON(upper_32_bits(vma->node.start));
	219	GEM_BUG_ON(upper_32_bits(vma->node.start + vma->node.size - 1));
	220	return lower_32_bits(vma->node.start);
	221	}
	222
dd18cedf JB	223	static inline u32 i915_ggtt_pin_bias(struct i915_vma *vma)
	224	{
	225	return i915_vm_to_ggtt(vma->vm)->pin_bias;
	226	}
	227
b42fe9ca JL	228	static inline struct i915_vma i915_vma_get(struct i915_vma vma)
	229	{
	230	i915_gem_object_get(vma->obj);
	231	return vma;
	232	}
	233
	234	static inline void i915_vma_put(struct i915_vma *vma)
	235	{
	236	i915_gem_object_put(vma->obj);
	237	}
	238
2bf0d267 TU	239	static __always_inline ptrdiff_t ptrdiff(const void a, const void b)
	240	{
	241	return a - b;
	242	}
	243
b42fe9ca JL	244	static inline long
	245	i915_vma_compare(struct i915_vma *vma,
	246	struct i915_address_space *vm,
	247	const struct i915_ggtt_view *view)
	248	{
2bf0d267 TU	249	ptrdiff_t cmp;
2bf0d267 TU	250
4d1368f7	251	GEM_BUG_ON(view && !i915_is_ggtt(vm));
b42fe9ca	252
2bf0d267 TU	253	cmp = ptrdiff(vma->vm, vm);
	254	if (cmp)
	255	return cmp;
b42fe9ca	256
992e418d CW	257	BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL != 0);
992e418d CW	258	cmp = vma->ggtt_view.type;
b42fe9ca	259	if (!view)
992e418d	260	return cmp;
b42fe9ca	261
992e418d CW	262	cmp -= view->type;
	263	if (cmp)
	264	return cmp;
b42fe9ca	265
ed11e415 VS	266	assert_i915_gem_gtt_types();
ed11e415 VS	267
992e418d CW	268	/* ggtt_view.type also encodes its size so that we both distinguish
	269	* different views using it as a "type" and also use a compact (no
	270	* accessing of uninitialised padding bytes) memcmp without storing
	271	* an extra parameter or adding more code.
8bab1193 CW	272	*
	273	* To ensure that the memcmp is valid for all branches of the union,
	274	* even though the code looks like it is just comparing one branch,
	275	* we assert above that all branches have the same address, and that
	276	* each branch has a unique type/size.
992e418d CW	277	*/
	278	BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL >= I915_GGTT_VIEW_PARTIAL);
	279	BUILD_BUG_ON(I915_GGTT_VIEW_PARTIAL >= I915_GGTT_VIEW_ROTATED);
8bab1193 CW	280	BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=
	281	offsetof(typeof(*view), partial));
	282	return memcmp(&vma->ggtt_view.partial, &view->partial, view->type);
b42fe9ca JL	283	}
	284
	285	int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
	286	u32 flags);
	287	bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level);
782a3e9e CW	288	bool i915_vma_misplaced(const struct i915_vma *vma,
782a3e9e CW	289	u64 size, u64 alignment, u64 flags);
b42fe9ca	290	void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
a65adaf8	291	void i915_vma_revoke_mmap(struct i915_vma *vma);
b42fe9ca	292	int __must_check i915_vma_unbind(struct i915_vma *vma);
4ff4b44c	293	void i915_vma_unlink_ctx(struct i915_vma *vma);
b42fe9ca	294	void i915_vma_close(struct i915_vma *vma);
3365e226 CW	295	void i915_vma_reopen(struct i915_vma *vma);
3365e226 CW	296	void i915_vma_destroy(struct i915_vma *vma);
b42fe9ca JL	297
	298	int __i915_vma_do_pin(struct i915_vma *vma,
	299	u64 size, u64 alignment, u64 flags);
	300	static inline int __must_check
	301	i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
	302	{
	303	BUILD_BUG_ON(PIN_MBZ != I915_VMA_PIN_OVERFLOW);
	304	BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
	305	BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
	306
	307	/* Pin early to prevent the shrinker/eviction logic from destroying
	308	* our vma as we insert and bind.
	309	*/
0325701a CW	310	if (likely(((++vma->flags ^ flags) & I915_VMA_BIND_MASK) == 0)) {
	311	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
	312	GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
b42fe9ca	313	return 0;
0325701a	314	}
b42fe9ca JL	315
	316	return __i915_vma_do_pin(vma, size, alignment, flags);
	317	}
	318
	319	static inline int i915_vma_pin_count(const struct i915_vma *vma)
	320	{
	321	return vma->flags & I915_VMA_PIN_MASK;
	322	}
	323
	324	static inline bool i915_vma_is_pinned(const struct i915_vma *vma)
	325	{
	326	return i915_vma_pin_count(vma);
	327	}
	328
	329	static inline void __i915_vma_pin(struct i915_vma *vma)
	330	{
	331	vma->flags++;
	332	GEM_BUG_ON(vma->flags & I915_VMA_PIN_OVERFLOW);
	333	}
	334
	335	static inline void __i915_vma_unpin(struct i915_vma *vma)
	336	{
b42fe9ca JL	337	vma->flags--;
	338	}
	339
	340	static inline void i915_vma_unpin(struct i915_vma *vma)
	341	{
67fddd90	342	GEM_BUG_ON(!i915_vma_is_pinned(vma));
b42fe9ca JL	343	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
	344	__i915_vma_unpin(vma);
	345	}
	346
e9e7dc41 CW	347	static inline bool i915_vma_is_bound(const struct i915_vma *vma,
	348	unsigned int where)
	349	{
	350	return vma->flags & where;
	351	}
	352
b42fe9ca JL	353	/**
	354	* i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
	355	* @vma: VMA to iomap
	356	*
	357	* The passed in VMA has to be pinned in the global GTT mappable region.
	358	* An extra pinning of the VMA is acquired for the return iomapping,
	359	* the caller must call i915_vma_unpin_iomap to relinquish the pinning
	360	* after the iomapping is no longer required.
	361	*
	362	* Callers must hold the struct_mutex.
	363	*
	364	* Returns a valid iomapped pointer or ERR_PTR.
	365	*/
	366	void __iomem i915_vma_pin_iomap(struct i915_vma vma);
	367	#define IO_ERR_PTR(x) ((void __iomem *)ERR_PTR(x))
	368
	369	/**
	370	* i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
	371	* @vma: VMA to unpin
	372	*
	373	* Unpins the previously iomapped VMA from i915_vma_pin_iomap().
	374	*
	375	* Callers must hold the struct_mutex. This function is only valid to be
	376	* called on a VMA previously iomapped by the caller with i915_vma_pin_iomap().
	377	*/
b4563f59	378	void i915_vma_unpin_iomap(struct i915_vma *vma);
b42fe9ca JL	379
	380	static inline struct page i915_vma_first_page(struct i915_vma vma)
	381	{
	382	GEM_BUG_ON(!vma->pages);
	383	return sg_page(vma->pages->sgl);
	384	}
	385
	386	/**
	387	* i915_vma_pin_fence - pin fencing state
	388	* @vma: vma to pin fencing for
	389	*
	390	* This pins the fencing state (whether tiled or untiled) to make sure the
	391	* vma (and its object) is ready to be used as a scanout target. Fencing
	392	* status must be synchronize first by calling i915_vma_get_fence():
	393	*
	394	* The resulting fence pin reference must be released again with
	395	* i915_vma_unpin_fence().
	396	*
	397	* Returns:
	398	*
	399	* True if the vma has a fence, false otherwise.
	400	*/
3bd40735 CW	401	int i915_vma_pin_fence(struct i915_vma *vma);
	402	int __must_check i915_vma_put_fence(struct i915_vma *vma);
	403
	404	static inline void __i915_vma_unpin_fence(struct i915_vma *vma)
b42fe9ca	405	{
3bd40735 CW	406	GEM_BUG_ON(vma->fence->pin_count <= 0);
3bd40735 CW	407	vma->fence->pin_count--;
b42fe9ca JL	408	}
	409
	410	/**
	411	* i915_vma_unpin_fence - unpin fencing state
	412	* @vma: vma to unpin fencing for
	413	*
	414	* This releases the fence pin reference acquired through
	415	* i915_vma_pin_fence. It will handle both objects with and without an
	416	* attached fence correctly, callers do not need to distinguish this.
	417	*/
	418	static inline void
	419	i915_vma_unpin_fence(struct i915_vma *vma)
	420	{
520ea7c5	421	/* lockdep_assert_held(&vma->vm->i915->drm.struct_mutex); */
3bd40735 CW	422	if (vma->fence)
3bd40735 CW	423	__i915_vma_unpin_fence(vma);
b42fe9ca JL	424	}
b42fe9ca JL	425
3365e226 CW	426	void i915_vma_parked(struct drm_i915_private *i915);
3365e226 CW	427
e2189dd0 CW	428	#define for_each_until(cond) if (cond) break; else
	429
	430	/**
	431	* for_each_ggtt_vma - Iterate over the GGTT VMA belonging to an object.
	432	* @V: the #i915_vma iterator
	433	* @OBJ: the #drm_i915_gem_object
	434	*
	435	* GGTT VMA are placed at the being of the object's vma_list, see
	436	* vma_create(), so we can stop our walk as soon as we see a ppgtt VMA,
	437	* or the list is empty ofc.
	438	*/
	439	#define for_each_ggtt_vma(V, OBJ) \
528cbd17	440	list_for_each_entry(V, &(OBJ)->vma.list, obj_link) \
e2189dd0	441	for_each_until(!i915_vma_is_ggtt(V))
b42fe9ca	442
13f1bfd3 CW	443	struct i915_vma *i915_vma_alloc(void);
	444	void i915_vma_free(struct i915_vma *vma);
	445
e2189dd0	446	#endif