[linux-block.git] / drivers / gpu / drm / i915 / gt / intel_gtt.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2020 Intel Corporation
 */

#include <linux/slab.h> /* fault-inject.h is not standalone! */

#include <linux/fault-inject.h>
#include <linux/sched/mm.h>

#include <drm/drm_cache.h>

#include "gem/i915_gem_lmem.h"
#include "i915_trace.h"
#include "intel_gt.h"
#include "intel_gtt.h"

struct drm_i915_gem_object *alloc_pt_lmem(struct i915_address_space *vm, int sz)
{
	struct drm_i915_gem_object *obj;

	/*
	 * To avoid severe over-allocation when dealing with min_page_size
	 * restrictions, we override that behaviour here by allowing an object
	 * size and page layout which can be smaller. In practice this should be
	 * totally fine, since GTT paging structures are not typically inserted
	 * into the GTT.
	 *
	 * Note that we also hit this path for the scratch page, and for this
	 * case it might need to be 64K, but that should work fine here since we
	 * used the passed in size for the page size, which should ensure it
	 * also has the same alignment.
	 */
	obj = __i915_gem_object_create_lmem_with_ps(vm->i915, sz, sz,
						    vm->lmem_pt_obj_flags);
	/*
	 * Ensure all paging structures for this vm share the same dma-resv
	 * object underneath, with the idea that one object_lock() will lock
	 * them all at once.
	 */
	if (!IS_ERR(obj)) {
		obj->base.resv = i915_vm_resv_get(vm);
		obj->shares_resv_from = vm;
	}

	return obj;
}

struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz)
{
	struct drm_i915_gem_object *obj;

	if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
		i915_gem_shrink_all(vm->i915);

	obj = i915_gem_object_create_internal(vm->i915, sz);
	/*
	 * Ensure all paging structures for this vm share the same dma-resv
	 * object underneath, with the idea that one object_lock() will lock
	 * them all at once.
	 */
	if (!IS_ERR(obj)) {
		obj->base.resv = i915_vm_resv_get(vm);
		obj->shares_resv_from = vm;
	}

	return obj;
}

int map_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj)
{
	enum i915_map_type type;
	void *vaddr;

	type = i915_coherent_map_type(vm->i915, obj, true);
	vaddr = i915_gem_object_pin_map_unlocked(obj, type);
	if (IS_ERR(vaddr))
		return PTR_ERR(vaddr);

	i915_gem_object_make_unshrinkable(obj);
	return 0;
}

int map_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj)
{
	enum i915_map_type type;
	void *vaddr;

	type = i915_coherent_map_type(vm->i915, obj, true);
	vaddr = i915_gem_object_pin_map(obj, type);
	if (IS_ERR(vaddr))
		return PTR_ERR(vaddr);

	i915_gem_object_make_unshrinkable(obj);
	return 0;
}

void __i915_vm_close(struct i915_address_space *vm)
{
	struct i915_vma *vma, *vn;

	if (!atomic_dec_and_mutex_lock(&vm->open, &vm->mutex))
		return;

	list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
		struct drm_i915_gem_object *obj = vma->obj;

		/* Keep the obj (and hence the vma) alive as _we_ destroy it */
		if (!kref_get_unless_zero(&obj->base.refcount))
			continue;

		atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
		WARN_ON(__i915_vma_unbind(vma));
		__i915_vma_put(vma);

		i915_gem_object_put(obj);
	}
	GEM_BUG_ON(!list_empty(&vm->bound_list));

	mutex_unlock(&vm->mutex);
}

/* lock the vm into the current ww, if we lock one, we lock all */
int i915_vm_lock_objects(struct i915_address_space *vm,
			 struct i915_gem_ww_ctx *ww)
{
	if (vm->scratch[0]->base.resv == &vm->_resv) {
		return i915_gem_object_lock(vm->scratch[0], ww);
	} else {
		struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);

		/* We borrowed the scratch page from ggtt, take the top level object */
		return i915_gem_object_lock(ppgtt->pd->pt.base, ww);
	}
}

void i915_address_space_fini(struct i915_address_space *vm)
{
	drm_mm_takedown(&vm->mm);
	mutex_destroy(&vm->mutex);
}

/**
 * i915_vm_resv_release - Final struct i915_address_space destructor
 * @kref: Pointer to the &i915_address_space.resv_ref member.
 *
 * This function is called when the last lock sharer no longer shares the
 * &i915_address_space._resv lock.
 */
void i915_vm_resv_release(struct kref *kref)
{
	struct i915_address_space *vm =
		container_of(kref, typeof(*vm), resv_ref);

	dma_resv_fini(&vm->_resv);
	kfree(vm);
}

static void __i915_vm_release(struct work_struct *work)
{
	struct i915_address_space *vm =
		container_of(work, struct i915_address_space, release_work);

	vm->cleanup(vm);
	i915_address_space_fini(vm);

	i915_vm_resv_put(vm);
}

void i915_vm_release(struct kref *kref)
{
	struct i915_address_space *vm =
		container_of(kref, struct i915_address_space, ref);

	GEM_BUG_ON(i915_is_ggtt(vm));
	trace_i915_ppgtt_release(vm);

	queue_work(vm->i915->wq, &vm->release_work);
}

void i915_address_space_init(struct i915_address_space *vm, int subclass)
{
	kref_init(&vm->ref);

	/*
	 * Special case for GGTT that has already done an early
	 * kref_init here.
	 */
	if (!kref_read(&vm->resv_ref))
		kref_init(&vm->resv_ref);

	INIT_WORK(&vm->release_work, __i915_vm_release);
	atomic_set(&vm->open, 1);

	/*
	 * The vm->mutex must be reclaim safe (for use in the shrinker).
	 * Do a dummy acquire now under fs_reclaim so that any allocation
	 * attempt holding the lock is immediately reported by lockdep.
	 */
	mutex_init(&vm->mutex);
	lockdep_set_subclass(&vm->mutex, subclass);

	if (!intel_vm_no_concurrent_access_wa(vm->i915)) {
		i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex);
	} else {
		/*
		 * CHV + BXT VTD workaround use stop_machine(),
		 * which is allowed to allocate memory. This means &vm->mutex
		 * is the outer lock, and in theory we can allocate memory inside
		 * it through stop_machine().
		 *
		 * Add the annotation for this, we use trylock in shrinker.
		 */
		mutex_acquire(&vm->mutex.dep_map, 0, 0, _THIS_IP_);
		might_alloc(GFP_KERNEL);
		mutex_release(&vm->mutex.dep_map, _THIS_IP_);
	}
	dma_resv_init(&vm->_resv);

	GEM_BUG_ON(!vm->total);
	drm_mm_init(&vm->mm, 0, vm->total);
	vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;

	INIT_LIST_HEAD(&vm->bound_list);
}

void *__px_vaddr(struct drm_i915_gem_object *p)
{
	enum i915_map_type type;

	GEM_BUG_ON(!i915_gem_object_has_pages(p));
	return page_unpack_bits(p->mm.mapping, &type);
}

dma_addr_t __px_dma(struct drm_i915_gem_object *p)
{
	GEM_BUG_ON(!i915_gem_object_has_pages(p));
	return sg_dma_address(p->mm.pages->sgl);
}

struct page *__px_page(struct drm_i915_gem_object *p)
{
	GEM_BUG_ON(!i915_gem_object_has_pages(p));
	return sg_page(p->mm.pages->sgl);
}

void
fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count)
{
	void *vaddr = __px_vaddr(p);

	memset64(vaddr, val, count);
	clflush_cache_range(vaddr, PAGE_SIZE);
}

static void poison_scratch_page(struct drm_i915_gem_object *scratch)
{
	void *vaddr = __px_vaddr(scratch);
	u8 val;

	val = 0;
	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
		val = POISON_FREE;

	memset(vaddr, val, scratch->base.size);
	drm_clflush_virt_range(vaddr, scratch->base.size);
}

int setup_scratch_page(struct i915_address_space *vm)
{
	unsigned long size;

	/*
	 * In order to utilize 64K pages for an object with a size < 2M, we will
	 * need to support a 64K scratch page, given that every 16th entry for a
	 * page-table operating in 64K mode must point to a properly aligned 64K
	 * region, including any PTEs which happen to point to scratch.
	 *
	 * This is only relevant for the 48b PPGTT where we support
	 * huge-gtt-pages, see also i915_vma_insert(). However, as we share the
	 * scratch (read-only) between all vm, we create one 64k scratch page
	 * for all.
	 */
	size = I915_GTT_PAGE_SIZE_4K;
	if (i915_vm_is_4lvl(vm) &&
	    HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K))
		size = I915_GTT_PAGE_SIZE_64K;

	do {
		struct drm_i915_gem_object *obj;

		obj = vm->alloc_scratch_dma(vm, size);
		if (IS_ERR(obj))
			goto skip;

		if (map_pt_dma(vm, obj))
			goto skip_obj;

		/* We need a single contiguous page for our scratch */
		if (obj->mm.page_sizes.sg < size)
			goto skip_obj;

		/* And it needs to be correspondingly aligned */
		if (__px_dma(obj) & (size - 1))
			goto skip_obj;

		/*
		 * Use a non-zero scratch page for debugging.
		 *
		 * We want a value that should be reasonably obvious
		 * to spot in the error state, while also causing a GPU hang
		 * if executed. We prefer using a clear page in production, so
		 * should it ever be accidentally used, the effect should be
		 * fairly benign.
		 */
		poison_scratch_page(obj);

		vm->scratch[0] = obj;
		vm->scratch_order = get_order(size);
		return 0;

skip_obj:
		i915_gem_object_put(obj);
skip:
		if (size == I915_GTT_PAGE_SIZE_4K)
			return -ENOMEM;

		/*
		 * If we need 64K minimum GTT pages for device local-memory,
		 * like on XEHPSDV, then we need to fail the allocation here,
		 * otherwise we can't safely support the insertion of
		 * local-memory pages for this vm, since the HW expects the
		 * correct physical alignment and size when the page-table is
		 * operating in 64K GTT mode, which includes any scratch PTEs,
		 * since userspace can still touch them.
		 */
		if (HAS_64K_PAGES(vm->i915))
			return -ENOMEM;

		size = I915_GTT_PAGE_SIZE_4K;
	} while (1);
}

void free_scratch(struct i915_address_space *vm)
{
	int i;

	for (i = 0; i <= vm->top; i++)
		i915_gem_object_put(vm->scratch[i]);
}

void gtt_write_workarounds(struct intel_gt *gt)
{
	struct drm_i915_private *i915 = gt->i915;
	struct intel_uncore *uncore = gt->uncore;

	/*
	 * This function is for gtt related workarounds. This function is
	 * called on driver load and after a GPU reset, so you can place
	 * workarounds here even if they get overwritten by GPU reset.
	 */
	/* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl,icl */
	if (IS_BROADWELL(i915))
		intel_uncore_write(uncore,
				   GEN8_L3_LRA_1_GPGPU,
				   GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
	else if (IS_CHERRYVIEW(i915))
		intel_uncore_write(uncore,
				   GEN8_L3_LRA_1_GPGPU,
				   GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
	else if (IS_GEN9_LP(i915))
		intel_uncore_write(uncore,
				   GEN8_L3_LRA_1_GPGPU,
				   GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
	else if (GRAPHICS_VER(i915) >= 9 && GRAPHICS_VER(i915) <= 11)
		intel_uncore_write(uncore,
				   GEN8_L3_LRA_1_GPGPU,
				   GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);

	/*
	 * To support 64K PTEs we need to first enable the use of the
	 * Intermediate-Page-Size(IPS) bit of the PDE field via some magical
	 * mmio, otherwise the page-walker will simply ignore the IPS bit. This
	 * shouldn't be needed after GEN10.
	 *
	 * 64K pages were first introduced from BDW+, although technically they
	 * only *work* from gen9+. For pre-BDW we instead have the option for
	 * 32K pages, but we don't currently have any support for it in our
	 * driver.
	 */
	if (HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K) &&
	    GRAPHICS_VER(i915) <= 10)
		intel_uncore_rmw(uncore,
				 GEN8_GAMW_ECO_DEV_RW_IA,
				 0,
				 GAMW_ECO_ENABLE_64K_IPS_FIELD);

	if (IS_GRAPHICS_VER(i915, 8, 11)) {
		bool can_use_gtt_cache = true;

		/*
		 * According to the BSpec if we use 2M/1G pages then we also
		 * need to disable the GTT cache. At least on BDW we can see
		 * visual corruption when using 2M pages, and not disabling the
		 * GTT cache.
		 */
		if (HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_2M))
			can_use_gtt_cache = false;

		/* WaGttCachingOffByDefault */
		intel_uncore_write(uncore,
				   HSW_GTT_CACHE_EN,
				   can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0);
		drm_WARN_ON_ONCE(&i915->drm, can_use_gtt_cache &&
				 intel_uncore_read(uncore,
						   HSW_GTT_CACHE_EN) == 0);
	}
}

static void tgl_setup_private_ppat(struct intel_uncore *uncore)
{
	/* TGL doesn't support LLC or AGE settings */
	intel_uncore_write(uncore, GEN12_PAT_INDEX(0), GEN8_PPAT_WB);
	intel_uncore_write(uncore, GEN12_PAT_INDEX(1), GEN8_PPAT_WC);
	intel_uncore_write(uncore, GEN12_PAT_INDEX(2), GEN8_PPAT_WT);
	intel_uncore_write(uncore, GEN12_PAT_INDEX(3), GEN8_PPAT_UC);
	intel_uncore_write(uncore, GEN12_PAT_INDEX(4), GEN8_PPAT_WB);
	intel_uncore_write(uncore, GEN12_PAT_INDEX(5), GEN8_PPAT_WB);
	intel_uncore_write(uncore, GEN12_PAT_INDEX(6), GEN8_PPAT_WB);
	intel_uncore_write(uncore, GEN12_PAT_INDEX(7), GEN8_PPAT_WB);
}

static void icl_setup_private_ppat(struct intel_uncore *uncore)
{
	intel_uncore_write(uncore,
			   GEN10_PAT_INDEX(0),
			   GEN8_PPAT_WB | GEN8_PPAT_LLC);
	intel_uncore_write(uncore,
			   GEN10_PAT_INDEX(1),
			   GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
	intel_uncore_write(uncore,
			   GEN10_PAT_INDEX(2),
			   GEN8_PPAT_WB | GEN8_PPAT_ELLC_OVERRIDE);
	intel_uncore_write(uncore,
			   GEN10_PAT_INDEX(3),
			   GEN8_PPAT_UC);
	intel_uncore_write(uncore,
			   GEN10_PAT_INDEX(4),
			   GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
	intel_uncore_write(uncore,
			   GEN10_PAT_INDEX(5),
			   GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
	intel_uncore_write(uncore,
			   GEN10_PAT_INDEX(6),
			   GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
	intel_uncore_write(uncore,
			   GEN10_PAT_INDEX(7),
			   GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
}

/*
 * The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
 * bits. When using advanced contexts each context stores its own PAT, but
 * writing this data shouldn't be harmful even in those cases.
 */
static void bdw_setup_private_ppat(struct intel_uncore *uncore)
{
	struct drm_i915_private *i915 = uncore->i915;
	u64 pat;

	pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) |	/* for normal objects, no eLLC */
	      GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) |	/* for something pointing to ptes? */
	      GEN8_PPAT(3, GEN8_PPAT_UC) |			/* Uncached objects, mostly for scanout */
	      GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
	      GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
	      GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
	      GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));

	/* for scanout with eLLC */
	if (GRAPHICS_VER(i915) >= 9)
		pat |= GEN8_PPAT(2, GEN8_PPAT_WB | GEN8_PPAT_ELLC_OVERRIDE);
	else
		pat |= GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);

	intel_uncore_write(uncore, GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
	intel_uncore_write(uncore, GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
}

static void chv_setup_private_ppat(struct intel_uncore *uncore)
{
	u64 pat;

	/*
	 * Map WB on BDW to snooped on CHV.
	 *
	 * Only the snoop bit has meaning for CHV, the rest is
	 * ignored.
	 *
	 * The hardware will never snoop for certain types of accesses:
	 * - CPU GTT (GMADR->GGTT->no snoop->memory)
	 * - PPGTT page tables
	 * - some other special cycles
	 *
	 * As with BDW, we also need to consider the following for GT accesses:
	 * "For GGTT, there is NO pat_sel[2:0] from the entry,
	 * so RTL will always use the value corresponding to
	 * pat_sel = 000".
	 * Which means we must set the snoop bit in PAT entry 0
	 * in order to keep the global status page working.
	 */

	pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(1, 0) |
	      GEN8_PPAT(2, 0) |
	      GEN8_PPAT(3, 0) |
	      GEN8_PPAT(4, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(5, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(6, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(7, CHV_PPAT_SNOOP);

	intel_uncore_write(uncore, GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
	intel_uncore_write(uncore, GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
}

void setup_private_pat(struct intel_uncore *uncore)
{
	struct drm_i915_private *i915 = uncore->i915;

	GEM_BUG_ON(GRAPHICS_VER(i915) < 8);

	if (GRAPHICS_VER(i915) >= 12)
		tgl_setup_private_ppat(uncore);
	else if (GRAPHICS_VER(i915) >= 11)
		icl_setup_private_ppat(uncore);
	else if (IS_CHERRYVIEW(i915) || IS_GEN9_LP(i915))
		chv_setup_private_ppat(uncore);
	else
		bdw_setup_private_ppat(uncore);
}

struct i915_vma *
__vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size)
{
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;

	obj = i915_gem_object_create_internal(vm->i915, PAGE_ALIGN(size));
	if (IS_ERR(obj))
		return ERR_CAST(obj);

	i915_gem_object_set_cache_coherency(obj, I915_CACHING_CACHED);

	vma = i915_vma_instance(obj, vm, NULL);
	if (IS_ERR(vma)) {
		i915_gem_object_put(obj);
		return vma;
	}

	return vma;
}

struct i915_vma *
__vm_create_scratch_for_read_pinned(struct i915_address_space *vm, unsigned long size)
{
	struct i915_vma *vma;
	int err;

	vma = __vm_create_scratch_for_read(vm, size);
	if (IS_ERR(vma))
		return vma;

	err = i915_vma_pin(vma, 0, 0,
			   i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
	if (err) {
		i915_vma_put(vma);
		return ERR_PTR(err);
	}

	return vma;
}

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/mock_gtt.c"
#endif
Commit	Line	Data
2c86e55d MA	1	// SPDX-License-Identifier: MIT
	2	/*
	3	* Copyright © 2020 Intel Corporation
	4	*/
	5
	6	#include <linux/slab.h> /* fault-inject.h is not standalone! */
	7
	8	#include <linux/fault-inject.h>
8581fd40	9	#include <linux/sched/mm.h>
2c86e55d	10
2ca77606 MA	11	#include <drm/drm_cache.h>
2ca77606 MA	12
6aed5673	13	#include "gem/i915_gem_lmem.h"
2c86e55d MA	14	#include "i915_trace.h"
	15	#include "intel_gt.h"
	16	#include "intel_gtt.h"
	17
6aed5673 MA	18	struct drm_i915_gem_object alloc_pt_lmem(struct i915_address_space vm, int sz)
	19	{
	20	struct drm_i915_gem_object *obj;
	21
32334c9b MA	22	/*
	23	* To avoid severe over-allocation when dealing with min_page_size
	24	* restrictions, we override that behaviour here by allowing an object
	25	* size and page layout which can be smaller. In practice this should be
	26	* totally fine, since GTT paging structures are not typically inserted
	27	* into the GTT.
	28	*
	29	* Note that we also hit this path for the scratch page, and for this
	30	* case it might need to be 64K, but that should work fine here since we
	31	* used the passed in size for the page size, which should ensure it
	32	* also has the same alignment.
	33	*/
a259cc14 TH	34	obj = __i915_gem_object_create_lmem_with_ps(vm->i915, sz, sz,
a259cc14 TH	35	vm->lmem_pt_obj_flags);
6aed5673 MA	36	/*
	37	* Ensure all paging structures for this vm share the same dma-resv
	38	* object underneath, with the idea that one object_lock() will lock
	39	* them all at once.
	40	*/
4d8151ae TH	41	if (!IS_ERR(obj)) {
	42	obj->base.resv = i915_vm_resv_get(vm);
	43	obj->shares_resv_from = vm;
	44	}
	45
6aed5673 MA	46	return obj;
	47	}
	48
89351925	49	struct drm_i915_gem_object alloc_pt_dma(struct i915_address_space vm, int sz)
2c86e55d	50	{
26ad4f8b ML	51	struct drm_i915_gem_object *obj;
26ad4f8b ML	52
2c86e55d MA	53	if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
	54	i915_gem_shrink_all(vm->i915);
	55
26ad4f8b	56	obj = i915_gem_object_create_internal(vm->i915, sz);
6aed5673 MA	57	/*
	58	* Ensure all paging structures for this vm share the same dma-resv
	59	* object underneath, with the idea that one object_lock() will lock
	60	* them all at once.
	61	*/
4d8151ae TH	62	if (!IS_ERR(obj)) {
	63	obj->base.resv = i915_vm_resv_get(vm);
	64	obj->shares_resv_from = vm;
	65	}
	66
26ad4f8b	67	return obj;
2c86e55d MA	68	}
2c86e55d MA	69
529b9ec8	70	int map_pt_dma(struct i915_address_space vm, struct drm_i915_gem_object obj)
2c86e55d	71	{
6aed5673	72	enum i915_map_type type;
529b9ec8	73	void *vaddr;
2c86e55d	74
6aed5673 MA	75	type = i915_coherent_map_type(vm->i915, obj, true);
6aed5673 MA	76	vaddr = i915_gem_object_pin_map_unlocked(obj, type);
529b9ec8 MA	77	if (IS_ERR(vaddr))
529b9ec8 MA	78	return PTR_ERR(vaddr);
26ad4f8b ML	79
	80	i915_gem_object_make_unshrinkable(obj);
	81	return 0;
	82	}
	83
529b9ec8	84	int map_pt_dma_locked(struct i915_address_space vm, struct drm_i915_gem_object obj)
26ad4f8b	85	{
6aed5673	86	enum i915_map_type type;
529b9ec8	87	void *vaddr;
26ad4f8b	88
6aed5673 MA	89	type = i915_coherent_map_type(vm->i915, obj, true);
6aed5673 MA	90	vaddr = i915_gem_object_pin_map(obj, type);
529b9ec8 MA	91	if (IS_ERR(vaddr))
529b9ec8 MA	92	return PTR_ERR(vaddr);
2c86e55d	93
89351925 CW	94	i915_gem_object_make_unshrinkable(obj);
89351925 CW	95	return 0;
2c86e55d MA	96	}
	97
	98	void __i915_vm_close(struct i915_address_space *vm)
	99	{
	100	struct i915_vma vma, vn;
	101
ad2f9bc9 CW	102	if (!atomic_dec_and_mutex_lock(&vm->open, &vm->mutex))
	103	return;
	104
2c86e55d MA	105	list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) {
	106	struct drm_i915_gem_object *obj = vma->obj;
	107
	108	/* Keep the obj (and hence the vma) alive as _we_ destroy it */
	109	if (!kref_get_unless_zero(&obj->base.refcount))
	110	continue;
	111
	112	atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
	113	WARN_ON(__i915_vma_unbind(vma));
	114	__i915_vma_put(vma);
	115
	116	i915_gem_object_put(obj);
	117	}
	118	GEM_BUG_ON(!list_empty(&vm->bound_list));
ad2f9bc9	119
2c86e55d MA	120	mutex_unlock(&vm->mutex);
	121	}
	122
26ad4f8b ML	123	/* lock the vm into the current ww, if we lock one, we lock all */
	124	int i915_vm_lock_objects(struct i915_address_space *vm,
	125	struct i915_gem_ww_ctx *ww)
	126	{
4d8151ae	127	if (vm->scratch[0]->base.resv == &vm->_resv) {
26ad4f8b ML	128	return i915_gem_object_lock(vm->scratch[0], ww);
	129	} else {
	130	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
	131
	132	/* We borrowed the scratch page from ggtt, take the top level object */
	133	return i915_gem_object_lock(ppgtt->pd->pt.base, ww);
	134	}
	135	}
	136
2c86e55d MA	137	void i915_address_space_fini(struct i915_address_space *vm)
2c86e55d MA	138	{
2c86e55d	139	drm_mm_takedown(&vm->mm);
2c86e55d MA	140	mutex_destroy(&vm->mutex);
	141	}
	142
4d8151ae TH	143	/**
	144	* i915_vm_resv_release - Final struct i915_address_space destructor
	145	* @kref: Pointer to the &i915_address_space.resv_ref member.
	146	*
	147	* This function is called when the last lock sharer no longer shares the
	148	* &i915_address_space._resv lock.
	149	*/
	150	void i915_vm_resv_release(struct kref *kref)
	151	{
	152	struct i915_address_space *vm =
	153	container_of(kref, typeof(*vm), resv_ref);
	154
	155	dma_resv_fini(&vm->_resv);
	156	kfree(vm);
	157	}
	158
2c86e55d MA	159	static void __i915_vm_release(struct work_struct *work)
	160	{
	161	struct i915_address_space *vm =
dcc5d820	162	container_of(work, struct i915_address_space, release_work);
2c86e55d MA	163
	164	vm->cleanup(vm);
	165	i915_address_space_fini(vm);
	166
4d8151ae	167	i915_vm_resv_put(vm);
2c86e55d MA	168	}
	169
	170	void i915_vm_release(struct kref *kref)
	171	{
	172	struct i915_address_space *vm =
	173	container_of(kref, struct i915_address_space, ref);
	174
	175	GEM_BUG_ON(i915_is_ggtt(vm));
	176	trace_i915_ppgtt_release(vm);
	177
dcc5d820	178	queue_work(vm->i915->wq, &vm->release_work);
2c86e55d MA	179	}
	180
	181	void i915_address_space_init(struct i915_address_space *vm, int subclass)
	182	{
	183	kref_init(&vm->ref);
4d8151ae TH	184
	185	/*
	186	* Special case for GGTT that has already done an early
	187	* kref_init here.
	188	*/
	189	if (!kref_read(&vm->resv_ref))
	190	kref_init(&vm->resv_ref);
	191
dcc5d820	192	INIT_WORK(&vm->release_work, __i915_vm_release);
2c86e55d MA	193	atomic_set(&vm->open, 1);
	194
	195	/*
	196	* The vm->mutex must be reclaim safe (for use in the shrinker).
	197	* Do a dummy acquire now under fs_reclaim so that any allocation
	198	* attempt holding the lock is immediately reported by lockdep.
	199	*/
	200	mutex_init(&vm->mutex);
	201	lockdep_set_subclass(&vm->mutex, subclass);
bc6f80cc ML	202
	203	if (!intel_vm_no_concurrent_access_wa(vm->i915)) {
	204	i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex);
	205	} else {
	206	/*
	207	* CHV + BXT VTD workaround use stop_machine(),
	208	* which is allowed to allocate memory. This means &vm->mutex
	209	* is the outer lock, and in theory we can allocate memory inside
	210	* it through stop_machine().
	211	*
	212	* Add the annotation for this, we use trylock in shrinker.
	213	*/
	214	mutex_acquire(&vm->mutex.dep_map, 0, 0, _THIS_IP_);
	215	might_alloc(GFP_KERNEL);
	216	mutex_release(&vm->mutex.dep_map, _THIS_IP_);
	217	}
4d8151ae	218	dma_resv_init(&vm->_resv);
2c86e55d MA	219
	220	GEM_BUG_ON(!vm->total);
	221	drm_mm_init(&vm->mm, 0, vm->total);
	222	vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
	223
2c86e55d MA	224	INIT_LIST_HEAD(&vm->bound_list);
	225	}
	226
529b9ec8 MA	227	void __px_vaddr(struct drm_i915_gem_object p)
	228	{
	229	enum i915_map_type type;
	230
	231	GEM_BUG_ON(!i915_gem_object_has_pages(p));
	232	return page_unpack_bits(p->mm.mapping, &type);
	233	}
	234
89351925	235	dma_addr_t __px_dma(struct drm_i915_gem_object *p)
2c86e55d	236	{
89351925 CW	237	GEM_BUG_ON(!i915_gem_object_has_pages(p));
89351925 CW	238	return sg_dma_address(p->mm.pages->sgl);
2c86e55d MA	239	}
2c86e55d MA	240
89351925	241	struct page __px_page(struct drm_i915_gem_object p)
2c86e55d	242	{
89351925 CW	243	GEM_BUG_ON(!i915_gem_object_has_pages(p));
89351925 CW	244	return sg_page(p->mm.pages->sgl);
2c86e55d MA	245	}
	246
	247	void
89351925	248	fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count)
2c86e55d	249	{
529b9ec8	250	void *vaddr = __px_vaddr(p);
89351925	251
89351925 CW	252	memset64(vaddr, val, count);
89351925 CW	253	clflush_cache_range(vaddr, PAGE_SIZE);
2c86e55d MA	254	}
2c86e55d MA	255
89351925	256	static void poison_scratch_page(struct drm_i915_gem_object *scratch)
82d71e31	257	{
529b9ec8	258	void *vaddr = __px_vaddr(scratch);
89351925	259	u8 val;
82d71e31	260
89351925 CW	261	val = 0;
	262	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
	263	val = POISON_FREE;
82d71e31	264
529b9ec8	265	memset(vaddr, val, scratch->base.size);
2ca77606	266	drm_clflush_virt_range(vaddr, scratch->base.size);
82d71e31 CW	267	}
82d71e31 CW	268
89351925	269	int setup_scratch_page(struct i915_address_space *vm)
2c86e55d MA	270	{
	271	unsigned long size;
	272
	273	/*
	274	* In order to utilize 64K pages for an object with a size < 2M, we will
	275	* need to support a 64K scratch page, given that every 16th entry for a
	276	* page-table operating in 64K mode must point to a properly aligned 64K
	277	* region, including any PTEs which happen to point to scratch.
	278	*
	279	* This is only relevant for the 48b PPGTT where we support
	280	* huge-gtt-pages, see also i915_vma_insert(). However, as we share the
	281	* scratch (read-only) between all vm, we create one 64k scratch page
	282	* for all.
	283	*/
	284	size = I915_GTT_PAGE_SIZE_4K;
	285	if (i915_vm_is_4lvl(vm) &&
89351925	286	HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K))
2c86e55d	287	size = I915_GTT_PAGE_SIZE_64K;
2c86e55d MA	288
2c86e55d MA	289	do {
89351925	290	struct drm_i915_gem_object *obj;
2c86e55d	291
fef53be0	292	obj = vm->alloc_scratch_dma(vm, size);
89351925	293	if (IS_ERR(obj))
2c86e55d MA	294	goto skip;
2c86e55d MA	295
529b9ec8	296	if (map_pt_dma(vm, obj))
89351925 CW	297	goto skip_obj;
	298
	299	/* We need a single contiguous page for our scratch */
	300	if (obj->mm.page_sizes.sg < size)
	301	goto skip_obj;
	302
	303	/* And it needs to be correspondingly aligned */
	304	if (__px_dma(obj) & (size - 1))
	305	goto skip_obj;
	306
82d71e31 CW	307	/*
	308	* Use a non-zero scratch page for debugging.
	309	*
	310	* We want a value that should be reasonably obvious
	311	* to spot in the error state, while also causing a GPU hang
	312	* if executed. We prefer using a clear page in production, so
	313	* should it ever be accidentally used, the effect should be
	314	* fairly benign.
	315	*/
89351925 CW	316	poison_scratch_page(obj);
	317
	318	vm->scratch[0] = obj;
	319	vm->scratch_order = get_order(size);
2c86e55d MA	320	return 0;
2c86e55d MA	321
89351925 CW	322	skip_obj:
89351925 CW	323	i915_gem_object_put(obj);
2c86e55d MA	324	skip:
	325	if (size == I915_GTT_PAGE_SIZE_4K)
	326	return -ENOMEM;
	327
f122a46a MA	328	/*
	329	* If we need 64K minimum GTT pages for device local-memory,
	330	* like on XEHPSDV, then we need to fail the allocation here,
	331	* otherwise we can't safely support the insertion of
	332	* local-memory pages for this vm, since the HW expects the
	333	* correct physical alignment and size when the page-table is
	334	* operating in 64K GTT mode, which includes any scratch PTEs,
	335	* since userspace can still touch them.
	336	*/
	337	if (HAS_64K_PAGES(vm->i915))
	338	return -ENOMEM;
	339
2c86e55d	340	size = I915_GTT_PAGE_SIZE_4K;
2c86e55d MA	341	} while (1);
	342	}
	343
2c86e55d MA	344	void free_scratch(struct i915_address_space *vm)
	345	{
	346	int i;
	347
89351925 CW	348	for (i = 0; i <= vm->top; i++)
89351925 CW	349	i915_gem_object_put(vm->scratch[i]);
2c86e55d MA	350	}
	351
	352	void gtt_write_workarounds(struct intel_gt *gt)
	353	{
	354	struct drm_i915_private *i915 = gt->i915;
	355	struct intel_uncore *uncore = gt->uncore;
	356
	357	/*
	358	* This function is for gtt related workarounds. This function is
	359	* called on driver load and after a GPU reset, so you can place
	360	* workarounds here even if they get overwritten by GPU reset.
	361	*/
	362	/* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl,icl */
	363	if (IS_BROADWELL(i915))
	364	intel_uncore_write(uncore,
	365	GEN8_L3_LRA_1_GPGPU,
	366	GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
	367	else if (IS_CHERRYVIEW(i915))
	368	intel_uncore_write(uncore,
	369	GEN8_L3_LRA_1_GPGPU,
	370	GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
	371	else if (IS_GEN9_LP(i915))
	372	intel_uncore_write(uncore,
	373	GEN8_L3_LRA_1_GPGPU,
	374	GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
c816723b	375	else if (GRAPHICS_VER(i915) >= 9 && GRAPHICS_VER(i915) <= 11)
2c86e55d MA	376	intel_uncore_write(uncore,
	377	GEN8_L3_LRA_1_GPGPU,
	378	GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
	379
	380	/*
	381	* To support 64K PTEs we need to first enable the use of the
	382	* Intermediate-Page-Size(IPS) bit of the PDE field via some magical
	383	* mmio, otherwise the page-walker will simply ignore the IPS bit. This
	384	* shouldn't be needed after GEN10.
	385	*
	386	* 64K pages were first introduced from BDW+, although technically they
	387	* only work from gen9+. For pre-BDW we instead have the option for
	388	* 32K pages, but we don't currently have any support for it in our
	389	* driver.
	390	*/
	391	if (HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K) &&
c816723b	392	GRAPHICS_VER(i915) <= 10)
2c86e55d MA	393	intel_uncore_rmw(uncore,
	394	GEN8_GAMW_ECO_DEV_RW_IA,
	395	0,
	396	GAMW_ECO_ENABLE_64K_IPS_FIELD);
	397
c816723b	398	if (IS_GRAPHICS_VER(i915, 8, 11)) {
2c86e55d MA	399	bool can_use_gtt_cache = true;
	400
	401	/*
	402	* According to the BSpec if we use 2M/1G pages then we also
	403	* need to disable the GTT cache. At least on BDW we can see
	404	* visual corruption when using 2M pages, and not disabling the
	405	* GTT cache.
	406	*/
	407	if (HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_2M))
	408	can_use_gtt_cache = false;
	409
	410	/* WaGttCachingOffByDefault */
	411	intel_uncore_write(uncore,
	412	HSW_GTT_CACHE_EN,
	413	can_use_gtt_cache ? GTT_CACHE_EN_ALL : 0);
0d4c351a PB	414	drm_WARN_ON_ONCE(&i915->drm, can_use_gtt_cache &&
	415	intel_uncore_read(uncore,
	416	HSW_GTT_CACHE_EN) == 0);
2c86e55d MA	417	}
	418	}
	419
2c86e55d MA	420	static void tgl_setup_private_ppat(struct intel_uncore *uncore)
	421	{
	422	/* TGL doesn't support LLC or AGE settings */
	423	intel_uncore_write(uncore, GEN12_PAT_INDEX(0), GEN8_PPAT_WB);
	424	intel_uncore_write(uncore, GEN12_PAT_INDEX(1), GEN8_PPAT_WC);
	425	intel_uncore_write(uncore, GEN12_PAT_INDEX(2), GEN8_PPAT_WT);
	426	intel_uncore_write(uncore, GEN12_PAT_INDEX(3), GEN8_PPAT_UC);
	427	intel_uncore_write(uncore, GEN12_PAT_INDEX(4), GEN8_PPAT_WB);
	428	intel_uncore_write(uncore, GEN12_PAT_INDEX(5), GEN8_PPAT_WB);
	429	intel_uncore_write(uncore, GEN12_PAT_INDEX(6), GEN8_PPAT_WB);
	430	intel_uncore_write(uncore, GEN12_PAT_INDEX(7), GEN8_PPAT_WB);
	431	}
	432
6266992c	433	static void icl_setup_private_ppat(struct intel_uncore *uncore)
2c86e55d MA	434	{
	435	intel_uncore_write(uncore,
	436	GEN10_PAT_INDEX(0),
	437	GEN8_PPAT_WB \| GEN8_PPAT_LLC);
	438	intel_uncore_write(uncore,
	439	GEN10_PAT_INDEX(1),
	440	GEN8_PPAT_WC \| GEN8_PPAT_LLCELLC);
	441	intel_uncore_write(uncore,
	442	GEN10_PAT_INDEX(2),
c0888e9e	443	GEN8_PPAT_WB \| GEN8_PPAT_ELLC_OVERRIDE);
2c86e55d MA	444	intel_uncore_write(uncore,
	445	GEN10_PAT_INDEX(3),
	446	GEN8_PPAT_UC);
	447	intel_uncore_write(uncore,
	448	GEN10_PAT_INDEX(4),
	449	GEN8_PPAT_WB \| GEN8_PPAT_LLCELLC \| GEN8_PPAT_AGE(0));
	450	intel_uncore_write(uncore,
	451	GEN10_PAT_INDEX(5),
	452	GEN8_PPAT_WB \| GEN8_PPAT_LLCELLC \| GEN8_PPAT_AGE(1));
	453	intel_uncore_write(uncore,
	454	GEN10_PAT_INDEX(6),
	455	GEN8_PPAT_WB \| GEN8_PPAT_LLCELLC \| GEN8_PPAT_AGE(2));
	456	intel_uncore_write(uncore,
	457	GEN10_PAT_INDEX(7),
	458	GEN8_PPAT_WB \| GEN8_PPAT_LLCELLC \| GEN8_PPAT_AGE(3));
	459	}
	460
	461	/*
	462	* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
	463	* bits. When using advanced contexts each context stores its own PAT, but
	464	* writing this data shouldn't be harmful even in those cases.
	465	*/
	466	static void bdw_setup_private_ppat(struct intel_uncore *uncore)
	467	{
c0888e9e	468	struct drm_i915_private *i915 = uncore->i915;
2c86e55d MA	469	u64 pat;
	470
	471	pat = GEN8_PPAT(0, GEN8_PPAT_WB \| GEN8_PPAT_LLC) \| /* for normal objects, no eLLC */
	472	GEN8_PPAT(1, GEN8_PPAT_WC \| GEN8_PPAT_LLCELLC) \| /* for something pointing to ptes? */
2c86e55d MA	473	GEN8_PPAT(3, GEN8_PPAT_UC) \| /* Uncached objects, mostly for scanout */
	474	GEN8_PPAT(4, GEN8_PPAT_WB \| GEN8_PPAT_LLCELLC \| GEN8_PPAT_AGE(0)) \|
	475	GEN8_PPAT(5, GEN8_PPAT_WB \| GEN8_PPAT_LLCELLC \| GEN8_PPAT_AGE(1)) \|
	476	GEN8_PPAT(6, GEN8_PPAT_WB \| GEN8_PPAT_LLCELLC \| GEN8_PPAT_AGE(2)) \|
	477	GEN8_PPAT(7, GEN8_PPAT_WB \| GEN8_PPAT_LLCELLC \| GEN8_PPAT_AGE(3));
	478
c0888e9e	479	/* for scanout with eLLC */
c816723b	480	if (GRAPHICS_VER(i915) >= 9)
c0888e9e VS	481	pat \|= GEN8_PPAT(2, GEN8_PPAT_WB \| GEN8_PPAT_ELLC_OVERRIDE);
	482	else
	483	pat \|= GEN8_PPAT(2, GEN8_PPAT_WT \| GEN8_PPAT_LLCELLC);
	484
2c86e55d MA	485	intel_uncore_write(uncore, GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
	486	intel_uncore_write(uncore, GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
	487	}
	488
	489	static void chv_setup_private_ppat(struct intel_uncore *uncore)
	490	{
	491	u64 pat;
	492
	493	/*
	494	* Map WB on BDW to snooped on CHV.
	495	*
	496	* Only the snoop bit has meaning for CHV, the rest is
	497	* ignored.
	498	*
	499	* The hardware will never snoop for certain types of accesses:
	500	* - CPU GTT (GMADR->GGTT->no snoop->memory)
	501	* - PPGTT page tables
	502	* - some other special cycles
	503	*
	504	* As with BDW, we also need to consider the following for GT accesses:
	505	* "For GGTT, there is NO pat_sel[2:0] from the entry,
	506	* so RTL will always use the value corresponding to
	507	* pat_sel = 000".
	508	* Which means we must set the snoop bit in PAT entry 0
	509	* in order to keep the global status page working.
	510	*/
	511
	512	pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) \|
	513	GEN8_PPAT(1, 0) \|
	514	GEN8_PPAT(2, 0) \|
	515	GEN8_PPAT(3, 0) \|
	516	GEN8_PPAT(4, CHV_PPAT_SNOOP) \|
	517	GEN8_PPAT(5, CHV_PPAT_SNOOP) \|
	518	GEN8_PPAT(6, CHV_PPAT_SNOOP) \|
	519	GEN8_PPAT(7, CHV_PPAT_SNOOP);
	520
	521	intel_uncore_write(uncore, GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
	522	intel_uncore_write(uncore, GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
	523	}
	524
	525	void setup_private_pat(struct intel_uncore *uncore)
	526	{
	527	struct drm_i915_private *i915 = uncore->i915;
	528
c816723b	529	GEM_BUG_ON(GRAPHICS_VER(i915) < 8);
2c86e55d	530
c816723b	531	if (GRAPHICS_VER(i915) >= 12)
2c86e55d	532	tgl_setup_private_ppat(uncore);
6266992c LDM	533	else if (GRAPHICS_VER(i915) >= 11)
6266992c LDM	534	icl_setup_private_ppat(uncore);
2c86e55d MA	535	else if (IS_CHERRYVIEW(i915) \|\| IS_GEN9_LP(i915))
	536	chv_setup_private_ppat(uncore);
	537	else
	538	bdw_setup_private_ppat(uncore);
	539	}
	540
a4d86249 CW	541	struct i915_vma *
	542	__vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size)
	543	{
	544	struct drm_i915_gem_object *obj;
	545	struct i915_vma *vma;
a4d86249 CW	546
	547	obj = i915_gem_object_create_internal(vm->i915, PAGE_ALIGN(size));
	548	if (IS_ERR(obj))
	549	return ERR_CAST(obj);
	550
	551	i915_gem_object_set_cache_coherency(obj, I915_CACHING_CACHED);
	552
	553	vma = i915_vma_instance(obj, vm, NULL);
	554	if (IS_ERR(vma)) {
	555	i915_gem_object_put(obj);
	556	return vma;
	557	}
	558
2a665968 ML	559	return vma;
	560	}
	561
	562	struct i915_vma *
	563	__vm_create_scratch_for_read_pinned(struct i915_address_space *vm, unsigned long size)
	564	{
	565	struct i915_vma *vma;
	566	int err;
	567
	568	vma = __vm_create_scratch_for_read(vm, size);
	569	if (IS_ERR(vma))
	570	return vma;
	571
a4d86249 CW	572	err = i915_vma_pin(vma, 0, 0,
	573	i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
	574	if (err) {
	575	i915_vma_put(vma);
	576	return ERR_PTR(err);
	577	}
	578
	579	return vma;
	580	}
	581
2c86e55d MA	582	#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
	583	#include "selftests/mock_gtt.c"
	584	#endif