[linux-block.git] / drivers / gpu / drm / i915 / gt / gen6_ppgtt.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2020 Intel Corporation
 */

#include <linux/log2.h>

#include "gem/i915_gem_internal.h"

#include "gen6_ppgtt.h"
#include "i915_scatterlist.h"
#include "i915_trace.h"
#include "i915_vgpu.h"
#include "intel_gt_regs.h"
#include "intel_engine_regs.h"
#include "intel_gt.h"

/* Write pde (index) from the page directory @pd to the page table @pt */
static void gen6_write_pde(const struct gen6_ppgtt *ppgtt,
			   const unsigned int pde,
			   const struct i915_page_table *pt)
{
	dma_addr_t addr = pt ? px_dma(pt) : px_dma(ppgtt->base.vm.scratch[1]);

	/* Caller needs to make sure the write completes if necessary */
	iowrite32(GEN6_PDE_ADDR_ENCODE(addr) | GEN6_PDE_VALID,
		  ppgtt->pd_addr + pde);
}

void gen7_ppgtt_enable(struct intel_gt *gt)
{
	struct drm_i915_private *i915 = gt->i915;
	struct intel_uncore *uncore = gt->uncore;
	u32 ecochk;

	intel_uncore_rmw(uncore, GAC_ECO_BITS, 0, ECOBITS_PPGTT_CACHE64B);

	ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
	if (IS_HASWELL(i915)) {
		ecochk |= ECOCHK_PPGTT_WB_HSW;
	} else {
		ecochk |= ECOCHK_PPGTT_LLC_IVB;
		ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
	}
	intel_uncore_write(uncore, GAM_ECOCHK, ecochk);
}

void gen6_ppgtt_enable(struct intel_gt *gt)
{
	struct intel_uncore *uncore = gt->uncore;

	intel_uncore_rmw(uncore,
			 GAC_ECO_BITS,
			 0,
			 ECOBITS_SNB_BIT | ECOBITS_PPGTT_CACHE64B);

	intel_uncore_rmw(uncore,
			 GAB_CTL,
			 0,
			 GAB_CTL_CONT_AFTER_PAGEFAULT);

	intel_uncore_rmw(uncore,
			 GAM_ECOCHK,
			 0,
			 ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);

	if (HAS_PPGTT(uncore->i915)) /* may be disabled for VT-d */
		intel_uncore_write(uncore,
				   GFX_MODE,
				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
}

/* PPGTT support for Sandybdrige/Gen6 and later */
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
				   u64 start, u64 length)
{
	struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
	const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
	const gen6_pte_t scratch_pte = vm->scratch[0]->encode;
	unsigned int pde = first_entry / GEN6_PTES;
	unsigned int pte = first_entry % GEN6_PTES;
	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;

	while (num_entries) {
		struct i915_page_table * const pt =
			i915_pt_entry(ppgtt->base.pd, pde++);
		const unsigned int count = min(num_entries, GEN6_PTES - pte);
		gen6_pte_t *vaddr;

		num_entries -= count;

		GEM_BUG_ON(count > atomic_read(&pt->used));
		if (!atomic_sub_return(count, &pt->used))
			ppgtt->scan_for_unused_pt = true;

		/*
		 * Note that the hw doesn't support removing PDE on the fly
		 * (they are cached inside the context with no means to
		 * invalidate the cache), so we can only reset the PTE
		 * entries back to scratch.
		 */

		vaddr = px_vaddr(pt);
		memset32(vaddr + pte, scratch_pte, count);

		pte = 0;
	}
}

static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
				      struct i915_vma_resource *vma_res,
				      unsigned int pat_index,
				      u32 flags)
{
	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
	struct i915_page_directory * const pd = ppgtt->pd;
	unsigned int first_entry = vma_res->start / I915_GTT_PAGE_SIZE;
	unsigned int act_pt = first_entry / GEN6_PTES;
	unsigned int act_pte = first_entry % GEN6_PTES;
	const u32 pte_encode = vm->pte_encode(0, pat_index, flags);
	struct sgt_dma iter = sgt_dma(vma_res);
	gen6_pte_t *vaddr;

	GEM_BUG_ON(!pd->entry[act_pt]);

	vaddr = px_vaddr(i915_pt_entry(pd, act_pt));
	do {
		GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE);
		vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);

		iter.dma += I915_GTT_PAGE_SIZE;
		if (iter.dma == iter.max) {
			iter.sg = __sg_next(iter.sg);
			if (!iter.sg || sg_dma_len(iter.sg) == 0)
				break;

			iter.dma = sg_dma_address(iter.sg);
			iter.max = iter.dma + sg_dma_len(iter.sg);
		}

		if (++act_pte == GEN6_PTES) {
			vaddr = px_vaddr(i915_pt_entry(pd, ++act_pt));
			act_pte = 0;
		}
	} while (1);

	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
}

static void gen6_flush_pd(struct gen6_ppgtt *ppgtt, u64 start, u64 end)
{
	struct i915_page_directory * const pd = ppgtt->base.pd;
	struct i915_page_table *pt;
	unsigned int pde;

	start = round_down(start, SZ_64K);
	end = round_up(end, SZ_64K) - start;

	mutex_lock(&ppgtt->flush);

	gen6_for_each_pde(pt, pd, start, end, pde)
		gen6_write_pde(ppgtt, pde, pt);

	mb();
	ioread32(ppgtt->pd_addr + pde - 1);
	gen6_ggtt_invalidate(ppgtt->base.vm.gt->ggtt);
	mb();

	mutex_unlock(&ppgtt->flush);
}

static void gen6_alloc_va_range(struct i915_address_space *vm,
				struct i915_vm_pt_stash *stash,
				u64 start, u64 length)
{
	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
	struct i915_page_directory * const pd = ppgtt->base.pd;
	struct i915_page_table *pt;
	bool flush = false;
	u64 from = start;
	unsigned int pde;

	spin_lock(&pd->lock);
	gen6_for_each_pde(pt, pd, start, length, pde) {
		const unsigned int count = gen6_pte_count(start, length);

		if (!pt) {
			spin_unlock(&pd->lock);

			pt = stash->pt[0];
			__i915_gem_object_pin_pages(pt->base);

			fill32_px(pt, vm->scratch[0]->encode);

			spin_lock(&pd->lock);
			if (!pd->entry[pde]) {
				stash->pt[0] = pt->stash;
				atomic_set(&pt->used, 0);
				pd->entry[pde] = pt;
			} else {
				pt = pd->entry[pde];
			}

			flush = true;
		}

		atomic_add(count, &pt->used);
	}
	spin_unlock(&pd->lock);

	if (flush && i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) {
		intel_wakeref_t wakeref;

		with_intel_runtime_pm(&vm->i915->runtime_pm, wakeref)
			gen6_flush_pd(ppgtt, from, start);
	}
}

static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
{
	struct i915_address_space * const vm = &ppgtt->base.vm;
	int ret;

	ret = setup_scratch_page(vm);
	if (ret)
		return ret;

	vm->scratch[0]->encode =
		vm->pte_encode(px_dma(vm->scratch[0]),
			       i915_gem_get_pat_index(vm->i915,
						      I915_CACHE_NONE),
			       PTE_READ_ONLY);

	vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);
	if (IS_ERR(vm->scratch[1])) {
		ret = PTR_ERR(vm->scratch[1]);
		goto err_scratch0;
	}

	ret = map_pt_dma(vm, vm->scratch[1]);
	if (ret)
		goto err_scratch1;

	fill32_px(vm->scratch[1], vm->scratch[0]->encode);

	return 0;

err_scratch1:
	i915_gem_object_put(vm->scratch[1]);
err_scratch0:
	i915_gem_object_put(vm->scratch[0]);
	vm->scratch[0] = NULL;
	return ret;
}

static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
{
	struct i915_page_directory * const pd = ppgtt->base.pd;
	struct i915_page_table *pt;
	u32 pde;

	gen6_for_all_pdes(pt, pd, pde)
		if (pt)
			free_pt(&ppgtt->base.vm, pt);
}

static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
{
	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));

	gen6_ppgtt_free_pd(ppgtt);
	free_scratch(vm);

	if (ppgtt->base.pd)
		free_pd(&ppgtt->base.vm, ppgtt->base.pd);

	mutex_destroy(&ppgtt->flush);
}

static void pd_vma_bind(struct i915_address_space *vm,
			struct i915_vm_pt_stash *stash,
			struct i915_vma_resource *vma_res,
			unsigned int pat_index,
			u32 unused)
{
	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
	struct gen6_ppgtt *ppgtt = vma_res->private;
	u32 ggtt_offset = vma_res->start / I915_GTT_PAGE_SIZE;

	ppgtt->pp_dir = ggtt_offset * sizeof(gen6_pte_t) << 10;
	ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;

	gen6_flush_pd(ppgtt, 0, ppgtt->base.vm.total);
}

static void pd_vma_unbind(struct i915_address_space *vm,
			  struct i915_vma_resource *vma_res)
{
	struct gen6_ppgtt *ppgtt = vma_res->private;
	struct i915_page_directory * const pd = ppgtt->base.pd;
	struct i915_page_table *pt;
	unsigned int pde;

	if (!ppgtt->scan_for_unused_pt)
		return;

	/* Free all no longer used page tables */
	gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {
		if (!pt || atomic_read(&pt->used))
			continue;

		free_pt(&ppgtt->base.vm, pt);
		pd->entry[pde] = NULL;
	}

	ppgtt->scan_for_unused_pt = false;
}

static const struct i915_vma_ops pd_vma_ops = {
	.bind_vma = pd_vma_bind,
	.unbind_vma = pd_vma_unbind,
};

int gen6_ppgtt_pin(struct i915_ppgtt *base, struct i915_gem_ww_ctx *ww)
{
	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
	int err;

	GEM_BUG_ON(!kref_read(&ppgtt->base.vm.ref));

	/*
	 * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt
	 * which will be pinned into every active context.
	 * (When vma->pin_count becomes atomic, I expect we will naturally
	 * need a larger, unpacked, type and kill this redundancy.)
	 */
	if (atomic_add_unless(&ppgtt->pin_count, 1, 0))
		return 0;

	/* grab the ppgtt resv to pin the object */
	err = i915_vm_lock_objects(&ppgtt->base.vm, ww);
	if (err)
		return err;

	/*
	 * PPGTT PDEs reside in the GGTT and consists of 512 entries. The
	 * allocator works in address space sizes, so it's multiplied by page
	 * size. We allocate at the top of the GTT to avoid fragmentation.
	 */
	if (!atomic_read(&ppgtt->pin_count)) {
		err = i915_ggtt_pin(ppgtt->vma, ww, GEN6_PD_ALIGN, PIN_HIGH);

		GEM_BUG_ON(ppgtt->vma->fence);
		clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(ppgtt->vma));
	}
	if (!err)
		atomic_inc(&ppgtt->pin_count);

	return err;
}

static int pd_dummy_obj_get_pages(struct drm_i915_gem_object *obj)
{
	obj->mm.pages = ZERO_SIZE_PTR;
	return 0;
}

static void pd_dummy_obj_put_pages(struct drm_i915_gem_object *obj,
				   struct sg_table *pages)
{
}

static const struct drm_i915_gem_object_ops pd_dummy_obj_ops = {
	.name = "pd_dummy_obj",
	.get_pages = pd_dummy_obj_get_pages,
	.put_pages = pd_dummy_obj_put_pages,
};

static struct i915_page_directory *
gen6_alloc_top_pd(struct gen6_ppgtt *ppgtt)
{
	struct i915_ggtt * const ggtt = ppgtt->base.vm.gt->ggtt;
	struct i915_page_directory *pd;
	int err;

	pd = __alloc_pd(I915_PDES);
	if (unlikely(!pd))
		return ERR_PTR(-ENOMEM);

	pd->pt.base = __i915_gem_object_create_internal(ppgtt->base.vm.gt->i915,
							&pd_dummy_obj_ops,
							I915_PDES * SZ_4K);
	if (IS_ERR(pd->pt.base)) {
		err = PTR_ERR(pd->pt.base);
		pd->pt.base = NULL;
		goto err_pd;
	}

	pd->pt.base->base.resv = i915_vm_resv_get(&ppgtt->base.vm);
	pd->pt.base->shares_resv_from = &ppgtt->base.vm;

	ppgtt->vma = i915_vma_instance(pd->pt.base, &ggtt->vm, NULL);
	if (IS_ERR(ppgtt->vma)) {
		err = PTR_ERR(ppgtt->vma);
		ppgtt->vma = NULL;
		goto err_pd;
	}

	/* The dummy object we create is special, override ops.. */
	ppgtt->vma->ops = &pd_vma_ops;
	ppgtt->vma->private = ppgtt;
	return pd;

err_pd:
	free_pd(&ppgtt->base.vm, pd);
	return ERR_PTR(err);
}

void gen6_ppgtt_unpin(struct i915_ppgtt *base)
{
	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);

	GEM_BUG_ON(!atomic_read(&ppgtt->pin_count));
	if (atomic_dec_and_test(&ppgtt->pin_count))
		i915_vma_unpin(ppgtt->vma);
}

struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt)
{
	struct i915_ggtt * const ggtt = gt->ggtt;
	struct gen6_ppgtt *ppgtt;
	int err;

	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
	if (!ppgtt)
		return ERR_PTR(-ENOMEM);

	mutex_init(&ppgtt->flush);

	ppgtt_init(&ppgtt->base, gt, 0);
	ppgtt->base.vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen6_pte_t));
	ppgtt->base.vm.top = 1;

	ppgtt->base.vm.bind_async_flags = I915_VMA_LOCAL_BIND;
	ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
	ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
	ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
	ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;

	ppgtt->base.vm.alloc_pt_dma = alloc_pt_dma;
	ppgtt->base.vm.alloc_scratch_dma = alloc_pt_dma;
	ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;

	err = gen6_ppgtt_init_scratch(ppgtt);
	if (err)
		goto err_put;

	ppgtt->base.pd = gen6_alloc_top_pd(ppgtt);
	if (IS_ERR(ppgtt->base.pd)) {
		err = PTR_ERR(ppgtt->base.pd);
		goto err_put;
	}

	return &ppgtt->base;

err_put:
	i915_vm_put(&ppgtt->base.vm);
	return ERR_PTR(err);
}
Commit	Line	Data
2c86e55d MA	1	// SPDX-License-Identifier: MIT
	2	/*
	3	* Copyright © 2020 Intel Corporation
	4	*/
	5
	6	#include <linux/log2.h>
	7
b508d01f JN	8	#include "gem/i915_gem_internal.h"
b508d01f JN	9
2c86e55d MA	10	#include "gen6_ppgtt.h"
	11	#include "i915_scatterlist.h"
	12	#include "i915_trace.h"
	13	#include "i915_vgpu.h"
0d6419e9	14	#include "intel_gt_regs.h"
202b1f4c	15	#include "intel_engine_regs.h"
2c86e55d MA	16	#include "intel_gt.h"
	17
	18	/* Write pde (index) from the page directory @pd to the page table @pt */
9834dfef CW	19	static void gen6_write_pde(const struct gen6_ppgtt *ppgtt,
	20	const unsigned int pde,
	21	const struct i915_page_table *pt)
2c86e55d	22	{
89351925 CW	23	dma_addr_t addr = pt ? px_dma(pt) : px_dma(ppgtt->base.vm.scratch[1]);
89351925 CW	24
2c86e55d	25	/* Caller needs to make sure the write completes if necessary */
89351925	26	iowrite32(GEN6_PDE_ADDR_ENCODE(addr) \| GEN6_PDE_VALID,
2c86e55d MA	27	ppgtt->pd_addr + pde);
	28	}
	29
	30	void gen7_ppgtt_enable(struct intel_gt *gt)
	31	{
	32	struct drm_i915_private *i915 = gt->i915;
	33	struct intel_uncore *uncore = gt->uncore;
2c86e55d MA	34	u32 ecochk;
	35
	36	intel_uncore_rmw(uncore, GAC_ECO_BITS, 0, ECOBITS_PPGTT_CACHE64B);
	37
	38	ecochk = intel_uncore_read(uncore, GAM_ECOCHK);
	39	if (IS_HASWELL(i915)) {
	40	ecochk \|= ECOCHK_PPGTT_WB_HSW;
	41	} else {
	42	ecochk \|= ECOCHK_PPGTT_LLC_IVB;
	43	ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
	44	}
	45	intel_uncore_write(uncore, GAM_ECOCHK, ecochk);
2c86e55d MA	46	}
	47
	48	void gen6_ppgtt_enable(struct intel_gt *gt)
	49	{
	50	struct intel_uncore *uncore = gt->uncore;
	51
	52	intel_uncore_rmw(uncore,
	53	GAC_ECO_BITS,
	54	0,
	55	ECOBITS_SNB_BIT \| ECOBITS_PPGTT_CACHE64B);
	56
	57	intel_uncore_rmw(uncore,
	58	GAB_CTL,
	59	0,
	60	GAB_CTL_CONT_AFTER_PAGEFAULT);
	61
	62	intel_uncore_rmw(uncore,
	63	GAM_ECOCHK,
	64	0,
	65	ECOCHK_SNB_BIT \| ECOCHK_PPGTT_CACHE64B);
	66
	67	if (HAS_PPGTT(uncore->i915)) /* may be disabled for VT-d */
	68	intel_uncore_write(uncore,
	69	GFX_MODE,
	70	_MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
	71	}
	72
	73	/* PPGTT support for Sandybdrige/Gen6 and later */
	74	static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
	75	u64 start, u64 length)
	76	{
	77	struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
	78	const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
89351925	79	const gen6_pte_t scratch_pte = vm->scratch[0]->encode;
2c86e55d MA	80	unsigned int pde = first_entry / GEN6_PTES;
	81	unsigned int pte = first_entry % GEN6_PTES;
	82	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
	83
	84	while (num_entries) {
	85	struct i915_page_table * const pt =
	86	i915_pt_entry(ppgtt->base.pd, pde++);
	87	const unsigned int count = min(num_entries, GEN6_PTES - pte);
	88	gen6_pte_t *vaddr;
	89
2c86e55d MA	90	num_entries -= count;
	91
	92	GEM_BUG_ON(count > atomic_read(&pt->used));
	93	if (!atomic_sub_return(count, &pt->used))
	94	ppgtt->scan_for_unused_pt = true;
	95
	96	/*
	97	* Note that the hw doesn't support removing PDE on the fly
	98	* (they are cached inside the context with no means to
	99	* invalidate the cache), so we can only reset the PTE
	100	* entries back to scratch.
	101	*/
	102
529b9ec8	103	vaddr = px_vaddr(pt);
2c86e55d	104	memset32(vaddr + pte, scratch_pte, count);
2c86e55d MA	105
	106	pte = 0;
	107	}
	108	}
	109
	110	static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
39a2bd34	111	struct i915_vma_resource *vma_res,
9275277d	112	unsigned int pat_index,
2c86e55d MA	113	u32 flags)
	114	{
	115	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
	116	struct i915_page_directory * const pd = ppgtt->pd;
39a2bd34	117	unsigned int first_entry = vma_res->start / I915_GTT_PAGE_SIZE;
2c86e55d MA	118	unsigned int act_pt = first_entry / GEN6_PTES;
2c86e55d MA	119	unsigned int act_pte = first_entry % GEN6_PTES;
9275277d	120	const u32 pte_encode = vm->pte_encode(0, pat_index, flags);
39a2bd34	121	struct sgt_dma iter = sgt_dma(vma_res);
2c86e55d MA	122	gen6_pte_t *vaddr;
2c86e55d MA	123
89351925	124	GEM_BUG_ON(!pd->entry[act_pt]);
2c86e55d	125
529b9ec8	126	vaddr = px_vaddr(i915_pt_entry(pd, act_pt));
2c86e55d	127	do {
8a473dba	128	GEM_BUG_ON(sg_dma_len(iter.sg) < I915_GTT_PAGE_SIZE);
2c86e55d MA	129	vaddr[act_pte] = pte_encode \| GEN6_PTE_ADDR_ENCODE(iter.dma);
	130
	131	iter.dma += I915_GTT_PAGE_SIZE;
	132	if (iter.dma == iter.max) {
	133	iter.sg = __sg_next(iter.sg);
8a473dba	134	if (!iter.sg \|\| sg_dma_len(iter.sg) == 0)
2c86e55d MA	135	break;
	136
	137	iter.dma = sg_dma_address(iter.sg);
8a473dba	138	iter.max = iter.dma + sg_dma_len(iter.sg);
2c86e55d MA	139	}
	140
	141	if (++act_pte == GEN6_PTES) {
529b9ec8	142	vaddr = px_vaddr(i915_pt_entry(pd, ++act_pt));
2c86e55d MA	143	act_pte = 0;
	144	}
	145	} while (1);
2c86e55d	146
39a2bd34	147	vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
2c86e55d MA	148	}
	149
	150	static void gen6_flush_pd(struct gen6_ppgtt *ppgtt, u64 start, u64 end)
	151	{
	152	struct i915_page_directory * const pd = ppgtt->base.pd;
	153	struct i915_page_table *pt;
	154	unsigned int pde;
	155
	156	start = round_down(start, SZ_64K);
	157	end = round_up(end, SZ_64K) - start;
	158
	159	mutex_lock(&ppgtt->flush);
	160
	161	gen6_for_each_pde(pt, pd, start, end, pde)
	162	gen6_write_pde(ppgtt, pde, pt);
	163
	164	mb();
	165	ioread32(ppgtt->pd_addr + pde - 1);
	166	gen6_ggtt_invalidate(ppgtt->base.vm.gt->ggtt);
	167	mb();
	168
	169	mutex_unlock(&ppgtt->flush);
	170	}
	171
cd0452aa CW	172	static void gen6_alloc_va_range(struct i915_address_space *vm,
	173	struct i915_vm_pt_stash *stash,
	174	u64 start, u64 length)
2c86e55d MA	175	{
	176	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
	177	struct i915_page_directory * const pd = ppgtt->base.pd;
cd0452aa	178	struct i915_page_table *pt;
b297bde1	179	bool flush = false;
2c86e55d MA	180	u64 from = start;
2c86e55d MA	181	unsigned int pde;
2c86e55d	182
2c86e55d MA	183	spin_lock(&pd->lock);
	184	gen6_for_each_pde(pt, pd, start, length, pde) {
	185	const unsigned int count = gen6_pte_count(start, length);
	186
89351925	187	if (!pt) {
2c86e55d MA	188	spin_unlock(&pd->lock);
2c86e55d MA	189
cd0452aa	190	pt = stash->pt[0];
89351925	191	__i915_gem_object_pin_pages(pt->base);
2c86e55d	192
89351925	193	fill32_px(pt, vm->scratch[0]->encode);
2c86e55d MA	194
2c86e55d MA	195	spin_lock(&pd->lock);
89351925	196	if (!pd->entry[pde]) {
cd0452aa CW	197	stash->pt[0] = pt->stash;
cd0452aa CW	198	atomic_set(&pt->used, 0);
2c86e55d MA	199	pd->entry[pde] = pt;
2c86e55d MA	200	} else {
2c86e55d MA	201	pt = pd->entry[pde];
2c86e55d MA	202	}
b297bde1 CW	203
b297bde1 CW	204	flush = true;
2c86e55d MA	205	}
	206
	207	atomic_add(count, &pt->used);
	208	}
	209	spin_unlock(&pd->lock);
	210
b297bde1 CW	211	if (flush && i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) {
	212	intel_wakeref_t wakeref;
	213
	214	with_intel_runtime_pm(&vm->i915->runtime_pm, wakeref)
	215	gen6_flush_pd(ppgtt, from, start);
	216	}
2c86e55d MA	217	}
	218
	219	static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
	220	{
	221	struct i915_address_space * const vm = &ppgtt->base.vm;
2c86e55d MA	222	int ret;
2c86e55d MA	223
89351925	224	ret = setup_scratch_page(vm);
2c86e55d MA	225	if (ret)
	226	return ret;
	227
89351925 CW	228	vm->scratch[0]->encode =
89351925 CW	229	vm->pte_encode(px_dma(vm->scratch[0]),
9275277d FY	230	i915_gem_get_pat_index(vm->i915,
	231	I915_CACHE_NONE),
	232	PTE_READ_ONLY);
2c86e55d	233
89351925	234	vm->scratch[1] = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);
fa812ce9 CW	235	if (IS_ERR(vm->scratch[1])) {
	236	ret = PTR_ERR(vm->scratch[1]);
	237	goto err_scratch0;
	238	}
89351925	239
529b9ec8	240	ret = map_pt_dma(vm, vm->scratch[1]);
fa812ce9 CW	241	if (ret)
fa812ce9 CW	242	goto err_scratch1;
2c86e55d	243
89351925	244	fill32_px(vm->scratch[1], vm->scratch[0]->encode);
2c86e55d MA	245
2c86e55d MA	246	return 0;
fa812ce9 CW	247
	248	err_scratch1:
	249	i915_gem_object_put(vm->scratch[1]);
	250	err_scratch0:
	251	i915_gem_object_put(vm->scratch[0]);
c286558f	252	vm->scratch[0] = NULL;
fa812ce9	253	return ret;
2c86e55d MA	254	}
	255
	256	static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
	257	{
	258	struct i915_page_directory * const pd = ppgtt->base.pd;
2c86e55d MA	259	struct i915_page_table *pt;
	260	u32 pde;
	261
	262	gen6_for_all_pdes(pt, pd, pde)
89351925	263	if (pt)
82adf901	264	free_pt(&ppgtt->base.vm, pt);
2c86e55d MA	265	}
	266
	267	static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
	268	{
	269	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
	270
2c86e55d MA	271	gen6_ppgtt_free_pd(ppgtt);
	272	free_scratch(vm);
	273
c286558f CW	274	if (ppgtt->base.pd)
c286558f CW	275	free_pd(&ppgtt->base.vm, ppgtt->base.pd);
82adf901	276
c286558f	277	mutex_destroy(&ppgtt->flush);
2c86e55d MA	278	}
2c86e55d MA	279
cd0452aa CW	280	static void pd_vma_bind(struct i915_address_space *vm,
cd0452aa CW	281	struct i915_vm_pt_stash *stash,
39a2bd34	282	struct i915_vma_resource *vma_res,
9275277d	283	unsigned int pat_index,
cd0452aa	284	u32 unused)
2c86e55d	285	{
12b07256	286	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
39a2bd34 TH	287	struct gen6_ppgtt *ppgtt = vma_res->private;
39a2bd34 TH	288	u32 ggtt_offset = vma_res->start / I915_GTT_PAGE_SIZE;
2c86e55d	289
89351925	290	ppgtt->pp_dir = ggtt_offset * sizeof(gen6_pte_t) << 10;
2c86e55d MA	291	ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;
	292
	293	gen6_flush_pd(ppgtt, 0, ppgtt->base.vm.total);
2c86e55d MA	294	}
2c86e55d MA	295
39a2bd34 TH	296	static void pd_vma_unbind(struct i915_address_space *vm,
39a2bd34 TH	297	struct i915_vma_resource *vma_res)
2c86e55d	298	{
39a2bd34	299	struct gen6_ppgtt *ppgtt = vma_res->private;
2c86e55d	300	struct i915_page_directory * const pd = ppgtt->base.pd;
2c86e55d MA	301	struct i915_page_table *pt;
	302	unsigned int pde;
	303
	304	if (!ppgtt->scan_for_unused_pt)
	305	return;
	306
	307	/* Free all no longer used page tables */
	308	gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {
89351925	309	if (!pt \|\| atomic_read(&pt->used))
2c86e55d MA	310	continue;
2c86e55d MA	311
82adf901	312	free_pt(&ppgtt->base.vm, pt);
89351925	313	pd->entry[pde] = NULL;
2c86e55d MA	314	}
	315
	316	ppgtt->scan_for_unused_pt = false;
	317	}
	318
	319	static const struct i915_vma_ops pd_vma_ops = {
2c86e55d MA	320	.bind_vma = pd_vma_bind,
	321	.unbind_vma = pd_vma_unbind,
	322	};
	323
47b08693	324	int gen6_ppgtt_pin(struct i915_ppgtt base, struct i915_gem_ww_ctx ww)
2c86e55d MA	325	{
	326	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
	327	int err;
	328
e1a7ab4f	329	GEM_BUG_ON(!kref_read(&ppgtt->base.vm.ref));
2c86e55d MA	330
	331	/*
	332	* Workaround the limited maximum vma->pin_count and the aliasing_ppgtt
	333	* which will be pinned into every active context.
	334	* (When vma->pin_count becomes atomic, I expect we will naturally
	335	* need a larger, unpacked, type and kill this redundancy.)
	336	*/
	337	if (atomic_add_unless(&ppgtt->pin_count, 1, 0))
	338	return 0;
	339
b0b0f2d2 ML	340	/* grab the ppgtt resv to pin the object */
	341	err = i915_vm_lock_objects(&ppgtt->base.vm, ww);
	342	if (err)
	343	return err;
2c86e55d MA	344
	345	/*
	346	* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
	347	* allocator works in address space sizes, so it's multiplied by page
	348	* size. We allocate at the top of the GTT to avoid fragmentation.
	349	*/
b0b0f2d2	350	if (!atomic_read(&ppgtt->pin_count)) {
47b08693	351	err = i915_ggtt_pin(ppgtt->vma, ww, GEN6_PD_ALIGN, PIN_HIGH);
b0b0f2d2 ML	352
	353	GEM_BUG_ON(ppgtt->vma->fence);
	354	clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(ppgtt->vma));
	355	}
2c86e55d MA	356	if (!err)
2c86e55d MA	357	atomic_inc(&ppgtt->pin_count);
2c86e55d MA	358
	359	return err;
	360	}
	361
b0b0f2d2	362	static int pd_dummy_obj_get_pages(struct drm_i915_gem_object *obj)
2c86e55d	363	{
b0b0f2d2 ML	364	obj->mm.pages = ZERO_SIZE_PTR;
	365	return 0;
	366	}
2c86e55d	367
b0b0f2d2 ML	368	static void pd_dummy_obj_put_pages(struct drm_i915_gem_object *obj,
	369	struct sg_table *pages)
	370	{
2c86e55d MA	371	}
2c86e55d MA	372
b0b0f2d2 ML	373	static const struct drm_i915_gem_object_ops pd_dummy_obj_ops = {
	374	.name = "pd_dummy_obj",
	375	.get_pages = pd_dummy_obj_get_pages,
	376	.put_pages = pd_dummy_obj_put_pages,
	377	};
	378
	379	static struct i915_page_directory *
	380	gen6_alloc_top_pd(struct gen6_ppgtt *ppgtt)
2c86e55d	381	{
b0b0f2d2 ML	382	struct i915_ggtt * const ggtt = ppgtt->base.vm.gt->ggtt;
	383	struct i915_page_directory *pd;
	384	int err;
2c86e55d	385
b0b0f2d2 ML	386	pd = __alloc_pd(I915_PDES);
	387	if (unlikely(!pd))
	388	return ERR_PTR(-ENOMEM);
2c86e55d	389
b0b0f2d2 ML	390	pd->pt.base = __i915_gem_object_create_internal(ppgtt->base.vm.gt->i915,
	391	&pd_dummy_obj_ops,
	392	I915_PDES * SZ_4K);
	393	if (IS_ERR(pd->pt.base)) {
	394	err = PTR_ERR(pd->pt.base);
	395	pd->pt.base = NULL;
	396	goto err_pd;
	397	}
	398
	399	pd->pt.base->base.resv = i915_vm_resv_get(&ppgtt->base.vm);
	400	pd->pt.base->shares_resv_from = &ppgtt->base.vm;
	401
	402	ppgtt->vma = i915_vma_instance(pd->pt.base, &ggtt->vm, NULL);
	403	if (IS_ERR(ppgtt->vma)) {
	404	err = PTR_ERR(ppgtt->vma);
	405	ppgtt->vma = NULL;
	406	goto err_pd;
	407	}
	408
	409	/* The dummy object we create is special, override ops.. */
	410	ppgtt->vma->ops = &pd_vma_ops;
	411	ppgtt->vma->private = ppgtt;
	412	return pd;
	413
	414	err_pd:
	415	free_pd(&ppgtt->base.vm, pd);
	416	return ERR_PTR(err);
	417	}
	418
2c86e55d MA	419	void gen6_ppgtt_unpin(struct i915_ppgtt *base)
	420	{
	421	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
	422
	423	GEM_BUG_ON(!atomic_read(&ppgtt->pin_count));
	424	if (atomic_dec_and_test(&ppgtt->pin_count))
	425	i915_vma_unpin(ppgtt->vma);
2c86e55d MA	426	}
	427
	428	struct i915_ppgtt gen6_ppgtt_create(struct intel_gt gt)
	429	{
	430	struct i915_ggtt * const ggtt = gt->ggtt;
	431	struct gen6_ppgtt *ppgtt;
	432	int err;
	433
	434	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
	435	if (!ppgtt)
	436	return ERR_PTR(-ENOMEM);
	437
	438	mutex_init(&ppgtt->flush);
2c86e55d	439
a259cc14	440	ppgtt_init(&ppgtt->base, gt, 0);
cd0452aa	441	ppgtt->base.vm.pd_shift = ilog2(SZ_4K * SZ_4K / sizeof(gen6_pte_t));
2c86e55d MA	442	ppgtt->base.vm.top = 1;
	443
	444	ppgtt->base.vm.bind_async_flags = I915_VMA_LOCAL_BIND;
	445	ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
	446	ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
	447	ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
	448	ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;
	449
89351925	450	ppgtt->base.vm.alloc_pt_dma = alloc_pt_dma;
fef53be0	451	ppgtt->base.vm.alloc_scratch_dma = alloc_pt_dma;
2c86e55d MA	452	ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;
2c86e55d MA	453
2c86e55d MA	454	err = gen6_ppgtt_init_scratch(ppgtt);
2c86e55d MA	455	if (err)
c286558f	456	goto err_put;
2c86e55d	457
b0b0f2d2 ML	458	ppgtt->base.pd = gen6_alloc_top_pd(ppgtt);
	459	if (IS_ERR(ppgtt->base.pd)) {
	460	err = PTR_ERR(ppgtt->base.pd);
c286558f	461	goto err_put;
2c86e55d MA	462	}
	463
	464	return &ppgtt->base;
	465
c286558f CW	466	err_put:
c286558f CW	467	i915_vm_put(&ppgtt->base.vm);
2c86e55d MA	468	return ERR_PTR(err);
2c86e55d MA	469	}