[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_gmc.h

/*
 * Copyright 2018 Advanced Micro Devices, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
#ifndef __AMDGPU_GMC_H__
#define __AMDGPU_GMC_H__

#include <linux/types.h>

#include "amdgpu_irq.h"

/* VA hole for 48bit addresses on Vega10 */
#define AMDGPU_GMC_HOLE_START	0x0000800000000000ULL
#define AMDGPU_GMC_HOLE_END	0xffff800000000000ULL

/*
 * Hardware is programmed as if the hole doesn't exists with start and end
 * address values.
 *
 * This mask is used to remove the upper 16bits of the VA and so come up with
 * the linear addr value.
 */
#define AMDGPU_GMC_HOLE_MASK	0x0000ffffffffffffULL

/*
 * Ring size as power of two for the log of recent faults.
 */
#define AMDGPU_GMC_FAULT_RING_ORDER	8
#define AMDGPU_GMC_FAULT_RING_SIZE	(1 << AMDGPU_GMC_FAULT_RING_ORDER)

/*
 * Hash size as power of two for the log of recent faults
 */
#define AMDGPU_GMC_FAULT_HASH_ORDER	8
#define AMDGPU_GMC_FAULT_HASH_SIZE	(1 << AMDGPU_GMC_FAULT_HASH_ORDER)

/*
 * Number of IH timestamp ticks until a fault is considered handled
 */
#define AMDGPU_GMC_FAULT_TIMEOUT	5000ULL

struct firmware;

/*
 * GMC page fault information
 */
struct amdgpu_gmc_fault {
	uint64_t	timestamp;
	uint64_t	next:AMDGPU_GMC_FAULT_RING_ORDER;
	uint64_t	key:52;
};

/*
 * VMHUB structures, functions & helpers
 */
struct amdgpu_vmhub_funcs {
	void (*print_l2_protection_fault_status)(struct amdgpu_device *adev,
						 uint32_t status);
	uint32_t (*get_invalidate_req)(unsigned int vmid, uint32_t flush_type);
};

struct amdgpu_vmhub {
	uint32_t	ctx0_ptb_addr_lo32;
	uint32_t	ctx0_ptb_addr_hi32;
	uint32_t	vm_inv_eng0_sem;
	uint32_t	vm_inv_eng0_req;
	uint32_t	vm_inv_eng0_ack;
	uint32_t	vm_context0_cntl;
	uint32_t	vm_l2_pro_fault_status;
	uint32_t	vm_l2_pro_fault_cntl;

	/*
	 * store the register distances between two continuous context domain
	 * and invalidation engine.
	 */
	uint32_t	ctx_distance;
	uint32_t	ctx_addr_distance; /* include LO32/HI32 */
	uint32_t	eng_distance;
	uint32_t	eng_addr_distance; /* include LO32/HI32 */

	uint32_t	vm_cntx_cntl_vm_fault;

	const struct amdgpu_vmhub_funcs *vmhub_funcs;
};

/*
 * GPU MC structures, functions & helpers
 */
struct amdgpu_gmc_funcs {
	/* flush the vm tlb via mmio */
	void (*flush_gpu_tlb)(struct amdgpu_device *adev, uint32_t vmid,
				uint32_t vmhub, uint32_t flush_type);
	/* flush the vm tlb via pasid */
	int (*flush_gpu_tlb_pasid)(struct amdgpu_device *adev, uint16_t pasid,
					uint32_t flush_type, bool all_hub);
	/* flush the vm tlb via ring */
	uint64_t (*emit_flush_gpu_tlb)(struct amdgpu_ring *ring, unsigned vmid,
				       uint64_t pd_addr);
	/* Change the VMID -> PASID mapping */
	void (*emit_pasid_mapping)(struct amdgpu_ring *ring, unsigned vmid,
				   unsigned pasid);
	/* enable/disable PRT support */
	void (*set_prt)(struct amdgpu_device *adev, bool enable);
	/* map mtype to hardware flags */
	uint64_t (*map_mtype)(struct amdgpu_device *adev, uint32_t flags);
	/* get the pde for a given mc addr */
	void (*get_vm_pde)(struct amdgpu_device *adev, int level,
			   u64 *dst, u64 *flags);
	/* get the pte flags to use for a BO VA mapping */
	void (*get_vm_pte)(struct amdgpu_device *adev,
			   struct amdgpu_bo_va_mapping *mapping,
			   uint64_t *flags);
	/* get the amount of memory used by the vbios for pre-OS console */
	unsigned int (*get_vbios_fb_size)(struct amdgpu_device *adev);
};

struct amdgpu_xgmi {
	/* from psp */
	u64 node_id;
	u64 hive_id;
	/* fixed per family */
	u64 node_segment_size;
	/* physical node (0-3) */
	unsigned physical_node_id;
	/* number of nodes (0-4) */
	unsigned num_physical_nodes;
	/* gpu list in the same hive */
	struct list_head head;
	bool supported;
	struct ras_common_if *ras_if;
};

struct amdgpu_gmc {
	/* FB's physical address in MMIO space (for CPU to
	 * map FB). This is different compared to the agp/
	 * gart/vram_start/end field as the later is from
	 * GPU's view and aper_base is from CPU's view.
	 */
	resource_size_t		aper_size;
	resource_size_t		aper_base;
	/* for some chips with <= 32MB we need to lie
	 * about vram size near mc fb location */
	u64			mc_vram_size;
	u64			visible_vram_size;
	/* AGP aperture start and end in MC address space
	 * Driver find a hole in the MC address space
	 * to place AGP by setting MC_VM_AGP_BOT/TOP registers
	 * Under VMID0, logical address == MC address. AGP
	 * aperture maps to physical bus or IOVA addressed.
	 * AGP aperture is used to simulate FB in ZFB case.
	 * AGP aperture is also used for page table in system
	 * memory (mainly for APU).
	 *
	 */
	u64			agp_size;
	u64			agp_start;
	u64			agp_end;
	/* GART aperture start and end in MC address space
	 * Driver find a hole in the MC address space
	 * to place GART by setting VM_CONTEXT0_PAGE_TABLE_START/END_ADDR
	 * registers
	 * Under VMID0, logical address inside GART aperture will
	 * be translated through gpuvm gart page table to access
	 * paged system memory
	 */
	u64			gart_size;
	u64			gart_start;
	u64			gart_end;
	/* Frame buffer aperture of this GPU device. Different from
	 * fb_start (see below), this only covers the local GPU device.
	 * Driver get fb_start from MC_VM_FB_LOCATION_BASE (set by vbios)
	 * and calculate vram_start of this local device by adding an
	 * offset inside the XGMI hive.
	 * Under VMID0, logical address == MC address
	 */
	u64			vram_start;
	u64			vram_end;
	/* FB region , it's same as local vram region in single GPU, in XGMI
	 * configuration, this region covers all GPUs in the same hive ,
	 * each GPU in the hive has the same view of this FB region .
	 * GPU0's vram starts at offset (0 * segment size) ,
	 * GPU1 starts at offset (1 * segment size), etc.
	 */
	u64			fb_start;
	u64			fb_end;
	unsigned		vram_width;
	u64			real_vram_size;
	int			vram_mtrr;
	u64                     mc_mask;
	const struct firmware   *fw;	/* MC firmware */
	uint32_t                fw_version;
	struct amdgpu_irq_src	vm_fault;
	uint32_t		vram_type;
	uint8_t			vram_vendor;
	uint32_t                srbm_soft_reset;
	bool			prt_warning;
	uint32_t		sdpif_register;
	/* apertures */
	u64			shared_aperture_start;
	u64			shared_aperture_end;
	u64			private_aperture_start;
	u64			private_aperture_end;
	/* protects concurrent invalidation */
	spinlock_t		invalidate_lock;
	bool			translate_further;
	struct kfd_vm_fault_info *vm_fault_info;
	atomic_t		vm_fault_info_updated;

	struct amdgpu_gmc_fault	fault_ring[AMDGPU_GMC_FAULT_RING_SIZE];
	struct {
		uint64_t	idx:AMDGPU_GMC_FAULT_RING_ORDER;
	} fault_hash[AMDGPU_GMC_FAULT_HASH_SIZE];
	uint64_t		last_fault:AMDGPU_GMC_FAULT_RING_ORDER;

	bool tmz_enabled;

	const struct amdgpu_gmc_funcs	*gmc_funcs;

	struct amdgpu_xgmi xgmi;
	struct amdgpu_irq_src	ecc_irq;
	int noretry;
};

#define amdgpu_gmc_flush_gpu_tlb(adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))
#define amdgpu_gmc_flush_gpu_tlb_pasid(adev, pasid, type, allhub) \
	((adev)->gmc.gmc_funcs->flush_gpu_tlb_pasid \
	((adev), (pasid), (type), (allhub)))
#define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
#define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))
#define amdgpu_gmc_map_mtype(adev, flags) (adev)->gmc.gmc_funcs->map_mtype((adev),(flags))
#define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags))
#define amdgpu_gmc_get_vm_pte(adev, mapping, flags) (adev)->gmc.gmc_funcs->get_vm_pte((adev), (mapping), (flags))
#define amdgpu_gmc_get_vbios_fb_size(adev) (adev)->gmc.gmc_funcs->get_vbios_fb_size((adev))

/**
 * amdgpu_gmc_vram_full_visible - Check if full VRAM is visible through the BAR
 *
 * @adev: amdgpu_device pointer
 *
 * Returns:
 * True if full VRAM is visible through the BAR
 */
static inline bool amdgpu_gmc_vram_full_visible(struct amdgpu_gmc *gmc)
{
	WARN_ON(gmc->real_vram_size < gmc->visible_vram_size);

	return (gmc->real_vram_size == gmc->visible_vram_size);
}

/**
 * amdgpu_gmc_sign_extend - sign extend the given gmc address
 *
 * @addr: address to extend
 */
static inline uint64_t amdgpu_gmc_sign_extend(uint64_t addr)
{
	if (addr >= AMDGPU_GMC_HOLE_START)
		addr |= AMDGPU_GMC_HOLE_END;

	return addr;
}

void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level,
			       uint64_t *addr, uint64_t *flags);
int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
				uint32_t gpu_page_idx, uint64_t addr,
				uint64_t flags);
uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo);
uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo);
void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,
			      u64 base);
void amdgpu_gmc_gart_location(struct amdgpu_device *adev,
			      struct amdgpu_gmc *mc);
void amdgpu_gmc_agp_location(struct amdgpu_device *adev,
			     struct amdgpu_gmc *mc);
bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr,
			      uint16_t pasid, uint64_t timestamp);
int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev);
void amdgpu_gmc_ras_fini(struct amdgpu_device *adev);
int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev);

extern void amdgpu_gmc_tmz_set(struct amdgpu_device *adev);
extern void amdgpu_gmc_noretry_set(struct amdgpu_device *adev);

extern void
amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type,
			      bool enable);

void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev);

#endif
Commit	Line	Data
770d13b1 CK	1	/*
	2	* Copyright 2018 Advanced Micro Devices, Inc.
	3	* All Rights Reserved.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the
	7	* "Software"), to deal in the Software without restriction, including
	8	* without limitation the rights to use, copy, modify, merge, publish,
	9	* distribute, sub license, and/or sell copies of the Software, and to
	10	* permit persons to whom the Software is furnished to do so, subject to
	11	* the following conditions:
	12	*
	13	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	14	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	15	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
	16	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
	17	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
	18	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
	19	* USE OR OTHER DEALINGS IN THE SOFTWARE.
	20	*
	21	* The above copyright notice and this permission notice (including the
	22	* next paragraph) shall be included in all copies or substantial portions
	23	* of the Software.
	24	*
	25	*/
	26	#ifndef __AMDGPU_GMC_H__
	27	#define __AMDGPU_GMC_H__
	28
	29	#include <linux/types.h>
	30
	31	#include "amdgpu_irq.h"
	32
ad9a5b78 CK	33	/* VA hole for 48bit addresses on Vega10 */
	34	#define AMDGPU_GMC_HOLE_START 0x0000800000000000ULL
	35	#define AMDGPU_GMC_HOLE_END 0xffff800000000000ULL
	36
	37	/*
	38	* Hardware is programmed as if the hole doesn't exists with start and end
	39	* address values.
	40	*
	41	* This mask is used to remove the upper 16bits of the VA and so come up with
	42	* the linear addr value.
	43	*/
	44	#define AMDGPU_GMC_HOLE_MASK 0x0000ffffffffffffULL
	45
c1a8abd9 CK	46	/*
	47	* Ring size as power of two for the log of recent faults.
	48	*/
	49	#define AMDGPU_GMC_FAULT_RING_ORDER 8
	50	#define AMDGPU_GMC_FAULT_RING_SIZE (1 << AMDGPU_GMC_FAULT_RING_ORDER)
	51
	52	/*
	53	* Hash size as power of two for the log of recent faults
	54	*/
	55	#define AMDGPU_GMC_FAULT_HASH_ORDER 8
	56	#define AMDGPU_GMC_FAULT_HASH_SIZE (1 << AMDGPU_GMC_FAULT_HASH_ORDER)
	57
	58	/*
	59	* Number of IH timestamp ticks until a fault is considered handled
	60	*/
	61	#define AMDGPU_GMC_FAULT_TIMEOUT 5000ULL
	62
770d13b1 CK	63	struct firmware;
770d13b1 CK	64
c1a8abd9 CK	65	/*
	66	* GMC page fault information
	67	*/
	68	struct amdgpu_gmc_fault {
	69	uint64_t timestamp;
	70	uint64_t next:AMDGPU_GMC_FAULT_RING_ORDER;
	71	uint64_t key:52;
	72	};
	73
770d13b1 CK	74	/*
	75	* VMHUB structures, functions & helpers
	76	*/
2577db91 HR	77	struct amdgpu_vmhub_funcs {
	78	void (print_l2_protection_fault_status)(struct amdgpu_device adev,
	79	uint32_t status);
caa9f483	80	uint32_t (*get_invalidate_req)(unsigned int vmid, uint32_t flush_type);
2577db91 HR	81	};
2577db91 HR	82
770d13b1 CK	83	struct amdgpu_vmhub {
	84	uint32_t ctx0_ptb_addr_lo32;
	85	uint32_t ctx0_ptb_addr_hi32;
6c2c8972	86	uint32_t vm_inv_eng0_sem;
770d13b1 CK	87	uint32_t vm_inv_eng0_req;
	88	uint32_t vm_inv_eng0_ack;
	89	uint32_t vm_context0_cntl;
	90	uint32_t vm_l2_pro_fault_status;
	91	uint32_t vm_l2_pro_fault_cntl;
1f9d56c3 HR	92
	93	/*
	94	* store the register distances between two continuous context domain
	95	* and invalidation engine.
	96	*/
	97	uint32_t ctx_distance;
	98	uint32_t ctx_addr_distance; /* include LO32/HI32 */
	99	uint32_t eng_distance;
	100	uint32_t eng_addr_distance; /* include LO32/HI32 */
5befb6fc HR	101
5befb6fc HR	102	uint32_t vm_cntx_cntl_vm_fault;
2577db91 HR	103
2577db91 HR	104	const struct amdgpu_vmhub_funcs *vmhub_funcs;
770d13b1 CK	105	};
	106
	107	/*
	108	* GPU MC structures, functions & helpers
	109	*/
132f34e4 CK	110	struct amdgpu_gmc_funcs {
132f34e4 CK	111	/* flush the vm tlb via mmio */
3ff98548 OZ	112	void (flush_gpu_tlb)(struct amdgpu_device adev, uint32_t vmid,
3ff98548 OZ	113	uint32_t vmhub, uint32_t flush_type);
ea930000 AS	114	/* flush the vm tlb via pasid */
	115	int (flush_gpu_tlb_pasid)(struct amdgpu_device adev, uint16_t pasid,
	116	uint32_t flush_type, bool all_hub);
7ef11047 CK	117	/* flush the vm tlb via ring */
7ef11047 CK	118	uint64_t (emit_flush_gpu_tlb)(struct amdgpu_ring ring, unsigned vmid,
c633c00b CK	119	uint64_t pd_addr);
	120	/* Change the VMID -> PASID mapping */
	121	void (emit_pasid_mapping)(struct amdgpu_ring ring, unsigned vmid,
	122	unsigned pasid);
132f34e4 CK	123	/* enable/disable PRT support */
132f34e4 CK	124	void (set_prt)(struct amdgpu_device adev, bool enable);
71776b6d CK	125	/* map mtype to hardware flags */
71776b6d CK	126	uint64_t (map_mtype)(struct amdgpu_device adev, uint32_t flags);
132f34e4 CK	127	/* get the pde for a given mc addr */
	128	void (get_vm_pde)(struct amdgpu_device adev, int level,
	129	u64 dst, u64 flags);
cbfae36c CK	130	/* get the pte flags to use for a BO VA mapping */
	131	void (get_vm_pte)(struct amdgpu_device adev,
	132	struct amdgpu_bo_va_mapping *mapping,
	133	uint64_t *flags);
dd285c5d AD	134	/* get the amount of memory used by the vbios for pre-OS console */
dd285c5d AD	135	unsigned int (get_vbios_fb_size)(struct amdgpu_device adev);
132f34e4 CK	136	};
132f34e4 CK	137
76a5b367 AD	138	struct amdgpu_xgmi {
76a5b367 AD	139	/* from psp */
dd3c45d3	140	u64 node_id;
76a5b367 AD	141	u64 hive_id;
	142	/* fixed per family */
	143	u64 node_segment_size;
	144	/* physical node (0-3) */
	145	unsigned physical_node_id;
	146	/* number of nodes (0-4) */
	147	unsigned num_physical_nodes;
fb30fc59 SL	148	/* gpu list in the same hive */
fb30fc59 SL	149	struct list_head head;
47622ba0	150	bool supported;
029fbd43	151	struct ras_common_if *ras_if;
76a5b367 AD	152	};
76a5b367 AD	153
770d13b1	154	struct amdgpu_gmc {
f6baa074 OZ	155	/* FB's physical address in MMIO space (for CPU to
	156	* map FB). This is different compared to the agp/
	157	* gart/vram_start/end field as the later is from
	158	* GPU's view and aper_base is from CPU's view.
	159	*/
770d13b1 CK	160	resource_size_t aper_size;
	161	resource_size_t aper_base;
	162	/* for some chips with <= 32MB we need to lie
	163	* about vram size near mc fb location */
	164	u64 mc_vram_size;
	165	u64 visible_vram_size;
f6baa074 OZ	166	/* AGP aperture start and end in MC address space
	167	* Driver find a hole in the MC address space
	168	* to place AGP by setting MC_VM_AGP_BOT/TOP registers
	169	* Under VMID0, logical address == MC address. AGP
	170	* aperture maps to physical bus or IOVA addressed.
	171	* AGP aperture is used to simulate FB in ZFB case.
	172	* AGP aperture is also used for page table in system
	173	* memory (mainly for APU).
	174	*
	175	*/
d76364fc CK	176	u64 agp_size;
	177	u64 agp_start;
	178	u64 agp_end;
f6baa074 OZ	179	/* GART aperture start and end in MC address space
	180	* Driver find a hole in the MC address space
	181	* to place GART by setting VM_CONTEXT0_PAGE_TABLE_START/END_ADDR
	182	* registers
	183	* Under VMID0, logical address inside GART aperture will
	184	* be translated through gpuvm gart page table to access
	185	* paged system memory
	186	*/
770d13b1 CK	187	u64 gart_size;
	188	u64 gart_start;
	189	u64 gart_end;
f6baa074 OZ	190	/* Frame buffer aperture of this GPU device. Different from
	191	* fb_start (see below), this only covers the local GPU device.
	192	* Driver get fb_start from MC_VM_FB_LOCATION_BASE (set by vbios)
	193	* and calculate vram_start of this local device by adding an
	194	* offset inside the XGMI hive.
	195	* Under VMID0, logical address == MC address
	196	*/
770d13b1 CK	197	u64 vram_start;
770d13b1 CK	198	u64 vram_end;
6fdd68b1 AD	199	/* FB region , it's same as local vram region in single GPU, in XGMI
	200	* configuration, this region covers all GPUs in the same hive ,
	201	* each GPU in the hive has the same view of this FB region .
	202	* GPU0's vram starts at offset (0 * segment size) ,
	203	* GPU1 starts at offset (1 * segment size), etc.
	204	*/
	205	u64 fb_start;
	206	u64 fb_end;
770d13b1 CK	207	unsigned vram_width;
	208	u64 real_vram_size;
	209	int vram_mtrr;
	210	u64 mc_mask;
	211	const struct firmware fw; / MC firmware */
	212	uint32_t fw_version;
	213	struct amdgpu_irq_src vm_fault;
	214	uint32_t vram_type;
ad02e08e	215	uint8_t vram_vendor;
770d13b1 CK	216	uint32_t srbm_soft_reset;
770d13b1 CK	217	bool prt_warning;
c2ecd79b	218	uint32_t sdpif_register;
770d13b1 CK	219	/* apertures */
	220	u64 shared_aperture_start;
	221	u64 shared_aperture_end;
	222	u64 private_aperture_start;
	223	u64 private_aperture_end;
	224	/* protects concurrent invalidation */
	225	spinlock_t invalidate_lock;
	226	bool translate_further;
b97dfa27	227	struct kfd_vm_fault_info *vm_fault_info;
b97dfa27	228	atomic_t vm_fault_info_updated;
132f34e4	229
c1a8abd9 CK	230	struct amdgpu_gmc_fault fault_ring[AMDGPU_GMC_FAULT_RING_SIZE];
	231	struct {
	232	uint64_t idx:AMDGPU_GMC_FAULT_RING_ORDER;
	233	} fault_hash[AMDGPU_GMC_FAULT_HASH_SIZE];
	234	uint64_t last_fault:AMDGPU_GMC_FAULT_RING_ORDER;
	235
c6252390 LT	236	bool tmz_enabled;
c6252390 LT	237
132f34e4	238	const struct amdgpu_gmc_funcs *gmc_funcs;
76a5b367 AD	239
76a5b367 AD	240	struct amdgpu_xgmi xgmi;
791c4769	241	struct amdgpu_irq_src ecc_irq;
9b498efa	242	int noretry;
770d13b1 CK	243	};
770d13b1 CK	244
3ff98548	245	#define amdgpu_gmc_flush_gpu_tlb(adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))
ea930000 AS	246	#define amdgpu_gmc_flush_gpu_tlb_pasid(adev, pasid, type, allhub) \
	247	((adev)->gmc.gmc_funcs->flush_gpu_tlb_pasid \
	248	((adev), (pasid), (type), (allhub)))
c082b998 HR	249	#define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
c082b998 HR	250	#define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))
71776b6d	251	#define amdgpu_gmc_map_mtype(adev, flags) (adev)->gmc.gmc_funcs->map_mtype((adev),(flags))
c082b998	252	#define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags))
cbfae36c	253	#define amdgpu_gmc_get_vm_pte(adev, mapping, flags) (adev)->gmc.gmc_funcs->get_vm_pte((adev), (mapping), (flags))
dd285c5d	254	#define amdgpu_gmc_get_vbios_fb_size(adev) (adev)->gmc.gmc_funcs->get_vbios_fb_size((adev))
c082b998	255
c8c5e569 AG	256	/**
	257	* amdgpu_gmc_vram_full_visible - Check if full VRAM is visible through the BAR
	258	*
	259	* @adev: amdgpu_device pointer
	260	*
	261	* Returns:
	262	* True if full VRAM is visible through the BAR
	263	*/
	264	static inline bool amdgpu_gmc_vram_full_visible(struct amdgpu_gmc *gmc)
	265	{
	266	WARN_ON(gmc->real_vram_size < gmc->visible_vram_size);
	267
	268	return (gmc->real_vram_size == gmc->visible_vram_size);
	269	}
	270
ad9a5b78 CK	271	/**
	272	* amdgpu_gmc_sign_extend - sign extend the given gmc address
	273	*
	274	* @addr: address to extend
	275	*/
	276	static inline uint64_t amdgpu_gmc_sign_extend(uint64_t addr)
	277	{
	278	if (addr >= AMDGPU_GMC_HOLE_START)
	279	addr \|= AMDGPU_GMC_HOLE_END;
	280
	281	return addr;
	282	}
	283
24a8d289 CK	284	void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level,
24a8d289 CK	285	uint64_t addr, uint64_t flags);
6490bd76 YZ	286	int amdgpu_gmc_set_pte_pde(struct amdgpu_device adev, void cpu_pt_addr,
	287	uint32_t gpu_page_idx, uint64_t addr,
	288	uint64_t flags);
11c3a249	289	uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo);
485fc361	290	uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo);
961c75cf CK	291	void amdgpu_gmc_vram_location(struct amdgpu_device adev, struct amdgpu_gmc mc,
	292	u64 base);
	293	void amdgpu_gmc_gart_location(struct amdgpu_device *adev,
	294	struct amdgpu_gmc *mc);
d76364fc CK	295	void amdgpu_gmc_agp_location(struct amdgpu_device *adev,
d76364fc CK	296	struct amdgpu_gmc *mc);
c1a8abd9 CK	297	bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr,
c1a8abd9 CK	298	uint16_t pasid, uint64_t timestamp);
ba083492	299	int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev);
2adf1344	300	void amdgpu_gmc_ras_fini(struct amdgpu_device *adev);
bdbe90f0	301	int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev);
11c3a249	302
c6252390	303	extern void amdgpu_gmc_tmz_set(struct amdgpu_device *adev);
9b498efa	304	extern void amdgpu_gmc_noretry_set(struct amdgpu_device *adev);
c6252390	305
f2c1b5c1 HR	306	extern void
	307	amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type,
	308	bool enable);
	309
dd285c5d AD	310	void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev);
dd285c5d AD	311
770d13b1	312	#endif