[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_amdkfd.h

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */

#ifndef AMDGPU_AMDKFD_H_INCLUDED
#define AMDGPU_AMDKFD_H_INCLUDED

#include <linux/types.h>
#include <linux/mm.h>
#include <linux/kthread.h>
#include <linux/workqueue.h>
#include <kgd_kfd_interface.h>
#include <drm/ttm/ttm_execbuf_util.h>
#include "amdgpu_sync.h"
#include "amdgpu_vm.h"

extern uint64_t amdgpu_amdkfd_total_mem_size;

struct amdgpu_device;

struct kfd_bo_va_list {
	struct list_head bo_list;
	struct amdgpu_bo_va *bo_va;
	void *kgd_dev;
	bool is_mapped;
	uint64_t va;
	uint64_t pte_flags;
};

struct kgd_mem {
	struct mutex lock;
	struct amdgpu_bo *bo;
	struct list_head bo_va_list;
	/* protected by amdkfd_process_info.lock */
	struct ttm_validate_buffer validate_list;
	struct ttm_validate_buffer resv_list;
	uint32_t domain;
	unsigned int mapped_to_gpu_memory;
	uint64_t va;

	uint32_t alloc_flags;

	atomic_t invalid;
	struct amdkfd_process_info *process_info;

	struct amdgpu_sync sync;

	bool aql_queue;
	bool is_imported;
};

/* KFD Memory Eviction */
struct amdgpu_amdkfd_fence {
	struct dma_fence base;
	struct mm_struct *mm;
	spinlock_t lock;
	char timeline_name[TASK_COMM_LEN];
};

struct amdgpu_kfd_dev {
	struct kfd_dev *dev;
	uint64_t vram_used;
	bool init_complete;
};

enum kgd_engine_type {
	KGD_ENGINE_PFP = 1,
	KGD_ENGINE_ME,
	KGD_ENGINE_CE,
	KGD_ENGINE_MEC1,
	KGD_ENGINE_MEC2,
	KGD_ENGINE_RLC,
	KGD_ENGINE_SDMA1,
	KGD_ENGINE_SDMA2,
	KGD_ENGINE_MAX
};


struct amdkfd_process_info {
	/* List head of all VMs that belong to a KFD process */
	struct list_head vm_list_head;
	/* List head for all KFD BOs that belong to a KFD process. */
	struct list_head kfd_bo_list;
	/* List of userptr BOs that are valid or invalid */
	struct list_head userptr_valid_list;
	struct list_head userptr_inval_list;
	/* Lock to protect kfd_bo_list */
	struct mutex lock;

	/* Number of VMs */
	unsigned int n_vms;
	/* Eviction Fence */
	struct amdgpu_amdkfd_fence *eviction_fence;

	/* MMU-notifier related fields */
	atomic_t evicted_bos;
	struct delayed_work restore_userptr_work;
	struct pid *pid;
};

int amdgpu_amdkfd_init(void);
void amdgpu_amdkfd_fini(void);

void amdgpu_amdkfd_suspend(struct amdgpu_device *adev, bool run_pm);
int amdgpu_amdkfd_resume(struct amdgpu_device *adev, bool run_pm);
void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
			const void *ih_ring_entry);
void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
				uint32_t vmid, uint64_t gpu_addr,
				uint32_t *ib_cmd, uint32_t ib_len);
void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd);
int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct kgd_dev *kgd, uint16_t vmid);
int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid);

bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);

int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);

int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);

void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);

int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
					int queue_bit);

struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
								struct mm_struct *mm);
#if IS_ENABLED(CONFIG_HSA_AMD)
bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);
int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo);
int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
#else
static inline
bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)
{
	return false;
}

static inline
struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)
{
	return NULL;
}

static inline
int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)
{
	return 0;
}

static inline
int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm)
{
	return 0;
}
#endif
/* Shared API */
int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
				void **mem_obj, uint64_t *gpu_addr,
				void **cpu_ptr, bool mqd_gfx9);
void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
int amdgpu_amdkfd_alloc_gws(struct kgd_dev *kgd, size_t size, void **mem_obj);
void amdgpu_amdkfd_free_gws(struct kgd_dev *kgd, void *mem_obj);
int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem);
int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem);
uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
				      enum kgd_engine_type type);
void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
				      struct kfd_local_mem_info *mem_info);
uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd);

uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
				  struct kgd_dev **dmabuf_kgd,
				  uint64_t *bo_size, void *metadata_buffer,
				  size_t buffer_size, uint32_t *metadata_size,
				  uint32_t *flags);
uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);
uint64_t amdgpu_amdkfd_get_unique_id(struct kgd_dev *kgd);
uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd);
uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);

/* Read user wptr from a specified user address space with page fault
 * disabled. The memory must be pinned and mapped to the hardware when
 * this is called in hqd_load functions, so it should never fault in
 * the first place. This resolves a circular lock dependency involving
 * four locks, including the DQM lock and mmap_lock.
 */
#define read_user_wptr(mmptr, wptr, dst)				\
	({								\
		bool valid = false;					\
		if ((mmptr) && (wptr)) {				\
			pagefault_disable();				\
			if ((mmptr) == current->mm) {			\
				valid = !get_user((dst), (wptr));	\
			} else if (current->flags & PF_KTHREAD) {	\
				kthread_use_mm(mmptr);			\
				valid = !get_user((dst), (wptr));	\
				kthread_unuse_mm(mmptr);		\
			}						\
			pagefault_enable();				\
		}							\
		valid;							\
	})

/* GPUVM API */
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
					struct file *filp, u32 pasid,
					void **vm, void **process_info,
					struct dma_fence **ef);
void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm);
uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
		struct kgd_dev *kgd, uint64_t va, uint64_t size,
		void *vm, struct kgd_mem **mem,
		uint64_t *offset, uint32_t flags);
int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem, uint64_t *size);
int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
int amdgpu_amdkfd_gpuvm_sync_memory(
		struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);
int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
		struct kgd_mem *mem, void **kptr, uint64_t *size);
int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
					    struct dma_fence **ef);
int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
					      struct kfd_vm_fault_info *info);
int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
				      struct dma_buf *dmabuf,
				      uint64_t va, void *vm,
				      struct kgd_mem **mem, uint64_t *size,
				      uint64_t *mmap_offset);
int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
				struct tile_config *config);
#if IS_ENABLED(CONFIG_HSA_AMD)
void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
				struct amdgpu_vm *vm);
void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo);
#else
static inline
void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
{
}

static inline
void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
					struct amdgpu_vm *vm)
{
}

static inline
void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo)
{
}
#endif
/* KGD2KFD callbacks */
int kgd2kfd_quiesce_mm(struct mm_struct *mm);
int kgd2kfd_resume_mm(struct mm_struct *mm);
int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
						struct dma_fence *fence);
#if IS_ENABLED(CONFIG_HSA_AMD)
int kgd2kfd_init(void);
void kgd2kfd_exit(void);
struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev,
			      unsigned int asic_type, bool vf);
bool kgd2kfd_device_init(struct kfd_dev *kfd,
			 struct drm_device *ddev,
			 const struct kgd2kfd_shared_resources *gpu_resources);
void kgd2kfd_device_exit(struct kfd_dev *kfd);
void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm);
int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm);
int kgd2kfd_pre_reset(struct kfd_dev *kfd);
int kgd2kfd_post_reset(struct kfd_dev *kfd);
void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd);
void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask);
#else
static inline int kgd2kfd_init(void)
{
	return -ENOENT;
}

static inline void kgd2kfd_exit(void)
{
}

static inline
struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev,
					unsigned int asic_type, bool vf)
{
	return NULL;
}

static inline
bool kgd2kfd_device_init(struct kfd_dev *kfd, struct drm_device *ddev,
				const struct kgd2kfd_shared_resources *gpu_resources)
{
	return false;
}

static inline void kgd2kfd_device_exit(struct kfd_dev *kfd)
{
}

static inline void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)
{
}

static inline int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)
{
	return 0;
}

static inline int kgd2kfd_pre_reset(struct kfd_dev *kfd)
{
	return 0;
}

static inline int kgd2kfd_post_reset(struct kfd_dev *kfd)
{
	return 0;
}

static inline
void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
{
}

static inline
void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
{
}

static inline
void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask)
{
}
#endif
#endif /* AMDGPU_AMDKFD_H_INCLUDED */
Commit	Line	Data
130e0371 OG	1	/*
	2	* Copyright 2014 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*/
	22
	23	/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */
	24
	25	#ifndef AMDGPU_AMDKFD_H_INCLUDED
	26	#define AMDGPU_AMDKFD_H_INCLUDED
	27
	28	#include <linux/types.h>
7420f482	29	#include <linux/mm.h>
9bf5b9eb	30	#include <linux/kthread.h>
5ae0283e	31	#include <linux/workqueue.h>
130e0371	32	#include <kgd_kfd_interface.h>
a46a2cd1 FK	33	#include <drm/ttm/ttm_execbuf_util.h>
	34	#include "amdgpu_sync.h"
	35	#include "amdgpu_vm.h"
130e0371	36
611736d8	37	extern uint64_t amdgpu_amdkfd_total_mem_size;
d8d019cc	38
130e0371 OG	39	struct amdgpu_device;
130e0371 OG	40
a46a2cd1 FK	41	struct kfd_bo_va_list {
	42	struct list_head bo_list;
	43	struct amdgpu_bo_va *bo_va;
	44	void *kgd_dev;
	45	bool is_mapped;
	46	uint64_t va;
	47	uint64_t pte_flags;
	48	};
	49
130e0371	50	struct kgd_mem {
a46a2cd1	51	struct mutex lock;
130e0371	52	struct amdgpu_bo *bo;
a46a2cd1 FK	53	struct list_head bo_va_list;
	54	/* protected by amdkfd_process_info.lock */
	55	struct ttm_validate_buffer validate_list;
	56	struct ttm_validate_buffer resv_list;
	57	uint32_t domain;
	58	unsigned int mapped_to_gpu_memory;
	59	uint64_t va;
	60
d0ba51b1	61	uint32_t alloc_flags;
a46a2cd1	62
5ae0283e	63	atomic_t invalid;
a46a2cd1 FK	64	struct amdkfd_process_info *process_info;
	65
	66	struct amdgpu_sync sync;
	67
	68	bool aql_queue;
d4566dee	69	bool is_imported;
130e0371 OG	70	};
130e0371 OG	71
d8d019cc FK	72	/* KFD Memory Eviction */
	73	struct amdgpu_amdkfd_fence {
	74	struct dma_fence base;
	75	struct mm_struct *mm;
	76	spinlock_t lock;
	77	char timeline_name[TASK_COMM_LEN];
	78	};
	79
611736d8 FK	80	struct amdgpu_kfd_dev {
	81	struct kfd_dev *dev;
	82	uint64_t vram_used;
8e2712e7	83	bool init_complete;
611736d8 FK	84	};
611736d8 FK	85
0da8b10e AL	86	enum kgd_engine_type {
	87	KGD_ENGINE_PFP = 1,
	88	KGD_ENGINE_ME,
	89	KGD_ENGINE_CE,
	90	KGD_ENGINE_MEC1,
	91	KGD_ENGINE_MEC2,
	92	KGD_ENGINE_RLC,
	93	KGD_ENGINE_SDMA1,
	94	KGD_ENGINE_SDMA2,
	95	KGD_ENGINE_MAX
	96	};
	97
d8d019cc	98
a46a2cd1 FK	99	struct amdkfd_process_info {
	100	/* List head of all VMs that belong to a KFD process */
	101	struct list_head vm_list_head;
	102	/* List head for all KFD BOs that belong to a KFD process. */
	103	struct list_head kfd_bo_list;
5ae0283e FK	104	/* List of userptr BOs that are valid or invalid */
	105	struct list_head userptr_valid_list;
	106	struct list_head userptr_inval_list;
a46a2cd1 FK	107	/* Lock to protect kfd_bo_list */
	108	struct mutex lock;
	109
	110	/* Number of VMs */
	111	unsigned int n_vms;
	112	/* Eviction Fence */
	113	struct amdgpu_amdkfd_fence *eviction_fence;
5ae0283e FK	114
	115	/* MMU-notifier related fields */
	116	atomic_t evicted_bos;
	117	struct delayed_work restore_userptr_work;
	118	struct pid *pid;
a46a2cd1 FK	119	};
a46a2cd1 FK	120
efb1c658	121	int amdgpu_amdkfd_init(void);
130e0371 OG	122	void amdgpu_amdkfd_fini(void);
130e0371 OG	123
9593f4d6 RB	124	void amdgpu_amdkfd_suspend(struct amdgpu_device *adev, bool run_pm);
9593f4d6 RB	125	int amdgpu_amdkfd_resume(struct amdgpu_device *adev, bool run_pm);
dc102c43	126	void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
130e0371	127	const void *ih_ring_entry);
dc102c43 AR	128	void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
	129	void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
	130	void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
4c660c8f FK	131	int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
	132	uint32_t vmid, uint64_t gpu_addr,
	133	uint32_t *ib_cmd, uint32_t ib_len);
01c097db	134	void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
aabf3a95	135	bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd);
ffa02269 AS	136	int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct kgd_dev *kgd, uint16_t vmid);
ffa02269 AS	137	int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid);
4c660c8f	138
155494db FK	139	bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
155494db FK	140
5c6dd71e SL	141	int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);
	142
	143	int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);
	144
24da5a9c SL	145	void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);
24da5a9c SL	146
d09f85d5 YZ	147	int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev,
	148	int queue_bit);
	149
cd63989e LY	150	struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
	151	struct mm_struct *mm);
	152	#if IS_ENABLED(CONFIG_HSA_AMD)
	153	bool amdkfd_fence_check_mm(struct dma_fence f, struct mm_struct mm);
	154	struct amdgpu_amdkfd_fence to_amdgpu_amdkfd_fence(struct dma_fence f);
	155	int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo);
	156	int amdgpu_amdkfd_evict_userptr(struct kgd_mem mem, struct mm_struct mm);
	157	#else
	158	static inline
	159	bool amdkfd_fence_check_mm(struct dma_fence f, struct mm_struct mm)
	160	{
	161	return false;
	162	}
	163
	164	static inline
	165	struct amdgpu_amdkfd_fence to_amdgpu_amdkfd_fence(struct dma_fence f)
	166	{
	167	return NULL;
	168	}
	169
	170	static inline
	171	int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo)
	172	{
	173	return 0;
	174	}
	175
	176	static inline
	177	int amdgpu_amdkfd_evict_userptr(struct kgd_mem mem, struct mm_struct mm)
	178	{
	179	return 0;
	180	}
	181	#endif
130e0371	182	/* Shared API */
7cd52c91 AL	183	int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
	184	void *mem_obj, uint64_t gpu_addr,
	185	void **cpu_ptr, bool mqd_gfx9);
	186	void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev kgd, void mem_obj);
ca66fb8f OZ	187	int amdgpu_amdkfd_alloc_gws(struct kgd_dev kgd, size_t size, void *mem_obj);
ca66fb8f OZ	188	void amdgpu_amdkfd_free_gws(struct kgd_dev kgd, void mem_obj);
71efab6a OZ	189	int amdgpu_amdkfd_add_gws_to_process(void info, void gws, struct kgd_mem **mem);
71efab6a OZ	190	int amdgpu_amdkfd_remove_gws_from_process(void info, void mem);
0da8b10e AL	191	uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
0da8b10e AL	192	enum kgd_engine_type type);
7cd52c91 AL	193	void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
	194	struct kfd_local_mem_info *mem_info);
	195	uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd);
	196
	197	uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
	198	void amdgpu_amdkfd_get_cu_info(struct kgd_dev kgd, struct kfd_cu_info cu_info);
1dde0ea9 FK	199	int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd,
	200	struct kgd_dev **dmabuf_kgd,
	201	uint64_t bo_size, void metadata_buffer,
	202	size_t buffer_size, uint32_t *metadata_size,
	203	uint32_t *flags);
9f0a0b41	204	uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
db8b62c0	205	uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);
0c663695	206	uint64_t amdgpu_amdkfd_get_unique_id(struct kgd_dev *kgd);
d8e408a8	207	uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd);
29e76462	208	uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
c6d1ec41	209	uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
9b498efa	210	int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
da361dd1	211	uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev dst, struct kgd_dev src);
130e0371	212
cd05c865 FK	213	/* Read user wptr from a specified user address space with page fault
	214	* disabled. The memory must be pinned and mapped to the hardware when
	215	* this is called in hqd_load functions, so it should never fault in
	216	* the first place. This resolves a circular lock dependency involving
c1e8d7c6	217	* four locks, including the DQM lock and mmap_lock.
cd05c865	218	*/
70539bd7 FK	219	#define read_user_wptr(mmptr, wptr, dst) \
	220	({ \
	221	bool valid = false; \
	222	if ((mmptr) && (wptr)) { \
cd05c865	223	pagefault_disable(); \
70539bd7 FK	224	if ((mmptr) == current->mm) { \
70539bd7 FK	225	valid = !get_user((dst), (wptr)); \
8449d150	226	} else if (current->flags & PF_KTHREAD) { \
f5678e7f	227	kthread_use_mm(mmptr); \
70539bd7	228	valid = !get_user((dst), (wptr)); \
f5678e7f	229	kthread_unuse_mm(mmptr); \
70539bd7	230	} \
cd05c865	231	pagefault_enable(); \
70539bd7 FK	232	} \
	233	valid; \
	234	})
	235
a46a2cd1	236	/* GPUVM API */
ede0dd86	237	int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
c7b6bac9	238	struct file *filp, u32 pasid,
fcdfa432 OG	239	void vm, void process_info,
fcdfa432 OG	240	struct dma_fence **ef);
bf47afba	241	void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev kgd, void vm);
e715c6d0	242	uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
a46a2cd1 FK	243	int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
	244	struct kgd_dev *kgd, uint64_t va, uint64_t size,
	245	void vm, struct kgd_mem *mem,
	246	uint64_t *offset, uint32_t flags);
	247	int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
d4566dee	248	struct kgd_dev kgd, struct kgd_mem mem, uint64_t *size);
a46a2cd1 FK	249	int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
	250	struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	251	int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
	252	struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	253	int amdgpu_amdkfd_gpuvm_sync_memory(
	254	struct kgd_dev kgd, struct kgd_mem mem, bool intr);
	255	int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
	256	struct kgd_mem mem, void kptr, uint64_t size);
	257	int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
	258	struct dma_fence **ef);
b97dfa27	259	int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
b97dfa27	260	struct kfd_vm_fault_info *info);
1dde0ea9 FK	261	int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
	262	struct dma_buf *dmabuf,
	263	uint64_t va, void *vm,
	264	struct kgd_mem *mem, uint64_t size,
	265	uint64_t *mmap_offset);
fd7d08ba YZ	266	int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
fd7d08ba YZ	267	struct tile_config *config);
cd63989e LY	268	#if IS_ENABLED(CONFIG_HSA_AMD)
	269	void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
	270	void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
	271	struct amdgpu_vm *vm);
	272	void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo);
	273	#else
	274	static inline
	275	void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
	276	{
	277	}
fd7d08ba	278
cd63989e LY	279	static inline
	280	void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
	281	struct amdgpu_vm *vm)
	282	{
	283	}
	284
	285	static inline
	286	void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo)
	287	{
	288	}
	289	#endif
2d3d25b6	290	/* KGD2KFD callbacks */
cd63989e LY	291	int kgd2kfd_quiesce_mm(struct mm_struct *mm);
	292	int kgd2kfd_resume_mm(struct mm_struct *mm);
	293	int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
	294	struct dma_fence *fence);
	295	#if IS_ENABLED(CONFIG_HSA_AMD)
308176d6	296	int kgd2kfd_init(void);
2d3d25b6 AL	297	void kgd2kfd_exit(void);
2d3d25b6 AL	298	struct kfd_dev kgd2kfd_probe(struct kgd_dev kgd, struct pci_dev *pdev,
050091ab	299	unsigned int asic_type, bool vf);
2d3d25b6	300	bool kgd2kfd_device_init(struct kfd_dev *kfd,
3a0c3423	301	struct drm_device *ddev,
2d3d25b6 AL	302	const struct kgd2kfd_shared_resources *gpu_resources);
2d3d25b6 AL	303	void kgd2kfd_device_exit(struct kfd_dev *kfd);
9593f4d6 RB	304	void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm);
9593f4d6 RB	305	int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm);
2d3d25b6 AL	306	int kgd2kfd_pre_reset(struct kfd_dev *kfd);
	307	int kgd2kfd_post_reset(struct kfd_dev *kfd);
	308	void kgd2kfd_interrupt(struct kfd_dev kfd, const void ih_ring_entry);
9b54d201	309	void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd);
2c2b0d88	310	void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask);
cd63989e LY	311	#else
	312	static inline int kgd2kfd_init(void)
	313	{
	314	return -ENOENT;
	315	}
2d3d25b6	316
cd63989e LY	317	static inline void kgd2kfd_exit(void)
	318	{
	319	}
	320
	321	static inline
	322	struct kfd_dev kgd2kfd_probe(struct kgd_dev kgd, struct pci_dev *pdev,
	323	unsigned int asic_type, bool vf)
	324	{
	325	return NULL;
	326	}
	327
	328	static inline
	329	bool kgd2kfd_device_init(struct kfd_dev kfd, struct drm_device ddev,
	330	const struct kgd2kfd_shared_resources *gpu_resources)
	331	{
	332	return false;
	333	}
	334
	335	static inline void kgd2kfd_device_exit(struct kfd_dev *kfd)
	336	{
	337	}
	338
	339	static inline void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)
	340	{
	341	}
	342
	343	static inline int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)
	344	{
	345	return 0;
	346	}
	347
	348	static inline int kgd2kfd_pre_reset(struct kfd_dev *kfd)
	349	{
	350	return 0;
	351	}
	352
	353	static inline int kgd2kfd_post_reset(struct kfd_dev *kfd)
	354	{
	355	return 0;
	356	}
	357
	358	static inline
	359	void kgd2kfd_interrupt(struct kfd_dev kfd, const void ih_ring_entry)
	360	{
	361	}
	362
	363	static inline
	364	void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
	365	{
	366	}
	367
	368	static inline
	369	void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask)
	370	{
	371	}
	372	#endif
130e0371	373	#endif /* AMDGPU_AMDKFD_H_INCLUDED */