[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_amdkfd.h

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */

#ifndef AMDGPU_AMDKFD_H_INCLUDED
#define AMDGPU_AMDKFD_H_INCLUDED

#include <linux/types.h>
#include <linux/mm.h>
#include <linux/workqueue.h>
#include <kgd_kfd_interface.h>
#include <drm/ttm/ttm_execbuf_util.h>
#include "amdgpu_sync.h"
#include "amdgpu_vm.h"

extern const struct kgd2kfd_calls *kgd2kfd;

struct amdgpu_device;

struct kfd_bo_va_list {
	struct list_head bo_list;
	struct amdgpu_bo_va *bo_va;
	void *kgd_dev;
	bool is_mapped;
	uint64_t va;
	uint64_t pte_flags;
};

struct kgd_mem {
	struct mutex lock;
	struct amdgpu_bo *bo;
	struct list_head bo_va_list;
	/* protected by amdkfd_process_info.lock */
	struct ttm_validate_buffer validate_list;
	struct ttm_validate_buffer resv_list;
	uint32_t domain;
	unsigned int mapped_to_gpu_memory;
	uint64_t va;

	uint32_t mapping_flags;

	atomic_t invalid;
	struct amdkfd_process_info *process_info;
	struct page **user_pages;

	struct amdgpu_sync sync;

	bool aql_queue;
};

/* KFD Memory Eviction */
struct amdgpu_amdkfd_fence {
	struct dma_fence base;
	struct mm_struct *mm;
	spinlock_t lock;
	char timeline_name[TASK_COMM_LEN];
};

struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
						       struct mm_struct *mm);
bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);

struct amdkfd_process_info {
	/* List head of all VMs that belong to a KFD process */
	struct list_head vm_list_head;
	/* List head for all KFD BOs that belong to a KFD process. */
	struct list_head kfd_bo_list;
	/* List of userptr BOs that are valid or invalid */
	struct list_head userptr_valid_list;
	struct list_head userptr_inval_list;
	/* Lock to protect kfd_bo_list */
	struct mutex lock;

	/* Number of VMs */
	unsigned int n_vms;
	/* Eviction Fence */
	struct amdgpu_amdkfd_fence *eviction_fence;

	/* MMU-notifier related fields */
	atomic_t evicted_bos;
	struct delayed_work restore_userptr_work;
	struct pid *pid;
};

int amdgpu_amdkfd_init(void);
void amdgpu_amdkfd_fini(void);

void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
			const void *ih_ring_entry);
void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);

int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
				uint32_t vmid, uint64_t gpu_addr,
				uint32_t *ib_cmd, uint32_t ib_len);
void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);

struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);

bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);

int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);

int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);

void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);

/* Shared API */
int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
				void **mem_obj, uint64_t *gpu_addr,
				void **cpu_ptr, bool mqd_gfx9);
void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
				      struct kfd_local_mem_info *mem_info);
uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd);

uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
void amdgpu_amdkfd_get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);

#define read_user_wptr(mmptr, wptr, dst)				\
	({								\
		bool valid = false;					\
		if ((mmptr) && (wptr)) {				\
			if ((mmptr) == current->mm) {			\
				valid = !get_user((dst), (wptr));	\
			} else if (current->mm == NULL) {		\
				use_mm(mmptr);				\
				valid = !get_user((dst), (wptr));	\
				unuse_mm(mmptr);			\
			}						\
		}							\
		valid;							\
	})

/* GPUVM API */
int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, unsigned int pasid,
					void **vm, void **process_info,
					struct dma_fence **ef);
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
					struct file *filp, unsigned int pasid,
					void **vm, void **process_info,
					struct dma_fence **ef);
void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
				struct amdgpu_vm *vm);
void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm);
void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm);
uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
		struct kgd_dev *kgd, uint64_t va, uint64_t size,
		void *vm, struct kgd_mem **mem,
		uint64_t *offset, uint32_t flags);
int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem);
int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
int amdgpu_amdkfd_gpuvm_sync_memory(
		struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);
int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
		struct kgd_mem *mem, void **kptr, uint64_t *size);
int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
					    struct dma_fence **ef);

int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
					      struct kfd_vm_fault_info *info);

void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);

#endif /* AMDGPU_AMDKFD_H_INCLUDED */
Commit	Line	Data
130e0371 OG	1	/*
	2	* Copyright 2014 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*/
	22
	23	/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */
	24
	25	#ifndef AMDGPU_AMDKFD_H_INCLUDED
	26	#define AMDGPU_AMDKFD_H_INCLUDED
	27
	28	#include <linux/types.h>
7420f482	29	#include <linux/mm.h>
5ae0283e	30	#include <linux/workqueue.h>
130e0371	31	#include <kgd_kfd_interface.h>
a46a2cd1 FK	32	#include <drm/ttm/ttm_execbuf_util.h>
	33	#include "amdgpu_sync.h"
	34	#include "amdgpu_vm.h"
130e0371	35
d8d019cc FK	36	extern const struct kgd2kfd_calls *kgd2kfd;
d8d019cc FK	37
130e0371 OG	38	struct amdgpu_device;
130e0371 OG	39
a46a2cd1 FK	40	struct kfd_bo_va_list {
	41	struct list_head bo_list;
	42	struct amdgpu_bo_va *bo_va;
	43	void *kgd_dev;
	44	bool is_mapped;
	45	uint64_t va;
	46	uint64_t pte_flags;
	47	};
	48
130e0371	49	struct kgd_mem {
a46a2cd1	50	struct mutex lock;
130e0371	51	struct amdgpu_bo *bo;
a46a2cd1 FK	52	struct list_head bo_va_list;
	53	/* protected by amdkfd_process_info.lock */
	54	struct ttm_validate_buffer validate_list;
	55	struct ttm_validate_buffer resv_list;
	56	uint32_t domain;
	57	unsigned int mapped_to_gpu_memory;
	58	uint64_t va;
	59
	60	uint32_t mapping_flags;
	61
5ae0283e	62	atomic_t invalid;
a46a2cd1	63	struct amdkfd_process_info *process_info;
5ae0283e	64	struct page **user_pages;
a46a2cd1 FK	65
	66	struct amdgpu_sync sync;
	67
	68	bool aql_queue;
130e0371 OG	69	};
130e0371 OG	70
d8d019cc FK	71	/* KFD Memory Eviction */
	72	struct amdgpu_amdkfd_fence {
	73	struct dma_fence base;
	74	struct mm_struct *mm;
	75	spinlock_t lock;
	76	char timeline_name[TASK_COMM_LEN];
	77	};
	78
	79	struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
	80	struct mm_struct *mm);
	81	bool amdkfd_fence_check_mm(struct dma_fence f, struct mm_struct mm);
	82	struct amdgpu_amdkfd_fence to_amdgpu_amdkfd_fence(struct dma_fence f);
	83
a46a2cd1 FK	84	struct amdkfd_process_info {
	85	/* List head of all VMs that belong to a KFD process */
	86	struct list_head vm_list_head;
	87	/* List head for all KFD BOs that belong to a KFD process. */
	88	struct list_head kfd_bo_list;
5ae0283e FK	89	/* List of userptr BOs that are valid or invalid */
	90	struct list_head userptr_valid_list;
	91	struct list_head userptr_inval_list;
a46a2cd1 FK	92	/* Lock to protect kfd_bo_list */
	93	struct mutex lock;
	94
	95	/* Number of VMs */
	96	unsigned int n_vms;
	97	/* Eviction Fence */
	98	struct amdgpu_amdkfd_fence *eviction_fence;
5ae0283e FK	99
	100	/* MMU-notifier related fields */
	101	atomic_t evicted_bos;
	102	struct delayed_work restore_userptr_work;
	103	struct pid *pid;
a46a2cd1 FK	104	};
a46a2cd1 FK	105
efb1c658	106	int amdgpu_amdkfd_init(void);
130e0371 OG	107	void amdgpu_amdkfd_fini(void);
130e0371 OG	108
dc102c43 AR	109	void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
	110	int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
	111	void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
130e0371	112	const void *ih_ring_entry);
dc102c43 AR	113	void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
	114	void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
	115	void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
130e0371	116
e52482de	117	int amdgpu_amdkfd_evict_userptr(struct kgd_mem mem, struct mm_struct mm);
4c660c8f FK	118	int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
	119	uint32_t vmid, uint64_t gpu_addr,
	120	uint32_t *ib_cmd, uint32_t ib_len);
01c097db	121	void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
4c660c8f	122
130e0371	123	struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
ff758a12	124	struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
d5a114a6	125	struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);
130e0371	126
155494db FK	127	bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
155494db FK	128
5c6dd71e SL	129	int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);
	130
	131	int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);
	132
24da5a9c SL	133	void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);
24da5a9c SL	134
130e0371	135	/* Shared API */
7cd52c91 AL	136	int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
	137	void *mem_obj, uint64_t gpu_addr,
	138	void **cpu_ptr, bool mqd_gfx9);
	139	void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev kgd, void mem_obj);
	140	void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
	141	struct kfd_local_mem_info *mem_info);
	142	uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd);
	143
	144	uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
	145	void amdgpu_amdkfd_get_cu_info(struct kgd_dev kgd, struct kfd_cu_info cu_info);
9f0a0b41	146	uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
db8b62c0	147	uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd);
130e0371	148
70539bd7 FK	149	#define read_user_wptr(mmptr, wptr, dst) \
	150	({ \
	151	bool valid = false; \
	152	if ((mmptr) && (wptr)) { \
	153	if ((mmptr) == current->mm) { \
	154	valid = !get_user((dst), (wptr)); \
	155	} else if (current->mm == NULL) { \
	156	use_mm(mmptr); \
	157	valid = !get_user((dst), (wptr)); \
	158	unuse_mm(mmptr); \
	159	} \
	160	} \
	161	valid; \
	162	})
	163
a46a2cd1	164	/* GPUVM API */
1685b01a OZ	165	int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, unsigned int pasid,
1685b01a OZ	166	void vm, void process_info,
fcdfa432	167	struct dma_fence **ef);
ede0dd86	168	int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
1685b01a	169	struct file *filp, unsigned int pasid,
fcdfa432 OG	170	void vm, void process_info,
fcdfa432 OG	171	struct dma_fence **ef);
ede0dd86	172	void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
fcdfa432	173	struct amdgpu_vm *vm);
a46a2cd1	174	void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev kgd, void vm);
bf47afba	175	void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev kgd, void vm);
e715c6d0	176	uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
a46a2cd1 FK	177	int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
	178	struct kgd_dev *kgd, uint64_t va, uint64_t size,
	179	void vm, struct kgd_mem *mem,
	180	uint64_t *offset, uint32_t flags);
	181	int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
	182	struct kgd_dev kgd, struct kgd_mem mem);
	183	int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
	184	struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	185	int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
	186	struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	187	int amdgpu_amdkfd_gpuvm_sync_memory(
	188	struct kgd_dev kgd, struct kgd_mem mem, bool intr);
	189	int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
	190	struct kgd_mem mem, void kptr, uint64_t size);
	191	int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
	192	struct dma_fence **ef);
	193
b97dfa27	194	int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
	195	struct kfd_vm_fault_info *info);
	196
a46a2cd1 FK	197	void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
	198	void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);
	199
130e0371	200	#endif /* AMDGPU_AMDKFD_H_INCLUDED */