[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_amdkfd.h

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */

#ifndef AMDGPU_AMDKFD_H_INCLUDED
#define AMDGPU_AMDKFD_H_INCLUDED

#include <linux/types.h>
#include <linux/mm.h>
#include <linux/mmu_context.h>
#include <linux/workqueue.h>
#include <kgd_kfd_interface.h>
#include <drm/ttm/ttm_execbuf_util.h>
#include "amdgpu_sync.h"
#include "amdgpu_vm.h"

extern const struct kgd2kfd_calls *kgd2kfd;

struct amdgpu_device;

struct kfd_bo_va_list {
	struct list_head bo_list;
	struct amdgpu_bo_va *bo_va;
	void *kgd_dev;
	bool is_mapped;
	uint64_t va;
	uint64_t pte_flags;
};

struct kgd_mem {
	struct mutex lock;
	struct amdgpu_bo *bo;
	struct list_head bo_va_list;
	/* protected by amdkfd_process_info.lock */
	struct ttm_validate_buffer validate_list;
	struct ttm_validate_buffer resv_list;
	uint32_t domain;
	unsigned int mapped_to_gpu_memory;
	uint64_t va;

	uint32_t mapping_flags;

	atomic_t invalid;
	struct amdkfd_process_info *process_info;
	struct page **user_pages;

	struct amdgpu_sync sync;

	bool aql_queue;
};

/* KFD Memory Eviction */
struct amdgpu_amdkfd_fence {
	struct dma_fence base;
	struct mm_struct *mm;
	spinlock_t lock;
	char timeline_name[TASK_COMM_LEN];
};

struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
						       struct mm_struct *mm);
bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);

struct amdkfd_process_info {
	/* List head of all VMs that belong to a KFD process */
	struct list_head vm_list_head;
	/* List head for all KFD BOs that belong to a KFD process. */
	struct list_head kfd_bo_list;
	/* List of userptr BOs that are valid or invalid */
	struct list_head userptr_valid_list;
	struct list_head userptr_inval_list;
	/* Lock to protect kfd_bo_list */
	struct mutex lock;

	/* Number of VMs */
	unsigned int n_vms;
	/* Eviction Fence */
	struct amdgpu_amdkfd_fence *eviction_fence;

	/* MMU-notifier related fields */
	atomic_t evicted_bos;
	struct delayed_work restore_userptr_work;
	struct pid *pid;
};

int amdgpu_amdkfd_init(void);
void amdgpu_amdkfd_fini(void);

void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
			const void *ih_ring_entry);
void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);

int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
				uint32_t vmid, uint64_t gpu_addr,
				uint32_t *ib_cmd, uint32_t ib_len);
void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);

struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);

bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);

int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);

int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);

void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);

/* Shared API */
int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
			void **mem_obj, uint64_t *gpu_addr,
			void **cpu_ptr, bool mqd_gfx9);
void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
void get_local_mem_info(struct kgd_dev *kgd,
			struct kfd_local_mem_info *mem_info);
uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);

uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);

#define read_user_wptr(mmptr, wptr, dst)				\
	({								\
		bool valid = false;					\
		if ((mmptr) && (wptr)) {				\
			if ((mmptr) == current->mm) {			\
				valid = !get_user((dst), (wptr));	\
			} else if (current->mm == NULL) {		\
				use_mm(mmptr);				\
				valid = !get_user((dst), (wptr));	\
				unuse_mm(mmptr);			\
			}						\
		}							\
		valid;							\
	})

/* GPUVM API */
int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
					void **process_info,
					struct dma_fence **ef);
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
					struct file *filp,
					void **vm, void **process_info,
					struct dma_fence **ef);
void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
				struct amdgpu_vm *vm);
void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm);
uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
		struct kgd_dev *kgd, uint64_t va, uint64_t size,
		void *vm, struct kgd_mem **mem,
		uint64_t *offset, uint32_t flags);
int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem);
int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
int amdgpu_amdkfd_gpuvm_sync_memory(
		struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);
int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
		struct kgd_mem *mem, void **kptr, uint64_t *size);
int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
					    struct dma_fence **ef);

int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
					      struct kfd_vm_fault_info *info);

void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);

#endif /* AMDGPU_AMDKFD_H_INCLUDED */
Commit	Line	Data
130e0371 OG	1	/*
	2	* Copyright 2014 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*/
	22
	23	/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */
	24
	25	#ifndef AMDGPU_AMDKFD_H_INCLUDED
	26	#define AMDGPU_AMDKFD_H_INCLUDED
	27
	28	#include <linux/types.h>
7420f482	29	#include <linux/mm.h>
70539bd7	30	#include <linux/mmu_context.h>
5ae0283e	31	#include <linux/workqueue.h>
130e0371	32	#include <kgd_kfd_interface.h>
a46a2cd1 FK	33	#include <drm/ttm/ttm_execbuf_util.h>
	34	#include "amdgpu_sync.h"
	35	#include "amdgpu_vm.h"
130e0371	36
d8d019cc FK	37	extern const struct kgd2kfd_calls *kgd2kfd;
d8d019cc FK	38
130e0371 OG	39	struct amdgpu_device;
130e0371 OG	40
a46a2cd1 FK	41	struct kfd_bo_va_list {
	42	struct list_head bo_list;
	43	struct amdgpu_bo_va *bo_va;
	44	void *kgd_dev;
	45	bool is_mapped;
	46	uint64_t va;
	47	uint64_t pte_flags;
	48	};
	49
130e0371	50	struct kgd_mem {
a46a2cd1	51	struct mutex lock;
130e0371	52	struct amdgpu_bo *bo;
a46a2cd1 FK	53	struct list_head bo_va_list;
	54	/* protected by amdkfd_process_info.lock */
	55	struct ttm_validate_buffer validate_list;
	56	struct ttm_validate_buffer resv_list;
	57	uint32_t domain;
	58	unsigned int mapped_to_gpu_memory;
	59	uint64_t va;
	60
	61	uint32_t mapping_flags;
	62
5ae0283e	63	atomic_t invalid;
a46a2cd1	64	struct amdkfd_process_info *process_info;
5ae0283e	65	struct page **user_pages;
a46a2cd1 FK	66
	67	struct amdgpu_sync sync;
	68
	69	bool aql_queue;
130e0371 OG	70	};
130e0371 OG	71
d8d019cc FK	72	/* KFD Memory Eviction */
	73	struct amdgpu_amdkfd_fence {
	74	struct dma_fence base;
	75	struct mm_struct *mm;
	76	spinlock_t lock;
	77	char timeline_name[TASK_COMM_LEN];
	78	};
	79
	80	struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
	81	struct mm_struct *mm);
	82	bool amdkfd_fence_check_mm(struct dma_fence f, struct mm_struct mm);
	83	struct amdgpu_amdkfd_fence to_amdgpu_amdkfd_fence(struct dma_fence f);
	84
a46a2cd1 FK	85	struct amdkfd_process_info {
	86	/* List head of all VMs that belong to a KFD process */
	87	struct list_head vm_list_head;
	88	/* List head for all KFD BOs that belong to a KFD process. */
	89	struct list_head kfd_bo_list;
5ae0283e FK	90	/* List of userptr BOs that are valid or invalid */
	91	struct list_head userptr_valid_list;
	92	struct list_head userptr_inval_list;
a46a2cd1 FK	93	/* Lock to protect kfd_bo_list */
	94	struct mutex lock;
	95
	96	/* Number of VMs */
	97	unsigned int n_vms;
	98	/* Eviction Fence */
	99	struct amdgpu_amdkfd_fence *eviction_fence;
5ae0283e FK	100
	101	/* MMU-notifier related fields */
	102	atomic_t evicted_bos;
	103	struct delayed_work restore_userptr_work;
	104	struct pid *pid;
a46a2cd1 FK	105	};
a46a2cd1 FK	106
efb1c658	107	int amdgpu_amdkfd_init(void);
130e0371 OG	108	void amdgpu_amdkfd_fini(void);
130e0371 OG	109
dc102c43 AR	110	void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
	111	int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
	112	void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
130e0371	113	const void *ih_ring_entry);
dc102c43 AR	114	void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
	115	void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
	116	void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
130e0371	117
e52482de	118	int amdgpu_amdkfd_evict_userptr(struct kgd_mem mem, struct mm_struct mm);
4c660c8f FK	119	int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
	120	uint32_t vmid, uint64_t gpu_addr,
	121	uint32_t *ib_cmd, uint32_t ib_len);
01c097db	122	void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
4c660c8f	123
130e0371	124	struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
ff758a12	125	struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
d5a114a6	126	struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void);
130e0371	127
155494db FK	128	bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
155494db FK	129
5c6dd71e SL	130	int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);
	131
	132	int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);
	133
24da5a9c SL	134	void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);
24da5a9c SL	135
130e0371 OG	136	/* Shared API */
	137	int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
	138	void *mem_obj, uint64_t gpu_addr,
15426dbb	139	void **cpu_ptr, bool mqd_gfx9);
130e0371	140	void free_gtt_mem(struct kgd_dev kgd, void mem_obj);
30f1c042 HK	141	void get_local_mem_info(struct kgd_dev *kgd,
30f1c042 HK	142	struct kfd_local_mem_info *mem_info);
130e0371 OG	143	uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
	144
	145	uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
ebdebf42	146	void get_cu_info(struct kgd_dev kgd, struct kfd_cu_info cu_info);
9f0a0b41	147	uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
130e0371	148
70539bd7 FK	149	#define read_user_wptr(mmptr, wptr, dst) \
	150	({ \
	151	bool valid = false; \
	152	if ((mmptr) && (wptr)) { \
	153	if ((mmptr) == current->mm) { \
	154	valid = !get_user((dst), (wptr)); \
	155	} else if (current->mm == NULL) { \
	156	use_mm(mmptr); \
	157	valid = !get_user((dst), (wptr)); \
	158	unuse_mm(mmptr); \
	159	} \
	160	} \
	161	valid; \
	162	})
	163
a46a2cd1 FK	164	/* GPUVM API */
a46a2cd1 FK	165	int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev kgd, void *vm,
fcdfa432 OG	166	void **process_info,
fcdfa432 OG	167	struct dma_fence **ef);
ede0dd86	168	int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
fcdfa432 OG	169	struct file *filp,
	170	void vm, void process_info,
	171	struct dma_fence **ef);
ede0dd86	172	void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
fcdfa432	173	struct amdgpu_vm *vm);
a46a2cd1 FK	174	void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev kgd, void vm);
	175	uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
	176	int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
	177	struct kgd_dev *kgd, uint64_t va, uint64_t size,
	178	void vm, struct kgd_mem *mem,
	179	uint64_t *offset, uint32_t flags);
	180	int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
	181	struct kgd_dev kgd, struct kgd_mem mem);
	182	int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
	183	struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	184	int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
	185	struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	186	int amdgpu_amdkfd_gpuvm_sync_memory(
	187	struct kgd_dev kgd, struct kgd_mem mem, bool intr);
	188	int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
	189	struct kgd_mem mem, void kptr, uint64_t size);
	190	int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
	191	struct dma_fence **ef);
	192
b97dfa27	193	int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
	194	struct kfd_vm_fault_info *info);
	195
a46a2cd1 FK	196	void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
	197	void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);
	198
130e0371	199	#endif /* AMDGPU_AMDKFD_H_INCLUDED */