[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_amdkfd.h

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */

#ifndef AMDGPU_AMDKFD_H_INCLUDED
#define AMDGPU_AMDKFD_H_INCLUDED

#include <linux/types.h>
#include <linux/mm.h>
#include <linux/mmu_context.h>
#include <kgd_kfd_interface.h>
#include <drm/ttm/ttm_execbuf_util.h>
#include "amdgpu_sync.h"
#include "amdgpu_vm.h"

extern const struct kgd2kfd_calls *kgd2kfd;

struct amdgpu_device;

struct kfd_bo_va_list {
	struct list_head bo_list;
	struct amdgpu_bo_va *bo_va;
	void *kgd_dev;
	bool is_mapped;
	uint64_t va;
	uint64_t pte_flags;
};

struct kgd_mem {
	struct mutex lock;
	struct amdgpu_bo *bo;
	struct list_head bo_va_list;
	/* protected by amdkfd_process_info.lock */
	struct ttm_validate_buffer validate_list;
	struct ttm_validate_buffer resv_list;
	uint32_t domain;
	unsigned int mapped_to_gpu_memory;
	uint64_t va;

	uint32_t mapping_flags;

	struct amdkfd_process_info *process_info;

	struct amdgpu_sync sync;

	bool aql_queue;
};

/* KFD Memory Eviction */
struct amdgpu_amdkfd_fence {
	struct dma_fence base;
	struct mm_struct *mm;
	spinlock_t lock;
	char timeline_name[TASK_COMM_LEN];
};

struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
						       struct mm_struct *mm);
bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);

struct amdkfd_process_info {
	/* List head of all VMs that belong to a KFD process */
	struct list_head vm_list_head;
	/* List head for all KFD BOs that belong to a KFD process. */
	struct list_head kfd_bo_list;
	/* Lock to protect kfd_bo_list */
	struct mutex lock;

	/* Number of VMs */
	unsigned int n_vms;
	/* Eviction Fence */
	struct amdgpu_amdkfd_fence *eviction_fence;
};

int amdgpu_amdkfd_init(void);
void amdgpu_amdkfd_fini(void);

void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
			const void *ih_ring_entry);
void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);

int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm);
int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
				uint32_t vmid, uint64_t gpu_addr,
				uint32_t *ib_cmd, uint32_t ib_len);

struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);

bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);

/* Shared API */
int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
			void **mem_obj, uint64_t *gpu_addr,
			void **cpu_ptr);
void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
void get_local_mem_info(struct kgd_dev *kgd,
			struct kfd_local_mem_info *mem_info);
uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);

uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);

#define read_user_wptr(mmptr, wptr, dst)				\
	({								\
		bool valid = false;					\
		if ((mmptr) && (wptr)) {				\
			if ((mmptr) == current->mm) {			\
				valid = !get_user((dst), (wptr));	\
			} else if (current->mm == NULL) {		\
				use_mm(mmptr);				\
				valid = !get_user((dst), (wptr));	\
				unuse_mm(mmptr);			\
			}						\
		}							\
		valid;							\
	})

/* GPUVM API */
int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm,
					  void **process_info,
					  struct dma_fence **ef);
int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
					   struct file *filp,
					   void **vm, void **process_info,
					   struct dma_fence **ef);
void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
				    struct amdgpu_vm *vm);
void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm);
uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
		struct kgd_dev *kgd, uint64_t va, uint64_t size,
		void *vm, struct kgd_mem **mem,
		uint64_t *offset, uint32_t flags);
int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem);
int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
int amdgpu_amdkfd_gpuvm_sync_memory(
		struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);
int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
		struct kgd_mem *mem, void **kptr, uint64_t *size);
int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
					    struct dma_fence **ef);

void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);

#endif /* AMDGPU_AMDKFD_H_INCLUDED */
Commit	Line	Data
130e0371 OG	1	/*
	2	* Copyright 2014 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*/
	22
	23	/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */
	24
	25	#ifndef AMDGPU_AMDKFD_H_INCLUDED
	26	#define AMDGPU_AMDKFD_H_INCLUDED
	27
	28	#include <linux/types.h>
7420f482	29	#include <linux/mm.h>
70539bd7	30	#include <linux/mmu_context.h>
130e0371	31	#include <kgd_kfd_interface.h>
a46a2cd1 FK	32	#include <drm/ttm/ttm_execbuf_util.h>
	33	#include "amdgpu_sync.h"
	34	#include "amdgpu_vm.h"
130e0371	35
d8d019cc FK	36	extern const struct kgd2kfd_calls *kgd2kfd;
d8d019cc FK	37
130e0371 OG	38	struct amdgpu_device;
130e0371 OG	39
a46a2cd1 FK	40	struct kfd_bo_va_list {
	41	struct list_head bo_list;
	42	struct amdgpu_bo_va *bo_va;
	43	void *kgd_dev;
	44	bool is_mapped;
	45	uint64_t va;
	46	uint64_t pte_flags;
	47	};
	48
130e0371	49	struct kgd_mem {
a46a2cd1	50	struct mutex lock;
130e0371	51	struct amdgpu_bo *bo;
a46a2cd1 FK	52	struct list_head bo_va_list;
	53	/* protected by amdkfd_process_info.lock */
	54	struct ttm_validate_buffer validate_list;
	55	struct ttm_validate_buffer resv_list;
	56	uint32_t domain;
	57	unsigned int mapped_to_gpu_memory;
	58	uint64_t va;
	59
	60	uint32_t mapping_flags;
	61
	62	struct amdkfd_process_info *process_info;
	63
	64	struct amdgpu_sync sync;
	65
	66	bool aql_queue;
130e0371 OG	67	};
130e0371 OG	68
d8d019cc FK	69	/* KFD Memory Eviction */
	70	struct amdgpu_amdkfd_fence {
	71	struct dma_fence base;
	72	struct mm_struct *mm;
	73	spinlock_t lock;
	74	char timeline_name[TASK_COMM_LEN];
	75	};
	76
	77	struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
	78	struct mm_struct *mm);
	79	bool amdkfd_fence_check_mm(struct dma_fence f, struct mm_struct mm);
	80	struct amdgpu_amdkfd_fence to_amdgpu_amdkfd_fence(struct dma_fence f);
	81
a46a2cd1 FK	82	struct amdkfd_process_info {
	83	/* List head of all VMs that belong to a KFD process */
	84	struct list_head vm_list_head;
	85	/* List head for all KFD BOs that belong to a KFD process. */
	86	struct list_head kfd_bo_list;
	87	/* Lock to protect kfd_bo_list */
	88	struct mutex lock;
	89
	90	/* Number of VMs */
	91	unsigned int n_vms;
	92	/* Eviction Fence */
	93	struct amdgpu_amdkfd_fence *eviction_fence;
	94	};
	95
efb1c658	96	int amdgpu_amdkfd_init(void);
130e0371 OG	97	void amdgpu_amdkfd_fini(void);
130e0371 OG	98
dc102c43 AR	99	void amdgpu_amdkfd_suspend(struct amdgpu_device *adev);
	100	int amdgpu_amdkfd_resume(struct amdgpu_device *adev);
	101	void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev,
130e0371	102	const void *ih_ring_entry);
dc102c43 AR	103	void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev);
	104	void amdgpu_amdkfd_device_init(struct amdgpu_device *adev);
	105	void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev);
130e0371	106
e52482de	107	int amdgpu_amdkfd_evict_userptr(struct kgd_mem mem, struct mm_struct mm);
4c660c8f FK	108	int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
	109	uint32_t vmid, uint64_t gpu_addr,
	110	uint32_t *ib_cmd, uint32_t ib_len);
	111
130e0371	112	struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
ff758a12	113	struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
130e0371	114
155494db FK	115	bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
155494db FK	116
130e0371 OG	117	/* Shared API */
	118	int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
	119	void *mem_obj, uint64_t gpu_addr,
	120	void **cpu_ptr);
	121	void free_gtt_mem(struct kgd_dev kgd, void mem_obj);
30f1c042 HK	122	void get_local_mem_info(struct kgd_dev *kgd,
30f1c042 HK	123	struct kfd_local_mem_info *mem_info);
130e0371 OG	124	uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
	125
	126	uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
ebdebf42	127	void get_cu_info(struct kgd_dev kgd, struct kfd_cu_info cu_info);
9f0a0b41	128	uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
130e0371	129
70539bd7 FK	130	#define read_user_wptr(mmptr, wptr, dst) \
	131	({ \
	132	bool valid = false; \
	133	if ((mmptr) && (wptr)) { \
	134	if ((mmptr) == current->mm) { \
	135	valid = !get_user((dst), (wptr)); \
	136	} else if (current->mm == NULL) { \
	137	use_mm(mmptr); \
	138	valid = !get_user((dst), (wptr)); \
	139	unuse_mm(mmptr); \
	140	} \
	141	} \
	142	valid; \
	143	})
	144
a46a2cd1 FK	145	/* GPUVM API */
	146	int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev kgd, void *vm,
	147	void **process_info,
	148	struct dma_fence **ef);
ede0dd86 FK	149	int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
	150	struct file *filp,
	151	void vm, void process_info,
	152	struct dma_fence **ef);
	153	void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
	154	struct amdgpu_vm *vm);
a46a2cd1 FK	155	void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev kgd, void vm);
	156	uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
	157	int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
	158	struct kgd_dev *kgd, uint64_t va, uint64_t size,
	159	void vm, struct kgd_mem *mem,
	160	uint64_t *offset, uint32_t flags);
	161	int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
	162	struct kgd_dev kgd, struct kgd_mem mem);
	163	int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
	164	struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	165	int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
	166	struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	167	int amdgpu_amdkfd_gpuvm_sync_memory(
	168	struct kgd_dev kgd, struct kgd_mem mem, bool intr);
	169	int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
	170	struct kgd_mem mem, void kptr, uint64_t size);
	171	int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
	172	struct dma_fence **ef);
	173
	174	void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
	175	void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);
	176
130e0371	177	#endif /* AMDGPU_AMDKFD_H_INCLUDED */