[linux-2.6-block.git] / drivers / gpu / drm / amd / amdkfd / kfd_priv.h

/*
 * Copyright 2014 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef KFD_PRIV_H_INCLUDED
#define KFD_PRIV_H_INCLUDED

#include <linux/hashtable.h>
#include <linux/mmu_notifier.h>
#include <linux/mutex.h>
#include <linux/types.h>
#include <linux/atomic.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
#include <linux/kfd_ioctl.h>
#include <kgd_kfd_interface.h>

#define KFD_SYSFS_FILE_MODE 0444

/*
 * When working with cp scheduler we should assign the HIQ manually or via
 * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot
 * definitions for Kaveri. In Kaveri only the first ME queues participates
 * in the cp scheduling taking that in mind we set the HIQ slot in the
 * second ME.
 */
#define KFD_CIK_HIQ_PIPE 4
#define KFD_CIK_HIQ_QUEUE 0

/* GPU ID hash width in bits */
#define KFD_GPU_ID_HASH_WIDTH 16

/* Macro for allocating structures */
#define kfd_alloc_struct(ptr_to_struct)	\
	((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))

/* Kernel module parameter to specify maximum number of supported processes */
extern int max_num_of_processes;

#define KFD_MAX_NUM_OF_PROCESSES_DEFAULT 32
#define KFD_MAX_NUM_OF_PROCESSES 512

/*
 * Kernel module parameter to specify maximum number of supported queues
 * per process
 */
extern int max_num_of_queues_per_process;

#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128
#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024

#define KFD_KERNEL_QUEUE_SIZE 2048

enum cache_policy {
	cache_policy_coherent,
	cache_policy_noncoherent
};

struct kfd_device_info {
	unsigned int max_pasid_bits;
	size_t ih_ring_entry_size;
	uint16_t mqd_size_aligned;
};

struct kfd_dev {
	struct kgd_dev *kgd;

	const struct kfd_device_info *device_info;
	struct pci_dev *pdev;

	unsigned int id;		/* topology stub index */

	phys_addr_t doorbell_base;	/* Start of actual doorbells used by
					 * KFD. It is aligned for mapping
					 * into user mode
					 */
	size_t doorbell_id_offset;	/* Doorbell offset (from KFD doorbell
					 * to HW doorbell, GFX reserved some
					 * at the start)
					 */
	size_t doorbell_process_limit;	/* Number of processes we have doorbell
					 * space for.
					 */
	u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells
					   * page used by kernel queue
					   */

	struct kgd2kfd_shared_resources shared_resources;

	/* QCM Device instance */
	struct device_queue_manager *dqm;

	bool init_complete;
};

/* KGD2KFD callbacks */
void kgd2kfd_exit(void);
struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev);
bool kgd2kfd_device_init(struct kfd_dev *kfd,
			 const struct kgd2kfd_shared_resources *gpu_resources);
void kgd2kfd_device_exit(struct kfd_dev *kfd);

extern const struct kfd2kgd_calls *kfd2kgd;

struct kfd_mem_obj {
	void *bo;
	uint64_t gpu_addr;
	uint32_t *cpu_ptr;
};

enum kfd_mempool {
	KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
	KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
	KFD_MEMPOOL_FRAMEBUFFER = 3,
};

/* Character device interface */
int kfd_chardev_init(void);
void kfd_chardev_exit(void);
struct device *kfd_chardev(void);


enum kfd_preempt_type {
	KFD_PREEMPT_TYPE_WAVEFRONT,
	KFD_PREEMPT_TYPE_WAVEFRONT_RESET
};

/**
 * enum kfd_queue_type
 *
 * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type.
 *
 * @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type.
 *
 * @KFD_QUEUE_TYPE_HIQ: HIQ queue type.
 *
 * @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
 */
enum kfd_queue_type  {
	KFD_QUEUE_TYPE_COMPUTE,
	KFD_QUEUE_TYPE_SDMA,
	KFD_QUEUE_TYPE_HIQ,
	KFD_QUEUE_TYPE_DIQ
};

enum kfd_queue_format {
	KFD_QUEUE_FORMAT_PM4,
	KFD_QUEUE_FORMAT_AQL
};

/**
 * struct queue_properties
 *
 * @type: The queue type.
 *
 * @queue_id: Queue identifier.
 *
 * @queue_address: Queue ring buffer address.
 *
 * @queue_size: Queue ring buffer size.
 *
 * @priority: Defines the queue priority relative to other queues in the
 * process.
 * This is just an indication and HW scheduling may override the priority as
 * necessary while keeping the relative prioritization.
 * the priority granularity is from 0 to f which f is the highest priority.
 * currently all queues are initialized with the highest priority.
 *
 * @queue_percent: This field is partially implemented and currently a zero in
 * this field defines that the queue is non active.
 *
 * @read_ptr: User space address which points to the number of dwords the
 * cp read from the ring buffer. This field updates automatically by the H/W.
 *
 * @write_ptr: Defines the number of dwords written to the ring buffer.
 *
 * @doorbell_ptr: This field aim is to notify the H/W of new packet written to
 * the queue ring buffer. This field should be similar to write_ptr and the user
 * should update this field after he updated the write_ptr.
 *
 * @doorbell_off: The doorbell offset in the doorbell pci-bar.
 *
 * @is_interop: Defines if this is a interop queue. Interop queue means that the
 * queue can access both graphics and compute resources.
 *
 * @is_active: Defines if the queue is active or not.
 *
 * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
 * of the queue.
 *
 * This structure represents the queue properties for each queue no matter if
 * it's user mode or kernel mode queue.
 *
 */
struct queue_properties {
	enum kfd_queue_type type;
	enum kfd_queue_format format;
	unsigned int queue_id;
	uint64_t queue_address;
	uint64_t  queue_size;
	uint32_t priority;
	uint32_t queue_percent;
	uint32_t *read_ptr;
	uint32_t *write_ptr;
	uint32_t *doorbell_ptr;
	uint32_t doorbell_off;
	bool is_interop;
	bool is_active;
	/* Not relevant for user mode queues in cp scheduling */
	unsigned int vmid;
};

/**
 * struct queue
 *
 * @list: Queue linked list.
 *
 * @mqd: The queue MQD.
 *
 * @mqd_mem_obj: The MQD local gpu memory object.
 *
 * @gart_mqd_addr: The MQD gart mc address.
 *
 * @properties: The queue properties.
 *
 * @mec: Used only in no cp scheduling mode and identifies to micro engine id
 * that the queue should be execute on.
 *
 * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe id.
 *
 * @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
 *
 * @process: The kfd process that created this queue.
 *
 * @device: The kfd device that created this queue.
 *
 * This structure represents user mode compute queues.
 * It contains all the necessary data to handle such queues.
 *
 */

struct queue {
	struct list_head list;
	void *mqd;
	struct kfd_mem_obj *mqd_mem_obj;
	uint64_t gart_mqd_addr;
	struct queue_properties properties;

	uint32_t mec;
	uint32_t pipe;
	uint32_t queue;

	struct kfd_process	*process;
	struct kfd_dev		*device;
};

/*
 * Please read the kfd_mqd_manager.h description.
 */
enum KFD_MQD_TYPE {
	KFD_MQD_TYPE_CIK_COMPUTE = 0, /* for no cp scheduling */
	KFD_MQD_TYPE_CIK_HIQ, /* for hiq */
	KFD_MQD_TYPE_CIK_CP, /* for cp queues and diq */
	KFD_MQD_TYPE_CIK_SDMA, /* for sdma queues */
	KFD_MQD_TYPE_MAX
};

/* Data that is per-process-per device. */
struct kfd_process_device {
	/*
	 * List of all per-device data for a process.
	 * Starts from kfd_process.per_device_data.
	 */
	struct list_head per_device_list;

	/* The device that owns this data. */
	struct kfd_dev *dev;


	/*Apertures*/
	uint64_t lds_base;
	uint64_t lds_limit;
	uint64_t gpuvm_base;
	uint64_t gpuvm_limit;
	uint64_t scratch_base;
	uint64_t scratch_limit;

	/* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
	bool bound;
};

/* Process data */
struct kfd_process {
	/*
	 * kfd_process are stored in an mm_struct*->kfd_process*
	 * hash table (kfd_processes in kfd_process.c)
	 */
	struct hlist_node kfd_processes;

	struct mm_struct *mm;

	struct mutex mutex;

	/*
	 * In any process, the thread that started main() is the lead
	 * thread and outlives the rest.
	 * It is here because amd_iommu_bind_pasid wants a task_struct.
	 */
	struct task_struct *lead_thread;

	/* We want to receive a notification when the mm_struct is destroyed */
	struct mmu_notifier mmu_notifier;

	/* Use for delayed freeing of kfd_process structure */
	struct rcu_head	rcu;

	unsigned int pasid;

	/*
	 * List of kfd_process_device structures,
	 * one for each device the process is using.
	 */
	struct list_head per_device_data;

	/* The process's queues. */
	size_t queue_array_size;

	/* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */
	struct kfd_queue **queues;

	unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)];

	/*Is the user space process 32 bit?*/
	bool is_32bit_user_mode;
};

void kfd_process_create_wq(void);
void kfd_process_destroy_wq(void);
struct kfd_process *kfd_create_process(const struct task_struct *);
struct kfd_process *kfd_get_process(const struct task_struct *);

void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid);
struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
							struct kfd_process *p,
							int create_pdd);

/* PASIDs */
int kfd_pasid_init(void);
void kfd_pasid_exit(void);
bool kfd_set_pasid_limit(unsigned int new_limit);
unsigned int kfd_get_pasid_limit(void);
unsigned int kfd_pasid_alloc(void);
void kfd_pasid_free(unsigned int pasid);

/* Doorbells */
void kfd_doorbell_init(struct kfd_dev *kfd);
int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma);
u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
					unsigned int *doorbell_off);
void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
u32 read_kernel_doorbell(u32 __iomem *db);
void write_kernel_doorbell(u32 __iomem *db, u32 value);
unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
					struct kfd_process *process,
					unsigned int queue_id);

extern struct device *kfd_device;

/* Topology */
int kfd_topology_init(void);
void kfd_topology_shutdown(void);
int kfd_topology_add_device(struct kfd_dev *gpu);
int kfd_topology_remove_device(struct kfd_dev *gpu);
struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx);

/* Interrupts */
void kgd2kfd_interrupt(struct kfd_dev *dev, const void *ih_ring_entry);

/* Power Management */
void kgd2kfd_suspend(struct kfd_dev *dev);
int kgd2kfd_resume(struct kfd_dev *dev);

/* amdkfd Apertures */
int kfd_init_apertures(struct kfd_process *process);

/* Queue Context Management */
int init_queue(struct queue **q, struct queue_properties properties);
void uninit_queue(struct queue *q);
void print_queue(struct queue *q);

/* Packet Manager */

struct packet_manager {
	struct device_queue_manager *dqm;
	struct kernel_queue *priv_queue;
	struct mutex lock;
	bool allocated;
	struct kfd_mem_obj *ib_buffer_obj;
};

uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
					struct kfd_process *process);

#endif
Commit	Line	Data
4a488a7a OG	1	/*
	2	* Copyright 2014 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*/
	22
	23	#ifndef KFD_PRIV_H_INCLUDED
	24	#define KFD_PRIV_H_INCLUDED
	25
	26	#include <linux/hashtable.h>
	27	#include <linux/mmu_notifier.h>
	28	#include <linux/mutex.h>
	29	#include <linux/types.h>
	30	#include <linux/atomic.h>
	31	#include <linux/workqueue.h>
	32	#include <linux/spinlock.h>
19f6d2a6	33	#include <linux/kfd_ioctl.h>
4a488a7a OG	34	#include <kgd_kfd_interface.h>
4a488a7a OG	35
5b5c4e40 EP	36	#define KFD_SYSFS_FILE_MODE 0444
5b5c4e40 EP	37
ed6e6a34 BG	38	/*
	39	* When working with cp scheduler we should assign the HIQ manually or via
	40	* the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot
	41	* definitions for Kaveri. In Kaveri only the first ME queues participates
	42	* in the cp scheduling taking that in mind we set the HIQ slot in the
	43	* second ME.
	44	*/
	45	#define KFD_CIK_HIQ_PIPE 4
	46	#define KFD_CIK_HIQ_QUEUE 0
	47
5b5c4e40 EP	48	/* GPU ID hash width in bits */
	49	#define KFD_GPU_ID_HASH_WIDTH 16
	50
	51	/* Macro for allocating structures */
	52	#define kfd_alloc_struct(ptr_to_struct) \
	53	((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
	54
19f6d2a6 OG	55	/* Kernel module parameter to specify maximum number of supported processes */
	56	extern int max_num_of_processes;
	57
	58	#define KFD_MAX_NUM_OF_PROCESSES_DEFAULT 32
	59	#define KFD_MAX_NUM_OF_PROCESSES 512
	60
	61	/*
	62	* Kernel module parameter to specify maximum number of supported queues
	63	* per process
	64	*/
	65	extern int max_num_of_queues_per_process;
	66
	67	#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128
	68	#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
	69
ed6e6a34 BG	70	#define KFD_KERNEL_QUEUE_SIZE 2048
	71
	72	enum cache_policy {
	73	cache_policy_coherent,
	74	cache_policy_noncoherent
	75	};
	76
4a488a7a OG	77	struct kfd_device_info {
	78	unsigned int max_pasid_bits;
	79	size_t ih_ring_entry_size;
19f6d2a6	80	uint16_t mqd_size_aligned;
4a488a7a OG	81	};
	82
	83	struct kfd_dev {
	84	struct kgd_dev *kgd;
	85
	86	const struct kfd_device_info *device_info;
	87	struct pci_dev *pdev;
	88
	89	unsigned int id; /* topology stub index */
	90
19f6d2a6 OG	91	phys_addr_t doorbell_base; /* Start of actual doorbells used by
	92	* KFD. It is aligned for mapping
	93	* into user mode
	94	*/
	95	size_t doorbell_id_offset; /* Doorbell offset (from KFD doorbell
	96	* to HW doorbell, GFX reserved some
	97	* at the start)
	98	*/
	99	size_t doorbell_process_limit; /* Number of processes we have doorbell
	100	* space for.
	101	*/
	102	u32 __iomem doorbell_kernel_ptr; / This is a pointer for a doorbells
	103	* page used by kernel queue
	104	*/
	105
4a488a7a OG	106	struct kgd2kfd_shared_resources shared_resources;
4a488a7a OG	107
ed6e6a34 BG	108	/* QCM Device instance */
ed6e6a34 BG	109	struct device_queue_manager *dqm;
4a488a7a	110
ed6e6a34	111	bool init_complete;
4a488a7a OG	112	};
	113
	114	/* KGD2KFD callbacks */
	115	void kgd2kfd_exit(void);
	116	struct kfd_dev kgd2kfd_probe(struct kgd_dev kgd, struct pci_dev *pdev);
	117	bool kgd2kfd_device_init(struct kfd_dev *kfd,
	118	const struct kgd2kfd_shared_resources *gpu_resources);
	119	void kgd2kfd_device_exit(struct kfd_dev *kfd);
	120
	121	extern const struct kfd2kgd_calls *kfd2kgd;
	122
19f6d2a6 OG	123	struct kfd_mem_obj {
	124	void *bo;
	125	uint64_t gpu_addr;
	126	uint32_t *cpu_ptr;
	127	};
	128
	129	enum kfd_mempool {
	130	KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
	131	KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
	132	KFD_MEMPOOL_FRAMEBUFFER = 3,
	133	};
	134
4a488a7a OG	135	/* Character device interface */
	136	int kfd_chardev_init(void);
	137	void kfd_chardev_exit(void);
	138	struct device *kfd_chardev(void);
	139
19f6d2a6	140
6e99df57 BG	141	enum kfd_preempt_type {
	142	KFD_PREEMPT_TYPE_WAVEFRONT,
	143	KFD_PREEMPT_TYPE_WAVEFRONT_RESET
	144	};
	145
ed8aab45 BG	146	/**
	147	* enum kfd_queue_type
	148	*
	149	* @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type.
	150	*
	151	* @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type.
	152	*
	153	* @KFD_QUEUE_TYPE_HIQ: HIQ queue type.
	154	*
	155	* @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
	156	*/
	157	enum kfd_queue_type {
	158	KFD_QUEUE_TYPE_COMPUTE,
	159	KFD_QUEUE_TYPE_SDMA,
	160	KFD_QUEUE_TYPE_HIQ,
	161	KFD_QUEUE_TYPE_DIQ
	162	};
	163
6e99df57 BG	164	enum kfd_queue_format {
	165	KFD_QUEUE_FORMAT_PM4,
	166	KFD_QUEUE_FORMAT_AQL
	167	};
	168
ed8aab45 BG	169	/**
	170	* struct queue_properties
	171	*
	172	* @type: The queue type.
	173	*
	174	* @queue_id: Queue identifier.
	175	*
	176	* @queue_address: Queue ring buffer address.
	177	*
	178	* @queue_size: Queue ring buffer size.
	179	*
	180	* @priority: Defines the queue priority relative to other queues in the
	181	* process.
	182	* This is just an indication and HW scheduling may override the priority as
	183	* necessary while keeping the relative prioritization.
	184	* the priority granularity is from 0 to f which f is the highest priority.
	185	* currently all queues are initialized with the highest priority.
	186	*
	187	* @queue_percent: This field is partially implemented and currently a zero in
	188	* this field defines that the queue is non active.
	189	*
	190	* @read_ptr: User space address which points to the number of dwords the
	191	* cp read from the ring buffer. This field updates automatically by the H/W.
	192	*
	193	* @write_ptr: Defines the number of dwords written to the ring buffer.
	194	*
	195	* @doorbell_ptr: This field aim is to notify the H/W of new packet written to
	196	* the queue ring buffer. This field should be similar to write_ptr and the user
	197	* should update this field after he updated the write_ptr.
	198	*
	199	* @doorbell_off: The doorbell offset in the doorbell pci-bar.
	200	*
	201	* @is_interop: Defines if this is a interop queue. Interop queue means that the
	202	* queue can access both graphics and compute resources.
	203	*
	204	* @is_active: Defines if the queue is active or not.
	205	*
	206	* @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
	207	* of the queue.
	208	*
	209	* This structure represents the queue properties for each queue no matter if
	210	* it's user mode or kernel mode queue.
	211	*
	212	*/
	213	struct queue_properties {
	214	enum kfd_queue_type type;
6e99df57	215	enum kfd_queue_format format;
ed8aab45 BG	216	unsigned int queue_id;
	217	uint64_t queue_address;
	218	uint64_t queue_size;
	219	uint32_t priority;
	220	uint32_t queue_percent;
	221	uint32_t *read_ptr;
	222	uint32_t *write_ptr;
	223	uint32_t *doorbell_ptr;
	224	uint32_t doorbell_off;
	225	bool is_interop;
	226	bool is_active;
	227	/* Not relevant for user mode queues in cp scheduling */
	228	unsigned int vmid;
	229	};
	230
	231	/**
	232	* struct queue
	233	*
	234	* @list: Queue linked list.
	235	*
	236	* @mqd: The queue MQD.
	237	*
	238	* @mqd_mem_obj: The MQD local gpu memory object.
	239	*
	240	* @gart_mqd_addr: The MQD gart mc address.
	241	*
	242	* @properties: The queue properties.
	243	*
	244	* @mec: Used only in no cp scheduling mode and identifies to micro engine id
	245	* that the queue should be execute on.
	246	*
	247	* @pipe: Used only in no cp scheduling mode and identifies the queue's pipe id.
	248	*
	249	* @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
	250	*
	251	* @process: The kfd process that created this queue.
	252	*
	253	* @device: The kfd device that created this queue.
	254	*
	255	* This structure represents user mode compute queues.
	256	* It contains all the necessary data to handle such queues.
	257	*
	258	*/
	259
	260	struct queue {
	261	struct list_head list;
	262	void *mqd;
	263	struct kfd_mem_obj *mqd_mem_obj;
	264	uint64_t gart_mqd_addr;
	265	struct queue_properties properties;
	266
	267	uint32_t mec;
	268	uint32_t pipe;
	269	uint32_t queue;
	270
	271	struct kfd_process *process;
	272	struct kfd_dev *device;
	273	};
	274
6e99df57 BG	275	/*
	276	* Please read the kfd_mqd_manager.h description.
	277	*/
	278	enum KFD_MQD_TYPE {
	279	KFD_MQD_TYPE_CIK_COMPUTE = 0, /* for no cp scheduling */
	280	KFD_MQD_TYPE_CIK_HIQ, /* for hiq */
	281	KFD_MQD_TYPE_CIK_CP, /* for cp queues and diq */
	282	KFD_MQD_TYPE_CIK_SDMA, /* for sdma queues */
	283	KFD_MQD_TYPE_MAX
	284	};
	285
19f6d2a6 OG	286	/* Data that is per-process-per device. */
	287	struct kfd_process_device {
	288	/*
	289	* List of all per-device data for a process.
	290	* Starts from kfd_process.per_device_data.
	291	*/
	292	struct list_head per_device_list;
	293
	294	/* The device that owns this data. */
	295	struct kfd_dev *dev;
	296
	297
	298	/Apertures/
	299	uint64_t lds_base;
	300	uint64_t lds_limit;
	301	uint64_t gpuvm_base;
	302	uint64_t gpuvm_limit;
	303	uint64_t scratch_base;
	304	uint64_t scratch_limit;
	305
	306	/* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
	307	bool bound;
	308	};
	309
4a488a7a OG	310	/* Process data */
4a488a7a OG	311	struct kfd_process {
19f6d2a6 OG	312	/*
	313	* kfd_process are stored in an mm_struct->kfd_process
	314	* hash table (kfd_processes in kfd_process.c)
	315	*/
	316	struct hlist_node kfd_processes;
	317
	318	struct mm_struct *mm;
	319
	320	struct mutex mutex;
	321
	322	/*
	323	* In any process, the thread that started main() is the lead
	324	* thread and outlives the rest.
	325	* It is here because amd_iommu_bind_pasid wants a task_struct.
	326	*/
	327	struct task_struct *lead_thread;
	328
	329	/* We want to receive a notification when the mm_struct is destroyed */
	330	struct mmu_notifier mmu_notifier;
	331
	332	/* Use for delayed freeing of kfd_process structure */
	333	struct rcu_head rcu;
	334
	335	unsigned int pasid;
	336
	337	/*
	338	* List of kfd_process_device structures,
	339	* one for each device the process is using.
	340	*/
	341	struct list_head per_device_data;
	342
	343	/* The process's queues. */
	344	size_t queue_array_size;
	345
	346	/* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */
	347	struct kfd_queue **queues;
	348
	349	unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)];
	350
	351	/Is the user space process 32 bit?/
	352	bool is_32bit_user_mode;
4a488a7a OG	353	};
4a488a7a OG	354
19f6d2a6 OG	355	void kfd_process_create_wq(void);
	356	void kfd_process_destroy_wq(void);
	357	struct kfd_process kfd_create_process(const struct task_struct );
	358	struct kfd_process kfd_get_process(const struct task_struct );
	359
b17f068a	360	void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid);
19f6d2a6 OG	361	struct kfd_process_device kfd_get_process_device_data(struct kfd_dev dev,
	362	struct kfd_process *p,
	363	int create_pdd);
	364
	365	/* PASIDs */
	366	int kfd_pasid_init(void);
	367	void kfd_pasid_exit(void);
	368	bool kfd_set_pasid_limit(unsigned int new_limit);
	369	unsigned int kfd_get_pasid_limit(void);
	370	unsigned int kfd_pasid_alloc(void);
	371	void kfd_pasid_free(unsigned int pasid);
	372
	373	/* Doorbells */
	374	void kfd_doorbell_init(struct kfd_dev *kfd);
	375	int kfd_doorbell_mmap(struct kfd_process process, struct vm_area_struct vma);
	376	u32 __iomem kfd_get_kernel_doorbell(struct kfd_dev kfd,
	377	unsigned int *doorbell_off);
	378	void kfd_release_kernel_doorbell(struct kfd_dev kfd, u32 __iomem db_addr);
	379	u32 read_kernel_doorbell(u32 __iomem *db);
	380	void write_kernel_doorbell(u32 __iomem *db, u32 value);
	381	unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd,
	382	struct kfd_process *process,
	383	unsigned int queue_id);
	384
4a488a7a OG	385	extern struct device *kfd_device;
4a488a7a OG	386
5b5c4e40 EP	387	/* Topology */
	388	int kfd_topology_init(void);
	389	void kfd_topology_shutdown(void);
	390	int kfd_topology_add_device(struct kfd_dev *gpu);
	391	int kfd_topology_remove_device(struct kfd_dev *gpu);
	392	struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
	393	struct kfd_dev kfd_device_by_pci_dev(const struct pci_dev pdev);
	394	struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx);
	395
4a488a7a OG	396	/* Interrupts */
	397	void kgd2kfd_interrupt(struct kfd_dev dev, const void ih_ring_entry);
	398
	399	/* Power Management */
	400	void kgd2kfd_suspend(struct kfd_dev *dev);
	401	int kgd2kfd_resume(struct kfd_dev *dev);
	402
19f6d2a6 OG	403	/* amdkfd Apertures */
	404	int kfd_init_apertures(struct kfd_process *process);
	405
ed6e6a34 BG	406	/* Queue Context Management */
	407	int init_queue(struct queue **q, struct queue_properties properties);
	408	void uninit_queue(struct queue *q);
	409	void print_queue(struct queue *q);
	410
	411	/* Packet Manager */
	412
	413	struct packet_manager {
	414	struct device_queue_manager *dqm;
	415	struct kernel_queue *priv_queue;
	416	struct mutex lock;
	417	bool allocated;
	418	struct kfd_mem_obj *ib_buffer_obj;
	419	};
	420
19f6d2a6 OG	421	uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
	422	phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
	423	struct kfd_process *process);
	424
4a488a7a	425	#endif