Commit | Line | Data |
---|---|---|
4a488a7a OG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #ifndef KFD_PRIV_H_INCLUDED | |
24 | #define KFD_PRIV_H_INCLUDED | |
25 | ||
26 | #include <linux/hashtable.h> | |
27 | #include <linux/mmu_notifier.h> | |
28 | #include <linux/mutex.h> | |
29 | #include <linux/types.h> | |
30 | #include <linux/atomic.h> | |
31 | #include <linux/workqueue.h> | |
32 | #include <linux/spinlock.h> | |
19f6d2a6 | 33 | #include <linux/kfd_ioctl.h> |
482f0777 | 34 | #include <linux/idr.h> |
4a488a7a OG |
35 | #include <kgd_kfd_interface.h> |
36 | ||
e596b903 YZ |
37 | #include "amd_shared.h" |
38 | ||
5b5c4e40 EP |
39 | #define KFD_SYSFS_FILE_MODE 0444 |
40 | ||
f3a39818 AL |
41 | #define KFD_MMAP_DOORBELL_MASK 0x8000000000000 |
42 | #define KFD_MMAP_EVENTS_MASK 0x4000000000000 | |
43 | ||
ed6e6a34 BG |
44 | /* |
45 | * When working with cp scheduler we should assign the HIQ manually or via | |
46 | * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot | |
47 | * definitions for Kaveri. In Kaveri only the first ME queues participates | |
48 | * in the cp scheduling taking that in mind we set the HIQ slot in the | |
49 | * second ME. | |
50 | */ | |
51 | #define KFD_CIK_HIQ_PIPE 4 | |
52 | #define KFD_CIK_HIQ_QUEUE 0 | |
53 | ||
5b5c4e40 EP |
54 | /* GPU ID hash width in bits */ |
55 | #define KFD_GPU_ID_HASH_WIDTH 16 | |
56 | ||
57 | /* Macro for allocating structures */ | |
58 | #define kfd_alloc_struct(ptr_to_struct) \ | |
59 | ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL)) | |
60 | ||
19f6d2a6 | 61 | #define KFD_MAX_NUM_OF_PROCESSES 512 |
b8cbab04 | 62 | #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 |
19f6d2a6 OG |
63 | |
64 | /* | |
b8cbab04 OG |
65 | * Kernel module parameter to specify maximum number of supported queues per |
66 | * device | |
19f6d2a6 | 67 | */ |
b8cbab04 | 68 | extern int max_num_of_queues_per_device; |
19f6d2a6 | 69 | |
b8cbab04 OG |
70 | #define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE_DEFAULT 4096 |
71 | #define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \ | |
72 | (KFD_MAX_NUM_OF_PROCESSES * \ | |
73 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) | |
19f6d2a6 | 74 | |
ed6e6a34 BG |
75 | #define KFD_KERNEL_QUEUE_SIZE 2048 |
76 | ||
31c21fec BG |
77 | /* Kernel module parameter to specify the scheduling policy */ |
78 | extern int sched_policy; | |
79 | ||
81663016 OG |
80 | /* |
81 | * Kernel module parameter to specify whether to send sigterm to HSA process on | |
82 | * unhandled exception | |
83 | */ | |
84 | extern int send_sigterm; | |
85 | ||
31c21fec BG |
86 | /** |
87 | * enum kfd_sched_policy | |
88 | * | |
89 | * @KFD_SCHED_POLICY_HWS: H/W scheduling policy known as command processor (cp) | |
90 | * scheduling. In this scheduling mode we're using the firmware code to | |
91 | * schedule the user mode queues and kernel queues such as HIQ and DIQ. | |
92 | * the HIQ queue is used as a special queue that dispatches the configuration | |
93 | * to the cp and the user mode queues list that are currently running. | |
94 | * the DIQ queue is a debugging queue that dispatches debugging commands to the | |
95 | * firmware. | |
96 | * in this scheduling mode user mode queues over subscription feature is | |
97 | * enabled. | |
98 | * | |
99 | * @KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION: The same as above but the over | |
100 | * subscription feature disabled. | |
101 | * | |
102 | * @KFD_SCHED_POLICY_NO_HWS: no H/W scheduling policy is a mode which directly | |
103 | * set the command processor registers and sets the queues "manually". This | |
104 | * mode is used *ONLY* for debugging proposes. | |
105 | * | |
106 | */ | |
107 | enum kfd_sched_policy { | |
108 | KFD_SCHED_POLICY_HWS = 0, | |
109 | KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION, | |
110 | KFD_SCHED_POLICY_NO_HWS | |
111 | }; | |
112 | ||
ed6e6a34 BG |
113 | enum cache_policy { |
114 | cache_policy_coherent, | |
115 | cache_policy_noncoherent | |
116 | }; | |
117 | ||
f3a39818 AL |
118 | struct kfd_event_interrupt_class { |
119 | bool (*interrupt_isr)(struct kfd_dev *dev, | |
120 | const uint32_t *ih_ring_entry); | |
121 | void (*interrupt_wq)(struct kfd_dev *dev, | |
122 | const uint32_t *ih_ring_entry); | |
123 | }; | |
124 | ||
4a488a7a | 125 | struct kfd_device_info { |
e596b903 | 126 | enum amd_asic_type asic_family; |
f3a39818 | 127 | const struct kfd_event_interrupt_class *event_interrupt_class; |
4a488a7a | 128 | unsigned int max_pasid_bits; |
992839ad | 129 | unsigned int max_no_of_hqd; |
4a488a7a | 130 | size_t ih_ring_entry_size; |
f7c826ad | 131 | uint8_t num_of_watch_points; |
19f6d2a6 | 132 | uint16_t mqd_size_aligned; |
4a488a7a OG |
133 | }; |
134 | ||
36b5c08f OG |
135 | struct kfd_mem_obj { |
136 | uint32_t range_start; | |
137 | uint32_t range_end; | |
138 | uint64_t gpu_addr; | |
139 | uint32_t *cpu_ptr; | |
140 | }; | |
141 | ||
44008d7a YZ |
142 | struct kfd_vmid_info { |
143 | uint32_t first_vmid_kfd; | |
144 | uint32_t last_vmid_kfd; | |
145 | uint32_t vmid_num_kfd; | |
146 | }; | |
147 | ||
4a488a7a OG |
148 | struct kfd_dev { |
149 | struct kgd_dev *kgd; | |
150 | ||
151 | const struct kfd_device_info *device_info; | |
152 | struct pci_dev *pdev; | |
153 | ||
154 | unsigned int id; /* topology stub index */ | |
155 | ||
19f6d2a6 OG |
156 | phys_addr_t doorbell_base; /* Start of actual doorbells used by |
157 | * KFD. It is aligned for mapping | |
158 | * into user mode | |
159 | */ | |
160 | size_t doorbell_id_offset; /* Doorbell offset (from KFD doorbell | |
161 | * to HW doorbell, GFX reserved some | |
162 | * at the start) | |
163 | */ | |
19f6d2a6 OG |
164 | u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells |
165 | * page used by kernel queue | |
166 | */ | |
167 | ||
4a488a7a | 168 | struct kgd2kfd_shared_resources shared_resources; |
44008d7a | 169 | struct kfd_vmid_info vm_info; |
4a488a7a | 170 | |
cea405b1 XZ |
171 | const struct kfd2kgd_calls *kfd2kgd; |
172 | struct mutex doorbell_mutex; | |
f761d8bd JP |
173 | DECLARE_BITMAP(doorbell_available_index, |
174 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); | |
cea405b1 | 175 | |
36b5c08f OG |
176 | void *gtt_mem; |
177 | uint64_t gtt_start_gpu_addr; | |
178 | void *gtt_start_cpu_ptr; | |
179 | void *gtt_sa_bitmap; | |
180 | struct mutex gtt_sa_lock; | |
181 | unsigned int gtt_sa_chunk_size; | |
182 | unsigned int gtt_sa_num_of_chunks; | |
183 | ||
2249d558 AL |
184 | /* Interrupts */ |
185 | void *interrupt_ring; | |
186 | size_t interrupt_ring_size; | |
187 | atomic_t interrupt_ring_rptr; | |
188 | atomic_t interrupt_ring_wptr; | |
189 | struct work_struct interrupt_work; | |
190 | spinlock_t interrupt_lock; | |
191 | ||
ed6e6a34 BG |
192 | /* QCM Device instance */ |
193 | struct device_queue_manager *dqm; | |
4a488a7a | 194 | |
ed6e6a34 | 195 | bool init_complete; |
2249d558 AL |
196 | /* |
197 | * Interrupts of interest to KFD are copied | |
198 | * from the HW ring into a SW ring. | |
199 | */ | |
200 | bool interrupts_active; | |
fbeb661b YS |
201 | |
202 | /* Debug manager */ | |
203 | struct kfd_dbgmgr *dbgmgr; | |
4a488a7a OG |
204 | }; |
205 | ||
206 | /* KGD2KFD callbacks */ | |
207 | void kgd2kfd_exit(void); | |
cea405b1 XZ |
208 | struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, |
209 | struct pci_dev *pdev, const struct kfd2kgd_calls *f2g); | |
4a488a7a | 210 | bool kgd2kfd_device_init(struct kfd_dev *kfd, |
cea405b1 | 211 | const struct kgd2kfd_shared_resources *gpu_resources); |
4a488a7a OG |
212 | void kgd2kfd_device_exit(struct kfd_dev *kfd); |
213 | ||
19f6d2a6 OG |
214 | enum kfd_mempool { |
215 | KFD_MEMPOOL_SYSTEM_CACHEABLE = 1, | |
216 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2, | |
217 | KFD_MEMPOOL_FRAMEBUFFER = 3, | |
218 | }; | |
219 | ||
4a488a7a OG |
220 | /* Character device interface */ |
221 | int kfd_chardev_init(void); | |
222 | void kfd_chardev_exit(void); | |
223 | struct device *kfd_chardev(void); | |
224 | ||
241f24f8 | 225 | /** |
7da2bcf8 | 226 | * enum kfd_unmap_queues_filter |
241f24f8 | 227 | * |
7da2bcf8 | 228 | * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue. |
241f24f8 | 229 | * |
7da2bcf8 | 230 | * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the |
241f24f8 BG |
231 | * running queues list. |
232 | * | |
7da2bcf8 | 233 | * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to |
241f24f8 BG |
234 | * specific process. |
235 | * | |
236 | */ | |
7da2bcf8 YZ |
237 | enum kfd_unmap_queues_filter { |
238 | KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE, | |
239 | KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, | |
240 | KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, | |
241 | KFD_UNMAP_QUEUES_FILTER_BY_PASID | |
241f24f8 | 242 | }; |
19f6d2a6 | 243 | |
ed8aab45 BG |
244 | /** |
245 | * enum kfd_queue_type | |
246 | * | |
247 | * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type. | |
248 | * | |
249 | * @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type. | |
250 | * | |
251 | * @KFD_QUEUE_TYPE_HIQ: HIQ queue type. | |
252 | * | |
253 | * @KFD_QUEUE_TYPE_DIQ: DIQ queue type. | |
254 | */ | |
255 | enum kfd_queue_type { | |
256 | KFD_QUEUE_TYPE_COMPUTE, | |
257 | KFD_QUEUE_TYPE_SDMA, | |
258 | KFD_QUEUE_TYPE_HIQ, | |
259 | KFD_QUEUE_TYPE_DIQ | |
260 | }; | |
261 | ||
6e99df57 BG |
262 | enum kfd_queue_format { |
263 | KFD_QUEUE_FORMAT_PM4, | |
264 | KFD_QUEUE_FORMAT_AQL | |
265 | }; | |
266 | ||
ed8aab45 BG |
267 | /** |
268 | * struct queue_properties | |
269 | * | |
270 | * @type: The queue type. | |
271 | * | |
272 | * @queue_id: Queue identifier. | |
273 | * | |
274 | * @queue_address: Queue ring buffer address. | |
275 | * | |
276 | * @queue_size: Queue ring buffer size. | |
277 | * | |
278 | * @priority: Defines the queue priority relative to other queues in the | |
279 | * process. | |
280 | * This is just an indication and HW scheduling may override the priority as | |
281 | * necessary while keeping the relative prioritization. | |
282 | * the priority granularity is from 0 to f which f is the highest priority. | |
283 | * currently all queues are initialized with the highest priority. | |
284 | * | |
285 | * @queue_percent: This field is partially implemented and currently a zero in | |
286 | * this field defines that the queue is non active. | |
287 | * | |
288 | * @read_ptr: User space address which points to the number of dwords the | |
289 | * cp read from the ring buffer. This field updates automatically by the H/W. | |
290 | * | |
291 | * @write_ptr: Defines the number of dwords written to the ring buffer. | |
292 | * | |
293 | * @doorbell_ptr: This field aim is to notify the H/W of new packet written to | |
8eabaf54 KR |
294 | * the queue ring buffer. This field should be similar to write_ptr and the |
295 | * user should update this field after he updated the write_ptr. | |
ed8aab45 BG |
296 | * |
297 | * @doorbell_off: The doorbell offset in the doorbell pci-bar. | |
298 | * | |
8eabaf54 KR |
299 | * @is_interop: Defines if this is a interop queue. Interop queue means that |
300 | * the queue can access both graphics and compute resources. | |
ed8aab45 BG |
301 | * |
302 | * @is_active: Defines if the queue is active or not. | |
303 | * | |
304 | * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid | |
305 | * of the queue. | |
306 | * | |
307 | * This structure represents the queue properties for each queue no matter if | |
308 | * it's user mode or kernel mode queue. | |
309 | * | |
310 | */ | |
311 | struct queue_properties { | |
312 | enum kfd_queue_type type; | |
6e99df57 | 313 | enum kfd_queue_format format; |
ed8aab45 BG |
314 | unsigned int queue_id; |
315 | uint64_t queue_address; | |
316 | uint64_t queue_size; | |
317 | uint32_t priority; | |
318 | uint32_t queue_percent; | |
319 | uint32_t *read_ptr; | |
320 | uint32_t *write_ptr; | |
5cd78de5 | 321 | uint32_t __iomem *doorbell_ptr; |
ed8aab45 BG |
322 | uint32_t doorbell_off; |
323 | bool is_interop; | |
324 | bool is_active; | |
325 | /* Not relevant for user mode queues in cp scheduling */ | |
326 | unsigned int vmid; | |
77669eb8 BG |
327 | /* Relevant only for sdma queues*/ |
328 | uint32_t sdma_engine_id; | |
329 | uint32_t sdma_queue_id; | |
330 | uint32_t sdma_vm_addr; | |
ff3d04a1 BG |
331 | /* Relevant only for VI */ |
332 | uint64_t eop_ring_buffer_address; | |
333 | uint32_t eop_ring_buffer_size; | |
334 | uint64_t ctx_save_restore_area_address; | |
335 | uint32_t ctx_save_restore_area_size; | |
ed8aab45 BG |
336 | }; |
337 | ||
338 | /** | |
339 | * struct queue | |
340 | * | |
341 | * @list: Queue linked list. | |
342 | * | |
343 | * @mqd: The queue MQD. | |
344 | * | |
345 | * @mqd_mem_obj: The MQD local gpu memory object. | |
346 | * | |
347 | * @gart_mqd_addr: The MQD gart mc address. | |
348 | * | |
349 | * @properties: The queue properties. | |
350 | * | |
351 | * @mec: Used only in no cp scheduling mode and identifies to micro engine id | |
8eabaf54 | 352 | * that the queue should be execute on. |
ed8aab45 | 353 | * |
8eabaf54 KR |
354 | * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe |
355 | * id. | |
ed8aab45 BG |
356 | * |
357 | * @queue: Used only in no cp scheduliong mode and identifies the queue's slot. | |
358 | * | |
359 | * @process: The kfd process that created this queue. | |
360 | * | |
361 | * @device: The kfd device that created this queue. | |
362 | * | |
363 | * This structure represents user mode compute queues. | |
364 | * It contains all the necessary data to handle such queues. | |
365 | * | |
366 | */ | |
367 | ||
368 | struct queue { | |
369 | struct list_head list; | |
370 | void *mqd; | |
371 | struct kfd_mem_obj *mqd_mem_obj; | |
372 | uint64_t gart_mqd_addr; | |
373 | struct queue_properties properties; | |
374 | ||
375 | uint32_t mec; | |
376 | uint32_t pipe; | |
377 | uint32_t queue; | |
378 | ||
77669eb8 BG |
379 | unsigned int sdma_id; |
380 | ||
ed8aab45 BG |
381 | struct kfd_process *process; |
382 | struct kfd_dev *device; | |
383 | }; | |
384 | ||
6e99df57 BG |
385 | /* |
386 | * Please read the kfd_mqd_manager.h description. | |
387 | */ | |
388 | enum KFD_MQD_TYPE { | |
85d258f9 BG |
389 | KFD_MQD_TYPE_COMPUTE = 0, /* for no cp scheduling */ |
390 | KFD_MQD_TYPE_HIQ, /* for hiq */ | |
391 | KFD_MQD_TYPE_CP, /* for cp queues and diq */ | |
392 | KFD_MQD_TYPE_SDMA, /* for sdma queues */ | |
6e99df57 BG |
393 | KFD_MQD_TYPE_MAX |
394 | }; | |
395 | ||
241f24f8 BG |
396 | struct scheduling_resources { |
397 | unsigned int vmid_mask; | |
398 | enum kfd_queue_type type; | |
399 | uint64_t queue_mask; | |
400 | uint64_t gws_mask; | |
401 | uint32_t oac_mask; | |
402 | uint32_t gds_heap_base; | |
403 | uint32_t gds_heap_size; | |
404 | }; | |
405 | ||
406 | struct process_queue_manager { | |
407 | /* data */ | |
408 | struct kfd_process *process; | |
241f24f8 BG |
409 | struct list_head queues; |
410 | unsigned long *queue_slot_bitmap; | |
411 | }; | |
412 | ||
413 | struct qcm_process_device { | |
414 | /* The Device Queue Manager that owns this data */ | |
415 | struct device_queue_manager *dqm; | |
416 | struct process_queue_manager *pqm; | |
241f24f8 BG |
417 | /* Queues list */ |
418 | struct list_head queues_list; | |
419 | struct list_head priv_queue_list; | |
420 | ||
421 | unsigned int queue_count; | |
422 | unsigned int vmid; | |
423 | bool is_debug; | |
9fd3f1bf FK |
424 | |
425 | /* This flag tells if we should reset all wavefronts on | |
426 | * process termination | |
427 | */ | |
428 | bool reset_wavefronts; | |
429 | ||
241f24f8 BG |
430 | /* |
431 | * All the memory management data should be here too | |
432 | */ | |
433 | uint64_t gds_context_area; | |
434 | uint32_t sh_mem_config; | |
435 | uint32_t sh_mem_bases; | |
436 | uint32_t sh_mem_ape1_base; | |
437 | uint32_t sh_mem_ape1_limit; | |
438 | uint32_t page_table_base; | |
439 | uint32_t gds_size; | |
440 | uint32_t num_gws; | |
441 | uint32_t num_oac; | |
6a1c9510 | 442 | uint32_t sh_hidden_private_base; |
241f24f8 BG |
443 | }; |
444 | ||
733fa1f7 YZ |
445 | |
446 | enum kfd_pdd_bound { | |
447 | PDD_UNBOUND = 0, | |
448 | PDD_BOUND, | |
449 | PDD_BOUND_SUSPENDED, | |
450 | }; | |
451 | ||
19f6d2a6 OG |
452 | /* Data that is per-process-per device. */ |
453 | struct kfd_process_device { | |
454 | /* | |
455 | * List of all per-device data for a process. | |
456 | * Starts from kfd_process.per_device_data. | |
457 | */ | |
458 | struct list_head per_device_list; | |
459 | ||
460 | /* The device that owns this data. */ | |
461 | struct kfd_dev *dev; | |
462 | ||
9fd3f1bf FK |
463 | /* The process that owns this kfd_process_device. */ |
464 | struct kfd_process *process; | |
19f6d2a6 | 465 | |
45102048 BG |
466 | /* per-process-per device QCM data structure */ |
467 | struct qcm_process_device qpd; | |
468 | ||
19f6d2a6 OG |
469 | /*Apertures*/ |
470 | uint64_t lds_base; | |
471 | uint64_t lds_limit; | |
472 | uint64_t gpuvm_base; | |
473 | uint64_t gpuvm_limit; | |
474 | uint64_t scratch_base; | |
475 | uint64_t scratch_limit; | |
476 | ||
477 | /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */ | |
733fa1f7 | 478 | enum kfd_pdd_bound bound; |
a82918f1 | 479 | |
9fd3f1bf FK |
480 | /* Flag used to tell the pdd has dequeued from the dqm. |
481 | * This is used to prevent dev->dqm->ops.process_termination() from | |
482 | * being called twice when it is already called in IOMMU callback | |
483 | * function. | |
a82918f1 | 484 | */ |
9fd3f1bf | 485 | bool already_dequeued; |
19f6d2a6 OG |
486 | }; |
487 | ||
52a5fdce AS |
488 | #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd) |
489 | ||
4a488a7a OG |
490 | /* Process data */ |
491 | struct kfd_process { | |
19f6d2a6 OG |
492 | /* |
493 | * kfd_process are stored in an mm_struct*->kfd_process* | |
494 | * hash table (kfd_processes in kfd_process.c) | |
495 | */ | |
496 | struct hlist_node kfd_processes; | |
497 | ||
9b56bb11 FK |
498 | /* |
499 | * Opaque pointer to mm_struct. We don't hold a reference to | |
500 | * it so it should never be dereferenced from here. This is | |
501 | * only used for looking up processes by their mm. | |
502 | */ | |
503 | void *mm; | |
19f6d2a6 OG |
504 | |
505 | struct mutex mutex; | |
506 | ||
507 | /* | |
508 | * In any process, the thread that started main() is the lead | |
509 | * thread and outlives the rest. | |
510 | * It is here because amd_iommu_bind_pasid wants a task_struct. | |
511 | */ | |
512 | struct task_struct *lead_thread; | |
513 | ||
514 | /* We want to receive a notification when the mm_struct is destroyed */ | |
515 | struct mmu_notifier mmu_notifier; | |
516 | ||
517 | /* Use for delayed freeing of kfd_process structure */ | |
518 | struct rcu_head rcu; | |
519 | ||
520 | unsigned int pasid; | |
a91e70e3 | 521 | unsigned int doorbell_index; |
19f6d2a6 OG |
522 | |
523 | /* | |
524 | * List of kfd_process_device structures, | |
525 | * one for each device the process is using. | |
526 | */ | |
527 | struct list_head per_device_data; | |
528 | ||
45102048 BG |
529 | struct process_queue_manager pqm; |
530 | ||
19f6d2a6 OG |
531 | /* The process's queues. */ |
532 | size_t queue_array_size; | |
533 | ||
534 | /* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */ | |
535 | struct kfd_queue **queues; | |
536 | ||
19f6d2a6 OG |
537 | /*Is the user space process 32 bit?*/ |
538 | bool is_32bit_user_mode; | |
f3a39818 AL |
539 | |
540 | /* Event-related data */ | |
541 | struct mutex event_mutex; | |
482f0777 FK |
542 | /* Event ID allocator and lookup */ |
543 | struct idr event_idr; | |
50cb7dd9 FK |
544 | /* Event page */ |
545 | struct kfd_signal_page *signal_page; | |
b9a5d0a5 | 546 | size_t signal_mapped_size; |
f3a39818 | 547 | size_t signal_event_count; |
c986169f | 548 | bool signal_event_limit_reached; |
4a488a7a OG |
549 | }; |
550 | ||
76baee6c OG |
551 | /** |
552 | * Ioctl function type. | |
553 | * | |
554 | * \param filep pointer to file structure. | |
555 | * \param p amdkfd process pointer. | |
556 | * \param data pointer to arg that was copied from user. | |
557 | */ | |
558 | typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p, | |
559 | void *data); | |
560 | ||
561 | struct amdkfd_ioctl_desc { | |
562 | unsigned int cmd; | |
563 | int flags; | |
564 | amdkfd_ioctl_t *func; | |
565 | unsigned int cmd_drv; | |
566 | const char *name; | |
567 | }; | |
568 | ||
19f6d2a6 OG |
569 | void kfd_process_create_wq(void); |
570 | void kfd_process_destroy_wq(void); | |
571 | struct kfd_process *kfd_create_process(const struct task_struct *); | |
572 | struct kfd_process *kfd_get_process(const struct task_struct *); | |
f3a39818 | 573 | struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid); |
19f6d2a6 | 574 | |
64c7f8cf | 575 | struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, |
733fa1f7 YZ |
576 | struct kfd_process *p); |
577 | int kfd_bind_processes_to_device(struct kfd_dev *dev); | |
578 | void kfd_unbind_processes_from_device(struct kfd_dev *dev); | |
579 | void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned int pasid); | |
19f6d2a6 | 580 | struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, |
093c7d8c AS |
581 | struct kfd_process *p); |
582 | struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, | |
583 | struct kfd_process *p); | |
19f6d2a6 | 584 | |
775921ed | 585 | /* Process device data iterator */ |
8eabaf54 KR |
586 | struct kfd_process_device *kfd_get_first_process_device_data( |
587 | struct kfd_process *p); | |
588 | struct kfd_process_device *kfd_get_next_process_device_data( | |
589 | struct kfd_process *p, | |
775921ed AS |
590 | struct kfd_process_device *pdd); |
591 | bool kfd_has_process_device_data(struct kfd_process *p); | |
592 | ||
19f6d2a6 OG |
593 | /* PASIDs */ |
594 | int kfd_pasid_init(void); | |
595 | void kfd_pasid_exit(void); | |
596 | bool kfd_set_pasid_limit(unsigned int new_limit); | |
597 | unsigned int kfd_get_pasid_limit(void); | |
598 | unsigned int kfd_pasid_alloc(void); | |
599 | void kfd_pasid_free(unsigned int pasid); | |
600 | ||
601 | /* Doorbells */ | |
735df2ba FK |
602 | int kfd_doorbell_init(struct kfd_dev *kfd); |
603 | void kfd_doorbell_fini(struct kfd_dev *kfd); | |
19f6d2a6 OG |
604 | int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma); |
605 | u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, | |
606 | unsigned int *doorbell_off); | |
607 | void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr); | |
608 | u32 read_kernel_doorbell(u32 __iomem *db); | |
609 | void write_kernel_doorbell(u32 __iomem *db, u32 value); | |
610 | unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd, | |
611 | struct kfd_process *process, | |
612 | unsigned int queue_id); | |
a91e70e3 FK |
613 | phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, |
614 | struct kfd_process *process); | |
615 | int kfd_alloc_process_doorbells(struct kfd_process *process); | |
616 | void kfd_free_process_doorbells(struct kfd_process *process); | |
19f6d2a6 | 617 | |
6e81090b OG |
618 | /* GTT Sub-Allocator */ |
619 | ||
620 | int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, | |
621 | struct kfd_mem_obj **mem_obj); | |
622 | ||
623 | int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj); | |
624 | ||
4a488a7a OG |
625 | extern struct device *kfd_device; |
626 | ||
5b5c4e40 EP |
627 | /* Topology */ |
628 | int kfd_topology_init(void); | |
629 | void kfd_topology_shutdown(void); | |
630 | int kfd_topology_add_device(struct kfd_dev *gpu); | |
631 | int kfd_topology_remove_device(struct kfd_dev *gpu); | |
632 | struct kfd_dev *kfd_device_by_id(uint32_t gpu_id); | |
633 | struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); | |
634 | struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx); | |
635 | ||
4a488a7a | 636 | /* Interrupts */ |
2249d558 AL |
637 | int kfd_interrupt_init(struct kfd_dev *dev); |
638 | void kfd_interrupt_exit(struct kfd_dev *dev); | |
b3f5e6b4 | 639 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); |
2249d558 AL |
640 | bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry); |
641 | bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry); | |
4a488a7a OG |
642 | |
643 | /* Power Management */ | |
b3f5e6b4 AL |
644 | void kgd2kfd_suspend(struct kfd_dev *kfd); |
645 | int kgd2kfd_resume(struct kfd_dev *kfd); | |
4a488a7a | 646 | |
19f6d2a6 OG |
647 | /* amdkfd Apertures */ |
648 | int kfd_init_apertures(struct kfd_process *process); | |
649 | ||
ed6e6a34 | 650 | /* Queue Context Management */ |
77669eb8 | 651 | struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd); |
241f24f8 | 652 | |
e88a614c | 653 | int init_queue(struct queue **q, const struct queue_properties *properties); |
ed6e6a34 | 654 | void uninit_queue(struct queue *q); |
45102048 | 655 | void print_queue_properties(struct queue_properties *q); |
ed6e6a34 BG |
656 | void print_queue(struct queue *q); |
657 | ||
64c7f8cf BG |
658 | struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, |
659 | struct kfd_dev *dev); | |
4b8f589b BG |
660 | struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, |
661 | struct kfd_dev *dev); | |
662 | struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type, | |
663 | struct kfd_dev *dev); | |
64c7f8cf BG |
664 | struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev); |
665 | void device_queue_manager_uninit(struct device_queue_manager *dqm); | |
241f24f8 BG |
666 | struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, |
667 | enum kfd_queue_type type); | |
668 | void kernel_queue_uninit(struct kernel_queue *kq); | |
669 | ||
45102048 BG |
670 | /* Process Queue Manager */ |
671 | struct process_queue_node { | |
672 | struct queue *q; | |
673 | struct kernel_queue *kq; | |
674 | struct list_head process_queue_list; | |
675 | }; | |
676 | ||
9fd3f1bf FK |
677 | void kfd_process_dequeue_from_device(struct kfd_process_device *pdd); |
678 | void kfd_process_dequeue_from_all_devices(struct kfd_process *p); | |
45102048 BG |
679 | int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p); |
680 | void pqm_uninit(struct process_queue_manager *pqm); | |
681 | int pqm_create_queue(struct process_queue_manager *pqm, | |
682 | struct kfd_dev *dev, | |
683 | struct file *f, | |
684 | struct queue_properties *properties, | |
45102048 BG |
685 | unsigned int *qid); |
686 | int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid); | |
687 | int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, | |
688 | struct queue_properties *p); | |
fbeb661b YS |
689 | struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm, |
690 | unsigned int qid); | |
45102048 | 691 | |
788bf83d YS |
692 | int amdkfd_fence_wait_timeout(unsigned int *fence_addr, |
693 | unsigned int fence_value, | |
8c72c3d7 | 694 | unsigned int timeout_ms); |
788bf83d | 695 | |
ed6e6a34 BG |
696 | /* Packet Manager */ |
697 | ||
64c7f8cf BG |
698 | #define KFD_FENCE_COMPLETED (100) |
699 | #define KFD_FENCE_INIT (10) | |
241f24f8 | 700 | |
ed6e6a34 BG |
701 | struct packet_manager { |
702 | struct device_queue_manager *dqm; | |
703 | struct kernel_queue *priv_queue; | |
704 | struct mutex lock; | |
705 | bool allocated; | |
706 | struct kfd_mem_obj *ib_buffer_obj; | |
707 | }; | |
708 | ||
64c7f8cf BG |
709 | int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm); |
710 | void pm_uninit(struct packet_manager *pm); | |
711 | int pm_send_set_resources(struct packet_manager *pm, | |
712 | struct scheduling_resources *res); | |
713 | int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues); | |
714 | int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, | |
715 | uint32_t fence_value); | |
716 | ||
717 | int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, | |
7da2bcf8 | 718 | enum kfd_unmap_queues_filter mode, |
64c7f8cf BG |
719 | uint32_t filter_param, bool reset, |
720 | unsigned int sdma_engine); | |
721 | ||
241f24f8 BG |
722 | void pm_release_ib(struct packet_manager *pm); |
723 | ||
19f6d2a6 | 724 | uint64_t kfd_get_number_elems(struct kfd_dev *kfd); |
19f6d2a6 | 725 | |
f3a39818 AL |
726 | /* Events */ |
727 | extern const struct kfd_event_interrupt_class event_interrupt_class_cik; | |
930c5ff4 | 728 | extern const struct kfd_device_global_init_class device_global_init_class_cik; |
f3a39818 | 729 | |
f3a39818 AL |
730 | void kfd_event_init_process(struct kfd_process *p); |
731 | void kfd_event_free_process(struct kfd_process *p); | |
732 | int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma); | |
733 | int kfd_wait_on_events(struct kfd_process *p, | |
59d3e8be | 734 | uint32_t num_events, void __user *data, |
f3a39818 | 735 | bool all, uint32_t user_timeout_ms, |
fdf0c833 | 736 | uint32_t *wait_result); |
f3a39818 AL |
737 | void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id, |
738 | uint32_t valid_id_bits); | |
59d3e8be AS |
739 | void kfd_signal_iommu_event(struct kfd_dev *dev, |
740 | unsigned int pasid, unsigned long address, | |
741 | bool is_write_requested, bool is_execute_requested); | |
930c5ff4 | 742 | void kfd_signal_hw_exception_event(unsigned int pasid); |
f3a39818 AL |
743 | int kfd_set_event(struct kfd_process *p, uint32_t event_id); |
744 | int kfd_reset_event(struct kfd_process *p, uint32_t event_id); | |
745 | int kfd_event_create(struct file *devkfd, struct kfd_process *p, | |
746 | uint32_t event_type, bool auto_reset, uint32_t node_id, | |
747 | uint32_t *event_id, uint32_t *event_trigger_data, | |
748 | uint64_t *event_page_offset, uint32_t *event_slot_index); | |
749 | int kfd_event_destroy(struct kfd_process *p, uint32_t event_id); | |
750 | ||
c3447e81 BG |
751 | int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p); |
752 | ||
4a488a7a | 753 | #endif |