Commit | Line | Data |
---|---|---|
4a488a7a OG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #ifndef KFD_PRIV_H_INCLUDED | |
24 | #define KFD_PRIV_H_INCLUDED | |
25 | ||
26 | #include <linux/hashtable.h> | |
27 | #include <linux/mmu_notifier.h> | |
28 | #include <linux/mutex.h> | |
29 | #include <linux/types.h> | |
30 | #include <linux/atomic.h> | |
31 | #include <linux/workqueue.h> | |
32 | #include <linux/spinlock.h> | |
19f6d2a6 | 33 | #include <linux/kfd_ioctl.h> |
482f0777 | 34 | #include <linux/idr.h> |
04ad47bd | 35 | #include <linux/kfifo.h> |
851a645e | 36 | #include <linux/seq_file.h> |
5ce10687 | 37 | #include <linux/kref.h> |
4a488a7a OG |
38 | #include <kgd_kfd_interface.h> |
39 | ||
e596b903 YZ |
40 | #include "amd_shared.h" |
41 | ||
af47b390 LA |
42 | #define KFD_MAX_RING_ENTRY_SIZE 8 |
43 | ||
5b5c4e40 EP |
44 | #define KFD_SYSFS_FILE_MODE 0444 |
45 | ||
df03ef93 HK |
46 | /* GPU ID hash width in bits */ |
47 | #define KFD_GPU_ID_HASH_WIDTH 16 | |
48 | ||
49 | /* Use upper bits of mmap offset to store KFD driver specific information. | |
50 | * BITS[63:62] - Encode MMAP type | |
51 | * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to | |
52 | * BITS[45:0] - MMAP offset value | |
53 | * | |
54 | * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these | |
55 | * defines are w.r.t to PAGE_SIZE | |
56 | */ | |
57 | #define KFD_MMAP_TYPE_SHIFT (62 - PAGE_SHIFT) | |
58 | #define KFD_MMAP_TYPE_MASK (0x3ULL << KFD_MMAP_TYPE_SHIFT) | |
59 | #define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT) | |
60 | #define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT) | |
61 | #define KFD_MMAP_TYPE_RESERVED_MEM (0x1ULL << KFD_MMAP_TYPE_SHIFT) | |
62 | ||
63 | #define KFD_MMAP_GPU_ID_SHIFT (46 - PAGE_SHIFT) | |
64 | #define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \ | |
65 | << KFD_MMAP_GPU_ID_SHIFT) | |
66 | #define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\ | |
67 | & KFD_MMAP_GPU_ID_MASK) | |
68 | #define KFD_MMAP_GPU_ID_GET(offset) ((offset & KFD_MMAP_GPU_ID_MASK) \ | |
69 | >> KFD_MMAP_GPU_ID_SHIFT) | |
70 | ||
71 | #define KFD_MMAP_OFFSET_VALUE_MASK (0x3FFFFFFFFFFFULL >> PAGE_SHIFT) | |
72 | #define KFD_MMAP_OFFSET_VALUE_GET(offset) (offset & KFD_MMAP_OFFSET_VALUE_MASK) | |
f3a39818 | 73 | |
ed6e6a34 BG |
74 | /* |
75 | * When working with cp scheduler we should assign the HIQ manually or via | |
e7016d8e | 76 | * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot |
ed6e6a34 BG |
77 | * definitions for Kaveri. In Kaveri only the first ME queues participates |
78 | * in the cp scheduling taking that in mind we set the HIQ slot in the | |
79 | * second ME. | |
80 | */ | |
81 | #define KFD_CIK_HIQ_PIPE 4 | |
82 | #define KFD_CIK_HIQ_QUEUE 0 | |
83 | ||
5b5c4e40 EP |
84 | /* Macro for allocating structures */ |
85 | #define kfd_alloc_struct(ptr_to_struct) \ | |
86 | ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL)) | |
87 | ||
19f6d2a6 | 88 | #define KFD_MAX_NUM_OF_PROCESSES 512 |
b8cbab04 | 89 | #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 |
19f6d2a6 | 90 | |
373d7080 FK |
91 | /* |
92 | * Size of the per-process TBA+TMA buffer: 2 pages | |
93 | * | |
94 | * The first page is the TBA used for the CWSR ISA code. The second | |
95 | * page is used as TMA for daisy changing a user-mode trap handler. | |
96 | */ | |
97 | #define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2) | |
98 | #define KFD_CWSR_TMA_OFFSET PAGE_SIZE | |
99 | ||
19f6d2a6 | 100 | /* |
b8cbab04 OG |
101 | * Kernel module parameter to specify maximum number of supported queues per |
102 | * device | |
19f6d2a6 | 103 | */ |
b8cbab04 | 104 | extern int max_num_of_queues_per_device; |
19f6d2a6 | 105 | |
b8cbab04 OG |
106 | #define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \ |
107 | (KFD_MAX_NUM_OF_PROCESSES * \ | |
108 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) | |
19f6d2a6 | 109 | |
ed6e6a34 BG |
110 | #define KFD_KERNEL_QUEUE_SIZE 2048 |
111 | ||
31c21fec BG |
112 | /* Kernel module parameter to specify the scheduling policy */ |
113 | extern int sched_policy; | |
114 | ||
a99c6d4f FK |
115 | /* |
116 | * Kernel module parameter to specify the maximum process | |
117 | * number per HW scheduler | |
118 | */ | |
119 | extern int hws_max_conc_proc; | |
120 | ||
373d7080 FK |
121 | extern int cwsr_enable; |
122 | ||
81663016 OG |
123 | /* |
124 | * Kernel module parameter to specify whether to send sigterm to HSA process on | |
125 | * unhandled exception | |
126 | */ | |
127 | extern int send_sigterm; | |
128 | ||
374200b1 FK |
129 | /* |
130 | * This kernel module is used to simulate large bar machine on non-large bar | |
131 | * enabled machines. | |
132 | */ | |
133 | extern int debug_largebar; | |
134 | ||
ebcfd1e2 FK |
135 | /* |
136 | * Ignore CRAT table during KFD initialization, can be used to work around | |
137 | * broken CRAT tables on some AMD systems | |
138 | */ | |
139 | extern int ignore_crat; | |
140 | ||
bed4f110 FK |
141 | /* |
142 | * Set sh_mem_config.retry_disable on Vega10 | |
143 | */ | |
2b281977 | 144 | extern int noretry; |
bed4f110 | 145 | |
0e9a860c YZ |
146 | /* |
147 | * Halt if HWS hang is detected | |
148 | */ | |
149 | extern int halt_if_hws_hang; | |
150 | ||
ed6e6a34 BG |
151 | enum cache_policy { |
152 | cache_policy_coherent, | |
153 | cache_policy_noncoherent | |
154 | }; | |
155 | ||
ef568db7 FK |
156 | #define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10) |
157 | ||
f3a39818 AL |
158 | struct kfd_event_interrupt_class { |
159 | bool (*interrupt_isr)(struct kfd_dev *dev, | |
58e69886 LX |
160 | const uint32_t *ih_ring_entry, uint32_t *patched_ihre, |
161 | bool *patched_flag); | |
f3a39818 | 162 | void (*interrupt_wq)(struct kfd_dev *dev, |
58e69886 | 163 | const uint32_t *ih_ring_entry); |
f3a39818 AL |
164 | }; |
165 | ||
4a488a7a | 166 | struct kfd_device_info { |
e596b903 | 167 | enum amd_asic_type asic_family; |
f3a39818 | 168 | const struct kfd_event_interrupt_class *event_interrupt_class; |
4a488a7a | 169 | unsigned int max_pasid_bits; |
992839ad | 170 | unsigned int max_no_of_hqd; |
ada2b29c | 171 | unsigned int doorbell_size; |
4a488a7a | 172 | size_t ih_ring_entry_size; |
f7c826ad | 173 | uint8_t num_of_watch_points; |
19f6d2a6 | 174 | uint16_t mqd_size_aligned; |
373d7080 | 175 | bool supports_cwsr; |
64d1c3a4 | 176 | bool needs_iommu_device; |
3ee2d00c | 177 | bool needs_pci_atomics; |
98bb9222 | 178 | unsigned int num_sdma_engines; |
d5094189 | 179 | unsigned int num_sdma_queues_per_engine; |
4a488a7a OG |
180 | }; |
181 | ||
36b5c08f OG |
182 | struct kfd_mem_obj { |
183 | uint32_t range_start; | |
184 | uint32_t range_end; | |
185 | uint64_t gpu_addr; | |
186 | uint32_t *cpu_ptr; | |
b91d43dd | 187 | void *gtt_mem; |
36b5c08f OG |
188 | }; |
189 | ||
44008d7a YZ |
190 | struct kfd_vmid_info { |
191 | uint32_t first_vmid_kfd; | |
192 | uint32_t last_vmid_kfd; | |
193 | uint32_t vmid_num_kfd; | |
194 | }; | |
195 | ||
4a488a7a OG |
196 | struct kfd_dev { |
197 | struct kgd_dev *kgd; | |
198 | ||
199 | const struct kfd_device_info *device_info; | |
200 | struct pci_dev *pdev; | |
201 | ||
202 | unsigned int id; /* topology stub index */ | |
203 | ||
19f6d2a6 OG |
204 | phys_addr_t doorbell_base; /* Start of actual doorbells used by |
205 | * KFD. It is aligned for mapping | |
206 | * into user mode | |
207 | */ | |
208 | size_t doorbell_id_offset; /* Doorbell offset (from KFD doorbell | |
209 | * to HW doorbell, GFX reserved some | |
210 | * at the start) | |
211 | */ | |
19f6d2a6 OG |
212 | u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells |
213 | * page used by kernel queue | |
214 | */ | |
215 | ||
4a488a7a | 216 | struct kgd2kfd_shared_resources shared_resources; |
44008d7a | 217 | struct kfd_vmid_info vm_info; |
4a488a7a | 218 | |
cea405b1 XZ |
219 | const struct kfd2kgd_calls *kfd2kgd; |
220 | struct mutex doorbell_mutex; | |
f761d8bd JP |
221 | DECLARE_BITMAP(doorbell_available_index, |
222 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); | |
cea405b1 | 223 | |
36b5c08f OG |
224 | void *gtt_mem; |
225 | uint64_t gtt_start_gpu_addr; | |
226 | void *gtt_start_cpu_ptr; | |
227 | void *gtt_sa_bitmap; | |
228 | struct mutex gtt_sa_lock; | |
229 | unsigned int gtt_sa_chunk_size; | |
230 | unsigned int gtt_sa_num_of_chunks; | |
231 | ||
2249d558 | 232 | /* Interrupts */ |
04ad47bd | 233 | struct kfifo ih_fifo; |
48e876a2 | 234 | struct workqueue_struct *ih_wq; |
2249d558 AL |
235 | struct work_struct interrupt_work; |
236 | spinlock_t interrupt_lock; | |
237 | ||
ed6e6a34 BG |
238 | /* QCM Device instance */ |
239 | struct device_queue_manager *dqm; | |
4a488a7a | 240 | |
ed6e6a34 | 241 | bool init_complete; |
2249d558 AL |
242 | /* |
243 | * Interrupts of interest to KFD are copied | |
244 | * from the HW ring into a SW ring. | |
245 | */ | |
246 | bool interrupts_active; | |
fbeb661b YS |
247 | |
248 | /* Debug manager */ | |
249 | struct kfd_dbgmgr *dbgmgr; | |
373d7080 | 250 | |
5ade6c9c FK |
251 | /* Firmware versions */ |
252 | uint16_t mec_fw_version; | |
253 | uint16_t sdma_fw_version; | |
254 | ||
a99c6d4f FK |
255 | /* Maximum process number mapped to HW scheduler */ |
256 | unsigned int max_proc_per_quantum; | |
257 | ||
373d7080 FK |
258 | /* CWSR */ |
259 | bool cwsr_enabled; | |
260 | const void *cwsr_isa; | |
261 | unsigned int cwsr_isa_size; | |
0c1690e3 SL |
262 | |
263 | /* xGMI */ | |
264 | uint64_t hive_id; | |
d35f00d8 EH |
265 | |
266 | bool pci_atomic_requested; | |
4a488a7a OG |
267 | }; |
268 | ||
269 | /* KGD2KFD callbacks */ | |
270 | void kgd2kfd_exit(void); | |
cea405b1 XZ |
271 | struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, |
272 | struct pci_dev *pdev, const struct kfd2kgd_calls *f2g); | |
4a488a7a | 273 | bool kgd2kfd_device_init(struct kfd_dev *kfd, |
cea405b1 | 274 | const struct kgd2kfd_shared_resources *gpu_resources); |
4a488a7a OG |
275 | void kgd2kfd_device_exit(struct kfd_dev *kfd); |
276 | ||
19f6d2a6 OG |
277 | enum kfd_mempool { |
278 | KFD_MEMPOOL_SYSTEM_CACHEABLE = 1, | |
279 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2, | |
280 | KFD_MEMPOOL_FRAMEBUFFER = 3, | |
281 | }; | |
282 | ||
4a488a7a OG |
283 | /* Character device interface */ |
284 | int kfd_chardev_init(void); | |
285 | void kfd_chardev_exit(void); | |
286 | struct device *kfd_chardev(void); | |
287 | ||
241f24f8 | 288 | /** |
7da2bcf8 | 289 | * enum kfd_unmap_queues_filter |
241f24f8 | 290 | * |
7da2bcf8 | 291 | * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue. |
241f24f8 | 292 | * |
7da2bcf8 | 293 | * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the |
241f24f8 BG |
294 | * running queues list. |
295 | * | |
7da2bcf8 | 296 | * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to |
241f24f8 BG |
297 | * specific process. |
298 | * | |
299 | */ | |
7da2bcf8 YZ |
300 | enum kfd_unmap_queues_filter { |
301 | KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE, | |
302 | KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, | |
303 | KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, | |
304 | KFD_UNMAP_QUEUES_FILTER_BY_PASID | |
241f24f8 | 305 | }; |
19f6d2a6 | 306 | |
ed8aab45 BG |
307 | /** |
308 | * enum kfd_queue_type | |
309 | * | |
310 | * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type. | |
311 | * | |
312 | * @KFD_QUEUE_TYPE_SDMA: Sdma user mode queue type. | |
313 | * | |
314 | * @KFD_QUEUE_TYPE_HIQ: HIQ queue type. | |
315 | * | |
316 | * @KFD_QUEUE_TYPE_DIQ: DIQ queue type. | |
317 | */ | |
318 | enum kfd_queue_type { | |
319 | KFD_QUEUE_TYPE_COMPUTE, | |
320 | KFD_QUEUE_TYPE_SDMA, | |
321 | KFD_QUEUE_TYPE_HIQ, | |
322 | KFD_QUEUE_TYPE_DIQ | |
323 | }; | |
324 | ||
6e99df57 BG |
325 | enum kfd_queue_format { |
326 | KFD_QUEUE_FORMAT_PM4, | |
327 | KFD_QUEUE_FORMAT_AQL | |
328 | }; | |
329 | ||
ed8aab45 BG |
330 | /** |
331 | * struct queue_properties | |
332 | * | |
333 | * @type: The queue type. | |
334 | * | |
335 | * @queue_id: Queue identifier. | |
336 | * | |
337 | * @queue_address: Queue ring buffer address. | |
338 | * | |
339 | * @queue_size: Queue ring buffer size. | |
340 | * | |
341 | * @priority: Defines the queue priority relative to other queues in the | |
342 | * process. | |
343 | * This is just an indication and HW scheduling may override the priority as | |
344 | * necessary while keeping the relative prioritization. | |
345 | * the priority granularity is from 0 to f which f is the highest priority. | |
346 | * currently all queues are initialized with the highest priority. | |
347 | * | |
348 | * @queue_percent: This field is partially implemented and currently a zero in | |
349 | * this field defines that the queue is non active. | |
350 | * | |
351 | * @read_ptr: User space address which points to the number of dwords the | |
352 | * cp read from the ring buffer. This field updates automatically by the H/W. | |
353 | * | |
354 | * @write_ptr: Defines the number of dwords written to the ring buffer. | |
355 | * | |
356 | * @doorbell_ptr: This field aim is to notify the H/W of new packet written to | |
8eabaf54 KR |
357 | * the queue ring buffer. This field should be similar to write_ptr and the |
358 | * user should update this field after he updated the write_ptr. | |
ed8aab45 BG |
359 | * |
360 | * @doorbell_off: The doorbell offset in the doorbell pci-bar. | |
361 | * | |
8eabaf54 KR |
362 | * @is_interop: Defines if this is a interop queue. Interop queue means that |
363 | * the queue can access both graphics and compute resources. | |
ed8aab45 | 364 | * |
26103436 FK |
365 | * @is_evicted: Defines if the queue is evicted. Only active queues |
366 | * are evicted, rendering them inactive. | |
367 | * | |
368 | * @is_active: Defines if the queue is active or not. @is_active and | |
369 | * @is_evicted are protected by the DQM lock. | |
ed8aab45 BG |
370 | * |
371 | * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid | |
372 | * of the queue. | |
373 | * | |
374 | * This structure represents the queue properties for each queue no matter if | |
375 | * it's user mode or kernel mode queue. | |
376 | * | |
377 | */ | |
378 | struct queue_properties { | |
379 | enum kfd_queue_type type; | |
6e99df57 | 380 | enum kfd_queue_format format; |
ed8aab45 BG |
381 | unsigned int queue_id; |
382 | uint64_t queue_address; | |
383 | uint64_t queue_size; | |
384 | uint32_t priority; | |
385 | uint32_t queue_percent; | |
386 | uint32_t *read_ptr; | |
387 | uint32_t *write_ptr; | |
ada2b29c | 388 | void __iomem *doorbell_ptr; |
ed8aab45 BG |
389 | uint32_t doorbell_off; |
390 | bool is_interop; | |
26103436 | 391 | bool is_evicted; |
ed8aab45 BG |
392 | bool is_active; |
393 | /* Not relevant for user mode queues in cp scheduling */ | |
394 | unsigned int vmid; | |
77669eb8 BG |
395 | /* Relevant only for sdma queues*/ |
396 | uint32_t sdma_engine_id; | |
397 | uint32_t sdma_queue_id; | |
398 | uint32_t sdma_vm_addr; | |
ff3d04a1 BG |
399 | /* Relevant only for VI */ |
400 | uint64_t eop_ring_buffer_address; | |
401 | uint32_t eop_ring_buffer_size; | |
402 | uint64_t ctx_save_restore_area_address; | |
403 | uint32_t ctx_save_restore_area_size; | |
373d7080 FK |
404 | uint32_t ctl_stack_size; |
405 | uint64_t tba_addr; | |
406 | uint64_t tma_addr; | |
39e7f331 FK |
407 | /* Relevant for CU */ |
408 | uint32_t cu_mask_count; /* Must be a multiple of 32 */ | |
409 | uint32_t *cu_mask; | |
ed8aab45 BG |
410 | }; |
411 | ||
412 | /** | |
413 | * struct queue | |
414 | * | |
415 | * @list: Queue linked list. | |
416 | * | |
417 | * @mqd: The queue MQD. | |
418 | * | |
419 | * @mqd_mem_obj: The MQD local gpu memory object. | |
420 | * | |
421 | * @gart_mqd_addr: The MQD gart mc address. | |
422 | * | |
423 | * @properties: The queue properties. | |
424 | * | |
425 | * @mec: Used only in no cp scheduling mode and identifies to micro engine id | |
8eabaf54 | 426 | * that the queue should be execute on. |
ed8aab45 | 427 | * |
8eabaf54 KR |
428 | * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe |
429 | * id. | |
ed8aab45 BG |
430 | * |
431 | * @queue: Used only in no cp scheduliong mode and identifies the queue's slot. | |
432 | * | |
433 | * @process: The kfd process that created this queue. | |
434 | * | |
435 | * @device: The kfd device that created this queue. | |
436 | * | |
437 | * This structure represents user mode compute queues. | |
438 | * It contains all the necessary data to handle such queues. | |
439 | * | |
440 | */ | |
441 | ||
442 | struct queue { | |
443 | struct list_head list; | |
444 | void *mqd; | |
445 | struct kfd_mem_obj *mqd_mem_obj; | |
446 | uint64_t gart_mqd_addr; | |
447 | struct queue_properties properties; | |
448 | ||
449 | uint32_t mec; | |
450 | uint32_t pipe; | |
451 | uint32_t queue; | |
452 | ||
77669eb8 | 453 | unsigned int sdma_id; |
ef568db7 | 454 | unsigned int doorbell_id; |
77669eb8 | 455 | |
ed8aab45 BG |
456 | struct kfd_process *process; |
457 | struct kfd_dev *device; | |
458 | }; | |
459 | ||
6e99df57 BG |
460 | /* |
461 | * Please read the kfd_mqd_manager.h description. | |
462 | */ | |
463 | enum KFD_MQD_TYPE { | |
85d258f9 BG |
464 | KFD_MQD_TYPE_COMPUTE = 0, /* for no cp scheduling */ |
465 | KFD_MQD_TYPE_HIQ, /* for hiq */ | |
466 | KFD_MQD_TYPE_CP, /* for cp queues and diq */ | |
467 | KFD_MQD_TYPE_SDMA, /* for sdma queues */ | |
6e99df57 BG |
468 | KFD_MQD_TYPE_MAX |
469 | }; | |
470 | ||
241f24f8 BG |
471 | struct scheduling_resources { |
472 | unsigned int vmid_mask; | |
473 | enum kfd_queue_type type; | |
474 | uint64_t queue_mask; | |
475 | uint64_t gws_mask; | |
476 | uint32_t oac_mask; | |
477 | uint32_t gds_heap_base; | |
478 | uint32_t gds_heap_size; | |
479 | }; | |
480 | ||
481 | struct process_queue_manager { | |
482 | /* data */ | |
483 | struct kfd_process *process; | |
241f24f8 BG |
484 | struct list_head queues; |
485 | unsigned long *queue_slot_bitmap; | |
486 | }; | |
487 | ||
488 | struct qcm_process_device { | |
489 | /* The Device Queue Manager that owns this data */ | |
490 | struct device_queue_manager *dqm; | |
491 | struct process_queue_manager *pqm; | |
241f24f8 BG |
492 | /* Queues list */ |
493 | struct list_head queues_list; | |
494 | struct list_head priv_queue_list; | |
495 | ||
496 | unsigned int queue_count; | |
497 | unsigned int vmid; | |
498 | bool is_debug; | |
26103436 | 499 | unsigned int evicted; /* eviction counter, 0=active */ |
9fd3f1bf FK |
500 | |
501 | /* This flag tells if we should reset all wavefronts on | |
502 | * process termination | |
503 | */ | |
504 | bool reset_wavefronts; | |
505 | ||
241f24f8 BG |
506 | /* |
507 | * All the memory management data should be here too | |
508 | */ | |
509 | uint64_t gds_context_area; | |
e715c6d0 | 510 | uint64_t page_table_base; |
241f24f8 BG |
511 | uint32_t sh_mem_config; |
512 | uint32_t sh_mem_bases; | |
513 | uint32_t sh_mem_ape1_base; | |
514 | uint32_t sh_mem_ape1_limit; | |
241f24f8 BG |
515 | uint32_t gds_size; |
516 | uint32_t num_gws; | |
517 | uint32_t num_oac; | |
6a1c9510 | 518 | uint32_t sh_hidden_private_base; |
373d7080 FK |
519 | |
520 | /* CWSR memory */ | |
521 | void *cwsr_kaddr; | |
d01994c2 | 522 | uint64_t cwsr_base; |
373d7080 FK |
523 | uint64_t tba_addr; |
524 | uint64_t tma_addr; | |
d01994c2 FK |
525 | |
526 | /* IB memory */ | |
527 | uint64_t ib_base; | |
552764b6 | 528 | void *ib_kaddr; |
ef568db7 FK |
529 | |
530 | /* doorbell resources per process per device */ | |
531 | unsigned long *doorbell_bitmap; | |
241f24f8 BG |
532 | }; |
533 | ||
26103436 FK |
534 | /* KFD Memory Eviction */ |
535 | ||
536 | /* Approx. wait time before attempting to restore evicted BOs */ | |
537 | #define PROCESS_RESTORE_TIME_MS 100 | |
538 | /* Approx. back off time if restore fails due to lack of memory */ | |
539 | #define PROCESS_BACK_OFF_TIME_MS 100 | |
540 | /* Approx. time before evicting the process again */ | |
541 | #define PROCESS_ACTIVE_TIME_MS 10 | |
542 | ||
6b95e797 FK |
543 | int kgd2kfd_quiesce_mm(struct mm_struct *mm); |
544 | int kgd2kfd_resume_mm(struct mm_struct *mm); | |
26103436 FK |
545 | int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, |
546 | struct dma_fence *fence); | |
733fa1f7 | 547 | |
5ec7e028 FK |
548 | /* 8 byte handle containing GPU ID in the most significant 4 bytes and |
549 | * idr_handle in the least significant 4 bytes | |
550 | */ | |
551 | #define MAKE_HANDLE(gpu_id, idr_handle) \ | |
552 | (((uint64_t)(gpu_id) << 32) + idr_handle) | |
553 | #define GET_GPU_ID(handle) (handle >> 32) | |
554 | #define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF) | |
555 | ||
733fa1f7 YZ |
556 | enum kfd_pdd_bound { |
557 | PDD_UNBOUND = 0, | |
558 | PDD_BOUND, | |
559 | PDD_BOUND_SUSPENDED, | |
560 | }; | |
561 | ||
19f6d2a6 OG |
562 | /* Data that is per-process-per device. */ |
563 | struct kfd_process_device { | |
564 | /* | |
565 | * List of all per-device data for a process. | |
566 | * Starts from kfd_process.per_device_data. | |
567 | */ | |
568 | struct list_head per_device_list; | |
569 | ||
570 | /* The device that owns this data. */ | |
571 | struct kfd_dev *dev; | |
572 | ||
9fd3f1bf FK |
573 | /* The process that owns this kfd_process_device. */ |
574 | struct kfd_process *process; | |
19f6d2a6 | 575 | |
45102048 BG |
576 | /* per-process-per device QCM data structure */ |
577 | struct qcm_process_device qpd; | |
578 | ||
19f6d2a6 OG |
579 | /*Apertures*/ |
580 | uint64_t lds_base; | |
581 | uint64_t lds_limit; | |
582 | uint64_t gpuvm_base; | |
583 | uint64_t gpuvm_limit; | |
584 | uint64_t scratch_base; | |
585 | uint64_t scratch_limit; | |
586 | ||
403575c4 | 587 | /* VM context for GPUVM allocations */ |
b84394e2 | 588 | struct file *drm_file; |
403575c4 FK |
589 | void *vm; |
590 | ||
52b29d73 FK |
591 | /* GPUVM allocations storage */ |
592 | struct idr alloc_idr; | |
593 | ||
9fd3f1bf FK |
594 | /* Flag used to tell the pdd has dequeued from the dqm. |
595 | * This is used to prevent dev->dqm->ops.process_termination() from | |
596 | * being called twice when it is already called in IOMMU callback | |
597 | * function. | |
a82918f1 | 598 | */ |
9fd3f1bf | 599 | bool already_dequeued; |
64d1c3a4 FK |
600 | |
601 | /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */ | |
602 | enum kfd_pdd_bound bound; | |
19f6d2a6 OG |
603 | }; |
604 | ||
52a5fdce AS |
605 | #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd) |
606 | ||
4a488a7a OG |
607 | /* Process data */ |
608 | struct kfd_process { | |
19f6d2a6 OG |
609 | /* |
610 | * kfd_process are stored in an mm_struct*->kfd_process* | |
611 | * hash table (kfd_processes in kfd_process.c) | |
612 | */ | |
613 | struct hlist_node kfd_processes; | |
614 | ||
9b56bb11 FK |
615 | /* |
616 | * Opaque pointer to mm_struct. We don't hold a reference to | |
617 | * it so it should never be dereferenced from here. This is | |
618 | * only used for looking up processes by their mm. | |
619 | */ | |
620 | void *mm; | |
19f6d2a6 | 621 | |
5ce10687 FK |
622 | struct kref ref; |
623 | struct work_struct release_work; | |
624 | ||
19f6d2a6 OG |
625 | struct mutex mutex; |
626 | ||
627 | /* | |
628 | * In any process, the thread that started main() is the lead | |
629 | * thread and outlives the rest. | |
630 | * It is here because amd_iommu_bind_pasid wants a task_struct. | |
894a8293 FK |
631 | * It can also be used for safely getting a reference to the |
632 | * mm_struct of the process. | |
19f6d2a6 OG |
633 | */ |
634 | struct task_struct *lead_thread; | |
635 | ||
636 | /* We want to receive a notification when the mm_struct is destroyed */ | |
637 | struct mmu_notifier mmu_notifier; | |
638 | ||
639 | /* Use for delayed freeing of kfd_process structure */ | |
640 | struct rcu_head rcu; | |
641 | ||
642 | unsigned int pasid; | |
a91e70e3 | 643 | unsigned int doorbell_index; |
19f6d2a6 OG |
644 | |
645 | /* | |
646 | * List of kfd_process_device structures, | |
647 | * one for each device the process is using. | |
648 | */ | |
649 | struct list_head per_device_data; | |
650 | ||
45102048 BG |
651 | struct process_queue_manager pqm; |
652 | ||
19f6d2a6 OG |
653 | /*Is the user space process 32 bit?*/ |
654 | bool is_32bit_user_mode; | |
f3a39818 AL |
655 | |
656 | /* Event-related data */ | |
657 | struct mutex event_mutex; | |
482f0777 FK |
658 | /* Event ID allocator and lookup */ |
659 | struct idr event_idr; | |
50cb7dd9 FK |
660 | /* Event page */ |
661 | struct kfd_signal_page *signal_page; | |
b9a5d0a5 | 662 | size_t signal_mapped_size; |
f3a39818 | 663 | size_t signal_event_count; |
c986169f | 664 | bool signal_event_limit_reached; |
403575c4 FK |
665 | |
666 | /* Information used for memory eviction */ | |
667 | void *kgd_process_info; | |
668 | /* Eviction fence that is attached to all the BOs of this process. The | |
669 | * fence will be triggered during eviction and new one will be created | |
670 | * during restore | |
671 | */ | |
672 | struct dma_fence *ef; | |
26103436 FK |
673 | |
674 | /* Work items for evicting and restoring BOs */ | |
675 | struct delayed_work eviction_work; | |
676 | struct delayed_work restore_work; | |
677 | /* seqno of the last scheduled eviction */ | |
678 | unsigned int last_eviction_seqno; | |
679 | /* Approx. the last timestamp (in jiffies) when the process was | |
680 | * restored after an eviction | |
681 | */ | |
682 | unsigned long last_restore_timestamp; | |
4a488a7a OG |
683 | }; |
684 | ||
64d1c3a4 FK |
685 | #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */ |
686 | extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); | |
687 | extern struct srcu_struct kfd_processes_srcu; | |
688 | ||
76baee6c OG |
689 | /** |
690 | * Ioctl function type. | |
691 | * | |
692 | * \param filep pointer to file structure. | |
693 | * \param p amdkfd process pointer. | |
694 | * \param data pointer to arg that was copied from user. | |
695 | */ | |
696 | typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p, | |
697 | void *data); | |
698 | ||
699 | struct amdkfd_ioctl_desc { | |
700 | unsigned int cmd; | |
701 | int flags; | |
702 | amdkfd_ioctl_t *func; | |
703 | unsigned int cmd_drv; | |
704 | const char *name; | |
705 | }; | |
67f7cf9f | 706 | bool kfd_dev_is_large_bar(struct kfd_dev *dev); |
76baee6c | 707 | |
1679ae8f | 708 | int kfd_process_create_wq(void); |
19f6d2a6 | 709 | void kfd_process_destroy_wq(void); |
373d7080 | 710 | struct kfd_process *kfd_create_process(struct file *filep); |
19f6d2a6 | 711 | struct kfd_process *kfd_get_process(const struct task_struct *); |
f3a39818 | 712 | struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid); |
26103436 | 713 | struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm); |
abb208a8 | 714 | void kfd_unref_process(struct kfd_process *p); |
6b95e797 FK |
715 | int kfd_process_evict_queues(struct kfd_process *p); |
716 | int kfd_process_restore_queues(struct kfd_process *p); | |
26103436 FK |
717 | void kfd_suspend_all_processes(void); |
718 | int kfd_resume_all_processes(void); | |
19f6d2a6 | 719 | |
b84394e2 FK |
720 | int kfd_process_device_init_vm(struct kfd_process_device *pdd, |
721 | struct file *drm_file); | |
64c7f8cf | 722 | struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, |
733fa1f7 | 723 | struct kfd_process *p); |
19f6d2a6 | 724 | struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, |
093c7d8c AS |
725 | struct kfd_process *p); |
726 | struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, | |
727 | struct kfd_process *p); | |
19f6d2a6 | 728 | |
df03ef93 | 729 | int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process, |
373d7080 FK |
730 | struct vm_area_struct *vma); |
731 | ||
52b29d73 FK |
732 | /* KFD process API for creating and translating handles */ |
733 | int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, | |
734 | void *mem); | |
735 | void *kfd_process_device_translate_handle(struct kfd_process_device *p, | |
736 | int handle); | |
737 | void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, | |
738 | int handle); | |
739 | ||
775921ed | 740 | /* Process device data iterator */ |
8eabaf54 KR |
741 | struct kfd_process_device *kfd_get_first_process_device_data( |
742 | struct kfd_process *p); | |
743 | struct kfd_process_device *kfd_get_next_process_device_data( | |
744 | struct kfd_process *p, | |
775921ed AS |
745 | struct kfd_process_device *pdd); |
746 | bool kfd_has_process_device_data(struct kfd_process *p); | |
747 | ||
19f6d2a6 OG |
748 | /* PASIDs */ |
749 | int kfd_pasid_init(void); | |
750 | void kfd_pasid_exit(void); | |
751 | bool kfd_set_pasid_limit(unsigned int new_limit); | |
752 | unsigned int kfd_get_pasid_limit(void); | |
753 | unsigned int kfd_pasid_alloc(void); | |
754 | void kfd_pasid_free(unsigned int pasid); | |
755 | ||
756 | /* Doorbells */ | |
ef568db7 | 757 | size_t kfd_doorbell_process_slice(struct kfd_dev *kfd); |
735df2ba FK |
758 | int kfd_doorbell_init(struct kfd_dev *kfd); |
759 | void kfd_doorbell_fini(struct kfd_dev *kfd); | |
df03ef93 HK |
760 | int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process, |
761 | struct vm_area_struct *vma); | |
ada2b29c | 762 | void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, |
19f6d2a6 OG |
763 | unsigned int *doorbell_off); |
764 | void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr); | |
765 | u32 read_kernel_doorbell(u32 __iomem *db); | |
ada2b29c | 766 | void write_kernel_doorbell(void __iomem *db, u32 value); |
9d7d0248 | 767 | void write_kernel_doorbell64(void __iomem *db, u64 value); |
ef568db7 | 768 | unsigned int kfd_doorbell_id_to_offset(struct kfd_dev *kfd, |
19f6d2a6 | 769 | struct kfd_process *process, |
ef568db7 | 770 | unsigned int doorbell_id); |
a91e70e3 FK |
771 | phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, |
772 | struct kfd_process *process); | |
773 | int kfd_alloc_process_doorbells(struct kfd_process *process); | |
774 | void kfd_free_process_doorbells(struct kfd_process *process); | |
19f6d2a6 | 775 | |
6e81090b OG |
776 | /* GTT Sub-Allocator */ |
777 | ||
778 | int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, | |
779 | struct kfd_mem_obj **mem_obj); | |
780 | ||
781 | int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj); | |
782 | ||
4a488a7a OG |
783 | extern struct device *kfd_device; |
784 | ||
5b5c4e40 EP |
785 | /* Topology */ |
786 | int kfd_topology_init(void); | |
787 | void kfd_topology_shutdown(void); | |
788 | int kfd_topology_add_device(struct kfd_dev *gpu); | |
789 | int kfd_topology_remove_device(struct kfd_dev *gpu); | |
3a87177e HK |
790 | struct kfd_topology_device *kfd_topology_device_by_proximity_domain( |
791 | uint32_t proximity_domain); | |
44d8cc6f | 792 | struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id); |
5b5c4e40 EP |
793 | struct kfd_dev *kfd_device_by_id(uint32_t gpu_id); |
794 | struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); | |
6d82eb0e | 795 | int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev); |
520b8fb7 | 796 | int kfd_numa_node_to_apic_id(int numa_node_id); |
5b5c4e40 | 797 | |
4a488a7a | 798 | /* Interrupts */ |
2249d558 AL |
799 | int kfd_interrupt_init(struct kfd_dev *dev); |
800 | void kfd_interrupt_exit(struct kfd_dev *dev); | |
b3f5e6b4 | 801 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); |
2249d558 | 802 | bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry); |
58e69886 LX |
803 | bool interrupt_is_wanted(struct kfd_dev *dev, |
804 | const uint32_t *ih_ring_entry, | |
805 | uint32_t *patched_ihre, bool *flag); | |
4a488a7a OG |
806 | |
807 | /* Power Management */ | |
b3f5e6b4 AL |
808 | void kgd2kfd_suspend(struct kfd_dev *kfd); |
809 | int kgd2kfd_resume(struct kfd_dev *kfd); | |
4a488a7a | 810 | |
e3b7a967 SL |
811 | /* GPU reset */ |
812 | int kgd2kfd_pre_reset(struct kfd_dev *kfd); | |
813 | int kgd2kfd_post_reset(struct kfd_dev *kfd); | |
814 | ||
19f6d2a6 OG |
815 | /* amdkfd Apertures */ |
816 | int kfd_init_apertures(struct kfd_process *process); | |
817 | ||
ed6e6a34 | 818 | /* Queue Context Management */ |
e88a614c | 819 | int init_queue(struct queue **q, const struct queue_properties *properties); |
ed6e6a34 | 820 | void uninit_queue(struct queue *q); |
45102048 | 821 | void print_queue_properties(struct queue_properties *q); |
ed6e6a34 BG |
822 | void print_queue(struct queue *q); |
823 | ||
64c7f8cf BG |
824 | struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, |
825 | struct kfd_dev *dev); | |
4b8f589b BG |
826 | struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, |
827 | struct kfd_dev *dev); | |
ee04955a FK |
828 | struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type, |
829 | struct kfd_dev *dev); | |
4b8f589b BG |
830 | struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type, |
831 | struct kfd_dev *dev); | |
ee04955a FK |
832 | struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type, |
833 | struct kfd_dev *dev); | |
b91d43dd FK |
834 | struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, |
835 | struct kfd_dev *dev); | |
64c7f8cf BG |
836 | struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev); |
837 | void device_queue_manager_uninit(struct device_queue_manager *dqm); | |
241f24f8 BG |
838 | struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, |
839 | enum kfd_queue_type type); | |
840 | void kernel_queue_uninit(struct kernel_queue *kq); | |
2640c3fa | 841 | int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid); |
241f24f8 | 842 | |
45102048 BG |
843 | /* Process Queue Manager */ |
844 | struct process_queue_node { | |
845 | struct queue *q; | |
846 | struct kernel_queue *kq; | |
847 | struct list_head process_queue_list; | |
848 | }; | |
849 | ||
9fd3f1bf FK |
850 | void kfd_process_dequeue_from_device(struct kfd_process_device *pdd); |
851 | void kfd_process_dequeue_from_all_devices(struct kfd_process *p); | |
45102048 BG |
852 | int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p); |
853 | void pqm_uninit(struct process_queue_manager *pqm); | |
854 | int pqm_create_queue(struct process_queue_manager *pqm, | |
855 | struct kfd_dev *dev, | |
856 | struct file *f, | |
857 | struct queue_properties *properties, | |
45102048 BG |
858 | unsigned int *qid); |
859 | int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid); | |
860 | int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, | |
861 | struct queue_properties *p); | |
39e7f331 FK |
862 | int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid, |
863 | struct queue_properties *p); | |
fbeb661b YS |
864 | struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm, |
865 | unsigned int qid); | |
5df099e8 JC |
866 | int pqm_get_wave_state(struct process_queue_manager *pqm, |
867 | unsigned int qid, | |
868 | void __user *ctl_stack, | |
869 | u32 *ctl_stack_used_size, | |
870 | u32 *save_area_used_size); | |
45102048 | 871 | |
788bf83d YS |
872 | int amdkfd_fence_wait_timeout(unsigned int *fence_addr, |
873 | unsigned int fence_value, | |
8c72c3d7 | 874 | unsigned int timeout_ms); |
788bf83d | 875 | |
ed6e6a34 BG |
876 | /* Packet Manager */ |
877 | ||
64c7f8cf BG |
878 | #define KFD_FENCE_COMPLETED (100) |
879 | #define KFD_FENCE_INIT (10) | |
241f24f8 | 880 | |
ed6e6a34 BG |
881 | struct packet_manager { |
882 | struct device_queue_manager *dqm; | |
883 | struct kernel_queue *priv_queue; | |
884 | struct mutex lock; | |
885 | bool allocated; | |
886 | struct kfd_mem_obj *ib_buffer_obj; | |
851a645e | 887 | unsigned int ib_size_bytes; |
f6e27ff1 FK |
888 | |
889 | const struct packet_manager_funcs *pmf; | |
890 | }; | |
891 | ||
892 | struct packet_manager_funcs { | |
893 | /* Support ASIC-specific packet formats for PM4 packets */ | |
894 | int (*map_process)(struct packet_manager *pm, uint32_t *buffer, | |
895 | struct qcm_process_device *qpd); | |
896 | int (*runlist)(struct packet_manager *pm, uint32_t *buffer, | |
897 | uint64_t ib, size_t ib_size_in_dwords, bool chain); | |
898 | int (*set_resources)(struct packet_manager *pm, uint32_t *buffer, | |
899 | struct scheduling_resources *res); | |
900 | int (*map_queues)(struct packet_manager *pm, uint32_t *buffer, | |
901 | struct queue *q, bool is_static); | |
902 | int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer, | |
903 | enum kfd_queue_type type, | |
904 | enum kfd_unmap_queues_filter mode, | |
905 | uint32_t filter_param, bool reset, | |
906 | unsigned int sdma_engine); | |
907 | int (*query_status)(struct packet_manager *pm, uint32_t *buffer, | |
908 | uint64_t fence_address, uint32_t fence_value); | |
909 | int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer); | |
910 | ||
911 | /* Packet sizes */ | |
912 | int map_process_size; | |
913 | int runlist_size; | |
914 | int set_resources_size; | |
915 | int map_queues_size; | |
916 | int unmap_queues_size; | |
917 | int query_status_size; | |
918 | int release_mem_size; | |
ed6e6a34 BG |
919 | }; |
920 | ||
f6e27ff1 | 921 | extern const struct packet_manager_funcs kfd_vi_pm_funcs; |
454150b1 | 922 | extern const struct packet_manager_funcs kfd_v9_pm_funcs; |
f6e27ff1 | 923 | |
64c7f8cf BG |
924 | int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm); |
925 | void pm_uninit(struct packet_manager *pm); | |
926 | int pm_send_set_resources(struct packet_manager *pm, | |
927 | struct scheduling_resources *res); | |
928 | int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues); | |
929 | int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, | |
930 | uint32_t fence_value); | |
931 | ||
932 | int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, | |
7da2bcf8 | 933 | enum kfd_unmap_queues_filter mode, |
64c7f8cf BG |
934 | uint32_t filter_param, bool reset, |
935 | unsigned int sdma_engine); | |
936 | ||
241f24f8 BG |
937 | void pm_release_ib(struct packet_manager *pm); |
938 | ||
454150b1 FK |
939 | /* Following PM funcs can be shared among VI and AI */ |
940 | unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size); | |
941 | int pm_set_resources_vi(struct packet_manager *pm, uint32_t *buffer, | |
942 | struct scheduling_resources *res); | |
943 | ||
19f6d2a6 | 944 | uint64_t kfd_get_number_elems(struct kfd_dev *kfd); |
19f6d2a6 | 945 | |
f3a39818 AL |
946 | /* Events */ |
947 | extern const struct kfd_event_interrupt_class event_interrupt_class_cik; | |
ca750681 FK |
948 | extern const struct kfd_event_interrupt_class event_interrupt_class_v9; |
949 | ||
930c5ff4 | 950 | extern const struct kfd_device_global_init_class device_global_init_class_cik; |
f3a39818 | 951 | |
f3a39818 AL |
952 | void kfd_event_init_process(struct kfd_process *p); |
953 | void kfd_event_free_process(struct kfd_process *p); | |
954 | int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma); | |
955 | int kfd_wait_on_events(struct kfd_process *p, | |
59d3e8be | 956 | uint32_t num_events, void __user *data, |
f3a39818 | 957 | bool all, uint32_t user_timeout_ms, |
fdf0c833 | 958 | uint32_t *wait_result); |
f3a39818 AL |
959 | void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id, |
960 | uint32_t valid_id_bits); | |
59d3e8be AS |
961 | void kfd_signal_iommu_event(struct kfd_dev *dev, |
962 | unsigned int pasid, unsigned long address, | |
963 | bool is_write_requested, bool is_execute_requested); | |
930c5ff4 | 964 | void kfd_signal_hw_exception_event(unsigned int pasid); |
f3a39818 AL |
965 | int kfd_set_event(struct kfd_process *p, uint32_t event_id); |
966 | int kfd_reset_event(struct kfd_process *p, uint32_t event_id); | |
0fc8011f FK |
967 | int kfd_event_page_set(struct kfd_process *p, void *kernel_address, |
968 | uint64_t size); | |
f3a39818 AL |
969 | int kfd_event_create(struct file *devkfd, struct kfd_process *p, |
970 | uint32_t event_type, bool auto_reset, uint32_t node_id, | |
971 | uint32_t *event_id, uint32_t *event_trigger_data, | |
972 | uint64_t *event_page_offset, uint32_t *event_slot_index); | |
973 | int kfd_event_destroy(struct kfd_process *p, uint32_t event_id); | |
974 | ||
2640c3fa | 975 | void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid, |
976 | struct kfd_vm_fault_info *info); | |
977 | ||
e42051d2 SL |
978 | void kfd_signal_reset_event(struct kfd_dev *dev); |
979 | ||
403575c4 FK |
980 | void kfd_flush_tlb(struct kfd_process_device *pdd); |
981 | ||
c3447e81 BG |
982 | int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p); |
983 | ||
e42051d2 SL |
984 | bool kfd_is_locked(void); |
985 | ||
851a645e FK |
986 | /* Debugfs */ |
987 | #if defined(CONFIG_DEBUG_FS) | |
988 | ||
989 | void kfd_debugfs_init(void); | |
990 | void kfd_debugfs_fini(void); | |
991 | int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data); | |
992 | int pqm_debugfs_mqds(struct seq_file *m, void *data); | |
993 | int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data); | |
994 | int dqm_debugfs_hqds(struct seq_file *m, void *data); | |
995 | int kfd_debugfs_rls_by_device(struct seq_file *m, void *data); | |
996 | int pm_debugfs_runlist(struct seq_file *m, void *data); | |
997 | ||
a29ec470 SL |
998 | int kfd_debugfs_hang_hws(struct kfd_dev *dev); |
999 | int pm_debugfs_hang_hws(struct packet_manager *pm); | |
1000 | int dqm_debugfs_execute_queues(struct device_queue_manager *dqm); | |
1001 | ||
851a645e FK |
1002 | #else |
1003 | ||
1004 | static inline void kfd_debugfs_init(void) {} | |
1005 | static inline void kfd_debugfs_fini(void) {} | |
1006 | ||
1007 | #endif | |
1008 | ||
4a488a7a | 1009 | #endif |