Commit | Line | Data |
---|---|---|
4a488a7a OG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #ifndef KFD_PRIV_H_INCLUDED | |
24 | #define KFD_PRIV_H_INCLUDED | |
25 | ||
26 | #include <linux/hashtable.h> | |
27 | #include <linux/mmu_notifier.h> | |
28 | #include <linux/mutex.h> | |
29 | #include <linux/types.h> | |
30 | #include <linux/atomic.h> | |
31 | #include <linux/workqueue.h> | |
32 | #include <linux/spinlock.h> | |
19f6d2a6 | 33 | #include <linux/kfd_ioctl.h> |
482f0777 | 34 | #include <linux/idr.h> |
04ad47bd | 35 | #include <linux/kfifo.h> |
851a645e | 36 | #include <linux/seq_file.h> |
5ce10687 | 37 | #include <linux/kref.h> |
de9f26bb | 38 | #include <linux/sysfs.h> |
6b855f7b | 39 | #include <linux/device_cgroup.h> |
1cd4d9ee SR |
40 | #include <drm/drm_file.h> |
41 | #include <drm/drm_drv.h> | |
42 | #include <drm/drm_device.h> | |
99c7b309 | 43 | #include <drm/drm_ioctl.h> |
4a488a7a | 44 | #include <kgd_kfd_interface.h> |
6d220a7e | 45 | #include <linux/swap.h> |
4a488a7a | 46 | |
e596b903 | 47 | #include "amd_shared.h" |
6ae27841 | 48 | #include "amdgpu.h" |
e596b903 | 49 | |
af47b390 LA |
50 | #define KFD_MAX_RING_ENTRY_SIZE 8 |
51 | ||
5b5c4e40 EP |
52 | #define KFD_SYSFS_FILE_MODE 0444 |
53 | ||
df03ef93 HK |
54 | /* GPU ID hash width in bits */ |
55 | #define KFD_GPU_ID_HASH_WIDTH 16 | |
56 | ||
57 | /* Use upper bits of mmap offset to store KFD driver specific information. | |
58 | * BITS[63:62] - Encode MMAP type | |
59 | * BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to | |
60 | * BITS[45:0] - MMAP offset value | |
61 | * | |
62 | * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these | |
63 | * defines are w.r.t to PAGE_SIZE | |
64 | */ | |
29453755 | 65 | #define KFD_MMAP_TYPE_SHIFT 62 |
df03ef93 HK |
66 | #define KFD_MMAP_TYPE_MASK (0x3ULL << KFD_MMAP_TYPE_SHIFT) |
67 | #define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT) | |
68 | #define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT) | |
69 | #define KFD_MMAP_TYPE_RESERVED_MEM (0x1ULL << KFD_MMAP_TYPE_SHIFT) | |
d33ea570 | 70 | #define KFD_MMAP_TYPE_MMIO (0x0ULL << KFD_MMAP_TYPE_SHIFT) |
df03ef93 | 71 | |
29453755 | 72 | #define KFD_MMAP_GPU_ID_SHIFT 46 |
df03ef93 HK |
73 | #define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \ |
74 | << KFD_MMAP_GPU_ID_SHIFT) | |
75 | #define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\ | |
76 | & KFD_MMAP_GPU_ID_MASK) | |
29453755 | 77 | #define KFD_MMAP_GET_GPU_ID(offset) ((offset & KFD_MMAP_GPU_ID_MASK) \ |
df03ef93 HK |
78 | >> KFD_MMAP_GPU_ID_SHIFT) |
79 | ||
ed6e6a34 BG |
80 | /* |
81 | * When working with cp scheduler we should assign the HIQ manually or via | |
e7016d8e | 82 | * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot |
ed6e6a34 BG |
83 | * definitions for Kaveri. In Kaveri only the first ME queues participates |
84 | * in the cp scheduling taking that in mind we set the HIQ slot in the | |
85 | * second ME. | |
86 | */ | |
87 | #define KFD_CIK_HIQ_PIPE 4 | |
88 | #define KFD_CIK_HIQ_QUEUE 0 | |
89 | ||
5b5c4e40 EP |
90 | /* Macro for allocating structures */ |
91 | #define kfd_alloc_struct(ptr_to_struct) \ | |
92 | ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL)) | |
93 | ||
19f6d2a6 | 94 | #define KFD_MAX_NUM_OF_PROCESSES 512 |
b8cbab04 | 95 | #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 |
19f6d2a6 | 96 | |
373d7080 FK |
97 | /* |
98 | * Size of the per-process TBA+TMA buffer: 2 pages | |
99 | * | |
100 | * The first page is the TBA used for the CWSR ISA code. The second | |
a4497974 | 101 | * page is used as TMA for user-mode trap handler setup in daisy-chain mode. |
373d7080 FK |
102 | */ |
103 | #define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2) | |
104 | #define KFD_CWSR_TMA_OFFSET PAGE_SIZE | |
105 | ||
74523943 YZ |
106 | #define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \ |
107 | (KFD_MAX_NUM_OF_PROCESSES * \ | |
108 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) | |
109 | ||
110 | #define KFD_KERNEL_QUEUE_SIZE 2048 | |
111 | ||
14328aa5 PC |
112 | #define KFD_UNMAP_LATENCY_MS (4000) |
113 | ||
1f86805a YZ |
114 | /* |
115 | * 512 = 0x200 | |
116 | * The doorbell index distance between SDMA RLC (2*i) and (2*i+1) in the | |
117 | * same SDMA engine on SOC15, which has 8-byte doorbells for SDMA. | |
118 | * 512 8-byte doorbell distance (i.e. one page away) ensures that SDMA RLC | |
119 | * (2*i+1) doorbells (in terms of the lower 12 bit address) lie exactly in | |
120 | * the OFFSET and SIZE set in registers like BIF_SDMA0_DOORBELL_RANGE. | |
121 | */ | |
122 | #define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512 | |
123 | ||
124 | ||
19f6d2a6 | 125 | /* |
b8cbab04 OG |
126 | * Kernel module parameter to specify maximum number of supported queues per |
127 | * device | |
19f6d2a6 | 128 | */ |
b8cbab04 | 129 | extern int max_num_of_queues_per_device; |
19f6d2a6 | 130 | |
ed6e6a34 | 131 | |
31c21fec BG |
132 | /* Kernel module parameter to specify the scheduling policy */ |
133 | extern int sched_policy; | |
134 | ||
a99c6d4f FK |
135 | /* |
136 | * Kernel module parameter to specify the maximum process | |
137 | * number per HW scheduler | |
138 | */ | |
139 | extern int hws_max_conc_proc; | |
140 | ||
373d7080 FK |
141 | extern int cwsr_enable; |
142 | ||
81663016 OG |
143 | /* |
144 | * Kernel module parameter to specify whether to send sigterm to HSA process on | |
145 | * unhandled exception | |
146 | */ | |
147 | extern int send_sigterm; | |
148 | ||
374200b1 FK |
149 | /* |
150 | * This kernel module is used to simulate large bar machine on non-large bar | |
151 | * enabled machines. | |
152 | */ | |
153 | extern int debug_largebar; | |
154 | ||
ebcfd1e2 FK |
155 | /* |
156 | * Ignore CRAT table during KFD initialization, can be used to work around | |
157 | * broken CRAT tables on some AMD systems | |
158 | */ | |
159 | extern int ignore_crat; | |
160 | ||
a4497974 | 161 | /* Set sh_mem_config.retry_disable on GFX v9 */ |
75ee6487 | 162 | extern int amdgpu_noretry; |
bed4f110 | 163 | |
a4497974 | 164 | /* Halt if HWS hang is detected */ |
0e9a860c YZ |
165 | extern int halt_if_hws_hang; |
166 | ||
a4497974 | 167 | /* Whether MEC FW support GWS barriers */ |
29e76462 OZ |
168 | extern bool hws_gws_support; |
169 | ||
a4497974 | 170 | /* Queue preemption timeout in ms */ |
14328aa5 PC |
171 | extern int queue_preemption_timeout_ms; |
172 | ||
6d909c5d OZ |
173 | /* |
174 | * Don't evict process queues on vm fault | |
175 | */ | |
176 | extern int amdgpu_no_queue_eviction_on_vm_fault; | |
177 | ||
a4497974 | 178 | /* Enable eviction debug messages */ |
b2057956 FK |
179 | extern bool debug_evictions; |
180 | ||
ed6e6a34 BG |
181 | enum cache_policy { |
182 | cache_policy_coherent, | |
183 | cache_policy_noncoherent | |
184 | }; | |
185 | ||
dd0ae064 GS |
186 | #define KFD_GC_VERSION(dev) ((dev)->adev->ip_versions[GC_HWIP][0]) |
187 | #define KFD_IS_SOC15(dev) ((KFD_GC_VERSION(dev)) >= (IP_VERSION(9, 0, 1))) | |
ef568db7 | 188 | |
f3a39818 AL |
189 | struct kfd_event_interrupt_class { |
190 | bool (*interrupt_isr)(struct kfd_dev *dev, | |
58e69886 LX |
191 | const uint32_t *ih_ring_entry, uint32_t *patched_ihre, |
192 | bool *patched_flag); | |
f3a39818 | 193 | void (*interrupt_wq)(struct kfd_dev *dev, |
58e69886 | 194 | const uint32_t *ih_ring_entry); |
f3a39818 AL |
195 | }; |
196 | ||
4a488a7a | 197 | struct kfd_device_info { |
c181159a | 198 | const char *asic_name; |
9d6fa9c7 | 199 | uint32_t gfx_target_version; |
f3a39818 | 200 | const struct kfd_event_interrupt_class *event_interrupt_class; |
4a488a7a | 201 | unsigned int max_pasid_bits; |
992839ad | 202 | unsigned int max_no_of_hqd; |
ada2b29c | 203 | unsigned int doorbell_size; |
4a488a7a | 204 | size_t ih_ring_entry_size; |
f7c826ad | 205 | uint8_t num_of_watch_points; |
19f6d2a6 | 206 | uint16_t mqd_size_aligned; |
373d7080 | 207 | bool supports_cwsr; |
64d1c3a4 | 208 | bool needs_iommu_device; |
3ee2d00c | 209 | bool needs_pci_atomics; |
fb932dfe | 210 | uint32_t no_atomic_fw_version; |
98bb9222 | 211 | unsigned int num_sdma_engines; |
1b4670f6 | 212 | unsigned int num_xgmi_sdma_engines; |
d5094189 | 213 | unsigned int num_sdma_queues_per_engine; |
4a488a7a OG |
214 | }; |
215 | ||
36b5c08f OG |
216 | struct kfd_mem_obj { |
217 | uint32_t range_start; | |
218 | uint32_t range_end; | |
219 | uint64_t gpu_addr; | |
220 | uint32_t *cpu_ptr; | |
b91d43dd | 221 | void *gtt_mem; |
36b5c08f OG |
222 | }; |
223 | ||
44008d7a YZ |
224 | struct kfd_vmid_info { |
225 | uint32_t first_vmid_kfd; | |
226 | uint32_t last_vmid_kfd; | |
227 | uint32_t vmid_num_kfd; | |
228 | }; | |
229 | ||
4a488a7a | 230 | struct kfd_dev { |
c6c57446 | 231 | struct amdgpu_device *adev; |
4a488a7a OG |
232 | |
233 | const struct kfd_device_info *device_info; | |
234 | struct pci_dev *pdev; | |
3a0c3423 | 235 | struct drm_device *ddev; |
4a488a7a OG |
236 | |
237 | unsigned int id; /* topology stub index */ | |
238 | ||
19f6d2a6 OG |
239 | phys_addr_t doorbell_base; /* Start of actual doorbells used by |
240 | * KFD. It is aligned for mapping | |
241 | * into user mode | |
242 | */ | |
339903fa YZ |
243 | size_t doorbell_base_dw_offset; /* Offset from the start of the PCI |
244 | * doorbell BAR to the first KFD | |
245 | * doorbell in dwords. GFX reserves | |
246 | * the segment before this offset. | |
19f6d2a6 | 247 | */ |
19f6d2a6 OG |
248 | u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells |
249 | * page used by kernel queue | |
250 | */ | |
251 | ||
4a488a7a | 252 | struct kgd2kfd_shared_resources shared_resources; |
44008d7a | 253 | struct kfd_vmid_info vm_info; |
4a488a7a | 254 | |
cea405b1 XZ |
255 | const struct kfd2kgd_calls *kfd2kgd; |
256 | struct mutex doorbell_mutex; | |
f761d8bd JP |
257 | DECLARE_BITMAP(doorbell_available_index, |
258 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); | |
cea405b1 | 259 | |
36b5c08f OG |
260 | void *gtt_mem; |
261 | uint64_t gtt_start_gpu_addr; | |
262 | void *gtt_start_cpu_ptr; | |
263 | void *gtt_sa_bitmap; | |
264 | struct mutex gtt_sa_lock; | |
265 | unsigned int gtt_sa_chunk_size; | |
266 | unsigned int gtt_sa_num_of_chunks; | |
267 | ||
2249d558 | 268 | /* Interrupts */ |
04ad47bd | 269 | struct kfifo ih_fifo; |
48e876a2 | 270 | struct workqueue_struct *ih_wq; |
2249d558 AL |
271 | struct work_struct interrupt_work; |
272 | spinlock_t interrupt_lock; | |
273 | ||
ed6e6a34 BG |
274 | /* QCM Device instance */ |
275 | struct device_queue_manager *dqm; | |
4a488a7a | 276 | |
ed6e6a34 | 277 | bool init_complete; |
2249d558 AL |
278 | /* |
279 | * Interrupts of interest to KFD are copied | |
280 | * from the HW ring into a SW ring. | |
281 | */ | |
282 | bool interrupts_active; | |
fbeb661b YS |
283 | |
284 | /* Debug manager */ | |
0d87c9cf | 285 | struct kfd_dbgmgr *dbgmgr; |
373d7080 | 286 | |
5ade6c9c FK |
287 | /* Firmware versions */ |
288 | uint16_t mec_fw_version; | |
29633d0e | 289 | uint16_t mec2_fw_version; |
5ade6c9c FK |
290 | uint16_t sdma_fw_version; |
291 | ||
a99c6d4f FK |
292 | /* Maximum process number mapped to HW scheduler */ |
293 | unsigned int max_proc_per_quantum; | |
294 | ||
373d7080 FK |
295 | /* CWSR */ |
296 | bool cwsr_enabled; | |
297 | const void *cwsr_isa; | |
298 | unsigned int cwsr_isa_size; | |
0c1690e3 SL |
299 | |
300 | /* xGMI */ | |
301 | uint64_t hive_id; | |
a4497974 | 302 | |
d35f00d8 | 303 | bool pci_atomic_requested; |
9b54d201 | 304 | |
6127896f HR |
305 | /* Use IOMMU v2 flag */ |
306 | bool use_iommu_v2; | |
307 | ||
9b54d201 EH |
308 | /* SRAM ECC flag */ |
309 | atomic_t sram_ecc_flag; | |
f756e631 HK |
310 | |
311 | /* Compute Profile ref. count */ | |
312 | atomic_t compute_profile; | |
e09d4fc8 | 313 | |
a4497974 | 314 | /* Global GWS resource shared between processes */ |
e09d4fc8 | 315 | void *gws; |
938a0650 AL |
316 | |
317 | /* Clients watching SMI events */ | |
318 | struct list_head smi_clients; | |
319 | spinlock_t smi_lock; | |
55977744 MJ |
320 | |
321 | uint32_t reset_seq_num; | |
59d7115d MJ |
322 | |
323 | struct ida doorbell_ida; | |
324 | unsigned int max_doorbell_slices; | |
9b498efa AD |
325 | |
326 | int noretry; | |
814ab993 PY |
327 | |
328 | /* HMM page migration MEMORY_DEVICE_PRIVATE mapping */ | |
329 | struct dev_pagemap pgmap; | |
4a488a7a OG |
330 | }; |
331 | ||
19f6d2a6 OG |
332 | enum kfd_mempool { |
333 | KFD_MEMPOOL_SYSTEM_CACHEABLE = 1, | |
334 | KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2, | |
335 | KFD_MEMPOOL_FRAMEBUFFER = 3, | |
336 | }; | |
337 | ||
4a488a7a OG |
338 | /* Character device interface */ |
339 | int kfd_chardev_init(void); | |
340 | void kfd_chardev_exit(void); | |
341 | struct device *kfd_chardev(void); | |
342 | ||
241f24f8 | 343 | /** |
a4497974 | 344 | * enum kfd_unmap_queues_filter - Enum for queue filters. |
241f24f8 | 345 | * |
7da2bcf8 | 346 | * @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue. |
241f24f8 | 347 | * |
7da2bcf8 | 348 | * @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the |
241f24f8 BG |
349 | * running queues list. |
350 | * | |
7da2bcf8 | 351 | * @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to |
241f24f8 BG |
352 | * specific process. |
353 | * | |
354 | */ | |
7da2bcf8 YZ |
355 | enum kfd_unmap_queues_filter { |
356 | KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE, | |
357 | KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, | |
358 | KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, | |
359 | KFD_UNMAP_QUEUES_FILTER_BY_PASID | |
241f24f8 | 360 | }; |
19f6d2a6 | 361 | |
ed8aab45 | 362 | /** |
a4497974 | 363 | * enum kfd_queue_type - Enum for various queue types. |
ed8aab45 BG |
364 | * |
365 | * @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type. | |
366 | * | |
a4497974 | 367 | * @KFD_QUEUE_TYPE_SDMA: SDMA user mode queue type. |
ed8aab45 BG |
368 | * |
369 | * @KFD_QUEUE_TYPE_HIQ: HIQ queue type. | |
370 | * | |
371 | * @KFD_QUEUE_TYPE_DIQ: DIQ queue type. | |
a4497974 RB |
372 | * |
373 | * @KFD_QUEUE_TYPE_SDMA_XGMI: Special SDMA queue for XGMI interface. | |
ed8aab45 BG |
374 | */ |
375 | enum kfd_queue_type { | |
376 | KFD_QUEUE_TYPE_COMPUTE, | |
377 | KFD_QUEUE_TYPE_SDMA, | |
378 | KFD_QUEUE_TYPE_HIQ, | |
1b4670f6 OZ |
379 | KFD_QUEUE_TYPE_DIQ, |
380 | KFD_QUEUE_TYPE_SDMA_XGMI | |
ed8aab45 BG |
381 | }; |
382 | ||
6e99df57 BG |
383 | enum kfd_queue_format { |
384 | KFD_QUEUE_FORMAT_PM4, | |
385 | KFD_QUEUE_FORMAT_AQL | |
386 | }; | |
387 | ||
0ccbc7cd OZ |
388 | enum KFD_QUEUE_PRIORITY { |
389 | KFD_QUEUE_PRIORITY_MINIMUM = 0, | |
390 | KFD_QUEUE_PRIORITY_MAXIMUM = 15 | |
391 | }; | |
392 | ||
ed8aab45 BG |
393 | /** |
394 | * struct queue_properties | |
395 | * | |
396 | * @type: The queue type. | |
397 | * | |
398 | * @queue_id: Queue identifier. | |
399 | * | |
400 | * @queue_address: Queue ring buffer address. | |
401 | * | |
402 | * @queue_size: Queue ring buffer size. | |
403 | * | |
404 | * @priority: Defines the queue priority relative to other queues in the | |
405 | * process. | |
406 | * This is just an indication and HW scheduling may override the priority as | |
407 | * necessary while keeping the relative prioritization. | |
408 | * the priority granularity is from 0 to f which f is the highest priority. | |
409 | * currently all queues are initialized with the highest priority. | |
410 | * | |
411 | * @queue_percent: This field is partially implemented and currently a zero in | |
412 | * this field defines that the queue is non active. | |
413 | * | |
414 | * @read_ptr: User space address which points to the number of dwords the | |
415 | * cp read from the ring buffer. This field updates automatically by the H/W. | |
416 | * | |
417 | * @write_ptr: Defines the number of dwords written to the ring buffer. | |
418 | * | |
a4497974 RB |
419 | * @doorbell_ptr: Notifies the H/W of new packet written to the queue ring |
420 | * buffer. This field should be similar to write_ptr and the user should | |
421 | * update this field after updating the write_ptr. | |
ed8aab45 BG |
422 | * |
423 | * @doorbell_off: The doorbell offset in the doorbell pci-bar. | |
424 | * | |
8eabaf54 KR |
425 | * @is_interop: Defines if this is a interop queue. Interop queue means that |
426 | * the queue can access both graphics and compute resources. | |
ed8aab45 | 427 | * |
26103436 FK |
428 | * @is_evicted: Defines if the queue is evicted. Only active queues |
429 | * are evicted, rendering them inactive. | |
430 | * | |
431 | * @is_active: Defines if the queue is active or not. @is_active and | |
432 | * @is_evicted are protected by the DQM lock. | |
ed8aab45 | 433 | * |
b8020b03 JG |
434 | * @is_gws: Defines if the queue has been updated to be GWS-capable or not. |
435 | * @is_gws should be protected by the DQM lock, since changing it can yield the | |
436 | * possibility of updating DQM state on number of GWS queues. | |
437 | * | |
ed8aab45 BG |
438 | * @vmid: If the scheduling mode is no cp scheduling the field defines the vmid |
439 | * of the queue. | |
440 | * | |
441 | * This structure represents the queue properties for each queue no matter if | |
442 | * it's user mode or kernel mode queue. | |
443 | * | |
444 | */ | |
445 | struct queue_properties { | |
446 | enum kfd_queue_type type; | |
6e99df57 | 447 | enum kfd_queue_format format; |
ed8aab45 BG |
448 | unsigned int queue_id; |
449 | uint64_t queue_address; | |
450 | uint64_t queue_size; | |
451 | uint32_t priority; | |
452 | uint32_t queue_percent; | |
453 | uint32_t *read_ptr; | |
454 | uint32_t *write_ptr; | |
ada2b29c | 455 | void __iomem *doorbell_ptr; |
ed8aab45 BG |
456 | uint32_t doorbell_off; |
457 | bool is_interop; | |
26103436 | 458 | bool is_evicted; |
ed8aab45 | 459 | bool is_active; |
b8020b03 | 460 | bool is_gws; |
ed8aab45 BG |
461 | /* Not relevant for user mode queues in cp scheduling */ |
462 | unsigned int vmid; | |
77669eb8 BG |
463 | /* Relevant only for sdma queues*/ |
464 | uint32_t sdma_engine_id; | |
465 | uint32_t sdma_queue_id; | |
466 | uint32_t sdma_vm_addr; | |
ff3d04a1 BG |
467 | /* Relevant only for VI */ |
468 | uint64_t eop_ring_buffer_address; | |
469 | uint32_t eop_ring_buffer_size; | |
470 | uint64_t ctx_save_restore_area_address; | |
471 | uint32_t ctx_save_restore_area_size; | |
373d7080 FK |
472 | uint32_t ctl_stack_size; |
473 | uint64_t tba_addr; | |
474 | uint64_t tma_addr; | |
ed8aab45 BG |
475 | }; |
476 | ||
bb2d2128 FK |
477 | #define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \ |
478 | (q).queue_address != 0 && \ | |
479 | (q).queue_percent > 0 && \ | |
480 | !(q).is_evicted) | |
481 | ||
7c695a2c LY |
482 | enum mqd_update_flag { |
483 | UPDATE_FLAG_CU_MASK = 0, | |
484 | }; | |
485 | ||
486 | struct mqd_update_info { | |
487 | union { | |
488 | struct { | |
489 | uint32_t count; /* Must be a multiple of 32 */ | |
490 | uint32_t *ptr; | |
491 | } cu_mask; | |
492 | }; | |
493 | enum mqd_update_flag update_flag; | |
494 | }; | |
c6e559eb | 495 | |
ed8aab45 BG |
496 | /** |
497 | * struct queue | |
498 | * | |
499 | * @list: Queue linked list. | |
500 | * | |
a4497974 | 501 | * @mqd: The queue MQD (memory queue descriptor). |
ed8aab45 BG |
502 | * |
503 | * @mqd_mem_obj: The MQD local gpu memory object. | |
504 | * | |
505 | * @gart_mqd_addr: The MQD gart mc address. | |
506 | * | |
507 | * @properties: The queue properties. | |
508 | * | |
509 | * @mec: Used only in no cp scheduling mode and identifies to micro engine id | |
a4497974 | 510 | * that the queue should be executed on. |
ed8aab45 | 511 | * |
8eabaf54 KR |
512 | * @pipe: Used only in no cp scheduling mode and identifies the queue's pipe |
513 | * id. | |
ed8aab45 BG |
514 | * |
515 | * @queue: Used only in no cp scheduliong mode and identifies the queue's slot. | |
516 | * | |
517 | * @process: The kfd process that created this queue. | |
518 | * | |
519 | * @device: The kfd device that created this queue. | |
520 | * | |
eb82da1d OZ |
521 | * @gws: Pointing to gws kgd_mem if this is a gws control queue; NULL |
522 | * otherwise. | |
523 | * | |
ed8aab45 BG |
524 | * This structure represents user mode compute queues. |
525 | * It contains all the necessary data to handle such queues. | |
526 | * | |
527 | */ | |
528 | ||
529 | struct queue { | |
530 | struct list_head list; | |
531 | void *mqd; | |
532 | struct kfd_mem_obj *mqd_mem_obj; | |
533 | uint64_t gart_mqd_addr; | |
534 | struct queue_properties properties; | |
535 | ||
536 | uint32_t mec; | |
537 | uint32_t pipe; | |
538 | uint32_t queue; | |
539 | ||
77669eb8 | 540 | unsigned int sdma_id; |
ef568db7 | 541 | unsigned int doorbell_id; |
77669eb8 | 542 | |
ed8aab45 BG |
543 | struct kfd_process *process; |
544 | struct kfd_dev *device; | |
eb82da1d | 545 | void *gws; |
6d220a7e AL |
546 | |
547 | /* procfs */ | |
548 | struct kobject kobj; | |
ed8aab45 BG |
549 | }; |
550 | ||
6e99df57 | 551 | enum KFD_MQD_TYPE { |
d7c0b047 | 552 | KFD_MQD_TYPE_HIQ = 0, /* for hiq */ |
85d258f9 BG |
553 | KFD_MQD_TYPE_CP, /* for cp queues and diq */ |
554 | KFD_MQD_TYPE_SDMA, /* for sdma queues */ | |
59f650a0 | 555 | KFD_MQD_TYPE_DIQ, /* for diq */ |
6e99df57 BG |
556 | KFD_MQD_TYPE_MAX |
557 | }; | |
558 | ||
0ccbc7cd OZ |
559 | enum KFD_PIPE_PRIORITY { |
560 | KFD_PIPE_PRIORITY_CS_LOW = 0, | |
561 | KFD_PIPE_PRIORITY_CS_MEDIUM, | |
562 | KFD_PIPE_PRIORITY_CS_HIGH | |
563 | }; | |
564 | ||
241f24f8 BG |
565 | struct scheduling_resources { |
566 | unsigned int vmid_mask; | |
567 | enum kfd_queue_type type; | |
568 | uint64_t queue_mask; | |
569 | uint64_t gws_mask; | |
570 | uint32_t oac_mask; | |
571 | uint32_t gds_heap_base; | |
572 | uint32_t gds_heap_size; | |
573 | }; | |
574 | ||
575 | struct process_queue_manager { | |
576 | /* data */ | |
577 | struct kfd_process *process; | |
241f24f8 BG |
578 | struct list_head queues; |
579 | unsigned long *queue_slot_bitmap; | |
580 | }; | |
581 | ||
582 | struct qcm_process_device { | |
583 | /* The Device Queue Manager that owns this data */ | |
584 | struct device_queue_manager *dqm; | |
585 | struct process_queue_manager *pqm; | |
241f24f8 BG |
586 | /* Queues list */ |
587 | struct list_head queues_list; | |
588 | struct list_head priv_queue_list; | |
589 | ||
590 | unsigned int queue_count; | |
591 | unsigned int vmid; | |
592 | bool is_debug; | |
26103436 | 593 | unsigned int evicted; /* eviction counter, 0=active */ |
9fd3f1bf FK |
594 | |
595 | /* This flag tells if we should reset all wavefronts on | |
596 | * process termination | |
597 | */ | |
598 | bool reset_wavefronts; | |
599 | ||
b8020b03 JG |
600 | /* This flag tells us if this process has a GWS-capable |
601 | * queue that will be mapped into the runlist. It's | |
602 | * possible to request a GWS BO, but not have the queue | |
603 | * currently mapped, and this changes how the MAP_PROCESS | |
604 | * PM4 packet is configured. | |
605 | */ | |
606 | bool mapped_gws_queue; | |
607 | ||
a4497974 | 608 | /* All the memory management data should be here too */ |
241f24f8 | 609 | uint64_t gds_context_area; |
435e2f97 | 610 | /* Contains page table flags such as AMDGPU_PTE_VALID since gfx9 */ |
e715c6d0 | 611 | uint64_t page_table_base; |
241f24f8 BG |
612 | uint32_t sh_mem_config; |
613 | uint32_t sh_mem_bases; | |
614 | uint32_t sh_mem_ape1_base; | |
615 | uint32_t sh_mem_ape1_limit; | |
241f24f8 BG |
616 | uint32_t gds_size; |
617 | uint32_t num_gws; | |
618 | uint32_t num_oac; | |
6a1c9510 | 619 | uint32_t sh_hidden_private_base; |
373d7080 FK |
620 | |
621 | /* CWSR memory */ | |
68df0f19 | 622 | struct kgd_mem *cwsr_mem; |
373d7080 | 623 | void *cwsr_kaddr; |
d01994c2 | 624 | uint64_t cwsr_base; |
373d7080 FK |
625 | uint64_t tba_addr; |
626 | uint64_t tma_addr; | |
d01994c2 FK |
627 | |
628 | /* IB memory */ | |
68df0f19 | 629 | struct kgd_mem *ib_mem; |
d01994c2 | 630 | uint64_t ib_base; |
552764b6 | 631 | void *ib_kaddr; |
ef568db7 FK |
632 | |
633 | /* doorbell resources per process per device */ | |
634 | unsigned long *doorbell_bitmap; | |
241f24f8 BG |
635 | }; |
636 | ||
26103436 FK |
637 | /* KFD Memory Eviction */ |
638 | ||
639 | /* Approx. wait time before attempting to restore evicted BOs */ | |
640 | #define PROCESS_RESTORE_TIME_MS 100 | |
641 | /* Approx. back off time if restore fails due to lack of memory */ | |
642 | #define PROCESS_BACK_OFF_TIME_MS 100 | |
643 | /* Approx. time before evicting the process again */ | |
644 | #define PROCESS_ACTIVE_TIME_MS 10 | |
645 | ||
5ec7e028 FK |
646 | /* 8 byte handle containing GPU ID in the most significant 4 bytes and |
647 | * idr_handle in the least significant 4 bytes | |
648 | */ | |
649 | #define MAKE_HANDLE(gpu_id, idr_handle) \ | |
650 | (((uint64_t)(gpu_id) << 32) + idr_handle) | |
651 | #define GET_GPU_ID(handle) (handle >> 32) | |
652 | #define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF) | |
653 | ||
733fa1f7 YZ |
654 | enum kfd_pdd_bound { |
655 | PDD_UNBOUND = 0, | |
656 | PDD_BOUND, | |
657 | PDD_BOUND_SUSPENDED, | |
658 | }; | |
659 | ||
4327bed2 | 660 | #define MAX_SYSFS_FILENAME_LEN 15 |
32cb59f3 MJ |
661 | |
662 | /* | |
663 | * SDMA counter runs at 100MHz frequency. | |
664 | * We display SDMA activity in microsecond granularity in sysfs. | |
665 | * As a result, the divisor is 100. | |
666 | */ | |
667 | #define SDMA_ACTIVITY_DIVISOR 100 | |
d4566dee | 668 | |
19f6d2a6 OG |
669 | /* Data that is per-process-per device. */ |
670 | struct kfd_process_device { | |
19f6d2a6 OG |
671 | /* The device that owns this data. */ |
672 | struct kfd_dev *dev; | |
673 | ||
9fd3f1bf FK |
674 | /* The process that owns this kfd_process_device. */ |
675 | struct kfd_process *process; | |
19f6d2a6 | 676 | |
45102048 BG |
677 | /* per-process-per device QCM data structure */ |
678 | struct qcm_process_device qpd; | |
679 | ||
19f6d2a6 OG |
680 | /*Apertures*/ |
681 | uint64_t lds_base; | |
682 | uint64_t lds_limit; | |
683 | uint64_t gpuvm_base; | |
684 | uint64_t gpuvm_limit; | |
685 | uint64_t scratch_base; | |
686 | uint64_t scratch_limit; | |
687 | ||
403575c4 | 688 | /* VM context for GPUVM allocations */ |
b84394e2 | 689 | struct file *drm_file; |
b40a6ab2 | 690 | void *drm_priv; |
403575c4 | 691 | |
52b29d73 FK |
692 | /* GPUVM allocations storage */ |
693 | struct idr alloc_idr; | |
694 | ||
9fd3f1bf FK |
695 | /* Flag used to tell the pdd has dequeued from the dqm. |
696 | * This is used to prevent dev->dqm->ops.process_termination() from | |
697 | * being called twice when it is already called in IOMMU callback | |
698 | * function. | |
a82918f1 | 699 | */ |
9fd3f1bf | 700 | bool already_dequeued; |
9593f4d6 | 701 | bool runtime_inuse; |
64d1c3a4 FK |
702 | |
703 | /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */ | |
704 | enum kfd_pdd_bound bound; | |
d4566dee MJ |
705 | |
706 | /* VRAM usage */ | |
707 | uint64_t vram_usage; | |
708 | struct attribute attr_vram; | |
32cb59f3 MJ |
709 | char vram_filename[MAX_SYSFS_FILENAME_LEN]; |
710 | ||
711 | /* SDMA activity tracking */ | |
712 | uint64_t sdma_past_activity_counter; | |
713 | struct attribute attr_sdma; | |
714 | char sdma_filename[MAX_SYSFS_FILENAME_LEN]; | |
4327bed2 PC |
715 | |
716 | /* Eviction activity tracking */ | |
717 | uint64_t last_evict_timestamp; | |
718 | atomic64_t evict_duration_counter; | |
719 | struct attribute attr_evict; | |
720 | ||
721 | struct kobject *kobj_stats; | |
59d7115d | 722 | unsigned int doorbell_index; |
f2fa07b3 RE |
723 | |
724 | /* | |
725 | * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process | |
726 | * that is associated with device encoded by "this" struct instance. The | |
727 | * value reflects CU usage by all of the waves launched by this process | |
728 | * on this device. A very important property of occupancy parameter is | |
729 | * that its value is a snapshot of current use. | |
730 | * | |
731 | * Following is to be noted regarding how this parameter is reported: | |
732 | * | |
733 | * The number of waves that a CU can launch is limited by couple of | |
734 | * parameters. These are encoded by struct amdgpu_cu_info instance | |
735 | * that is part of every device definition. For GFX9 devices this | |
736 | * translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves | |
737 | * do not use scratch memory and 32 waves (max_scratch_slots_per_cu) | |
738 | * when they do use scratch memory. This could change for future | |
739 | * devices and therefore this example should be considered as a guide. | |
740 | * | |
741 | * All CU's of a device are available for the process. This may not be true | |
742 | * under certain conditions - e.g. CU masking. | |
743 | * | |
744 | * Finally number of CU's that are occupied by a process is affected by both | |
745 | * number of CU's a device has along with number of other competing processes | |
746 | */ | |
747 | struct attribute attr_cu_occupancy; | |
751580b3 PY |
748 | |
749 | /* sysfs counters for GPU retry fault and page migration tracking */ | |
750 | struct kobject *kobj_counters; | |
751 | struct attribute attr_faults; | |
752 | struct attribute attr_page_in; | |
753 | struct attribute attr_page_out; | |
754 | uint64_t faults; | |
755 | uint64_t page_in; | |
756 | uint64_t page_out; | |
19f6d2a6 OG |
757 | }; |
758 | ||
52a5fdce AS |
759 | #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd) |
760 | ||
42de677f PY |
761 | struct svm_range_list { |
762 | struct mutex lock; | |
763 | struct rb_root_cached objects; | |
764 | struct list_head list; | |
4683cfec PY |
765 | struct work_struct deferred_list_work; |
766 | struct list_head deferred_range_list; | |
767 | spinlock_t deferred_list_lock; | |
8a7c184a | 768 | atomic_t evicted_ranges; |
a44fe9ee | 769 | bool drain_pagefaults; |
8a7c184a | 770 | struct delayed_work restore_work; |
5a75ea56 | 771 | DECLARE_BITMAP(bitmap_supported, MAX_GPU_INSTANCE); |
a6283010 | 772 | struct task_struct *faulting_task; |
42de677f PY |
773 | }; |
774 | ||
4a488a7a OG |
775 | /* Process data */ |
776 | struct kfd_process { | |
19f6d2a6 OG |
777 | /* |
778 | * kfd_process are stored in an mm_struct*->kfd_process* | |
779 | * hash table (kfd_processes in kfd_process.c) | |
780 | */ | |
781 | struct hlist_node kfd_processes; | |
782 | ||
9b56bb11 FK |
783 | /* |
784 | * Opaque pointer to mm_struct. We don't hold a reference to | |
785 | * it so it should never be dereferenced from here. This is | |
786 | * only used for looking up processes by their mm. | |
787 | */ | |
788 | void *mm; | |
19f6d2a6 | 789 | |
5ce10687 FK |
790 | struct kref ref; |
791 | struct work_struct release_work; | |
792 | ||
19f6d2a6 OG |
793 | struct mutex mutex; |
794 | ||
795 | /* | |
796 | * In any process, the thread that started main() is the lead | |
797 | * thread and outlives the rest. | |
798 | * It is here because amd_iommu_bind_pasid wants a task_struct. | |
894a8293 FK |
799 | * It can also be used for safely getting a reference to the |
800 | * mm_struct of the process. | |
19f6d2a6 OG |
801 | */ |
802 | struct task_struct *lead_thread; | |
803 | ||
804 | /* We want to receive a notification when the mm_struct is destroyed */ | |
805 | struct mmu_notifier mmu_notifier; | |
806 | ||
c7b6bac9 | 807 | u32 pasid; |
19f6d2a6 OG |
808 | |
809 | /* | |
6ae27841 | 810 | * Array of kfd_process_device pointers, |
19f6d2a6 OG |
811 | * one for each device the process is using. |
812 | */ | |
6ae27841 AS |
813 | struct kfd_process_device *pdds[MAX_GPU_INSTANCE]; |
814 | uint32_t n_pdds; | |
19f6d2a6 | 815 | |
45102048 BG |
816 | struct process_queue_manager pqm; |
817 | ||
19f6d2a6 OG |
818 | /*Is the user space process 32 bit?*/ |
819 | bool is_32bit_user_mode; | |
f3a39818 AL |
820 | |
821 | /* Event-related data */ | |
822 | struct mutex event_mutex; | |
482f0777 FK |
823 | /* Event ID allocator and lookup */ |
824 | struct idr event_idr; | |
50cb7dd9 | 825 | /* Event page */ |
68df0f19 | 826 | u64 signal_handle; |
50cb7dd9 | 827 | struct kfd_signal_page *signal_page; |
b9a5d0a5 | 828 | size_t signal_mapped_size; |
f3a39818 | 829 | size_t signal_event_count; |
c986169f | 830 | bool signal_event_limit_reached; |
403575c4 FK |
831 | |
832 | /* Information used for memory eviction */ | |
833 | void *kgd_process_info; | |
834 | /* Eviction fence that is attached to all the BOs of this process. The | |
835 | * fence will be triggered during eviction and new one will be created | |
836 | * during restore | |
837 | */ | |
838 | struct dma_fence *ef; | |
26103436 FK |
839 | |
840 | /* Work items for evicting and restoring BOs */ | |
841 | struct delayed_work eviction_work; | |
842 | struct delayed_work restore_work; | |
843 | /* seqno of the last scheduled eviction */ | |
844 | unsigned int last_eviction_seqno; | |
845 | /* Approx. the last timestamp (in jiffies) when the process was | |
846 | * restored after an eviction | |
847 | */ | |
848 | unsigned long last_restore_timestamp; | |
de9f26bb KR |
849 | |
850 | /* Kobj for our procfs */ | |
851 | struct kobject *kobj; | |
6d220a7e | 852 | struct kobject *kobj_queues; |
de9f26bb | 853 | struct attribute attr_pasid; |
40ce74d1 | 854 | |
42de677f PY |
855 | /* shared virtual memory registered by this process */ |
856 | struct svm_range_list svms; | |
063e33c5 AS |
857 | |
858 | bool xnack_enabled; | |
4a488a7a OG |
859 | }; |
860 | ||
64d1c3a4 FK |
861 | #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */ |
862 | extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); | |
863 | extern struct srcu_struct kfd_processes_srcu; | |
864 | ||
76baee6c | 865 | /** |
a4497974 RB |
866 | * typedef amdkfd_ioctl_t - typedef for ioctl function pointer. |
867 | * | |
868 | * @filep: pointer to file structure. | |
869 | * @p: amdkfd process pointer. | |
870 | * @data: pointer to arg that was copied from user. | |
76baee6c | 871 | * |
a4497974 | 872 | * Return: returns ioctl completion code. |
76baee6c OG |
873 | */ |
874 | typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p, | |
875 | void *data); | |
876 | ||
877 | struct amdkfd_ioctl_desc { | |
878 | unsigned int cmd; | |
879 | int flags; | |
880 | amdkfd_ioctl_t *func; | |
881 | unsigned int cmd_drv; | |
882 | const char *name; | |
883 | }; | |
67f7cf9f | 884 | bool kfd_dev_is_large_bar(struct kfd_dev *dev); |
76baee6c | 885 | |
1679ae8f | 886 | int kfd_process_create_wq(void); |
19f6d2a6 | 887 | void kfd_process_destroy_wq(void); |
373d7080 | 888 | struct kfd_process *kfd_create_process(struct file *filep); |
19f6d2a6 | 889 | struct kfd_process *kfd_get_process(const struct task_struct *); |
c7b6bac9 | 890 | struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid); |
26103436 | 891 | struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm); |
2aeb742b AS |
892 | |
893 | int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id); | |
56c5977e | 894 | int kfd_process_gpuid_from_adev(struct kfd_process *p, |
cda0f85b FK |
895 | struct amdgpu_device *adev, uint32_t *gpuid, |
896 | uint32_t *gpuidx); | |
2aeb742b AS |
897 | static inline int kfd_process_gpuid_from_gpuidx(struct kfd_process *p, |
898 | uint32_t gpuidx, uint32_t *gpuid) { | |
899 | return gpuidx < p->n_pdds ? p->pdds[gpuidx]->dev->id : -EINVAL; | |
900 | } | |
901 | static inline struct kfd_process_device *kfd_process_device_from_gpuidx( | |
902 | struct kfd_process *p, uint32_t gpuidx) { | |
903 | return gpuidx < p->n_pdds ? p->pdds[gpuidx] : NULL; | |
904 | } | |
905 | ||
abb208a8 | 906 | void kfd_unref_process(struct kfd_process *p); |
6b95e797 FK |
907 | int kfd_process_evict_queues(struct kfd_process *p); |
908 | int kfd_process_restore_queues(struct kfd_process *p); | |
26103436 FK |
909 | void kfd_suspend_all_processes(void); |
910 | int kfd_resume_all_processes(void); | |
19f6d2a6 | 911 | |
b84394e2 FK |
912 | int kfd_process_device_init_vm(struct kfd_process_device *pdd, |
913 | struct file *drm_file); | |
64c7f8cf | 914 | struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, |
733fa1f7 | 915 | struct kfd_process *p); |
19f6d2a6 | 916 | struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, |
093c7d8c AS |
917 | struct kfd_process *p); |
918 | struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, | |
919 | struct kfd_process *p); | |
19f6d2a6 | 920 | |
063e33c5 AS |
921 | bool kfd_process_xnack_mode(struct kfd_process *p, bool supported); |
922 | ||
df03ef93 | 923 | int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process, |
373d7080 FK |
924 | struct vm_area_struct *vma); |
925 | ||
52b29d73 FK |
926 | /* KFD process API for creating and translating handles */ |
927 | int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, | |
928 | void *mem); | |
929 | void *kfd_process_device_translate_handle(struct kfd_process_device *p, | |
930 | int handle); | |
931 | void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, | |
932 | int handle); | |
933 | ||
19f6d2a6 OG |
934 | /* PASIDs */ |
935 | int kfd_pasid_init(void); | |
936 | void kfd_pasid_exit(void); | |
937 | bool kfd_set_pasid_limit(unsigned int new_limit); | |
938 | unsigned int kfd_get_pasid_limit(void); | |
c7b6bac9 FY |
939 | u32 kfd_pasid_alloc(void); |
940 | void kfd_pasid_free(u32 pasid); | |
19f6d2a6 OG |
941 | |
942 | /* Doorbells */ | |
ef568db7 | 943 | size_t kfd_doorbell_process_slice(struct kfd_dev *kfd); |
735df2ba FK |
944 | int kfd_doorbell_init(struct kfd_dev *kfd); |
945 | void kfd_doorbell_fini(struct kfd_dev *kfd); | |
df03ef93 HK |
946 | int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process, |
947 | struct vm_area_struct *vma); | |
ada2b29c | 948 | void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, |
19f6d2a6 OG |
949 | unsigned int *doorbell_off); |
950 | void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr); | |
951 | u32 read_kernel_doorbell(u32 __iomem *db); | |
ada2b29c | 952 | void write_kernel_doorbell(void __iomem *db, u32 value); |
9d7d0248 | 953 | void write_kernel_doorbell64(void __iomem *db, u64 value); |
339903fa | 954 | unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd, |
59d7115d | 955 | struct kfd_process_device *pdd, |
ef568db7 | 956 | unsigned int doorbell_id); |
59d7115d MJ |
957 | phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd); |
958 | int kfd_alloc_process_doorbells(struct kfd_dev *kfd, | |
959 | unsigned int *doorbell_index); | |
960 | void kfd_free_process_doorbells(struct kfd_dev *kfd, | |
961 | unsigned int doorbell_index); | |
6e81090b OG |
962 | /* GTT Sub-Allocator */ |
963 | ||
964 | int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, | |
965 | struct kfd_mem_obj **mem_obj); | |
966 | ||
967 | int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj); | |
968 | ||
4a488a7a OG |
969 | extern struct device *kfd_device; |
970 | ||
de9f26bb KR |
971 | /* KFD's procfs */ |
972 | void kfd_procfs_init(void); | |
973 | void kfd_procfs_shutdown(void); | |
6d220a7e AL |
974 | int kfd_procfs_add_queue(struct queue *q); |
975 | void kfd_procfs_del_queue(struct queue *q); | |
de9f26bb | 976 | |
5b5c4e40 EP |
977 | /* Topology */ |
978 | int kfd_topology_init(void); | |
979 | void kfd_topology_shutdown(void); | |
980 | int kfd_topology_add_device(struct kfd_dev *gpu); | |
981 | int kfd_topology_remove_device(struct kfd_dev *gpu); | |
3a87177e HK |
982 | struct kfd_topology_device *kfd_topology_device_by_proximity_domain( |
983 | uint32_t proximity_domain); | |
44d8cc6f | 984 | struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id); |
5b5c4e40 EP |
985 | struct kfd_dev *kfd_device_by_id(uint32_t gpu_id); |
986 | struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); | |
574c4183 | 987 | struct kfd_dev *kfd_device_by_adev(const struct amdgpu_device *adev); |
6d82eb0e | 988 | int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev); |
520b8fb7 | 989 | int kfd_numa_node_to_apic_id(int numa_node_id); |
6127896f | 990 | void kfd_double_confirm_iommu_support(struct kfd_dev *gpu); |
5b5c4e40 | 991 | |
4a488a7a | 992 | /* Interrupts */ |
2249d558 AL |
993 | int kfd_interrupt_init(struct kfd_dev *dev); |
994 | void kfd_interrupt_exit(struct kfd_dev *dev); | |
2249d558 | 995 | bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry); |
58e69886 LX |
996 | bool interrupt_is_wanted(struct kfd_dev *dev, |
997 | const uint32_t *ih_ring_entry, | |
998 | uint32_t *patched_ihre, bool *flag); | |
4a488a7a | 999 | |
19f6d2a6 OG |
1000 | /* amdkfd Apertures */ |
1001 | int kfd_init_apertures(struct kfd_process *process); | |
1002 | ||
7c9631af JC |
1003 | void kfd_process_set_trap_handler(struct qcm_process_device *qpd, |
1004 | uint64_t tba_addr, | |
1005 | uint64_t tma_addr); | |
1006 | ||
ed6e6a34 | 1007 | /* Queue Context Management */ |
e88a614c | 1008 | int init_queue(struct queue **q, const struct queue_properties *properties); |
ed6e6a34 | 1009 | void uninit_queue(struct queue *q); |
45102048 | 1010 | void print_queue_properties(struct queue_properties *q); |
ed6e6a34 BG |
1011 | void print_queue(struct queue *q); |
1012 | ||
4b8f589b BG |
1013 | struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, |
1014 | struct kfd_dev *dev); | |
ee04955a FK |
1015 | struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type, |
1016 | struct kfd_dev *dev); | |
4b8f589b BG |
1017 | struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type, |
1018 | struct kfd_dev *dev); | |
ee04955a FK |
1019 | struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type, |
1020 | struct kfd_dev *dev); | |
b91d43dd FK |
1021 | struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, |
1022 | struct kfd_dev *dev); | |
14328aa5 PC |
1023 | struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type, |
1024 | struct kfd_dev *dev); | |
64c7f8cf BG |
1025 | struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev); |
1026 | void device_queue_manager_uninit(struct device_queue_manager *dqm); | |
241f24f8 BG |
1027 | struct kernel_queue *kernel_queue_init(struct kfd_dev *dev, |
1028 | enum kfd_queue_type type); | |
c2a77fde | 1029 | void kernel_queue_uninit(struct kernel_queue *kq, bool hanging); |
c7b6bac9 | 1030 | int kfd_process_vm_fault(struct device_queue_manager *dqm, u32 pasid); |
241f24f8 | 1031 | |
45102048 BG |
1032 | /* Process Queue Manager */ |
1033 | struct process_queue_node { | |
1034 | struct queue *q; | |
1035 | struct kernel_queue *kq; | |
1036 | struct list_head process_queue_list; | |
1037 | }; | |
1038 | ||
9fd3f1bf FK |
1039 | void kfd_process_dequeue_from_device(struct kfd_process_device *pdd); |
1040 | void kfd_process_dequeue_from_all_devices(struct kfd_process *p); | |
45102048 BG |
1041 | int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p); |
1042 | void pqm_uninit(struct process_queue_manager *pqm); | |
1043 | int pqm_create_queue(struct process_queue_manager *pqm, | |
1044 | struct kfd_dev *dev, | |
1045 | struct file *f, | |
1046 | struct queue_properties *properties, | |
e47a8b52 YZ |
1047 | unsigned int *qid, |
1048 | uint32_t *p_doorbell_offset_in_process); | |
45102048 | 1049 | int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid); |
7c695a2c | 1050 | int pqm_update_queue_properties(struct process_queue_manager *pqm, unsigned int qid, |
39e7f331 | 1051 | struct queue_properties *p); |
7c695a2c LY |
1052 | int pqm_update_mqd(struct process_queue_manager *pqm, unsigned int qid, |
1053 | struct mqd_update_info *minfo); | |
eb82da1d OZ |
1054 | int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid, |
1055 | void *gws); | |
fbeb661b YS |
1056 | struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm, |
1057 | unsigned int qid); | |
5bb4b78b OZ |
1058 | struct queue *pqm_get_user_queue(struct process_queue_manager *pqm, |
1059 | unsigned int qid); | |
5df099e8 JC |
1060 | int pqm_get_wave_state(struct process_queue_manager *pqm, |
1061 | unsigned int qid, | |
1062 | void __user *ctl_stack, | |
1063 | u32 *ctl_stack_used_size, | |
1064 | u32 *save_area_used_size); | |
45102048 | 1065 | |
b010affe QH |
1066 | int amdkfd_fence_wait_timeout(uint64_t *fence_addr, |
1067 | uint64_t fence_value, | |
14328aa5 | 1068 | unsigned int timeout_ms); |
788bf83d | 1069 | |
ed6e6a34 BG |
1070 | /* Packet Manager */ |
1071 | ||
64c7f8cf BG |
1072 | #define KFD_FENCE_COMPLETED (100) |
1073 | #define KFD_FENCE_INIT (10) | |
241f24f8 | 1074 | |
ed6e6a34 BG |
1075 | struct packet_manager { |
1076 | struct device_queue_manager *dqm; | |
1077 | struct kernel_queue *priv_queue; | |
1078 | struct mutex lock; | |
1079 | bool allocated; | |
1080 | struct kfd_mem_obj *ib_buffer_obj; | |
851a645e | 1081 | unsigned int ib_size_bytes; |
819ec5ac | 1082 | bool is_over_subscription; |
f6e27ff1 FK |
1083 | |
1084 | const struct packet_manager_funcs *pmf; | |
1085 | }; | |
1086 | ||
1087 | struct packet_manager_funcs { | |
1088 | /* Support ASIC-specific packet formats for PM4 packets */ | |
1089 | int (*map_process)(struct packet_manager *pm, uint32_t *buffer, | |
1090 | struct qcm_process_device *qpd); | |
1091 | int (*runlist)(struct packet_manager *pm, uint32_t *buffer, | |
1092 | uint64_t ib, size_t ib_size_in_dwords, bool chain); | |
1093 | int (*set_resources)(struct packet_manager *pm, uint32_t *buffer, | |
1094 | struct scheduling_resources *res); | |
1095 | int (*map_queues)(struct packet_manager *pm, uint32_t *buffer, | |
1096 | struct queue *q, bool is_static); | |
1097 | int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer, | |
1098 | enum kfd_queue_type type, | |
1099 | enum kfd_unmap_queues_filter mode, | |
1100 | uint32_t filter_param, bool reset, | |
1101 | unsigned int sdma_engine); | |
1102 | int (*query_status)(struct packet_manager *pm, uint32_t *buffer, | |
b010affe | 1103 | uint64_t fence_address, uint64_t fence_value); |
f6e27ff1 FK |
1104 | int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer); |
1105 | ||
1106 | /* Packet sizes */ | |
1107 | int map_process_size; | |
1108 | int runlist_size; | |
1109 | int set_resources_size; | |
1110 | int map_queues_size; | |
1111 | int unmap_queues_size; | |
1112 | int query_status_size; | |
1113 | int release_mem_size; | |
ed6e6a34 BG |
1114 | }; |
1115 | ||
f6e27ff1 | 1116 | extern const struct packet_manager_funcs kfd_vi_pm_funcs; |
454150b1 | 1117 | extern const struct packet_manager_funcs kfd_v9_pm_funcs; |
fd6a440e | 1118 | extern const struct packet_manager_funcs kfd_aldebaran_pm_funcs; |
f6e27ff1 | 1119 | |
64c7f8cf | 1120 | int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm); |
c2a77fde | 1121 | void pm_uninit(struct packet_manager *pm, bool hanging); |
64c7f8cf BG |
1122 | int pm_send_set_resources(struct packet_manager *pm, |
1123 | struct scheduling_resources *res); | |
1124 | int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues); | |
1125 | int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address, | |
b010affe | 1126 | uint64_t fence_value); |
64c7f8cf BG |
1127 | |
1128 | int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type, | |
7da2bcf8 | 1129 | enum kfd_unmap_queues_filter mode, |
64c7f8cf BG |
1130 | uint32_t filter_param, bool reset, |
1131 | unsigned int sdma_engine); | |
1132 | ||
241f24f8 BG |
1133 | void pm_release_ib(struct packet_manager *pm); |
1134 | ||
454150b1 FK |
1135 | /* Following PM funcs can be shared among VI and AI */ |
1136 | unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size); | |
454150b1 | 1137 | |
19f6d2a6 | 1138 | uint64_t kfd_get_number_elems(struct kfd_dev *kfd); |
19f6d2a6 | 1139 | |
f3a39818 AL |
1140 | /* Events */ |
1141 | extern const struct kfd_event_interrupt_class event_interrupt_class_cik; | |
ca750681 FK |
1142 | extern const struct kfd_event_interrupt_class event_interrupt_class_v9; |
1143 | ||
930c5ff4 | 1144 | extern const struct kfd_device_global_init_class device_global_init_class_cik; |
f3a39818 | 1145 | |
f3a39818 AL |
1146 | void kfd_event_init_process(struct kfd_process *p); |
1147 | void kfd_event_free_process(struct kfd_process *p); | |
1148 | int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma); | |
1149 | int kfd_wait_on_events(struct kfd_process *p, | |
59d3e8be | 1150 | uint32_t num_events, void __user *data, |
f3a39818 | 1151 | bool all, uint32_t user_timeout_ms, |
fdf0c833 | 1152 | uint32_t *wait_result); |
c7b6bac9 | 1153 | void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, |
f3a39818 | 1154 | uint32_t valid_id_bits); |
59d3e8be | 1155 | void kfd_signal_iommu_event(struct kfd_dev *dev, |
c7b6bac9 FY |
1156 | u32 pasid, unsigned long address, |
1157 | bool is_write_requested, bool is_execute_requested); | |
1158 | void kfd_signal_hw_exception_event(u32 pasid); | |
f3a39818 AL |
1159 | int kfd_set_event(struct kfd_process *p, uint32_t event_id); |
1160 | int kfd_reset_event(struct kfd_process *p, uint32_t event_id); | |
0fc8011f FK |
1161 | int kfd_event_page_set(struct kfd_process *p, void *kernel_address, |
1162 | uint64_t size); | |
f3a39818 AL |
1163 | int kfd_event_create(struct file *devkfd, struct kfd_process *p, |
1164 | uint32_t event_type, bool auto_reset, uint32_t node_id, | |
1165 | uint32_t *event_id, uint32_t *event_trigger_data, | |
1166 | uint64_t *event_page_offset, uint32_t *event_slot_index); | |
1167 | int kfd_event_destroy(struct kfd_process *p, uint32_t event_id); | |
1168 | ||
c7b6bac9 | 1169 | void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid, |
2640c3fa | 1170 | struct kfd_vm_fault_info *info); |
1171 | ||
e42051d2 SL |
1172 | void kfd_signal_reset_event(struct kfd_dev *dev); |
1173 | ||
e2b1f9f5 DL |
1174 | void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid); |
1175 | ||
3543b055 | 1176 | void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type); |
403575c4 | 1177 | |
c3447e81 BG |
1178 | int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p); |
1179 | ||
e42051d2 SL |
1180 | bool kfd_is_locked(void); |
1181 | ||
f756e631 HK |
1182 | /* Compute profile */ |
1183 | void kfd_inc_compute_active(struct kfd_dev *dev); | |
1184 | void kfd_dec_compute_active(struct kfd_dev *dev); | |
1185 | ||
6b855f7b HK |
1186 | /* Cgroup Support */ |
1187 | /* Check with device cgroup if @kfd device is accessible */ | |
1188 | static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd) | |
1189 | { | |
eec8fd02 | 1190 | #if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF) |
6b855f7b HK |
1191 | struct drm_device *ddev = kfd->ddev; |
1192 | ||
99c7b309 | 1193 | return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR, |
6b855f7b HK |
1194 | ddev->render->index, |
1195 | DEVCG_ACC_WRITE | DEVCG_ACC_READ); | |
1196 | #else | |
1197 | return 0; | |
1198 | #endif | |
1199 | } | |
1200 | ||
851a645e FK |
1201 | /* Debugfs */ |
1202 | #if defined(CONFIG_DEBUG_FS) | |
1203 | ||
1204 | void kfd_debugfs_init(void); | |
1205 | void kfd_debugfs_fini(void); | |
1206 | int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data); | |
1207 | int pqm_debugfs_mqds(struct seq_file *m, void *data); | |
1208 | int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data); | |
1209 | int dqm_debugfs_hqds(struct seq_file *m, void *data); | |
1210 | int kfd_debugfs_rls_by_device(struct seq_file *m, void *data); | |
1211 | int pm_debugfs_runlist(struct seq_file *m, void *data); | |
1212 | ||
a29ec470 SL |
1213 | int kfd_debugfs_hang_hws(struct kfd_dev *dev); |
1214 | int pm_debugfs_hang_hws(struct packet_manager *pm); | |
4f942aae | 1215 | int dqm_debugfs_hang_hws(struct device_queue_manager *dqm); |
a29ec470 | 1216 | |
851a645e FK |
1217 | #else |
1218 | ||
1219 | static inline void kfd_debugfs_init(void) {} | |
1220 | static inline void kfd_debugfs_fini(void) {} | |
1221 | ||
1222 | #endif | |
1223 | ||
4a488a7a | 1224 | #endif |