Commit | Line | Data |
---|---|---|
4b8f589b BG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | ||
24 | #include <linux/printk.h> | |
d696d536 | 25 | #include <linux/slab.h> |
589ee628 IM |
26 | #include <linux/mm_types.h> |
27 | ||
4b8f589b BG |
28 | #include "kfd_priv.h" |
29 | #include "kfd_mqd_manager.h" | |
d696d536 BG |
30 | #include "vi_structs.h" |
31 | #include "gca/gfx_8_0_sh_mask.h" | |
32 | #include "gca/gfx_8_0_enum.h" | |
5aaf2bef | 33 | #include "oss/oss_3_0_sh_mask.h" |
0ccbc7cd | 34 | |
d696d536 BG |
35 | #define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8 |
36 | ||
37 | static inline struct vi_mqd *get_mqd(void *mqd) | |
38 | { | |
39 | return (struct vi_mqd *)mqd; | |
40 | } | |
41 | ||
97b9ad12 FK |
42 | static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd) |
43 | { | |
44 | return (struct vi_sdma_mqd *)mqd; | |
45 | } | |
46 | ||
39e7f331 FK |
47 | static void update_cu_mask(struct mqd_manager *mm, void *mqd, |
48 | struct queue_properties *q) | |
49 | { | |
50 | struct vi_mqd *m; | |
51 | uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */ | |
52 | ||
53 | if (q->cu_mask_count == 0) | |
54 | return; | |
55 | ||
56 | mqd_symmetrically_map_cu_mask(mm, | |
57 | q->cu_mask, q->cu_mask_count, se_mask); | |
58 | ||
59 | m = get_mqd(mqd); | |
60 | m->compute_static_thread_mgmt_se0 = se_mask[0]; | |
61 | m->compute_static_thread_mgmt_se1 = se_mask[1]; | |
62 | m->compute_static_thread_mgmt_se2 = se_mask[2]; | |
63 | m->compute_static_thread_mgmt_se3 = se_mask[3]; | |
64 | ||
65 | pr_debug("Update cu mask to %#x %#x %#x %#x\n", | |
66 | m->compute_static_thread_mgmt_se0, | |
67 | m->compute_static_thread_mgmt_se1, | |
68 | m->compute_static_thread_mgmt_se2, | |
69 | m->compute_static_thread_mgmt_se3); | |
70 | } | |
71 | ||
0ccbc7cd OZ |
72 | static void set_priority(struct vi_mqd *m, struct queue_properties *q) |
73 | { | |
74 | m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; | |
75 | m->cp_hqd_queue_priority = q->priority; | |
76 | } | |
77 | ||
d1f8f0d1 OZ |
78 | static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd, |
79 | struct queue_properties *q) | |
80 | { | |
81 | struct kfd_mem_obj *mqd_mem_obj; | |
82 | ||
83 | if (kfd_gtt_sa_allocate(kfd, sizeof(struct vi_mqd), | |
84 | &mqd_mem_obj)) | |
85 | return NULL; | |
86 | ||
87 | return mqd_mem_obj; | |
88 | } | |
89 | ||
8636e53c OZ |
90 | static void init_mqd(struct mqd_manager *mm, void **mqd, |
91 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, | |
d696d536 BG |
92 | struct queue_properties *q) |
93 | { | |
d696d536 BG |
94 | uint64_t addr; |
95 | struct vi_mqd *m; | |
d696d536 | 96 | |
8636e53c OZ |
97 | m = (struct vi_mqd *) mqd_mem_obj->cpu_ptr; |
98 | addr = mqd_mem_obj->gpu_addr; | |
d696d536 BG |
99 | |
100 | memset(m, 0, sizeof(struct vi_mqd)); | |
101 | ||
102 | m->header = 0xC0310800; | |
103 | m->compute_pipelinestat_enable = 1; | |
104 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
105 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
106 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
107 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
108 | ||
109 | m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK | | |
110 | 0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT; | |
111 | ||
112 | m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT | | |
113 | MTYPE_UC << CP_MQD_CONTROL__MTYPE__SHIFT; | |
114 | ||
115 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
116 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
117 | ||
118 | m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT | | |
119 | 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT | | |
5d7c6f18 | 120 | 1 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT; |
d696d536 | 121 | |
0ccbc7cd | 122 | set_priority(m, q); |
d696d536 BG |
123 | m->cp_hqd_eop_rptr = 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT; |
124 | ||
125 | if (q->format == KFD_QUEUE_FORMAT_AQL) | |
126 | m->cp_hqd_iq_rptr = 1; | |
127 | ||
373d7080 FK |
128 | if (q->tba_addr) { |
129 | m->compute_tba_lo = lower_32_bits(q->tba_addr >> 8); | |
130 | m->compute_tba_hi = upper_32_bits(q->tba_addr >> 8); | |
131 | m->compute_tma_lo = lower_32_bits(q->tma_addr >> 8); | |
132 | m->compute_tma_hi = upper_32_bits(q->tma_addr >> 8); | |
133 | m->compute_pgm_rsrc2 |= | |
134 | (1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT); | |
135 | } | |
136 | ||
137 | if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address) { | |
138 | m->cp_hqd_persistent_state |= | |
139 | (1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT); | |
140 | m->cp_hqd_ctx_save_base_addr_lo = | |
141 | lower_32_bits(q->ctx_save_restore_area_address); | |
142 | m->cp_hqd_ctx_save_base_addr_hi = | |
143 | upper_32_bits(q->ctx_save_restore_area_address); | |
144 | m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size; | |
145 | m->cp_hqd_cntl_stack_size = q->ctl_stack_size; | |
146 | m->cp_hqd_cntl_stack_offset = q->ctl_stack_size; | |
147 | m->cp_hqd_wg_state_offset = q->ctl_stack_size; | |
148 | } | |
149 | ||
d696d536 | 150 | *mqd = m; |
4eacc26b | 151 | if (gart_addr) |
d696d536 | 152 | *gart_addr = addr; |
8636e53c | 153 | mm->update_mqd(mm, m, q); |
d696d536 BG |
154 | } |
155 | ||
156 | static int load_mqd(struct mqd_manager *mm, void *mqd, | |
157 | uint32_t pipe_id, uint32_t queue_id, | |
70539bd7 | 158 | struct queue_properties *p, struct mm_struct *mms) |
d696d536 | 159 | { |
70539bd7 FK |
160 | /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */ |
161 | uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0); | |
6d566930 | 162 | uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1); |
70539bd7 FK |
163 | |
164 | return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, | |
165 | (uint32_t __user *)p->write_ptr, | |
166 | wptr_shift, wptr_mask, mms); | |
d696d536 BG |
167 | } |
168 | ||
8636e53c | 169 | static void __update_mqd(struct mqd_manager *mm, void *mqd, |
d696d536 BG |
170 | struct queue_properties *q, unsigned int mtype, |
171 | unsigned int atc_bit) | |
172 | { | |
173 | struct vi_mqd *m; | |
174 | ||
d696d536 BG |
175 | m = get_mqd(mqd); |
176 | ||
177 | m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT | | |
178 | atc_bit << CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT | | |
179 | mtype << CP_HQD_PQ_CONTROL__MTYPE__SHIFT; | |
115c8c41 | 180 | m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1; |
79775b62 | 181 | pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control); |
d696d536 BG |
182 | |
183 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); | |
184 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
185 | ||
186 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
187 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
ee04955a FK |
188 | m->cp_hqd_pq_wptr_poll_addr_lo = lower_32_bits((uint64_t)q->write_ptr); |
189 | m->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits((uint64_t)q->write_ptr); | |
d696d536 BG |
190 | |
191 | m->cp_hqd_pq_doorbell_control = | |
d696d536 BG |
192 | q->doorbell_off << |
193 | CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; | |
79775b62 | 194 | pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", |
d696d536 BG |
195 | m->cp_hqd_pq_doorbell_control); |
196 | ||
197 | m->cp_hqd_eop_control = atc_bit << CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT | | |
198 | mtype << CP_HQD_EOP_CONTROL__MTYPE__SHIFT; | |
199 | ||
200 | m->cp_hqd_ib_control = atc_bit << CP_HQD_IB_CONTROL__IB_ATC__SHIFT | | |
201 | 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT | | |
202 | mtype << CP_HQD_IB_CONTROL__MTYPE__SHIFT; | |
203 | ||
af68d87c JC |
204 | /* |
205 | * HW does not clamp this field correctly. Maximum EOP queue size | |
206 | * is constrained by per-SE EOP done signal count, which is 8-bit. | |
207 | * Limit is 0xFF EOP entries (= 0x7F8 dwords). CP will not submit | |
208 | * more than (EOP entry count - 1) so a queue size of 0x800 dwords | |
209 | * is safe, giving a maximum field value of 0xA. | |
210 | */ | |
211 | m->cp_hqd_eop_control |= min(0xA, | |
115c8c41 | 212 | order_base_2(q->eop_ring_buffer_size / 4) - 1); |
d696d536 BG |
213 | m->cp_hqd_eop_base_addr_lo = |
214 | lower_32_bits(q->eop_ring_buffer_address >> 8); | |
215 | m->cp_hqd_eop_base_addr_hi = | |
216 | upper_32_bits(q->eop_ring_buffer_address >> 8); | |
217 | ||
218 | m->cp_hqd_iq_timer = atc_bit << CP_HQD_IQ_TIMER__IQ_ATC__SHIFT | | |
219 | mtype << CP_HQD_IQ_TIMER__MTYPE__SHIFT; | |
220 | ||
221 | m->cp_hqd_vmid = q->vmid; | |
222 | ||
223 | if (q->format == KFD_QUEUE_FORMAT_AQL) { | |
224 | m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK | | |
225 | 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT; | |
226 | } | |
227 | ||
373d7080 FK |
228 | if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address) |
229 | m->cp_hqd_ctx_save_control = | |
230 | atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT | | |
231 | mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT; | |
232 | ||
39e7f331 | 233 | update_cu_mask(mm, mqd, q); |
0ccbc7cd | 234 | set_priority(m, q); |
39e7f331 | 235 | |
bb2d2128 | 236 | q->is_active = QUEUE_IS_ACTIVE(*q); |
d696d536 BG |
237 | } |
238 | ||
239 | ||
8636e53c | 240 | static void update_mqd(struct mqd_manager *mm, void *mqd, |
d696d536 BG |
241 | struct queue_properties *q) |
242 | { | |
8636e53c | 243 | __update_mqd(mm, mqd, q, MTYPE_CC, 1); |
d696d536 BG |
244 | } |
245 | ||
51a0f459 OZ |
246 | static uint32_t read_doorbell_id(void *mqd) |
247 | { | |
248 | struct vi_mqd *m = (struct vi_mqd *)mqd; | |
249 | ||
250 | return m->queue_doorbell_id0; | |
251 | } | |
252 | ||
8636e53c | 253 | static void update_mqd_tonga(struct mqd_manager *mm, void *mqd, |
ee04955a FK |
254 | struct queue_properties *q) |
255 | { | |
8636e53c | 256 | __update_mqd(mm, mqd, q, MTYPE_UC, 0); |
ee04955a FK |
257 | } |
258 | ||
d696d536 BG |
259 | static int destroy_mqd(struct mqd_manager *mm, void *mqd, |
260 | enum kfd_preempt_type type, | |
261 | unsigned int timeout, uint32_t pipe_id, | |
262 | uint32_t queue_id) | |
263 | { | |
264 | return mm->dev->kfd2kgd->hqd_destroy | |
70539bd7 | 265 | (mm->dev->kgd, mqd, type, timeout, |
d696d536 BG |
266 | pipe_id, queue_id); |
267 | } | |
268 | ||
8636e53c | 269 | static void free_mqd(struct mqd_manager *mm, void *mqd, |
d696d536 BG |
270 | struct kfd_mem_obj *mqd_mem_obj) |
271 | { | |
d696d536 BG |
272 | kfd_gtt_sa_free(mm->dev, mqd_mem_obj); |
273 | } | |
274 | ||
275 | static bool is_occupied(struct mqd_manager *mm, void *mqd, | |
276 | uint64_t queue_address, uint32_t pipe_id, | |
277 | uint32_t queue_id) | |
278 | { | |
279 | return mm->dev->kfd2kgd->hqd_is_occupied( | |
280 | mm->dev->kgd, queue_address, | |
281 | pipe_id, queue_id); | |
282 | } | |
283 | ||
5df099e8 JC |
284 | static int get_wave_state(struct mqd_manager *mm, void *mqd, |
285 | void __user *ctl_stack, | |
286 | u32 *ctl_stack_used_size, | |
287 | u32 *save_area_used_size) | |
288 | { | |
289 | struct vi_mqd *m; | |
290 | ||
291 | m = get_mqd(mqd); | |
292 | ||
293 | *ctl_stack_used_size = m->cp_hqd_cntl_stack_size - | |
294 | m->cp_hqd_cntl_stack_offset; | |
295 | *save_area_used_size = m->cp_hqd_wg_state_offset - | |
296 | m->cp_hqd_cntl_stack_size; | |
297 | ||
298 | /* Control stack is not copied to user mode for GFXv8 because | |
299 | * it's part of the context save area that is already | |
300 | * accessible to user mode | |
301 | */ | |
302 | ||
303 | return 0; | |
304 | } | |
305 | ||
8636e53c OZ |
306 | static void init_mqd_hiq(struct mqd_manager *mm, void **mqd, |
307 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, | |
d696d536 BG |
308 | struct queue_properties *q) |
309 | { | |
310 | struct vi_mqd *m; | |
8636e53c | 311 | init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q); |
d696d536 BG |
312 | |
313 | m = get_mqd(*mqd); | |
314 | ||
315 | m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT | | |
316 | 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT; | |
d696d536 BG |
317 | } |
318 | ||
8636e53c | 319 | static void update_mqd_hiq(struct mqd_manager *mm, void *mqd, |
d696d536 BG |
320 | struct queue_properties *q) |
321 | { | |
8636e53c | 322 | __update_mqd(mm, mqd, q, MTYPE_UC, 0); |
d696d536 | 323 | } |
4b8f589b | 324 | |
8636e53c OZ |
325 | static void init_mqd_sdma(struct mqd_manager *mm, void **mqd, |
326 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, | |
5aaf2bef PC |
327 | struct queue_properties *q) |
328 | { | |
5aaf2bef | 329 | struct vi_sdma_mqd *m; |
5aaf2bef | 330 | |
8636e53c | 331 | m = (struct vi_sdma_mqd *) mqd_mem_obj->cpu_ptr; |
5aaf2bef PC |
332 | |
333 | memset(m, 0, sizeof(struct vi_sdma_mqd)); | |
334 | ||
335 | *mqd = m; | |
0d87c9cf | 336 | if (gart_addr) |
8636e53c | 337 | *gart_addr = mqd_mem_obj->gpu_addr; |
5aaf2bef | 338 | |
8636e53c | 339 | mm->update_mqd(mm, m, q); |
5aaf2bef PC |
340 | } |
341 | ||
5aaf2bef PC |
342 | static int load_mqd_sdma(struct mqd_manager *mm, void *mqd, |
343 | uint32_t pipe_id, uint32_t queue_id, | |
344 | struct queue_properties *p, struct mm_struct *mms) | |
345 | { | |
346 | return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd, | |
347 | (uint32_t __user *)p->write_ptr, | |
348 | mms); | |
349 | } | |
350 | ||
8636e53c | 351 | static void update_mqd_sdma(struct mqd_manager *mm, void *mqd, |
5aaf2bef PC |
352 | struct queue_properties *q) |
353 | { | |
354 | struct vi_sdma_mqd *m; | |
355 | ||
356 | m = get_sdma_mqd(mqd); | |
115c8c41 | 357 | m->sdmax_rlcx_rb_cntl = order_base_2(q->queue_size / 4) |
5aaf2bef PC |
358 | << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT | |
359 | q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT | | |
360 | 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | | |
361 | 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT; | |
362 | ||
363 | m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8); | |
364 | m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8); | |
365 | m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
366 | m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
367 | m->sdmax_rlcx_doorbell = | |
368 | q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT; | |
369 | ||
370 | m->sdmax_rlcx_virtual_addr = q->sdma_vm_addr; | |
371 | ||
372 | m->sdma_engine_id = q->sdma_engine_id; | |
373 | m->sdma_queue_id = q->sdma_queue_id; | |
374 | ||
bb2d2128 | 375 | q->is_active = QUEUE_IS_ACTIVE(*q); |
5aaf2bef PC |
376 | } |
377 | ||
378 | /* | |
379 | * * preempt type here is ignored because there is only one way | |
380 | * * to preempt sdma queue | |
381 | */ | |
382 | static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd, | |
383 | enum kfd_preempt_type type, | |
384 | unsigned int timeout, uint32_t pipe_id, | |
385 | uint32_t queue_id) | |
386 | { | |
387 | return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout); | |
388 | } | |
389 | ||
390 | static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd, | |
391 | uint64_t queue_address, uint32_t pipe_id, | |
392 | uint32_t queue_id) | |
393 | { | |
394 | return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd); | |
395 | } | |
396 | ||
851a645e | 397 | #if defined(CONFIG_DEBUG_FS) |
5aaf2bef | 398 | |
851a645e FK |
399 | static int debugfs_show_mqd(struct seq_file *m, void *data) |
400 | { | |
401 | seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4, | |
402 | data, sizeof(struct vi_mqd), false); | |
403 | return 0; | |
404 | } | |
405 | ||
406 | static int debugfs_show_mqd_sdma(struct seq_file *m, void *data) | |
407 | { | |
408 | seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4, | |
409 | data, sizeof(struct vi_sdma_mqd), false); | |
410 | return 0; | |
411 | } | |
412 | ||
413 | #endif | |
5aaf2bef | 414 | |
4b8f589b | 415 | struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type, |
d696d536 | 416 | struct kfd_dev *dev) |
4b8f589b | 417 | { |
d696d536 BG |
418 | struct mqd_manager *mqd; |
419 | ||
32fa8219 FK |
420 | if (WARN_ON(type >= KFD_MQD_TYPE_MAX)) |
421 | return NULL; | |
d696d536 | 422 | |
1cd106ec | 423 | mqd = kzalloc(sizeof(*mqd), GFP_KERNEL); |
d696d536 BG |
424 | if (!mqd) |
425 | return NULL; | |
426 | ||
427 | mqd->dev = dev; | |
428 | ||
429 | switch (type) { | |
430 | case KFD_MQD_TYPE_CP: | |
8636e53c | 431 | mqd->allocate_mqd = allocate_mqd; |
d696d536 | 432 | mqd->init_mqd = init_mqd; |
8636e53c | 433 | mqd->free_mqd = free_mqd; |
d696d536 BG |
434 | mqd->load_mqd = load_mqd; |
435 | mqd->update_mqd = update_mqd; | |
436 | mqd->destroy_mqd = destroy_mqd; | |
437 | mqd->is_occupied = is_occupied; | |
5df099e8 | 438 | mqd->get_wave_state = get_wave_state; |
6c6cde55 | 439 | mqd->mqd_size = sizeof(struct vi_mqd); |
851a645e FK |
440 | #if defined(CONFIG_DEBUG_FS) |
441 | mqd->debugfs_show_mqd = debugfs_show_mqd; | |
442 | #endif | |
d696d536 BG |
443 | break; |
444 | case KFD_MQD_TYPE_HIQ: | |
8636e53c | 445 | mqd->allocate_mqd = allocate_hiq_mqd; |
d696d536 | 446 | mqd->init_mqd = init_mqd_hiq; |
8636e53c | 447 | mqd->free_mqd = free_mqd_hiq_sdma; |
d696d536 BG |
448 | mqd->load_mqd = load_mqd; |
449 | mqd->update_mqd = update_mqd_hiq; | |
450 | mqd->destroy_mqd = destroy_mqd; | |
451 | mqd->is_occupied = is_occupied; | |
6c6cde55 | 452 | mqd->mqd_size = sizeof(struct vi_mqd); |
851a645e FK |
453 | #if defined(CONFIG_DEBUG_FS) |
454 | mqd->debugfs_show_mqd = debugfs_show_mqd; | |
59f650a0 | 455 | #endif |
51a0f459 | 456 | mqd->read_doorbell_id = read_doorbell_id; |
59f650a0 OZ |
457 | break; |
458 | case KFD_MQD_TYPE_DIQ: | |
7633c5e0 | 459 | mqd->allocate_mqd = allocate_mqd; |
59f650a0 | 460 | mqd->init_mqd = init_mqd_hiq; |
8636e53c | 461 | mqd->free_mqd = free_mqd; |
59f650a0 OZ |
462 | mqd->load_mqd = load_mqd; |
463 | mqd->update_mqd = update_mqd_hiq; | |
464 | mqd->destroy_mqd = destroy_mqd; | |
465 | mqd->is_occupied = is_occupied; | |
6c6cde55 | 466 | mqd->mqd_size = sizeof(struct vi_mqd); |
59f650a0 OZ |
467 | #if defined(CONFIG_DEBUG_FS) |
468 | mqd->debugfs_show_mqd = debugfs_show_mqd; | |
851a645e | 469 | #endif |
d696d536 BG |
470 | break; |
471 | case KFD_MQD_TYPE_SDMA: | |
8636e53c | 472 | mqd->allocate_mqd = allocate_sdma_mqd; |
5aaf2bef | 473 | mqd->init_mqd = init_mqd_sdma; |
8636e53c | 474 | mqd->free_mqd = free_mqd_hiq_sdma; |
5aaf2bef PC |
475 | mqd->load_mqd = load_mqd_sdma; |
476 | mqd->update_mqd = update_mqd_sdma; | |
477 | mqd->destroy_mqd = destroy_mqd_sdma; | |
478 | mqd->is_occupied = is_occupied_sdma; | |
6c6cde55 | 479 | mqd->mqd_size = sizeof(struct vi_sdma_mqd); |
851a645e FK |
480 | #if defined(CONFIG_DEBUG_FS) |
481 | mqd->debugfs_show_mqd = debugfs_show_mqd_sdma; | |
482 | #endif | |
d696d536 BG |
483 | break; |
484 | default: | |
485 | kfree(mqd); | |
486 | return NULL; | |
487 | } | |
488 | ||
489 | return mqd; | |
4b8f589b | 490 | } |
ee04955a FK |
491 | |
492 | struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type, | |
493 | struct kfd_dev *dev) | |
494 | { | |
495 | struct mqd_manager *mqd; | |
496 | ||
497 | mqd = mqd_manager_init_vi(type, dev); | |
498 | if (!mqd) | |
499 | return NULL; | |
d7c0b047 | 500 | if (type == KFD_MQD_TYPE_CP) |
ee04955a FK |
501 | mqd->update_mqd = update_mqd_tonga; |
502 | return mqd; | |
503 | } |