Commit | Line | Data |
---|---|---|
d87f36a0 | 1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
4b8f589b | 2 | /* |
d87f36a0 | 3 | * Copyright 2014-2022 Advanced Micro Devices, Inc. |
4b8f589b BG |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the "Software"), | |
7 | * to deal in the Software without restriction, including without limitation | |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
9 | * and/or sell copies of the Software, and to permit persons to whom the | |
10 | * Software is furnished to do so, subject to the following conditions: | |
11 | * | |
12 | * The above copyright notice and this permission notice shall be included in | |
13 | * all copies or substantial portions of the Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
21 | * OTHER DEALINGS IN THE SOFTWARE. | |
22 | * | |
23 | */ | |
24 | ||
25 | #include <linux/printk.h> | |
26 | #include <linux/slab.h> | |
589ee628 IM |
27 | #include <linux/mm_types.h> |
28 | ||
4b8f589b BG |
29 | #include "kfd_priv.h" |
30 | #include "kfd_mqd_manager.h" | |
31 | #include "cik_regs.h" | |
32 | #include "cik_structs.h" | |
3d30b28b | 33 | #include "oss/oss_2_4_sh_mask.h" |
4b8f589b | 34 | |
4b8f589b BG |
35 | static inline struct cik_mqd *get_mqd(void *mqd) |
36 | { | |
37 | return (struct cik_mqd *)mqd; | |
38 | } | |
39 | ||
97b9ad12 FK |
40 | static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) |
41 | { | |
42 | return (struct cik_sdma_rlc_registers *)mqd; | |
43 | } | |
44 | ||
39e7f331 | 45 | static void update_cu_mask(struct mqd_manager *mm, void *mqd, |
7c695a2c | 46 | struct mqd_update_info *minfo) |
39e7f331 FK |
47 | { |
48 | struct cik_mqd *m; | |
49 | uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */ | |
50 | ||
69a8c3ae | 51 | if (!minfo || !minfo->cu_mask.ptr) |
39e7f331 FK |
52 | return; |
53 | ||
54 | mqd_symmetrically_map_cu_mask(mm, | |
fc6efed2 | 55 | minfo->cu_mask.ptr, minfo->cu_mask.count, se_mask, 0); |
39e7f331 FK |
56 | |
57 | m = get_mqd(mqd); | |
58 | m->compute_static_thread_mgmt_se0 = se_mask[0]; | |
59 | m->compute_static_thread_mgmt_se1 = se_mask[1]; | |
60 | m->compute_static_thread_mgmt_se2 = se_mask[2]; | |
61 | m->compute_static_thread_mgmt_se3 = se_mask[3]; | |
62 | ||
63 | pr_debug("Update cu mask to %#x %#x %#x %#x\n", | |
64 | m->compute_static_thread_mgmt_se0, | |
65 | m->compute_static_thread_mgmt_se1, | |
66 | m->compute_static_thread_mgmt_se2, | |
67 | m->compute_static_thread_mgmt_se3); | |
68 | } | |
69 | ||
0ccbc7cd OZ |
70 | static void set_priority(struct cik_mqd *m, struct queue_properties *q) |
71 | { | |
72 | m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; | |
73 | m->cp_hqd_queue_priority = q->priority; | |
74 | } | |
75 | ||
8dc1db31 | 76 | static struct kfd_mem_obj *allocate_mqd(struct kfd_node *kfd, |
d1f8f0d1 OZ |
77 | struct queue_properties *q) |
78 | { | |
79 | struct kfd_mem_obj *mqd_mem_obj; | |
80 | ||
81 | if (kfd_gtt_sa_allocate(kfd, sizeof(struct cik_mqd), | |
82 | &mqd_mem_obj)) | |
83 | return NULL; | |
84 | ||
85 | return mqd_mem_obj; | |
86 | } | |
87 | ||
8636e53c OZ |
88 | static void init_mqd(struct mqd_manager *mm, void **mqd, |
89 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, | |
4b8f589b BG |
90 | struct queue_properties *q) |
91 | { | |
92 | uint64_t addr; | |
93 | struct cik_mqd *m; | |
4b8f589b | 94 | |
8636e53c OZ |
95 | m = (struct cik_mqd *) mqd_mem_obj->cpu_ptr; |
96 | addr = mqd_mem_obj->gpu_addr; | |
4b8f589b BG |
97 | |
98 | memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); | |
99 | ||
100 | m->header = 0xC0310800; | |
101 | m->compute_pipelinestat_enable = 1; | |
102 | m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; | |
103 | m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; | |
104 | m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; | |
105 | m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; | |
106 | ||
107 | /* | |
108 | * Make sure to use the last queue state saved on mqd when the cp | |
109 | * reassigns the queue, so when queue is switched on/off (e.g over | |
110 | * subscription or quantum timeout) the context will be consistent | |
111 | */ | |
112 | m->cp_hqd_persistent_state = | |
113 | DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; | |
114 | ||
115 | m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; | |
116 | m->cp_mqd_base_addr_lo = lower_32_bits(addr); | |
117 | m->cp_mqd_base_addr_hi = upper_32_bits(addr); | |
118 | ||
4b8f589b BG |
119 | m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | |
120 | QUANTUM_DURATION(10); | |
121 | ||
122 | /* | |
123 | * Pipe Priority | |
124 | * Identifies the pipe relative priority when this queue is connected | |
125 | * to the pipeline. The pipe priority is against the GFX pipe and HP3D. | |
126 | * In KFD we are using a fixed pipe priority set to CS_MEDIUM. | |
127 | * 0 = CS_LOW (typically below GFX) | |
128 | * 1 = CS_MEDIUM (typically between HP3D and GFX | |
129 | * 2 = CS_HIGH (typically above HP3D) | |
130 | */ | |
0ccbc7cd | 131 | set_priority(m, q); |
4b8f589b | 132 | |
d752f95e JC |
133 | if (q->format == KFD_QUEUE_FORMAT_AQL) |
134 | m->cp_hqd_iq_rptr = AQL_ENABLE; | |
135 | ||
4b8f589b | 136 | *mqd = m; |
4eacc26b | 137 | if (gart_addr) |
4b8f589b | 138 | *gart_addr = addr; |
c6e559eb | 139 | mm->update_mqd(mm, m, q, NULL); |
4b8f589b BG |
140 | } |
141 | ||
8636e53c OZ |
142 | static void init_mqd_sdma(struct mqd_manager *mm, void **mqd, |
143 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, | |
4b8f589b BG |
144 | struct queue_properties *q) |
145 | { | |
4b8f589b BG |
146 | struct cik_sdma_rlc_registers *m; |
147 | ||
8636e53c | 148 | m = (struct cik_sdma_rlc_registers *) mqd_mem_obj->cpu_ptr; |
4b8f589b BG |
149 | |
150 | memset(m, 0, sizeof(struct cik_sdma_rlc_registers)); | |
151 | ||
152 | *mqd = m; | |
4eacc26b | 153 | if (gart_addr) |
8636e53c | 154 | *gart_addr = mqd_mem_obj->gpu_addr; |
4b8f589b | 155 | |
c6e559eb | 156 | mm->update_mqd(mm, m, q, NULL); |
4b8f589b BG |
157 | } |
158 | ||
4b8f589b | 159 | static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, |
70539bd7 FK |
160 | uint32_t queue_id, struct queue_properties *p, |
161 | struct mm_struct *mms) | |
4b8f589b | 162 | { |
70539bd7 FK |
163 | /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */ |
164 | uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0); | |
6d566930 | 165 | uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1); |
70539bd7 | 166 | |
420185fd | 167 | return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id, |
70539bd7 | 168 | (uint32_t __user *)p->write_ptr, |
e2069a7b | 169 | wptr_shift, wptr_mask, mms, 0); |
4b8f589b BG |
170 | } |
171 | ||
8636e53c | 172 | static void __update_mqd(struct mqd_manager *mm, void *mqd, |
c6e559eb LY |
173 | struct queue_properties *q, struct mqd_update_info *minfo, |
174 | unsigned int atc_bit) | |
4b8f589b BG |
175 | { |
176 | struct cik_mqd *m; | |
177 | ||
4b8f589b BG |
178 | m = get_mqd(mqd); |
179 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
ee04955a FK |
180 | DEFAULT_MIN_AVAIL_SIZE; |
181 | m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; | |
182 | if (atc_bit) { | |
183 | m->cp_hqd_pq_control |= PQ_ATC_EN; | |
184 | m->cp_hqd_ib_control |= IB_ATC_EN; | |
185 | } | |
4b8f589b BG |
186 | |
187 | /* | |
188 | * Calculating queue size which is log base 2 of actual queue size -1 | |
189 | * dwords and another -1 for ffs | |
190 | */ | |
115c8c41 | 191 | m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1; |
4b8f589b BG |
192 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); |
193 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
194 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
195 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
70539bd7 | 196 | m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off); |
4b8f589b BG |
197 | |
198 | m->cp_hqd_vmid = q->vmid; | |
199 | ||
8eabaf54 | 200 | if (q->format == KFD_QUEUE_FORMAT_AQL) |
4b8f589b | 201 | m->cp_hqd_pq_control |= NO_UPDATE_RPTR; |
4b8f589b | 202 | |
7c695a2c | 203 | update_cu_mask(mm, mqd, minfo); |
0ccbc7cd | 204 | set_priority(m, q); |
39e7f331 | 205 | |
bb2d2128 | 206 | q->is_active = QUEUE_IS_ACTIVE(*q); |
4b8f589b BG |
207 | } |
208 | ||
0991a4c1 | 209 | static bool check_preemption_failed(struct mqd_manager *mm, void *mqd) |
51a0f459 OZ |
210 | { |
211 | struct cik_mqd *m = (struct cik_mqd *)mqd; | |
212 | ||
0991a4c1 | 213 | return kfd_check_hiq_mqd_doorbell_id(mm->dev, m->queue_doorbell_id0, 0); |
51a0f459 OZ |
214 | } |
215 | ||
c99a2e7a AD |
216 | static void update_mqd(struct mqd_manager *mm, void *mqd, |
217 | struct queue_properties *q, | |
218 | struct mqd_update_info *minfo) | |
ee04955a | 219 | { |
c6e559eb | 220 | __update_mqd(mm, mqd, q, minfo, 0); |
ee04955a FK |
221 | } |
222 | ||
8636e53c | 223 | static void update_mqd_sdma(struct mqd_manager *mm, void *mqd, |
c6e559eb LY |
224 | struct queue_properties *q, |
225 | struct mqd_update_info *minfo) | |
4b8f589b BG |
226 | { |
227 | struct cik_sdma_rlc_registers *m; | |
228 | ||
4b8f589b | 229 | m = get_sdma_mqd(mqd); |
115c8c41 | 230 | m->sdma_rlc_rb_cntl = order_base_2(q->queue_size / 4) |
d12fb13f | 231 | << SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT | |
3d30b28b OG |
232 | q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT | |
233 | 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | | |
234 | 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT; | |
4b8f589b BG |
235 | |
236 | m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8); | |
237 | m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8); | |
238 | m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
239 | m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
bba9662d JC |
240 | m->sdma_rlc_doorbell = |
241 | q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT; | |
3d30b28b | 242 | |
4b8f589b BG |
243 | m->sdma_rlc_virtual_addr = q->sdma_vm_addr; |
244 | ||
245 | m->sdma_engine_id = q->sdma_engine_id; | |
246 | m->sdma_queue_id = q->sdma_queue_id; | |
247 | ||
bb2d2128 | 248 | q->is_active = QUEUE_IS_ACTIVE(*q); |
4b8f589b BG |
249 | } |
250 | ||
3a9822d7 | 251 | static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst) |
42c6c482 DYS |
252 | { |
253 | struct cik_mqd *m; | |
254 | ||
255 | m = get_mqd(mqd); | |
256 | ||
257 | memcpy(mqd_dst, m, sizeof(struct cik_mqd)); | |
258 | } | |
259 | ||
260 | static void restore_mqd(struct mqd_manager *mm, void **mqd, | |
261 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, | |
262 | struct queue_properties *qp, | |
3a9822d7 DYS |
263 | const void *mqd_src, |
264 | const void *ctl_stack_src, const u32 ctl_stack_size) | |
42c6c482 DYS |
265 | { |
266 | uint64_t addr; | |
267 | struct cik_mqd *m; | |
268 | ||
269 | m = (struct cik_mqd *) mqd_mem_obj->cpu_ptr; | |
270 | addr = mqd_mem_obj->gpu_addr; | |
271 | ||
272 | memcpy(m, mqd_src, sizeof(*m)); | |
273 | ||
274 | *mqd = m; | |
275 | if (gart_addr) | |
276 | *gart_addr = addr; | |
277 | ||
278 | m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(qp->doorbell_off); | |
279 | ||
280 | pr_debug("cp_hqd_pq_doorbell_control 0x%x\n", | |
281 | m->cp_hqd_pq_doorbell_control); | |
282 | ||
283 | qp->is_active = 0; | |
284 | } | |
285 | ||
3a9822d7 DYS |
286 | static void checkpoint_mqd_sdma(struct mqd_manager *mm, |
287 | void *mqd, | |
288 | void *mqd_dst, | |
289 | void *ctl_stack_dst) | |
42c6c482 DYS |
290 | { |
291 | struct cik_sdma_rlc_registers *m; | |
292 | ||
293 | m = get_sdma_mqd(mqd); | |
294 | ||
295 | memcpy(mqd_dst, m, sizeof(struct cik_sdma_rlc_registers)); | |
296 | } | |
297 | ||
298 | static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd, | |
299 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, | |
300 | struct queue_properties *qp, | |
3a9822d7 DYS |
301 | const void *mqd_src, |
302 | const void *ctl_stack_src, const u32 ctl_stack_size) | |
42c6c482 DYS |
303 | { |
304 | uint64_t addr; | |
305 | struct cik_sdma_rlc_registers *m; | |
306 | ||
307 | m = (struct cik_sdma_rlc_registers *) mqd_mem_obj->cpu_ptr; | |
308 | addr = mqd_mem_obj->gpu_addr; | |
309 | ||
310 | memcpy(m, mqd_src, sizeof(*m)); | |
311 | ||
312 | m->sdma_rlc_doorbell = | |
313 | qp->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT; | |
314 | ||
315 | *mqd = m; | |
316 | if (gart_addr) | |
317 | *gart_addr = addr; | |
318 | ||
319 | qp->is_active = 0; | |
320 | } | |
321 | ||
4b8f589b BG |
322 | /* |
323 | * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. | |
324 | * The HIQ queue in Kaveri is using the same MQD structure as all the user mode | |
325 | * queues but with different initial values. | |
326 | */ | |
327 | ||
8636e53c OZ |
328 | static void init_mqd_hiq(struct mqd_manager *mm, void **mqd, |
329 | struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, | |
4b8f589b BG |
330 | struct queue_properties *q) |
331 | { | |
8636e53c | 332 | init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q); |
4b8f589b BG |
333 | } |
334 | ||
8636e53c | 335 | static void update_mqd_hiq(struct mqd_manager *mm, void *mqd, |
c6e559eb LY |
336 | struct queue_properties *q, |
337 | struct mqd_update_info *minfo) | |
4b8f589b BG |
338 | { |
339 | struct cik_mqd *m; | |
340 | ||
4b8f589b BG |
341 | m = get_mqd(mqd); |
342 | m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | | |
343 | DEFAULT_MIN_AVAIL_SIZE | | |
344 | PRIV_STATE | | |
345 | KMD_QUEUE; | |
346 | ||
347 | /* | |
348 | * Calculating queue size which is log base 2 of actual queue | |
349 | * size -1 dwords | |
350 | */ | |
115c8c41 | 351 | m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1; |
4b8f589b BG |
352 | m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); |
353 | m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); | |
354 | m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); | |
355 | m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); | |
bba9662d | 356 | m->cp_hqd_pq_doorbell_control = DOORBELL_OFFSET(q->doorbell_off); |
4b8f589b BG |
357 | |
358 | m->cp_hqd_vmid = q->vmid; | |
359 | ||
bb2d2128 | 360 | q->is_active = QUEUE_IS_ACTIVE(*q); |
4b8f589b | 361 | |
0ccbc7cd | 362 | set_priority(m, q); |
4b8f589b BG |
363 | } |
364 | ||
851a645e FK |
365 | #if defined(CONFIG_DEBUG_FS) |
366 | ||
367 | static int debugfs_show_mqd(struct seq_file *m, void *data) | |
368 | { | |
369 | seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4, | |
370 | data, sizeof(struct cik_mqd), false); | |
371 | return 0; | |
372 | } | |
373 | ||
374 | static int debugfs_show_mqd_sdma(struct seq_file *m, void *data) | |
375 | { | |
376 | seq_hex_dump(m, " ", DUMP_PREFIX_OFFSET, 32, 4, | |
377 | data, sizeof(struct cik_sdma_rlc_registers), false); | |
378 | return 0; | |
379 | } | |
380 | ||
381 | #endif | |
382 | ||
4b8f589b | 383 | struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, |
8dc1db31 | 384 | struct kfd_node *dev) |
4b8f589b BG |
385 | { |
386 | struct mqd_manager *mqd; | |
387 | ||
32fa8219 FK |
388 | if (WARN_ON(type >= KFD_MQD_TYPE_MAX)) |
389 | return NULL; | |
4b8f589b | 390 | |
1cd106ec | 391 | mqd = kzalloc(sizeof(*mqd), GFP_KERNEL); |
4b8f589b BG |
392 | if (!mqd) |
393 | return NULL; | |
394 | ||
395 | mqd->dev = dev; | |
396 | ||
397 | switch (type) { | |
398 | case KFD_MQD_TYPE_CP: | |
8636e53c | 399 | mqd->allocate_mqd = allocate_mqd; |
4b8f589b | 400 | mqd->init_mqd = init_mqd; |
a439b890 | 401 | mqd->free_mqd = kfd_free_mqd_cp; |
4b8f589b BG |
402 | mqd->load_mqd = load_mqd; |
403 | mqd->update_mqd = update_mqd; | |
a439b890 MJ |
404 | mqd->destroy_mqd = kfd_destroy_mqd_cp; |
405 | mqd->is_occupied = kfd_is_occupied_cp; | |
42c6c482 DYS |
406 | mqd->checkpoint_mqd = checkpoint_mqd; |
407 | mqd->restore_mqd = restore_mqd; | |
6c6cde55 | 408 | mqd->mqd_size = sizeof(struct cik_mqd); |
851a645e FK |
409 | #if defined(CONFIG_DEBUG_FS) |
410 | mqd->debugfs_show_mqd = debugfs_show_mqd; | |
411 | #endif | |
4b8f589b BG |
412 | break; |
413 | case KFD_MQD_TYPE_HIQ: | |
8636e53c | 414 | mqd->allocate_mqd = allocate_hiq_mqd; |
4b8f589b | 415 | mqd->init_mqd = init_mqd_hiq; |
8636e53c | 416 | mqd->free_mqd = free_mqd_hiq_sdma; |
4b8f589b BG |
417 | mqd->load_mqd = load_mqd; |
418 | mqd->update_mqd = update_mqd_hiq; | |
a439b890 MJ |
419 | mqd->destroy_mqd = kfd_destroy_mqd_cp; |
420 | mqd->is_occupied = kfd_is_occupied_cp; | |
6c6cde55 | 421 | mqd->mqd_size = sizeof(struct cik_mqd); |
2f77b9a2 | 422 | mqd->mqd_stride = kfd_mqd_stride; |
851a645e FK |
423 | #if defined(CONFIG_DEBUG_FS) |
424 | mqd->debugfs_show_mqd = debugfs_show_mqd; | |
59f650a0 | 425 | #endif |
26d97182 | 426 | mqd->check_preemption_failed = check_preemption_failed; |
59f650a0 OZ |
427 | break; |
428 | case KFD_MQD_TYPE_DIQ: | |
7633c5e0 | 429 | mqd->allocate_mqd = allocate_mqd; |
59f650a0 | 430 | mqd->init_mqd = init_mqd_hiq; |
a439b890 | 431 | mqd->free_mqd = kfd_free_mqd_cp; |
59f650a0 OZ |
432 | mqd->load_mqd = load_mqd; |
433 | mqd->update_mqd = update_mqd_hiq; | |
a439b890 MJ |
434 | mqd->destroy_mqd = kfd_destroy_mqd_cp; |
435 | mqd->is_occupied = kfd_is_occupied_cp; | |
6c6cde55 | 436 | mqd->mqd_size = sizeof(struct cik_mqd); |
2f77b9a2 | 437 | mqd->mqd_stride = kfd_mqd_stride; |
59f650a0 OZ |
438 | #if defined(CONFIG_DEBUG_FS) |
439 | mqd->debugfs_show_mqd = debugfs_show_mqd; | |
851a645e | 440 | #endif |
4b8f589b BG |
441 | break; |
442 | case KFD_MQD_TYPE_SDMA: | |
8636e53c | 443 | mqd->allocate_mqd = allocate_sdma_mqd; |
4b8f589b | 444 | mqd->init_mqd = init_mqd_sdma; |
8636e53c | 445 | mqd->free_mqd = free_mqd_hiq_sdma; |
a439b890 | 446 | mqd->load_mqd = kfd_load_mqd_sdma; |
4b8f589b | 447 | mqd->update_mqd = update_mqd_sdma; |
a439b890 MJ |
448 | mqd->destroy_mqd = kfd_destroy_mqd_sdma; |
449 | mqd->is_occupied = kfd_is_occupied_sdma; | |
42c6c482 DYS |
450 | mqd->checkpoint_mqd = checkpoint_mqd_sdma; |
451 | mqd->restore_mqd = restore_mqd_sdma; | |
6c6cde55 | 452 | mqd->mqd_size = sizeof(struct cik_sdma_rlc_registers); |
2f77b9a2 | 453 | mqd->mqd_stride = kfd_mqd_stride; |
851a645e FK |
454 | #if defined(CONFIG_DEBUG_FS) |
455 | mqd->debugfs_show_mqd = debugfs_show_mqd_sdma; | |
456 | #endif | |
4b8f589b BG |
457 | break; |
458 | default: | |
459 | kfree(mqd); | |
460 | return NULL; | |
461 | } | |
462 | ||
463 | return mqd; | |
464 | } |