projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/amdgpu: Fix MMIO HDP flush on SRIOV
[linux-block.git] / drivers / gpu / drm / amd / amdkfd / kfd_device_queue_manager.c
CommitLineData
64c7f8cf
BG		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
26103436
FK		 24#include <linux/ratelimit.h>
25#include <linux/printk.h>
64c7f8cf
BG		 26#include <linux/slab.h>
27#include <linux/list.h>
28#include <linux/types.h>
64c7f8cf 29#include <linux/bitops.h>
99331a51 30#include <linux/sched.h>
64c7f8cf
BG		 31#include "kfd_priv.h"
32#include "kfd_device_queue_manager.h"
33#include "kfd_mqd_manager.h"
34#include "cik_regs.h"
35#include "kfd_kernel_queue.h"
5b87245f 36#include "amdgpu_amdkfd.h"
64c7f8cf
BG		 37
38/* Size of the per-pipe EOP queue */
39#define CIK_HPD_EOP_BYTES_LOG2 11
40#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
41
64c7f8cf 42static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
c7b6bac9 43 u32 pasid, unsigned int vmid);
64c7f8cf 44
c4744e24
YZ		 45static int execute_queues_cpsch(struct device_queue_manager *dqm,
46 enum kfd_unmap_queues_filter filter,
47 uint32_t filter_param);
7da2bcf8 48static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 49 enum kfd_unmap_queues_filter filter,
50 uint32_t filter_param);
64c7f8cf 51
60a00956
FK		 52static int map_queues_cpsch(struct device_queue_manager *dqm);
53
bcea3081 54static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1b4670f6 55 struct queue *q);
64c7f8cf 56
d39b7737
OZ		 57static inline void deallocate_hqd(struct device_queue_manager *dqm,
58 struct queue *q);
59static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
60static int allocate_sdma_queue(struct device_queue_manager *dqm,
61 struct queue *q);
73ea648d
SL		 62static void kfd_process_hw_exception(struct work_struct *work);
63
bcea3081
BG		 64static inline
65enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
64c7f8cf 66{
1b4670f6 67 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
85d258f9
BG		 68 return KFD_MQD_TYPE_SDMA;
69 return KFD_MQD_TYPE_CP;
64c7f8cf
BG		 70}
71
d0b63bb3
AR		 72static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
73{
74 int i;
0d801007
JC		 75 int pipe_offset = (mec * dqm->dev->shared_resources.num_pipe_per_mec
76 + pipe) * dqm->dev->shared_resources.num_queue_per_pipe;
d0b63bb3
AR		 77
78 /* queue is available for KFD usage if bit is 1 */
79 for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i)
80 if (test_bit(pipe_offset + i,
e6945304 81 dqm->dev->shared_resources.cp_queue_bitmap))
d0b63bb3
AR		 82 return true;
83 return false;
84}
85
e6945304 86unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
64ea8f4a 87{
e6945304 88 return bitmap_weight(dqm->dev->shared_resources.cp_queue_bitmap,
d0b63bb3 89 KGD_MAX_QUEUES);
64ea8f4a
OG		 90}
91
d0b63bb3 92unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
64c7f8cf 93{
d0b63bb3
AR		 94 return dqm->dev->shared_resources.num_queue_per_pipe;
95}
96
97unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
98{
d0b63bb3 99 return dqm->dev->shared_resources.num_pipe_per_mec;
64c7f8cf
BG		 100}
101
98bb9222
YZ		 102static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
103{
104 return dqm->dev->device_info->num_sdma_engines;
105}
106
1b4670f6
OZ		 107static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm)
108{
109 return dqm->dev->device_info->num_xgmi_sdma_engines;
110}
111
c7637c95
YZ		 112static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
113{
114 return get_num_sdma_engines(dqm) + get_num_xgmi_sdma_engines(dqm);
115}
116
98bb9222
YZ		 117unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
118{
119 return dqm->dev->device_info->num_sdma_engines
d5094189 120 * dqm->dev->device_info->num_sdma_queues_per_engine;
98bb9222
YZ		 121}
122
1b4670f6
OZ		 123unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
124{
125 return dqm->dev->device_info->num_xgmi_sdma_engines
126 * dqm->dev->device_info->num_sdma_queues_per_engine;
127}
128
a22fc854 129void program_sh_mem_settings(struct device_queue_manager *dqm,
64c7f8cf
BG		 130 struct qcm_process_device *qpd)
131{
cea405b1 132 return dqm->dev->kfd2kgd->program_sh_mem_settings(
3356c38d 133 dqm->dev->adev, qpd->vmid,
64c7f8cf
BG		 134 qpd->sh_mem_config,
135 qpd->sh_mem_ape1_base,
136 qpd->sh_mem_ape1_limit,
137 qpd->sh_mem_bases);
138}
139
204d8998 140static void increment_queue_count(struct device_queue_manager *dqm,
b42902f4
YZ		 141 enum kfd_queue_type type)
142{
143 dqm->active_queue_count++;
144 if (type == KFD_QUEUE_TYPE_COMPUTE || type == KFD_QUEUE_TYPE_DIQ)
145 dqm->active_cp_queue_count++;
146}
147
204d8998 148static void decrement_queue_count(struct device_queue_manager *dqm,
b42902f4
YZ		 149 enum kfd_queue_type type)
150{
151 dqm->active_queue_count--;
152 if (type == KFD_QUEUE_TYPE_COMPUTE || type == KFD_QUEUE_TYPE_DIQ)
153 dqm->active_cp_queue_count--;
154}
155
ef568db7
FK		 156static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
157{
158 struct kfd_dev *dev = qpd->dqm->dev;
159
dd0ae064 160 if (!KFD_IS_SOC15(dev)) {
ef568db7
FK		 161 /* On pre-SOC15 chips we need to use the queue ID to
162 * preserve the user mode ABI.
163 */
164 q->doorbell_id = q->properties.queue_id;
1b4670f6
OZ		 165 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
166 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
234441dd
YZ		 167 /* For SDMA queues on SOC15 with 8-byte doorbell, use static
168 * doorbell assignments based on the engine and queue id.
169 * The doobell index distance between RLC (2*i) and (2*i+1)
170 * for a SDMA engine is 512.
ef568db7 171 */
234441dd
YZ		 172 uint32_t *idx_offset =
173 dev->shared_resources.sdma_doorbell_idx;
174
175 q->doorbell_id = idx_offset[q->properties.sdma_engine_id]
176 + (q->properties.sdma_queue_id & 1)
177 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET
178 + (q->properties.sdma_queue_id >> 1);
ef568db7
FK		 179 } else {
180 /* For CP queues on SOC15 reserve a free doorbell ID */
181 unsigned int found;
182
183 found = find_first_zero_bit(qpd->doorbell_bitmap,
184 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
185 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
186 pr_debug("No doorbells available");
187 return -EBUSY;
188 }
189 set_bit(found, qpd->doorbell_bitmap);
190 q->doorbell_id = found;
191 }
192
193 q->properties.doorbell_off =
59d7115d 194 kfd_get_doorbell_dw_offset_in_bar(dev, qpd_to_pdd(qpd),
ef568db7 195 q->doorbell_id);
ef568db7
FK		 196 return 0;
197}
198
199static void deallocate_doorbell(struct qcm_process_device *qpd,
200 struct queue *q)
201{
202 unsigned int old;
203 struct kfd_dev *dev = qpd->dqm->dev;
204
dd0ae064 205 if (!KFD_IS_SOC15(dev) ||
1b4670f6
OZ		 206 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
207 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
ef568db7
FK		 208 return;
209
210 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
211 WARN_ON(!old);
212}
213
b53ef0df
MJ		 214static void program_trap_handler_settings(struct device_queue_manager *dqm,
215 struct qcm_process_device *qpd)
216{
217 if (dqm->dev->kfd2kgd->program_trap_handler_settings)
218 dqm->dev->kfd2kgd->program_trap_handler_settings(
3356c38d 219 dqm->dev->adev, qpd->vmid,
b53ef0df
MJ		 220 qpd->tba_addr, qpd->tma_addr);
221}
222
64c7f8cf
BG		 223static int allocate_vmid(struct device_queue_manager *dqm,
224 struct qcm_process_device *qpd,
225 struct queue *q)
226{
d9d4623c 227 int allocated_vmid = -1, i;
64c7f8cf 228
d9d4623c
YZ		 229 for (i = dqm->dev->vm_info.first_vmid_kfd;
230 i <= dqm->dev->vm_info.last_vmid_kfd; i++) {
231 if (!dqm->vmid_pasid[i]) {
232 allocated_vmid = i;
233 break;
234 }
235 }
236
237 if (allocated_vmid < 0) {
238 pr_err("no more vmid to allocate\n");
239 return -ENOSPC;
240 }
241
242 pr_debug("vmid allocated: %d\n", allocated_vmid);
243
244 dqm->vmid_pasid[allocated_vmid] = q->process->pasid;
64c7f8cf 245
d9d4623c 246 set_pasid_vmid_mapping(dqm, q->process->pasid, allocated_vmid);
64c7f8cf 247
64c7f8cf
BG		 248 qpd->vmid = allocated_vmid;
249 q->properties.vmid = allocated_vmid;
250
64c7f8cf
BG		 251 program_sh_mem_settings(dqm, qpd);
252
046e674b 253 if (KFD_IS_SOC15(dqm->dev) && dqm->dev->cwsr_enabled)
b53ef0df
MJ		 254 program_trap_handler_settings(dqm, qpd);
255
403575c4
FK		 256 /* qpd->page_table_base is set earlier when register_process()
257 * is called, i.e. when the first queue is created.
258 */
3356c38d 259 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->adev,
403575c4
FK		 260 qpd->vmid,
261 qpd->page_table_base);
262 /* invalidate the VM context after pasid and vmid mapping is set up */
3543b055 263 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
403575c4 264
c637b36a 265 if (dqm->dev->kfd2kgd->set_scratch_backing_va)
3356c38d 266 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev,
c637b36a 267 qpd->sh_hidden_private_base, qpd->vmid);
d39b7737 268
64c7f8cf
BG		 269 return 0;
270}
271
552764b6
FK		 272static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
273 struct qcm_process_device *qpd)
274{
9af5379c 275 const struct packet_manager_funcs *pmf = qpd->dqm->packet_mgr.pmf;
f6e27ff1 276 int ret;
552764b6
FK		 277
278 if (!qpd->ib_kaddr)
279 return -ENOMEM;
280
f6e27ff1
FK		 281 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
282 if (ret)
283 return ret;
552764b6 284
6bfc7c7e 285 return amdgpu_amdkfd_submit_ib(kdev->adev, KGD_ENGINE_MEC1, qpd->vmid,
f6e27ff1
FK		 286 qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
287 pmf->release_mem_size / sizeof(uint32_t));
552764b6
FK		 288}
289
64c7f8cf
BG		 290static void deallocate_vmid(struct device_queue_manager *dqm,
291 struct qcm_process_device *qpd,
292 struct queue *q)
293{
552764b6 294 /* On GFX v7, CP doesn't flush TC at dequeue */
7eb0502a 295 if (q->device->adev->asic_type == CHIP_HAWAII)
552764b6
FK		 296 if (flush_texture_cache_nocpsch(q->device, qpd))
297 pr_err("Failed to flush TC\n");
298
3543b055 299 kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY);
403575c4 300
2030664b
BG		 301 /* Release the vmid mapping */
302 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
d9d4623c 303 dqm->vmid_pasid[qpd->vmid] = 0;
2030664b 304
64c7f8cf
BG		 305 qpd->vmid = 0;
306 q->properties.vmid = 0;
307}
308
309static int create_queue_nocpsch(struct device_queue_manager *dqm,
310 struct queue *q,
b46cb7d7 311 struct qcm_process_device *qpd)
64c7f8cf 312{
d39b7737 313 struct mqd_manager *mqd_mgr;
64c7f8cf
BG		 314 int retval;
315
efeaed4d 316 dqm_lock(dqm);
64c7f8cf 317
b8cbab04 318 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 319 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04 320 dqm->total_queue_count);
ab7c1648
KR		 321 retval = -EPERM;
322 goto out_unlock;
b8cbab04
OG		 323 }
324
64c7f8cf
BG		 325 if (list_empty(&qpd->queues_list)) {
326 retval = allocate_vmid(dqm, qpd, q);
ab7c1648
KR		 327 if (retval)
328 goto out_unlock;
64c7f8cf 329 }
64c7f8cf 330 q->properties.vmid = qpd->vmid;
26103436 331 /*
bb2d2128
FK		 332 * Eviction state logic: mark all queues as evicted, even ones
333 * not currently active. Restoring inactive queues later only
334 * updates the is_evicted flag but is a no-op otherwise.
26103436 335 */
bb2d2128 336 q->properties.is_evicted = !!qpd->evicted;
64c7f8cf 337
373d7080
FK		 338 q->properties.tba_addr = qpd->tba_addr;
339 q->properties.tma_addr = qpd->tma_addr;
340
d091bc0a
OZ		 341 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
342 q->properties.type)];
d39b7737
OZ		 343 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
344 retval = allocate_hqd(dqm, q);
345 if (retval)
346 goto deallocate_vmid;
347 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
348 q->pipe, q->queue);
349 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
350 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
351 retval = allocate_sdma_queue(dqm, q);
352 if (retval)
353 goto deallocate_vmid;
354 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
355 }
356
357 retval = allocate_doorbell(qpd, q);
358 if (retval)
359 goto out_deallocate_hqd;
360
6a6ef5ee
OZ		 361 /* Temporarily release dqm lock to avoid a circular lock dependency */
362 dqm_unlock(dqm);
d091bc0a 363 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
6a6ef5ee
OZ		 364 dqm_lock(dqm);
365
d091bc0a
OZ		 366 if (!q->mqd_mem_obj) {
367 retval = -ENOMEM;
368 goto out_deallocate_doorbell;
369 }
8636e53c
OZ		 370 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
371 &q->gart_mqd_addr, &q->properties);
d39b7737 372 if (q->properties.is_active) {
2c99a547
PY		 373 if (!dqm->sched_running) {
374 WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
375 goto add_queue_to_list;
376 }
d39b7737
OZ		 377
378 if (WARN(q->process->mm != current->mm,
379 "should only run in user thread"))
380 retval = -EFAULT;
381 else
382 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
383 q->queue, &q->properties, current->mm);
384 if (retval)
d091bc0a 385 goto out_free_mqd;
64c7f8cf
BG		 386 }
387
2c99a547 388add_queue_to_list:
64c7f8cf 389 list_add(&q->list, &qpd->queues_list);
bc920fd4 390 qpd->queue_count++;
b6819cec 391 if (q->properties.is_active)
b42902f4 392 increment_queue_count(dqm, q->properties.type);
64c7f8cf 393
b8cbab04
OG		 394 /*
395 * Unconditionally increment this counter, regardless of the queue's
396 * type or whether the queue is active.
397 */
398 dqm->total_queue_count++;
399 pr_debug("Total of %d queues are accountable so far\n",
400 dqm->total_queue_count);
d091bc0a 401 goto out_unlock;
b8cbab04 402
d091bc0a
OZ		 403out_free_mqd:
404 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
d39b7737
OZ		 405out_deallocate_doorbell:
406 deallocate_doorbell(qpd, q);
407out_deallocate_hqd:
408 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
409 deallocate_hqd(dqm, q);
410 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
411 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
412 deallocate_sdma_queue(dqm, q);
413deallocate_vmid:
414 if (list_empty(&qpd->queues_list))
415 deallocate_vmid(dqm, qpd, q);
ab7c1648 416out_unlock:
efeaed4d 417 dqm_unlock(dqm);
ab7c1648 418 return retval;
64c7f8cf
BG		 419}
420
421static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
422{
423 bool set;
f0ec5b99 424 int pipe, bit, i;
64c7f8cf
BG		 425
426 set = false;
427
8eabaf54
KR		 428 for (pipe = dqm->next_pipe_to_allocate, i = 0;
429 i < get_pipes_per_mec(dqm);
d0b63bb3
AR		 430 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
431
432 if (!is_pipe_enabled(dqm, 0, pipe))
433 continue;
434
64c7f8cf 435 if (dqm->allocated_queues[pipe] != 0) {
4252bf68
HK		 436 bit = ffs(dqm->allocated_queues[pipe]) - 1;
437 dqm->allocated_queues[pipe] &= ~(1 << bit);
64c7f8cf
BG		 438 q->pipe = pipe;
439 q->queue = bit;
440 set = true;
441 break;
442 }
443 }
444
991ca8ee 445 if (!set)
64c7f8cf
BG		 446 return -EBUSY;
447
79775b62 448 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
64c7f8cf 449 /* horizontal hqd allocation */
d0b63bb3 450 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
64c7f8cf
BG		 451
452 return 0;
453}
454
455static inline void deallocate_hqd(struct device_queue_manager *dqm,
456 struct queue *q)
457{
4252bf68 458 dqm->allocated_queues[q->pipe] |= (1 << q->queue);
64c7f8cf
BG		 459}
460
9fd3f1bf
FK		 461/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
462 * to avoid asynchronized access
463 */
464static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
64c7f8cf
BG		 465 struct qcm_process_device *qpd,
466 struct queue *q)
467{
468 int retval;
8d5f3552 469 struct mqd_manager *mqd_mgr;
64c7f8cf 470
fdfa090b
OZ		 471 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
472 q->properties.type)];
64c7f8cf 473
c7637c95 474 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
c2e1b3a4 475 deallocate_hqd(dqm, q);
c7637c95 476 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1b4670f6 477 deallocate_sdma_queue(dqm, q);
c7637c95 478 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1b4670f6 479 deallocate_sdma_queue(dqm, q);
c7637c95 480 else {
79775b62 481 pr_debug("q->properties.type %d is invalid\n",
7113cd65 482 q->properties.type);
9fd3f1bf 483 return -EINVAL;
64c7f8cf 484 }
9fd3f1bf 485 dqm->total_queue_count--;
64c7f8cf 486
ef568db7
FK		 487 deallocate_doorbell(qpd, q);
488
2c99a547
PY		 489 if (!dqm->sched_running) {
490 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
491 return 0;
492 }
493
8d5f3552 494 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
c2e1b3a4 495 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
b90e3fbe 496 KFD_UNMAP_LATENCY_MS,
64c7f8cf 497 q->pipe, q->queue);
9fd3f1bf
FK		 498 if (retval == -ETIME)
499 qpd->reset_wavefronts = true;
64c7f8cf 500
64c7f8cf 501 list_del(&q->list);
9fd3f1bf
FK		 502 if (list_empty(&qpd->queues_list)) {
503 if (qpd->reset_wavefronts) {
504 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
505 dqm->dev);
506 /* dbgdev_wave_reset_wavefronts has to be called before
507 * deallocate_vmid(), i.e. when vmid is still in use.
508 */
509 dbgdev_wave_reset_wavefronts(dqm->dev,
510 qpd->pqm->process);
511 qpd->reset_wavefronts = false;
512 }
513
64c7f8cf 514 deallocate_vmid(dqm, qpd, q);
9fd3f1bf 515 }
bc920fd4 516 qpd->queue_count--;
b8020b03 517 if (q->properties.is_active) {
b42902f4 518 decrement_queue_count(dqm, q->properties.type);
b8020b03
JG		 519 if (q->properties.is_gws) {
520 dqm->gws_queue_count--;
521 qpd->mapped_gws_queue = false;
522 }
523 }
b8cbab04 524
9fd3f1bf
FK		 525 return retval;
526}
b8cbab04 527
9fd3f1bf
FK		 528static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
529 struct qcm_process_device *qpd,
530 struct queue *q)
531{
532 int retval;
d69fd951
MJ		 533 uint64_t sdma_val = 0;
534 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
a7b2451d
AL		 535 struct mqd_manager *mqd_mgr =
536 dqm->mqd_mgrs[get_mqd_type_from_queue_type(q->properties.type)];
d69fd951
MJ		 537
538 /* Get the SDMA queue stats */
539 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
540 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
818b0324 541 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
d69fd951
MJ		 542 &sdma_val);
543 if (retval)
544 pr_err("Failed to read SDMA queue counter for queue: %d\n",
545 q->properties.queue_id);
546 }
9fd3f1bf 547
efeaed4d 548 dqm_lock(dqm);
9fd3f1bf 549 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
d69fd951
MJ		 550 if (!retval)
551 pdd->sdma_past_activity_counter += sdma_val;
efeaed4d 552 dqm_unlock(dqm);
9fd3f1bf 553
a7b2451d
AL		 554 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
555
64c7f8cf
BG		 556 return retval;
557}
558
c6e559eb
LY		 559static int update_queue(struct device_queue_manager *dqm, struct queue *q,
560 struct mqd_update_info *minfo)
64c7f8cf 561{
8636e53c 562 int retval = 0;
8d5f3552 563 struct mqd_manager *mqd_mgr;
26103436 564 struct kfd_process_device *pdd;
b6ffbab8 565 bool prev_active = false;
64c7f8cf 566
efeaed4d 567 dqm_lock(dqm);
26103436
FK		 568 pdd = kfd_get_process_device_data(q->device, q->process);
569 if (!pdd) {
570 retval = -ENODEV;
571 goto out_unlock;
572 }
fdfa090b
OZ		 573 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
574 q->properties.type)];
64c7f8cf 575
60a00956
FK		 576 /* Save previous activity state for counters */
577 prev_active = q->properties.is_active;
578
579 /* Make sure the queue is unmapped before updating the MQD */
d146c5a7 580 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
60a00956
FK		 581 retval = unmap_queues_cpsch(dqm,
582 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
894a8293 583 if (retval) {
60a00956
FK		 584 pr_err("unmap queue failed\n");
585 goto out_unlock;
586 }
894a8293 587 } else if (prev_active &&
60a00956 588 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1b4670f6
OZ		 589 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
590 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2c99a547
PY		 591
592 if (!dqm->sched_running) {
593 WARN_ONCE(1, "Update non-HWS queue while stopped\n");
594 goto out_unlock;
595 }
596
8d5f3552 597 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
b53ef0df
MJ		 598 (dqm->dev->cwsr_enabled?
599 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE:
600 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
60a00956
FK		 601 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
602 if (retval) {
603 pr_err("destroy mqd failed\n");
604 goto out_unlock;
605 }
606 }
607
c6e559eb 608 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties, minfo);
60a00956 609
096d1a3e
FK		 610 /*
611 * check active state vs. the previous state and modify
612 * counter accordingly. map_queues_cpsch uses the
81b820b3 613 * dqm->active_queue_count to determine whether a new runlist must be
096d1a3e
FK		 614 * uploaded.
615 */
616 if (q->properties.is_active && !prev_active)
b42902f4 617 increment_queue_count(dqm, q->properties.type);
096d1a3e 618 else if (!q->properties.is_active && prev_active)
b42902f4 619 decrement_queue_count(dqm, q->properties.type);
096d1a3e 620
b8020b03
JG		 621 if (q->gws && !q->properties.is_gws) {
622 if (q->properties.is_active) {
623 dqm->gws_queue_count++;
624 pdd->qpd.mapped_gws_queue = true;
625 }
626 q->properties.is_gws = true;
627 } else if (!q->gws && q->properties.is_gws) {
628 if (q->properties.is_active) {
629 dqm->gws_queue_count--;
630 pdd->qpd.mapped_gws_queue = false;
631 }
632 q->properties.is_gws = false;
633 }
634
d146c5a7 635 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
60a00956 636 retval = map_queues_cpsch(dqm);
894a8293 637 else if (q->properties.is_active &&
60a00956 638 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1b4670f6
OZ		 639 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
640 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1b19aa5a
FK		 641 if (WARN(q->process->mm != current->mm,
642 "should only run in user thread"))
643 retval = -EFAULT;
644 else
645 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
646 q->pipe, q->queue,
647 &q->properties, current->mm);
648 }
b6ffbab8 649
ab7c1648 650out_unlock:
efeaed4d 651 dqm_unlock(dqm);
64c7f8cf
BG		 652 return retval;
653}
654
26103436
FK		 655static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
656 struct qcm_process_device *qpd)
657{
658 struct queue *q;
8d5f3552 659 struct mqd_manager *mqd_mgr;
26103436 660 struct kfd_process_device *pdd;
bb2d2128 661 int retval, ret = 0;
26103436 662
efeaed4d 663 dqm_lock(dqm);
26103436
FK		 664 if (qpd->evicted++ > 0) /* already evicted, do nothing */
665 goto out;
666
667 pdd = qpd_to_pdd(qpd);
783a25f4 668 pr_debug_ratelimited("Evicting PASID 0x%x queues\n",
26103436
FK		 669 pdd->process->pasid);
670
4327bed2 671 pdd->last_evict_timestamp = get_jiffies_64();
bb2d2128
FK		 672 /* Mark all queues as evicted. Deactivate all active queues on
673 * the qpd.
674 */
26103436 675 list_for_each_entry(q, &qpd->queues_list, list) {
bb2d2128 676 q->properties.is_evicted = true;
26103436
FK		 677 if (!q->properties.is_active)
678 continue;
bb2d2128 679
fdfa090b
OZ		 680 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
681 q->properties.type)];
26103436 682 q->properties.is_active = false;
b42902f4 683 decrement_queue_count(dqm, q->properties.type);
b8020b03
JG		 684 if (q->properties.is_gws) {
685 dqm->gws_queue_count--;
686 qpd->mapped_gws_queue = false;
687 }
2c99a547
PY		 688
689 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
690 continue;
691
8d5f3552 692 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
b53ef0df
MJ		 693 (dqm->dev->cwsr_enabled?
694 KFD_PREEMPT_TYPE_WAVEFRONT_SAVE:
695 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN),
26103436 696 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
bb2d2128
FK		 697 if (retval && !ret)
698 /* Return the first error, but keep going to
699 * maintain a consistent eviction state
700 */
701 ret = retval;
26103436
FK		 702 }
703
704out:
efeaed4d 705 dqm_unlock(dqm);
bb2d2128 706 return ret;
26103436
FK		 707}
708
709static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
710 struct qcm_process_device *qpd)
711{
712 struct queue *q;
713 struct kfd_process_device *pdd;
714 int retval = 0;
715
efeaed4d 716 dqm_lock(dqm);
26103436
FK		 717 if (qpd->evicted++ > 0) /* already evicted, do nothing */
718 goto out;
719
720 pdd = qpd_to_pdd(qpd);
783a25f4 721 pr_debug_ratelimited("Evicting PASID 0x%x queues\n",
26103436
FK		 722 pdd->process->pasid);
723
bb2d2128
FK		 724 /* Mark all queues as evicted. Deactivate all active queues on
725 * the qpd.
726 */
26103436 727 list_for_each_entry(q, &qpd->queues_list, list) {
bb2d2128 728 q->properties.is_evicted = true;
26103436
FK		 729 if (!q->properties.is_active)
730 continue;
bb2d2128 731
26103436 732 q->properties.is_active = false;
b42902f4 733 decrement_queue_count(dqm, q->properties.type);
26103436 734 }
4327bed2 735 pdd->last_evict_timestamp = get_jiffies_64();
26103436
FK		 736 retval = execute_queues_cpsch(dqm,
737 qpd->is_debug ?
738 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
739 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
740
741out:
efeaed4d 742 dqm_unlock(dqm);
26103436
FK		 743 return retval;
744}
745
746static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
747 struct qcm_process_device *qpd)
748{
1b19aa5a 749 struct mm_struct *mm = NULL;
26103436 750 struct queue *q;
8d5f3552 751 struct mqd_manager *mqd_mgr;
26103436 752 struct kfd_process_device *pdd;
e715c6d0 753 uint64_t pd_base;
4327bed2 754 uint64_t eviction_duration;
bb2d2128 755 int retval, ret = 0;
26103436
FK		 756
757 pdd = qpd_to_pdd(qpd);
758 /* Retrieve PD base */
b40a6ab2 759 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
26103436 760
efeaed4d 761 dqm_lock(dqm);
26103436
FK		 762 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
763 goto out;
764 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
765 qpd->evicted--;
766 goto out;
767 }
768
783a25f4 769 pr_debug_ratelimited("Restoring PASID 0x%x queues\n",
26103436
FK		 770 pdd->process->pasid);
771
772 /* Update PD Base in QPD */
773 qpd->page_table_base = pd_base;
e715c6d0 774 pr_debug("Updated PD address to 0x%llx\n", pd_base);
26103436
FK		 775
776 if (!list_empty(&qpd->queues_list)) {
777 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
3356c38d 778 dqm->dev->adev,
26103436
FK		 779 qpd->vmid,
780 qpd->page_table_base);
3543b055 781 kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY);
26103436
FK		 782 }
783
1b19aa5a
FK		 784 /* Take a safe reference to the mm_struct, which may otherwise
785 * disappear even while the kfd_process is still referenced.
786 */
787 mm = get_task_mm(pdd->process->lead_thread);
788 if (!mm) {
bb2d2128 789 ret = -EFAULT;
1b19aa5a
FK		 790 goto out;
791 }
792
bb2d2128
FK		 793 /* Remove the eviction flags. Activate queues that are not
794 * inactive for other reasons.
795 */
26103436 796 list_for_each_entry(q, &qpd->queues_list, list) {
bb2d2128
FK		 797 q->properties.is_evicted = false;
798 if (!QUEUE_IS_ACTIVE(q->properties))
26103436 799 continue;
bb2d2128 800
fdfa090b
OZ		 801 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
802 q->properties.type)];
26103436 803 q->properties.is_active = true;
b42902f4 804 increment_queue_count(dqm, q->properties.type);
b8020b03
JG		 805 if (q->properties.is_gws) {
806 dqm->gws_queue_count++;
807 qpd->mapped_gws_queue = true;
808 }
2c99a547
PY		 809
810 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
811 continue;
812
8d5f3552 813 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
1b19aa5a 814 q->queue, &q->properties, mm);
bb2d2128
FK		 815 if (retval && !ret)
816 /* Return the first error, but keep going to
817 * maintain a consistent eviction state
818 */
819 ret = retval;
26103436
FK		 820 }
821 qpd->evicted = 0;
4327bed2
PC		 822 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
823 atomic64_add(eviction_duration, &pdd->evict_duration_counter);
26103436 824out:
1b19aa5a
FK		 825 if (mm)
826 mmput(mm);
efeaed4d 827 dqm_unlock(dqm);
bb2d2128 828 return ret;
26103436
FK		 829}
830
831static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
832 struct qcm_process_device *qpd)
833{
834 struct queue *q;
835 struct kfd_process_device *pdd;
e715c6d0 836 uint64_t pd_base;
4327bed2 837 uint64_t eviction_duration;
26103436
FK		 838 int retval = 0;
839
840 pdd = qpd_to_pdd(qpd);
841 /* Retrieve PD base */
b40a6ab2 842 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
26103436 843
efeaed4d 844 dqm_lock(dqm);
26103436
FK		 845 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
846 goto out;
847 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
848 qpd->evicted--;
849 goto out;
850 }
851
783a25f4 852 pr_debug_ratelimited("Restoring PASID 0x%x queues\n",
26103436
FK		 853 pdd->process->pasid);
854
855 /* Update PD Base in QPD */
856 qpd->page_table_base = pd_base;
e715c6d0 857 pr_debug("Updated PD address to 0x%llx\n", pd_base);
26103436
FK		 858
859 /* activate all active queues on the qpd */
860 list_for_each_entry(q, &qpd->queues_list, list) {
26103436 861 q->properties.is_evicted = false;
bb2d2128
FK		 862 if (!QUEUE_IS_ACTIVE(q->properties))
863 continue;
864
26103436 865 q->properties.is_active = true;
b42902f4 866 increment_queue_count(dqm, q->properties.type);
26103436
FK		 867 }
868 retval = execute_queues_cpsch(dqm,
869 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
bb2d2128 870 qpd->evicted = 0;
4327bed2
PC		 871 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
872 atomic64_add(eviction_duration, &pdd->evict_duration_counter);
26103436 873out:
efeaed4d 874 dqm_unlock(dqm);
26103436
FK		 875 return retval;
876}
877
58dcd5bf 878static int register_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 879 struct qcm_process_device *qpd)
880{
881 struct device_process_node *n;
403575c4 882 struct kfd_process_device *pdd;
e715c6d0 883 uint64_t pd_base;
a22fc854 884 int retval;
64c7f8cf 885
dbf56ab1 886 n = kzalloc(sizeof(*n), GFP_KERNEL);
64c7f8cf
BG		 887 if (!n)
888 return -ENOMEM;
889
890 n->qpd = qpd;
891
403575c4
FK		 892 pdd = qpd_to_pdd(qpd);
893 /* Retrieve PD base */
b40a6ab2 894 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
403575c4 895
efeaed4d 896 dqm_lock(dqm);
64c7f8cf
BG		 897 list_add(&n->list, &dqm->queues);
898
403575c4
FK		 899 /* Update PD Base in QPD */
900 qpd->page_table_base = pd_base;
e715c6d0 901 pr_debug("Updated PD address to 0x%llx\n", pd_base);
403575c4 902
bfd5e378 903 retval = dqm->asic_ops.update_qpd(dqm, qpd);
a22fc854 904
f756e631 905 dqm->processes_count++;
64c7f8cf 906
efeaed4d 907 dqm_unlock(dqm);
64c7f8cf 908
32cce8bc
FK		 909 /* Outside the DQM lock because under the DQM lock we can't do
910 * reclaim or take other locks that others hold while reclaiming.
911 */
912 kfd_inc_compute_active(dqm->dev);
913
a22fc854 914 return retval;
64c7f8cf
BG		 915}
916
58dcd5bf 917static int unregister_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 918 struct qcm_process_device *qpd)
919{
920 int retval;
921 struct device_process_node *cur, *next;
922
1e5ec956
OG		 923 pr_debug("qpd->queues_list is %s\n",
924 list_empty(&qpd->queues_list) ? "empty" : "not empty");
64c7f8cf
BG		 925
926 retval = 0;
efeaed4d 927 dqm_lock(dqm);
64c7f8cf
BG		 928
929 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
930 if (qpd == cur->qpd) {
931 list_del(&cur->list);
f5d896bb 932 kfree(cur);
f756e631 933 dqm->processes_count--;
64c7f8cf
BG		 934 goto out;
935 }
936 }
937 /* qpd not found in dqm list */
938 retval = 1;
939out:
efeaed4d 940 dqm_unlock(dqm);
32cce8bc
FK		 941
942 /* Outside the DQM lock because under the DQM lock we can't do
943 * reclaim or take other locks that others hold while reclaiming.
944 */
945 if (!retval)
946 kfd_dec_compute_active(dqm->dev);
947
64c7f8cf
BG		 948 return retval;
949}
950
951static int
c7b6bac9 952set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
64c7f8cf
BG		 953 unsigned int vmid)
954{
cea405b1 955 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
3356c38d 956 dqm->dev->adev, pasid, vmid);
64c7f8cf
BG		 957}
958
2249d558
AL		 959static void init_interrupts(struct device_queue_manager *dqm)
960{
961 unsigned int i;
962
d0b63bb3
AR		 963 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
964 if (is_pipe_enabled(dqm, 0, i))
3356c38d 965 dqm->dev->kfd2kgd->init_interrupts(dqm->dev->adev, i);
2249d558
AL		 966}
967
64c7f8cf
BG		 968static int initialize_nocpsch(struct device_queue_manager *dqm)
969{
86194cf8 970 int pipe, queue;
64c7f8cf 971
79775b62 972 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf 973
ab7c1648
KR		 974 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
975 sizeof(unsigned int), GFP_KERNEL);
976 if (!dqm->allocated_queues)
977 return -ENOMEM;
978
efeaed4d 979 mutex_init(&dqm->lock_hidden);
64c7f8cf 980 INIT_LIST_HEAD(&dqm->queues);
81b820b3 981 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;
b42902f4 982 dqm->active_cp_queue_count = 0;
b8020b03 983 dqm->gws_queue_count = 0;
64c7f8cf 984
86194cf8
FK		 985 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
986 int pipe_offset = pipe * get_queues_per_pipe(dqm);
987
988 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
989 if (test_bit(pipe_offset + queue,
e6945304 990 dqm->dev->shared_resources.cp_queue_bitmap))
86194cf8
FK		 991 dqm->allocated_queues[pipe] |= 1 << queue;
992 }
64c7f8cf 993
d9d4623c
YZ		 994 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
995
35cdc81b
OZ		 996 dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm));
997 dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm));
64c7f8cf 998
64c7f8cf
BG		 999 return 0;
1000}
1001
58dcd5bf 1002static void uninitialize(struct device_queue_manager *dqm)
64c7f8cf 1003{
6f9d54fd
OG		 1004 int i;
1005
81b820b3 1006 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);
64c7f8cf
BG		 1007
1008 kfree(dqm->allocated_queues);
6f9d54fd 1009 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
8d5f3552 1010 kfree(dqm->mqd_mgrs[i]);
efeaed4d 1011 mutex_destroy(&dqm->lock_hidden);
64c7f8cf
BG		 1012}
1013
1014static int start_nocpsch(struct device_queue_manager *dqm)
1015{
52055039 1016 pr_info("SW scheduler is used");
2249d558 1017 init_interrupts(dqm);
424b5442 1018
7eb0502a 1019 if (dqm->dev->adev->asic_type == CHIP_HAWAII)
9af5379c 1020 return pm_init(&dqm->packet_mgr, dqm);
2c99a547
PY		 1021 dqm->sched_running = true;
1022
424b5442 1023 return 0;
64c7f8cf
BG		 1024}
1025
1026static int stop_nocpsch(struct device_queue_manager *dqm)
1027{
7eb0502a 1028 if (dqm->dev->adev->asic_type == CHIP_HAWAII)
9af5379c 1029 pm_uninit(&dqm->packet_mgr, false);
2c99a547
PY		 1030 dqm->sched_running = false;
1031
64c7f8cf
BG		 1032 return 0;
1033}
1034
09c34e8d
FK		 1035static void pre_reset(struct device_queue_manager *dqm)
1036{
1037 dqm_lock(dqm);
1038 dqm->is_resetting = true;
1039 dqm_unlock(dqm);
1040}
1041
bcea3081 1042static int allocate_sdma_queue(struct device_queue_manager *dqm,
e78579aa 1043 struct queue *q)
bcea3081
BG		 1044{
1045 int bit;
1046
1b4670f6 1047 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
c7637c95
YZ		 1048 if (dqm->sdma_bitmap == 0) {
1049 pr_err("No more SDMA queue to allocate\n");
1b4670f6 1050 return -ENOMEM;
c7637c95
YZ		 1051 }
1052
1b4670f6
OZ		 1053 bit = __ffs64(dqm->sdma_bitmap);
1054 dqm->sdma_bitmap &= ~(1ULL << bit);
1055 q->sdma_id = bit;
1056 q->properties.sdma_engine_id = q->sdma_id %
1057 get_num_sdma_engines(dqm);
1058 q->properties.sdma_queue_id = q->sdma_id /
1059 get_num_sdma_engines(dqm);
1060 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
c7637c95
YZ		 1061 if (dqm->xgmi_sdma_bitmap == 0) {
1062 pr_err("No more XGMI SDMA queue to allocate\n");
1b4670f6 1063 return -ENOMEM;
c7637c95 1064 }
1b4670f6
OZ		 1065 bit = __ffs64(dqm->xgmi_sdma_bitmap);
1066 dqm->xgmi_sdma_bitmap &= ~(1ULL << bit);
1067 q->sdma_id = bit;
1068 /* sdma_engine_id is sdma id including
1069 * both PCIe-optimized SDMAs and XGMI-
1070 * optimized SDMAs. The calculation below
1071 * assumes the first N engines are always
1072 * PCIe-optimized ones
1073 */
1074 q->properties.sdma_engine_id = get_num_sdma_engines(dqm) +
1075 q->sdma_id % get_num_xgmi_sdma_engines(dqm);
1076 q->properties.sdma_queue_id = q->sdma_id /
1077 get_num_xgmi_sdma_engines(dqm);
1078 }
e78579aa 1079
e78579aa
YZ		 1080 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
1081 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
bcea3081
BG		 1082
1083 return 0;
1084}
1085
1086static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1b4670f6 1087 struct queue *q)
bcea3081 1088{
1b4670f6
OZ		 1089 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1090 if (q->sdma_id >= get_num_sdma_queues(dqm))
1091 return;
1092 dqm->sdma_bitmap |= (1ULL << q->sdma_id);
1093 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1094 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
1095 return;
1096 dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id);
1097 }
bcea3081
BG		 1098}
1099
64c7f8cf
BG		 1100/*
1101 * Device Queue Manager implementation for cp scheduler
1102 */
1103
1104static int set_sched_resources(struct device_queue_manager *dqm)
1105{
d0b63bb3 1106 int i, mec;
64c7f8cf 1107 struct scheduling_resources res;
64c7f8cf 1108
44008d7a 1109 res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
d0b63bb3
AR		 1110
1111 res.queue_mask = 0;
1112 for (i = 0; i < KGD_MAX_QUEUES; ++i) {
1113 mec = (i / dqm->dev->shared_resources.num_queue_per_pipe)
1114 / dqm->dev->shared_resources.num_pipe_per_mec;
1115
e6945304 1116 if (!test_bit(i, dqm->dev->shared_resources.cp_queue_bitmap))
d0b63bb3
AR		 1117 continue;
1118
1119 /* only acquire queues from the first MEC */
1120 if (mec > 0)
1121 continue;
1122
1123 /* This situation may be hit in the future if a new HW
1124 * generation exposes more than 64 queues. If so, the
8eabaf54
KR		 1125 * definition of res.queue_mask needs updating
1126 */
1d11ee89 1127 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
d0b63bb3
AR		 1128 pr_err("Invalid queue enabled by amdgpu: %d\n", i);
1129 break;
1130 }
1131
d09f85d5
YZ		 1132 res.queue_mask |= 1ull
1133 << amdgpu_queue_mask_bit_to_set_resource_bit(
56c5977e 1134 dqm->dev->adev, i);
d0b63bb3 1135 }
d9848e14
OZ		 1136 res.gws_mask = ~0ull;
1137 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
64c7f8cf 1138
79775b62
KR		 1139 pr_debug("Scheduling resources:\n"
1140 "vmid mask: 0x%8X\n"
1141 "queue mask: 0x%8llX\n",
64c7f8cf
BG		 1142 res.vmid_mask, res.queue_mask);
1143
9af5379c 1144 return pm_send_set_resources(&dqm->packet_mgr, &res);
64c7f8cf
BG		 1145}
1146
1147static int initialize_cpsch(struct device_queue_manager *dqm)
1148{
50e2fc36
AJ		 1149 uint64_t num_sdma_queues;
1150 uint64_t num_xgmi_sdma_queues;
1151
79775b62 1152 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf 1153
efeaed4d 1154 mutex_init(&dqm->lock_hidden);
64c7f8cf 1155 INIT_LIST_HEAD(&dqm->queues);
81b820b3 1156 dqm->active_queue_count = dqm->processes_count = 0;
b42902f4 1157 dqm->active_cp_queue_count = 0;
b8020b03 1158 dqm->gws_queue_count = 0;
64c7f8cf 1159 dqm->active_runlist = false;
50e2fc36
AJ		 1160
1161 num_sdma_queues = get_num_sdma_queues(dqm);
1162 if (num_sdma_queues >= BITS_PER_TYPE(dqm->sdma_bitmap))
1163 dqm->sdma_bitmap = ULLONG_MAX;
1164 else
1165 dqm->sdma_bitmap = (BIT_ULL(num_sdma_queues) - 1);
1166
1167 num_xgmi_sdma_queues = get_num_xgmi_sdma_queues(dqm);
1168 if (num_xgmi_sdma_queues >= BITS_PER_TYPE(dqm->xgmi_sdma_bitmap))
1169 dqm->xgmi_sdma_bitmap = ULLONG_MAX;
1170 else
1171 dqm->xgmi_sdma_bitmap = (BIT_ULL(num_xgmi_sdma_queues) - 1);
64c7f8cf 1172
73ea648d
SL		 1173 INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
1174
bfd5e378 1175 return 0;
64c7f8cf
BG		 1176}
1177
1178static int start_cpsch(struct device_queue_manager *dqm)
1179{
64c7f8cf
BG		 1180 int retval;
1181
64c7f8cf
BG		 1182 retval = 0;
1183
4f942aae 1184 dqm_lock(dqm);
9af5379c 1185 retval = pm_init(&dqm->packet_mgr, dqm);
4eacc26b 1186 if (retval)
64c7f8cf
BG		 1187 goto fail_packet_manager_init;
1188
1189 retval = set_sched_resources(dqm);
4eacc26b 1190 if (retval)
64c7f8cf
BG		 1191 goto fail_set_sched_resources;
1192
79775b62 1193 pr_debug("Allocating fence memory\n");
64c7f8cf
BG		 1194
1195 /* allocate fence memory on the gart */
a86aa3ca
OG		 1196 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
1197 &dqm->fence_mem);
64c7f8cf 1198
4eacc26b 1199 if (retval)
64c7f8cf
BG		 1200 goto fail_allocate_vidmem;
1201
b010affe 1202 dqm->fence_addr = (uint64_t *)dqm->fence_mem->cpu_ptr;
64c7f8cf 1203 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
2249d558
AL		 1204
1205 init_interrupts(dqm);
1206
73ea648d
SL		 1207 /* clear hang status when driver try to start the hw scheduler */
1208 dqm->is_hws_hang = false;
09c34e8d 1209 dqm->is_resetting = false;
2c99a547 1210 dqm->sched_running = true;
c4744e24 1211 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
efeaed4d 1212 dqm_unlock(dqm);
64c7f8cf
BG		 1213
1214 return 0;
1215fail_allocate_vidmem:
1216fail_set_sched_resources:
9af5379c 1217 pm_uninit(&dqm->packet_mgr, false);
64c7f8cf 1218fail_packet_manager_init:
4f942aae 1219 dqm_unlock(dqm);
64c7f8cf
BG		 1220 return retval;
1221}
1222
1223static int stop_cpsch(struct device_queue_manager *dqm)
1224{
c2a77fde
FK		 1225 bool hanging;
1226
efeaed4d 1227 dqm_lock(dqm);
c96cb659 1228 if (!dqm->sched_running) {
1229 dqm_unlock(dqm);
1230 return 0;
1231 }
1232
c2a77fde
FK		 1233 if (!dqm->is_hws_hang)
1234 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
1235 hanging = dqm->is_hws_hang || dqm->is_resetting;
2c99a547 1236 dqm->sched_running = false;
64c7f8cf 1237
9af5379c 1238 pm_release_ib(&dqm->packet_mgr);
087d7641 1239
a86aa3ca 1240 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
9af5379c 1241 pm_uninit(&dqm->packet_mgr, hanging);
4f942aae 1242 dqm_unlock(dqm);
64c7f8cf
BG		 1243
1244 return 0;
1245}
1246
1247static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1248 struct kernel_queue *kq,
1249 struct qcm_process_device *qpd)
1250{
efeaed4d 1251 dqm_lock(dqm);
b8cbab04 1252 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1253 pr_warn("Can't create new kernel queue because %d queues were already created\n",
b8cbab04 1254 dqm->total_queue_count);
efeaed4d 1255 dqm_unlock(dqm);
b8cbab04
OG		 1256 return -EPERM;
1257 }
1258
1259 /*
1260 * Unconditionally increment this counter, regardless of the queue's
1261 * type or whether the queue is active.
1262 */
1263 dqm->total_queue_count++;
1264 pr_debug("Total of %d queues are accountable so far\n",
1265 dqm->total_queue_count);
1266
64c7f8cf 1267 list_add(&kq->list, &qpd->priv_queue_list);
b42902f4 1268 increment_queue_count(dqm, kq->queue->properties.type);
64c7f8cf 1269 qpd->is_debug = true;
c4744e24 1270 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
efeaed4d 1271 dqm_unlock(dqm);
64c7f8cf
BG		 1272
1273 return 0;
1274}
1275
1276static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1277 struct kernel_queue *kq,
1278 struct qcm_process_device *qpd)
1279{
efeaed4d 1280 dqm_lock(dqm);
64c7f8cf 1281 list_del(&kq->list);
b42902f4 1282 decrement_queue_count(dqm, kq->queue->properties.type);
64c7f8cf 1283 qpd->is_debug = false;
c4744e24 1284 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
b8cbab04
OG		 1285 /*
1286 * Unconditionally decrement this counter, regardless of the queue's
1287 * type.
1288 */
8b58f261 1289 dqm->total_queue_count--;
b8cbab04
OG		 1290 pr_debug("Total of %d queues are accountable so far\n",
1291 dqm->total_queue_count);
efeaed4d 1292 dqm_unlock(dqm);
64c7f8cf
BG		 1293}
1294
1295static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
b46cb7d7 1296 struct qcm_process_device *qpd)
64c7f8cf
BG		 1297{
1298 int retval;
8d5f3552 1299 struct mqd_manager *mqd_mgr;
64c7f8cf 1300
b8cbab04 1301 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1302 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04 1303 dqm->total_queue_count);
70d488fb
OZ		 1304 retval = -EPERM;
1305 goto out;
b8cbab04
OG		 1306 }
1307
1b4670f6
OZ		 1308 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1309 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
38bb4226 1310 dqm_lock(dqm);
e78579aa 1311 retval = allocate_sdma_queue(dqm, q);
38bb4226 1312 dqm_unlock(dqm);
894a8293 1313 if (retval)
70d488fb 1314 goto out;
e139cd2a 1315 }
ef568db7
FK		 1316
1317 retval = allocate_doorbell(qpd, q);
1318 if (retval)
1319 goto out_deallocate_sdma_queue;
1320
70d488fb
OZ		 1321 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1322 q->properties.type)];
70df8273 1323
eec0b4cf
OZ		 1324 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1325 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1326 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
373d7080
FK		 1327 q->properties.tba_addr = qpd->tba_addr;
1328 q->properties.tma_addr = qpd->tma_addr;
70d488fb
OZ		 1329 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
1330 if (!q->mqd_mem_obj) {
1331 retval = -ENOMEM;
1332 goto out_deallocate_doorbell;
1333 }
70df8273
EH		 1334
1335 dqm_lock(dqm);
1336 /*
1337 * Eviction state logic: mark all queues as evicted, even ones
1338 * not currently active. Restoring inactive queues later only
1339 * updates the is_evicted flag but is a no-op otherwise.
1340 */
1341 q->properties.is_evicted = !!qpd->evicted;
8636e53c
OZ		 1342 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
1343 &q->gart_mqd_addr, &q->properties);
89cd9d23 1344
64c7f8cf 1345 list_add(&q->list, &qpd->queues_list);
bc920fd4 1346 qpd->queue_count++;
f38abc15 1347
64c7f8cf 1348 if (q->properties.is_active) {
b42902f4
YZ		 1349 increment_queue_count(dqm, q->properties.type);
1350
66a5710b 1351 execute_queues_cpsch(dqm,
c4744e24 1352 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
64c7f8cf
BG		 1353 }
1354
b8cbab04
OG		 1355 /*
1356 * Unconditionally increment this counter, regardless of the queue's
1357 * type or whether the queue is active.
1358 */
1359 dqm->total_queue_count++;
1360
1361 pr_debug("Total of %d queues are accountable so far\n",
1362 dqm->total_queue_count);
1363
efeaed4d 1364 dqm_unlock(dqm);
72a01d23
FK		 1365 return retval;
1366
70d488fb
OZ		 1367out_deallocate_doorbell:
1368 deallocate_doorbell(qpd, q);
72a01d23 1369out_deallocate_sdma_queue:
1b4670f6 1370 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
38bb4226
OZ		 1371 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1372 dqm_lock(dqm);
1b4670f6 1373 deallocate_sdma_queue(dqm, q);
38bb4226
OZ		 1374 dqm_unlock(dqm);
1375 }
70d488fb 1376out:
64c7f8cf
BG		 1377 return retval;
1378}
1379
b010affe
QH		 1380int amdkfd_fence_wait_timeout(uint64_t *fence_addr,
1381 uint64_t fence_value,
8c72c3d7 1382 unsigned int timeout_ms)
64c7f8cf 1383{
8c72c3d7 1384 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
64c7f8cf
BG		 1385
1386 while (*fence_addr != fence_value) {
8c72c3d7 1387 if (time_after(jiffies, end_jiffies)) {
79775b62 1388 pr_err("qcm fence wait loop timeout expired\n");
0e9a860c
YZ		 1389 /* In HWS case, this is used to halt the driver thread
1390 * in order not to mess up CP states before doing
1391 * scandumps for FW debugging.
1392 */
1393 while (halt_if_hws_hang)
1394 schedule();
1395
64c7f8cf
BG		 1396 return -ETIME;
1397 }
99331a51 1398 schedule();
64c7f8cf
BG		 1399 }
1400
1401 return 0;
1402}
1403
60a00956
FK		 1404/* dqm->lock mutex has to be locked before calling this function */
1405static int map_queues_cpsch(struct device_queue_manager *dqm)
1406{
1407 int retval;
1408
2c99a547
PY		 1409 if (!dqm->sched_running)
1410 return 0;
81b820b3 1411 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)
60a00956 1412 return 0;
60a00956
FK		 1413 if (dqm->active_runlist)
1414 return 0;
1415
9af5379c 1416 retval = pm_send_runlist(&dqm->packet_mgr, &dqm->queues);
14328aa5 1417 pr_debug("%s sent runlist\n", __func__);
60a00956
FK		 1418 if (retval) {
1419 pr_err("failed to execute runlist\n");
1420 return retval;
1421 }
1422 dqm->active_runlist = true;
1423
1424 return retval;
1425}
1426
ac30c783 1427/* dqm->lock mutex has to be locked before calling this function */
7da2bcf8 1428static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 1429 enum kfd_unmap_queues_filter filter,
1430 uint32_t filter_param)
64c7f8cf 1431{
9fd3f1bf 1432 int retval = 0;
51a0f459 1433 struct mqd_manager *mqd_mgr;
64c7f8cf 1434
2c99a547
PY		 1435 if (!dqm->sched_running)
1436 return 0;
b8c20c74 1437 if (dqm->is_hws_hang || dqm->is_resetting)
73ea648d 1438 return -EIO;
991ca8ee 1439 if (!dqm->active_runlist)
ac30c783 1440 return retval;
bcea3081 1441
9af5379c 1442 retval = pm_send_unmap_queue(&dqm->packet_mgr, KFD_QUEUE_TYPE_COMPUTE,
4465f466 1443 filter, filter_param, false, 0);
4eacc26b 1444 if (retval)
ac30c783 1445 return retval;
64c7f8cf
BG		 1446
1447 *dqm->fence_addr = KFD_FENCE_INIT;
9af5379c 1448 pm_send_query_status(&dqm->packet_mgr, dqm->fence_gpu_addr,
64c7f8cf
BG		 1449 KFD_FENCE_COMPLETED);
1450 /* should be timed out */
c3447e81 1451 retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
14328aa5 1452 queue_preemption_timeout_ms);
09c34e8d
FK		 1453 if (retval) {
1454 pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
1455 dqm->is_hws_hang = true;
1456 /* It's possible we're detecting a HWS hang in the
1457 * middle of a GPU reset. No need to schedule another
1458 * reset in this case.
1459 */
1460 if (!dqm->is_resetting)
1461 schedule_work(&dqm->hw_exception_work);
ac30c783 1462 return retval;
09c34e8d 1463 }
9fd3f1bf 1464
51a0f459
OZ		 1465 /* In the current MEC firmware implementation, if compute queue
1466 * doesn't response to the preemption request in time, HIQ will
1467 * abandon the unmap request without returning any timeout error
1468 * to driver. Instead, MEC firmware will log the doorbell of the
1469 * unresponding compute queue to HIQ.MQD.queue_doorbell_id fields.
1470 * To make sure the queue unmap was successful, driver need to
1471 * check those fields
1472 */
1473 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ];
9af5379c 1474 if (mqd_mgr->read_doorbell_id(dqm->packet_mgr.priv_queue->queue->mqd)) {
51a0f459
OZ		 1475 pr_err("HIQ MQD's queue_doorbell_id0 is not 0, Queue preemption time out\n");
1476 while (halt_if_hws_hang)
1477 schedule();
1478 return -ETIME;
1479 }
1480
9af5379c 1481 pm_release_ib(&dqm->packet_mgr);
64c7f8cf
BG		 1482 dqm->active_runlist = false;
1483
64c7f8cf
BG		 1484 return retval;
1485}
1486
ac30c783 1487/* dqm->lock mutex has to be locked before calling this function */
c4744e24
YZ		 1488static int execute_queues_cpsch(struct device_queue_manager *dqm,
1489 enum kfd_unmap_queues_filter filter,
1490 uint32_t filter_param)
64c7f8cf
BG		 1491{
1492 int retval;
1493
73ea648d
SL		 1494 if (dqm->is_hws_hang)
1495 return -EIO;
c4744e24 1496 retval = unmap_queues_cpsch(dqm, filter, filter_param);
09c34e8d 1497 if (retval)
ac30c783 1498 return retval;
64c7f8cf 1499
60a00956 1500 return map_queues_cpsch(dqm);
64c7f8cf
BG		 1501}
1502
1503static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1504 struct qcm_process_device *qpd,
1505 struct queue *q)
1506{
1507 int retval;
8d5f3552 1508 struct mqd_manager *mqd_mgr;
d69fd951
MJ		 1509 uint64_t sdma_val = 0;
1510 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
1511
1512 /* Get the SDMA queue stats */
1513 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
1514 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
818b0324 1515 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
d69fd951
MJ		 1516 &sdma_val);
1517 if (retval)
1518 pr_err("Failed to read SDMA queue counter for queue: %d\n",
1519 q->properties.queue_id);
1520 }
992839ad 1521
64c7f8cf
BG		 1522 retval = 0;
1523
1524 /* remove queue from list to prevent rescheduling after preemption */
efeaed4d 1525 dqm_lock(dqm);
992839ad
YS		 1526
1527 if (qpd->is_debug) {
1528 /*
1529 * error, currently we do not allow to destroy a queue
1530 * of a currently debugged process
1531 */
1532 retval = -EBUSY;
1533 goto failed_try_destroy_debugged_queue;
1534
1535 }
1536
fdfa090b
OZ		 1537 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1538 q->properties.type)];
64c7f8cf 1539
ef568db7
FK		 1540 deallocate_doorbell(qpd, q);
1541
d69fd951
MJ		 1542 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
1543 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1b4670f6 1544 deallocate_sdma_queue(dqm, q);
d69fd951
MJ		 1545 pdd->sdma_past_activity_counter += sdma_val;
1546 }
bcea3081 1547
64c7f8cf 1548 list_del(&q->list);
bc920fd4 1549 qpd->queue_count--;
40a526dc 1550 if (q->properties.is_active) {
b42902f4 1551 decrement_queue_count(dqm, q->properties.type);
40a526dc 1552 retval = execute_queues_cpsch(dqm,
9fd3f1bf 1553 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
40a526dc
YZ		 1554 if (retval == -ETIME)
1555 qpd->reset_wavefronts = true;
b8020b03
JG		 1556 if (q->properties.is_gws) {
1557 dqm->gws_queue_count--;
1558 qpd->mapped_gws_queue = false;
1559 }
40a526dc 1560 }
64c7f8cf 1561
b8cbab04
OG		 1562 /*
1563 * Unconditionally decrement this counter, regardless of the queue's
1564 * type
1565 */
1566 dqm->total_queue_count--;
1567 pr_debug("Total of %d queues are accountable so far\n",
1568 dqm->total_queue_count);
64c7f8cf 1569
efeaed4d 1570 dqm_unlock(dqm);
64c7f8cf 1571
8636e53c
OZ		 1572 /* Do free_mqd after dqm_unlock(dqm) to avoid circular locking */
1573 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
89cd9d23 1574
9e827224 1575 return retval;
64c7f8cf 1576
992839ad
YS		 1577failed_try_destroy_debugged_queue:
1578
efeaed4d 1579 dqm_unlock(dqm);
64c7f8cf
BG		 1580 return retval;
1581}
1582
1583/*
1584 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1585 * stay in user mode.
1586 */
1587#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1588/* APE1 limit is inclusive and 64K aligned. */
1589#define APE1_LIMIT_ALIGNMENT 0xFFFF
1590
1591static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1592 struct qcm_process_device *qpd,
1593 enum cache_policy default_policy,
1594 enum cache_policy alternate_policy,
1595 void __user *alternate_aperture_base,
1596 uint64_t alternate_aperture_size)
1597{
bed4f110
FK		 1598 bool retval = true;
1599
1600 if (!dqm->asic_ops.set_cache_memory_policy)
1601 return retval;
64c7f8cf 1602
efeaed4d 1603 dqm_lock(dqm);
64c7f8cf
BG		 1604
1605 if (alternate_aperture_size == 0) {
1606 /* base > limit disables APE1 */
1607 qpd->sh_mem_ape1_base = 1;
1608 qpd->sh_mem_ape1_limit = 0;
1609 } else {
1610 /*
1611 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1612 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1613 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1614 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1615 * Verify that the base and size parameters can be
1616 * represented in this format and convert them.
1617 * Additionally restrict APE1 to user-mode addresses.
1618 */
1619
1620 uint64_t base = (uintptr_t)alternate_aperture_base;
1621 uint64_t limit = base + alternate_aperture_size - 1;
1622
ab7c1648
KR		 1623 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
1624 (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
1625 retval = false;
64c7f8cf 1626 goto out;
ab7c1648 1627 }
64c7f8cf
BG		 1628
1629 qpd->sh_mem_ape1_base = base >> 16;
1630 qpd->sh_mem_ape1_limit = limit >> 16;
1631 }
1632
bfd5e378 1633 retval = dqm->asic_ops.set_cache_memory_policy(
a22fc854
BG		 1634 dqm,
1635 qpd,
1636 default_policy,
1637 alternate_policy,
1638 alternate_aperture_base,
1639 alternate_aperture_size);
64c7f8cf 1640
d146c5a7 1641 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
64c7f8cf
BG		 1642 program_sh_mem_settings(dqm, qpd);
1643
79775b62 1644 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
64c7f8cf
BG		 1645 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1646 qpd->sh_mem_ape1_limit);
1647
64c7f8cf 1648out:
efeaed4d 1649 dqm_unlock(dqm);
ab7c1648 1650 return retval;
64c7f8cf
BG		 1651}
1652
9fd3f1bf
FK		 1653static int process_termination_nocpsch(struct device_queue_manager *dqm,
1654 struct qcm_process_device *qpd)
1655{
a7b2451d 1656 struct queue *q;
9fd3f1bf
FK		 1657 struct device_process_node *cur, *next_dpn;
1658 int retval = 0;
32cce8bc 1659 bool found = false;
9fd3f1bf 1660
efeaed4d 1661 dqm_lock(dqm);
9fd3f1bf
FK		 1662
1663 /* Clear all user mode queues */
a7b2451d
AL		 1664 while (!list_empty(&qpd->queues_list)) {
1665 struct mqd_manager *mqd_mgr;
9fd3f1bf
FK		 1666 int ret;
1667
a7b2451d
AL		 1668 q = list_first_entry(&qpd->queues_list, struct queue, list);
1669 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1670 q->properties.type)];
9fd3f1bf
FK		 1671 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
1672 if (ret)
1673 retval = ret;
a7b2451d
AL		 1674 dqm_unlock(dqm);
1675 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
1676 dqm_lock(dqm);
9fd3f1bf
FK		 1677 }
1678
1679 /* Unregister process */
1680 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1681 if (qpd == cur->qpd) {
1682 list_del(&cur->list);
1683 kfree(cur);
1684 dqm->processes_count--;
32cce8bc 1685 found = true;
9fd3f1bf
FK		 1686 break;
1687 }
1688 }
1689
efeaed4d 1690 dqm_unlock(dqm);
32cce8bc
FK		 1691
1692 /* Outside the DQM lock because under the DQM lock we can't do
1693 * reclaim or take other locks that others hold while reclaiming.
1694 */
1695 if (found)
1696 kfd_dec_compute_active(dqm->dev);
1697
9fd3f1bf
FK		 1698 return retval;
1699}
1700
5df099e8
JC		 1701static int get_wave_state(struct device_queue_manager *dqm,
1702 struct queue *q,
1703 void __user *ctl_stack,
1704 u32 *ctl_stack_used_size,
1705 u32 *save_area_used_size)
1706{
4e6c6fc1 1707 struct mqd_manager *mqd_mgr;
5df099e8
JC		 1708
1709 dqm_lock(dqm);
1710
d7c0b047 1711 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
5df099e8 1712
63f6e012
JK		 1713 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
1714 q->properties.is_active || !q->device->cwsr_enabled ||
1715 !mqd_mgr->get_wave_state) {
1716 dqm_unlock(dqm);
1717 return -EINVAL;
5df099e8
JC		 1718 }
1719
5df099e8 1720 dqm_unlock(dqm);
63f6e012
JK		 1721
1722 /*
1723 * get_wave_state is outside the dqm lock to prevent circular locking
1724 * and the queue should be protected against destruction by the process
1725 * lock.
1726 */
1727 return mqd_mgr->get_wave_state(mqd_mgr, q->mqd, ctl_stack,
1728 ctl_stack_used_size, save_area_used_size);
5df099e8 1729}
9fd3f1bf
FK		 1730
1731static int process_termination_cpsch(struct device_queue_manager *dqm,
1732 struct qcm_process_device *qpd)
1733{
1734 int retval;
56f221b6 1735 struct queue *q;
9fd3f1bf 1736 struct kernel_queue *kq, *kq_next;
8d5f3552 1737 struct mqd_manager *mqd_mgr;
9fd3f1bf
FK		 1738 struct device_process_node *cur, *next_dpn;
1739 enum kfd_unmap_queues_filter filter =
1740 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
32cce8bc 1741 bool found = false;
9fd3f1bf
FK		 1742
1743 retval = 0;
1744
efeaed4d 1745 dqm_lock(dqm);
9fd3f1bf
FK		 1746
1747 /* Clean all kernel queues */
1748 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
1749 list_del(&kq->list);
b42902f4 1750 decrement_queue_count(dqm, kq->queue->properties.type);
9fd3f1bf
FK		 1751 qpd->is_debug = false;
1752 dqm->total_queue_count--;
1753 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
1754 }
1755
1756 /* Clear all user mode queues */
1757 list_for_each_entry(q, &qpd->queues_list, list) {
c7637c95 1758 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1b4670f6 1759 deallocate_sdma_queue(dqm, q);
c7637c95 1760 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1b4670f6 1761 deallocate_sdma_queue(dqm, q);
9fd3f1bf 1762
b8020b03 1763 if (q->properties.is_active) {
b42902f4 1764 decrement_queue_count(dqm, q->properties.type);
b8020b03
JG		 1765 if (q->properties.is_gws) {
1766 dqm->gws_queue_count--;
1767 qpd->mapped_gws_queue = false;
1768 }
1769 }
9fd3f1bf
FK		 1770
1771 dqm->total_queue_count--;
1772 }
1773
1774 /* Unregister process */
1775 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1776 if (qpd == cur->qpd) {
1777 list_del(&cur->list);
1778 kfree(cur);
1779 dqm->processes_count--;
32cce8bc 1780 found = true;
9fd3f1bf
FK		 1781 break;
1782 }
1783 }
1784
1785 retval = execute_queues_cpsch(dqm, filter, 0);
73ea648d 1786 if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
9fd3f1bf
FK		 1787 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
1788 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
1789 qpd->reset_wavefronts = false;
1790 }
1791
89cd9d23 1792 /* Lastly, free mqd resources.
8636e53c 1793 * Do free_mqd() after dqm_unlock to avoid circular locking.
89cd9d23 1794 */
56f221b6 1795 while (!list_empty(&qpd->queues_list)) {
1796 q = list_first_entry(&qpd->queues_list, struct queue, list);
fdfa090b
OZ		 1797 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1798 q->properties.type)];
9fd3f1bf 1799 list_del(&q->list);
bc920fd4 1800 qpd->queue_count--;
56f221b6 1801 dqm_unlock(dqm);
8636e53c 1802 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
56f221b6 1803 dqm_lock(dqm);
9fd3f1bf 1804 }
56f221b6 1805 dqm_unlock(dqm);
1806
1807 /* Outside the DQM lock because under the DQM lock we can't do
1808 * reclaim or take other locks that others hold while reclaiming.
1809 */
1810 if (found)
1811 kfd_dec_compute_active(dqm->dev);
9fd3f1bf 1812
9fd3f1bf
FK		 1813 return retval;
1814}
1815
fdfa090b
OZ		 1816static int init_mqd_managers(struct device_queue_manager *dqm)
1817{
1818 int i, j;
1819 struct mqd_manager *mqd_mgr;
1820
1821 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
1822 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
1823 if (!mqd_mgr) {
1824 pr_err("mqd manager [%d] initialization failed\n", i);
1825 goto out_free;
1826 }
1827 dqm->mqd_mgrs[i] = mqd_mgr;
1828 }
1829
1830 return 0;
1831
1832out_free:
1833 for (j = 0; j < i; j++) {
1834 kfree(dqm->mqd_mgrs[j]);
1835 dqm->mqd_mgrs[j] = NULL;
1836 }
1837
1838 return -ENOMEM;
1839}
11614c36
OZ		 1840
1841/* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
1842static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
1843{
1844 int retval;
1845 struct kfd_dev *dev = dqm->dev;
1846 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
1847 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
c7637c95 1848 get_num_all_sdma_engines(dqm) *
11614c36
OZ		 1849 dev->device_info->num_sdma_queues_per_engine +
1850 dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size;
1851
6bfc7c7e 1852 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size,
11614c36 1853 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr),
f2cc50ce 1854 (void *)&(mem_obj->cpu_ptr), false);
11614c36
OZ		 1855
1856 return retval;
1857}
1858
64c7f8cf
BG		 1859struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1860{
1861 struct device_queue_manager *dqm;
1862
79775b62 1863 pr_debug("Loading device queue manager\n");
a22fc854 1864
dbf56ab1 1865 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
64c7f8cf
BG		 1866 if (!dqm)
1867 return NULL;
1868
7eb0502a 1869 switch (dev->adev->asic_type) {
d146c5a7
FK		 1870 /* HWS is not available on Hawaii. */
1871 case CHIP_HAWAII:
1872 /* HWS depends on CWSR for timely dequeue. CWSR is not
1873 * available on Tonga.
1874 *
1875 * FIXME: This argument also applies to Kaveri.
1876 */
1877 case CHIP_TONGA:
1878 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
1879 break;
1880 default:
1881 dqm->sched_policy = sched_policy;
1882 break;
1883 }
1884
64c7f8cf 1885 dqm->dev = dev;
d146c5a7 1886 switch (dqm->sched_policy) {
64c7f8cf
BG		 1887 case KFD_SCHED_POLICY_HWS:
1888 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1889 /* initialize dqm for cp scheduling */
45c9a5e4
OG		 1890 dqm->ops.create_queue = create_queue_cpsch;
1891 dqm->ops.initialize = initialize_cpsch;
1892 dqm->ops.start = start_cpsch;
1893 dqm->ops.stop = stop_cpsch;
09c34e8d 1894 dqm->ops.pre_reset = pre_reset;
45c9a5e4
OG		 1895 dqm->ops.destroy_queue = destroy_queue_cpsch;
1896 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1897 dqm->ops.register_process = register_process;
1898 dqm->ops.unregister_process = unregister_process;
1899 dqm->ops.uninitialize = uninitialize;
45c9a5e4
OG		 1900 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1901 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1902 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
9fd3f1bf 1903 dqm->ops.process_termination = process_termination_cpsch;
26103436
FK		 1904 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
1905 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
5df099e8 1906 dqm->ops.get_wave_state = get_wave_state;
64c7f8cf
BG		 1907 break;
1908 case KFD_SCHED_POLICY_NO_HWS:
1909 /* initialize dqm for no cp scheduling */
45c9a5e4
OG		 1910 dqm->ops.start = start_nocpsch;
1911 dqm->ops.stop = stop_nocpsch;
09c34e8d 1912 dqm->ops.pre_reset = pre_reset;
45c9a5e4
OG		 1913 dqm->ops.create_queue = create_queue_nocpsch;
1914 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1915 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1916 dqm->ops.register_process = register_process;
1917 dqm->ops.unregister_process = unregister_process;
45c9a5e4 1918 dqm->ops.initialize = initialize_nocpsch;
58dcd5bf 1919 dqm->ops.uninitialize = uninitialize;
45c9a5e4 1920 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
9fd3f1bf 1921 dqm->ops.process_termination = process_termination_nocpsch;
26103436
FK		 1922 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
1923 dqm->ops.restore_process_queues =
1924 restore_process_queues_nocpsch;
5df099e8 1925 dqm->ops.get_wave_state = get_wave_state;
64c7f8cf
BG		 1926 break;
1927 default:
d146c5a7 1928 pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
32fa8219 1929 goto out_free;
64c7f8cf
BG		 1930 }
1931
7eb0502a 1932 switch (dev->adev->asic_type) {
a22fc854 1933 case CHIP_CARRIZO:
bfd5e378 1934 device_queue_manager_init_vi(&dqm->asic_ops);
300dec95
OG		 1935 break;
1936
a22fc854 1937 case CHIP_KAVERI:
bfd5e378 1938 device_queue_manager_init_cik(&dqm->asic_ops);
300dec95 1939 break;
97672cbe
FK		 1940
1941 case CHIP_HAWAII:
1942 device_queue_manager_init_cik_hawaii(&dqm->asic_ops);
1943 break;
1944
1945 case CHIP_TONGA:
1946 case CHIP_FIJI:
1947 case CHIP_POLARIS10:
1948 case CHIP_POLARIS11:
846a44d7 1949 case CHIP_POLARIS12:
ed81cd6e 1950 case CHIP_VEGAM:
97672cbe
FK		 1951 device_queue_manager_init_vi_tonga(&dqm->asic_ops);
1952 break;
bed4f110 1953
e596b903 1954 default:
e4804a39
GS		 1955 if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
1956 device_queue_manager_init_v10_navi10(&dqm->asic_ops);
1957 else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))
1958 device_queue_manager_init_v9(&dqm->asic_ops);
1959 else {
1960 WARN(1, "Unexpected ASIC family %u",
7eb0502a 1961 dev->adev->asic_type);
e4804a39
GS		 1962 goto out_free;
1963 }
a22fc854
BG		 1964 }
1965
fdfa090b
OZ		 1966 if (init_mqd_managers(dqm))
1967 goto out_free;
1968
11614c36
OZ		 1969 if (allocate_hiq_sdma_mqd(dqm)) {
1970 pr_err("Failed to allocate hiq sdma mqd trunk buffer\n");
1971 goto out_free;
1972 }
1973
32fa8219
FK		 1974 if (!dqm->ops.initialize(dqm))
1975 return dqm;
64c7f8cf 1976
32fa8219
FK		 1977out_free:
1978 kfree(dqm);
1979 return NULL;
64c7f8cf
BG		 1980}
1981
7fd5a6fb
Y		 1982static void deallocate_hiq_sdma_mqd(struct kfd_dev *dev,
1983 struct kfd_mem_obj *mqd)
11614c36
OZ		 1984{
1985 WARN(!mqd, "No hiq sdma mqd trunk to free");
1986
6bfc7c7e 1987 amdgpu_amdkfd_free_gtt_mem(dev->adev, mqd->gtt_mem);
11614c36
OZ		 1988}
1989
64c7f8cf
BG		 1990void device_queue_manager_uninit(struct device_queue_manager *dqm)
1991{
45c9a5e4 1992 dqm->ops.uninitialize(dqm);
11614c36 1993 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
64c7f8cf
BG		 1994 kfree(dqm);
1995}
851a645e 1996
c7b6bac9 1997int kfd_process_vm_fault(struct device_queue_manager *dqm, u32 pasid)
2640c3fa 1998{
1999 struct kfd_process_device *pdd;
2000 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
2001 int ret = 0;
2002
2003 if (!p)
2004 return -EINVAL;
8a491bb3 2005 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
2640c3fa 2006 pdd = kfd_get_process_device_data(dqm->dev, p);
2007 if (pdd)
2008 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
2009 kfd_unref_process(p);
2010
2011 return ret;
2012}
2013
73ea648d
SL		 2014static void kfd_process_hw_exception(struct work_struct *work)
2015{
2016 struct device_queue_manager *dqm = container_of(work,
2017 struct device_queue_manager, hw_exception_work);
6bfc7c7e 2018 amdgpu_amdkfd_gpu_reset(dqm->dev->adev);
73ea648d
SL		 2019}
2020
851a645e
FK		 2021#if defined(CONFIG_DEBUG_FS)
2022
2023static void seq_reg_dump(struct seq_file *m,
2024 uint32_t (*dump)[2], uint32_t n_regs)
2025{
2026 uint32_t i, count;
2027
2028 for (i = 0, count = 0; i < n_regs; i++) {
2029 if (count == 0 ||
2030 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
2031 seq_printf(m, "%s %08x: %08x",
2032 i ? "\n" : "",
2033 dump[i][0], dump[i][1]);
2034 count = 7;
2035 } else {
2036 seq_printf(m, " %08x", dump[i][1]);
2037 count--;
2038 }
2039 }
2040
2041 seq_puts(m, "\n");
2042}
2043
2044int dqm_debugfs_hqds(struct seq_file *m, void *data)
2045{
2046 struct device_queue_manager *dqm = data;
2047 uint32_t (*dump)[2], n_regs;
2048 int pipe, queue;
2049 int r = 0;
2050
2c99a547
PY		 2051 if (!dqm->sched_running) {
2052 seq_printf(m, " Device is stopped\n");
2053
2054 return 0;
2055 }
2056
420185fd 2057 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->adev,
14328aa5
PC		 2058 KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
2059 &dump, &n_regs);
24f48a42
OZ		 2060 if (!r) {
2061 seq_printf(m, " HIQ on MEC %d Pipe %d Queue %d\n",
14328aa5
PC		 2062 KFD_CIK_HIQ_PIPE/get_pipes_per_mec(dqm)+1,
2063 KFD_CIK_HIQ_PIPE%get_pipes_per_mec(dqm),
2064 KFD_CIK_HIQ_QUEUE);
24f48a42
OZ		 2065 seq_reg_dump(m, dump, n_regs);
2066
2067 kfree(dump);
2068 }
2069
851a645e
FK		 2070 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
2071 int pipe_offset = pipe * get_queues_per_pipe(dqm);
2072
2073 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
2074 if (!test_bit(pipe_offset + queue,
e6945304 2075 dqm->dev->shared_resources.cp_queue_bitmap))
851a645e
FK		 2076 continue;
2077
2078 r = dqm->dev->kfd2kgd->hqd_dump(
420185fd 2079 dqm->dev->adev, pipe, queue, &dump, &n_regs);
851a645e
FK		 2080 if (r)
2081 break;
2082
2083 seq_printf(m, " CP Pipe %d, Queue %d\n",
2084 pipe, queue);
2085 seq_reg_dump(m, dump, n_regs);
2086
2087 kfree(dump);
2088 }
2089 }
2090
c7637c95 2091 for (pipe = 0; pipe < get_num_all_sdma_engines(dqm); pipe++) {
d5094189
SL		 2092 for (queue = 0;
2093 queue < dqm->dev->device_info->num_sdma_queues_per_engine;
2094 queue++) {
851a645e 2095 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
420185fd 2096 dqm->dev->adev, pipe, queue, &dump, &n_regs);
851a645e
FK		 2097 if (r)
2098 break;
2099
2100 seq_printf(m, " SDMA Engine %d, RLC %d\n",
2101 pipe, queue);
2102 seq_reg_dump(m, dump, n_regs);
2103
2104 kfree(dump);
2105 }
2106 }
2107
2108 return r;
2109}
2110
4f942aae 2111int dqm_debugfs_hang_hws(struct device_queue_manager *dqm)
a29ec470
SL		 2112{
2113 int r = 0;
2114
2115 dqm_lock(dqm);
4f942aae
OZ		 2116 r = pm_debugfs_hang_hws(&dqm->packet_mgr);
2117 if (r) {
2118 dqm_unlock(dqm);
2119 return r;
2120 }
a29ec470
SL		 2121 dqm->active_runlist = true;
2122 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
2123 dqm_unlock(dqm);
2124
2125 return r;
2126}
2127
851a645e 2128#endif
Linux 6.x block layer and io_uring tree(s)