projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Revert "drm/amdkfd: Fix a circular lock dependency"
[linux-block.git] / drivers / gpu / drm / amd / amdkfd / kfd_device_queue_manager.c
CommitLineData
64c7f8cf
BG		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
26103436
FK		 24#include <linux/ratelimit.h>
25#include <linux/printk.h>
64c7f8cf
BG		 26#include <linux/slab.h>
27#include <linux/list.h>
28#include <linux/types.h>
64c7f8cf 29#include <linux/bitops.h>
99331a51 30#include <linux/sched.h>
64c7f8cf
BG		 31#include "kfd_priv.h"
32#include "kfd_device_queue_manager.h"
33#include "kfd_mqd_manager.h"
34#include "cik_regs.h"
35#include "kfd_kernel_queue.h"
5b87245f 36#include "amdgpu_amdkfd.h"
64c7f8cf
BG		 37
38/* Size of the per-pipe EOP queue */
39#define CIK_HPD_EOP_BYTES_LOG2 11
40#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
41
64c7f8cf
BG		 42static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
43 unsigned int pasid, unsigned int vmid);
44
c4744e24
YZ		 45static int execute_queues_cpsch(struct device_queue_manager *dqm,
46 enum kfd_unmap_queues_filter filter,
47 uint32_t filter_param);
7da2bcf8 48static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 49 enum kfd_unmap_queues_filter filter,
50 uint32_t filter_param);
64c7f8cf 51
60a00956
FK		 52static int map_queues_cpsch(struct device_queue_manager *dqm);
53
bcea3081 54static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1b4670f6 55 struct queue *q);
64c7f8cf 56
d39b7737
OZ		 57static inline void deallocate_hqd(struct device_queue_manager *dqm,
58 struct queue *q);
59static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
60static int allocate_sdma_queue(struct device_queue_manager *dqm,
61 struct queue *q);
73ea648d
SL		 62static void kfd_process_hw_exception(struct work_struct *work);
63
bcea3081
BG		 64static inline
65enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
64c7f8cf 66{
1b4670f6 67 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
85d258f9
BG		 68 return KFD_MQD_TYPE_SDMA;
69 return KFD_MQD_TYPE_CP;
64c7f8cf
BG		 70}
71
d0b63bb3
AR		 72static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
73{
74 int i;
75 int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec
76 + pipe * dqm->dev->shared_resources.num_queue_per_pipe;
77
78 /* queue is available for KFD usage if bit is 1 */
79 for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i)
80 if (test_bit(pipe_offset + i,
81 dqm->dev->shared_resources.queue_bitmap))
82 return true;
83 return false;
84}
85
d0b63bb3 86unsigned int get_queues_num(struct device_queue_manager *dqm)
64ea8f4a 87{
d0b63bb3
AR		 88 return bitmap_weight(dqm->dev->shared_resources.queue_bitmap,
89 KGD_MAX_QUEUES);
64ea8f4a
OG		 90}
91
d0b63bb3 92unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
64c7f8cf 93{
d0b63bb3
AR		 94 return dqm->dev->shared_resources.num_queue_per_pipe;
95}
96
97unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
98{
d0b63bb3 99 return dqm->dev->shared_resources.num_pipe_per_mec;
64c7f8cf
BG		 100}
101
98bb9222
YZ		 102static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
103{
104 return dqm->dev->device_info->num_sdma_engines;
105}
106
1b4670f6
OZ		 107static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm)
108{
109 return dqm->dev->device_info->num_xgmi_sdma_engines;
110}
111
98bb9222
YZ		 112unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
113{
114 return dqm->dev->device_info->num_sdma_engines
d5094189 115 * dqm->dev->device_info->num_sdma_queues_per_engine;
98bb9222
YZ		 116}
117
1b4670f6
OZ		 118unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
119{
120 return dqm->dev->device_info->num_xgmi_sdma_engines
121 * dqm->dev->device_info->num_sdma_queues_per_engine;
122}
123
a22fc854 124void program_sh_mem_settings(struct device_queue_manager *dqm,
64c7f8cf
BG		 125 struct qcm_process_device *qpd)
126{
cea405b1
XZ		 127 return dqm->dev->kfd2kgd->program_sh_mem_settings(
128 dqm->dev->kgd, qpd->vmid,
64c7f8cf
BG		 129 qpd->sh_mem_config,
130 qpd->sh_mem_ape1_base,
131 qpd->sh_mem_ape1_limit,
132 qpd->sh_mem_bases);
133}
134
ef568db7
FK		 135static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
136{
137 struct kfd_dev *dev = qpd->dqm->dev;
138
139 if (!KFD_IS_SOC15(dev->device_info->asic_family)) {
140 /* On pre-SOC15 chips we need to use the queue ID to
141 * preserve the user mode ABI.
142 */
143 q->doorbell_id = q->properties.queue_id;
1b4670f6
OZ		 144 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
145 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
234441dd
YZ		 146 /* For SDMA queues on SOC15 with 8-byte doorbell, use static
147 * doorbell assignments based on the engine and queue id.
148 * The doobell index distance between RLC (2*i) and (2*i+1)
149 * for a SDMA engine is 512.
ef568db7 150 */
234441dd
YZ		 151 uint32_t *idx_offset =
152 dev->shared_resources.sdma_doorbell_idx;
153
154 q->doorbell_id = idx_offset[q->properties.sdma_engine_id]
155 + (q->properties.sdma_queue_id & 1)
156 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET
157 + (q->properties.sdma_queue_id >> 1);
ef568db7
FK		 158 } else {
159 /* For CP queues on SOC15 reserve a free doorbell ID */
160 unsigned int found;
161
162 found = find_first_zero_bit(qpd->doorbell_bitmap,
163 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
164 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
165 pr_debug("No doorbells available");
166 return -EBUSY;
167 }
168 set_bit(found, qpd->doorbell_bitmap);
169 q->doorbell_id = found;
170 }
171
172 q->properties.doorbell_off =
173 kfd_doorbell_id_to_offset(dev, q->process,
174 q->doorbell_id);
175
176 return 0;
177}
178
179static void deallocate_doorbell(struct qcm_process_device *qpd,
180 struct queue *q)
181{
182 unsigned int old;
183 struct kfd_dev *dev = qpd->dqm->dev;
184
185 if (!KFD_IS_SOC15(dev->device_info->asic_family) ||
1b4670f6
OZ		 186 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
187 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
ef568db7
FK		 188 return;
189
190 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
191 WARN_ON(!old);
192}
193
64c7f8cf
BG		 194static int allocate_vmid(struct device_queue_manager *dqm,
195 struct qcm_process_device *qpd,
196 struct queue *q)
197{
198 int bit, allocated_vmid;
199
200 if (dqm->vmid_bitmap == 0)
201 return -ENOMEM;
202
4252bf68
HK		 203 bit = ffs(dqm->vmid_bitmap) - 1;
204 dqm->vmid_bitmap &= ~(1 << bit);
64c7f8cf 205
44008d7a 206 allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd;
79775b62 207 pr_debug("vmid allocation %d\n", allocated_vmid);
64c7f8cf
BG		 208 qpd->vmid = allocated_vmid;
209 q->properties.vmid = allocated_vmid;
210
211 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
212 program_sh_mem_settings(dqm, qpd);
213
403575c4
FK		 214 /* qpd->page_table_base is set earlier when register_process()
215 * is called, i.e. when the first queue is created.
216 */
217 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd,
218 qpd->vmid,
219 qpd->page_table_base);
220 /* invalidate the VM context after pasid and vmid mapping is set up */
221 kfd_flush_tlb(qpd_to_pdd(qpd));
222
d39b7737
OZ		 223 dqm->dev->kfd2kgd->set_scratch_backing_va(
224 dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
225
64c7f8cf
BG		 226 return 0;
227}
228
552764b6
FK		 229static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
230 struct qcm_process_device *qpd)
231{
f6e27ff1
FK		 232 const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf;
233 int ret;
552764b6
FK		 234
235 if (!qpd->ib_kaddr)
236 return -ENOMEM;
237
f6e27ff1
FK		 238 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
239 if (ret)
240 return ret;
552764b6 241
5b87245f 242 return amdgpu_amdkfd_submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid,
f6e27ff1
FK		 243 qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
244 pmf->release_mem_size / sizeof(uint32_t));
552764b6
FK		 245}
246
64c7f8cf
BG		 247static void deallocate_vmid(struct device_queue_manager *dqm,
248 struct qcm_process_device *qpd,
249 struct queue *q)
250{
44008d7a 251 int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd;
64c7f8cf 252
552764b6
FK		 253 /* On GFX v7, CP doesn't flush TC at dequeue */
254 if (q->device->device_info->asic_family == CHIP_HAWAII)
255 if (flush_texture_cache_nocpsch(q->device, qpd))
256 pr_err("Failed to flush TC\n");
257
403575c4
FK		 258 kfd_flush_tlb(qpd_to_pdd(qpd));
259
2030664b
BG		 260 /* Release the vmid mapping */
261 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
262
4252bf68 263 dqm->vmid_bitmap |= (1 << bit);
64c7f8cf
BG		 264 qpd->vmid = 0;
265 q->properties.vmid = 0;
266}
267
268static int create_queue_nocpsch(struct device_queue_manager *dqm,
269 struct queue *q,
b46cb7d7 270 struct qcm_process_device *qpd)
64c7f8cf 271{
d39b7737 272 struct mqd_manager *mqd_mgr;
64c7f8cf
BG		 273 int retval;
274
64c7f8cf
BG		 275 print_queue(q);
276
efeaed4d 277 dqm_lock(dqm);
64c7f8cf 278
b8cbab04 279 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 280 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04 281 dqm->total_queue_count);
ab7c1648
KR		 282 retval = -EPERM;
283 goto out_unlock;
b8cbab04
OG		 284 }
285
64c7f8cf
BG		 286 if (list_empty(&qpd->queues_list)) {
287 retval = allocate_vmid(dqm, qpd, q);
ab7c1648
KR		 288 if (retval)
289 goto out_unlock;
64c7f8cf 290 }
64c7f8cf 291 q->properties.vmid = qpd->vmid;
26103436 292 /*
bb2d2128
FK		 293 * Eviction state logic: mark all queues as evicted, even ones
294 * not currently active. Restoring inactive queues later only
295 * updates the is_evicted flag but is a no-op otherwise.
26103436 296 */
bb2d2128 297 q->properties.is_evicted = !!qpd->evicted;
64c7f8cf 298
373d7080
FK		 299 q->properties.tba_addr = qpd->tba_addr;
300 q->properties.tma_addr = qpd->tma_addr;
301
d091bc0a
OZ		 302 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
303 q->properties.type)];
d39b7737
OZ		 304 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
305 retval = allocate_hqd(dqm, q);
306 if (retval)
307 goto deallocate_vmid;
308 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
309 q->pipe, q->queue);
310 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
311 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
312 retval = allocate_sdma_queue(dqm, q);
313 if (retval)
314 goto deallocate_vmid;
315 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
316 }
317
318 retval = allocate_doorbell(qpd, q);
319 if (retval)
320 goto out_deallocate_hqd;
321
d091bc0a
OZ		 322 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
323 if (!q->mqd_mem_obj) {
324 retval = -ENOMEM;
325 goto out_deallocate_doorbell;
326 }
8636e53c
OZ		 327 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
328 &q->gart_mqd_addr, &q->properties);
d39b7737
OZ		 329 if (q->properties.is_active) {
330
331 if (WARN(q->process->mm != current->mm,
332 "should only run in user thread"))
333 retval = -EFAULT;
334 else
335 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
336 q->queue, &q->properties, current->mm);
337 if (retval)
d091bc0a 338 goto out_free_mqd;
64c7f8cf
BG		 339 }
340
341 list_add(&q->list, &qpd->queues_list);
bc920fd4 342 qpd->queue_count++;
b6819cec
JC		 343 if (q->properties.is_active)
344 dqm->queue_count++;
64c7f8cf 345
bcea3081
BG		 346 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
347 dqm->sdma_queue_count++;
1b4670f6
OZ		 348 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
349 dqm->xgmi_sdma_queue_count++;
64c7f8cf 350
b8cbab04
OG		 351 /*
352 * Unconditionally increment this counter, regardless of the queue's
353 * type or whether the queue is active.
354 */
355 dqm->total_queue_count++;
356 pr_debug("Total of %d queues are accountable so far\n",
357 dqm->total_queue_count);
d091bc0a 358 goto out_unlock;
b8cbab04 359
d091bc0a
OZ		 360out_free_mqd:
361 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
d39b7737
OZ		 362out_deallocate_doorbell:
363 deallocate_doorbell(qpd, q);
364out_deallocate_hqd:
365 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
366 deallocate_hqd(dqm, q);
367 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
368 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
369 deallocate_sdma_queue(dqm, q);
370deallocate_vmid:
371 if (list_empty(&qpd->queues_list))
372 deallocate_vmid(dqm, qpd, q);
ab7c1648 373out_unlock:
efeaed4d 374 dqm_unlock(dqm);
ab7c1648 375 return retval;
64c7f8cf
BG		 376}
377
378static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
379{
380 bool set;
f0ec5b99 381 int pipe, bit, i;
64c7f8cf
BG		 382
383 set = false;
384
8eabaf54
KR		 385 for (pipe = dqm->next_pipe_to_allocate, i = 0;
386 i < get_pipes_per_mec(dqm);
d0b63bb3
AR		 387 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
388
389 if (!is_pipe_enabled(dqm, 0, pipe))
390 continue;
391
64c7f8cf 392 if (dqm->allocated_queues[pipe] != 0) {
4252bf68
HK		 393 bit = ffs(dqm->allocated_queues[pipe]) - 1;
394 dqm->allocated_queues[pipe] &= ~(1 << bit);
64c7f8cf
BG		 395 q->pipe = pipe;
396 q->queue = bit;
397 set = true;
398 break;
399 }
400 }
401
991ca8ee 402 if (!set)
64c7f8cf
BG		 403 return -EBUSY;
404
79775b62 405 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
64c7f8cf 406 /* horizontal hqd allocation */
d0b63bb3 407 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
64c7f8cf
BG		 408
409 return 0;
410}
411
412static inline void deallocate_hqd(struct device_queue_manager *dqm,
413 struct queue *q)
414{
4252bf68 415 dqm->allocated_queues[q->pipe] |= (1 << q->queue);
64c7f8cf
BG		 416}
417
9fd3f1bf
FK		 418/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
419 * to avoid asynchronized access
420 */
421static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
64c7f8cf
BG		 422 struct qcm_process_device *qpd,
423 struct queue *q)
424{
425 int retval;
8d5f3552 426 struct mqd_manager *mqd_mgr;
64c7f8cf 427
fdfa090b
OZ		 428 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
429 q->properties.type)];
64c7f8cf 430
c2e1b3a4 431 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
c2e1b3a4
BG		 432 deallocate_hqd(dqm, q);
433 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
c2e1b3a4 434 dqm->sdma_queue_count--;
1b4670f6
OZ		 435 deallocate_sdma_queue(dqm, q);
436 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
437 dqm->xgmi_sdma_queue_count--;
438 deallocate_sdma_queue(dqm, q);
7113cd65 439 } else {
79775b62 440 pr_debug("q->properties.type %d is invalid\n",
7113cd65 441 q->properties.type);
9fd3f1bf 442 return -EINVAL;
64c7f8cf 443 }
9fd3f1bf 444 dqm->total_queue_count--;
64c7f8cf 445
ef568db7
FK		 446 deallocate_doorbell(qpd, q);
447
8d5f3552 448 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
c2e1b3a4 449 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
b90e3fbe 450 KFD_UNMAP_LATENCY_MS,
64c7f8cf 451 q->pipe, q->queue);
9fd3f1bf
FK		 452 if (retval == -ETIME)
453 qpd->reset_wavefronts = true;
64c7f8cf 454
8636e53c 455 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
64c7f8cf
BG		 456
457 list_del(&q->list);
9fd3f1bf
FK		 458 if (list_empty(&qpd->queues_list)) {
459 if (qpd->reset_wavefronts) {
460 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
461 dqm->dev);
462 /* dbgdev_wave_reset_wavefronts has to be called before
463 * deallocate_vmid(), i.e. when vmid is still in use.
464 */
465 dbgdev_wave_reset_wavefronts(dqm->dev,
466 qpd->pqm->process);
467 qpd->reset_wavefronts = false;
468 }
469
64c7f8cf 470 deallocate_vmid(dqm, qpd, q);
9fd3f1bf 471 }
bc920fd4 472 qpd->queue_count--;
b6819cec
JC		 473 if (q->properties.is_active)
474 dqm->queue_count--;
b8cbab04 475
9fd3f1bf
FK		 476 return retval;
477}
b8cbab04 478
9fd3f1bf
FK		 479static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
480 struct qcm_process_device *qpd,
481 struct queue *q)
482{
483 int retval;
484
efeaed4d 485 dqm_lock(dqm);
9fd3f1bf 486 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
efeaed4d 487 dqm_unlock(dqm);
9fd3f1bf 488
64c7f8cf
BG		 489 return retval;
490}
491
492static int update_queue(struct device_queue_manager *dqm, struct queue *q)
493{
8636e53c 494 int retval = 0;
8d5f3552 495 struct mqd_manager *mqd_mgr;
26103436 496 struct kfd_process_device *pdd;
b6ffbab8 497 bool prev_active = false;
64c7f8cf 498
efeaed4d 499 dqm_lock(dqm);
26103436
FK		 500 pdd = kfd_get_process_device_data(q->device, q->process);
501 if (!pdd) {
502 retval = -ENODEV;
503 goto out_unlock;
504 }
fdfa090b
OZ		 505 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
506 q->properties.type)];
64c7f8cf 507
60a00956
FK		 508 /* Save previous activity state for counters */
509 prev_active = q->properties.is_active;
510
511 /* Make sure the queue is unmapped before updating the MQD */
d146c5a7 512 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
60a00956
FK		 513 retval = unmap_queues_cpsch(dqm,
514 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
894a8293 515 if (retval) {
60a00956
FK		 516 pr_err("unmap queue failed\n");
517 goto out_unlock;
518 }
894a8293 519 } else if (prev_active &&
60a00956 520 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1b4670f6
OZ		 521 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
522 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
8d5f3552 523 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
60a00956
FK		 524 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
525 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
526 if (retval) {
527 pr_err("destroy mqd failed\n");
528 goto out_unlock;
529 }
530 }
531
8636e53c 532 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties);
60a00956 533
096d1a3e
FK		 534 /*
535 * check active state vs. the previous state and modify
536 * counter accordingly. map_queues_cpsch uses the
537 * dqm->queue_count to determine whether a new runlist must be
538 * uploaded.
539 */
540 if (q->properties.is_active && !prev_active)
541 dqm->queue_count++;
542 else if (!q->properties.is_active && prev_active)
543 dqm->queue_count--;
544
d146c5a7 545 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
60a00956 546 retval = map_queues_cpsch(dqm);
894a8293 547 else if (q->properties.is_active &&
60a00956 548 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1b4670f6
OZ		 549 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
550 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1b19aa5a
FK		 551 if (WARN(q->process->mm != current->mm,
552 "should only run in user thread"))
553 retval = -EFAULT;
554 else
555 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
556 q->pipe, q->queue,
557 &q->properties, current->mm);
558 }
b6ffbab8 559
ab7c1648 560out_unlock:
efeaed4d 561 dqm_unlock(dqm);
64c7f8cf
BG		 562 return retval;
563}
564
26103436
FK		 565static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
566 struct qcm_process_device *qpd)
567{
568 struct queue *q;
8d5f3552 569 struct mqd_manager *mqd_mgr;
26103436 570 struct kfd_process_device *pdd;
bb2d2128 571 int retval, ret = 0;
26103436 572
efeaed4d 573 dqm_lock(dqm);
26103436
FK		 574 if (qpd->evicted++ > 0) /* already evicted, do nothing */
575 goto out;
576
577 pdd = qpd_to_pdd(qpd);
578 pr_info_ratelimited("Evicting PASID %u queues\n",
579 pdd->process->pasid);
580
bb2d2128
FK		 581 /* Mark all queues as evicted. Deactivate all active queues on
582 * the qpd.
583 */
26103436 584 list_for_each_entry(q, &qpd->queues_list, list) {
bb2d2128 585 q->properties.is_evicted = true;
26103436
FK		 586 if (!q->properties.is_active)
587 continue;
bb2d2128 588
fdfa090b
OZ		 589 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
590 q->properties.type)];
26103436 591 q->properties.is_active = false;
8d5f3552 592 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
26103436
FK		 593 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
594 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
bb2d2128
FK		 595 if (retval && !ret)
596 /* Return the first error, but keep going to
597 * maintain a consistent eviction state
598 */
599 ret = retval;
26103436
FK		 600 dqm->queue_count--;
601 }
602
603out:
efeaed4d 604 dqm_unlock(dqm);
bb2d2128 605 return ret;
26103436
FK		 606}
607
608static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
609 struct qcm_process_device *qpd)
610{
611 struct queue *q;
612 struct kfd_process_device *pdd;
613 int retval = 0;
614
efeaed4d 615 dqm_lock(dqm);
26103436
FK		 616 if (qpd->evicted++ > 0) /* already evicted, do nothing */
617 goto out;
618
619 pdd = qpd_to_pdd(qpd);
620 pr_info_ratelimited("Evicting PASID %u queues\n",
621 pdd->process->pasid);
622
bb2d2128
FK		 623 /* Mark all queues as evicted. Deactivate all active queues on
624 * the qpd.
625 */
26103436 626 list_for_each_entry(q, &qpd->queues_list, list) {
bb2d2128 627 q->properties.is_evicted = true;
26103436
FK		 628 if (!q->properties.is_active)
629 continue;
bb2d2128 630
26103436
FK		 631 q->properties.is_active = false;
632 dqm->queue_count--;
633 }
634 retval = execute_queues_cpsch(dqm,
635 qpd->is_debug ?
636 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
637 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
638
639out:
efeaed4d 640 dqm_unlock(dqm);
26103436
FK		 641 return retval;
642}
643
644static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
645 struct qcm_process_device *qpd)
646{
1b19aa5a 647 struct mm_struct *mm = NULL;
26103436 648 struct queue *q;
8d5f3552 649 struct mqd_manager *mqd_mgr;
26103436 650 struct kfd_process_device *pdd;
e715c6d0 651 uint64_t pd_base;
bb2d2128 652 int retval, ret = 0;
26103436
FK		 653
654 pdd = qpd_to_pdd(qpd);
655 /* Retrieve PD base */
5b87245f 656 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
26103436 657
efeaed4d 658 dqm_lock(dqm);
26103436
FK		 659 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
660 goto out;
661 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
662 qpd->evicted--;
663 goto out;
664 }
665
666 pr_info_ratelimited("Restoring PASID %u queues\n",
667 pdd->process->pasid);
668
669 /* Update PD Base in QPD */
670 qpd->page_table_base = pd_base;
e715c6d0 671 pr_debug("Updated PD address to 0x%llx\n", pd_base);
26103436
FK		 672
673 if (!list_empty(&qpd->queues_list)) {
674 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
675 dqm->dev->kgd,
676 qpd->vmid,
677 qpd->page_table_base);
678 kfd_flush_tlb(pdd);
679 }
680
1b19aa5a
FK		 681 /* Take a safe reference to the mm_struct, which may otherwise
682 * disappear even while the kfd_process is still referenced.
683 */
684 mm = get_task_mm(pdd->process->lead_thread);
685 if (!mm) {
bb2d2128 686 ret = -EFAULT;
1b19aa5a
FK		 687 goto out;
688 }
689
bb2d2128
FK		 690 /* Remove the eviction flags. Activate queues that are not
691 * inactive for other reasons.
692 */
26103436 693 list_for_each_entry(q, &qpd->queues_list, list) {
bb2d2128
FK		 694 q->properties.is_evicted = false;
695 if (!QUEUE_IS_ACTIVE(q->properties))
26103436 696 continue;
bb2d2128 697
fdfa090b
OZ		 698 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
699 q->properties.type)];
26103436 700 q->properties.is_active = true;
8d5f3552 701 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
1b19aa5a 702 q->queue, &q->properties, mm);
bb2d2128
FK		 703 if (retval && !ret)
704 /* Return the first error, but keep going to
705 * maintain a consistent eviction state
706 */
707 ret = retval;
26103436
FK		 708 dqm->queue_count++;
709 }
710 qpd->evicted = 0;
711out:
1b19aa5a
FK		 712 if (mm)
713 mmput(mm);
efeaed4d 714 dqm_unlock(dqm);
bb2d2128 715 return ret;
26103436
FK		 716}
717
718static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
719 struct qcm_process_device *qpd)
720{
721 struct queue *q;
722 struct kfd_process_device *pdd;
e715c6d0 723 uint64_t pd_base;
26103436
FK		 724 int retval = 0;
725
726 pdd = qpd_to_pdd(qpd);
727 /* Retrieve PD base */
5b87245f 728 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
26103436 729
efeaed4d 730 dqm_lock(dqm);
26103436
FK		 731 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
732 goto out;
733 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
734 qpd->evicted--;
735 goto out;
736 }
737
738 pr_info_ratelimited("Restoring PASID %u queues\n",
739 pdd->process->pasid);
740
741 /* Update PD Base in QPD */
742 qpd->page_table_base = pd_base;
e715c6d0 743 pr_debug("Updated PD address to 0x%llx\n", pd_base);
26103436
FK		 744
745 /* activate all active queues on the qpd */
746 list_for_each_entry(q, &qpd->queues_list, list) {
26103436 747 q->properties.is_evicted = false;
bb2d2128
FK		 748 if (!QUEUE_IS_ACTIVE(q->properties))
749 continue;
750
26103436
FK		 751 q->properties.is_active = true;
752 dqm->queue_count++;
753 }
754 retval = execute_queues_cpsch(dqm,
755 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
bb2d2128 756 qpd->evicted = 0;
26103436 757out:
efeaed4d 758 dqm_unlock(dqm);
26103436
FK		 759 return retval;
760}
761
58dcd5bf 762static int register_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 763 struct qcm_process_device *qpd)
764{
765 struct device_process_node *n;
403575c4 766 struct kfd_process_device *pdd;
e715c6d0 767 uint64_t pd_base;
a22fc854 768 int retval;
64c7f8cf 769
dbf56ab1 770 n = kzalloc(sizeof(*n), GFP_KERNEL);
64c7f8cf
BG		 771 if (!n)
772 return -ENOMEM;
773
774 n->qpd = qpd;
775
403575c4
FK		 776 pdd = qpd_to_pdd(qpd);
777 /* Retrieve PD base */
5b87245f 778 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
403575c4 779
efeaed4d 780 dqm_lock(dqm);
64c7f8cf
BG		 781 list_add(&n->list, &dqm->queues);
782
403575c4
FK		 783 /* Update PD Base in QPD */
784 qpd->page_table_base = pd_base;
e715c6d0 785 pr_debug("Updated PD address to 0x%llx\n", pd_base);
403575c4 786
bfd5e378 787 retval = dqm->asic_ops.update_qpd(dqm, qpd);
a22fc854 788
f756e631 789 dqm->processes_count++;
64c7f8cf 790
efeaed4d 791 dqm_unlock(dqm);
64c7f8cf 792
32cce8bc
FK		 793 /* Outside the DQM lock because under the DQM lock we can't do
794 * reclaim or take other locks that others hold while reclaiming.
795 */
796 kfd_inc_compute_active(dqm->dev);
797
a22fc854 798 return retval;
64c7f8cf
BG		 799}
800
58dcd5bf 801static int unregister_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 802 struct qcm_process_device *qpd)
803{
804 int retval;
805 struct device_process_node *cur, *next;
806
1e5ec956
OG		 807 pr_debug("qpd->queues_list is %s\n",
808 list_empty(&qpd->queues_list) ? "empty" : "not empty");
64c7f8cf
BG		 809
810 retval = 0;
efeaed4d 811 dqm_lock(dqm);
64c7f8cf
BG		 812
813 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
814 if (qpd == cur->qpd) {
815 list_del(&cur->list);
f5d896bb 816 kfree(cur);
f756e631 817 dqm->processes_count--;
64c7f8cf
BG		 818 goto out;
819 }
820 }
821 /* qpd not found in dqm list */
822 retval = 1;
823out:
efeaed4d 824 dqm_unlock(dqm);
32cce8bc
FK		 825
826 /* Outside the DQM lock because under the DQM lock we can't do
827 * reclaim or take other locks that others hold while reclaiming.
828 */
829 if (!retval)
830 kfd_dec_compute_active(dqm->dev);
831
64c7f8cf
BG		 832 return retval;
833}
834
835static int
836set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
837 unsigned int vmid)
838{
cea405b1 839 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
deb99d7c 840 dqm->dev->kgd, pasid, vmid);
64c7f8cf
BG		 841}
842
2249d558
AL		 843static void init_interrupts(struct device_queue_manager *dqm)
844{
845 unsigned int i;
846
d0b63bb3
AR		 847 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
848 if (is_pipe_enabled(dqm, 0, i))
849 dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
2249d558
AL		 850}
851
64c7f8cf
BG		 852static int initialize_nocpsch(struct device_queue_manager *dqm)
853{
86194cf8 854 int pipe, queue;
64c7f8cf 855
79775b62 856 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf 857
ab7c1648
KR		 858 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
859 sizeof(unsigned int), GFP_KERNEL);
860 if (!dqm->allocated_queues)
861 return -ENOMEM;
862
efeaed4d 863 mutex_init(&dqm->lock_hidden);
64c7f8cf
BG		 864 INIT_LIST_HEAD(&dqm->queues);
865 dqm->queue_count = dqm->next_pipe_to_allocate = 0;
bcea3081 866 dqm->sdma_queue_count = 0;
1b4670f6 867 dqm->xgmi_sdma_queue_count = 0;
64c7f8cf 868
86194cf8
FK		 869 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
870 int pipe_offset = pipe * get_queues_per_pipe(dqm);
871
872 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
873 if (test_bit(pipe_offset + queue,
874 dqm->dev->shared_resources.queue_bitmap))
875 dqm->allocated_queues[pipe] |= 1 << queue;
876 }
64c7f8cf 877
44008d7a 878 dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
cb77ee7c 879 dqm->sdma_bitmap = (1ULL << get_num_sdma_queues(dqm)) - 1;
1b4670f6 880 dqm->xgmi_sdma_bitmap = (1ULL << get_num_xgmi_sdma_queues(dqm)) - 1;
64c7f8cf 881
64c7f8cf
BG		 882 return 0;
883}
884
58dcd5bf 885static void uninitialize(struct device_queue_manager *dqm)
64c7f8cf 886{
6f9d54fd
OG		 887 int i;
888
32fa8219 889 WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
64c7f8cf
BG		 890
891 kfree(dqm->allocated_queues);
6f9d54fd 892 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
8d5f3552 893 kfree(dqm->mqd_mgrs[i]);
efeaed4d 894 mutex_destroy(&dqm->lock_hidden);
a86aa3ca 895 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
64c7f8cf
BG		 896}
897
898static int start_nocpsch(struct device_queue_manager *dqm)
899{
2249d558 900 init_interrupts(dqm);
552764b6 901 return pm_init(&dqm->packets, dqm);
64c7f8cf
BG		 902}
903
904static int stop_nocpsch(struct device_queue_manager *dqm)
905{
552764b6 906 pm_uninit(&dqm->packets);
64c7f8cf
BG		 907 return 0;
908}
909
bcea3081 910static int allocate_sdma_queue(struct device_queue_manager *dqm,
e78579aa 911 struct queue *q)
bcea3081
BG		 912{
913 int bit;
914
1b4670f6
OZ		 915 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
916 if (dqm->sdma_bitmap == 0)
917 return -ENOMEM;
918 bit = __ffs64(dqm->sdma_bitmap);
919 dqm->sdma_bitmap &= ~(1ULL << bit);
920 q->sdma_id = bit;
921 q->properties.sdma_engine_id = q->sdma_id %
922 get_num_sdma_engines(dqm);
923 q->properties.sdma_queue_id = q->sdma_id /
924 get_num_sdma_engines(dqm);
925 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
926 if (dqm->xgmi_sdma_bitmap == 0)
927 return -ENOMEM;
928 bit = __ffs64(dqm->xgmi_sdma_bitmap);
929 dqm->xgmi_sdma_bitmap &= ~(1ULL << bit);
930 q->sdma_id = bit;
931 /* sdma_engine_id is sdma id including
932 * both PCIe-optimized SDMAs and XGMI-
933 * optimized SDMAs. The calculation below
934 * assumes the first N engines are always
935 * PCIe-optimized ones
936 */
937 q->properties.sdma_engine_id = get_num_sdma_engines(dqm) +
938 q->sdma_id % get_num_xgmi_sdma_engines(dqm);
939 q->properties.sdma_queue_id = q->sdma_id /
940 get_num_xgmi_sdma_engines(dqm);
941 }
e78579aa 942
e78579aa
YZ		 943 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
944 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
bcea3081
BG		 945
946 return 0;
947}
948
949static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1b4670f6 950 struct queue *q)
bcea3081 951{
1b4670f6
OZ		 952 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
953 if (q->sdma_id >= get_num_sdma_queues(dqm))
954 return;
955 dqm->sdma_bitmap |= (1ULL << q->sdma_id);
956 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
957 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
958 return;
959 dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id);
960 }
bcea3081
BG		 961}
962
64c7f8cf
BG		 963/*
964 * Device Queue Manager implementation for cp scheduler
965 */
966
967static int set_sched_resources(struct device_queue_manager *dqm)
968{
d0b63bb3 969 int i, mec;
64c7f8cf 970 struct scheduling_resources res;
64c7f8cf 971
44008d7a 972 res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
d0b63bb3
AR		 973
974 res.queue_mask = 0;
975 for (i = 0; i < KGD_MAX_QUEUES; ++i) {
976 mec = (i / dqm->dev->shared_resources.num_queue_per_pipe)
977 / dqm->dev->shared_resources.num_pipe_per_mec;
978
979 if (!test_bit(i, dqm->dev->shared_resources.queue_bitmap))
980 continue;
981
982 /* only acquire queues from the first MEC */
983 if (mec > 0)
984 continue;
985
986 /* This situation may be hit in the future if a new HW
987 * generation exposes more than 64 queues. If so, the
8eabaf54
KR		 988 * definition of res.queue_mask needs updating
989 */
1d11ee89 990 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
d0b63bb3
AR		 991 pr_err("Invalid queue enabled by amdgpu: %d\n", i);
992 break;
993 }
994
995 res.queue_mask |= (1ull << i);
996 }
64c7f8cf
BG		 997 res.gws_mask = res.oac_mask = res.gds_heap_base =
998 res.gds_heap_size = 0;
999
79775b62
KR		 1000 pr_debug("Scheduling resources:\n"
1001 "vmid mask: 0x%8X\n"
1002 "queue mask: 0x%8llX\n",
64c7f8cf
BG		 1003 res.vmid_mask, res.queue_mask);
1004
1005 return pm_send_set_resources(&dqm->packets, &res);
1006}
1007
1008static int initialize_cpsch(struct device_queue_manager *dqm)
1009{
79775b62 1010 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf 1011
efeaed4d 1012 mutex_init(&dqm->lock_hidden);
64c7f8cf
BG		 1013 INIT_LIST_HEAD(&dqm->queues);
1014 dqm->queue_count = dqm->processes_count = 0;
bcea3081 1015 dqm->sdma_queue_count = 0;
1b4670f6 1016 dqm->xgmi_sdma_queue_count = 0;
64c7f8cf 1017 dqm->active_runlist = false;
cb77ee7c 1018 dqm->sdma_bitmap = (1ULL << get_num_sdma_queues(dqm)) - 1;
1b4670f6 1019 dqm->xgmi_sdma_bitmap = (1ULL << get_num_xgmi_sdma_queues(dqm)) - 1;
64c7f8cf 1020
73ea648d
SL		 1021 INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
1022
bfd5e378 1023 return 0;
64c7f8cf
BG		 1024}
1025
1026static int start_cpsch(struct device_queue_manager *dqm)
1027{
64c7f8cf
BG		 1028 int retval;
1029
64c7f8cf
BG		 1030 retval = 0;
1031
1032 retval = pm_init(&dqm->packets, dqm);
4eacc26b 1033 if (retval)
64c7f8cf
BG		 1034 goto fail_packet_manager_init;
1035
1036 retval = set_sched_resources(dqm);
4eacc26b 1037 if (retval)
64c7f8cf
BG		 1038 goto fail_set_sched_resources;
1039
79775b62 1040 pr_debug("Allocating fence memory\n");
64c7f8cf
BG		 1041
1042 /* allocate fence memory on the gart */
a86aa3ca
OG		 1043 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
1044 &dqm->fence_mem);
64c7f8cf 1045
4eacc26b 1046 if (retval)
64c7f8cf
BG		 1047 goto fail_allocate_vidmem;
1048
1049 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
1050 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
2249d558
AL		 1051
1052 init_interrupts(dqm);
1053
efeaed4d 1054 dqm_lock(dqm);
73ea648d
SL		 1055 /* clear hang status when driver try to start the hw scheduler */
1056 dqm->is_hws_hang = false;
c4744e24 1057 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
efeaed4d 1058 dqm_unlock(dqm);
64c7f8cf
BG		 1059
1060 return 0;
1061fail_allocate_vidmem:
1062fail_set_sched_resources:
1063 pm_uninit(&dqm->packets);
1064fail_packet_manager_init:
1065 return retval;
1066}
1067
1068static int stop_cpsch(struct device_queue_manager *dqm)
1069{
efeaed4d 1070 dqm_lock(dqm);
4465f466 1071 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
efeaed4d 1072 dqm_unlock(dqm);
64c7f8cf 1073
a86aa3ca 1074 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
64c7f8cf
BG		 1075 pm_uninit(&dqm->packets);
1076
1077 return 0;
1078}
1079
1080static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1081 struct kernel_queue *kq,
1082 struct qcm_process_device *qpd)
1083{
efeaed4d 1084 dqm_lock(dqm);
b8cbab04 1085 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1086 pr_warn("Can't create new kernel queue because %d queues were already created\n",
b8cbab04 1087 dqm->total_queue_count);
efeaed4d 1088 dqm_unlock(dqm);
b8cbab04
OG		 1089 return -EPERM;
1090 }
1091
1092 /*
1093 * Unconditionally increment this counter, regardless of the queue's
1094 * type or whether the queue is active.
1095 */
1096 dqm->total_queue_count++;
1097 pr_debug("Total of %d queues are accountable so far\n",
1098 dqm->total_queue_count);
1099
64c7f8cf
BG		 1100 list_add(&kq->list, &qpd->priv_queue_list);
1101 dqm->queue_count++;
1102 qpd->is_debug = true;
c4744e24 1103 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
efeaed4d 1104 dqm_unlock(dqm);
64c7f8cf
BG		 1105
1106 return 0;
1107}
1108
1109static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1110 struct kernel_queue *kq,
1111 struct qcm_process_device *qpd)
1112{
efeaed4d 1113 dqm_lock(dqm);
64c7f8cf
BG		 1114 list_del(&kq->list);
1115 dqm->queue_count--;
1116 qpd->is_debug = false;
c4744e24 1117 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
b8cbab04
OG		 1118 /*
1119 * Unconditionally decrement this counter, regardless of the queue's
1120 * type.
1121 */
8b58f261 1122 dqm->total_queue_count--;
b8cbab04
OG		 1123 pr_debug("Total of %d queues are accountable so far\n",
1124 dqm->total_queue_count);
efeaed4d 1125 dqm_unlock(dqm);
64c7f8cf
BG		 1126}
1127
1128static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
b46cb7d7 1129 struct qcm_process_device *qpd)
64c7f8cf
BG		 1130{
1131 int retval;
8d5f3552 1132 struct mqd_manager *mqd_mgr;
64c7f8cf 1133
b8cbab04 1134 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1135 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04 1136 dqm->total_queue_count);
70d488fb
OZ		 1137 retval = -EPERM;
1138 goto out;
b8cbab04
OG		 1139 }
1140
1b4670f6
OZ		 1141 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1142 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
e78579aa 1143 retval = allocate_sdma_queue(dqm, q);
894a8293 1144 if (retval)
70d488fb 1145 goto out;
e139cd2a 1146 }
ef568db7
FK		 1147
1148 retval = allocate_doorbell(qpd, q);
1149 if (retval)
1150 goto out_deallocate_sdma_queue;
1151
70d488fb
OZ		 1152 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1153 q->properties.type)];
26103436 1154 /*
bb2d2128
FK		 1155 * Eviction state logic: mark all queues as evicted, even ones
1156 * not currently active. Restoring inactive queues later only
1157 * updates the is_evicted flag but is a no-op otherwise.
26103436 1158 */
bb2d2128 1159 q->properties.is_evicted = !!qpd->evicted;
eec0b4cf
OZ		 1160 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1161 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1162 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
373d7080
FK		 1163 q->properties.tba_addr = qpd->tba_addr;
1164 q->properties.tma_addr = qpd->tma_addr;
70d488fb
OZ		 1165 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
1166 if (!q->mqd_mem_obj) {
1167 retval = -ENOMEM;
1168 goto out_deallocate_doorbell;
1169 }
8636e53c
OZ		 1170 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
1171 &q->gart_mqd_addr, &q->properties);
70d488fb 1172 dqm_lock(dqm);
89cd9d23 1173
64c7f8cf 1174 list_add(&q->list, &qpd->queues_list);
bc920fd4 1175 qpd->queue_count++;
64c7f8cf
BG		 1176 if (q->properties.is_active) {
1177 dqm->queue_count++;
c4744e24
YZ		 1178 retval = execute_queues_cpsch(dqm,
1179 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
64c7f8cf
BG		 1180 }
1181
bcea3081 1182 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
8eabaf54 1183 dqm->sdma_queue_count++;
1b4670f6
OZ		 1184 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1185 dqm->xgmi_sdma_queue_count++;
b8cbab04
OG		 1186 /*
1187 * Unconditionally increment this counter, regardless of the queue's
1188 * type or whether the queue is active.
1189 */
1190 dqm->total_queue_count++;
1191
1192 pr_debug("Total of %d queues are accountable so far\n",
1193 dqm->total_queue_count);
1194
efeaed4d 1195 dqm_unlock(dqm);
72a01d23
FK		 1196 return retval;
1197
70d488fb
OZ		 1198out_deallocate_doorbell:
1199 deallocate_doorbell(qpd, q);
72a01d23 1200out_deallocate_sdma_queue:
1b4670f6
OZ		 1201 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1202 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1203 deallocate_sdma_queue(dqm, q);
70d488fb 1204out:
64c7f8cf
BG		 1205 return retval;
1206}
1207
788bf83d 1208int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
d80d19bd 1209 unsigned int fence_value,
8c72c3d7 1210 unsigned int timeout_ms)
64c7f8cf 1211{
8c72c3d7 1212 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
64c7f8cf
BG		 1213
1214 while (*fence_addr != fence_value) {
8c72c3d7 1215 if (time_after(jiffies, end_jiffies)) {
79775b62 1216 pr_err("qcm fence wait loop timeout expired\n");
0e9a860c
YZ		 1217 /* In HWS case, this is used to halt the driver thread
1218 * in order not to mess up CP states before doing
1219 * scandumps for FW debugging.
1220 */
1221 while (halt_if_hws_hang)
1222 schedule();
1223
64c7f8cf
BG		 1224 return -ETIME;
1225 }
99331a51 1226 schedule();
64c7f8cf
BG		 1227 }
1228
1229 return 0;
1230}
1231
065e4bdf 1232static int unmap_sdma_queues(struct device_queue_manager *dqm)
bcea3081 1233{
065e4bdf
OZ		 1234 int i, retval = 0;
1235
1b4670f6
OZ		 1236 for (i = 0; i < dqm->dev->device_info->num_sdma_engines +
1237 dqm->dev->device_info->num_xgmi_sdma_engines; i++) {
065e4bdf
OZ		 1238 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
1239 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false, i);
1240 if (retval)
1241 return retval;
1242 }
1243 return retval;
bcea3081
BG		 1244}
1245
60a00956
FK		 1246/* dqm->lock mutex has to be locked before calling this function */
1247static int map_queues_cpsch(struct device_queue_manager *dqm)
1248{
1249 int retval;
1250
1251 if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
1252 return 0;
1253
1254 if (dqm->active_runlist)
1255 return 0;
1256
1257 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1258 if (retval) {
1259 pr_err("failed to execute runlist\n");
1260 return retval;
1261 }
1262 dqm->active_runlist = true;
1263
1264 return retval;
1265}
1266
ac30c783 1267/* dqm->lock mutex has to be locked before calling this function */
7da2bcf8 1268static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 1269 enum kfd_unmap_queues_filter filter,
1270 uint32_t filter_param)
64c7f8cf 1271{
9fd3f1bf 1272 int retval = 0;
64c7f8cf 1273
73ea648d
SL		 1274 if (dqm->is_hws_hang)
1275 return -EIO;
991ca8ee 1276 if (!dqm->active_runlist)
ac30c783 1277 return retval;
bcea3081 1278
1b4670f6
OZ		 1279 pr_debug("Before destroying queues, sdma queue count is : %u, xgmi sdma queue count is : %u\n",
1280 dqm->sdma_queue_count, dqm->xgmi_sdma_queue_count);
bcea3081 1281
1b4670f6 1282 if (dqm->sdma_queue_count > 0 || dqm->xgmi_sdma_queue_count)
065e4bdf 1283 unmap_sdma_queues(dqm);
bcea3081 1284
64c7f8cf 1285 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
4465f466 1286 filter, filter_param, false, 0);
4eacc26b 1287 if (retval)
ac30c783 1288 return retval;
64c7f8cf
BG		 1289
1290 *dqm->fence_addr = KFD_FENCE_INIT;
1291 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
1292 KFD_FENCE_COMPLETED);
1293 /* should be timed out */
c3447e81 1294 retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
64c7f8cf 1295 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
9fd3f1bf 1296 if (retval)
ac30c783 1297 return retval;
9fd3f1bf 1298
64c7f8cf
BG		 1299 pm_release_ib(&dqm->packets);
1300 dqm->active_runlist = false;
1301
64c7f8cf
BG		 1302 return retval;
1303}
1304
ac30c783 1305/* dqm->lock mutex has to be locked before calling this function */
c4744e24
YZ		 1306static int execute_queues_cpsch(struct device_queue_manager *dqm,
1307 enum kfd_unmap_queues_filter filter,
1308 uint32_t filter_param)
64c7f8cf
BG		 1309{
1310 int retval;
1311
73ea648d
SL		 1312 if (dqm->is_hws_hang)
1313 return -EIO;
c4744e24 1314 retval = unmap_queues_cpsch(dqm, filter, filter_param);
4eacc26b 1315 if (retval) {
c4744e24 1316 pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
73ea648d
SL		 1317 dqm->is_hws_hang = true;
1318 schedule_work(&dqm->hw_exception_work);
ac30c783 1319 return retval;
64c7f8cf
BG		 1320 }
1321
60a00956 1322 return map_queues_cpsch(dqm);
64c7f8cf
BG		 1323}
1324
1325static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1326 struct qcm_process_device *qpd,
1327 struct queue *q)
1328{
1329 int retval;
8d5f3552 1330 struct mqd_manager *mqd_mgr;
992839ad 1331
64c7f8cf
BG		 1332 retval = 0;
1333
1334 /* remove queue from list to prevent rescheduling after preemption */
efeaed4d 1335 dqm_lock(dqm);
992839ad
YS		 1336
1337 if (qpd->is_debug) {
1338 /*
1339 * error, currently we do not allow to destroy a queue
1340 * of a currently debugged process
1341 */
1342 retval = -EBUSY;
1343 goto failed_try_destroy_debugged_queue;
1344
1345 }
1346
fdfa090b
OZ		 1347 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1348 q->properties.type)];
64c7f8cf 1349
ef568db7
FK		 1350 deallocate_doorbell(qpd, q);
1351
e139cd2a 1352 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
bcea3081 1353 dqm->sdma_queue_count--;
1b4670f6
OZ		 1354 deallocate_sdma_queue(dqm, q);
1355 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1356 dqm->xgmi_sdma_queue_count--;
1357 deallocate_sdma_queue(dqm, q);
e139cd2a 1358 }
bcea3081 1359
64c7f8cf 1360 list_del(&q->list);
bc920fd4 1361 qpd->queue_count--;
40a526dc 1362 if (q->properties.is_active) {
b6819cec 1363 dqm->queue_count--;
40a526dc 1364 retval = execute_queues_cpsch(dqm,
9fd3f1bf 1365 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
40a526dc
YZ		 1366 if (retval == -ETIME)
1367 qpd->reset_wavefronts = true;
1368 }
64c7f8cf 1369
b8cbab04
OG		 1370 /*
1371 * Unconditionally decrement this counter, regardless of the queue's
1372 * type
1373 */
1374 dqm->total_queue_count--;
1375 pr_debug("Total of %d queues are accountable so far\n",
1376 dqm->total_queue_count);
64c7f8cf 1377
efeaed4d 1378 dqm_unlock(dqm);
64c7f8cf 1379
8636e53c
OZ		 1380 /* Do free_mqd after dqm_unlock(dqm) to avoid circular locking */
1381 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
89cd9d23 1382
9e827224 1383 return retval;
64c7f8cf 1384
992839ad
YS		 1385failed_try_destroy_debugged_queue:
1386
efeaed4d 1387 dqm_unlock(dqm);
64c7f8cf
BG		 1388 return retval;
1389}
1390
1391/*
1392 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1393 * stay in user mode.
1394 */
1395#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1396/* APE1 limit is inclusive and 64K aligned. */
1397#define APE1_LIMIT_ALIGNMENT 0xFFFF
1398
1399static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1400 struct qcm_process_device *qpd,
1401 enum cache_policy default_policy,
1402 enum cache_policy alternate_policy,
1403 void __user *alternate_aperture_base,
1404 uint64_t alternate_aperture_size)
1405{
bed4f110
FK		 1406 bool retval = true;
1407
1408 if (!dqm->asic_ops.set_cache_memory_policy)
1409 return retval;
64c7f8cf 1410
efeaed4d 1411 dqm_lock(dqm);
64c7f8cf
BG		 1412
1413 if (alternate_aperture_size == 0) {
1414 /* base > limit disables APE1 */
1415 qpd->sh_mem_ape1_base = 1;
1416 qpd->sh_mem_ape1_limit = 0;
1417 } else {
1418 /*
1419 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1420 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1421 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1422 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1423 * Verify that the base and size parameters can be
1424 * represented in this format and convert them.
1425 * Additionally restrict APE1 to user-mode addresses.
1426 */
1427
1428 uint64_t base = (uintptr_t)alternate_aperture_base;
1429 uint64_t limit = base + alternate_aperture_size - 1;
1430
ab7c1648
KR		 1431 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
1432 (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
1433 retval = false;
64c7f8cf 1434 goto out;
ab7c1648 1435 }
64c7f8cf
BG		 1436
1437 qpd->sh_mem_ape1_base = base >> 16;
1438 qpd->sh_mem_ape1_limit = limit >> 16;
1439 }
1440
bfd5e378 1441 retval = dqm->asic_ops.set_cache_memory_policy(
a22fc854
BG		 1442 dqm,
1443 qpd,
1444 default_policy,
1445 alternate_policy,
1446 alternate_aperture_base,
1447 alternate_aperture_size);
64c7f8cf 1448
d146c5a7 1449 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
64c7f8cf
BG		 1450 program_sh_mem_settings(dqm, qpd);
1451
79775b62 1452 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
64c7f8cf
BG		 1453 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1454 qpd->sh_mem_ape1_limit);
1455
64c7f8cf 1456out:
efeaed4d 1457 dqm_unlock(dqm);
ab7c1648 1458 return retval;
64c7f8cf
BG		 1459}
1460
d7b9bd22
FK		 1461static int set_trap_handler(struct device_queue_manager *dqm,
1462 struct qcm_process_device *qpd,
1463 uint64_t tba_addr,
1464 uint64_t tma_addr)
1465{
1466 uint64_t *tma;
1467
1468 if (dqm->dev->cwsr_enabled) {
1469 /* Jump from CWSR trap handler to user trap */
1470 tma = (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1471 tma[0] = tba_addr;
1472 tma[1] = tma_addr;
1473 } else {
1474 qpd->tba_addr = tba_addr;
1475 qpd->tma_addr = tma_addr;
1476 }
1477
1478 return 0;
1479}
1480
9fd3f1bf
FK		 1481static int process_termination_nocpsch(struct device_queue_manager *dqm,
1482 struct qcm_process_device *qpd)
1483{
1484 struct queue *q, *next;
1485 struct device_process_node *cur, *next_dpn;
1486 int retval = 0;
32cce8bc 1487 bool found = false;
9fd3f1bf 1488
efeaed4d 1489 dqm_lock(dqm);
9fd3f1bf
FK		 1490
1491 /* Clear all user mode queues */
1492 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1493 int ret;
1494
1495 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
1496 if (ret)
1497 retval = ret;
1498 }
1499
1500 /* Unregister process */
1501 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1502 if (qpd == cur->qpd) {
1503 list_del(&cur->list);
1504 kfree(cur);
1505 dqm->processes_count--;
32cce8bc 1506 found = true;
9fd3f1bf
FK		 1507 break;
1508 }
1509 }
1510
efeaed4d 1511 dqm_unlock(dqm);
32cce8bc
FK		 1512
1513 /* Outside the DQM lock because under the DQM lock we can't do
1514 * reclaim or take other locks that others hold while reclaiming.
1515 */
1516 if (found)
1517 kfd_dec_compute_active(dqm->dev);
1518
9fd3f1bf
FK		 1519 return retval;
1520}
1521
5df099e8
JC		 1522static int get_wave_state(struct device_queue_manager *dqm,
1523 struct queue *q,
1524 void __user *ctl_stack,
1525 u32 *ctl_stack_used_size,
1526 u32 *save_area_used_size)
1527{
4e6c6fc1 1528 struct mqd_manager *mqd_mgr;
5df099e8
JC		 1529 int r;
1530
1531 dqm_lock(dqm);
1532
1533 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
1534 q->properties.is_active || !q->device->cwsr_enabled) {
1535 r = -EINVAL;
1536 goto dqm_unlock;
1537 }
1538
fdfa090b 1539 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_COMPUTE];
5df099e8 1540
4e6c6fc1 1541 if (!mqd_mgr->get_wave_state) {
5df099e8
JC		 1542 r = -EINVAL;
1543 goto dqm_unlock;
1544 }
1545
4e6c6fc1
YZ		 1546 r = mqd_mgr->get_wave_state(mqd_mgr, q->mqd, ctl_stack,
1547 ctl_stack_used_size, save_area_used_size);
5df099e8
JC		 1548
1549dqm_unlock:
1550 dqm_unlock(dqm);
1551 return r;
1552}
9fd3f1bf
FK		 1553
1554static int process_termination_cpsch(struct device_queue_manager *dqm,
1555 struct qcm_process_device *qpd)
1556{
1557 int retval;
1558 struct queue *q, *next;
1559 struct kernel_queue *kq, *kq_next;
8d5f3552 1560 struct mqd_manager *mqd_mgr;
9fd3f1bf
FK		 1561 struct device_process_node *cur, *next_dpn;
1562 enum kfd_unmap_queues_filter filter =
1563 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
32cce8bc 1564 bool found = false;
9fd3f1bf
FK		 1565
1566 retval = 0;
1567
efeaed4d 1568 dqm_lock(dqm);
9fd3f1bf
FK		 1569
1570 /* Clean all kernel queues */
1571 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
1572 list_del(&kq->list);
1573 dqm->queue_count--;
1574 qpd->is_debug = false;
1575 dqm->total_queue_count--;
1576 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
1577 }
1578
1579 /* Clear all user mode queues */
1580 list_for_each_entry(q, &qpd->queues_list, list) {
72a01d23 1581 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
9fd3f1bf 1582 dqm->sdma_queue_count--;
1b4670f6
OZ		 1583 deallocate_sdma_queue(dqm, q);
1584 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1585 dqm->xgmi_sdma_queue_count--;
1586 deallocate_sdma_queue(dqm, q);
72a01d23 1587 }
9fd3f1bf
FK		 1588
1589 if (q->properties.is_active)
1590 dqm->queue_count--;
1591
1592 dqm->total_queue_count--;
1593 }
1594
1595 /* Unregister process */
1596 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1597 if (qpd == cur->qpd) {
1598 list_del(&cur->list);
1599 kfree(cur);
1600 dqm->processes_count--;
32cce8bc 1601 found = true;
9fd3f1bf
FK		 1602 break;
1603 }
1604 }
1605
1606 retval = execute_queues_cpsch(dqm, filter, 0);
73ea648d 1607 if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
9fd3f1bf
FK		 1608 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
1609 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
1610 qpd->reset_wavefronts = false;
1611 }
1612
89cd9d23
PY		 1613 dqm_unlock(dqm);
1614
32cce8bc
FK		 1615 /* Outside the DQM lock because under the DQM lock we can't do
1616 * reclaim or take other locks that others hold while reclaiming.
1617 */
1618 if (found)
1619 kfd_dec_compute_active(dqm->dev);
1620
89cd9d23 1621 /* Lastly, free mqd resources.
8636e53c 1622 * Do free_mqd() after dqm_unlock to avoid circular locking.
89cd9d23 1623 */
9fd3f1bf 1624 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
fdfa090b
OZ		 1625 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1626 q->properties.type)];
9fd3f1bf 1627 list_del(&q->list);
bc920fd4 1628 qpd->queue_count--;
8636e53c 1629 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
9fd3f1bf
FK		 1630 }
1631
9fd3f1bf
FK		 1632 return retval;
1633}
1634
fdfa090b
OZ		 1635static int init_mqd_managers(struct device_queue_manager *dqm)
1636{
1637 int i, j;
1638 struct mqd_manager *mqd_mgr;
1639
1640 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
1641 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
1642 if (!mqd_mgr) {
1643 pr_err("mqd manager [%d] initialization failed\n", i);
1644 goto out_free;
1645 }
1646 dqm->mqd_mgrs[i] = mqd_mgr;
1647 }
1648
1649 return 0;
1650
1651out_free:
1652 for (j = 0; j < i; j++) {
1653 kfree(dqm->mqd_mgrs[j]);
1654 dqm->mqd_mgrs[j] = NULL;
1655 }
1656
1657 return -ENOMEM;
1658}
11614c36
OZ		 1659
1660/* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
1661static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
1662{
1663 int retval;
1664 struct kfd_dev *dev = dqm->dev;
1665 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
1666 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
1667 dev->device_info->num_sdma_engines *
1668 dev->device_info->num_sdma_queues_per_engine +
1669 dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size;
1670
1671 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->kgd, size,
1672 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr),
1673 (void *)&(mem_obj->cpu_ptr), true);
1674
1675 return retval;
1676}
1677
64c7f8cf
BG		 1678struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1679{
1680 struct device_queue_manager *dqm;
1681
79775b62 1682 pr_debug("Loading device queue manager\n");
a22fc854 1683
dbf56ab1 1684 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
64c7f8cf
BG		 1685 if (!dqm)
1686 return NULL;
1687
d146c5a7
FK		 1688 switch (dev->device_info->asic_family) {
1689 /* HWS is not available on Hawaii. */
1690 case CHIP_HAWAII:
1691 /* HWS depends on CWSR for timely dequeue. CWSR is not
1692 * available on Tonga.
1693 *
1694 * FIXME: This argument also applies to Kaveri.
1695 */
1696 case CHIP_TONGA:
1697 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
1698 break;
1699 default:
1700 dqm->sched_policy = sched_policy;
1701 break;
1702 }
1703
64c7f8cf 1704 dqm->dev = dev;
d146c5a7 1705 switch (dqm->sched_policy) {
64c7f8cf
BG		 1706 case KFD_SCHED_POLICY_HWS:
1707 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1708 /* initialize dqm for cp scheduling */
45c9a5e4
OG		 1709 dqm->ops.create_queue = create_queue_cpsch;
1710 dqm->ops.initialize = initialize_cpsch;
1711 dqm->ops.start = start_cpsch;
1712 dqm->ops.stop = stop_cpsch;
1713 dqm->ops.destroy_queue = destroy_queue_cpsch;
1714 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1715 dqm->ops.register_process = register_process;
1716 dqm->ops.unregister_process = unregister_process;
1717 dqm->ops.uninitialize = uninitialize;
45c9a5e4
OG		 1718 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1719 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1720 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
d7b9bd22 1721 dqm->ops.set_trap_handler = set_trap_handler;
9fd3f1bf 1722 dqm->ops.process_termination = process_termination_cpsch;
26103436
FK		 1723 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
1724 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
5df099e8 1725 dqm->ops.get_wave_state = get_wave_state;
64c7f8cf
BG		 1726 break;
1727 case KFD_SCHED_POLICY_NO_HWS:
1728 /* initialize dqm for no cp scheduling */
45c9a5e4
OG		 1729 dqm->ops.start = start_nocpsch;
1730 dqm->ops.stop = stop_nocpsch;
1731 dqm->ops.create_queue = create_queue_nocpsch;
1732 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1733 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1734 dqm->ops.register_process = register_process;
1735 dqm->ops.unregister_process = unregister_process;
45c9a5e4 1736 dqm->ops.initialize = initialize_nocpsch;
58dcd5bf 1737 dqm->ops.uninitialize = uninitialize;
45c9a5e4 1738 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
d7b9bd22 1739 dqm->ops.set_trap_handler = set_trap_handler;
9fd3f1bf 1740 dqm->ops.process_termination = process_termination_nocpsch;
26103436
FK		 1741 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
1742 dqm->ops.restore_process_queues =
1743 restore_process_queues_nocpsch;
5df099e8 1744 dqm->ops.get_wave_state = get_wave_state;
64c7f8cf
BG		 1745 break;
1746 default:
d146c5a7 1747 pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
32fa8219 1748 goto out_free;
64c7f8cf
BG		 1749 }
1750
a22fc854
BG		 1751 switch (dev->device_info->asic_family) {
1752 case CHIP_CARRIZO:
bfd5e378 1753 device_queue_manager_init_vi(&dqm->asic_ops);
300dec95
OG		 1754 break;
1755
a22fc854 1756 case CHIP_KAVERI:
bfd5e378 1757 device_queue_manager_init_cik(&dqm->asic_ops);
300dec95 1758 break;
97672cbe
FK		 1759
1760 case CHIP_HAWAII:
1761 device_queue_manager_init_cik_hawaii(&dqm->asic_ops);
1762 break;
1763
1764 case CHIP_TONGA:
1765 case CHIP_FIJI:
1766 case CHIP_POLARIS10:
1767 case CHIP_POLARIS11:
846a44d7 1768 case CHIP_POLARIS12:
ed81cd6e 1769 case CHIP_VEGAM:
97672cbe
FK		 1770 device_queue_manager_init_vi_tonga(&dqm->asic_ops);
1771 break;
bed4f110
FK		 1772
1773 case CHIP_VEGA10:
846a44d7 1774 case CHIP_VEGA12:
22a3a294 1775 case CHIP_VEGA20:
bed4f110
FK		 1776 case CHIP_RAVEN:
1777 device_queue_manager_init_v9(&dqm->asic_ops);
1778 break;
e596b903
YZ		 1779 default:
1780 WARN(1, "Unexpected ASIC family %u",
1781 dev->device_info->asic_family);
1782 goto out_free;
a22fc854
BG		 1783 }
1784
fdfa090b
OZ		 1785 if (init_mqd_managers(dqm))
1786 goto out_free;
1787
11614c36
OZ		 1788 if (allocate_hiq_sdma_mqd(dqm)) {
1789 pr_err("Failed to allocate hiq sdma mqd trunk buffer\n");
1790 goto out_free;
1791 }
1792
32fa8219
FK		 1793 if (!dqm->ops.initialize(dqm))
1794 return dqm;
64c7f8cf 1795
32fa8219
FK		 1796out_free:
1797 kfree(dqm);
1798 return NULL;
64c7f8cf
BG		 1799}
1800
11614c36
OZ		 1801void deallocate_hiq_sdma_mqd(struct kfd_dev *dev, struct kfd_mem_obj *mqd)
1802{
1803 WARN(!mqd, "No hiq sdma mqd trunk to free");
1804
1805 amdgpu_amdkfd_free_gtt_mem(dev->kgd, mqd->gtt_mem);
1806}
1807
64c7f8cf
BG		 1808void device_queue_manager_uninit(struct device_queue_manager *dqm)
1809{
45c9a5e4 1810 dqm->ops.uninitialize(dqm);
11614c36 1811 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
64c7f8cf
BG		 1812 kfree(dqm);
1813}
851a645e 1814
2640c3fa 1815int kfd_process_vm_fault(struct device_queue_manager *dqm,
1816 unsigned int pasid)
1817{
1818 struct kfd_process_device *pdd;
1819 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
1820 int ret = 0;
1821
1822 if (!p)
1823 return -EINVAL;
1824 pdd = kfd_get_process_device_data(dqm->dev, p);
1825 if (pdd)
1826 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
1827 kfd_unref_process(p);
1828
1829 return ret;
1830}
1831
73ea648d
SL		 1832static void kfd_process_hw_exception(struct work_struct *work)
1833{
1834 struct device_queue_manager *dqm = container_of(work,
1835 struct device_queue_manager, hw_exception_work);
5b87245f 1836 amdgpu_amdkfd_gpu_reset(dqm->dev->kgd);
73ea648d
SL		 1837}
1838
851a645e
FK		 1839#if defined(CONFIG_DEBUG_FS)
1840
1841static void seq_reg_dump(struct seq_file *m,
1842 uint32_t (*dump)[2], uint32_t n_regs)
1843{
1844 uint32_t i, count;
1845
1846 for (i = 0, count = 0; i < n_regs; i++) {
1847 if (count == 0 ||
1848 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
1849 seq_printf(m, "%s %08x: %08x",
1850 i ? "\n" : "",
1851 dump[i][0], dump[i][1]);
1852 count = 7;
1853 } else {
1854 seq_printf(m, " %08x", dump[i][1]);
1855 count--;
1856 }
1857 }
1858
1859 seq_puts(m, "\n");
1860}
1861
1862int dqm_debugfs_hqds(struct seq_file *m, void *data)
1863{
1864 struct device_queue_manager *dqm = data;
1865 uint32_t (*dump)[2], n_regs;
1866 int pipe, queue;
1867 int r = 0;
1868
24f48a42
OZ		 1869 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
1870 KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE, &dump, &n_regs);
1871 if (!r) {
1872 seq_printf(m, " HIQ on MEC %d Pipe %d Queue %d\n",
1873 KFD_CIK_HIQ_PIPE/get_pipes_per_mec(dqm)+1,
1874 KFD_CIK_HIQ_PIPE%get_pipes_per_mec(dqm),
1875 KFD_CIK_HIQ_QUEUE);
1876 seq_reg_dump(m, dump, n_regs);
1877
1878 kfree(dump);
1879 }
1880
851a645e
FK		 1881 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1882 int pipe_offset = pipe * get_queues_per_pipe(dqm);
1883
1884 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
1885 if (!test_bit(pipe_offset + queue,
1886 dqm->dev->shared_resources.queue_bitmap))
1887 continue;
1888
1889 r = dqm->dev->kfd2kgd->hqd_dump(
1890 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
1891 if (r)
1892 break;
1893
1894 seq_printf(m, " CP Pipe %d, Queue %d\n",
1895 pipe, queue);
1896 seq_reg_dump(m, dump, n_regs);
1897
1898 kfree(dump);
1899 }
1900 }
1901
98bb9222 1902 for (pipe = 0; pipe < get_num_sdma_engines(dqm); pipe++) {
d5094189
SL		 1903 for (queue = 0;
1904 queue < dqm->dev->device_info->num_sdma_queues_per_engine;
1905 queue++) {
851a645e
FK		 1906 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
1907 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
1908 if (r)
1909 break;
1910
1911 seq_printf(m, " SDMA Engine %d, RLC %d\n",
1912 pipe, queue);
1913 seq_reg_dump(m, dump, n_regs);
1914
1915 kfree(dump);
1916 }
1917 }
1918
1919 return r;
1920}
1921
a29ec470
SL		 1922int dqm_debugfs_execute_queues(struct device_queue_manager *dqm)
1923{
1924 int r = 0;
1925
1926 dqm_lock(dqm);
1927 dqm->active_runlist = true;
1928 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
1929 dqm_unlock(dqm);
1930
1931 return r;
1932}
1933
851a645e 1934#endif
Linux 6.x block layer and io_uring tree(s)