projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Linux 5.9-rc5
[linux-block.git] / drivers / gpu / drm / amd / amdkfd / kfd_device_queue_manager.c
CommitLineData
64c7f8cf
BG		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
26103436
FK		 24#include <linux/ratelimit.h>
25#include <linux/printk.h>
64c7f8cf
BG		 26#include <linux/slab.h>
27#include <linux/list.h>
28#include <linux/types.h>
64c7f8cf 29#include <linux/bitops.h>
99331a51 30#include <linux/sched.h>
64c7f8cf
BG		 31#include "kfd_priv.h"
32#include "kfd_device_queue_manager.h"
33#include "kfd_mqd_manager.h"
34#include "cik_regs.h"
35#include "kfd_kernel_queue.h"
5b87245f 36#include "amdgpu_amdkfd.h"
64c7f8cf
BG		 37
38/* Size of the per-pipe EOP queue */
39#define CIK_HPD_EOP_BYTES_LOG2 11
40#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
41
64c7f8cf
BG		 42static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
43 unsigned int pasid, unsigned int vmid);
44
c4744e24
YZ		 45static int execute_queues_cpsch(struct device_queue_manager *dqm,
46 enum kfd_unmap_queues_filter filter,
47 uint32_t filter_param);
7da2bcf8 48static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 49 enum kfd_unmap_queues_filter filter,
50 uint32_t filter_param);
64c7f8cf 51
60a00956
FK		 52static int map_queues_cpsch(struct device_queue_manager *dqm);
53
bcea3081 54static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1b4670f6 55 struct queue *q);
64c7f8cf 56
d39b7737
OZ		 57static inline void deallocate_hqd(struct device_queue_manager *dqm,
58 struct queue *q);
59static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
60static int allocate_sdma_queue(struct device_queue_manager *dqm,
61 struct queue *q);
73ea648d
SL		 62static void kfd_process_hw_exception(struct work_struct *work);
63
bcea3081
BG		 64static inline
65enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
64c7f8cf 66{
1b4670f6 67 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
85d258f9
BG		 68 return KFD_MQD_TYPE_SDMA;
69 return KFD_MQD_TYPE_CP;
64c7f8cf
BG		 70}
71
d0b63bb3
AR		 72static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
73{
74 int i;
75 int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec
76 + pipe * dqm->dev->shared_resources.num_queue_per_pipe;
77
78 /* queue is available for KFD usage if bit is 1 */
79 for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i)
80 if (test_bit(pipe_offset + i,
e6945304 81 dqm->dev->shared_resources.cp_queue_bitmap))
d0b63bb3
AR		 82 return true;
83 return false;
84}
85
e6945304 86unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
64ea8f4a 87{
e6945304 88 return bitmap_weight(dqm->dev->shared_resources.cp_queue_bitmap,
d0b63bb3 89 KGD_MAX_QUEUES);
64ea8f4a
OG		 90}
91
d0b63bb3 92unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
64c7f8cf 93{
d0b63bb3
AR		 94 return dqm->dev->shared_resources.num_queue_per_pipe;
95}
96
97unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
98{
d0b63bb3 99 return dqm->dev->shared_resources.num_pipe_per_mec;
64c7f8cf
BG		 100}
101
98bb9222
YZ		 102static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
103{
104 return dqm->dev->device_info->num_sdma_engines;
105}
106
1b4670f6
OZ		 107static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm)
108{
109 return dqm->dev->device_info->num_xgmi_sdma_engines;
110}
111
c7637c95
YZ		 112static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
113{
114 return get_num_sdma_engines(dqm) + get_num_xgmi_sdma_engines(dqm);
115}
116
98bb9222
YZ		 117unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
118{
119 return dqm->dev->device_info->num_sdma_engines
d5094189 120 * dqm->dev->device_info->num_sdma_queues_per_engine;
98bb9222
YZ		 121}
122
1b4670f6
OZ		 123unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
124{
125 return dqm->dev->device_info->num_xgmi_sdma_engines
126 * dqm->dev->device_info->num_sdma_queues_per_engine;
127}
128
a22fc854 129void program_sh_mem_settings(struct device_queue_manager *dqm,
64c7f8cf
BG		 130 struct qcm_process_device *qpd)
131{
cea405b1
XZ		 132 return dqm->dev->kfd2kgd->program_sh_mem_settings(
133 dqm->dev->kgd, qpd->vmid,
64c7f8cf
BG		 134 qpd->sh_mem_config,
135 qpd->sh_mem_ape1_base,
136 qpd->sh_mem_ape1_limit,
137 qpd->sh_mem_bases);
138}
139
204d8998 140static void increment_queue_count(struct device_queue_manager *dqm,
b42902f4
YZ		 141 enum kfd_queue_type type)
142{
143 dqm->active_queue_count++;
144 if (type == KFD_QUEUE_TYPE_COMPUTE || type == KFD_QUEUE_TYPE_DIQ)
145 dqm->active_cp_queue_count++;
146}
147
204d8998 148static void decrement_queue_count(struct device_queue_manager *dqm,
b42902f4
YZ		 149 enum kfd_queue_type type)
150{
151 dqm->active_queue_count--;
152 if (type == KFD_QUEUE_TYPE_COMPUTE || type == KFD_QUEUE_TYPE_DIQ)
153 dqm->active_cp_queue_count--;
154}
155
d69fd951 156int read_sdma_queue_counter(uint64_t q_rptr, uint64_t *val)
32cb59f3
MJ		 157{
158 int ret;
159 uint64_t tmp = 0;
160
d69fd951 161 if (!val)
32cb59f3
MJ		 162 return -EINVAL;
163 /*
164 * SDMA activity counter is stored at queue's RPTR + 0x8 location.
165 */
d69fd951 166 if (!access_ok((const void __user *)(q_rptr +
32cb59f3
MJ		 167 sizeof(uint64_t)), sizeof(uint64_t))) {
168 pr_err("Can't access sdma queue activity counter\n");
169 return -EFAULT;
170 }
171
d69fd951 172 ret = get_user(tmp, (uint64_t *)(q_rptr + sizeof(uint64_t)));
32cb59f3
MJ		 173 if (!ret) {
174 *val = tmp;
175 }
176
177 return ret;
178}
179
ef568db7
FK		 180static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
181{
182 struct kfd_dev *dev = qpd->dqm->dev;
183
184 if (!KFD_IS_SOC15(dev->device_info->asic_family)) {
185 /* On pre-SOC15 chips we need to use the queue ID to
186 * preserve the user mode ABI.
187 */
188 q->doorbell_id = q->properties.queue_id;
1b4670f6
OZ		 189 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
190 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
234441dd
YZ		 191 /* For SDMA queues on SOC15 with 8-byte doorbell, use static
192 * doorbell assignments based on the engine and queue id.
193 * The doobell index distance between RLC (2*i) and (2*i+1)
194 * for a SDMA engine is 512.
ef568db7 195 */
234441dd
YZ		 196 uint32_t *idx_offset =
197 dev->shared_resources.sdma_doorbell_idx;
198
199 q->doorbell_id = idx_offset[q->properties.sdma_engine_id]
200 + (q->properties.sdma_queue_id & 1)
201 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET
202 + (q->properties.sdma_queue_id >> 1);
ef568db7
FK		 203 } else {
204 /* For CP queues on SOC15 reserve a free doorbell ID */
205 unsigned int found;
206
207 found = find_first_zero_bit(qpd->doorbell_bitmap,
208 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
209 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
210 pr_debug("No doorbells available");
211 return -EBUSY;
212 }
213 set_bit(found, qpd->doorbell_bitmap);
214 q->doorbell_id = found;
215 }
216
217 q->properties.doorbell_off =
339903fa 218 kfd_get_doorbell_dw_offset_in_bar(dev, q->process,
ef568db7
FK		 219 q->doorbell_id);
220
221 return 0;
222}
223
224static void deallocate_doorbell(struct qcm_process_device *qpd,
225 struct queue *q)
226{
227 unsigned int old;
228 struct kfd_dev *dev = qpd->dqm->dev;
229
230 if (!KFD_IS_SOC15(dev->device_info->asic_family) ||
1b4670f6
OZ		 231 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
232 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
ef568db7
FK		 233 return;
234
235 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
236 WARN_ON(!old);
237}
238
64c7f8cf
BG		 239static int allocate_vmid(struct device_queue_manager *dqm,
240 struct qcm_process_device *qpd,
241 struct queue *q)
242{
d9d4623c 243 int allocated_vmid = -1, i;
64c7f8cf 244
d9d4623c
YZ		 245 for (i = dqm->dev->vm_info.first_vmid_kfd;
246 i <= dqm->dev->vm_info.last_vmid_kfd; i++) {
247 if (!dqm->vmid_pasid[i]) {
248 allocated_vmid = i;
249 break;
250 }
251 }
252
253 if (allocated_vmid < 0) {
254 pr_err("no more vmid to allocate\n");
255 return -ENOSPC;
256 }
257
258 pr_debug("vmid allocated: %d\n", allocated_vmid);
259
260 dqm->vmid_pasid[allocated_vmid] = q->process->pasid;
64c7f8cf 261
d9d4623c 262 set_pasid_vmid_mapping(dqm, q->process->pasid, allocated_vmid);
64c7f8cf 263
64c7f8cf
BG		 264 qpd->vmid = allocated_vmid;
265 q->properties.vmid = allocated_vmid;
266
64c7f8cf
BG		 267 program_sh_mem_settings(dqm, qpd);
268
403575c4
FK		 269 /* qpd->page_table_base is set earlier when register_process()
270 * is called, i.e. when the first queue is created.
271 */
272 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd,
273 qpd->vmid,
274 qpd->page_table_base);
275 /* invalidate the VM context after pasid and vmid mapping is set up */
276 kfd_flush_tlb(qpd_to_pdd(qpd));
277
c637b36a
YZ		 278 if (dqm->dev->kfd2kgd->set_scratch_backing_va)
279 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->kgd,
280 qpd->sh_hidden_private_base, qpd->vmid);
d39b7737 281
64c7f8cf
BG		 282 return 0;
283}
284
552764b6
FK		 285static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
286 struct qcm_process_device *qpd)
287{
f6e27ff1
FK		 288 const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf;
289 int ret;
552764b6
FK		 290
291 if (!qpd->ib_kaddr)
292 return -ENOMEM;
293
f6e27ff1
FK		 294 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
295 if (ret)
296 return ret;
552764b6 297
5b87245f 298 return amdgpu_amdkfd_submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid,
f6e27ff1
FK		 299 qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
300 pmf->release_mem_size / sizeof(uint32_t));
552764b6
FK		 301}
302
64c7f8cf
BG		 303static void deallocate_vmid(struct device_queue_manager *dqm,
304 struct qcm_process_device *qpd,
305 struct queue *q)
306{
552764b6
FK		 307 /* On GFX v7, CP doesn't flush TC at dequeue */
308 if (q->device->device_info->asic_family == CHIP_HAWAII)
309 if (flush_texture_cache_nocpsch(q->device, qpd))
310 pr_err("Failed to flush TC\n");
311
403575c4
FK		 312 kfd_flush_tlb(qpd_to_pdd(qpd));
313
2030664b
BG		 314 /* Release the vmid mapping */
315 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
d9d4623c 316 dqm->vmid_pasid[qpd->vmid] = 0;
2030664b 317
64c7f8cf
BG		 318 qpd->vmid = 0;
319 q->properties.vmid = 0;
320}
321
322static int create_queue_nocpsch(struct device_queue_manager *dqm,
323 struct queue *q,
b46cb7d7 324 struct qcm_process_device *qpd)
64c7f8cf 325{
d39b7737 326 struct mqd_manager *mqd_mgr;
64c7f8cf
BG		 327 int retval;
328
efeaed4d 329 dqm_lock(dqm);
64c7f8cf 330
b8cbab04 331 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 332 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04 333 dqm->total_queue_count);
ab7c1648
KR		 334 retval = -EPERM;
335 goto out_unlock;
b8cbab04
OG		 336 }
337
64c7f8cf
BG		 338 if (list_empty(&qpd->queues_list)) {
339 retval = allocate_vmid(dqm, qpd, q);
ab7c1648
KR		 340 if (retval)
341 goto out_unlock;
64c7f8cf 342 }
64c7f8cf 343 q->properties.vmid = qpd->vmid;
26103436 344 /*
bb2d2128
FK		 345 * Eviction state logic: mark all queues as evicted, even ones
346 * not currently active. Restoring inactive queues later only
347 * updates the is_evicted flag but is a no-op otherwise.
26103436 348 */
bb2d2128 349 q->properties.is_evicted = !!qpd->evicted;
64c7f8cf 350
373d7080
FK		 351 q->properties.tba_addr = qpd->tba_addr;
352 q->properties.tma_addr = qpd->tma_addr;
353
d091bc0a
OZ		 354 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
355 q->properties.type)];
d39b7737
OZ		 356 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
357 retval = allocate_hqd(dqm, q);
358 if (retval)
359 goto deallocate_vmid;
360 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
361 q->pipe, q->queue);
362 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
363 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
364 retval = allocate_sdma_queue(dqm, q);
365 if (retval)
366 goto deallocate_vmid;
367 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
368 }
369
370 retval = allocate_doorbell(qpd, q);
371 if (retval)
372 goto out_deallocate_hqd;
373
6a6ef5ee
OZ		 374 /* Temporarily release dqm lock to avoid a circular lock dependency */
375 dqm_unlock(dqm);
d091bc0a 376 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
6a6ef5ee
OZ		 377 dqm_lock(dqm);
378
d091bc0a
OZ		 379 if (!q->mqd_mem_obj) {
380 retval = -ENOMEM;
381 goto out_deallocate_doorbell;
382 }
8636e53c
OZ		 383 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
384 &q->gart_mqd_addr, &q->properties);
d39b7737 385 if (q->properties.is_active) {
2c99a547
PY		 386 if (!dqm->sched_running) {
387 WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
388 goto add_queue_to_list;
389 }
d39b7737
OZ		 390
391 if (WARN(q->process->mm != current->mm,
392 "should only run in user thread"))
393 retval = -EFAULT;
394 else
395 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
396 q->queue, &q->properties, current->mm);
397 if (retval)
d091bc0a 398 goto out_free_mqd;
64c7f8cf
BG		 399 }
400
2c99a547 401add_queue_to_list:
64c7f8cf 402 list_add(&q->list, &qpd->queues_list);
bc920fd4 403 qpd->queue_count++;
b6819cec 404 if (q->properties.is_active)
b42902f4 405 increment_queue_count(dqm, q->properties.type);
64c7f8cf 406
b8cbab04
OG		 407 /*
408 * Unconditionally increment this counter, regardless of the queue's
409 * type or whether the queue is active.
410 */
411 dqm->total_queue_count++;
412 pr_debug("Total of %d queues are accountable so far\n",
413 dqm->total_queue_count);
d091bc0a 414 goto out_unlock;
b8cbab04 415
d091bc0a
OZ		 416out_free_mqd:
417 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
d39b7737
OZ		 418out_deallocate_doorbell:
419 deallocate_doorbell(qpd, q);
420out_deallocate_hqd:
421 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
422 deallocate_hqd(dqm, q);
423 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
424 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
425 deallocate_sdma_queue(dqm, q);
426deallocate_vmid:
427 if (list_empty(&qpd->queues_list))
428 deallocate_vmid(dqm, qpd, q);
ab7c1648 429out_unlock:
efeaed4d 430 dqm_unlock(dqm);
ab7c1648 431 return retval;
64c7f8cf
BG		 432}
433
434static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
435{
436 bool set;
f0ec5b99 437 int pipe, bit, i;
64c7f8cf
BG		 438
439 set = false;
440
8eabaf54
KR		 441 for (pipe = dqm->next_pipe_to_allocate, i = 0;
442 i < get_pipes_per_mec(dqm);
d0b63bb3
AR		 443 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
444
445 if (!is_pipe_enabled(dqm, 0, pipe))
446 continue;
447
64c7f8cf 448 if (dqm->allocated_queues[pipe] != 0) {
4252bf68
HK		 449 bit = ffs(dqm->allocated_queues[pipe]) - 1;
450 dqm->allocated_queues[pipe] &= ~(1 << bit);
64c7f8cf
BG		 451 q->pipe = pipe;
452 q->queue = bit;
453 set = true;
454 break;
455 }
456 }
457
991ca8ee 458 if (!set)
64c7f8cf
BG		 459 return -EBUSY;
460
79775b62 461 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
64c7f8cf 462 /* horizontal hqd allocation */
d0b63bb3 463 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
64c7f8cf
BG		 464
465 return 0;
466}
467
468static inline void deallocate_hqd(struct device_queue_manager *dqm,
469 struct queue *q)
470{
4252bf68 471 dqm->allocated_queues[q->pipe] |= (1 << q->queue);
64c7f8cf
BG		 472}
473
9fd3f1bf
FK		 474/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
475 * to avoid asynchronized access
476 */
477static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
64c7f8cf
BG		 478 struct qcm_process_device *qpd,
479 struct queue *q)
480{
481 int retval;
8d5f3552 482 struct mqd_manager *mqd_mgr;
64c7f8cf 483
fdfa090b
OZ		 484 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
485 q->properties.type)];
64c7f8cf 486
c7637c95 487 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
c2e1b3a4 488 deallocate_hqd(dqm, q);
c7637c95 489 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1b4670f6 490 deallocate_sdma_queue(dqm, q);
c7637c95 491 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1b4670f6 492 deallocate_sdma_queue(dqm, q);
c7637c95 493 else {
79775b62 494 pr_debug("q->properties.type %d is invalid\n",
7113cd65 495 q->properties.type);
9fd3f1bf 496 return -EINVAL;
64c7f8cf 497 }
9fd3f1bf 498 dqm->total_queue_count--;
64c7f8cf 499
ef568db7
FK		 500 deallocate_doorbell(qpd, q);
501
2c99a547
PY		 502 if (!dqm->sched_running) {
503 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
504 return 0;
505 }
506
8d5f3552 507 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
c2e1b3a4 508 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
b90e3fbe 509 KFD_UNMAP_LATENCY_MS,
64c7f8cf 510 q->pipe, q->queue);
9fd3f1bf
FK		 511 if (retval == -ETIME)
512 qpd->reset_wavefronts = true;
64c7f8cf 513
32cb59f3 514
8636e53c 515 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
64c7f8cf
BG		 516
517 list_del(&q->list);
9fd3f1bf
FK		 518 if (list_empty(&qpd->queues_list)) {
519 if (qpd->reset_wavefronts) {
520 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
521 dqm->dev);
522 /* dbgdev_wave_reset_wavefronts has to be called before
523 * deallocate_vmid(), i.e. when vmid is still in use.
524 */
525 dbgdev_wave_reset_wavefronts(dqm->dev,
526 qpd->pqm->process);
527 qpd->reset_wavefronts = false;
528 }
529
64c7f8cf 530 deallocate_vmid(dqm, qpd, q);
9fd3f1bf 531 }
bc920fd4 532 qpd->queue_count--;
b8020b03 533 if (q->properties.is_active) {
b42902f4 534 decrement_queue_count(dqm, q->properties.type);
b8020b03
JG		 535 if (q->properties.is_gws) {
536 dqm->gws_queue_count--;
537 qpd->mapped_gws_queue = false;
538 }
539 }
b8cbab04 540
9fd3f1bf
FK		 541 return retval;
542}
b8cbab04 543
9fd3f1bf
FK		 544static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
545 struct qcm_process_device *qpd,
546 struct queue *q)
547{
548 int retval;
d69fd951
MJ		 549 uint64_t sdma_val = 0;
550 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
551
552 /* Get the SDMA queue stats */
553 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
554 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
555 retval = read_sdma_queue_counter((uint64_t)q->properties.read_ptr,
556 &sdma_val);
557 if (retval)
558 pr_err("Failed to read SDMA queue counter for queue: %d\n",
559 q->properties.queue_id);
560 }
9fd3f1bf 561
efeaed4d 562 dqm_lock(dqm);
9fd3f1bf 563 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
d69fd951
MJ		 564 if (!retval)
565 pdd->sdma_past_activity_counter += sdma_val;
efeaed4d 566 dqm_unlock(dqm);
9fd3f1bf 567
64c7f8cf
BG		 568 return retval;
569}
570
571static int update_queue(struct device_queue_manager *dqm, struct queue *q)
572{
8636e53c 573 int retval = 0;
8d5f3552 574 struct mqd_manager *mqd_mgr;
26103436 575 struct kfd_process_device *pdd;
b6ffbab8 576 bool prev_active = false;
64c7f8cf 577
efeaed4d 578 dqm_lock(dqm);
26103436
FK		 579 pdd = kfd_get_process_device_data(q->device, q->process);
580 if (!pdd) {
581 retval = -ENODEV;
582 goto out_unlock;
583 }
fdfa090b
OZ		 584 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
585 q->properties.type)];
64c7f8cf 586
60a00956
FK		 587 /* Save previous activity state for counters */
588 prev_active = q->properties.is_active;
589
590 /* Make sure the queue is unmapped before updating the MQD */
d146c5a7 591 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
60a00956
FK		 592 retval = unmap_queues_cpsch(dqm,
593 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
894a8293 594 if (retval) {
60a00956
FK		 595 pr_err("unmap queue failed\n");
596 goto out_unlock;
597 }
894a8293 598 } else if (prev_active &&
60a00956 599 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1b4670f6
OZ		 600 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
601 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
2c99a547
PY		 602
603 if (!dqm->sched_running) {
604 WARN_ONCE(1, "Update non-HWS queue while stopped\n");
605 goto out_unlock;
606 }
607
8d5f3552 608 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
60a00956
FK		 609 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
610 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
611 if (retval) {
612 pr_err("destroy mqd failed\n");
613 goto out_unlock;
614 }
615 }
616
8636e53c 617 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties);
60a00956 618
096d1a3e
FK		 619 /*
620 * check active state vs. the previous state and modify
621 * counter accordingly. map_queues_cpsch uses the
81b820b3 622 * dqm->active_queue_count to determine whether a new runlist must be
096d1a3e
FK		 623 * uploaded.
624 */
625 if (q->properties.is_active && !prev_active)
b42902f4 626 increment_queue_count(dqm, q->properties.type);
096d1a3e 627 else if (!q->properties.is_active && prev_active)
b42902f4 628 decrement_queue_count(dqm, q->properties.type);
096d1a3e 629
b8020b03
JG		 630 if (q->gws && !q->properties.is_gws) {
631 if (q->properties.is_active) {
632 dqm->gws_queue_count++;
633 pdd->qpd.mapped_gws_queue = true;
634 }
635 q->properties.is_gws = true;
636 } else if (!q->gws && q->properties.is_gws) {
637 if (q->properties.is_active) {
638 dqm->gws_queue_count--;
639 pdd->qpd.mapped_gws_queue = false;
640 }
641 q->properties.is_gws = false;
642 }
643
d146c5a7 644 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
60a00956 645 retval = map_queues_cpsch(dqm);
894a8293 646 else if (q->properties.is_active &&
60a00956 647 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
1b4670f6
OZ		 648 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
649 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1b19aa5a
FK		 650 if (WARN(q->process->mm != current->mm,
651 "should only run in user thread"))
652 retval = -EFAULT;
653 else
654 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
655 q->pipe, q->queue,
656 &q->properties, current->mm);
657 }
b6ffbab8 658
ab7c1648 659out_unlock:
efeaed4d 660 dqm_unlock(dqm);
64c7f8cf
BG		 661 return retval;
662}
663
26103436
FK		 664static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
665 struct qcm_process_device *qpd)
666{
667 struct queue *q;
8d5f3552 668 struct mqd_manager *mqd_mgr;
26103436 669 struct kfd_process_device *pdd;
bb2d2128 670 int retval, ret = 0;
26103436 671
efeaed4d 672 dqm_lock(dqm);
26103436
FK		 673 if (qpd->evicted++ > 0) /* already evicted, do nothing */
674 goto out;
675
676 pdd = qpd_to_pdd(qpd);
6027b1bf 677 pr_info_ratelimited("Evicting PASID 0x%x queues\n",
26103436
FK		 678 pdd->process->pasid);
679
bb2d2128
FK		 680 /* Mark all queues as evicted. Deactivate all active queues on
681 * the qpd.
682 */
26103436 683 list_for_each_entry(q, &qpd->queues_list, list) {
bb2d2128 684 q->properties.is_evicted = true;
26103436
FK		 685 if (!q->properties.is_active)
686 continue;
bb2d2128 687
fdfa090b
OZ		 688 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
689 q->properties.type)];
26103436 690 q->properties.is_active = false;
b42902f4 691 decrement_queue_count(dqm, q->properties.type);
b8020b03
JG		 692 if (q->properties.is_gws) {
693 dqm->gws_queue_count--;
694 qpd->mapped_gws_queue = false;
695 }
2c99a547
PY		 696
697 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
698 continue;
699
8d5f3552 700 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
26103436
FK		 701 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
702 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
bb2d2128
FK		 703 if (retval && !ret)
704 /* Return the first error, but keep going to
705 * maintain a consistent eviction state
706 */
707 ret = retval;
26103436
FK		 708 }
709
710out:
efeaed4d 711 dqm_unlock(dqm);
bb2d2128 712 return ret;
26103436
FK		 713}
714
715static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
716 struct qcm_process_device *qpd)
717{
718 struct queue *q;
719 struct kfd_process_device *pdd;
720 int retval = 0;
721
efeaed4d 722 dqm_lock(dqm);
26103436
FK		 723 if (qpd->evicted++ > 0) /* already evicted, do nothing */
724 goto out;
725
726 pdd = qpd_to_pdd(qpd);
6027b1bf 727 pr_info_ratelimited("Evicting PASID 0x%x queues\n",
26103436
FK		 728 pdd->process->pasid);
729
bb2d2128
FK		 730 /* Mark all queues as evicted. Deactivate all active queues on
731 * the qpd.
732 */
26103436 733 list_for_each_entry(q, &qpd->queues_list, list) {
bb2d2128 734 q->properties.is_evicted = true;
26103436
FK		 735 if (!q->properties.is_active)
736 continue;
bb2d2128 737
26103436 738 q->properties.is_active = false;
b42902f4 739 decrement_queue_count(dqm, q->properties.type);
26103436
FK		 740 }
741 retval = execute_queues_cpsch(dqm,
742 qpd->is_debug ?
743 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
744 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
745
746out:
efeaed4d 747 dqm_unlock(dqm);
26103436
FK		 748 return retval;
749}
750
751static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
752 struct qcm_process_device *qpd)
753{
1b19aa5a 754 struct mm_struct *mm = NULL;
26103436 755 struct queue *q;
8d5f3552 756 struct mqd_manager *mqd_mgr;
26103436 757 struct kfd_process_device *pdd;
e715c6d0 758 uint64_t pd_base;
bb2d2128 759 int retval, ret = 0;
26103436
FK		 760
761 pdd = qpd_to_pdd(qpd);
762 /* Retrieve PD base */
5b87245f 763 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
26103436 764
efeaed4d 765 dqm_lock(dqm);
26103436
FK		 766 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
767 goto out;
768 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
769 qpd->evicted--;
770 goto out;
771 }
772
6027b1bf 773 pr_info_ratelimited("Restoring PASID 0x%x queues\n",
26103436
FK		 774 pdd->process->pasid);
775
776 /* Update PD Base in QPD */
777 qpd->page_table_base = pd_base;
e715c6d0 778 pr_debug("Updated PD address to 0x%llx\n", pd_base);
26103436
FK		 779
780 if (!list_empty(&qpd->queues_list)) {
781 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
782 dqm->dev->kgd,
783 qpd->vmid,
784 qpd->page_table_base);
785 kfd_flush_tlb(pdd);
786 }
787
1b19aa5a
FK		 788 /* Take a safe reference to the mm_struct, which may otherwise
789 * disappear even while the kfd_process is still referenced.
790 */
791 mm = get_task_mm(pdd->process->lead_thread);
792 if (!mm) {
bb2d2128 793 ret = -EFAULT;
1b19aa5a
FK		 794 goto out;
795 }
796
bb2d2128
FK		 797 /* Remove the eviction flags. Activate queues that are not
798 * inactive for other reasons.
799 */
26103436 800 list_for_each_entry(q, &qpd->queues_list, list) {
bb2d2128
FK		 801 q->properties.is_evicted = false;
802 if (!QUEUE_IS_ACTIVE(q->properties))
26103436 803 continue;
bb2d2128 804
fdfa090b
OZ		 805 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
806 q->properties.type)];
26103436 807 q->properties.is_active = true;
b42902f4 808 increment_queue_count(dqm, q->properties.type);
b8020b03
JG		 809 if (q->properties.is_gws) {
810 dqm->gws_queue_count++;
811 qpd->mapped_gws_queue = true;
812 }
2c99a547
PY		 813
814 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
815 continue;
816
8d5f3552 817 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
1b19aa5a 818 q->queue, &q->properties, mm);
bb2d2128
FK		 819 if (retval && !ret)
820 /* Return the first error, but keep going to
821 * maintain a consistent eviction state
822 */
823 ret = retval;
26103436
FK		 824 }
825 qpd->evicted = 0;
826out:
1b19aa5a
FK		 827 if (mm)
828 mmput(mm);
efeaed4d 829 dqm_unlock(dqm);
bb2d2128 830 return ret;
26103436
FK		 831}
832
833static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
834 struct qcm_process_device *qpd)
835{
836 struct queue *q;
837 struct kfd_process_device *pdd;
e715c6d0 838 uint64_t pd_base;
26103436
FK		 839 int retval = 0;
840
841 pdd = qpd_to_pdd(qpd);
842 /* Retrieve PD base */
5b87245f 843 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
26103436 844
efeaed4d 845 dqm_lock(dqm);
26103436
FK		 846 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
847 goto out;
848 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
849 qpd->evicted--;
850 goto out;
851 }
852
6027b1bf 853 pr_info_ratelimited("Restoring PASID 0x%x queues\n",
26103436
FK		 854 pdd->process->pasid);
855
856 /* Update PD Base in QPD */
857 qpd->page_table_base = pd_base;
e715c6d0 858 pr_debug("Updated PD address to 0x%llx\n", pd_base);
26103436
FK		 859
860 /* activate all active queues on the qpd */
861 list_for_each_entry(q, &qpd->queues_list, list) {
26103436 862 q->properties.is_evicted = false;
bb2d2128
FK		 863 if (!QUEUE_IS_ACTIVE(q->properties))
864 continue;
865
26103436 866 q->properties.is_active = true;
b42902f4 867 increment_queue_count(dqm, q->properties.type);
26103436
FK		 868 }
869 retval = execute_queues_cpsch(dqm,
870 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
bb2d2128 871 qpd->evicted = 0;
26103436 872out:
efeaed4d 873 dqm_unlock(dqm);
26103436
FK		 874 return retval;
875}
876
58dcd5bf 877static int register_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 878 struct qcm_process_device *qpd)
879{
880 struct device_process_node *n;
403575c4 881 struct kfd_process_device *pdd;
e715c6d0 882 uint64_t pd_base;
a22fc854 883 int retval;
64c7f8cf 884
dbf56ab1 885 n = kzalloc(sizeof(*n), GFP_KERNEL);
64c7f8cf
BG		 886 if (!n)
887 return -ENOMEM;
888
889 n->qpd = qpd;
890
403575c4
FK		 891 pdd = qpd_to_pdd(qpd);
892 /* Retrieve PD base */
5b87245f 893 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
403575c4 894
efeaed4d 895 dqm_lock(dqm);
64c7f8cf
BG		 896 list_add(&n->list, &dqm->queues);
897
403575c4
FK		 898 /* Update PD Base in QPD */
899 qpd->page_table_base = pd_base;
e715c6d0 900 pr_debug("Updated PD address to 0x%llx\n", pd_base);
403575c4 901
bfd5e378 902 retval = dqm->asic_ops.update_qpd(dqm, qpd);
a22fc854 903
f756e631 904 dqm->processes_count++;
64c7f8cf 905
efeaed4d 906 dqm_unlock(dqm);
64c7f8cf 907
32cce8bc
FK		 908 /* Outside the DQM lock because under the DQM lock we can't do
909 * reclaim or take other locks that others hold while reclaiming.
910 */
911 kfd_inc_compute_active(dqm->dev);
912
a22fc854 913 return retval;
64c7f8cf
BG		 914}
915
58dcd5bf 916static int unregister_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 917 struct qcm_process_device *qpd)
918{
919 int retval;
920 struct device_process_node *cur, *next;
921
1e5ec956
OG		 922 pr_debug("qpd->queues_list is %s\n",
923 list_empty(&qpd->queues_list) ? "empty" : "not empty");
64c7f8cf
BG		 924
925 retval = 0;
efeaed4d 926 dqm_lock(dqm);
64c7f8cf
BG		 927
928 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
929 if (qpd == cur->qpd) {
930 list_del(&cur->list);
f5d896bb 931 kfree(cur);
f756e631 932 dqm->processes_count--;
64c7f8cf
BG		 933 goto out;
934 }
935 }
936 /* qpd not found in dqm list */
937 retval = 1;
938out:
efeaed4d 939 dqm_unlock(dqm);
32cce8bc
FK		 940
941 /* Outside the DQM lock because under the DQM lock we can't do
942 * reclaim or take other locks that others hold while reclaiming.
943 */
944 if (!retval)
945 kfd_dec_compute_active(dqm->dev);
946
64c7f8cf
BG		 947 return retval;
948}
949
950static int
951set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
952 unsigned int vmid)
953{
cea405b1 954 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
deb99d7c 955 dqm->dev->kgd, pasid, vmid);
64c7f8cf
BG		 956}
957
2249d558
AL		 958static void init_interrupts(struct device_queue_manager *dqm)
959{
960 unsigned int i;
961
d0b63bb3
AR		 962 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
963 if (is_pipe_enabled(dqm, 0, i))
964 dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
2249d558
AL		 965}
966
64c7f8cf
BG		 967static int initialize_nocpsch(struct device_queue_manager *dqm)
968{
86194cf8 969 int pipe, queue;
64c7f8cf 970
79775b62 971 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf 972
ab7c1648
KR		 973 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
974 sizeof(unsigned int), GFP_KERNEL);
975 if (!dqm->allocated_queues)
976 return -ENOMEM;
977
efeaed4d 978 mutex_init(&dqm->lock_hidden);
64c7f8cf 979 INIT_LIST_HEAD(&dqm->queues);
81b820b3 980 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;
b42902f4 981 dqm->active_cp_queue_count = 0;
b8020b03 982 dqm->gws_queue_count = 0;
64c7f8cf 983
86194cf8
FK		 984 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
985 int pipe_offset = pipe * get_queues_per_pipe(dqm);
986
987 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
988 if (test_bit(pipe_offset + queue,
e6945304 989 dqm->dev->shared_resources.cp_queue_bitmap))
86194cf8
FK		 990 dqm->allocated_queues[pipe] |= 1 << queue;
991 }
64c7f8cf 992
d9d4623c
YZ		 993 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
994
35cdc81b
OZ		 995 dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm));
996 dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm));
64c7f8cf 997
64c7f8cf
BG		 998 return 0;
999}
1000
58dcd5bf 1001static void uninitialize(struct device_queue_manager *dqm)
64c7f8cf 1002{
6f9d54fd
OG		 1003 int i;
1004
81b820b3 1005 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);
64c7f8cf
BG		 1006
1007 kfree(dqm->allocated_queues);
6f9d54fd 1008 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
8d5f3552 1009 kfree(dqm->mqd_mgrs[i]);
efeaed4d 1010 mutex_destroy(&dqm->lock_hidden);
64c7f8cf
BG		 1011}
1012
1013static int start_nocpsch(struct device_queue_manager *dqm)
1014{
52055039 1015 pr_info("SW scheduler is used");
2249d558 1016 init_interrupts(dqm);
424b5442
YZ		 1017
1018 if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
1019 return pm_init(&dqm->packets, dqm);
2c99a547
PY		 1020 dqm->sched_running = true;
1021
424b5442 1022 return 0;
64c7f8cf
BG		 1023}
1024
1025static int stop_nocpsch(struct device_queue_manager *dqm)
1026{
424b5442 1027 if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
c2a77fde 1028 pm_uninit(&dqm->packets, false);
2c99a547
PY		 1029 dqm->sched_running = false;
1030
64c7f8cf
BG		 1031 return 0;
1032}
1033
09c34e8d
FK		 1034static void pre_reset(struct device_queue_manager *dqm)
1035{
1036 dqm_lock(dqm);
1037 dqm->is_resetting = true;
1038 dqm_unlock(dqm);
1039}
1040
bcea3081 1041static int allocate_sdma_queue(struct device_queue_manager *dqm,
e78579aa 1042 struct queue *q)
bcea3081
BG		 1043{
1044 int bit;
1045
1b4670f6 1046 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
c7637c95
YZ		 1047 if (dqm->sdma_bitmap == 0) {
1048 pr_err("No more SDMA queue to allocate\n");
1b4670f6 1049 return -ENOMEM;
c7637c95
YZ		 1050 }
1051
1b4670f6
OZ		 1052 bit = __ffs64(dqm->sdma_bitmap);
1053 dqm->sdma_bitmap &= ~(1ULL << bit);
1054 q->sdma_id = bit;
1055 q->properties.sdma_engine_id = q->sdma_id %
1056 get_num_sdma_engines(dqm);
1057 q->properties.sdma_queue_id = q->sdma_id /
1058 get_num_sdma_engines(dqm);
1059 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
c7637c95
YZ		 1060 if (dqm->xgmi_sdma_bitmap == 0) {
1061 pr_err("No more XGMI SDMA queue to allocate\n");
1b4670f6 1062 return -ENOMEM;
c7637c95 1063 }
1b4670f6
OZ		 1064 bit = __ffs64(dqm->xgmi_sdma_bitmap);
1065 dqm->xgmi_sdma_bitmap &= ~(1ULL << bit);
1066 q->sdma_id = bit;
1067 /* sdma_engine_id is sdma id including
1068 * both PCIe-optimized SDMAs and XGMI-
1069 * optimized SDMAs. The calculation below
1070 * assumes the first N engines are always
1071 * PCIe-optimized ones
1072 */
1073 q->properties.sdma_engine_id = get_num_sdma_engines(dqm) +
1074 q->sdma_id % get_num_xgmi_sdma_engines(dqm);
1075 q->properties.sdma_queue_id = q->sdma_id /
1076 get_num_xgmi_sdma_engines(dqm);
1077 }
e78579aa 1078
e78579aa
YZ		 1079 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
1080 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
bcea3081
BG		 1081
1082 return 0;
1083}
1084
1085static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1b4670f6 1086 struct queue *q)
bcea3081 1087{
1b4670f6
OZ		 1088 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1089 if (q->sdma_id >= get_num_sdma_queues(dqm))
1090 return;
1091 dqm->sdma_bitmap |= (1ULL << q->sdma_id);
1092 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1093 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
1094 return;
1095 dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id);
1096 }
bcea3081
BG		 1097}
1098
64c7f8cf
BG		 1099/*
1100 * Device Queue Manager implementation for cp scheduler
1101 */
1102
1103static int set_sched_resources(struct device_queue_manager *dqm)
1104{
d0b63bb3 1105 int i, mec;
64c7f8cf 1106 struct scheduling_resources res;
64c7f8cf 1107
44008d7a 1108 res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
d0b63bb3
AR		 1109
1110 res.queue_mask = 0;
1111 for (i = 0; i < KGD_MAX_QUEUES; ++i) {
1112 mec = (i / dqm->dev->shared_resources.num_queue_per_pipe)
1113 / dqm->dev->shared_resources.num_pipe_per_mec;
1114
e6945304 1115 if (!test_bit(i, dqm->dev->shared_resources.cp_queue_bitmap))
d0b63bb3
AR		 1116 continue;
1117
1118 /* only acquire queues from the first MEC */
1119 if (mec > 0)
1120 continue;
1121
1122 /* This situation may be hit in the future if a new HW
1123 * generation exposes more than 64 queues. If so, the
8eabaf54
KR		 1124 * definition of res.queue_mask needs updating
1125 */
1d11ee89 1126 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
d0b63bb3
AR		 1127 pr_err("Invalid queue enabled by amdgpu: %d\n", i);
1128 break;
1129 }
1130
d09f85d5
YZ		 1131 res.queue_mask |= 1ull
1132 << amdgpu_queue_mask_bit_to_set_resource_bit(
1133 (struct amdgpu_device *)dqm->dev->kgd, i);
d0b63bb3 1134 }
d9848e14
OZ		 1135 res.gws_mask = ~0ull;
1136 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
64c7f8cf 1137
79775b62
KR		 1138 pr_debug("Scheduling resources:\n"
1139 "vmid mask: 0x%8X\n"
1140 "queue mask: 0x%8llX\n",
64c7f8cf
BG		 1141 res.vmid_mask, res.queue_mask);
1142
1143 return pm_send_set_resources(&dqm->packets, &res);
1144}
1145
1146static int initialize_cpsch(struct device_queue_manager *dqm)
1147{
79775b62 1148 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf 1149
efeaed4d 1150 mutex_init(&dqm->lock_hidden);
64c7f8cf 1151 INIT_LIST_HEAD(&dqm->queues);
81b820b3 1152 dqm->active_queue_count = dqm->processes_count = 0;
b42902f4 1153 dqm->active_cp_queue_count = 0;
b8020b03 1154 dqm->gws_queue_count = 0;
64c7f8cf 1155 dqm->active_runlist = false;
35cdc81b
OZ		 1156 dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm));
1157 dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm));
64c7f8cf 1158
73ea648d
SL		 1159 INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
1160
bfd5e378 1161 return 0;
64c7f8cf
BG		 1162}
1163
1164static int start_cpsch(struct device_queue_manager *dqm)
1165{
64c7f8cf
BG		 1166 int retval;
1167
64c7f8cf
BG		 1168 retval = 0;
1169
1170 retval = pm_init(&dqm->packets, dqm);
4eacc26b 1171 if (retval)
64c7f8cf
BG		 1172 goto fail_packet_manager_init;
1173
1174 retval = set_sched_resources(dqm);
4eacc26b 1175 if (retval)
64c7f8cf
BG		 1176 goto fail_set_sched_resources;
1177
79775b62 1178 pr_debug("Allocating fence memory\n");
64c7f8cf
BG		 1179
1180 /* allocate fence memory on the gart */
a86aa3ca
OG		 1181 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
1182 &dqm->fence_mem);
64c7f8cf 1183
4eacc26b 1184 if (retval)
64c7f8cf
BG		 1185 goto fail_allocate_vidmem;
1186
1187 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
1188 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
2249d558
AL		 1189
1190 init_interrupts(dqm);
1191
efeaed4d 1192 dqm_lock(dqm);
73ea648d
SL		 1193 /* clear hang status when driver try to start the hw scheduler */
1194 dqm->is_hws_hang = false;
09c34e8d 1195 dqm->is_resetting = false;
2c99a547 1196 dqm->sched_running = true;
c4744e24 1197 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
efeaed4d 1198 dqm_unlock(dqm);
64c7f8cf
BG		 1199
1200 return 0;
1201fail_allocate_vidmem:
1202fail_set_sched_resources:
c2a77fde 1203 pm_uninit(&dqm->packets, false);
64c7f8cf
BG		 1204fail_packet_manager_init:
1205 return retval;
1206}
1207
1208static int stop_cpsch(struct device_queue_manager *dqm)
1209{
c2a77fde
FK		 1210 bool hanging;
1211
efeaed4d 1212 dqm_lock(dqm);
c2a77fde
FK		 1213 if (!dqm->is_hws_hang)
1214 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
1215 hanging = dqm->is_hws_hang || dqm->is_resetting;
2c99a547 1216 dqm->sched_running = false;
efeaed4d 1217 dqm_unlock(dqm);
64c7f8cf 1218
a86aa3ca 1219 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
c2a77fde 1220 pm_uninit(&dqm->packets, hanging);
64c7f8cf
BG		 1221
1222 return 0;
1223}
1224
1225static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1226 struct kernel_queue *kq,
1227 struct qcm_process_device *qpd)
1228{
efeaed4d 1229 dqm_lock(dqm);
b8cbab04 1230 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1231 pr_warn("Can't create new kernel queue because %d queues were already created\n",
b8cbab04 1232 dqm->total_queue_count);
efeaed4d 1233 dqm_unlock(dqm);
b8cbab04
OG		 1234 return -EPERM;
1235 }
1236
1237 /*
1238 * Unconditionally increment this counter, regardless of the queue's
1239 * type or whether the queue is active.
1240 */
1241 dqm->total_queue_count++;
1242 pr_debug("Total of %d queues are accountable so far\n",
1243 dqm->total_queue_count);
1244
64c7f8cf 1245 list_add(&kq->list, &qpd->priv_queue_list);
b42902f4 1246 increment_queue_count(dqm, kq->queue->properties.type);
64c7f8cf 1247 qpd->is_debug = true;
c4744e24 1248 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
efeaed4d 1249 dqm_unlock(dqm);
64c7f8cf
BG		 1250
1251 return 0;
1252}
1253
1254static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1255 struct kernel_queue *kq,
1256 struct qcm_process_device *qpd)
1257{
efeaed4d 1258 dqm_lock(dqm);
64c7f8cf 1259 list_del(&kq->list);
b42902f4 1260 decrement_queue_count(dqm, kq->queue->properties.type);
64c7f8cf 1261 qpd->is_debug = false;
c4744e24 1262 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
b8cbab04
OG		 1263 /*
1264 * Unconditionally decrement this counter, regardless of the queue's
1265 * type.
1266 */
8b58f261 1267 dqm->total_queue_count--;
b8cbab04
OG		 1268 pr_debug("Total of %d queues are accountable so far\n",
1269 dqm->total_queue_count);
efeaed4d 1270 dqm_unlock(dqm);
64c7f8cf
BG		 1271}
1272
1273static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
b46cb7d7 1274 struct qcm_process_device *qpd)
64c7f8cf
BG		 1275{
1276 int retval;
8d5f3552 1277 struct mqd_manager *mqd_mgr;
64c7f8cf 1278
b8cbab04 1279 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1280 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04 1281 dqm->total_queue_count);
70d488fb
OZ		 1282 retval = -EPERM;
1283 goto out;
b8cbab04
OG		 1284 }
1285
1b4670f6
OZ		 1286 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1287 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
38bb4226 1288 dqm_lock(dqm);
e78579aa 1289 retval = allocate_sdma_queue(dqm, q);
38bb4226 1290 dqm_unlock(dqm);
894a8293 1291 if (retval)
70d488fb 1292 goto out;
e139cd2a 1293 }
ef568db7
FK		 1294
1295 retval = allocate_doorbell(qpd, q);
1296 if (retval)
1297 goto out_deallocate_sdma_queue;
1298
70d488fb
OZ		 1299 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1300 q->properties.type)];
70df8273 1301
eec0b4cf
OZ		 1302 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1303 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1304 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
373d7080
FK		 1305 q->properties.tba_addr = qpd->tba_addr;
1306 q->properties.tma_addr = qpd->tma_addr;
70d488fb
OZ		 1307 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
1308 if (!q->mqd_mem_obj) {
1309 retval = -ENOMEM;
1310 goto out_deallocate_doorbell;
1311 }
70df8273
EH		 1312
1313 dqm_lock(dqm);
1314 /*
1315 * Eviction state logic: mark all queues as evicted, even ones
1316 * not currently active. Restoring inactive queues later only
1317 * updates the is_evicted flag but is a no-op otherwise.
1318 */
1319 q->properties.is_evicted = !!qpd->evicted;
8636e53c
OZ		 1320 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
1321 &q->gart_mqd_addr, &q->properties);
89cd9d23 1322
64c7f8cf 1323 list_add(&q->list, &qpd->queues_list);
bc920fd4 1324 qpd->queue_count++;
f38abc15 1325
64c7f8cf 1326 if (q->properties.is_active) {
b42902f4
YZ		 1327 increment_queue_count(dqm, q->properties.type);
1328
c4744e24
YZ		 1329 retval = execute_queues_cpsch(dqm,
1330 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
64c7f8cf
BG		 1331 }
1332
b8cbab04
OG		 1333 /*
1334 * Unconditionally increment this counter, regardless of the queue's
1335 * type or whether the queue is active.
1336 */
1337 dqm->total_queue_count++;
1338
1339 pr_debug("Total of %d queues are accountable so far\n",
1340 dqm->total_queue_count);
1341
efeaed4d 1342 dqm_unlock(dqm);
72a01d23
FK		 1343 return retval;
1344
70d488fb
OZ		 1345out_deallocate_doorbell:
1346 deallocate_doorbell(qpd, q);
72a01d23 1347out_deallocate_sdma_queue:
1b4670f6 1348 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
38bb4226
OZ		 1349 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1350 dqm_lock(dqm);
1b4670f6 1351 deallocate_sdma_queue(dqm, q);
38bb4226
OZ		 1352 dqm_unlock(dqm);
1353 }
70d488fb 1354out:
64c7f8cf
BG		 1355 return retval;
1356}
1357
788bf83d 1358int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
d80d19bd 1359 unsigned int fence_value,
8c72c3d7 1360 unsigned int timeout_ms)
64c7f8cf 1361{
8c72c3d7 1362 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
64c7f8cf
BG		 1363
1364 while (*fence_addr != fence_value) {
8c72c3d7 1365 if (time_after(jiffies, end_jiffies)) {
79775b62 1366 pr_err("qcm fence wait loop timeout expired\n");
0e9a860c
YZ		 1367 /* In HWS case, this is used to halt the driver thread
1368 * in order not to mess up CP states before doing
1369 * scandumps for FW debugging.
1370 */
1371 while (halt_if_hws_hang)
1372 schedule();
1373
64c7f8cf
BG		 1374 return -ETIME;
1375 }
99331a51 1376 schedule();
64c7f8cf
BG		 1377 }
1378
1379 return 0;
1380}
1381
60a00956
FK		 1382/* dqm->lock mutex has to be locked before calling this function */
1383static int map_queues_cpsch(struct device_queue_manager *dqm)
1384{
1385 int retval;
1386
2c99a547
PY		 1387 if (!dqm->sched_running)
1388 return 0;
81b820b3 1389 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)
60a00956 1390 return 0;
60a00956
FK		 1391 if (dqm->active_runlist)
1392 return 0;
1393
1394 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
14328aa5 1395 pr_debug("%s sent runlist\n", __func__);
60a00956
FK		 1396 if (retval) {
1397 pr_err("failed to execute runlist\n");
1398 return retval;
1399 }
1400 dqm->active_runlist = true;
1401
1402 return retval;
1403}
1404
ac30c783 1405/* dqm->lock mutex has to be locked before calling this function */
7da2bcf8 1406static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 1407 enum kfd_unmap_queues_filter filter,
1408 uint32_t filter_param)
64c7f8cf 1409{
9fd3f1bf 1410 int retval = 0;
64c7f8cf 1411
2c99a547
PY		 1412 if (!dqm->sched_running)
1413 return 0;
73ea648d
SL		 1414 if (dqm->is_hws_hang)
1415 return -EIO;
991ca8ee 1416 if (!dqm->active_runlist)
ac30c783 1417 return retval;
bcea3081 1418
64c7f8cf 1419 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
4465f466 1420 filter, filter_param, false, 0);
4eacc26b 1421 if (retval)
ac30c783 1422 return retval;
64c7f8cf
BG		 1423
1424 *dqm->fence_addr = KFD_FENCE_INIT;
1425 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
1426 KFD_FENCE_COMPLETED);
1427 /* should be timed out */
c3447e81 1428 retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
14328aa5 1429 queue_preemption_timeout_ms);
09c34e8d
FK		 1430 if (retval) {
1431 pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
1432 dqm->is_hws_hang = true;
1433 /* It's possible we're detecting a HWS hang in the
1434 * middle of a GPU reset. No need to schedule another
1435 * reset in this case.
1436 */
1437 if (!dqm->is_resetting)
1438 schedule_work(&dqm->hw_exception_work);
ac30c783 1439 return retval;
09c34e8d 1440 }
9fd3f1bf 1441
64c7f8cf
BG		 1442 pm_release_ib(&dqm->packets);
1443 dqm->active_runlist = false;
1444
64c7f8cf
BG		 1445 return retval;
1446}
1447
ac30c783 1448/* dqm->lock mutex has to be locked before calling this function */
c4744e24
YZ		 1449static int execute_queues_cpsch(struct device_queue_manager *dqm,
1450 enum kfd_unmap_queues_filter filter,
1451 uint32_t filter_param)
64c7f8cf
BG		 1452{
1453 int retval;
1454
73ea648d
SL		 1455 if (dqm->is_hws_hang)
1456 return -EIO;
c4744e24 1457 retval = unmap_queues_cpsch(dqm, filter, filter_param);
09c34e8d 1458 if (retval)
ac30c783 1459 return retval;
64c7f8cf 1460
60a00956 1461 return map_queues_cpsch(dqm);
64c7f8cf
BG		 1462}
1463
1464static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1465 struct qcm_process_device *qpd,
1466 struct queue *q)
1467{
1468 int retval;
8d5f3552 1469 struct mqd_manager *mqd_mgr;
d69fd951
MJ		 1470 uint64_t sdma_val = 0;
1471 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
1472
1473 /* Get the SDMA queue stats */
1474 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
1475 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1476 retval = read_sdma_queue_counter((uint64_t)q->properties.read_ptr,
1477 &sdma_val);
1478 if (retval)
1479 pr_err("Failed to read SDMA queue counter for queue: %d\n",
1480 q->properties.queue_id);
1481 }
992839ad 1482
64c7f8cf
BG		 1483 retval = 0;
1484
1485 /* remove queue from list to prevent rescheduling after preemption */
efeaed4d 1486 dqm_lock(dqm);
992839ad
YS		 1487
1488 if (qpd->is_debug) {
1489 /*
1490 * error, currently we do not allow to destroy a queue
1491 * of a currently debugged process
1492 */
1493 retval = -EBUSY;
1494 goto failed_try_destroy_debugged_queue;
1495
1496 }
1497
fdfa090b
OZ		 1498 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1499 q->properties.type)];
64c7f8cf 1500
ef568db7
FK		 1501 deallocate_doorbell(qpd, q);
1502
d69fd951
MJ		 1503 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
1504 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1b4670f6 1505 deallocate_sdma_queue(dqm, q);
d69fd951
MJ		 1506 pdd->sdma_past_activity_counter += sdma_val;
1507 }
bcea3081 1508
64c7f8cf 1509 list_del(&q->list);
bc920fd4 1510 qpd->queue_count--;
40a526dc 1511 if (q->properties.is_active) {
b42902f4 1512 decrement_queue_count(dqm, q->properties.type);
40a526dc 1513 retval = execute_queues_cpsch(dqm,
9fd3f1bf 1514 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
40a526dc
YZ		 1515 if (retval == -ETIME)
1516 qpd->reset_wavefronts = true;
b8020b03
JG		 1517 if (q->properties.is_gws) {
1518 dqm->gws_queue_count--;
1519 qpd->mapped_gws_queue = false;
1520 }
40a526dc 1521 }
64c7f8cf 1522
b8cbab04
OG		 1523 /*
1524 * Unconditionally decrement this counter, regardless of the queue's
1525 * type
1526 */
1527 dqm->total_queue_count--;
1528 pr_debug("Total of %d queues are accountable so far\n",
1529 dqm->total_queue_count);
64c7f8cf 1530
efeaed4d 1531 dqm_unlock(dqm);
64c7f8cf 1532
8636e53c
OZ		 1533 /* Do free_mqd after dqm_unlock(dqm) to avoid circular locking */
1534 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
89cd9d23 1535
9e827224 1536 return retval;
64c7f8cf 1537
992839ad
YS		 1538failed_try_destroy_debugged_queue:
1539
efeaed4d 1540 dqm_unlock(dqm);
64c7f8cf
BG		 1541 return retval;
1542}
1543
1544/*
1545 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1546 * stay in user mode.
1547 */
1548#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1549/* APE1 limit is inclusive and 64K aligned. */
1550#define APE1_LIMIT_ALIGNMENT 0xFFFF
1551
1552static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1553 struct qcm_process_device *qpd,
1554 enum cache_policy default_policy,
1555 enum cache_policy alternate_policy,
1556 void __user *alternate_aperture_base,
1557 uint64_t alternate_aperture_size)
1558{
bed4f110
FK		 1559 bool retval = true;
1560
1561 if (!dqm->asic_ops.set_cache_memory_policy)
1562 return retval;
64c7f8cf 1563
efeaed4d 1564 dqm_lock(dqm);
64c7f8cf
BG		 1565
1566 if (alternate_aperture_size == 0) {
1567 /* base > limit disables APE1 */
1568 qpd->sh_mem_ape1_base = 1;
1569 qpd->sh_mem_ape1_limit = 0;
1570 } else {
1571 /*
1572 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1573 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1574 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1575 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1576 * Verify that the base and size parameters can be
1577 * represented in this format and convert them.
1578 * Additionally restrict APE1 to user-mode addresses.
1579 */
1580
1581 uint64_t base = (uintptr_t)alternate_aperture_base;
1582 uint64_t limit = base + alternate_aperture_size - 1;
1583
ab7c1648
KR		 1584 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
1585 (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
1586 retval = false;
64c7f8cf 1587 goto out;
ab7c1648 1588 }
64c7f8cf
BG		 1589
1590 qpd->sh_mem_ape1_base = base >> 16;
1591 qpd->sh_mem_ape1_limit = limit >> 16;
1592 }
1593
bfd5e378 1594 retval = dqm->asic_ops.set_cache_memory_policy(
a22fc854
BG		 1595 dqm,
1596 qpd,
1597 default_policy,
1598 alternate_policy,
1599 alternate_aperture_base,
1600 alternate_aperture_size);
64c7f8cf 1601
d146c5a7 1602 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
64c7f8cf
BG		 1603 program_sh_mem_settings(dqm, qpd);
1604
79775b62 1605 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
64c7f8cf
BG		 1606 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1607 qpd->sh_mem_ape1_limit);
1608
64c7f8cf 1609out:
efeaed4d 1610 dqm_unlock(dqm);
ab7c1648 1611 return retval;
64c7f8cf
BG		 1612}
1613
d7b9bd22
FK		 1614static int set_trap_handler(struct device_queue_manager *dqm,
1615 struct qcm_process_device *qpd,
1616 uint64_t tba_addr,
1617 uint64_t tma_addr)
1618{
1619 uint64_t *tma;
1620
1621 if (dqm->dev->cwsr_enabled) {
1622 /* Jump from CWSR trap handler to user trap */
1623 tma = (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1624 tma[0] = tba_addr;
1625 tma[1] = tma_addr;
1626 } else {
1627 qpd->tba_addr = tba_addr;
1628 qpd->tma_addr = tma_addr;
1629 }
1630
1631 return 0;
1632}
1633
9fd3f1bf
FK		 1634static int process_termination_nocpsch(struct device_queue_manager *dqm,
1635 struct qcm_process_device *qpd)
1636{
1637 struct queue *q, *next;
1638 struct device_process_node *cur, *next_dpn;
1639 int retval = 0;
32cce8bc 1640 bool found = false;
9fd3f1bf 1641
efeaed4d 1642 dqm_lock(dqm);
9fd3f1bf
FK		 1643
1644 /* Clear all user mode queues */
1645 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1646 int ret;
1647
1648 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
1649 if (ret)
1650 retval = ret;
1651 }
1652
1653 /* Unregister process */
1654 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1655 if (qpd == cur->qpd) {
1656 list_del(&cur->list);
1657 kfree(cur);
1658 dqm->processes_count--;
32cce8bc 1659 found = true;
9fd3f1bf
FK		 1660 break;
1661 }
1662 }
1663
efeaed4d 1664 dqm_unlock(dqm);
32cce8bc
FK		 1665
1666 /* Outside the DQM lock because under the DQM lock we can't do
1667 * reclaim or take other locks that others hold while reclaiming.
1668 */
1669 if (found)
1670 kfd_dec_compute_active(dqm->dev);
1671
9fd3f1bf
FK		 1672 return retval;
1673}
1674
5df099e8
JC		 1675static int get_wave_state(struct device_queue_manager *dqm,
1676 struct queue *q,
1677 void __user *ctl_stack,
1678 u32 *ctl_stack_used_size,
1679 u32 *save_area_used_size)
1680{
4e6c6fc1 1681 struct mqd_manager *mqd_mgr;
5df099e8
JC		 1682 int r;
1683
1684 dqm_lock(dqm);
1685
1686 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
1687 q->properties.is_active || !q->device->cwsr_enabled) {
1688 r = -EINVAL;
1689 goto dqm_unlock;
1690 }
1691
d7c0b047 1692 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
5df099e8 1693
4e6c6fc1 1694 if (!mqd_mgr->get_wave_state) {
5df099e8
JC		 1695 r = -EINVAL;
1696 goto dqm_unlock;
1697 }
1698
4e6c6fc1
YZ		 1699 r = mqd_mgr->get_wave_state(mqd_mgr, q->mqd, ctl_stack,
1700 ctl_stack_used_size, save_area_used_size);
5df099e8
JC		 1701
1702dqm_unlock:
1703 dqm_unlock(dqm);
1704 return r;
1705}
9fd3f1bf
FK		 1706
1707static int process_termination_cpsch(struct device_queue_manager *dqm,
1708 struct qcm_process_device *qpd)
1709{
1710 int retval;
1711 struct queue *q, *next;
1712 struct kernel_queue *kq, *kq_next;
8d5f3552 1713 struct mqd_manager *mqd_mgr;
9fd3f1bf
FK		 1714 struct device_process_node *cur, *next_dpn;
1715 enum kfd_unmap_queues_filter filter =
1716 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
32cce8bc 1717 bool found = false;
9fd3f1bf
FK		 1718
1719 retval = 0;
1720
efeaed4d 1721 dqm_lock(dqm);
9fd3f1bf
FK		 1722
1723 /* Clean all kernel queues */
1724 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
1725 list_del(&kq->list);
b42902f4 1726 decrement_queue_count(dqm, kq->queue->properties.type);
9fd3f1bf
FK		 1727 qpd->is_debug = false;
1728 dqm->total_queue_count--;
1729 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
1730 }
1731
1732 /* Clear all user mode queues */
1733 list_for_each_entry(q, &qpd->queues_list, list) {
c7637c95 1734 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1b4670f6 1735 deallocate_sdma_queue(dqm, q);
c7637c95 1736 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1b4670f6 1737 deallocate_sdma_queue(dqm, q);
9fd3f1bf 1738
b8020b03 1739 if (q->properties.is_active) {
b42902f4 1740 decrement_queue_count(dqm, q->properties.type);
b8020b03
JG		 1741 if (q->properties.is_gws) {
1742 dqm->gws_queue_count--;
1743 qpd->mapped_gws_queue = false;
1744 }
1745 }
9fd3f1bf
FK		 1746
1747 dqm->total_queue_count--;
1748 }
1749
1750 /* Unregister process */
1751 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1752 if (qpd == cur->qpd) {
1753 list_del(&cur->list);
1754 kfree(cur);
1755 dqm->processes_count--;
32cce8bc 1756 found = true;
9fd3f1bf
FK		 1757 break;
1758 }
1759 }
1760
1761 retval = execute_queues_cpsch(dqm, filter, 0);
73ea648d 1762 if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
9fd3f1bf
FK		 1763 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
1764 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
1765 qpd->reset_wavefronts = false;
1766 }
1767
89cd9d23
PY		 1768 dqm_unlock(dqm);
1769
32cce8bc
FK		 1770 /* Outside the DQM lock because under the DQM lock we can't do
1771 * reclaim or take other locks that others hold while reclaiming.
1772 */
1773 if (found)
1774 kfd_dec_compute_active(dqm->dev);
1775
89cd9d23 1776 /* Lastly, free mqd resources.
8636e53c 1777 * Do free_mqd() after dqm_unlock to avoid circular locking.
89cd9d23 1778 */
9fd3f1bf 1779 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
fdfa090b
OZ		 1780 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1781 q->properties.type)];
9fd3f1bf 1782 list_del(&q->list);
bc920fd4 1783 qpd->queue_count--;
8636e53c 1784 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
9fd3f1bf
FK		 1785 }
1786
9fd3f1bf
FK		 1787 return retval;
1788}
1789
fdfa090b
OZ		 1790static int init_mqd_managers(struct device_queue_manager *dqm)
1791{
1792 int i, j;
1793 struct mqd_manager *mqd_mgr;
1794
1795 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
1796 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
1797 if (!mqd_mgr) {
1798 pr_err("mqd manager [%d] initialization failed\n", i);
1799 goto out_free;
1800 }
1801 dqm->mqd_mgrs[i] = mqd_mgr;
1802 }
1803
1804 return 0;
1805
1806out_free:
1807 for (j = 0; j < i; j++) {
1808 kfree(dqm->mqd_mgrs[j]);
1809 dqm->mqd_mgrs[j] = NULL;
1810 }
1811
1812 return -ENOMEM;
1813}
11614c36
OZ		 1814
1815/* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
1816static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
1817{
1818 int retval;
1819 struct kfd_dev *dev = dqm->dev;
1820 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
1821 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
c7637c95 1822 get_num_all_sdma_engines(dqm) *
11614c36
OZ		 1823 dev->device_info->num_sdma_queues_per_engine +
1824 dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size;
1825
1826 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->kgd, size,
1827 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr),
f2cc50ce 1828 (void *)&(mem_obj->cpu_ptr), false);
11614c36
OZ		 1829
1830 return retval;
1831}
1832
64c7f8cf
BG		 1833struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1834{
1835 struct device_queue_manager *dqm;
1836
79775b62 1837 pr_debug("Loading device queue manager\n");
a22fc854 1838
dbf56ab1 1839 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
64c7f8cf
BG		 1840 if (!dqm)
1841 return NULL;
1842
d146c5a7
FK		 1843 switch (dev->device_info->asic_family) {
1844 /* HWS is not available on Hawaii. */
1845 case CHIP_HAWAII:
1846 /* HWS depends on CWSR for timely dequeue. CWSR is not
1847 * available on Tonga.
1848 *
1849 * FIXME: This argument also applies to Kaveri.
1850 */
1851 case CHIP_TONGA:
1852 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
1853 break;
1854 default:
1855 dqm->sched_policy = sched_policy;
1856 break;
1857 }
1858
64c7f8cf 1859 dqm->dev = dev;
d146c5a7 1860 switch (dqm->sched_policy) {
64c7f8cf
BG		 1861 case KFD_SCHED_POLICY_HWS:
1862 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1863 /* initialize dqm for cp scheduling */
45c9a5e4
OG		 1864 dqm->ops.create_queue = create_queue_cpsch;
1865 dqm->ops.initialize = initialize_cpsch;
1866 dqm->ops.start = start_cpsch;
1867 dqm->ops.stop = stop_cpsch;
09c34e8d 1868 dqm->ops.pre_reset = pre_reset;
45c9a5e4
OG		 1869 dqm->ops.destroy_queue = destroy_queue_cpsch;
1870 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1871 dqm->ops.register_process = register_process;
1872 dqm->ops.unregister_process = unregister_process;
1873 dqm->ops.uninitialize = uninitialize;
45c9a5e4
OG		 1874 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1875 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1876 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
d7b9bd22 1877 dqm->ops.set_trap_handler = set_trap_handler;
9fd3f1bf 1878 dqm->ops.process_termination = process_termination_cpsch;
26103436
FK		 1879 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
1880 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
5df099e8 1881 dqm->ops.get_wave_state = get_wave_state;
64c7f8cf
BG		 1882 break;
1883 case KFD_SCHED_POLICY_NO_HWS:
1884 /* initialize dqm for no cp scheduling */
45c9a5e4
OG		 1885 dqm->ops.start = start_nocpsch;
1886 dqm->ops.stop = stop_nocpsch;
09c34e8d 1887 dqm->ops.pre_reset = pre_reset;
45c9a5e4
OG		 1888 dqm->ops.create_queue = create_queue_nocpsch;
1889 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1890 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1891 dqm->ops.register_process = register_process;
1892 dqm->ops.unregister_process = unregister_process;
45c9a5e4 1893 dqm->ops.initialize = initialize_nocpsch;
58dcd5bf 1894 dqm->ops.uninitialize = uninitialize;
45c9a5e4 1895 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
d7b9bd22 1896 dqm->ops.set_trap_handler = set_trap_handler;
9fd3f1bf 1897 dqm->ops.process_termination = process_termination_nocpsch;
26103436
FK		 1898 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
1899 dqm->ops.restore_process_queues =
1900 restore_process_queues_nocpsch;
5df099e8 1901 dqm->ops.get_wave_state = get_wave_state;
64c7f8cf
BG		 1902 break;
1903 default:
d146c5a7 1904 pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
32fa8219 1905 goto out_free;
64c7f8cf
BG		 1906 }
1907
a22fc854
BG		 1908 switch (dev->device_info->asic_family) {
1909 case CHIP_CARRIZO:
bfd5e378 1910 device_queue_manager_init_vi(&dqm->asic_ops);
300dec95
OG		 1911 break;
1912
a22fc854 1913 case CHIP_KAVERI:
bfd5e378 1914 device_queue_manager_init_cik(&dqm->asic_ops);
300dec95 1915 break;
97672cbe
FK		 1916
1917 case CHIP_HAWAII:
1918 device_queue_manager_init_cik_hawaii(&dqm->asic_ops);
1919 break;
1920
1921 case CHIP_TONGA:
1922 case CHIP_FIJI:
1923 case CHIP_POLARIS10:
1924 case CHIP_POLARIS11:
846a44d7 1925 case CHIP_POLARIS12:
ed81cd6e 1926 case CHIP_VEGAM:
97672cbe
FK		 1927 device_queue_manager_init_vi_tonga(&dqm->asic_ops);
1928 break;
bed4f110
FK		 1929
1930 case CHIP_VEGA10:
846a44d7 1931 case CHIP_VEGA12:
22a3a294 1932 case CHIP_VEGA20:
bed4f110 1933 case CHIP_RAVEN:
5a959a89 1934 case CHIP_RENOIR:
49adcf8a 1935 case CHIP_ARCTURUS:
bed4f110
FK		 1936 device_queue_manager_init_v9(&dqm->asic_ops);
1937 break;
14328aa5 1938 case CHIP_NAVI10:
0e94b564 1939 case CHIP_NAVI12:
8099ae40 1940 case CHIP_NAVI14:
3a2f0c81 1941 case CHIP_SIENNA_CICHLID:
de89b2e4 1942 case CHIP_NAVY_FLOUNDER:
14328aa5
PC		 1943 device_queue_manager_init_v10_navi10(&dqm->asic_ops);
1944 break;
e596b903
YZ		 1945 default:
1946 WARN(1, "Unexpected ASIC family %u",
1947 dev->device_info->asic_family);
1948 goto out_free;
a22fc854
BG		 1949 }
1950
fdfa090b
OZ		 1951 if (init_mqd_managers(dqm))
1952 goto out_free;
1953
11614c36
OZ		 1954 if (allocate_hiq_sdma_mqd(dqm)) {
1955 pr_err("Failed to allocate hiq sdma mqd trunk buffer\n");
1956 goto out_free;
1957 }
1958
32fa8219
FK		 1959 if (!dqm->ops.initialize(dqm))
1960 return dqm;
64c7f8cf 1961
32fa8219
FK		 1962out_free:
1963 kfree(dqm);
1964 return NULL;
64c7f8cf
BG		 1965}
1966
7fd5a6fb
Y		 1967static void deallocate_hiq_sdma_mqd(struct kfd_dev *dev,
1968 struct kfd_mem_obj *mqd)
11614c36
OZ		 1969{
1970 WARN(!mqd, "No hiq sdma mqd trunk to free");
1971
1972 amdgpu_amdkfd_free_gtt_mem(dev->kgd, mqd->gtt_mem);
1973}
1974
64c7f8cf
BG		 1975void device_queue_manager_uninit(struct device_queue_manager *dqm)
1976{
45c9a5e4 1977 dqm->ops.uninitialize(dqm);
11614c36 1978 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
64c7f8cf
BG		 1979 kfree(dqm);
1980}
851a645e 1981
2640c3fa 1982int kfd_process_vm_fault(struct device_queue_manager *dqm,
1983 unsigned int pasid)
1984{
1985 struct kfd_process_device *pdd;
1986 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
1987 int ret = 0;
1988
1989 if (!p)
1990 return -EINVAL;
1991 pdd = kfd_get_process_device_data(dqm->dev, p);
1992 if (pdd)
1993 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
1994 kfd_unref_process(p);
1995
1996 return ret;
1997}
1998
73ea648d
SL		 1999static void kfd_process_hw_exception(struct work_struct *work)
2000{
2001 struct device_queue_manager *dqm = container_of(work,
2002 struct device_queue_manager, hw_exception_work);
5b87245f 2003 amdgpu_amdkfd_gpu_reset(dqm->dev->kgd);
73ea648d
SL		 2004}
2005
851a645e
FK		 2006#if defined(CONFIG_DEBUG_FS)
2007
2008static void seq_reg_dump(struct seq_file *m,
2009 uint32_t (*dump)[2], uint32_t n_regs)
2010{
2011 uint32_t i, count;
2012
2013 for (i = 0, count = 0; i < n_regs; i++) {
2014 if (count == 0 ||
2015 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
2016 seq_printf(m, "%s %08x: %08x",
2017 i ? "\n" : "",
2018 dump[i][0], dump[i][1]);
2019 count = 7;
2020 } else {
2021 seq_printf(m, " %08x", dump[i][1]);
2022 count--;
2023 }
2024 }
2025
2026 seq_puts(m, "\n");
2027}
2028
2029int dqm_debugfs_hqds(struct seq_file *m, void *data)
2030{
2031 struct device_queue_manager *dqm = data;
2032 uint32_t (*dump)[2], n_regs;
2033 int pipe, queue;
2034 int r = 0;
2035
2c99a547
PY		 2036 if (!dqm->sched_running) {
2037 seq_printf(m, " Device is stopped\n");
2038
2039 return 0;
2040 }
2041
24f48a42 2042 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
14328aa5
PC		 2043 KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
2044 &dump, &n_regs);
24f48a42
OZ		 2045 if (!r) {
2046 seq_printf(m, " HIQ on MEC %d Pipe %d Queue %d\n",
14328aa5
PC		 2047 KFD_CIK_HIQ_PIPE/get_pipes_per_mec(dqm)+1,
2048 KFD_CIK_HIQ_PIPE%get_pipes_per_mec(dqm),
2049 KFD_CIK_HIQ_QUEUE);
24f48a42
OZ		 2050 seq_reg_dump(m, dump, n_regs);
2051
2052 kfree(dump);
2053 }
2054
851a645e
FK		 2055 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
2056 int pipe_offset = pipe * get_queues_per_pipe(dqm);
2057
2058 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
2059 if (!test_bit(pipe_offset + queue,
e6945304 2060 dqm->dev->shared_resources.cp_queue_bitmap))
851a645e
FK		 2061 continue;
2062
2063 r = dqm->dev->kfd2kgd->hqd_dump(
2064 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
2065 if (r)
2066 break;
2067
2068 seq_printf(m, " CP Pipe %d, Queue %d\n",
2069 pipe, queue);
2070 seq_reg_dump(m, dump, n_regs);
2071
2072 kfree(dump);
2073 }
2074 }
2075
c7637c95 2076 for (pipe = 0; pipe < get_num_all_sdma_engines(dqm); pipe++) {
d5094189
SL		 2077 for (queue = 0;
2078 queue < dqm->dev->device_info->num_sdma_queues_per_engine;
2079 queue++) {
851a645e
FK		 2080 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
2081 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
2082 if (r)
2083 break;
2084
2085 seq_printf(m, " SDMA Engine %d, RLC %d\n",
2086 pipe, queue);
2087 seq_reg_dump(m, dump, n_regs);
2088
2089 kfree(dump);
2090 }
2091 }
2092
2093 return r;
2094}
2095
a29ec470
SL		 2096int dqm_debugfs_execute_queues(struct device_queue_manager *dqm)
2097{
2098 int r = 0;
2099
2100 dqm_lock(dqm);
2101 dqm->active_runlist = true;
2102 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
2103 dqm_unlock(dqm);
2104
2105 return r;
2106}
2107
851a645e 2108#endif
Linux 6.x block layer and io_uring tree(s)