projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/amdkfd: Add TC flush on VMID deallocation for Hawaii
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdkfd / kfd_device_queue_manager.c
CommitLineData
64c7f8cf
BG		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
26103436
FK		 24#include <linux/ratelimit.h>
25#include <linux/printk.h>
64c7f8cf
BG		 26#include <linux/slab.h>
27#include <linux/list.h>
28#include <linux/types.h>
64c7f8cf 29#include <linux/bitops.h>
99331a51 30#include <linux/sched.h>
64c7f8cf
BG		 31#include "kfd_priv.h"
32#include "kfd_device_queue_manager.h"
33#include "kfd_mqd_manager.h"
34#include "cik_regs.h"
35#include "kfd_kernel_queue.h"
64c7f8cf
BG		 36
37/* Size of the per-pipe EOP queue */
38#define CIK_HPD_EOP_BYTES_LOG2 11
39#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
40
64c7f8cf
BG		 41static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
42 unsigned int pasid, unsigned int vmid);
43
44static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
45 struct queue *q,
46 struct qcm_process_device *qpd);
bcea3081 47
c4744e24
YZ		 48static int execute_queues_cpsch(struct device_queue_manager *dqm,
49 enum kfd_unmap_queues_filter filter,
50 uint32_t filter_param);
7da2bcf8 51static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 52 enum kfd_unmap_queues_filter filter,
53 uint32_t filter_param);
64c7f8cf 54
60a00956
FK		 55static int map_queues_cpsch(struct device_queue_manager *dqm);
56
bcea3081
BG		 57static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
58 struct queue *q,
59 struct qcm_process_device *qpd);
60
61static void deallocate_sdma_queue(struct device_queue_manager *dqm,
62 unsigned int sdma_queue_id);
64c7f8cf 63
bcea3081
BG		 64static inline
65enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
64c7f8cf 66{
bcea3081 67 if (type == KFD_QUEUE_TYPE_SDMA)
85d258f9
BG		 68 return KFD_MQD_TYPE_SDMA;
69 return KFD_MQD_TYPE_CP;
64c7f8cf
BG		 70}
71
d0b63bb3
AR		 72static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
73{
74 int i;
75 int pipe_offset = mec * dqm->dev->shared_resources.num_pipe_per_mec
76 + pipe * dqm->dev->shared_resources.num_queue_per_pipe;
77
78 /* queue is available for KFD usage if bit is 1 */
79 for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i)
80 if (test_bit(pipe_offset + i,
81 dqm->dev->shared_resources.queue_bitmap))
82 return true;
83 return false;
84}
85
d0b63bb3 86unsigned int get_queues_num(struct device_queue_manager *dqm)
64ea8f4a 87{
d0b63bb3
AR		 88 return bitmap_weight(dqm->dev->shared_resources.queue_bitmap,
89 KGD_MAX_QUEUES);
64ea8f4a
OG		 90}
91
d0b63bb3 92unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
64c7f8cf 93{
d0b63bb3
AR		 94 return dqm->dev->shared_resources.num_queue_per_pipe;
95}
96
97unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
98{
d0b63bb3 99 return dqm->dev->shared_resources.num_pipe_per_mec;
64c7f8cf
BG		 100}
101
a22fc854 102void program_sh_mem_settings(struct device_queue_manager *dqm,
64c7f8cf
BG		 103 struct qcm_process_device *qpd)
104{
cea405b1
XZ		 105 return dqm->dev->kfd2kgd->program_sh_mem_settings(
106 dqm->dev->kgd, qpd->vmid,
64c7f8cf
BG		 107 qpd->sh_mem_config,
108 qpd->sh_mem_ape1_base,
109 qpd->sh_mem_ape1_limit,
110 qpd->sh_mem_bases);
111}
112
113static int allocate_vmid(struct device_queue_manager *dqm,
114 struct qcm_process_device *qpd,
115 struct queue *q)
116{
117 int bit, allocated_vmid;
118
119 if (dqm->vmid_bitmap == 0)
120 return -ENOMEM;
121
4252bf68
HK		 122 bit = ffs(dqm->vmid_bitmap) - 1;
123 dqm->vmid_bitmap &= ~(1 << bit);
64c7f8cf 124
44008d7a 125 allocated_vmid = bit + dqm->dev->vm_info.first_vmid_kfd;
79775b62 126 pr_debug("vmid allocation %d\n", allocated_vmid);
64c7f8cf
BG		 127 qpd->vmid = allocated_vmid;
128 q->properties.vmid = allocated_vmid;
129
130 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
131 program_sh_mem_settings(dqm, qpd);
132
403575c4
FK		 133 /* qpd->page_table_base is set earlier when register_process()
134 * is called, i.e. when the first queue is created.
135 */
136 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd,
137 qpd->vmid,
138 qpd->page_table_base);
139 /* invalidate the VM context after pasid and vmid mapping is set up */
140 kfd_flush_tlb(qpd_to_pdd(qpd));
141
64c7f8cf
BG		 142 return 0;
143}
144
552764b6
FK		 145static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
146 struct qcm_process_device *qpd)
147{
148 uint32_t len;
149
150 if (!qpd->ib_kaddr)
151 return -ENOMEM;
152
153 len = pm_create_release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
154
155 return kdev->kfd2kgd->submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid,
156 qpd->ib_base, (uint32_t *)qpd->ib_kaddr, len);
157}
158
64c7f8cf
BG		 159static void deallocate_vmid(struct device_queue_manager *dqm,
160 struct qcm_process_device *qpd,
161 struct queue *q)
162{
44008d7a 163 int bit = qpd->vmid - dqm->dev->vm_info.first_vmid_kfd;
64c7f8cf 164
552764b6
FK		 165 /* On GFX v7, CP doesn't flush TC at dequeue */
166 if (q->device->device_info->asic_family == CHIP_HAWAII)
167 if (flush_texture_cache_nocpsch(q->device, qpd))
168 pr_err("Failed to flush TC\n");
169
403575c4
FK		 170 kfd_flush_tlb(qpd_to_pdd(qpd));
171
2030664b
BG		 172 /* Release the vmid mapping */
173 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
174
4252bf68 175 dqm->vmid_bitmap |= (1 << bit);
64c7f8cf
BG		 176 qpd->vmid = 0;
177 q->properties.vmid = 0;
178}
179
180static int create_queue_nocpsch(struct device_queue_manager *dqm,
181 struct queue *q,
b46cb7d7 182 struct qcm_process_device *qpd)
64c7f8cf
BG		 183{
184 int retval;
185
64c7f8cf
BG		 186 print_queue(q);
187
188 mutex_lock(&dqm->lock);
189
b8cbab04 190 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 191 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04 192 dqm->total_queue_count);
ab7c1648
KR		 193 retval = -EPERM;
194 goto out_unlock;
b8cbab04
OG		 195 }
196
64c7f8cf
BG		 197 if (list_empty(&qpd->queues_list)) {
198 retval = allocate_vmid(dqm, qpd, q);
ab7c1648
KR		 199 if (retval)
200 goto out_unlock;
64c7f8cf 201 }
64c7f8cf 202 q->properties.vmid = qpd->vmid;
26103436
FK		 203 /*
204 * Eviction state logic: we only mark active queues as evicted
205 * to avoid the overhead of restoring inactive queues later
206 */
207 if (qpd->evicted)
208 q->properties.is_evicted = (q->properties.queue_size > 0 &&
209 q->properties.queue_percent > 0 &&
210 q->properties.queue_address != 0);
64c7f8cf 211
373d7080
FK		 212 q->properties.tba_addr = qpd->tba_addr;
213 q->properties.tma_addr = qpd->tma_addr;
214
bcea3081
BG		 215 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
216 retval = create_compute_queue_nocpsch(dqm, q, qpd);
ab7c1648 217 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
bcea3081 218 retval = create_sdma_queue_nocpsch(dqm, q, qpd);
ab7c1648
KR		 219 else
220 retval = -EINVAL;
64c7f8cf 221
4eacc26b 222 if (retval) {
b46cb7d7 223 if (list_empty(&qpd->queues_list))
64c7f8cf 224 deallocate_vmid(dqm, qpd, q);
ab7c1648 225 goto out_unlock;
64c7f8cf
BG		 226 }
227
228 list_add(&q->list, &qpd->queues_list);
bc920fd4 229 qpd->queue_count++;
b6819cec
JC		 230 if (q->properties.is_active)
231 dqm->queue_count++;
64c7f8cf 232
bcea3081
BG		 233 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
234 dqm->sdma_queue_count++;
64c7f8cf 235
b8cbab04
OG		 236 /*
237 * Unconditionally increment this counter, regardless of the queue's
238 * type or whether the queue is active.
239 */
240 dqm->total_queue_count++;
241 pr_debug("Total of %d queues are accountable so far\n",
242 dqm->total_queue_count);
243
ab7c1648 244out_unlock:
64c7f8cf 245 mutex_unlock(&dqm->lock);
ab7c1648 246 return retval;
64c7f8cf
BG		 247}
248
249static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
250{
251 bool set;
f0ec5b99 252 int pipe, bit, i;
64c7f8cf
BG		 253
254 set = false;
255
8eabaf54
KR		 256 for (pipe = dqm->next_pipe_to_allocate, i = 0;
257 i < get_pipes_per_mec(dqm);
d0b63bb3
AR		 258 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
259
260 if (!is_pipe_enabled(dqm, 0, pipe))
261 continue;
262
64c7f8cf 263 if (dqm->allocated_queues[pipe] != 0) {
4252bf68
HK		 264 bit = ffs(dqm->allocated_queues[pipe]) - 1;
265 dqm->allocated_queues[pipe] &= ~(1 << bit);
64c7f8cf
BG		 266 q->pipe = pipe;
267 q->queue = bit;
268 set = true;
269 break;
270 }
271 }
272
991ca8ee 273 if (!set)
64c7f8cf
BG		 274 return -EBUSY;
275
79775b62 276 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
64c7f8cf 277 /* horizontal hqd allocation */
d0b63bb3 278 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
64c7f8cf
BG		 279
280 return 0;
281}
282
283static inline void deallocate_hqd(struct device_queue_manager *dqm,
284 struct queue *q)
285{
4252bf68 286 dqm->allocated_queues[q->pipe] |= (1 << q->queue);
64c7f8cf
BG		 287}
288
289static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
290 struct queue *q,
291 struct qcm_process_device *qpd)
292{
293 int retval;
294 struct mqd_manager *mqd;
295
45c9a5e4 296 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
4eacc26b 297 if (!mqd)
64c7f8cf
BG		 298 return -ENOMEM;
299
300 retval = allocate_hqd(dqm, q);
4eacc26b 301 if (retval)
64c7f8cf
BG		 302 return retval;
303
304 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
305 &q->gart_mqd_addr, &q->properties);
ab7c1648
KR		 306 if (retval)
307 goto out_deallocate_hqd;
64c7f8cf 308
79775b62
KR		 309 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
310 q->pipe, q->queue);
030e416b 311
6a1c9510
MR		 312 dqm->dev->kfd2kgd->set_scratch_backing_va(
313 dqm->dev->kgd, qpd->sh_hidden_private_base, qpd->vmid);
314
60a00956
FK		 315 if (!q->properties.is_active)
316 return 0;
317
70539bd7
FK		 318 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue, &q->properties,
319 q->process->mm);
ab7c1648
KR		 320 if (retval)
321 goto out_uninit_mqd;
030e416b 322
64c7f8cf 323 return 0;
ab7c1648
KR		 324
325out_uninit_mqd:
326 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
327out_deallocate_hqd:
328 deallocate_hqd(dqm, q);
329
330 return retval;
64c7f8cf
BG		 331}
332
9fd3f1bf
FK		 333/* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
334 * to avoid asynchronized access
335 */
336static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
64c7f8cf
BG		 337 struct qcm_process_device *qpd,
338 struct queue *q)
339{
340 int retval;
341 struct mqd_manager *mqd;
342
9fd3f1bf
FK		 343 mqd = dqm->ops.get_mqd_manager(dqm,
344 get_mqd_type_from_queue_type(q->properties.type));
345 if (!mqd)
346 return -ENOMEM;
64c7f8cf 347
c2e1b3a4 348 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
c2e1b3a4
BG		 349 deallocate_hqd(dqm, q);
350 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
c2e1b3a4
BG		 351 dqm->sdma_queue_count--;
352 deallocate_sdma_queue(dqm, q->sdma_id);
7113cd65 353 } else {
79775b62 354 pr_debug("q->properties.type %d is invalid\n",
7113cd65 355 q->properties.type);
9fd3f1bf 356 return -EINVAL;
64c7f8cf 357 }
9fd3f1bf 358 dqm->total_queue_count--;
64c7f8cf
BG		 359
360 retval = mqd->destroy_mqd(mqd, q->mqd,
c2e1b3a4 361 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
b90e3fbe 362 KFD_UNMAP_LATENCY_MS,
64c7f8cf 363 q->pipe, q->queue);
9fd3f1bf
FK		 364 if (retval == -ETIME)
365 qpd->reset_wavefronts = true;
64c7f8cf 366
64c7f8cf
BG		 367 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
368
369 list_del(&q->list);
9fd3f1bf
FK		 370 if (list_empty(&qpd->queues_list)) {
371 if (qpd->reset_wavefronts) {
372 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
373 dqm->dev);
374 /* dbgdev_wave_reset_wavefronts has to be called before
375 * deallocate_vmid(), i.e. when vmid is still in use.
376 */
377 dbgdev_wave_reset_wavefronts(dqm->dev,
378 qpd->pqm->process);
379 qpd->reset_wavefronts = false;
380 }
381
64c7f8cf 382 deallocate_vmid(dqm, qpd, q);
9fd3f1bf 383 }
bc920fd4 384 qpd->queue_count--;
b6819cec
JC		 385 if (q->properties.is_active)
386 dqm->queue_count--;
b8cbab04 387
9fd3f1bf
FK		 388 return retval;
389}
b8cbab04 390
9fd3f1bf
FK		 391static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
392 struct qcm_process_device *qpd,
393 struct queue *q)
394{
395 int retval;
396
397 mutex_lock(&dqm->lock);
398 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
64c7f8cf 399 mutex_unlock(&dqm->lock);
9fd3f1bf 400
64c7f8cf
BG		 401 return retval;
402}
403
404static int update_queue(struct device_queue_manager *dqm, struct queue *q)
405{
406 int retval;
407 struct mqd_manager *mqd;
26103436 408 struct kfd_process_device *pdd;
b6ffbab8 409 bool prev_active = false;
64c7f8cf 410
64c7f8cf 411 mutex_lock(&dqm->lock);
26103436
FK		 412 pdd = kfd_get_process_device_data(q->device, q->process);
413 if (!pdd) {
414 retval = -ENODEV;
415 goto out_unlock;
416 }
0b3674ae
OG		 417 mqd = dqm->ops.get_mqd_manager(dqm,
418 get_mqd_type_from_queue_type(q->properties.type));
4eacc26b 419 if (!mqd) {
ab7c1648
KR		 420 retval = -ENOMEM;
421 goto out_unlock;
64c7f8cf 422 }
26103436
FK		 423 /*
424 * Eviction state logic: we only mark active queues as evicted
425 * to avoid the overhead of restoring inactive queues later
426 */
427 if (pdd->qpd.evicted)
428 q->properties.is_evicted = (q->properties.queue_size > 0 &&
429 q->properties.queue_percent > 0 &&
430 q->properties.queue_address != 0);
64c7f8cf 431
60a00956
FK		 432 /* Save previous activity state for counters */
433 prev_active = q->properties.is_active;
434
435 /* Make sure the queue is unmapped before updating the MQD */
d146c5a7 436 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
60a00956
FK		 437 retval = unmap_queues_cpsch(dqm,
438 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
894a8293 439 if (retval) {
60a00956
FK		 440 pr_err("unmap queue failed\n");
441 goto out_unlock;
442 }
894a8293 443 } else if (prev_active &&
60a00956
FK		 444 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
445 q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
446 retval = mqd->destroy_mqd(mqd, q->mqd,
447 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
448 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
449 if (retval) {
450 pr_err("destroy mqd failed\n");
451 goto out_unlock;
452 }
453 }
454
455 retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
456
096d1a3e
FK		 457 /*
458 * check active state vs. the previous state and modify
459 * counter accordingly. map_queues_cpsch uses the
460 * dqm->queue_count to determine whether a new runlist must be
461 * uploaded.
462 */
463 if (q->properties.is_active && !prev_active)
464 dqm->queue_count++;
465 else if (!q->properties.is_active && prev_active)
466 dqm->queue_count--;
467
d146c5a7 468 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
60a00956 469 retval = map_queues_cpsch(dqm);
894a8293 470 else if (q->properties.is_active &&
60a00956
FK		 471 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
472 q->properties.type == KFD_QUEUE_TYPE_SDMA))
473 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue,
474 &q->properties, q->process->mm);
b6ffbab8 475
ab7c1648 476out_unlock:
64c7f8cf
BG		 477 mutex_unlock(&dqm->lock);
478 return retval;
479}
480
58dcd5bf 481static struct mqd_manager *get_mqd_manager(
64c7f8cf
BG		 482 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
483{
484 struct mqd_manager *mqd;
485
32fa8219
FK		 486 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
487 return NULL;
64c7f8cf 488
79775b62 489 pr_debug("mqd type %d\n", type);
64c7f8cf
BG		 490
491 mqd = dqm->mqds[type];
492 if (!mqd) {
493 mqd = mqd_manager_init(type, dqm->dev);
4eacc26b 494 if (!mqd)
79775b62 495 pr_err("mqd manager is NULL");
64c7f8cf
BG		 496 dqm->mqds[type] = mqd;
497 }
498
499 return mqd;
500}
501
26103436
FK		 502static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
503 struct qcm_process_device *qpd)
504{
505 struct queue *q;
506 struct mqd_manager *mqd;
507 struct kfd_process_device *pdd;
508 int retval = 0;
509
510 mutex_lock(&dqm->lock);
511 if (qpd->evicted++ > 0) /* already evicted, do nothing */
512 goto out;
513
514 pdd = qpd_to_pdd(qpd);
515 pr_info_ratelimited("Evicting PASID %u queues\n",
516 pdd->process->pasid);
517
518 /* unactivate all active queues on the qpd */
519 list_for_each_entry(q, &qpd->queues_list, list) {
520 if (!q->properties.is_active)
521 continue;
522 mqd = dqm->ops.get_mqd_manager(dqm,
523 get_mqd_type_from_queue_type(q->properties.type));
524 if (!mqd) { /* should not be here */
525 pr_err("Cannot evict queue, mqd mgr is NULL\n");
526 retval = -ENOMEM;
527 goto out;
528 }
529 q->properties.is_evicted = true;
530 q->properties.is_active = false;
531 retval = mqd->destroy_mqd(mqd, q->mqd,
532 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
533 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
534 if (retval)
535 goto out;
536 dqm->queue_count--;
537 }
538
539out:
540 mutex_unlock(&dqm->lock);
541 return retval;
542}
543
544static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
545 struct qcm_process_device *qpd)
546{
547 struct queue *q;
548 struct kfd_process_device *pdd;
549 int retval = 0;
550
551 mutex_lock(&dqm->lock);
552 if (qpd->evicted++ > 0) /* already evicted, do nothing */
553 goto out;
554
555 pdd = qpd_to_pdd(qpd);
556 pr_info_ratelimited("Evicting PASID %u queues\n",
557 pdd->process->pasid);
558
559 /* unactivate all active queues on the qpd */
560 list_for_each_entry(q, &qpd->queues_list, list) {
561 if (!q->properties.is_active)
562 continue;
563 q->properties.is_evicted = true;
564 q->properties.is_active = false;
565 dqm->queue_count--;
566 }
567 retval = execute_queues_cpsch(dqm,
568 qpd->is_debug ?
569 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
570 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
571
572out:
573 mutex_unlock(&dqm->lock);
574 return retval;
575}
576
577static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
578 struct qcm_process_device *qpd)
579{
580 struct queue *q;
581 struct mqd_manager *mqd;
582 struct kfd_process_device *pdd;
583 uint32_t pd_base;
584 int retval = 0;
585
586 pdd = qpd_to_pdd(qpd);
587 /* Retrieve PD base */
588 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
589
590 mutex_lock(&dqm->lock);
591 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
592 goto out;
593 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
594 qpd->evicted--;
595 goto out;
596 }
597
598 pr_info_ratelimited("Restoring PASID %u queues\n",
599 pdd->process->pasid);
600
601 /* Update PD Base in QPD */
602 qpd->page_table_base = pd_base;
603 pr_debug("Updated PD address to 0x%08x\n", pd_base);
604
605 if (!list_empty(&qpd->queues_list)) {
606 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
607 dqm->dev->kgd,
608 qpd->vmid,
609 qpd->page_table_base);
610 kfd_flush_tlb(pdd);
611 }
612
613 /* activate all active queues on the qpd */
614 list_for_each_entry(q, &qpd->queues_list, list) {
615 if (!q->properties.is_evicted)
616 continue;
617 mqd = dqm->ops.get_mqd_manager(dqm,
618 get_mqd_type_from_queue_type(q->properties.type));
619 if (!mqd) { /* should not be here */
620 pr_err("Cannot restore queue, mqd mgr is NULL\n");
621 retval = -ENOMEM;
622 goto out;
623 }
624 q->properties.is_evicted = false;
625 q->properties.is_active = true;
626 retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
627 q->queue, &q->properties,
628 q->process->mm);
629 if (retval)
630 goto out;
631 dqm->queue_count++;
632 }
633 qpd->evicted = 0;
634out:
635 mutex_unlock(&dqm->lock);
636 return retval;
637}
638
639static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
640 struct qcm_process_device *qpd)
641{
642 struct queue *q;
643 struct kfd_process_device *pdd;
644 uint32_t pd_base;
645 int retval = 0;
646
647 pdd = qpd_to_pdd(qpd);
648 /* Retrieve PD base */
649 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
650
651 mutex_lock(&dqm->lock);
652 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
653 goto out;
654 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
655 qpd->evicted--;
656 goto out;
657 }
658
659 pr_info_ratelimited("Restoring PASID %u queues\n",
660 pdd->process->pasid);
661
662 /* Update PD Base in QPD */
663 qpd->page_table_base = pd_base;
664 pr_debug("Updated PD address to 0x%08x\n", pd_base);
665
666 /* activate all active queues on the qpd */
667 list_for_each_entry(q, &qpd->queues_list, list) {
668 if (!q->properties.is_evicted)
669 continue;
670 q->properties.is_evicted = false;
671 q->properties.is_active = true;
672 dqm->queue_count++;
673 }
674 retval = execute_queues_cpsch(dqm,
675 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
676 if (!retval)
677 qpd->evicted = 0;
678out:
679 mutex_unlock(&dqm->lock);
680 return retval;
681}
682
58dcd5bf 683static int register_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 684 struct qcm_process_device *qpd)
685{
686 struct device_process_node *n;
403575c4
FK		 687 struct kfd_process_device *pdd;
688 uint32_t pd_base;
a22fc854 689 int retval;
64c7f8cf 690
dbf56ab1 691 n = kzalloc(sizeof(*n), GFP_KERNEL);
64c7f8cf
BG		 692 if (!n)
693 return -ENOMEM;
694
695 n->qpd = qpd;
696
403575c4
FK		 697 pdd = qpd_to_pdd(qpd);
698 /* Retrieve PD base */
699 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
700
64c7f8cf
BG		 701 mutex_lock(&dqm->lock);
702 list_add(&n->list, &dqm->queues);
703
403575c4
FK		 704 /* Update PD Base in QPD */
705 qpd->page_table_base = pd_base;
706
bfd5e378 707 retval = dqm->asic_ops.update_qpd(dqm, qpd);
a22fc854 708
64c7f8cf
BG		 709 dqm->processes_count++;
710
711 mutex_unlock(&dqm->lock);
712
a22fc854 713 return retval;
64c7f8cf
BG		 714}
715
58dcd5bf 716static int unregister_process(struct device_queue_manager *dqm,
64c7f8cf
BG		 717 struct qcm_process_device *qpd)
718{
719 int retval;
720 struct device_process_node *cur, *next;
721
1e5ec956
OG		 722 pr_debug("qpd->queues_list is %s\n",
723 list_empty(&qpd->queues_list) ? "empty" : "not empty");
64c7f8cf
BG		 724
725 retval = 0;
726 mutex_lock(&dqm->lock);
727
728 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
729 if (qpd == cur->qpd) {
730 list_del(&cur->list);
f5d896bb 731 kfree(cur);
64c7f8cf
BG		 732 dqm->processes_count--;
733 goto out;
734 }
735 }
736 /* qpd not found in dqm list */
737 retval = 1;
738out:
739 mutex_unlock(&dqm->lock);
740 return retval;
741}
742
743static int
744set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
745 unsigned int vmid)
746{
747 uint32_t pasid_mapping;
748
cea405b1
XZ		 749 pasid_mapping = (pasid == 0) ? 0 :
750 (uint32_t)pasid |
751 ATC_VMID_PASID_MAPPING_VALID;
752
753 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
754 dqm->dev->kgd, pasid_mapping,
64c7f8cf
BG		 755 vmid);
756}
757
2249d558
AL		 758static void init_interrupts(struct device_queue_manager *dqm)
759{
760 unsigned int i;
761
d0b63bb3
AR		 762 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
763 if (is_pipe_enabled(dqm, 0, i))
764 dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
2249d558
AL		 765}
766
64c7f8cf
BG		 767static int initialize_nocpsch(struct device_queue_manager *dqm)
768{
86194cf8 769 int pipe, queue;
64c7f8cf 770
79775b62 771 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf 772
ab7c1648
KR		 773 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
774 sizeof(unsigned int), GFP_KERNEL);
775 if (!dqm->allocated_queues)
776 return -ENOMEM;
777
64c7f8cf
BG		 778 mutex_init(&dqm->lock);
779 INIT_LIST_HEAD(&dqm->queues);
780 dqm->queue_count = dqm->next_pipe_to_allocate = 0;
bcea3081 781 dqm->sdma_queue_count = 0;
64c7f8cf 782
86194cf8
FK		 783 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
784 int pipe_offset = pipe * get_queues_per_pipe(dqm);
785
786 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
787 if (test_bit(pipe_offset + queue,
788 dqm->dev->shared_resources.queue_bitmap))
789 dqm->allocated_queues[pipe] |= 1 << queue;
790 }
64c7f8cf 791
44008d7a 792 dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
bcea3081 793 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
64c7f8cf 794
64c7f8cf
BG		 795 return 0;
796}
797
58dcd5bf 798static void uninitialize(struct device_queue_manager *dqm)
64c7f8cf 799{
6f9d54fd
OG		 800 int i;
801
32fa8219 802 WARN_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
64c7f8cf
BG		 803
804 kfree(dqm->allocated_queues);
6f9d54fd
OG		 805 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
806 kfree(dqm->mqds[i]);
64c7f8cf 807 mutex_destroy(&dqm->lock);
a86aa3ca 808 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
64c7f8cf
BG		 809}
810
811static int start_nocpsch(struct device_queue_manager *dqm)
812{
2249d558 813 init_interrupts(dqm);
552764b6 814 return pm_init(&dqm->packets, dqm);
64c7f8cf
BG		 815}
816
817static int stop_nocpsch(struct device_queue_manager *dqm)
818{
552764b6 819 pm_uninit(&dqm->packets);
64c7f8cf
BG		 820 return 0;
821}
822
bcea3081
BG		 823static int allocate_sdma_queue(struct device_queue_manager *dqm,
824 unsigned int *sdma_queue_id)
825{
826 int bit;
827
828 if (dqm->sdma_bitmap == 0)
829 return -ENOMEM;
830
4252bf68
HK		 831 bit = ffs(dqm->sdma_bitmap) - 1;
832 dqm->sdma_bitmap &= ~(1 << bit);
bcea3081
BG		 833 *sdma_queue_id = bit;
834
835 return 0;
836}
837
838static void deallocate_sdma_queue(struct device_queue_manager *dqm,
839 unsigned int sdma_queue_id)
840{
010b82e7 841 if (sdma_queue_id >= CIK_SDMA_QUEUES)
bcea3081 842 return;
4252bf68 843 dqm->sdma_bitmap |= (1 << sdma_queue_id);
bcea3081
BG		 844}
845
bcea3081
BG		 846static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
847 struct queue *q,
848 struct qcm_process_device *qpd)
849{
850 struct mqd_manager *mqd;
851 int retval;
852
45c9a5e4 853 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
bcea3081
BG		 854 if (!mqd)
855 return -ENOMEM;
856
857 retval = allocate_sdma_queue(dqm, &q->sdma_id);
4eacc26b 858 if (retval)
bcea3081
BG		 859 return retval;
860
e139cd2a 861 q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
862 q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
bcea3081 863
79775b62
KR		 864 pr_debug("SDMA id is: %d\n", q->sdma_id);
865 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
866 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
bcea3081 867
bfd5e378 868 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
bcea3081
BG		 869 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
870 &q->gart_mqd_addr, &q->properties);
ab7c1648
KR		 871 if (retval)
872 goto out_deallocate_sdma_queue;
bcea3081 873
70539bd7 874 retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);
ab7c1648
KR		 875 if (retval)
876 goto out_uninit_mqd;
4fadf6b6 877
bcea3081 878 return 0;
ab7c1648
KR		 879
880out_uninit_mqd:
881 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
882out_deallocate_sdma_queue:
883 deallocate_sdma_queue(dqm, q->sdma_id);
884
885 return retval;
bcea3081
BG		 886}
887
64c7f8cf
BG		 888/*
889 * Device Queue Manager implementation for cp scheduler
890 */
891
892static int set_sched_resources(struct device_queue_manager *dqm)
893{
d0b63bb3 894 int i, mec;
64c7f8cf 895 struct scheduling_resources res;
64c7f8cf 896
44008d7a 897 res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
d0b63bb3
AR		 898
899 res.queue_mask = 0;
900 for (i = 0; i < KGD_MAX_QUEUES; ++i) {
901 mec = (i / dqm->dev->shared_resources.num_queue_per_pipe)
902 / dqm->dev->shared_resources.num_pipe_per_mec;
903
904 if (!test_bit(i, dqm->dev->shared_resources.queue_bitmap))
905 continue;
906
907 /* only acquire queues from the first MEC */
908 if (mec > 0)
909 continue;
910
911 /* This situation may be hit in the future if a new HW
912 * generation exposes more than 64 queues. If so, the
8eabaf54
KR		 913 * definition of res.queue_mask needs updating
914 */
1d11ee89 915 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
d0b63bb3
AR		 916 pr_err("Invalid queue enabled by amdgpu: %d\n", i);
917 break;
918 }
919
920 res.queue_mask |= (1ull << i);
921 }
64c7f8cf
BG		 922 res.gws_mask = res.oac_mask = res.gds_heap_base =
923 res.gds_heap_size = 0;
924
79775b62
KR		 925 pr_debug("Scheduling resources:\n"
926 "vmid mask: 0x%8X\n"
927 "queue mask: 0x%8llX\n",
64c7f8cf
BG		 928 res.vmid_mask, res.queue_mask);
929
930 return pm_send_set_resources(&dqm->packets, &res);
931}
932
933static int initialize_cpsch(struct device_queue_manager *dqm)
934{
79775b62 935 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
64c7f8cf
BG		 936
937 mutex_init(&dqm->lock);
938 INIT_LIST_HEAD(&dqm->queues);
939 dqm->queue_count = dqm->processes_count = 0;
bcea3081 940 dqm->sdma_queue_count = 0;
64c7f8cf 941 dqm->active_runlist = false;
e139cd2a 942 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
64c7f8cf 943
bfd5e378 944 return 0;
64c7f8cf
BG		 945}
946
947static int start_cpsch(struct device_queue_manager *dqm)
948{
64c7f8cf
BG		 949 int retval;
950
64c7f8cf
BG		 951 retval = 0;
952
953 retval = pm_init(&dqm->packets, dqm);
4eacc26b 954 if (retval)
64c7f8cf
BG		 955 goto fail_packet_manager_init;
956
957 retval = set_sched_resources(dqm);
4eacc26b 958 if (retval)
64c7f8cf
BG		 959 goto fail_set_sched_resources;
960
79775b62 961 pr_debug("Allocating fence memory\n");
64c7f8cf
BG		 962
963 /* allocate fence memory on the gart */
a86aa3ca
OG		 964 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
965 &dqm->fence_mem);
64c7f8cf 966
4eacc26b 967 if (retval)
64c7f8cf
BG		 968 goto fail_allocate_vidmem;
969
970 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
971 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
2249d558
AL		 972
973 init_interrupts(dqm);
974
ac30c783 975 mutex_lock(&dqm->lock);
c4744e24 976 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
ac30c783 977 mutex_unlock(&dqm->lock);
64c7f8cf
BG		 978
979 return 0;
980fail_allocate_vidmem:
981fail_set_sched_resources:
982 pm_uninit(&dqm->packets);
983fail_packet_manager_init:
984 return retval;
985}
986
987static int stop_cpsch(struct device_queue_manager *dqm)
988{
ac30c783 989 mutex_lock(&dqm->lock);
4465f466 990 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
ac30c783 991 mutex_unlock(&dqm->lock);
64c7f8cf 992
a86aa3ca 993 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
64c7f8cf
BG		 994 pm_uninit(&dqm->packets);
995
996 return 0;
997}
998
999static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1000 struct kernel_queue *kq,
1001 struct qcm_process_device *qpd)
1002{
64c7f8cf 1003 mutex_lock(&dqm->lock);
b8cbab04 1004 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1005 pr_warn("Can't create new kernel queue because %d queues were already created\n",
b8cbab04
OG		 1006 dqm->total_queue_count);
1007 mutex_unlock(&dqm->lock);
1008 return -EPERM;
1009 }
1010
1011 /*
1012 * Unconditionally increment this counter, regardless of the queue's
1013 * type or whether the queue is active.
1014 */
1015 dqm->total_queue_count++;
1016 pr_debug("Total of %d queues are accountable so far\n",
1017 dqm->total_queue_count);
1018
64c7f8cf
BG		 1019 list_add(&kq->list, &qpd->priv_queue_list);
1020 dqm->queue_count++;
1021 qpd->is_debug = true;
c4744e24 1022 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
64c7f8cf
BG		 1023 mutex_unlock(&dqm->lock);
1024
1025 return 0;
1026}
1027
1028static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1029 struct kernel_queue *kq,
1030 struct qcm_process_device *qpd)
1031{
64c7f8cf 1032 mutex_lock(&dqm->lock);
64c7f8cf
BG		 1033 list_del(&kq->list);
1034 dqm->queue_count--;
1035 qpd->is_debug = false;
c4744e24 1036 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
b8cbab04
OG		 1037 /*
1038 * Unconditionally decrement this counter, regardless of the queue's
1039 * type.
1040 */
8b58f261 1041 dqm->total_queue_count--;
b8cbab04
OG		 1042 pr_debug("Total of %d queues are accountable so far\n",
1043 dqm->total_queue_count);
64c7f8cf
BG		 1044 mutex_unlock(&dqm->lock);
1045}
1046
1047static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
b46cb7d7 1048 struct qcm_process_device *qpd)
64c7f8cf
BG		 1049{
1050 int retval;
1051 struct mqd_manager *mqd;
1052
64c7f8cf
BG		 1053 retval = 0;
1054
64c7f8cf
BG		 1055 mutex_lock(&dqm->lock);
1056
b8cbab04 1057 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
79775b62 1058 pr_warn("Can't create new usermode queue because %d queues were already created\n",
b8cbab04
OG		 1059 dqm->total_queue_count);
1060 retval = -EPERM;
1061 goto out;
1062 }
1063
e139cd2a 1064 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1065 retval = allocate_sdma_queue(dqm, &q->sdma_id);
894a8293 1066 if (retval)
e139cd2a 1067 goto out;
1068 q->properties.sdma_queue_id =
1069 q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
1070 q->properties.sdma_engine_id =
1071 q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
1072 }
45c9a5e4 1073 mqd = dqm->ops.get_mqd_manager(dqm,
bcea3081
BG		 1074 get_mqd_type_from_queue_type(q->properties.type));
1075
4eacc26b 1076 if (!mqd) {
ab7c1648
KR		 1077 retval = -ENOMEM;
1078 goto out;
64c7f8cf 1079 }
26103436
FK		 1080 /*
1081 * Eviction state logic: we only mark active queues as evicted
1082 * to avoid the overhead of restoring inactive queues later
1083 */
1084 if (qpd->evicted)
1085 q->properties.is_evicted = (q->properties.queue_size > 0 &&
1086 q->properties.queue_percent > 0 &&
1087 q->properties.queue_address != 0);
64c7f8cf 1088
bfd5e378 1089 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
373d7080
FK		 1090
1091 q->properties.tba_addr = qpd->tba_addr;
1092 q->properties.tma_addr = qpd->tma_addr;
64c7f8cf
BG		 1093 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
1094 &q->gart_mqd_addr, &q->properties);
4eacc26b 1095 if (retval)
64c7f8cf
BG		 1096 goto out;
1097
1098 list_add(&q->list, &qpd->queues_list);
bc920fd4 1099 qpd->queue_count++;
64c7f8cf
BG		 1100 if (q->properties.is_active) {
1101 dqm->queue_count++;
c4744e24
YZ		 1102 retval = execute_queues_cpsch(dqm,
1103 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
64c7f8cf
BG		 1104 }
1105
bcea3081 1106 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
8eabaf54 1107 dqm->sdma_queue_count++;
b8cbab04
OG		 1108 /*
1109 * Unconditionally increment this counter, regardless of the queue's
1110 * type or whether the queue is active.
1111 */
1112 dqm->total_queue_count++;
1113
1114 pr_debug("Total of %d queues are accountable so far\n",
1115 dqm->total_queue_count);
1116
64c7f8cf
BG		 1117out:
1118 mutex_unlock(&dqm->lock);
1119 return retval;
1120}
1121
788bf83d 1122int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
d80d19bd 1123 unsigned int fence_value,
8c72c3d7 1124 unsigned int timeout_ms)
64c7f8cf 1125{
8c72c3d7 1126 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
64c7f8cf
BG		 1127
1128 while (*fence_addr != fence_value) {
8c72c3d7 1129 if (time_after(jiffies, end_jiffies)) {
79775b62 1130 pr_err("qcm fence wait loop timeout expired\n");
64c7f8cf
BG		 1131 return -ETIME;
1132 }
99331a51 1133 schedule();
64c7f8cf
BG		 1134 }
1135
1136 return 0;
1137}
1138
7da2bcf8 1139static int unmap_sdma_queues(struct device_queue_manager *dqm,
bcea3081
BG		 1140 unsigned int sdma_engine)
1141{
1142 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
7da2bcf8 1143 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false,
bcea3081
BG		 1144 sdma_engine);
1145}
1146
60a00956
FK		 1147/* dqm->lock mutex has to be locked before calling this function */
1148static int map_queues_cpsch(struct device_queue_manager *dqm)
1149{
1150 int retval;
1151
1152 if (dqm->queue_count <= 0 || dqm->processes_count <= 0)
1153 return 0;
1154
1155 if (dqm->active_runlist)
1156 return 0;
1157
1158 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1159 if (retval) {
1160 pr_err("failed to execute runlist\n");
1161 return retval;
1162 }
1163 dqm->active_runlist = true;
1164
1165 return retval;
1166}
1167
ac30c783 1168/* dqm->lock mutex has to be locked before calling this function */
7da2bcf8 1169static int unmap_queues_cpsch(struct device_queue_manager *dqm,
4465f466
YZ		 1170 enum kfd_unmap_queues_filter filter,
1171 uint32_t filter_param)
64c7f8cf 1172{
9fd3f1bf 1173 int retval = 0;
64c7f8cf 1174
991ca8ee 1175 if (!dqm->active_runlist)
ac30c783 1176 return retval;
bcea3081 1177
79775b62 1178 pr_debug("Before destroying queues, sdma queue count is : %u\n",
bcea3081
BG		 1179 dqm->sdma_queue_count);
1180
1181 if (dqm->sdma_queue_count > 0) {
7da2bcf8
YZ		 1182 unmap_sdma_queues(dqm, 0);
1183 unmap_sdma_queues(dqm, 1);
bcea3081
BG		 1184 }
1185
64c7f8cf 1186 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
4465f466 1187 filter, filter_param, false, 0);
4eacc26b 1188 if (retval)
ac30c783 1189 return retval;
64c7f8cf
BG		 1190
1191 *dqm->fence_addr = KFD_FENCE_INIT;
1192 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
1193 KFD_FENCE_COMPLETED);
1194 /* should be timed out */
c3447e81 1195 retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
64c7f8cf 1196 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
9fd3f1bf 1197 if (retval)
ac30c783 1198 return retval;
9fd3f1bf 1199
64c7f8cf
BG		 1200 pm_release_ib(&dqm->packets);
1201 dqm->active_runlist = false;
1202
64c7f8cf
BG		 1203 return retval;
1204}
1205
ac30c783 1206/* dqm->lock mutex has to be locked before calling this function */
c4744e24
YZ		 1207static int execute_queues_cpsch(struct device_queue_manager *dqm,
1208 enum kfd_unmap_queues_filter filter,
1209 uint32_t filter_param)
64c7f8cf
BG		 1210{
1211 int retval;
1212
c4744e24 1213 retval = unmap_queues_cpsch(dqm, filter, filter_param);
4eacc26b 1214 if (retval) {
c4744e24 1215 pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
ac30c783 1216 return retval;
64c7f8cf
BG		 1217 }
1218
60a00956 1219 return map_queues_cpsch(dqm);
64c7f8cf
BG		 1220}
1221
1222static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1223 struct qcm_process_device *qpd,
1224 struct queue *q)
1225{
1226 int retval;
1227 struct mqd_manager *mqd;
992839ad 1228 bool preempt_all_queues;
64c7f8cf 1229
992839ad
YS		 1230 preempt_all_queues = false;
1231
64c7f8cf
BG		 1232 retval = 0;
1233
1234 /* remove queue from list to prevent rescheduling after preemption */
1235 mutex_lock(&dqm->lock);
992839ad
YS		 1236
1237 if (qpd->is_debug) {
1238 /*
1239 * error, currently we do not allow to destroy a queue
1240 * of a currently debugged process
1241 */
1242 retval = -EBUSY;
1243 goto failed_try_destroy_debugged_queue;
1244
1245 }
1246
45c9a5e4 1247 mqd = dqm->ops.get_mqd_manager(dqm,
bcea3081 1248 get_mqd_type_from_queue_type(q->properties.type));
64c7f8cf
BG		 1249 if (!mqd) {
1250 retval = -ENOMEM;
1251 goto failed;
1252 }
1253
e139cd2a 1254 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
bcea3081 1255 dqm->sdma_queue_count--;
e139cd2a 1256 deallocate_sdma_queue(dqm, q->sdma_id);
1257 }
bcea3081 1258
64c7f8cf 1259 list_del(&q->list);
bc920fd4 1260 qpd->queue_count--;
40a526dc 1261 if (q->properties.is_active) {
b6819cec 1262 dqm->queue_count--;
40a526dc 1263 retval = execute_queues_cpsch(dqm,
9fd3f1bf 1264 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
40a526dc
YZ		 1265 if (retval == -ETIME)
1266 qpd->reset_wavefronts = true;
1267 }
64c7f8cf
BG		 1268
1269 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
b8cbab04
OG		 1270
1271 /*
1272 * Unconditionally decrement this counter, regardless of the queue's
1273 * type
1274 */
1275 dqm->total_queue_count--;
1276 pr_debug("Total of %d queues are accountable so far\n",
1277 dqm->total_queue_count);
64c7f8cf
BG		 1278
1279 mutex_unlock(&dqm->lock);
1280
9e827224 1281 return retval;
64c7f8cf
BG		 1282
1283failed:
992839ad
YS		 1284failed_try_destroy_debugged_queue:
1285
64c7f8cf
BG		 1286 mutex_unlock(&dqm->lock);
1287 return retval;
1288}
1289
1290/*
1291 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1292 * stay in user mode.
1293 */
1294#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1295/* APE1 limit is inclusive and 64K aligned. */
1296#define APE1_LIMIT_ALIGNMENT 0xFFFF
1297
1298static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1299 struct qcm_process_device *qpd,
1300 enum cache_policy default_policy,
1301 enum cache_policy alternate_policy,
1302 void __user *alternate_aperture_base,
1303 uint64_t alternate_aperture_size)
1304{
a22fc854 1305 bool retval;
64c7f8cf 1306
64c7f8cf
BG		 1307 mutex_lock(&dqm->lock);
1308
1309 if (alternate_aperture_size == 0) {
1310 /* base > limit disables APE1 */
1311 qpd->sh_mem_ape1_base = 1;
1312 qpd->sh_mem_ape1_limit = 0;
1313 } else {
1314 /*
1315 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1316 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1317 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1318 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1319 * Verify that the base and size parameters can be
1320 * represented in this format and convert them.
1321 * Additionally restrict APE1 to user-mode addresses.
1322 */
1323
1324 uint64_t base = (uintptr_t)alternate_aperture_base;
1325 uint64_t limit = base + alternate_aperture_size - 1;
1326
ab7c1648
KR		 1327 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
1328 (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
1329 retval = false;
64c7f8cf 1330 goto out;
ab7c1648 1331 }
64c7f8cf
BG		 1332
1333 qpd->sh_mem_ape1_base = base >> 16;
1334 qpd->sh_mem_ape1_limit = limit >> 16;
1335 }
1336
bfd5e378 1337 retval = dqm->asic_ops.set_cache_memory_policy(
a22fc854
BG		 1338 dqm,
1339 qpd,
1340 default_policy,
1341 alternate_policy,
1342 alternate_aperture_base,
1343 alternate_aperture_size);
64c7f8cf 1344
d146c5a7 1345 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
64c7f8cf
BG		 1346 program_sh_mem_settings(dqm, qpd);
1347
79775b62 1348 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
64c7f8cf
BG		 1349 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1350 qpd->sh_mem_ape1_limit);
1351
64c7f8cf
BG		 1352out:
1353 mutex_unlock(&dqm->lock);
ab7c1648 1354 return retval;
64c7f8cf
BG		 1355}
1356
d7b9bd22
FK		 1357static int set_trap_handler(struct device_queue_manager *dqm,
1358 struct qcm_process_device *qpd,
1359 uint64_t tba_addr,
1360 uint64_t tma_addr)
1361{
1362 uint64_t *tma;
1363
1364 if (dqm->dev->cwsr_enabled) {
1365 /* Jump from CWSR trap handler to user trap */
1366 tma = (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1367 tma[0] = tba_addr;
1368 tma[1] = tma_addr;
1369 } else {
1370 qpd->tba_addr = tba_addr;
1371 qpd->tma_addr = tma_addr;
1372 }
1373
1374 return 0;
1375}
1376
9fd3f1bf
FK		 1377static int process_termination_nocpsch(struct device_queue_manager *dqm,
1378 struct qcm_process_device *qpd)
1379{
1380 struct queue *q, *next;
1381 struct device_process_node *cur, *next_dpn;
1382 int retval = 0;
1383
1384 mutex_lock(&dqm->lock);
1385
1386 /* Clear all user mode queues */
1387 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1388 int ret;
1389
1390 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
1391 if (ret)
1392 retval = ret;
1393 }
1394
1395 /* Unregister process */
1396 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1397 if (qpd == cur->qpd) {
1398 list_del(&cur->list);
1399 kfree(cur);
1400 dqm->processes_count--;
1401 break;
1402 }
1403 }
1404
1405 mutex_unlock(&dqm->lock);
1406 return retval;
1407}
1408
1409
1410static int process_termination_cpsch(struct device_queue_manager *dqm,
1411 struct qcm_process_device *qpd)
1412{
1413 int retval;
1414 struct queue *q, *next;
1415 struct kernel_queue *kq, *kq_next;
1416 struct mqd_manager *mqd;
1417 struct device_process_node *cur, *next_dpn;
1418 enum kfd_unmap_queues_filter filter =
1419 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
1420
1421 retval = 0;
1422
1423 mutex_lock(&dqm->lock);
1424
1425 /* Clean all kernel queues */
1426 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
1427 list_del(&kq->list);
1428 dqm->queue_count--;
1429 qpd->is_debug = false;
1430 dqm->total_queue_count--;
1431 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
1432 }
1433
1434 /* Clear all user mode queues */
1435 list_for_each_entry(q, &qpd->queues_list, list) {
1436 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1437 dqm->sdma_queue_count--;
1438
1439 if (q->properties.is_active)
1440 dqm->queue_count--;
1441
1442 dqm->total_queue_count--;
1443 }
1444
1445 /* Unregister process */
1446 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1447 if (qpd == cur->qpd) {
1448 list_del(&cur->list);
1449 kfree(cur);
1450 dqm->processes_count--;
1451 break;
1452 }
1453 }
1454
1455 retval = execute_queues_cpsch(dqm, filter, 0);
1456 if (retval || qpd->reset_wavefronts) {
1457 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
1458 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
1459 qpd->reset_wavefronts = false;
1460 }
1461
1462 /* lastly, free mqd resources */
1463 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1464 mqd = dqm->ops.get_mqd_manager(dqm,
1465 get_mqd_type_from_queue_type(q->properties.type));
1466 if (!mqd) {
1467 retval = -ENOMEM;
1468 goto out;
1469 }
1470 list_del(&q->list);
bc920fd4 1471 qpd->queue_count--;
9fd3f1bf
FK		 1472 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1473 }
1474
1475out:
1476 mutex_unlock(&dqm->lock);
1477 return retval;
1478}
1479
64c7f8cf
BG		 1480struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1481{
1482 struct device_queue_manager *dqm;
1483
79775b62 1484 pr_debug("Loading device queue manager\n");
a22fc854 1485
dbf56ab1 1486 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
64c7f8cf
BG		 1487 if (!dqm)
1488 return NULL;
1489
d146c5a7
FK		 1490 switch (dev->device_info->asic_family) {
1491 /* HWS is not available on Hawaii. */
1492 case CHIP_HAWAII:
1493 /* HWS depends on CWSR for timely dequeue. CWSR is not
1494 * available on Tonga.
1495 *
1496 * FIXME: This argument also applies to Kaveri.
1497 */
1498 case CHIP_TONGA:
1499 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
1500 break;
1501 default:
1502 dqm->sched_policy = sched_policy;
1503 break;
1504 }
1505
64c7f8cf 1506 dqm->dev = dev;
d146c5a7 1507 switch (dqm->sched_policy) {
64c7f8cf
BG		 1508 case KFD_SCHED_POLICY_HWS:
1509 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1510 /* initialize dqm for cp scheduling */
45c9a5e4
OG		 1511 dqm->ops.create_queue = create_queue_cpsch;
1512 dqm->ops.initialize = initialize_cpsch;
1513 dqm->ops.start = start_cpsch;
1514 dqm->ops.stop = stop_cpsch;
1515 dqm->ops.destroy_queue = destroy_queue_cpsch;
1516 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1517 dqm->ops.get_mqd_manager = get_mqd_manager;
1518 dqm->ops.register_process = register_process;
1519 dqm->ops.unregister_process = unregister_process;
1520 dqm->ops.uninitialize = uninitialize;
45c9a5e4
OG		 1521 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1522 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1523 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
d7b9bd22 1524 dqm->ops.set_trap_handler = set_trap_handler;
9fd3f1bf 1525 dqm->ops.process_termination = process_termination_cpsch;
26103436
FK		 1526 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
1527 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
64c7f8cf
BG		 1528 break;
1529 case KFD_SCHED_POLICY_NO_HWS:
1530 /* initialize dqm for no cp scheduling */
45c9a5e4
OG		 1531 dqm->ops.start = start_nocpsch;
1532 dqm->ops.stop = stop_nocpsch;
1533 dqm->ops.create_queue = create_queue_nocpsch;
1534 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1535 dqm->ops.update_queue = update_queue;
58dcd5bf
YZ		 1536 dqm->ops.get_mqd_manager = get_mqd_manager;
1537 dqm->ops.register_process = register_process;
1538 dqm->ops.unregister_process = unregister_process;
45c9a5e4 1539 dqm->ops.initialize = initialize_nocpsch;
58dcd5bf 1540 dqm->ops.uninitialize = uninitialize;
45c9a5e4 1541 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
d7b9bd22 1542 dqm->ops.set_trap_handler = set_trap_handler;
9fd3f1bf 1543 dqm->ops.process_termination = process_termination_nocpsch;
26103436
FK		 1544 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
1545 dqm->ops.restore_process_queues =
1546 restore_process_queues_nocpsch;
64c7f8cf
BG		 1547 break;
1548 default:
d146c5a7 1549 pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
32fa8219 1550 goto out_free;
64c7f8cf
BG		 1551 }
1552
a22fc854
BG		 1553 switch (dev->device_info->asic_family) {
1554 case CHIP_CARRIZO:
bfd5e378 1555 device_queue_manager_init_vi(&dqm->asic_ops);
300dec95
OG		 1556 break;
1557
a22fc854 1558 case CHIP_KAVERI:
bfd5e378 1559 device_queue_manager_init_cik(&dqm->asic_ops);
300dec95 1560 break;
97672cbe
FK		 1561
1562 case CHIP_HAWAII:
1563 device_queue_manager_init_cik_hawaii(&dqm->asic_ops);
1564 break;
1565
1566 case CHIP_TONGA:
1567 case CHIP_FIJI:
1568 case CHIP_POLARIS10:
1569 case CHIP_POLARIS11:
1570 device_queue_manager_init_vi_tonga(&dqm->asic_ops);
1571 break;
e596b903
YZ		 1572 default:
1573 WARN(1, "Unexpected ASIC family %u",
1574 dev->device_info->asic_family);
1575 goto out_free;
a22fc854
BG		 1576 }
1577
32fa8219
FK		 1578 if (!dqm->ops.initialize(dqm))
1579 return dqm;
64c7f8cf 1580
32fa8219
FK		 1581out_free:
1582 kfree(dqm);
1583 return NULL;
64c7f8cf
BG		 1584}
1585
1586void device_queue_manager_uninit(struct device_queue_manager *dqm)
1587{
45c9a5e4 1588 dqm->ops.uninitialize(dqm);
64c7f8cf
BG		 1589 kfree(dqm);
1590}
851a645e
FK		 1591
1592#if defined(CONFIG_DEBUG_FS)
1593
1594static void seq_reg_dump(struct seq_file *m,
1595 uint32_t (*dump)[2], uint32_t n_regs)
1596{
1597 uint32_t i, count;
1598
1599 for (i = 0, count = 0; i < n_regs; i++) {
1600 if (count == 0 ||
1601 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
1602 seq_printf(m, "%s %08x: %08x",
1603 i ? "\n" : "",
1604 dump[i][0], dump[i][1]);
1605 count = 7;
1606 } else {
1607 seq_printf(m, " %08x", dump[i][1]);
1608 count--;
1609 }
1610 }
1611
1612 seq_puts(m, "\n");
1613}
1614
1615int dqm_debugfs_hqds(struct seq_file *m, void *data)
1616{
1617 struct device_queue_manager *dqm = data;
1618 uint32_t (*dump)[2], n_regs;
1619 int pipe, queue;
1620 int r = 0;
1621
1622 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
1623 int pipe_offset = pipe * get_queues_per_pipe(dqm);
1624
1625 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
1626 if (!test_bit(pipe_offset + queue,
1627 dqm->dev->shared_resources.queue_bitmap))
1628 continue;
1629
1630 r = dqm->dev->kfd2kgd->hqd_dump(
1631 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
1632 if (r)
1633 break;
1634
1635 seq_printf(m, " CP Pipe %d, Queue %d\n",
1636 pipe, queue);
1637 seq_reg_dump(m, dump, n_regs);
1638
1639 kfree(dump);
1640 }
1641 }
1642
1643 for (pipe = 0; pipe < CIK_SDMA_ENGINE_NUM; pipe++) {
1644 for (queue = 0; queue < CIK_SDMA_QUEUES_PER_ENGINE; queue++) {
1645 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
1646 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
1647 if (r)
1648 break;
1649
1650 seq_printf(m, " SDMA Engine %d, RLC %d\n",
1651 pipe, queue);
1652 seq_reg_dump(m, dump, n_regs);
1653
1654 kfree(dump);
1655 }
1656 }
1657
1658 return r;
1659}
1660
1661#endif
Linux 6.x block layer and io_uring tree(s)