projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branches 'acpi-video', 'device-properties', 'pm-sleep' and 'pm-cpuidle'
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdkfd / kfd_device_queue_manager.c
CommitLineData
64c7f8cf
BG		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/slab.h>
25#include <linux/list.h>
26#include <linux/types.h>
27#include <linux/printk.h>
28#include <linux/bitops.h>
99331a51 29#include <linux/sched.h>
64c7f8cf
BG		 30#include "kfd_priv.h"
31#include "kfd_device_queue_manager.h"
32#include "kfd_mqd_manager.h"
33#include "cik_regs.h"
34#include "kfd_kernel_queue.h"
64c7f8cf
BG		 35
36/* Size of the per-pipe EOP queue */
37#define CIK_HPD_EOP_BYTES_LOG2 11
38#define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
39
64c7f8cf
BG		 40static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
41 unsigned int pasid, unsigned int vmid);
42
43static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
44 struct queue *q,
45 struct qcm_process_device *qpd);
bcea3081 46
64c7f8cf
BG		 47static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
48static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock);
49
bcea3081
BG		 50static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
51 struct queue *q,
52 struct qcm_process_device *qpd);
53
54static void deallocate_sdma_queue(struct device_queue_manager *dqm,
55 unsigned int sdma_queue_id);
64c7f8cf 56
bcea3081
BG		 57static inline
58enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
64c7f8cf 59{
bcea3081 60 if (type == KFD_QUEUE_TYPE_SDMA)
85d258f9
BG		 61 return KFD_MQD_TYPE_SDMA;
62 return KFD_MQD_TYPE_CP;
64c7f8cf
BG		 63}
64
1365aa62 65unsigned int get_first_pipe(struct device_queue_manager *dqm)
64c7f8cf 66{
1365aa62 67 BUG_ON(!dqm || !dqm->dev);
64c7f8cf
BG		 68 return dqm->dev->shared_resources.first_compute_pipe;
69}
70
64ea8f4a
OG		 71unsigned int get_pipes_num(struct device_queue_manager *dqm)
72{
73 BUG_ON(!dqm || !dqm->dev);
74 return dqm->dev->shared_resources.compute_pipe_count;
75}
76
64c7f8cf
BG		 77static inline unsigned int get_pipes_num_cpsch(void)
78{
79 return PIPE_PER_ME_CP_SCHEDULING;
80}
81
a22fc854 82void program_sh_mem_settings(struct device_queue_manager *dqm,
64c7f8cf
BG		 83 struct qcm_process_device *qpd)
84{
cea405b1
XZ		 85 return dqm->dev->kfd2kgd->program_sh_mem_settings(
86 dqm->dev->kgd, qpd->vmid,
64c7f8cf
BG		 87 qpd->sh_mem_config,
88 qpd->sh_mem_ape1_base,
89 qpd->sh_mem_ape1_limit,
90 qpd->sh_mem_bases);
91}
92
93static int allocate_vmid(struct device_queue_manager *dqm,
94 struct qcm_process_device *qpd,
95 struct queue *q)
96{
97 int bit, allocated_vmid;
98
99 if (dqm->vmid_bitmap == 0)
100 return -ENOMEM;
101
102 bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
103 clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
104
105 /* Kaveri kfd vmid's starts from vmid 8 */
106 allocated_vmid = bit + KFD_VMID_START_OFFSET;
107 pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
108 qpd->vmid = allocated_vmid;
109 q->properties.vmid = allocated_vmid;
110
111 set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
112 program_sh_mem_settings(dqm, qpd);
113
114 return 0;
115}
116
117static void deallocate_vmid(struct device_queue_manager *dqm,
118 struct qcm_process_device *qpd,
119 struct queue *q)
120{
121 int bit = qpd->vmid - KFD_VMID_START_OFFSET;
122
2030664b
BG		 123 /* Release the vmid mapping */
124 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
125
64c7f8cf
BG		 126 set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
127 qpd->vmid = 0;
128 q->properties.vmid = 0;
129}
130
131static int create_queue_nocpsch(struct device_queue_manager *dqm,
132 struct queue *q,
133 struct qcm_process_device *qpd,
134 int *allocated_vmid)
135{
136 int retval;
137
138 BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
139
140 pr_debug("kfd: In func %s\n", __func__);
141 print_queue(q);
142
143 mutex_lock(&dqm->lock);
144
b8cbab04
OG		 145 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
146 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
147 dqm->total_queue_count);
148 mutex_unlock(&dqm->lock);
149 return -EPERM;
150 }
151
64c7f8cf
BG		 152 if (list_empty(&qpd->queues_list)) {
153 retval = allocate_vmid(dqm, qpd, q);
154 if (retval != 0) {
155 mutex_unlock(&dqm->lock);
156 return retval;
157 }
158 }
159 *allocated_vmid = qpd->vmid;
160 q->properties.vmid = qpd->vmid;
161
bcea3081
BG		 162 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
163 retval = create_compute_queue_nocpsch(dqm, q, qpd);
164 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
165 retval = create_sdma_queue_nocpsch(dqm, q, qpd);
64c7f8cf
BG		 166
167 if (retval != 0) {
168 if (list_empty(&qpd->queues_list)) {
169 deallocate_vmid(dqm, qpd, q);
170 *allocated_vmid = 0;
171 }
172 mutex_unlock(&dqm->lock);
173 return retval;
174 }
175
176 list_add(&q->list, &qpd->queues_list);
b6819cec
JC		 177 if (q->properties.is_active)
178 dqm->queue_count++;
64c7f8cf 179
bcea3081
BG		 180 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
181 dqm->sdma_queue_count++;
64c7f8cf 182
b8cbab04
OG		 183 /*
184 * Unconditionally increment this counter, regardless of the queue's
185 * type or whether the queue is active.
186 */
187 dqm->total_queue_count++;
188 pr_debug("Total of %d queues are accountable so far\n",
189 dqm->total_queue_count);
190
64c7f8cf
BG		 191 mutex_unlock(&dqm->lock);
192 return 0;
193}
194
195static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
196{
197 bool set;
f0ec5b99 198 int pipe, bit, i;
64c7f8cf
BG		 199
200 set = false;
201
f0ec5b99
BG		 202 for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm);
203 pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) {
64c7f8cf
BG		 204 if (dqm->allocated_queues[pipe] != 0) {
205 bit = find_first_bit(
206 (unsigned long *)&dqm->allocated_queues[pipe],
207 QUEUES_PER_PIPE);
208
209 clear_bit(bit,
210 (unsigned long *)&dqm->allocated_queues[pipe]);
211 q->pipe = pipe;
212 q->queue = bit;
213 set = true;
214 break;
215 }
216 }
217
218 if (set == false)
219 return -EBUSY;
220
221 pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
222 __func__, q->pipe, q->queue);
223 /* horizontal hqd allocation */
224 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm);
225
226 return 0;
227}
228
229static inline void deallocate_hqd(struct device_queue_manager *dqm,
230 struct queue *q)
231{
232 set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
233}
234
235static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
236 struct queue *q,
237 struct qcm_process_device *qpd)
238{
239 int retval;
240 struct mqd_manager *mqd;
241
242 BUG_ON(!dqm || !q || !qpd);
243
45c9a5e4 244 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
64c7f8cf
BG		 245 if (mqd == NULL)
246 return -ENOMEM;
247
248 retval = allocate_hqd(dqm, q);
249 if (retval != 0)
250 return retval;
251
252 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
253 &q->gart_mqd_addr, &q->properties);
254 if (retval != 0) {
255 deallocate_hqd(dqm, q);
256 return retval;
257 }
258
030e416b
BG		 259 pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",
260 q->pipe,
261 q->queue);
262
263 retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
8dfe58b2 264 q->queue, (uint32_t __user *) q->properties.write_ptr);
030e416b
BG		 265 if (retval != 0) {
266 deallocate_hqd(dqm, q);
267 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
268 return retval;
269 }
270
64c7f8cf
BG		 271 return 0;
272}
273
274static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
275 struct qcm_process_device *qpd,
276 struct queue *q)
277{
278 int retval;
279 struct mqd_manager *mqd;
280
281 BUG_ON(!dqm || !q || !q->mqd || !qpd);
282
283 retval = 0;
284
285 pr_debug("kfd: In Func %s\n", __func__);
286
287 mutex_lock(&dqm->lock);
64c7f8cf 288
c2e1b3a4 289 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
45c9a5e4 290 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
c2e1b3a4
BG		 291 if (mqd == NULL) {
292 retval = -ENOMEM;
293 goto out;
294 }
295 deallocate_hqd(dqm, q);
296 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
45c9a5e4 297 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
c2e1b3a4
BG		 298 if (mqd == NULL) {
299 retval = -ENOMEM;
300 goto out;
301 }
302 dqm->sdma_queue_count--;
303 deallocate_sdma_queue(dqm, q->sdma_id);
7113cd65
OG		 304 } else {
305 pr_debug("q->properties.type is invalid (%d)\n",
306 q->properties.type);
307 retval = -EINVAL;
64c7f8cf
BG		 308 goto out;
309 }
310
311 retval = mqd->destroy_mqd(mqd, q->mqd,
c2e1b3a4 312 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
64c7f8cf
BG		 313 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
314 q->pipe, q->queue);
315
316 if (retval != 0)
317 goto out;
318
64c7f8cf
BG		 319 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
320
321 list_del(&q->list);
322 if (list_empty(&qpd->queues_list))
323 deallocate_vmid(dqm, qpd, q);
b6819cec
JC		 324 if (q->properties.is_active)
325 dqm->queue_count--;
b8cbab04
OG		 326
327 /*
328 * Unconditionally decrement this counter, regardless of the queue's
329 * type
330 */
331 dqm->total_queue_count--;
332 pr_debug("Total of %d queues are accountable so far\n",
333 dqm->total_queue_count);
334
64c7f8cf
BG		 335out:
336 mutex_unlock(&dqm->lock);
337 return retval;
338}
339
340static int update_queue(struct device_queue_manager *dqm, struct queue *q)
341{
342 int retval;
343 struct mqd_manager *mqd;
b6ffbab8 344 bool prev_active = false;
64c7f8cf
BG		 345
346 BUG_ON(!dqm || !q || !q->mqd);
347
348 mutex_lock(&dqm->lock);
0b3674ae
OG		 349 mqd = dqm->ops.get_mqd_manager(dqm,
350 get_mqd_type_from_queue_type(q->properties.type));
64c7f8cf
BG		 351 if (mqd == NULL) {
352 mutex_unlock(&dqm->lock);
353 return -ENOMEM;
354 }
355
64c7f8cf 356 if (q->properties.is_active == true)
b6ffbab8
OG		 357 prev_active = true;
358
359 /*
360 *
361 * check active state vs. the previous state
362 * and modify counter accordingly
363 */
364 retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
365 if ((q->properties.is_active == true) && (prev_active == false))
64c7f8cf 366 dqm->queue_count++;
b6ffbab8 367 else if ((q->properties.is_active == false) && (prev_active == true))
64c7f8cf
BG		 368 dqm->queue_count--;
369
370 if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
371 retval = execute_queues_cpsch(dqm, false);
372
373 mutex_unlock(&dqm->lock);
374 return retval;
375}
376
377static struct mqd_manager *get_mqd_manager_nocpsch(
378 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
379{
380 struct mqd_manager *mqd;
381
382 BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
383
384 pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
385
386 mqd = dqm->mqds[type];
387 if (!mqd) {
388 mqd = mqd_manager_init(type, dqm->dev);
389 if (mqd == NULL)
390 pr_err("kfd: mqd manager is NULL");
391 dqm->mqds[type] = mqd;
392 }
393
394 return mqd;
395}
396
397static int register_process_nocpsch(struct device_queue_manager *dqm,
398 struct qcm_process_device *qpd)
399{
400 struct device_process_node *n;
a22fc854 401 int retval;
64c7f8cf
BG		 402
403 BUG_ON(!dqm || !qpd);
404
405 pr_debug("kfd: In func %s\n", __func__);
406
407 n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
408 if (!n)
409 return -ENOMEM;
410
411 n->qpd = qpd;
412
413 mutex_lock(&dqm->lock);
414 list_add(&n->list, &dqm->queues);
415
a22fc854
BG		 416 retval = dqm->ops_asic_specific.register_process(dqm, qpd);
417
64c7f8cf
BG		 418 dqm->processes_count++;
419
420 mutex_unlock(&dqm->lock);
421
a22fc854 422 return retval;
64c7f8cf
BG		 423}
424
425static int unregister_process_nocpsch(struct device_queue_manager *dqm,
426 struct qcm_process_device *qpd)
427{
428 int retval;
429 struct device_process_node *cur, *next;
430
431 BUG_ON(!dqm || !qpd);
432
1e5ec956 433 pr_debug("In func %s\n", __func__);
64c7f8cf 434
1e5ec956
OG		 435 pr_debug("qpd->queues_list is %s\n",
436 list_empty(&qpd->queues_list) ? "empty" : "not empty");
64c7f8cf
BG		 437
438 retval = 0;
439 mutex_lock(&dqm->lock);
440
441 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
442 if (qpd == cur->qpd) {
443 list_del(&cur->list);
f5d896bb 444 kfree(cur);
64c7f8cf
BG		 445 dqm->processes_count--;
446 goto out;
447 }
448 }
449 /* qpd not found in dqm list */
450 retval = 1;
451out:
452 mutex_unlock(&dqm->lock);
453 return retval;
454}
455
456static int
457set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
458 unsigned int vmid)
459{
460 uint32_t pasid_mapping;
461
cea405b1
XZ		 462 pasid_mapping = (pasid == 0) ? 0 :
463 (uint32_t)pasid |
464 ATC_VMID_PASID_MAPPING_VALID;
465
466 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
467 dqm->dev->kgd, pasid_mapping,
64c7f8cf
BG		 468 vmid);
469}
470
a22fc854 471int init_pipelines(struct device_queue_manager *dqm,
64c7f8cf
BG		 472 unsigned int pipes_num, unsigned int first_pipe)
473{
474 void *hpdptr;
475 struct mqd_manager *mqd;
476 unsigned int i, err, inx;
477 uint64_t pipe_hpd_addr;
478
479 BUG_ON(!dqm || !dqm->dev);
480
481 pr_debug("kfd: In func %s\n", __func__);
482
483 /*
484 * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
485 * The driver never accesses this memory after zeroing it.
486 * It doesn't even have to be saved/restored on suspend/resume
487 * because it contains no data when there are no active queues.
488 */
489
a86aa3ca
OG		 490 err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num,
491 &dqm->pipeline_mem);
64c7f8cf
BG		 492
493 if (err) {
494 pr_err("kfd: error allocate vidmem num pipes: %d\n",
495 pipes_num);
496 return -ENOMEM;
497 }
498
499 hpdptr = dqm->pipeline_mem->cpu_ptr;
500 dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr;
501
502 memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num);
503
45c9a5e4 504 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
64c7f8cf 505 if (mqd == NULL) {
a86aa3ca 506 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
64c7f8cf
BG		 507 return -ENOMEM;
508 }
509
510 for (i = 0; i < pipes_num; i++) {
511 inx = i + first_pipe;
749042b0
OG		 512 /*
513 * HPD buffer on GTT is allocated by amdkfd, no need to waste
514 * space in GTT for pipelines we don't initialize
515 */
64c7f8cf
BG		 516 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES;
517 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr);
518 /* = log2(bytes/4)-1 */
cea405b1 519 dqm->dev->kfd2kgd->init_pipeline(dqm->dev->kgd, inx,
64c7f8cf
BG		 520 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr);
521 }
522
523 return 0;
524}
525
64c7f8cf
BG		 526static int init_scheduler(struct device_queue_manager *dqm)
527{
528 int retval;
529
530 BUG_ON(!dqm);
531
532 pr_debug("kfd: In %s\n", __func__);
533
9fa843e7 534 retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm));
64c7f8cf
BG		 535 return retval;
536}
537
538static int initialize_nocpsch(struct device_queue_manager *dqm)
539{
540 int i;
541
542 BUG_ON(!dqm);
543
544 pr_debug("kfd: In func %s num of pipes: %d\n",
545 __func__, get_pipes_num(dqm));
546
547 mutex_init(&dqm->lock);
548 INIT_LIST_HEAD(&dqm->queues);
549 dqm->queue_count = dqm->next_pipe_to_allocate = 0;
bcea3081 550 dqm->sdma_queue_count = 0;
64c7f8cf
BG		 551 dqm->allocated_queues = kcalloc(get_pipes_num(dqm),
552 sizeof(unsigned int), GFP_KERNEL);
553 if (!dqm->allocated_queues) {
554 mutex_destroy(&dqm->lock);
555 return -ENOMEM;
556 }
557
558 for (i = 0; i < get_pipes_num(dqm); i++)
559 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1;
560
561 dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
bcea3081 562 dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
64c7f8cf
BG		 563
564 init_scheduler(dqm);
565 return 0;
566}
567
568static void uninitialize_nocpsch(struct device_queue_manager *dqm)
569{
6f9d54fd
OG		 570 int i;
571
64c7f8cf
BG		 572 BUG_ON(!dqm);
573
574 BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
575
576 kfree(dqm->allocated_queues);
6f9d54fd
OG		 577 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
578 kfree(dqm->mqds[i]);
64c7f8cf 579 mutex_destroy(&dqm->lock);
a86aa3ca 580 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
64c7f8cf
BG		 581}
582
583static int start_nocpsch(struct device_queue_manager *dqm)
584{
585 return 0;
586}
587
588static int stop_nocpsch(struct device_queue_manager *dqm)
589{
590 return 0;
591}
592
bcea3081
BG		 593static int allocate_sdma_queue(struct device_queue_manager *dqm,
594 unsigned int *sdma_queue_id)
595{
596 int bit;
597
598 if (dqm->sdma_bitmap == 0)
599 return -ENOMEM;
600
601 bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap,
602 CIK_SDMA_QUEUES);
603
604 clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap);
605 *sdma_queue_id = bit;
606
607 return 0;
608}
609
610static void deallocate_sdma_queue(struct device_queue_manager *dqm,
611 unsigned int sdma_queue_id)
612{
010b82e7 613 if (sdma_queue_id >= CIK_SDMA_QUEUES)
bcea3081
BG		 614 return;
615 set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap);
616}
617
618static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
619 struct qcm_process_device *qpd)
620{
621 uint32_t value = SDMA_ATC;
622
623 if (q->process->is_32bit_user_mode)
624 value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd));
625 else
626 value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64(
627 qpd_to_pdd(qpd)));
628 q->properties.sdma_vm_addr = value;
629}
630
631static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
632 struct queue *q,
633 struct qcm_process_device *qpd)
634{
635 struct mqd_manager *mqd;
636 int retval;
637
45c9a5e4 638 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
bcea3081
BG		 639 if (!mqd)
640 return -ENOMEM;
641
642 retval = allocate_sdma_queue(dqm, &q->sdma_id);
643 if (retval != 0)
644 return retval;
645
646 q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
647 q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
648
649 pr_debug("kfd: sdma id is: %d\n", q->sdma_id);
650 pr_debug(" sdma queue id: %d\n", q->properties.sdma_queue_id);
651 pr_debug(" sdma engine id: %d\n", q->properties.sdma_engine_id);
652
4fadf6b6 653 init_sdma_vm(dqm, q, qpd);
bcea3081
BG		 654 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
655 &q->gart_mqd_addr, &q->properties);
656 if (retval != 0) {
657 deallocate_sdma_queue(dqm, q->sdma_id);
658 return retval;
659 }
660
4fadf6b6
BG		 661 retval = mqd->load_mqd(mqd, q->mqd, 0,
662 0, NULL);
663 if (retval != 0) {
664 deallocate_sdma_queue(dqm, q->sdma_id);
665 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
666 return retval;
667 }
668
bcea3081
BG		 669 return 0;
670}
671
64c7f8cf
BG		 672/*
673 * Device Queue Manager implementation for cp scheduler
674 */
675
676static int set_sched_resources(struct device_queue_manager *dqm)
677{
678 struct scheduling_resources res;
679 unsigned int queue_num, queue_mask;
680
681 BUG_ON(!dqm);
682
683 pr_debug("kfd: In func %s\n", __func__);
684
685 queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE;
686 queue_mask = (1 << queue_num) - 1;
687 res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
688 res.vmid_mask <<= KFD_VMID_START_OFFSET;
689 res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE);
690 res.gws_mask = res.oac_mask = res.gds_heap_base =
691 res.gds_heap_size = 0;
692
693 pr_debug("kfd: scheduling resources:\n"
694 " vmid mask: 0x%8X\n"
695 " queue mask: 0x%8llX\n",
696 res.vmid_mask, res.queue_mask);
697
698 return pm_send_set_resources(&dqm->packets, &res);
699}
700
701static int initialize_cpsch(struct device_queue_manager *dqm)
702{
703 int retval;
704
705 BUG_ON(!dqm);
706
707 pr_debug("kfd: In func %s num of pipes: %d\n",
708 __func__, get_pipes_num_cpsch());
709
710 mutex_init(&dqm->lock);
711 INIT_LIST_HEAD(&dqm->queues);
712 dqm->queue_count = dqm->processes_count = 0;
bcea3081 713 dqm->sdma_queue_count = 0;
64c7f8cf 714 dqm->active_runlist = false;
a22fc854 715 retval = dqm->ops_asic_specific.initialize(dqm);
64c7f8cf
BG		 716 if (retval != 0)
717 goto fail_init_pipelines;
718
719 return 0;
720
721fail_init_pipelines:
722 mutex_destroy(&dqm->lock);
723 return retval;
724}
725
726static int start_cpsch(struct device_queue_manager *dqm)
727{
728 struct device_process_node *node;
729 int retval;
730
731 BUG_ON(!dqm);
732
733 retval = 0;
734
735 retval = pm_init(&dqm->packets, dqm);
736 if (retval != 0)
737 goto fail_packet_manager_init;
738
739 retval = set_sched_resources(dqm);
740 if (retval != 0)
741 goto fail_set_sched_resources;
742
743 pr_debug("kfd: allocating fence memory\n");
744
745 /* allocate fence memory on the gart */
a86aa3ca
OG		 746 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
747 &dqm->fence_mem);
64c7f8cf
BG		 748
749 if (retval != 0)
750 goto fail_allocate_vidmem;
751
752 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
753 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
64c7f8cf
BG		 754 list_for_each_entry(node, &dqm->queues, list)
755 if (node->qpd->pqm->process && dqm->dev)
756 kfd_bind_process_to_device(dqm->dev,
757 node->qpd->pqm->process);
758
759 execute_queues_cpsch(dqm, true);
760
761 return 0;
762fail_allocate_vidmem:
763fail_set_sched_resources:
764 pm_uninit(&dqm->packets);
765fail_packet_manager_init:
766 return retval;
767}
768
769static int stop_cpsch(struct device_queue_manager *dqm)
770{
771 struct device_process_node *node;
772 struct kfd_process_device *pdd;
773
774 BUG_ON(!dqm);
775
776 destroy_queues_cpsch(dqm, true);
777
778 list_for_each_entry(node, &dqm->queues, list) {
52a5fdce 779 pdd = qpd_to_pdd(node->qpd);
64c7f8cf
BG		 780 pdd->bound = false;
781 }
a86aa3ca 782 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
64c7f8cf
BG		 783 pm_uninit(&dqm->packets);
784
785 return 0;
786}
787
788static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
789 struct kernel_queue *kq,
790 struct qcm_process_device *qpd)
791{
792 BUG_ON(!dqm || !kq || !qpd);
793
794 pr_debug("kfd: In func %s\n", __func__);
795
796 mutex_lock(&dqm->lock);
b8cbab04
OG		 797 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
798 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n",
799 dqm->total_queue_count);
800 mutex_unlock(&dqm->lock);
801 return -EPERM;
802 }
803
804 /*
805 * Unconditionally increment this counter, regardless of the queue's
806 * type or whether the queue is active.
807 */
808 dqm->total_queue_count++;
809 pr_debug("Total of %d queues are accountable so far\n",
810 dqm->total_queue_count);
811
64c7f8cf
BG		 812 list_add(&kq->list, &qpd->priv_queue_list);
813 dqm->queue_count++;
814 qpd->is_debug = true;
815 execute_queues_cpsch(dqm, false);
816 mutex_unlock(&dqm->lock);
817
818 return 0;
819}
820
821static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
822 struct kernel_queue *kq,
823 struct qcm_process_device *qpd)
824{
825 BUG_ON(!dqm || !kq);
826
827 pr_debug("kfd: In %s\n", __func__);
828
829 mutex_lock(&dqm->lock);
830 destroy_queues_cpsch(dqm, false);
831 list_del(&kq->list);
832 dqm->queue_count--;
833 qpd->is_debug = false;
834 execute_queues_cpsch(dqm, false);
b8cbab04
OG		 835 /*
836 * Unconditionally decrement this counter, regardless of the queue's
837 * type.
838 */
8b58f261 839 dqm->total_queue_count--;
b8cbab04
OG		 840 pr_debug("Total of %d queues are accountable so far\n",
841 dqm->total_queue_count);
64c7f8cf
BG		 842 mutex_unlock(&dqm->lock);
843}
844
bcea3081
BG		 845static void select_sdma_engine_id(struct queue *q)
846{
847 static int sdma_id;
848
849 q->sdma_id = sdma_id;
850 sdma_id = (sdma_id + 1) % 2;
851}
852
64c7f8cf
BG		 853static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
854 struct qcm_process_device *qpd, int *allocate_vmid)
855{
856 int retval;
857 struct mqd_manager *mqd;
858
859 BUG_ON(!dqm || !q || !qpd);
860
861 retval = 0;
862
863 if (allocate_vmid)
864 *allocate_vmid = 0;
865
866 mutex_lock(&dqm->lock);
867
b8cbab04
OG		 868 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
869 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
870 dqm->total_queue_count);
871 retval = -EPERM;
872 goto out;
873 }
874
bcea3081
BG		 875 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
876 select_sdma_engine_id(q);
877
45c9a5e4 878 mqd = dqm->ops.get_mqd_manager(dqm,
bcea3081
BG		 879 get_mqd_type_from_queue_type(q->properties.type));
880
64c7f8cf
BG		 881 if (mqd == NULL) {
882 mutex_unlock(&dqm->lock);
883 return -ENOMEM;
884 }
885
79b066bd
XZ		 886 init_sdma_vm(dqm, q, qpd);
887
64c7f8cf
BG		 888 retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
889 &q->gart_mqd_addr, &q->properties);
890 if (retval != 0)
891 goto out;
892
893 list_add(&q->list, &qpd->queues_list);
894 if (q->properties.is_active) {
895 dqm->queue_count++;
896 retval = execute_queues_cpsch(dqm, false);
897 }
898
bcea3081
BG		 899 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
900 dqm->sdma_queue_count++;
b8cbab04
OG		 901 /*
902 * Unconditionally increment this counter, regardless of the queue's
903 * type or whether the queue is active.
904 */
905 dqm->total_queue_count++;
906
907 pr_debug("Total of %d queues are accountable so far\n",
908 dqm->total_queue_count);
909
64c7f8cf
BG		 910out:
911 mutex_unlock(&dqm->lock);
912 return retval;
913}
914
a9243ede 915static int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
d80d19bd
OG		 916 unsigned int fence_value,
917 unsigned long timeout)
64c7f8cf
BG		 918{
919 BUG_ON(!fence_addr);
920 timeout += jiffies;
921
922 while (*fence_addr != fence_value) {
923 if (time_after(jiffies, timeout)) {
924 pr_err("kfd: qcm fence wait loop timeout expired\n");
925 return -ETIME;
926 }
99331a51 927 schedule();
64c7f8cf
BG		 928 }
929
930 return 0;
931}
932
bcea3081
BG		 933static int destroy_sdma_queues(struct device_queue_manager *dqm,
934 unsigned int sdma_engine)
935{
936 return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
937 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false,
938 sdma_engine);
939}
940
64c7f8cf
BG		 941static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock)
942{
943 int retval;
944
945 BUG_ON(!dqm);
946
947 retval = 0;
948
949 if (lock)
950 mutex_lock(&dqm->lock);
951 if (dqm->active_runlist == false)
952 goto out;
bcea3081
BG		 953
954 pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n",
955 dqm->sdma_queue_count);
956
957 if (dqm->sdma_queue_count > 0) {
958 destroy_sdma_queues(dqm, 0);
959 destroy_sdma_queues(dqm, 1);
960 }
961
64c7f8cf
BG		 962 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
963 KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0);
964 if (retval != 0)
965 goto out;
966
967 *dqm->fence_addr = KFD_FENCE_INIT;
968 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
969 KFD_FENCE_COMPLETED);
970 /* should be timed out */
a9243ede 971 amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
64c7f8cf
BG		 972 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
973 pm_release_ib(&dqm->packets);
974 dqm->active_runlist = false;
975
976out:
977 if (lock)
978 mutex_unlock(&dqm->lock);
979 return retval;
980}
981
982static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
983{
984 int retval;
985
986 BUG_ON(!dqm);
987
988 if (lock)
989 mutex_lock(&dqm->lock);
990
991 retval = destroy_queues_cpsch(dqm, false);
992 if (retval != 0) {
993 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
994 goto out;
995 }
996
997 if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
998 retval = 0;
999 goto out;
1000 }
1001
1002 if (dqm->active_runlist) {
1003 retval = 0;
1004 goto out;
1005 }
1006
1007 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1008 if (retval != 0) {
1009 pr_err("kfd: failed to execute runlist");
1010 goto out;
1011 }
1012 dqm->active_runlist = true;
1013
1014out:
1015 if (lock)
1016 mutex_unlock(&dqm->lock);
1017 return retval;
1018}
1019
1020static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1021 struct qcm_process_device *qpd,
1022 struct queue *q)
1023{
1024 int retval;
1025 struct mqd_manager *mqd;
1026
1027 BUG_ON(!dqm || !qpd || !q);
1028
1029 retval = 0;
1030
1031 /* remove queue from list to prevent rescheduling after preemption */
1032 mutex_lock(&dqm->lock);
45c9a5e4 1033 mqd = dqm->ops.get_mqd_manager(dqm,
bcea3081 1034 get_mqd_type_from_queue_type(q->properties.type));
64c7f8cf
BG		 1035 if (!mqd) {
1036 retval = -ENOMEM;
1037 goto failed;
1038 }
1039
bcea3081
BG		 1040 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1041 dqm->sdma_queue_count--;
1042
64c7f8cf 1043 list_del(&q->list);
b6819cec
JC		 1044 if (q->properties.is_active)
1045 dqm->queue_count--;
64c7f8cf
BG		 1046
1047 execute_queues_cpsch(dqm, false);
1048
1049 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
b8cbab04
OG		 1050
1051 /*
1052 * Unconditionally decrement this counter, regardless of the queue's
1053 * type
1054 */
1055 dqm->total_queue_count--;
1056 pr_debug("Total of %d queues are accountable so far\n",
1057 dqm->total_queue_count);
64c7f8cf
BG		 1058
1059 mutex_unlock(&dqm->lock);
1060
1061 return 0;
1062
1063failed:
1064 mutex_unlock(&dqm->lock);
1065 return retval;
1066}
1067
1068/*
1069 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1070 * stay in user mode.
1071 */
1072#define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1073/* APE1 limit is inclusive and 64K aligned. */
1074#define APE1_LIMIT_ALIGNMENT 0xFFFF
1075
1076static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1077 struct qcm_process_device *qpd,
1078 enum cache_policy default_policy,
1079 enum cache_policy alternate_policy,
1080 void __user *alternate_aperture_base,
1081 uint64_t alternate_aperture_size)
1082{
a22fc854 1083 bool retval;
64c7f8cf
BG		 1084
1085 pr_debug("kfd: In func %s\n", __func__);
1086
1087 mutex_lock(&dqm->lock);
1088
1089 if (alternate_aperture_size == 0) {
1090 /* base > limit disables APE1 */
1091 qpd->sh_mem_ape1_base = 1;
1092 qpd->sh_mem_ape1_limit = 0;
1093 } else {
1094 /*
1095 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1096 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1097 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1098 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1099 * Verify that the base and size parameters can be
1100 * represented in this format and convert them.
1101 * Additionally restrict APE1 to user-mode addresses.
1102 */
1103
1104 uint64_t base = (uintptr_t)alternate_aperture_base;
1105 uint64_t limit = base + alternate_aperture_size - 1;
1106
1107 if (limit <= base)
1108 goto out;
1109
1110 if ((base & APE1_FIXED_BITS_MASK) != 0)
1111 goto out;
1112
1113 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
1114 goto out;
1115
1116 qpd->sh_mem_ape1_base = base >> 16;
1117 qpd->sh_mem_ape1_limit = limit >> 16;
1118 }
1119
a22fc854
BG		 1120 retval = dqm->ops_asic_specific.set_cache_memory_policy(
1121 dqm,
1122 qpd,
1123 default_policy,
1124 alternate_policy,
1125 alternate_aperture_base,
1126 alternate_aperture_size);
64c7f8cf
BG		 1127
1128 if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
1129 program_sh_mem_settings(dqm, qpd);
1130
1131 pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
1132 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1133 qpd->sh_mem_ape1_limit);
1134
1135 mutex_unlock(&dqm->lock);
a22fc854 1136 return retval;
64c7f8cf
BG		 1137
1138out:
1139 mutex_unlock(&dqm->lock);
1140 return false;
1141}
1142
1143struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1144{
1145 struct device_queue_manager *dqm;
1146
1147 BUG_ON(!dev);
1148
a22fc854
BG		 1149 pr_debug("kfd: loading device queue manager\n");
1150
64c7f8cf
BG		 1151 dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
1152 if (!dqm)
1153 return NULL;
1154
1155 dqm->dev = dev;
1156 switch (sched_policy) {
1157 case KFD_SCHED_POLICY_HWS:
1158 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1159 /* initialize dqm for cp scheduling */
45c9a5e4
OG		 1160 dqm->ops.create_queue = create_queue_cpsch;
1161 dqm->ops.initialize = initialize_cpsch;
1162 dqm->ops.start = start_cpsch;
1163 dqm->ops.stop = stop_cpsch;
1164 dqm->ops.destroy_queue = destroy_queue_cpsch;
1165 dqm->ops.update_queue = update_queue;
1166 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1167 dqm->ops.register_process = register_process_nocpsch;
1168 dqm->ops.unregister_process = unregister_process_nocpsch;
1169 dqm->ops.uninitialize = uninitialize_nocpsch;
1170 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1171 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1172 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
64c7f8cf
BG		 1173 break;
1174 case KFD_SCHED_POLICY_NO_HWS:
1175 /* initialize dqm for no cp scheduling */
45c9a5e4
OG		 1176 dqm->ops.start = start_nocpsch;
1177 dqm->ops.stop = stop_nocpsch;
1178 dqm->ops.create_queue = create_queue_nocpsch;
1179 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1180 dqm->ops.update_queue = update_queue;
1181 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1182 dqm->ops.register_process = register_process_nocpsch;
1183 dqm->ops.unregister_process = unregister_process_nocpsch;
1184 dqm->ops.initialize = initialize_nocpsch;
1185 dqm->ops.uninitialize = uninitialize_nocpsch;
1186 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
64c7f8cf
BG		 1187 break;
1188 default:
1189 BUG();
1190 break;
1191 }
1192
a22fc854
BG		 1193 switch (dev->device_info->asic_family) {
1194 case CHIP_CARRIZO:
1195 device_queue_manager_init_vi(&dqm->ops_asic_specific);
300dec95
OG		 1196 break;
1197
a22fc854
BG		 1198 case CHIP_KAVERI:
1199 device_queue_manager_init_cik(&dqm->ops_asic_specific);
300dec95 1200 break;
a22fc854
BG		 1201 }
1202
45c9a5e4 1203 if (dqm->ops.initialize(dqm) != 0) {
64c7f8cf
BG		 1204 kfree(dqm);
1205 return NULL;
1206 }
1207
1208 return dqm;
1209}
1210
1211void device_queue_manager_uninit(struct device_queue_manager *dqm)
1212{
1213 BUG_ON(!dqm);
1214
45c9a5e4 1215 dqm->ops.uninitialize(dqm);
64c7f8cf
BG		 1216 kfree(dqm);
1217}
Linux 6.x block layer and io_uring tree(s)