Commit | Line | Data |
---|---|---|
45102048 BG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | ||
24 | #include <linux/slab.h> | |
25 | #include <linux/list.h> | |
26 | #include "kfd_device_queue_manager.h" | |
27 | #include "kfd_priv.h" | |
28 | #include "kfd_kernel_queue.h" | |
29 | ||
30 | static inline struct process_queue_node *get_queue_by_qid( | |
31 | struct process_queue_manager *pqm, unsigned int qid) | |
32 | { | |
33 | struct process_queue_node *pqn; | |
34 | ||
45102048 | 35 | list_for_each_entry(pqn, &pqm->queues, process_queue_list) { |
ab7c1648 KR |
36 | if ((pqn->q && pqn->q->properties.queue_id == qid) || |
37 | (pqn->kq && pqn->kq->queue->properties.queue_id == qid)) | |
45102048 BG |
38 | return pqn; |
39 | } | |
40 | ||
41 | return NULL; | |
42 | } | |
43 | ||
44 | static int find_available_queue_slot(struct process_queue_manager *pqm, | |
45 | unsigned int *qid) | |
46 | { | |
47 | unsigned long found; | |
48 | ||
45102048 | 49 | found = find_first_zero_bit(pqm->queue_slot_bitmap, |
b8cbab04 | 50 | KFD_MAX_NUM_OF_QUEUES_PER_PROCESS); |
45102048 | 51 | |
79775b62 | 52 | pr_debug("The new slot id %lu\n", found); |
45102048 | 53 | |
b8cbab04 | 54 | if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { |
79775b62 | 55 | pr_info("Cannot open more queues for process with pasid %d\n", |
45102048 BG |
56 | pqm->process->pasid); |
57 | return -ENOMEM; | |
58 | } | |
59 | ||
60 | set_bit(found, pqm->queue_slot_bitmap); | |
61 | *qid = found; | |
62 | ||
63 | return 0; | |
64 | } | |
65 | ||
9fd3f1bf FK |
66 | void kfd_process_dequeue_from_device(struct kfd_process_device *pdd) |
67 | { | |
68 | struct kfd_dev *dev = pdd->dev; | |
69 | ||
70 | if (pdd->already_dequeued) | |
71 | return; | |
72 | ||
73 | dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd); | |
74 | pdd->already_dequeued = true; | |
75 | } | |
76 | ||
77 | void kfd_process_dequeue_from_all_devices(struct kfd_process *p) | |
78 | { | |
79 | struct kfd_process_device *pdd; | |
80 | ||
81 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) | |
82 | kfd_process_dequeue_from_device(pdd); | |
83 | } | |
84 | ||
45102048 BG |
85 | int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p) |
86 | { | |
45102048 BG |
87 | INIT_LIST_HEAD(&pqm->queues); |
88 | pqm->queue_slot_bitmap = | |
b8cbab04 | 89 | kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, |
45102048 | 90 | BITS_PER_BYTE), GFP_KERNEL); |
4eacc26b | 91 | if (!pqm->queue_slot_bitmap) |
45102048 BG |
92 | return -ENOMEM; |
93 | pqm->process = p; | |
94 | ||
95 | return 0; | |
96 | } | |
97 | ||
98 | void pqm_uninit(struct process_queue_manager *pqm) | |
99 | { | |
45102048 BG |
100 | struct process_queue_node *pqn, *next; |
101 | ||
45102048 | 102 | list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) { |
9fd3f1bf FK |
103 | uninit_queue(pqn->q); |
104 | list_del(&pqn->process_queue_list); | |
105 | kfree(pqn); | |
45102048 | 106 | } |
9fd3f1bf | 107 | |
45102048 BG |
108 | kfree(pqm->queue_slot_bitmap); |
109 | pqm->queue_slot_bitmap = NULL; | |
110 | } | |
111 | ||
112 | static int create_cp_queue(struct process_queue_manager *pqm, | |
113 | struct kfd_dev *dev, struct queue **q, | |
114 | struct queue_properties *q_properties, | |
115 | struct file *f, unsigned int qid) | |
116 | { | |
117 | int retval; | |
118 | ||
45102048 BG |
119 | /* Doorbell initialized in user space*/ |
120 | q_properties->doorbell_ptr = NULL; | |
121 | ||
122 | q_properties->doorbell_off = | |
123 | kfd_queue_id_to_doorbell(dev, pqm->process, qid); | |
124 | ||
125 | /* let DQM handle it*/ | |
126 | q_properties->vmid = 0; | |
127 | q_properties->queue_id = qid; | |
45102048 | 128 | |
e88a614c | 129 | retval = init_queue(q, q_properties); |
45102048 | 130 | if (retval != 0) |
ab7c1648 | 131 | return retval; |
45102048 BG |
132 | |
133 | (*q)->device = dev; | |
134 | (*q)->process = pqm->process; | |
135 | ||
79775b62 | 136 | pr_debug("PQM After init queue"); |
45102048 BG |
137 | |
138 | return retval; | |
45102048 BG |
139 | } |
140 | ||
141 | int pqm_create_queue(struct process_queue_manager *pqm, | |
142 | struct kfd_dev *dev, | |
143 | struct file *f, | |
144 | struct queue_properties *properties, | |
45102048 BG |
145 | unsigned int *qid) |
146 | { | |
147 | int retval; | |
148 | struct kfd_process_device *pdd; | |
45102048 BG |
149 | struct queue *q; |
150 | struct process_queue_node *pqn; | |
151 | struct kernel_queue *kq; | |
e6f791b1 | 152 | enum kfd_queue_type type = properties->type; |
36c2d7eb | 153 | unsigned int max_queues = 127; /* HWS limit */ |
45102048 | 154 | |
45102048 BG |
155 | q = NULL; |
156 | kq = NULL; | |
157 | ||
093c7d8c AS |
158 | pdd = kfd_get_process_device_data(dev, pqm->process); |
159 | if (!pdd) { | |
160 | pr_err("Process device data doesn't exist\n"); | |
161 | return -1; | |
162 | } | |
45102048 | 163 | |
992839ad YS |
164 | /* |
165 | * for debug process, verify that it is within the static queues limit | |
166 | * currently limit is set to half of the total avail HQD slots | |
167 | * If we are just about to create DIQ, the is_debug flag is not set yet | |
168 | * Hence we also check the type as well | |
169 | */ | |
36c2d7eb FK |
170 | if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ)) |
171 | max_queues = dev->device_info->max_no_of_hqd/2; | |
172 | ||
173 | if (pdd->qpd.queue_count >= max_queues) | |
174 | return -ENOSPC; | |
992839ad | 175 | |
45102048 BG |
176 | retval = find_available_queue_slot(pqm, qid); |
177 | if (retval != 0) | |
178 | return retval; | |
179 | ||
5a29ad6b BG |
180 | if (list_empty(&pdd->qpd.queues_list) && |
181 | list_empty(&pdd->qpd.priv_queue_list)) { | |
45102048 | 182 | pdd->qpd.pqm = pqm; |
45c9a5e4 | 183 | dev->dqm->ops.register_process(dev->dqm, &pdd->qpd); |
45102048 BG |
184 | } |
185 | ||
dbf56ab1 | 186 | pqn = kzalloc(sizeof(*pqn), GFP_KERNEL); |
45102048 BG |
187 | if (!pqn) { |
188 | retval = -ENOMEM; | |
189 | goto err_allocate_pqn; | |
190 | } | |
191 | ||
192 | switch (type) { | |
bcea3081 | 193 | case KFD_QUEUE_TYPE_SDMA: |
8c946b89 FK |
194 | if (dev->dqm->queue_count >= |
195 | CIK_SDMA_QUEUES_PER_ENGINE * CIK_SDMA_ENGINE_NUM) { | |
196 | pr_err("Over-subscription is not allowed for SDMA.\n"); | |
197 | retval = -EPERM; | |
198 | goto err_create_queue; | |
199 | } | |
200 | ||
201 | retval = create_cp_queue(pqm, dev, &q, properties, f, *qid); | |
202 | if (retval != 0) | |
203 | goto err_create_queue; | |
204 | pqn->q = q; | |
205 | pqn->kq = NULL; | |
206 | retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, | |
207 | &q->properties.vmid); | |
208 | pr_debug("DQM returned %d for create_queue\n", retval); | |
209 | print_queue(q); | |
210 | break; | |
211 | ||
45102048 BG |
212 | case KFD_QUEUE_TYPE_COMPUTE: |
213 | /* check if there is over subscription */ | |
214 | if ((sched_policy == KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) && | |
44008d7a | 215 | ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) || |
d0b63bb3 | 216 | (dev->dqm->queue_count >= get_queues_num(dev->dqm)))) { |
79775b62 | 217 | pr_err("Over-subscription is not allowed in radeon_kfd.sched_policy == 1\n"); |
45102048 BG |
218 | retval = -EPERM; |
219 | goto err_create_queue; | |
220 | } | |
221 | ||
e6f791b1 | 222 | retval = create_cp_queue(pqm, dev, &q, properties, f, *qid); |
45102048 BG |
223 | if (retval != 0) |
224 | goto err_create_queue; | |
225 | pqn->q = q; | |
226 | pqn->kq = NULL; | |
45c9a5e4 | 227 | retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, |
45102048 | 228 | &q->properties.vmid); |
b8cbab04 | 229 | pr_debug("DQM returned %d for create_queue\n", retval); |
45102048 BG |
230 | print_queue(q); |
231 | break; | |
232 | case KFD_QUEUE_TYPE_DIQ: | |
233 | kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ); | |
4eacc26b | 234 | if (!kq) { |
e048a0b2 | 235 | retval = -ENOMEM; |
45102048 BG |
236 | goto err_create_queue; |
237 | } | |
238 | kq->queue->properties.queue_id = *qid; | |
239 | pqn->kq = kq; | |
240 | pqn->q = NULL; | |
45c9a5e4 OG |
241 | retval = dev->dqm->ops.create_kernel_queue(dev->dqm, |
242 | kq, &pdd->qpd); | |
45102048 BG |
243 | break; |
244 | default: | |
32fa8219 FK |
245 | WARN(1, "Invalid queue type %d", type); |
246 | retval = -EINVAL; | |
45102048 BG |
247 | } |
248 | ||
249 | if (retval != 0) { | |
79775b62 | 250 | pr_err("DQM create queue failed\n"); |
45102048 BG |
251 | goto err_create_queue; |
252 | } | |
253 | ||
79775b62 | 254 | pr_debug("PQM After DQM create queue\n"); |
45102048 BG |
255 | |
256 | list_add(&pqn->process_queue_list, &pqm->queues); | |
257 | ||
258 | if (q) { | |
79775b62 | 259 | pr_debug("PQM done creating queue\n"); |
e6f791b1 | 260 | print_queue_properties(&q->properties); |
45102048 BG |
261 | } |
262 | ||
263 | return retval; | |
264 | ||
265 | err_create_queue: | |
266 | kfree(pqn); | |
267 | err_allocate_pqn: | |
f046bfdf | 268 | /* check if queues list is empty unregister process from device */ |
45102048 | 269 | clear_bit(*qid, pqm->queue_slot_bitmap); |
5a29ad6b BG |
270 | if (list_empty(&pdd->qpd.queues_list) && |
271 | list_empty(&pdd->qpd.priv_queue_list)) | |
b3869b17 | 272 | dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd); |
45102048 BG |
273 | return retval; |
274 | } | |
275 | ||
276 | int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid) | |
277 | { | |
278 | struct process_queue_node *pqn; | |
279 | struct kfd_process_device *pdd; | |
280 | struct device_queue_manager *dqm; | |
281 | struct kfd_dev *dev; | |
282 | int retval; | |
283 | ||
284 | dqm = NULL; | |
285 | ||
45102048 BG |
286 | retval = 0; |
287 | ||
45102048 | 288 | pqn = get_queue_by_qid(pqm, qid); |
4eacc26b | 289 | if (!pqn) { |
79775b62 | 290 | pr_err("Queue id does not match any known queue\n"); |
45102048 BG |
291 | return -EINVAL; |
292 | } | |
293 | ||
294 | dev = NULL; | |
295 | if (pqn->kq) | |
296 | dev = pqn->kq->dev; | |
297 | if (pqn->q) | |
298 | dev = pqn->q->device; | |
32fa8219 FK |
299 | if (WARN_ON(!dev)) |
300 | return -ENODEV; | |
45102048 | 301 | |
093c7d8c AS |
302 | pdd = kfd_get_process_device_data(dev, pqm->process); |
303 | if (!pdd) { | |
304 | pr_err("Process device data doesn't exist\n"); | |
305 | return -1; | |
306 | } | |
45102048 BG |
307 | |
308 | if (pqn->kq) { | |
309 | /* destroy kernel queue (DIQ) */ | |
310 | dqm = pqn->kq->dev->dqm; | |
45c9a5e4 | 311 | dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd); |
45102048 BG |
312 | kernel_queue_uninit(pqn->kq); |
313 | } | |
314 | ||
315 | if (pqn->q) { | |
316 | dqm = pqn->q->device->dqm; | |
45c9a5e4 | 317 | retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q); |
45102048 BG |
318 | uninit_queue(pqn->q); |
319 | } | |
320 | ||
321 | list_del(&pqn->process_queue_list); | |
322 | kfree(pqn); | |
323 | clear_bit(qid, pqm->queue_slot_bitmap); | |
324 | ||
5a29ad6b BG |
325 | if (list_empty(&pdd->qpd.queues_list) && |
326 | list_empty(&pdd->qpd.priv_queue_list)) | |
45c9a5e4 | 327 | dqm->ops.unregister_process(dqm, &pdd->qpd); |
45102048 BG |
328 | |
329 | return retval; | |
330 | } | |
331 | ||
332 | int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, | |
333 | struct queue_properties *p) | |
334 | { | |
335 | int retval; | |
336 | struct process_queue_node *pqn; | |
337 | ||
45102048 | 338 | pqn = get_queue_by_qid(pqm, qid); |
b9dce23d | 339 | if (!pqn) { |
79775b62 | 340 | pr_debug("No queue %d exists for update operation\n", qid); |
b9dce23d OG |
341 | return -EFAULT; |
342 | } | |
45102048 BG |
343 | |
344 | pqn->q->properties.queue_address = p->queue_address; | |
345 | pqn->q->properties.queue_size = p->queue_size; | |
346 | pqn->q->properties.queue_percent = p->queue_percent; | |
347 | pqn->q->properties.priority = p->priority; | |
348 | ||
45c9a5e4 OG |
349 | retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm, |
350 | pqn->q); | |
45102048 BG |
351 | if (retval != 0) |
352 | return retval; | |
353 | ||
354 | return 0; | |
355 | } | |
356 | ||
fbeb661b | 357 | struct kernel_queue *pqm_get_kernel_queue( |
7347a6cb | 358 | struct process_queue_manager *pqm, |
45102048 BG |
359 | unsigned int qid) |
360 | { | |
361 | struct process_queue_node *pqn; | |
362 | ||
45102048 BG |
363 | pqn = get_queue_by_qid(pqm, qid); |
364 | if (pqn && pqn->kq) | |
365 | return pqn->kq; | |
366 | ||
367 | return NULL; | |
368 | } | |
369 | ||
370 |