Commit | Line | Data |
---|---|---|
19f6d2a6 OG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #include <linux/mutex.h> | |
24 | #include <linux/log2.h> | |
25 | #include <linux/sched.h> | |
6e84f315 | 26 | #include <linux/sched/mm.h> |
c7b1243e | 27 | #include <linux/sched/task.h> |
32cb59f3 | 28 | #include <linux/mmu_context.h> |
19f6d2a6 | 29 | #include <linux/slab.h> |
b17f068a | 30 | #include <linux/amd-iommu.h> |
19f6d2a6 | 31 | #include <linux/notifier.h> |
dd59239a | 32 | #include <linux/compat.h> |
373d7080 | 33 | #include <linux/mman.h> |
b84394e2 | 34 | #include <linux/file.h> |
9593f4d6 | 35 | #include <linux/pm_runtime.h> |
5b87245f | 36 | #include "amdgpu_amdkfd.h" |
ffa02269 | 37 | #include "amdgpu.h" |
dd59239a | 38 | |
19f6d2a6 OG |
39 | struct mm_struct; |
40 | ||
41 | #include "kfd_priv.h" | |
403575c4 | 42 | #include "kfd_device_queue_manager.h" |
c3447e81 | 43 | #include "kfd_dbgmgr.h" |
64d1c3a4 | 44 | #include "kfd_iommu.h" |
19f6d2a6 | 45 | |
19f6d2a6 OG |
46 | /* |
47 | * List of struct kfd_process (field kfd_process). | |
48 | * Unique/indexed by mm_struct* | |
49 | */ | |
64d1c3a4 | 50 | DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); |
19f6d2a6 OG |
51 | static DEFINE_MUTEX(kfd_processes_mutex); |
52 | ||
64d1c3a4 | 53 | DEFINE_SRCU(kfd_processes_srcu); |
19f6d2a6 | 54 | |
1679ae8f | 55 | /* For process termination handling */ |
19f6d2a6 OG |
56 | static struct workqueue_struct *kfd_process_wq; |
57 | ||
1679ae8f FK |
58 | /* Ordered, single-threaded workqueue for restoring evicted |
59 | * processes. Restoring multiple processes concurrently under memory | |
60 | * pressure can lead to processes blocking each other from validating | |
61 | * their BOs and result in a live-lock situation where processes | |
62 | * remain evicted indefinitely. | |
63 | */ | |
64 | static struct workqueue_struct *kfd_restore_wq; | |
65 | ||
19f6d2a6 | 66 | static struct kfd_process *find_process(const struct task_struct *thread); |
abb208a8 | 67 | static void kfd_process_ref_release(struct kref *ref); |
0029cab3 JG |
68 | static struct kfd_process *create_process(const struct task_struct *thread); |
69 | static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep); | |
373d7080 | 70 | |
26103436 FK |
71 | static void evict_process_worker(struct work_struct *work); |
72 | static void restore_process_worker(struct work_struct *work); | |
73 | ||
de9f26bb KR |
74 | struct kfd_procfs_tree { |
75 | struct kobject *kobj; | |
76 | }; | |
77 | ||
78 | static struct kfd_procfs_tree procfs; | |
79 | ||
32cb59f3 MJ |
80 | /* |
81 | * Structure for SDMA activity tracking | |
82 | */ | |
83 | struct kfd_sdma_activity_handler_workarea { | |
84 | struct work_struct sdma_activity_work; | |
85 | struct kfd_process_device *pdd; | |
86 | uint64_t sdma_activity_counter; | |
87 | }; | |
88 | ||
d69fd951 | 89 | struct temp_sdma_queue_list { |
818b0324 | 90 | uint64_t __user *rptr; |
d69fd951 MJ |
91 | uint64_t sdma_val; |
92 | unsigned int queue_id; | |
93 | struct list_head list; | |
94 | }; | |
95 | ||
32cb59f3 MJ |
96 | static void kfd_sdma_activity_worker(struct work_struct *work) |
97 | { | |
98 | struct kfd_sdma_activity_handler_workarea *workarea; | |
99 | struct kfd_process_device *pdd; | |
100 | uint64_t val; | |
101 | struct mm_struct *mm; | |
102 | struct queue *q; | |
103 | struct qcm_process_device *qpd; | |
104 | struct device_queue_manager *dqm; | |
105 | int ret = 0; | |
d69fd951 MJ |
106 | struct temp_sdma_queue_list sdma_q_list; |
107 | struct temp_sdma_queue_list *sdma_q, *next; | |
32cb59f3 MJ |
108 | |
109 | workarea = container_of(work, struct kfd_sdma_activity_handler_workarea, | |
110 | sdma_activity_work); | |
111 | if (!workarea) | |
112 | return; | |
113 | ||
114 | pdd = workarea->pdd; | |
2652bda7 CIK |
115 | if (!pdd) |
116 | return; | |
32cb59f3 MJ |
117 | dqm = pdd->dev->dqm; |
118 | qpd = &pdd->qpd; | |
2652bda7 | 119 | if (!dqm || !qpd) |
32cb59f3 | 120 | return; |
d69fd951 MJ |
121 | /* |
122 | * Total SDMA activity is current SDMA activity + past SDMA activity | |
123 | * Past SDMA count is stored in pdd. | |
124 | * To get the current activity counters for all active SDMA queues, | |
125 | * we loop over all SDMA queues and get their counts from user-space. | |
126 | * | |
127 | * We cannot call get_user() with dqm_lock held as it can cause | |
128 | * a circular lock dependency situation. To read the SDMA stats, | |
129 | * we need to do the following: | |
130 | * | |
131 | * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list, | |
132 | * with dqm_lock/dqm_unlock(). | |
133 | * 2. Call get_user() for each node in temporary list without dqm_lock. | |
134 | * Save the SDMA count for each node and also add the count to the total | |
135 | * SDMA count counter. | |
136 | * Its possible, during this step, a few SDMA queue nodes got deleted | |
137 | * from the qpd->queues_list. | |
138 | * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted. | |
139 | * If any node got deleted, its SDMA count would be captured in the sdma | |
140 | * past activity counter. So subtract the SDMA counter stored in step 2 | |
141 | * for this node from the total SDMA count. | |
142 | */ | |
143 | INIT_LIST_HEAD(&sdma_q_list.list); | |
32cb59f3 | 144 | |
d69fd951 MJ |
145 | /* |
146 | * Create the temp list of all SDMA queues | |
147 | */ | |
148 | dqm_lock(dqm); | |
149 | ||
150 | list_for_each_entry(q, &qpd->queues_list, list) { | |
151 | if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && | |
152 | (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) | |
153 | continue; | |
154 | ||
155 | sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL); | |
156 | if (!sdma_q) { | |
157 | dqm_unlock(dqm); | |
158 | goto cleanup; | |
159 | } | |
160 | ||
161 | INIT_LIST_HEAD(&sdma_q->list); | |
818b0324 | 162 | sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr; |
d69fd951 MJ |
163 | sdma_q->queue_id = q->properties.queue_id; |
164 | list_add_tail(&sdma_q->list, &sdma_q_list.list); | |
32cb59f3 MJ |
165 | } |
166 | ||
d69fd951 MJ |
167 | /* |
168 | * If the temp list is empty, then no SDMA queues nodes were found in | |
169 | * qpd->queues_list. Return the past activity count as the total sdma | |
170 | * count | |
171 | */ | |
172 | if (list_empty(&sdma_q_list.list)) { | |
173 | workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; | |
174 | dqm_unlock(dqm); | |
175 | return; | |
176 | } | |
32cb59f3 | 177 | |
d69fd951 | 178 | dqm_unlock(dqm); |
32cb59f3 MJ |
179 | |
180 | /* | |
d69fd951 | 181 | * Get the usage count for each SDMA queue in temp_list. |
32cb59f3 | 182 | */ |
d69fd951 MJ |
183 | mm = get_task_mm(pdd->process->lead_thread); |
184 | if (!mm) | |
185 | goto cleanup; | |
186 | ||
9555152b | 187 | kthread_use_mm(mm); |
d69fd951 MJ |
188 | |
189 | list_for_each_entry(sdma_q, &sdma_q_list.list, list) { | |
190 | val = 0; | |
191 | ret = read_sdma_queue_counter(sdma_q->rptr, &val); | |
192 | if (ret) { | |
193 | pr_debug("Failed to read SDMA queue active counter for queue id: %d", | |
194 | sdma_q->queue_id); | |
195 | } else { | |
196 | sdma_q->sdma_val = val; | |
197 | workarea->sdma_activity_counter += val; | |
198 | } | |
199 | } | |
200 | ||
9555152b | 201 | kthread_unuse_mm(mm); |
d69fd951 | 202 | mmput(mm); |
32cb59f3 MJ |
203 | |
204 | /* | |
d69fd951 MJ |
205 | * Do a second iteration over qpd_queues_list to check if any SDMA |
206 | * nodes got deleted while fetching SDMA counter. | |
32cb59f3 | 207 | */ |
d69fd951 MJ |
208 | dqm_lock(dqm); |
209 | ||
210 | workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; | |
211 | ||
32cb59f3 | 212 | list_for_each_entry(q, &qpd->queues_list, list) { |
d69fd951 MJ |
213 | if (list_empty(&sdma_q_list.list)) |
214 | break; | |
215 | ||
216 | if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && | |
217 | (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) | |
218 | continue; | |
219 | ||
220 | list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { | |
818b0324 | 221 | if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) && |
d69fd951 MJ |
222 | (sdma_q->queue_id == q->properties.queue_id)) { |
223 | list_del(&sdma_q->list); | |
224 | kfree(sdma_q); | |
225 | break; | |
226 | } | |
32cb59f3 MJ |
227 | } |
228 | } | |
229 | ||
230 | dqm_unlock(dqm); | |
d69fd951 MJ |
231 | |
232 | /* | |
233 | * If temp list is not empty, it implies some queues got deleted | |
234 | * from qpd->queues_list during SDMA usage read. Subtract the SDMA | |
235 | * count for each node from the total SDMA count. | |
236 | */ | |
237 | list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { | |
238 | workarea->sdma_activity_counter -= sdma_q->sdma_val; | |
239 | list_del(&sdma_q->list); | |
240 | kfree(sdma_q); | |
241 | } | |
242 | ||
243 | return; | |
244 | ||
245 | cleanup: | |
246 | list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { | |
247 | list_del(&sdma_q->list); | |
248 | kfree(sdma_q); | |
249 | } | |
32cb59f3 MJ |
250 | } |
251 | ||
f2fa07b3 RE |
252 | /** |
253 | * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device | |
254 | * by current process. Translates acquired wave count into number of compute units | |
255 | * that are occupied. | |
256 | * | |
257 | * @atr: Handle of attribute that allows reporting of wave count. The attribute | |
258 | * handle encapsulates GPU device it is associated with, thereby allowing collection | |
259 | * of waves in flight, etc | |
260 | * | |
261 | * @buffer: Handle of user provided buffer updated with wave count | |
262 | * | |
263 | * Return: Number of bytes written to user buffer or an error value | |
264 | */ | |
265 | static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer) | |
266 | { | |
267 | int cu_cnt; | |
268 | int wave_cnt; | |
269 | int max_waves_per_cu; | |
270 | struct kfd_dev *dev = NULL; | |
271 | struct kfd_process *proc = NULL; | |
272 | struct kfd_process_device *pdd = NULL; | |
273 | ||
274 | pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy); | |
275 | dev = pdd->dev; | |
276 | if (dev->kfd2kgd->get_cu_occupancy == NULL) | |
277 | return -EINVAL; | |
278 | ||
279 | cu_cnt = 0; | |
280 | proc = pdd->process; | |
281 | if (pdd->qpd.queue_count == 0) { | |
282 | pr_debug("Gpu-Id: %d has no active queues for process %d\n", | |
283 | dev->id, proc->pasid); | |
284 | return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt); | |
285 | } | |
286 | ||
287 | /* Collect wave count from device if it supports */ | |
288 | wave_cnt = 0; | |
289 | max_waves_per_cu = 0; | |
290 | dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt, | |
291 | &max_waves_per_cu); | |
292 | ||
293 | /* Translate wave count to number of compute units */ | |
294 | cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu; | |
295 | return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt); | |
296 | } | |
297 | ||
de9f26bb KR |
298 | static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr, |
299 | char *buffer) | |
300 | { | |
de9f26bb KR |
301 | if (strcmp(attr->name, "pasid") == 0) { |
302 | struct kfd_process *p = container_of(attr, struct kfd_process, | |
303 | attr_pasid); | |
d4566dee MJ |
304 | |
305 | return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid); | |
306 | } else if (strncmp(attr->name, "vram_", 5) == 0) { | |
307 | struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, | |
308 | attr_vram); | |
32cb59f3 MJ |
309 | return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage)); |
310 | } else if (strncmp(attr->name, "sdma_", 5) == 0) { | |
311 | struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, | |
312 | attr_sdma); | |
313 | struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler; | |
314 | ||
315 | INIT_WORK(&sdma_activity_work_handler.sdma_activity_work, | |
316 | kfd_sdma_activity_worker); | |
317 | ||
318 | sdma_activity_work_handler.pdd = pdd; | |
5960e022 | 319 | sdma_activity_work_handler.sdma_activity_counter = 0; |
32cb59f3 MJ |
320 | |
321 | schedule_work(&sdma_activity_work_handler.sdma_activity_work); | |
322 | ||
323 | flush_work(&sdma_activity_work_handler.sdma_activity_work); | |
324 | ||
325 | return snprintf(buffer, PAGE_SIZE, "%llu\n", | |
326 | (sdma_activity_work_handler.sdma_activity_counter)/ | |
327 | SDMA_ACTIVITY_DIVISOR); | |
de9f26bb KR |
328 | } else { |
329 | pr_err("Invalid attribute"); | |
330 | return -EINVAL; | |
331 | } | |
332 | ||
d4566dee | 333 | return 0; |
de9f26bb KR |
334 | } |
335 | ||
336 | static void kfd_procfs_kobj_release(struct kobject *kobj) | |
337 | { | |
338 | kfree(kobj); | |
339 | } | |
340 | ||
341 | static const struct sysfs_ops kfd_procfs_ops = { | |
342 | .show = kfd_procfs_show, | |
343 | }; | |
344 | ||
345 | static struct kobj_type procfs_type = { | |
346 | .release = kfd_procfs_kobj_release, | |
347 | .sysfs_ops = &kfd_procfs_ops, | |
348 | }; | |
349 | ||
350 | void kfd_procfs_init(void) | |
351 | { | |
352 | int ret = 0; | |
353 | ||
354 | procfs.kobj = kfd_alloc_struct(procfs.kobj); | |
355 | if (!procfs.kobj) | |
356 | return; | |
357 | ||
358 | ret = kobject_init_and_add(procfs.kobj, &procfs_type, | |
359 | &kfd_device->kobj, "proc"); | |
360 | if (ret) { | |
361 | pr_warn("Could not create procfs proc folder"); | |
362 | /* If we fail to create the procfs, clean up */ | |
363 | kfd_procfs_shutdown(); | |
364 | } | |
365 | } | |
366 | ||
367 | void kfd_procfs_shutdown(void) | |
368 | { | |
369 | if (procfs.kobj) { | |
370 | kobject_del(procfs.kobj); | |
371 | kobject_put(procfs.kobj); | |
372 | procfs.kobj = NULL; | |
373 | } | |
374 | } | |
19f6d2a6 | 375 | |
6d220a7e AL |
376 | static ssize_t kfd_procfs_queue_show(struct kobject *kobj, |
377 | struct attribute *attr, char *buffer) | |
378 | { | |
379 | struct queue *q = container_of(kobj, struct queue, kobj); | |
380 | ||
381 | if (!strcmp(attr->name, "size")) | |
382 | return snprintf(buffer, PAGE_SIZE, "%llu", | |
383 | q->properties.queue_size); | |
384 | else if (!strcmp(attr->name, "type")) | |
385 | return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type); | |
386 | else if (!strcmp(attr->name, "gpuid")) | |
387 | return snprintf(buffer, PAGE_SIZE, "%u", q->device->id); | |
388 | else | |
389 | pr_err("Invalid attribute"); | |
390 | ||
391 | return 0; | |
392 | } | |
393 | ||
4327bed2 PC |
394 | static ssize_t kfd_procfs_stats_show(struct kobject *kobj, |
395 | struct attribute *attr, char *buffer) | |
396 | { | |
397 | if (strcmp(attr->name, "evicted_ms") == 0) { | |
398 | struct kfd_process_device *pdd = container_of(attr, | |
399 | struct kfd_process_device, | |
400 | attr_evict); | |
401 | uint64_t evict_jiffies; | |
402 | ||
403 | evict_jiffies = atomic64_read(&pdd->evict_duration_counter); | |
404 | ||
405 | return snprintf(buffer, | |
406 | PAGE_SIZE, | |
407 | "%llu\n", | |
408 | jiffies64_to_msecs(evict_jiffies)); | |
f2fa07b3 RE |
409 | |
410 | /* Sysfs handle that gets CU occupancy is per device */ | |
411 | } else if (strcmp(attr->name, "cu_occupancy") == 0) { | |
412 | return kfd_get_cu_occupancy(attr, buffer); | |
413 | } else { | |
4327bed2 | 414 | pr_err("Invalid attribute"); |
f2fa07b3 | 415 | } |
4327bed2 PC |
416 | |
417 | return 0; | |
418 | } | |
6d220a7e AL |
419 | |
420 | static struct attribute attr_queue_size = { | |
421 | .name = "size", | |
422 | .mode = KFD_SYSFS_FILE_MODE | |
423 | }; | |
424 | ||
425 | static struct attribute attr_queue_type = { | |
426 | .name = "type", | |
427 | .mode = KFD_SYSFS_FILE_MODE | |
428 | }; | |
429 | ||
430 | static struct attribute attr_queue_gpuid = { | |
431 | .name = "gpuid", | |
432 | .mode = KFD_SYSFS_FILE_MODE | |
433 | }; | |
434 | ||
435 | static struct attribute *procfs_queue_attrs[] = { | |
436 | &attr_queue_size, | |
437 | &attr_queue_type, | |
438 | &attr_queue_gpuid, | |
439 | NULL | |
440 | }; | |
441 | ||
442 | static const struct sysfs_ops procfs_queue_ops = { | |
443 | .show = kfd_procfs_queue_show, | |
444 | }; | |
445 | ||
446 | static struct kobj_type procfs_queue_type = { | |
447 | .sysfs_ops = &procfs_queue_ops, | |
448 | .default_attrs = procfs_queue_attrs, | |
449 | }; | |
450 | ||
4327bed2 PC |
451 | static const struct sysfs_ops procfs_stats_ops = { |
452 | .show = kfd_procfs_stats_show, | |
453 | }; | |
454 | ||
455 | static struct attribute *procfs_stats_attrs[] = { | |
456 | NULL | |
457 | }; | |
458 | ||
459 | static struct kobj_type procfs_stats_type = { | |
460 | .sysfs_ops = &procfs_stats_ops, | |
461 | .default_attrs = procfs_stats_attrs, | |
462 | }; | |
463 | ||
6d220a7e AL |
464 | int kfd_procfs_add_queue(struct queue *q) |
465 | { | |
466 | struct kfd_process *proc; | |
467 | int ret; | |
468 | ||
469 | if (!q || !q->process) | |
470 | return -EINVAL; | |
471 | proc = q->process; | |
472 | ||
473 | /* Create proc/<pid>/queues/<queue id> folder */ | |
474 | if (!proc->kobj_queues) | |
475 | return -EFAULT; | |
476 | ret = kobject_init_and_add(&q->kobj, &procfs_queue_type, | |
477 | proc->kobj_queues, "%u", q->properties.queue_id); | |
478 | if (ret < 0) { | |
479 | pr_warn("Creating proc/<pid>/queues/%u failed", | |
480 | q->properties.queue_id); | |
481 | kobject_put(&q->kobj); | |
482 | return ret; | |
483 | } | |
484 | ||
485 | return 0; | |
486 | } | |
487 | ||
32cb59f3 MJ |
488 | static int kfd_sysfs_create_file(struct kfd_process *p, struct attribute *attr, |
489 | char *name) | |
490 | { | |
491 | int ret = 0; | |
492 | ||
493 | if (!p || !attr || !name) | |
494 | return -EINVAL; | |
495 | ||
496 | attr->name = name; | |
497 | attr->mode = KFD_SYSFS_FILE_MODE; | |
498 | sysfs_attr_init(attr); | |
499 | ||
500 | ret = sysfs_create_file(p->kobj, attr); | |
501 | ||
502 | return ret; | |
503 | } | |
504 | ||
4327bed2 PC |
505 | static int kfd_procfs_add_sysfs_stats(struct kfd_process *p) |
506 | { | |
507 | int ret = 0; | |
508 | struct kfd_process_device *pdd; | |
509 | char stats_dir_filename[MAX_SYSFS_FILENAME_LEN]; | |
510 | ||
511 | if (!p) | |
512 | return -EINVAL; | |
513 | ||
514 | if (!p->kobj) | |
515 | return -EFAULT; | |
516 | ||
517 | /* | |
518 | * Create sysfs files for each GPU: | |
519 | * - proc/<pid>/stats_<gpuid>/ | |
520 | * - proc/<pid>/stats_<gpuid>/evicted_ms | |
f2fa07b3 | 521 | * - proc/<pid>/stats_<gpuid>/cu_occupancy |
4327bed2 PC |
522 | */ |
523 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) { | |
524 | struct kobject *kobj_stats; | |
525 | ||
526 | snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN, | |
527 | "stats_%u", pdd->dev->id); | |
528 | kobj_stats = kfd_alloc_struct(kobj_stats); | |
529 | if (!kobj_stats) | |
530 | return -ENOMEM; | |
531 | ||
532 | ret = kobject_init_and_add(kobj_stats, | |
533 | &procfs_stats_type, | |
534 | p->kobj, | |
535 | stats_dir_filename); | |
536 | ||
537 | if (ret) { | |
538 | pr_warn("Creating KFD proc/stats_%s folder failed", | |
539 | stats_dir_filename); | |
540 | kobject_put(kobj_stats); | |
541 | goto err; | |
542 | } | |
543 | ||
544 | pdd->kobj_stats = kobj_stats; | |
545 | pdd->attr_evict.name = "evicted_ms"; | |
546 | pdd->attr_evict.mode = KFD_SYSFS_FILE_MODE; | |
547 | sysfs_attr_init(&pdd->attr_evict); | |
548 | ret = sysfs_create_file(kobj_stats, &pdd->attr_evict); | |
549 | if (ret) | |
550 | pr_warn("Creating eviction stats for gpuid %d failed", | |
551 | (int)pdd->dev->id); | |
f2fa07b3 RE |
552 | |
553 | /* Add sysfs file to report compute unit occupancy */ | |
554 | if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) { | |
555 | pdd->attr_cu_occupancy.name = "cu_occupancy"; | |
556 | pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE; | |
557 | sysfs_attr_init(&pdd->attr_cu_occupancy); | |
558 | ret = sysfs_create_file(kobj_stats, | |
559 | &pdd->attr_cu_occupancy); | |
560 | if (ret) | |
561 | pr_warn("Creating %s failed for gpuid: %d", | |
562 | pdd->attr_cu_occupancy.name, | |
563 | (int)pdd->dev->id); | |
564 | } | |
4327bed2 PC |
565 | } |
566 | err: | |
567 | return ret; | |
568 | } | |
569 | ||
570 | ||
204d8998 | 571 | static int kfd_procfs_add_sysfs_files(struct kfd_process *p) |
d4566dee MJ |
572 | { |
573 | int ret = 0; | |
574 | struct kfd_process_device *pdd; | |
575 | ||
576 | if (!p) | |
577 | return -EINVAL; | |
578 | ||
579 | if (!p->kobj) | |
580 | return -EFAULT; | |
581 | ||
32cb59f3 MJ |
582 | /* |
583 | * Create sysfs files for each GPU: | |
584 | * - proc/<pid>/vram_<gpuid> | |
585 | * - proc/<pid>/sdma_<gpuid> | |
586 | */ | |
d4566dee | 587 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) { |
32cb59f3 | 588 | snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u", |
d4566dee | 589 | pdd->dev->id); |
32cb59f3 | 590 | ret = kfd_sysfs_create_file(p, &pdd->attr_vram, pdd->vram_filename); |
d4566dee MJ |
591 | if (ret) |
592 | pr_warn("Creating vram usage for gpu id %d failed", | |
593 | (int)pdd->dev->id); | |
32cb59f3 MJ |
594 | |
595 | snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u", | |
596 | pdd->dev->id); | |
597 | ret = kfd_sysfs_create_file(p, &pdd->attr_sdma, pdd->sdma_filename); | |
598 | if (ret) | |
599 | pr_warn("Creating sdma usage for gpu id %d failed", | |
600 | (int)pdd->dev->id); | |
d4566dee MJ |
601 | } |
602 | ||
603 | return ret; | |
604 | } | |
605 | ||
6d220a7e AL |
606 | void kfd_procfs_del_queue(struct queue *q) |
607 | { | |
608 | if (!q) | |
609 | return; | |
610 | ||
611 | kobject_del(&q->kobj); | |
612 | kobject_put(&q->kobj); | |
613 | } | |
614 | ||
1679ae8f | 615 | int kfd_process_create_wq(void) |
19f6d2a6 OG |
616 | { |
617 | if (!kfd_process_wq) | |
fd320bf6 | 618 | kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0); |
1679ae8f FK |
619 | if (!kfd_restore_wq) |
620 | kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0); | |
621 | ||
622 | if (!kfd_process_wq || !kfd_restore_wq) { | |
623 | kfd_process_destroy_wq(); | |
624 | return -ENOMEM; | |
625 | } | |
626 | ||
627 | return 0; | |
19f6d2a6 OG |
628 | } |
629 | ||
630 | void kfd_process_destroy_wq(void) | |
631 | { | |
632 | if (kfd_process_wq) { | |
19f6d2a6 OG |
633 | destroy_workqueue(kfd_process_wq); |
634 | kfd_process_wq = NULL; | |
635 | } | |
1679ae8f FK |
636 | if (kfd_restore_wq) { |
637 | destroy_workqueue(kfd_restore_wq); | |
638 | kfd_restore_wq = NULL; | |
639 | } | |
19f6d2a6 OG |
640 | } |
641 | ||
f35751b8 FK |
642 | static void kfd_process_free_gpuvm(struct kgd_mem *mem, |
643 | struct kfd_process_device *pdd) | |
644 | { | |
645 | struct kfd_dev *dev = pdd->dev; | |
646 | ||
5b87245f | 647 | amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->vm); |
d4566dee | 648 | amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, NULL); |
f35751b8 FK |
649 | } |
650 | ||
651 | /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process | |
652 | * This function should be only called right after the process | |
653 | * is created and when kfd_processes_mutex is still being held | |
654 | * to avoid concurrency. Because of that exclusiveness, we do | |
655 | * not need to take p->mutex. | |
656 | */ | |
657 | static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, | |
658 | uint64_t gpu_va, uint32_t size, | |
659 | uint32_t flags, void **kptr) | |
660 | { | |
661 | struct kfd_dev *kdev = pdd->dev; | |
662 | struct kgd_mem *mem = NULL; | |
663 | int handle; | |
664 | int err; | |
665 | ||
5b87245f | 666 | err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size, |
f35751b8 FK |
667 | pdd->vm, &mem, NULL, flags); |
668 | if (err) | |
669 | goto err_alloc_mem; | |
670 | ||
5b87245f | 671 | err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->vm); |
f35751b8 FK |
672 | if (err) |
673 | goto err_map_mem; | |
674 | ||
5b87245f | 675 | err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true); |
f35751b8 FK |
676 | if (err) { |
677 | pr_debug("Sync memory failed, wait interrupted by user signal\n"); | |
678 | goto sync_memory_failed; | |
679 | } | |
680 | ||
681 | /* Create an obj handle so kfd_process_device_remove_obj_handle | |
682 | * will take care of the bo removal when the process finishes. | |
683 | * We do not need to take p->mutex, because the process is just | |
684 | * created and the ioctls have not had the chance to run. | |
685 | */ | |
686 | handle = kfd_process_device_create_obj_handle(pdd, mem); | |
687 | ||
688 | if (handle < 0) { | |
689 | err = handle; | |
690 | goto free_gpuvm; | |
691 | } | |
692 | ||
693 | if (kptr) { | |
5b87245f | 694 | err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd, |
f35751b8 FK |
695 | (struct kgd_mem *)mem, kptr, NULL); |
696 | if (err) { | |
697 | pr_debug("Map GTT BO to kernel failed\n"); | |
698 | goto free_obj_handle; | |
699 | } | |
700 | } | |
701 | ||
702 | return err; | |
703 | ||
704 | free_obj_handle: | |
705 | kfd_process_device_remove_obj_handle(pdd, handle); | |
706 | free_gpuvm: | |
707 | sync_memory_failed: | |
708 | kfd_process_free_gpuvm(mem, pdd); | |
709 | return err; | |
710 | ||
711 | err_map_mem: | |
d4566dee | 712 | amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, NULL); |
f35751b8 FK |
713 | err_alloc_mem: |
714 | *kptr = NULL; | |
715 | return err; | |
716 | } | |
717 | ||
552764b6 FK |
718 | /* kfd_process_device_reserve_ib_mem - Reserve memory inside the |
719 | * process for IB usage The memory reserved is for KFD to submit | |
720 | * IB to AMDGPU from kernel. If the memory is reserved | |
721 | * successfully, ib_kaddr will have the CPU/kernel | |
722 | * address. Check ib_kaddr before accessing the memory. | |
723 | */ | |
724 | static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd) | |
725 | { | |
726 | struct qcm_process_device *qpd = &pdd->qpd; | |
1d251d90 YZ |
727 | uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT | |
728 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE | | |
729 | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | | |
730 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; | |
552764b6 FK |
731 | void *kaddr; |
732 | int ret; | |
733 | ||
734 | if (qpd->ib_kaddr || !qpd->ib_base) | |
735 | return 0; | |
736 | ||
737 | /* ib_base is only set for dGPU */ | |
738 | ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags, | |
739 | &kaddr); | |
740 | if (ret) | |
741 | return ret; | |
742 | ||
743 | qpd->ib_kaddr = kaddr; | |
744 | ||
745 | return 0; | |
746 | } | |
747 | ||
373d7080 | 748 | struct kfd_process *kfd_create_process(struct file *filep) |
19f6d2a6 OG |
749 | { |
750 | struct kfd_process *process; | |
373d7080 | 751 | struct task_struct *thread = current; |
de9f26bb | 752 | int ret; |
19f6d2a6 | 753 | |
4eacc26b | 754 | if (!thread->mm) |
19f6d2a6 OG |
755 | return ERR_PTR(-EINVAL); |
756 | ||
757 | /* Only the pthreads threading model is supported. */ | |
758 | if (thread->group_leader->mm != thread->mm) | |
759 | return ERR_PTR(-EINVAL); | |
760 | ||
19f6d2a6 OG |
761 | /* |
762 | * take kfd processes mutex before starting of process creation | |
763 | * so there won't be a case where two threads of the same process | |
764 | * create two kfd_process structures | |
765 | */ | |
766 | mutex_lock(&kfd_processes_mutex); | |
767 | ||
768 | /* A prior open of /dev/kfd could have already created the process. */ | |
769 | process = find_process(thread); | |
de9f26bb | 770 | if (process) { |
79775b62 | 771 | pr_debug("Process already found\n"); |
de9f26bb | 772 | } else { |
0029cab3 JG |
773 | process = create_process(thread); |
774 | if (IS_ERR(process)) | |
775 | goto out; | |
776 | ||
777 | ret = kfd_process_init_cwsr_apu(process, filep); | |
778 | if (ret) { | |
779 | process = ERR_PTR(ret); | |
780 | goto out; | |
781 | } | |
19f6d2a6 | 782 | |
de9f26bb KR |
783 | if (!procfs.kobj) |
784 | goto out; | |
785 | ||
786 | process->kobj = kfd_alloc_struct(process->kobj); | |
787 | if (!process->kobj) { | |
788 | pr_warn("Creating procfs kobject failed"); | |
789 | goto out; | |
790 | } | |
791 | ret = kobject_init_and_add(process->kobj, &procfs_type, | |
792 | procfs.kobj, "%d", | |
793 | (int)process->lead_thread->pid); | |
794 | if (ret) { | |
795 | pr_warn("Creating procfs pid directory failed"); | |
dc2f832e | 796 | kobject_put(process->kobj); |
de9f26bb KR |
797 | goto out; |
798 | } | |
799 | ||
800 | process->attr_pasid.name = "pasid"; | |
801 | process->attr_pasid.mode = KFD_SYSFS_FILE_MODE; | |
802 | sysfs_attr_init(&process->attr_pasid); | |
803 | ret = sysfs_create_file(process->kobj, &process->attr_pasid); | |
804 | if (ret) | |
805 | pr_warn("Creating pasid for pid %d failed", | |
806 | (int)process->lead_thread->pid); | |
6d220a7e AL |
807 | |
808 | process->kobj_queues = kobject_create_and_add("queues", | |
809 | process->kobj); | |
810 | if (!process->kobj_queues) | |
811 | pr_warn("Creating KFD proc/queues folder failed"); | |
d4566dee | 812 | |
4327bed2 PC |
813 | ret = kfd_procfs_add_sysfs_stats(process); |
814 | if (ret) | |
815 | pr_warn("Creating sysfs stats dir for pid %d failed", | |
816 | (int)process->lead_thread->pid); | |
817 | ||
32cb59f3 | 818 | ret = kfd_procfs_add_sysfs_files(process); |
d4566dee | 819 | if (ret) |
32cb59f3 | 820 | pr_warn("Creating sysfs usage file for pid %d failed", |
d4566dee | 821 | (int)process->lead_thread->pid); |
de9f26bb KR |
822 | } |
823 | out: | |
0f899fd4 FK |
824 | if (!IS_ERR(process)) |
825 | kref_get(&process->ref); | |
19f6d2a6 OG |
826 | mutex_unlock(&kfd_processes_mutex); |
827 | ||
19f6d2a6 OG |
828 | return process; |
829 | } | |
830 | ||
831 | struct kfd_process *kfd_get_process(const struct task_struct *thread) | |
832 | { | |
833 | struct kfd_process *process; | |
834 | ||
4eacc26b | 835 | if (!thread->mm) |
19f6d2a6 OG |
836 | return ERR_PTR(-EINVAL); |
837 | ||
838 | /* Only the pthreads threading model is supported. */ | |
839 | if (thread->group_leader->mm != thread->mm) | |
840 | return ERR_PTR(-EINVAL); | |
841 | ||
842 | process = find_process(thread); | |
e47cb828 WL |
843 | if (!process) |
844 | return ERR_PTR(-EINVAL); | |
19f6d2a6 OG |
845 | |
846 | return process; | |
847 | } | |
848 | ||
849 | static struct kfd_process *find_process_by_mm(const struct mm_struct *mm) | |
850 | { | |
851 | struct kfd_process *process; | |
852 | ||
853 | hash_for_each_possible_rcu(kfd_processes_table, process, | |
854 | kfd_processes, (uintptr_t)mm) | |
855 | if (process->mm == mm) | |
856 | return process; | |
857 | ||
858 | return NULL; | |
859 | } | |
860 | ||
861 | static struct kfd_process *find_process(const struct task_struct *thread) | |
862 | { | |
863 | struct kfd_process *p; | |
864 | int idx; | |
865 | ||
866 | idx = srcu_read_lock(&kfd_processes_srcu); | |
867 | p = find_process_by_mm(thread->mm); | |
868 | srcu_read_unlock(&kfd_processes_srcu, idx); | |
869 | ||
870 | return p; | |
871 | } | |
872 | ||
abb208a8 FK |
873 | void kfd_unref_process(struct kfd_process *p) |
874 | { | |
875 | kref_put(&p->ref, kfd_process_ref_release); | |
876 | } | |
877 | ||
52b29d73 FK |
878 | static void kfd_process_device_free_bos(struct kfd_process_device *pdd) |
879 | { | |
880 | struct kfd_process *p = pdd->process; | |
881 | void *mem; | |
882 | int id; | |
883 | ||
884 | /* | |
885 | * Remove all handles from idr and release appropriate | |
886 | * local memory object | |
887 | */ | |
888 | idr_for_each_entry(&pdd->alloc_idr, mem, id) { | |
889 | struct kfd_process_device *peer_pdd; | |
890 | ||
891 | list_for_each_entry(peer_pdd, &p->per_device_data, | |
892 | per_device_list) { | |
893 | if (!peer_pdd->vm) | |
894 | continue; | |
5b87245f | 895 | amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( |
52b29d73 FK |
896 | peer_pdd->dev->kgd, mem, peer_pdd->vm); |
897 | } | |
898 | ||
d4566dee | 899 | amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, NULL); |
52b29d73 FK |
900 | kfd_process_device_remove_obj_handle(pdd, id); |
901 | } | |
902 | } | |
903 | ||
904 | static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p) | |
905 | { | |
906 | struct kfd_process_device *pdd; | |
907 | ||
908 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) | |
909 | kfd_process_device_free_bos(pdd); | |
910 | } | |
911 | ||
de1450a5 | 912 | static void kfd_process_destroy_pdds(struct kfd_process *p) |
19f6d2a6 | 913 | { |
19f6d2a6 | 914 | struct kfd_process_device *pdd, *temp; |
19f6d2a6 | 915 | |
19f6d2a6 | 916 | list_for_each_entry_safe(pdd, temp, &p->per_device_data, |
de1450a5 | 917 | per_device_list) { |
6027b1bf | 918 | pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n", |
94a1ee09 OG |
919 | pdd->dev->id, p->pasid); |
920 | ||
bf47afba | 921 | if (pdd->drm_file) { |
5b87245f AL |
922 | amdgpu_amdkfd_gpuvm_release_process_vm( |
923 | pdd->dev->kgd, pdd->vm); | |
b84394e2 | 924 | fput(pdd->drm_file); |
bf47afba | 925 | } |
b84394e2 | 926 | else if (pdd->vm) |
5b87245f | 927 | amdgpu_amdkfd_gpuvm_destroy_process_vm( |
403575c4 FK |
928 | pdd->dev->kgd, pdd->vm); |
929 | ||
733fa1f7 | 930 | list_del(&pdd->per_device_list); |
373d7080 | 931 | |
f35751b8 | 932 | if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) |
373d7080 FK |
933 | free_pages((unsigned long)pdd->qpd.cwsr_kaddr, |
934 | get_order(KFD_CWSR_TBA_TMA_SIZE)); | |
935 | ||
ef568db7 | 936 | kfree(pdd->qpd.doorbell_bitmap); |
52b29d73 FK |
937 | idr_destroy(&pdd->alloc_idr); |
938 | ||
59d7115d MJ |
939 | kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index); |
940 | ||
9593f4d6 RB |
941 | /* |
942 | * before destroying pdd, make sure to report availability | |
943 | * for auto suspend | |
944 | */ | |
945 | if (pdd->runtime_inuse) { | |
946 | pm_runtime_mark_last_busy(pdd->dev->ddev->dev); | |
947 | pm_runtime_put_autosuspend(pdd->dev->ddev->dev); | |
948 | pdd->runtime_inuse = false; | |
949 | } | |
950 | ||
19f6d2a6 OG |
951 | kfree(pdd); |
952 | } | |
de1450a5 FK |
953 | } |
954 | ||
955 | /* No process locking is needed in this function, because the process | |
956 | * is not findable any more. We must assume that no other thread is | |
957 | * using it any more, otherwise we couldn't safely free the process | |
958 | * structure in the end. | |
959 | */ | |
960 | static void kfd_process_wq_release(struct work_struct *work) | |
961 | { | |
962 | struct kfd_process *p = container_of(work, struct kfd_process, | |
963 | release_work); | |
d4566dee | 964 | struct kfd_process_device *pdd; |
de1450a5 | 965 | |
de9f26bb KR |
966 | /* Remove the procfs files */ |
967 | if (p->kobj) { | |
968 | sysfs_remove_file(p->kobj, &p->attr_pasid); | |
6d220a7e AL |
969 | kobject_del(p->kobj_queues); |
970 | kobject_put(p->kobj_queues); | |
971 | p->kobj_queues = NULL; | |
d4566dee | 972 | |
32cb59f3 | 973 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) { |
d4566dee | 974 | sysfs_remove_file(p->kobj, &pdd->attr_vram); |
32cb59f3 | 975 | sysfs_remove_file(p->kobj, &pdd->attr_sdma); |
4327bed2 | 976 | sysfs_remove_file(p->kobj, &pdd->attr_evict); |
f2fa07b3 RE |
977 | if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) |
978 | sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy); | |
4327bed2 PC |
979 | kobject_del(pdd->kobj_stats); |
980 | kobject_put(pdd->kobj_stats); | |
981 | pdd->kobj_stats = NULL; | |
32cb59f3 | 982 | } |
d4566dee | 983 | |
de9f26bb KR |
984 | kobject_del(p->kobj); |
985 | kobject_put(p->kobj); | |
986 | p->kobj = NULL; | |
987 | } | |
988 | ||
64d1c3a4 | 989 | kfd_iommu_unbind_process(p); |
de1450a5 | 990 | |
52b29d73 FK |
991 | kfd_process_free_outstanding_kfd_bos(p); |
992 | ||
de1450a5 | 993 | kfd_process_destroy_pdds(p); |
403575c4 | 994 | dma_fence_put(p->ef); |
19f6d2a6 | 995 | |
f3a39818 AL |
996 | kfd_event_free_process(p); |
997 | ||
19f6d2a6 | 998 | kfd_pasid_free(p->pasid); |
19f6d2a6 OG |
999 | mutex_destroy(&p->mutex); |
1000 | ||
c7b1243e FK |
1001 | put_task_struct(p->lead_thread); |
1002 | ||
19f6d2a6 | 1003 | kfree(p); |
19f6d2a6 OG |
1004 | } |
1005 | ||
5ce10687 | 1006 | static void kfd_process_ref_release(struct kref *ref) |
19f6d2a6 | 1007 | { |
5ce10687 | 1008 | struct kfd_process *p = container_of(ref, struct kfd_process, ref); |
19f6d2a6 | 1009 | |
5ce10687 FK |
1010 | INIT_WORK(&p->release_work, kfd_process_wq_release); |
1011 | queue_work(kfd_process_wq, &p->release_work); | |
1012 | } | |
19f6d2a6 | 1013 | |
471f3902 | 1014 | static void kfd_process_free_notifier(struct mmu_notifier *mn) |
5ce10687 | 1015 | { |
471f3902 | 1016 | kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier)); |
19f6d2a6 OG |
1017 | } |
1018 | ||
1019 | static void kfd_process_notifier_release(struct mmu_notifier *mn, | |
1020 | struct mm_struct *mm) | |
1021 | { | |
1022 | struct kfd_process *p; | |
a82918f1 | 1023 | struct kfd_process_device *pdd = NULL; |
19f6d2a6 OG |
1024 | |
1025 | /* | |
1026 | * The kfd_process structure can not be free because the | |
1027 | * mmu_notifier srcu is read locked | |
1028 | */ | |
1029 | p = container_of(mn, struct kfd_process, mmu_notifier); | |
32fa8219 FK |
1030 | if (WARN_ON(p->mm != mm)) |
1031 | return; | |
19f6d2a6 OG |
1032 | |
1033 | mutex_lock(&kfd_processes_mutex); | |
1034 | hash_del_rcu(&p->kfd_processes); | |
1035 | mutex_unlock(&kfd_processes_mutex); | |
1036 | synchronize_srcu(&kfd_processes_srcu); | |
1037 | ||
26103436 FK |
1038 | cancel_delayed_work_sync(&p->eviction_work); |
1039 | cancel_delayed_work_sync(&p->restore_work); | |
1040 | ||
45102048 BG |
1041 | mutex_lock(&p->mutex); |
1042 | ||
062c5672 YS |
1043 | /* Iterate over all process device data structures and if the |
1044 | * pdd is in debug mode, we should first force unregistration, | |
1045 | * then we will be able to destroy the queues | |
1046 | */ | |
1047 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) { | |
1048 | struct kfd_dev *dev = pdd->dev; | |
1049 | ||
1050 | mutex_lock(kfd_get_dbgmgr_mutex()); | |
1051 | if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) { | |
1052 | if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) { | |
1053 | kfd_dbgmgr_destroy(dev->dbgmgr); | |
1054 | dev->dbgmgr = NULL; | |
1055 | } | |
1056 | } | |
1057 | mutex_unlock(kfd_get_dbgmgr_mutex()); | |
1058 | } | |
1059 | ||
9fd3f1bf | 1060 | kfd_process_dequeue_from_all_devices(p); |
45102048 BG |
1061 | pqm_uninit(&p->pqm); |
1062 | ||
5ce10687 FK |
1063 | /* Indicate to other users that MM is no longer valid */ |
1064 | p->mm = NULL; | |
129657c8 FK |
1065 | /* Signal the eviction fence after user mode queues are |
1066 | * destroyed. This allows any BOs to be freed without | |
1067 | * triggering pointless evictions or waiting for fences. | |
1068 | */ | |
1069 | dma_fence_signal(p->ef); | |
5ce10687 | 1070 | |
45102048 BG |
1071 | mutex_unlock(&p->mutex); |
1072 | ||
471f3902 | 1073 | mmu_notifier_put(&p->mmu_notifier); |
19f6d2a6 OG |
1074 | } |
1075 | ||
1076 | static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = { | |
1077 | .release = kfd_process_notifier_release, | |
471f3902 | 1078 | .free_notifier = kfd_process_free_notifier, |
19f6d2a6 OG |
1079 | }; |
1080 | ||
f35751b8 | 1081 | static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep) |
373d7080 | 1082 | { |
373d7080 | 1083 | unsigned long offset; |
f35751b8 | 1084 | struct kfd_process_device *pdd; |
373d7080 | 1085 | |
c0ede1f8 | 1086 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) { |
f35751b8 FK |
1087 | struct kfd_dev *dev = pdd->dev; |
1088 | struct qcm_process_device *qpd = &pdd->qpd; | |
1089 | ||
1090 | if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base) | |
373d7080 | 1091 | continue; |
f35751b8 | 1092 | |
29453755 | 1093 | offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id); |
373d7080 FK |
1094 | qpd->tba_addr = (int64_t)vm_mmap(filep, 0, |
1095 | KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC, | |
1096 | MAP_SHARED, offset); | |
1097 | ||
1098 | if (IS_ERR_VALUE(qpd->tba_addr)) { | |
c0ede1f8 YZ |
1099 | int err = qpd->tba_addr; |
1100 | ||
1101 | pr_err("Failure to set tba address. error %d.\n", err); | |
373d7080 FK |
1102 | qpd->tba_addr = 0; |
1103 | qpd->cwsr_kaddr = NULL; | |
c0ede1f8 | 1104 | return err; |
373d7080 FK |
1105 | } |
1106 | ||
1107 | memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size); | |
1108 | ||
1109 | qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; | |
1110 | pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n", | |
1111 | qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); | |
1112 | } | |
c0ede1f8 YZ |
1113 | |
1114 | return 0; | |
373d7080 FK |
1115 | } |
1116 | ||
f35751b8 FK |
1117 | static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) |
1118 | { | |
1119 | struct kfd_dev *dev = pdd->dev; | |
1120 | struct qcm_process_device *qpd = &pdd->qpd; | |
1d251d90 YZ |
1121 | uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
1122 | | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE | |
1123 | | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; | |
f35751b8 FK |
1124 | void *kaddr; |
1125 | int ret; | |
1126 | ||
1127 | if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base) | |
1128 | return 0; | |
1129 | ||
1130 | /* cwsr_base is only set for dGPU */ | |
1131 | ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, | |
1132 | KFD_CWSR_TBA_TMA_SIZE, flags, &kaddr); | |
1133 | if (ret) | |
1134 | return ret; | |
1135 | ||
1136 | qpd->cwsr_kaddr = kaddr; | |
1137 | qpd->tba_addr = qpd->cwsr_base; | |
1138 | ||
1139 | memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size); | |
1140 | ||
1141 | qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; | |
1142 | pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n", | |
1143 | qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); | |
1144 | ||
1145 | return 0; | |
1146 | } | |
1147 | ||
0029cab3 JG |
1148 | /* |
1149 | * On return the kfd_process is fully operational and will be freed when the | |
1150 | * mm is released | |
1151 | */ | |
1152 | static struct kfd_process *create_process(const struct task_struct *thread) | |
19f6d2a6 OG |
1153 | { |
1154 | struct kfd_process *process; | |
1155 | int err = -ENOMEM; | |
1156 | ||
1157 | process = kzalloc(sizeof(*process), GFP_KERNEL); | |
19f6d2a6 OG |
1158 | if (!process) |
1159 | goto err_alloc_process; | |
1160 | ||
5ce10687 | 1161 | kref_init(&process->ref); |
19f6d2a6 | 1162 | mutex_init(&process->mutex); |
19f6d2a6 | 1163 | process->mm = thread->mm; |
19f6d2a6 | 1164 | process->lead_thread = thread->group_leader; |
19f6d2a6 | 1165 | INIT_LIST_HEAD(&process->per_device_data); |
0029cab3 JG |
1166 | INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker); |
1167 | INIT_DELAYED_WORK(&process->restore_work, restore_process_worker); | |
1168 | process->last_restore_timestamp = get_jiffies_64(); | |
f3a39818 | 1169 | kfd_event_init_process(process); |
0029cab3 JG |
1170 | process->is_32bit_user_mode = in_compat_syscall(); |
1171 | ||
1172 | process->pasid = kfd_pasid_alloc(); | |
1173 | if (process->pasid == 0) | |
1174 | goto err_alloc_pasid; | |
1175 | ||
45102048 BG |
1176 | err = pqm_init(&process->pqm, process); |
1177 | if (err != 0) | |
1178 | goto err_process_pqm_init; | |
1179 | ||
dd59239a | 1180 | /* init process apertures*/ |
b312b2b2 DC |
1181 | err = kfd_init_apertures(process); |
1182 | if (err != 0) | |
7a10d63f | 1183 | goto err_init_apertures; |
dd59239a | 1184 | |
0029cab3 JG |
1185 | /* Must be last, have to use release destruction after this */ |
1186 | process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops; | |
1187 | err = mmu_notifier_register(&process->mmu_notifier, process->mm); | |
c0ede1f8 | 1188 | if (err) |
0029cab3 JG |
1189 | goto err_register_notifier; |
1190 | ||
1191 | get_task_struct(process->lead_thread); | |
1192 | hash_add_rcu(kfd_processes_table, &process->kfd_processes, | |
1193 | (uintptr_t)process->mm); | |
c0ede1f8 | 1194 | |
19f6d2a6 OG |
1195 | return process; |
1196 | ||
0029cab3 | 1197 | err_register_notifier: |
52b29d73 | 1198 | kfd_process_free_outstanding_kfd_bos(process); |
c0ede1f8 | 1199 | kfd_process_destroy_pdds(process); |
7a10d63f | 1200 | err_init_apertures: |
dd59239a | 1201 | pqm_uninit(&process->pqm); |
45102048 | 1202 | err_process_pqm_init: |
19f6d2a6 OG |
1203 | kfd_pasid_free(process->pasid); |
1204 | err_alloc_pasid: | |
0029cab3 | 1205 | mutex_destroy(&process->mutex); |
19f6d2a6 OG |
1206 | kfree(process); |
1207 | err_alloc_process: | |
1208 | return ERR_PTR(err); | |
1209 | } | |
1210 | ||
ef568db7 FK |
1211 | static int init_doorbell_bitmap(struct qcm_process_device *qpd, |
1212 | struct kfd_dev *dev) | |
1213 | { | |
1214 | unsigned int i; | |
89b0679b YZ |
1215 | int range_start = dev->shared_resources.non_cp_doorbells_start; |
1216 | int range_end = dev->shared_resources.non_cp_doorbells_end; | |
ef568db7 FK |
1217 | |
1218 | if (!KFD_IS_SOC15(dev->device_info->asic_family)) | |
1219 | return 0; | |
1220 | ||
1221 | qpd->doorbell_bitmap = | |
1222 | kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, | |
1223 | BITS_PER_BYTE), GFP_KERNEL); | |
1224 | if (!qpd->doorbell_bitmap) | |
1225 | return -ENOMEM; | |
1226 | ||
1f86805a | 1227 | /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */ |
89b0679b YZ |
1228 | pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end); |
1229 | pr_debug("reserved doorbell 0x%03x - 0x%03x\n", | |
1230 | range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET, | |
1231 | range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET); | |
1232 | ||
1f86805a | 1233 | for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) { |
89b0679b | 1234 | if (i >= range_start && i <= range_end) { |
ef568db7 | 1235 | set_bit(i, qpd->doorbell_bitmap); |
1f86805a YZ |
1236 | set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET, |
1237 | qpd->doorbell_bitmap); | |
ef568db7 | 1238 | } |
1f86805a | 1239 | } |
ef568db7 FK |
1240 | |
1241 | return 0; | |
1242 | } | |
1243 | ||
19f6d2a6 | 1244 | struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, |
093c7d8c | 1245 | struct kfd_process *p) |
19f6d2a6 OG |
1246 | { |
1247 | struct kfd_process_device *pdd = NULL; | |
1248 | ||
1249 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) | |
1250 | if (pdd->dev == dev) | |
733fa1f7 | 1251 | return pdd; |
093c7d8c | 1252 | |
733fa1f7 | 1253 | return NULL; |
093c7d8c AS |
1254 | } |
1255 | ||
1256 | struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, | |
1257 | struct kfd_process *p) | |
1258 | { | |
1259 | struct kfd_process_device *pdd = NULL; | |
1260 | ||
1261 | pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); | |
2d9b36f9 FK |
1262 | if (!pdd) |
1263 | return NULL; | |
1264 | ||
59d7115d MJ |
1265 | if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) { |
1266 | pr_err("Failed to alloc doorbell for pdd\n"); | |
1267 | goto err_free_pdd; | |
1268 | } | |
1269 | ||
ef568db7 FK |
1270 | if (init_doorbell_bitmap(&pdd->qpd, dev)) { |
1271 | pr_err("Failed to init doorbell for process\n"); | |
59d7115d | 1272 | goto err_free_pdd; |
ef568db7 FK |
1273 | } |
1274 | ||
2d9b36f9 FK |
1275 | pdd->dev = dev; |
1276 | INIT_LIST_HEAD(&pdd->qpd.queues_list); | |
1277 | INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); | |
1278 | pdd->qpd.dqm = dev->dqm; | |
1279 | pdd->qpd.pqm = &p->pqm; | |
26103436 | 1280 | pdd->qpd.evicted = 0; |
b8020b03 | 1281 | pdd->qpd.mapped_gws_queue = false; |
2d9b36f9 FK |
1282 | pdd->process = p; |
1283 | pdd->bound = PDD_UNBOUND; | |
1284 | pdd->already_dequeued = false; | |
9593f4d6 | 1285 | pdd->runtime_inuse = false; |
d4566dee | 1286 | pdd->vram_usage = 0; |
32cb59f3 | 1287 | pdd->sdma_past_activity_counter = 0; |
4327bed2 | 1288 | atomic64_set(&pdd->evict_duration_counter, 0); |
2d9b36f9 | 1289 | list_add(&pdd->per_device_list, &p->per_device_data); |
19f6d2a6 | 1290 | |
52b29d73 FK |
1291 | /* Init idr used for memory handle translation */ |
1292 | idr_init(&pdd->alloc_idr); | |
1293 | ||
b84394e2 | 1294 | return pdd; |
59d7115d MJ |
1295 | |
1296 | err_free_pdd: | |
1297 | kfree(pdd); | |
1298 | return NULL; | |
b84394e2 FK |
1299 | } |
1300 | ||
1301 | /** | |
1302 | * kfd_process_device_init_vm - Initialize a VM for a process-device | |
1303 | * | |
1304 | * @pdd: The process-device | |
1305 | * @drm_file: Optional pointer to a DRM file descriptor | |
1306 | * | |
1307 | * If @drm_file is specified, it will be used to acquire the VM from | |
1308 | * that file descriptor. If successful, the @pdd takes ownership of | |
1309 | * the file descriptor. | |
1310 | * | |
1311 | * If @drm_file is NULL, a new VM is created. | |
1312 | * | |
1313 | * Returns 0 on success, -errno on failure. | |
1314 | */ | |
1315 | int kfd_process_device_init_vm(struct kfd_process_device *pdd, | |
1316 | struct file *drm_file) | |
1317 | { | |
1318 | struct kfd_process *p; | |
1319 | struct kfd_dev *dev; | |
1320 | int ret; | |
1321 | ||
1322 | if (pdd->vm) | |
1323 | return drm_file ? -EBUSY : 0; | |
1324 | ||
1325 | p = pdd->process; | |
1326 | dev = pdd->dev; | |
1327 | ||
1328 | if (drm_file) | |
5b87245f | 1329 | ret = amdgpu_amdkfd_gpuvm_acquire_process_vm( |
1685b01a | 1330 | dev->kgd, drm_file, p->pasid, |
b84394e2 FK |
1331 | &pdd->vm, &p->kgd_process_info, &p->ef); |
1332 | else | |
5b87245f AL |
1333 | ret = amdgpu_amdkfd_gpuvm_create_process_vm(dev->kgd, p->pasid, |
1334 | &pdd->vm, &p->kgd_process_info, &p->ef); | |
b84394e2 | 1335 | if (ret) { |
403575c4 | 1336 | pr_err("Failed to create process VM object\n"); |
b84394e2 | 1337 | return ret; |
403575c4 | 1338 | } |
403575c4 | 1339 | |
f40c6912 YZ |
1340 | amdgpu_vm_set_task_info(pdd->vm); |
1341 | ||
552764b6 FK |
1342 | ret = kfd_process_device_reserve_ib_mem(pdd); |
1343 | if (ret) | |
1344 | goto err_reserve_ib_mem; | |
f35751b8 FK |
1345 | ret = kfd_process_device_init_cwsr_dgpu(pdd); |
1346 | if (ret) | |
1347 | goto err_init_cwsr; | |
1348 | ||
b84394e2 FK |
1349 | pdd->drm_file = drm_file; |
1350 | ||
1351 | return 0; | |
f35751b8 FK |
1352 | |
1353 | err_init_cwsr: | |
552764b6 | 1354 | err_reserve_ib_mem: |
f35751b8 FK |
1355 | kfd_process_device_free_bos(pdd); |
1356 | if (!drm_file) | |
5b87245f | 1357 | amdgpu_amdkfd_gpuvm_destroy_process_vm(dev->kgd, pdd->vm); |
f35751b8 FK |
1358 | pdd->vm = NULL; |
1359 | ||
1360 | return ret; | |
19f6d2a6 OG |
1361 | } |
1362 | ||
1363 | /* | |
1364 | * Direct the IOMMU to bind the process (specifically the pasid->mm) | |
1365 | * to the device. | |
1366 | * Unbinding occurs when the process dies or the device is removed. | |
1367 | * | |
1368 | * Assumes that the process lock is held. | |
1369 | */ | |
1370 | struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, | |
1371 | struct kfd_process *p) | |
1372 | { | |
093c7d8c | 1373 | struct kfd_process_device *pdd; |
b17f068a | 1374 | int err; |
19f6d2a6 | 1375 | |
093c7d8c AS |
1376 | pdd = kfd_get_process_device_data(dev, p); |
1377 | if (!pdd) { | |
1378 | pr_err("Process device data doesn't exist\n"); | |
19f6d2a6 | 1379 | return ERR_PTR(-ENOMEM); |
093c7d8c | 1380 | } |
19f6d2a6 | 1381 | |
9593f4d6 RB |
1382 | /* |
1383 | * signal runtime-pm system to auto resume and prevent | |
1384 | * further runtime suspend once device pdd is created until | |
1385 | * pdd is destroyed. | |
1386 | */ | |
1387 | if (!pdd->runtime_inuse) { | |
1388 | err = pm_runtime_get_sync(dev->ddev->dev); | |
1c1ada37 AD |
1389 | if (err < 0) { |
1390 | pm_runtime_put_autosuspend(dev->ddev->dev); | |
9593f4d6 | 1391 | return ERR_PTR(err); |
1c1ada37 | 1392 | } |
9593f4d6 RB |
1393 | } |
1394 | ||
64d1c3a4 FK |
1395 | err = kfd_iommu_bind_process_to_device(pdd); |
1396 | if (err) | |
9593f4d6 | 1397 | goto out; |
b17f068a | 1398 | |
b84394e2 FK |
1399 | err = kfd_process_device_init_vm(pdd, NULL); |
1400 | if (err) | |
9593f4d6 RB |
1401 | goto out; |
1402 | ||
1403 | /* | |
1404 | * make sure that runtime_usage counter is incremented just once | |
1405 | * per pdd | |
1406 | */ | |
1407 | pdd->runtime_inuse = true; | |
b84394e2 | 1408 | |
19f6d2a6 | 1409 | return pdd; |
9593f4d6 RB |
1410 | |
1411 | out: | |
1412 | /* balance runpm reference count and exit with error */ | |
1413 | if (!pdd->runtime_inuse) { | |
1414 | pm_runtime_mark_last_busy(dev->ddev->dev); | |
1415 | pm_runtime_put_autosuspend(dev->ddev->dev); | |
1416 | } | |
1417 | ||
1418 | return ERR_PTR(err); | |
19f6d2a6 OG |
1419 | } |
1420 | ||
8eabaf54 KR |
1421 | struct kfd_process_device *kfd_get_first_process_device_data( |
1422 | struct kfd_process *p) | |
19f6d2a6 OG |
1423 | { |
1424 | return list_first_entry(&p->per_device_data, | |
1425 | struct kfd_process_device, | |
1426 | per_device_list); | |
1427 | } | |
1428 | ||
8eabaf54 KR |
1429 | struct kfd_process_device *kfd_get_next_process_device_data( |
1430 | struct kfd_process *p, | |
19f6d2a6 OG |
1431 | struct kfd_process_device *pdd) |
1432 | { | |
1433 | if (list_is_last(&pdd->per_device_list, &p->per_device_data)) | |
1434 | return NULL; | |
1435 | return list_next_entry(pdd, per_device_list); | |
1436 | } | |
1437 | ||
1438 | bool kfd_has_process_device_data(struct kfd_process *p) | |
1439 | { | |
1440 | return !(list_empty(&p->per_device_data)); | |
1441 | } | |
f3a39818 | 1442 | |
52b29d73 FK |
1443 | /* Create specific handle mapped to mem from process local memory idr |
1444 | * Assumes that the process lock is held. | |
1445 | */ | |
1446 | int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, | |
1447 | void *mem) | |
1448 | { | |
1449 | return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL); | |
1450 | } | |
1451 | ||
1452 | /* Translate specific handle from process local memory idr | |
1453 | * Assumes that the process lock is held. | |
1454 | */ | |
1455 | void *kfd_process_device_translate_handle(struct kfd_process_device *pdd, | |
1456 | int handle) | |
1457 | { | |
1458 | if (handle < 0) | |
1459 | return NULL; | |
1460 | ||
1461 | return idr_find(&pdd->alloc_idr, handle); | |
1462 | } | |
1463 | ||
1464 | /* Remove specific handle from process local memory idr | |
1465 | * Assumes that the process lock is held. | |
1466 | */ | |
1467 | void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, | |
1468 | int handle) | |
1469 | { | |
1470 | if (handle >= 0) | |
1471 | idr_remove(&pdd->alloc_idr, handle); | |
1472 | } | |
1473 | ||
abb208a8 | 1474 | /* This increments the process->ref counter. */ |
c7b6bac9 | 1475 | struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid) |
f3a39818 | 1476 | { |
82c16b42 | 1477 | struct kfd_process *p, *ret_p = NULL; |
f3a39818 AL |
1478 | unsigned int temp; |
1479 | ||
1480 | int idx = srcu_read_lock(&kfd_processes_srcu); | |
1481 | ||
1482 | hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { | |
1483 | if (p->pasid == pasid) { | |
abb208a8 | 1484 | kref_get(&p->ref); |
82c16b42 | 1485 | ret_p = p; |
f3a39818 AL |
1486 | break; |
1487 | } | |
1488 | } | |
1489 | ||
1490 | srcu_read_unlock(&kfd_processes_srcu, idx); | |
1491 | ||
82c16b42 | 1492 | return ret_p; |
f3a39818 | 1493 | } |
373d7080 | 1494 | |
26103436 FK |
1495 | /* This increments the process->ref counter. */ |
1496 | struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm) | |
1497 | { | |
1498 | struct kfd_process *p; | |
1499 | ||
1500 | int idx = srcu_read_lock(&kfd_processes_srcu); | |
1501 | ||
1502 | p = find_process_by_mm(mm); | |
1503 | if (p) | |
1504 | kref_get(&p->ref); | |
1505 | ||
1506 | srcu_read_unlock(&kfd_processes_srcu, idx); | |
1507 | ||
1508 | return p; | |
1509 | } | |
1510 | ||
0aeaaf64 | 1511 | /* kfd_process_evict_queues - Evict all user queues of a process |
26103436 FK |
1512 | * |
1513 | * Eviction is reference-counted per process-device. This means multiple | |
1514 | * evictions from different sources can be nested safely. | |
1515 | */ | |
6b95e797 | 1516 | int kfd_process_evict_queues(struct kfd_process *p) |
26103436 FK |
1517 | { |
1518 | struct kfd_process_device *pdd; | |
1519 | int r = 0; | |
1520 | unsigned int n_evicted = 0; | |
1521 | ||
1522 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) { | |
1523 | r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, | |
1524 | &pdd->qpd); | |
1525 | if (r) { | |
1526 | pr_err("Failed to evict process queues\n"); | |
1527 | goto fail; | |
1528 | } | |
1529 | n_evicted++; | |
1530 | } | |
1531 | ||
1532 | return r; | |
1533 | ||
1534 | fail: | |
1535 | /* To keep state consistent, roll back partial eviction by | |
1536 | * restoring queues | |
1537 | */ | |
1538 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) { | |
1539 | if (n_evicted == 0) | |
1540 | break; | |
1541 | if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, | |
1542 | &pdd->qpd)) | |
1543 | pr_err("Failed to restore queues\n"); | |
1544 | ||
1545 | n_evicted--; | |
1546 | } | |
1547 | ||
1548 | return r; | |
1549 | } | |
1550 | ||
0aeaaf64 | 1551 | /* kfd_process_restore_queues - Restore all user queues of a process */ |
6b95e797 | 1552 | int kfd_process_restore_queues(struct kfd_process *p) |
26103436 FK |
1553 | { |
1554 | struct kfd_process_device *pdd; | |
1555 | int r, ret = 0; | |
1556 | ||
1557 | list_for_each_entry(pdd, &p->per_device_data, per_device_list) { | |
1558 | r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, | |
1559 | &pdd->qpd); | |
1560 | if (r) { | |
1561 | pr_err("Failed to restore process queues\n"); | |
1562 | if (!ret) | |
1563 | ret = r; | |
1564 | } | |
1565 | } | |
1566 | ||
1567 | return ret; | |
1568 | } | |
1569 | ||
1570 | static void evict_process_worker(struct work_struct *work) | |
1571 | { | |
1572 | int ret; | |
1573 | struct kfd_process *p; | |
1574 | struct delayed_work *dwork; | |
1575 | ||
1576 | dwork = to_delayed_work(work); | |
1577 | ||
1578 | /* Process termination destroys this worker thread. So during the | |
1579 | * lifetime of this thread, kfd_process p will be valid | |
1580 | */ | |
1581 | p = container_of(dwork, struct kfd_process, eviction_work); | |
1582 | WARN_ONCE(p->last_eviction_seqno != p->ef->seqno, | |
1583 | "Eviction fence mismatch\n"); | |
1584 | ||
1585 | /* Narrow window of overlap between restore and evict work | |
1586 | * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos | |
1587 | * unreserves KFD BOs, it is possible to evicted again. But | |
1588 | * restore has few more steps of finish. So lets wait for any | |
1589 | * previous restore work to complete | |
1590 | */ | |
1591 | flush_delayed_work(&p->restore_work); | |
1592 | ||
6027b1bf | 1593 | pr_debug("Started evicting pasid 0x%x\n", p->pasid); |
6b95e797 | 1594 | ret = kfd_process_evict_queues(p); |
26103436 FK |
1595 | if (!ret) { |
1596 | dma_fence_signal(p->ef); | |
1597 | dma_fence_put(p->ef); | |
1598 | p->ef = NULL; | |
1679ae8f | 1599 | queue_delayed_work(kfd_restore_wq, &p->restore_work, |
26103436 FK |
1600 | msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)); |
1601 | ||
6027b1bf | 1602 | pr_debug("Finished evicting pasid 0x%x\n", p->pasid); |
26103436 | 1603 | } else |
6027b1bf | 1604 | pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid); |
26103436 FK |
1605 | } |
1606 | ||
1607 | static void restore_process_worker(struct work_struct *work) | |
1608 | { | |
1609 | struct delayed_work *dwork; | |
1610 | struct kfd_process *p; | |
26103436 FK |
1611 | int ret = 0; |
1612 | ||
1613 | dwork = to_delayed_work(work); | |
1614 | ||
1615 | /* Process termination destroys this worker thread. So during the | |
1616 | * lifetime of this thread, kfd_process p will be valid | |
1617 | */ | |
1618 | p = container_of(dwork, struct kfd_process, restore_work); | |
6027b1bf | 1619 | pr_debug("Started restoring pasid 0x%x\n", p->pasid); |
26103436 FK |
1620 | |
1621 | /* Setting last_restore_timestamp before successful restoration. | |
1622 | * Otherwise this would have to be set by KGD (restore_process_bos) | |
1623 | * before KFD BOs are unreserved. If not, the process can be evicted | |
1624 | * again before the timestamp is set. | |
1625 | * If restore fails, the timestamp will be set again in the next | |
1626 | * attempt. This would mean that the minimum GPU quanta would be | |
1627 | * PROCESS_ACTIVE_TIME_MS - (time to execute the following two | |
1628 | * functions) | |
1629 | */ | |
1630 | ||
1631 | p->last_restore_timestamp = get_jiffies_64(); | |
5b87245f | 1632 | ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info, |
26103436 FK |
1633 | &p->ef); |
1634 | if (ret) { | |
6027b1bf | 1635 | pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n", |
26103436 | 1636 | p->pasid, PROCESS_BACK_OFF_TIME_MS); |
1679ae8f | 1637 | ret = queue_delayed_work(kfd_restore_wq, &p->restore_work, |
26103436 FK |
1638 | msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS)); |
1639 | WARN(!ret, "reschedule restore work failed\n"); | |
1640 | return; | |
1641 | } | |
1642 | ||
6b95e797 | 1643 | ret = kfd_process_restore_queues(p); |
26103436 | 1644 | if (!ret) |
6027b1bf | 1645 | pr_debug("Finished restoring pasid 0x%x\n", p->pasid); |
26103436 | 1646 | else |
6027b1bf | 1647 | pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid); |
26103436 FK |
1648 | } |
1649 | ||
1650 | void kfd_suspend_all_processes(void) | |
1651 | { | |
1652 | struct kfd_process *p; | |
1653 | unsigned int temp; | |
1654 | int idx = srcu_read_lock(&kfd_processes_srcu); | |
1655 | ||
8a491bb3 | 1656 | WARN(debug_evictions, "Evicting all processes"); |
26103436 FK |
1657 | hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { |
1658 | cancel_delayed_work_sync(&p->eviction_work); | |
1659 | cancel_delayed_work_sync(&p->restore_work); | |
1660 | ||
6b95e797 | 1661 | if (kfd_process_evict_queues(p)) |
6027b1bf | 1662 | pr_err("Failed to suspend process 0x%x\n", p->pasid); |
26103436 FK |
1663 | dma_fence_signal(p->ef); |
1664 | dma_fence_put(p->ef); | |
1665 | p->ef = NULL; | |
1666 | } | |
1667 | srcu_read_unlock(&kfd_processes_srcu, idx); | |
1668 | } | |
1669 | ||
1670 | int kfd_resume_all_processes(void) | |
1671 | { | |
1672 | struct kfd_process *p; | |
1673 | unsigned int temp; | |
1674 | int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu); | |
1675 | ||
1676 | hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { | |
1679ae8f | 1677 | if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) { |
26103436 FK |
1678 | pr_err("Restore process %d failed during resume\n", |
1679 | p->pasid); | |
1680 | ret = -EFAULT; | |
1681 | } | |
1682 | } | |
1683 | srcu_read_unlock(&kfd_processes_srcu, idx); | |
1684 | return ret; | |
1685 | } | |
1686 | ||
df03ef93 | 1687 | int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process, |
373d7080 FK |
1688 | struct vm_area_struct *vma) |
1689 | { | |
373d7080 FK |
1690 | struct kfd_process_device *pdd; |
1691 | struct qcm_process_device *qpd; | |
1692 | ||
373d7080 FK |
1693 | if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) { |
1694 | pr_err("Incorrect CWSR mapping size.\n"); | |
1695 | return -EINVAL; | |
1696 | } | |
1697 | ||
1698 | pdd = kfd_get_process_device_data(dev, process); | |
1699 | if (!pdd) | |
1700 | return -EINVAL; | |
1701 | qpd = &pdd->qpd; | |
1702 | ||
1703 | qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, | |
1704 | get_order(KFD_CWSR_TBA_TMA_SIZE)); | |
1705 | if (!qpd->cwsr_kaddr) { | |
1706 | pr_err("Error allocating per process CWSR buffer.\n"); | |
1707 | return -ENOMEM; | |
1708 | } | |
1709 | ||
1710 | vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | |
1711 | | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP; | |
1712 | /* Mapping pages to user process */ | |
1713 | return remap_pfn_range(vma, vma->vm_start, | |
1714 | PFN_DOWN(__pa(qpd->cwsr_kaddr)), | |
1715 | KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot); | |
1716 | } | |
851a645e | 1717 | |
403575c4 FK |
1718 | void kfd_flush_tlb(struct kfd_process_device *pdd) |
1719 | { | |
1720 | struct kfd_dev *dev = pdd->dev; | |
403575c4 FK |
1721 | |
1722 | if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { | |
1723 | /* Nothing to flush until a VMID is assigned, which | |
1724 | * only happens when the first queue is created. | |
1725 | */ | |
1726 | if (pdd->qpd.vmid) | |
ffa02269 AS |
1727 | amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd, |
1728 | pdd->qpd.vmid); | |
403575c4 | 1729 | } else { |
ffa02269 AS |
1730 | amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd, |
1731 | pdd->process->pasid); | |
403575c4 FK |
1732 | } |
1733 | } | |
1734 | ||
851a645e FK |
1735 | #if defined(CONFIG_DEBUG_FS) |
1736 | ||
1737 | int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data) | |
1738 | { | |
1739 | struct kfd_process *p; | |
1740 | unsigned int temp; | |
1741 | int r = 0; | |
1742 | ||
1743 | int idx = srcu_read_lock(&kfd_processes_srcu); | |
1744 | ||
1745 | hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { | |
6027b1bf | 1746 | seq_printf(m, "Process %d PASID 0x%x:\n", |
851a645e FK |
1747 | p->lead_thread->tgid, p->pasid); |
1748 | ||
1749 | mutex_lock(&p->mutex); | |
1750 | r = pqm_debugfs_mqds(m, &p->pqm); | |
1751 | mutex_unlock(&p->mutex); | |
1752 | ||
1753 | if (r) | |
1754 | break; | |
1755 | } | |
1756 | ||
1757 | srcu_read_unlock(&kfd_processes_srcu, idx); | |
1758 | ||
1759 | return r; | |
1760 | } | |
1761 | ||
1762 | #endif | |
14328aa5 | 1763 |