Commit | Line | Data |
---|---|---|
4a488a7a OG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #include <linux/device.h> | |
24 | #include <linux/export.h> | |
25 | #include <linux/err.h> | |
26 | #include <linux/fs.h> | |
27 | #include <linux/sched.h> | |
28 | #include <linux/slab.h> | |
29 | #include <linux/uaccess.h> | |
30 | #include <linux/compat.h> | |
31 | #include <uapi/linux/kfd_ioctl.h> | |
32 | #include <linux/time.h> | |
33 | #include <linux/mm.h> | |
34 | #include <linux/uaccess.h> | |
35 | #include <uapi/asm-generic/mman-common.h> | |
36 | #include <asm/processor.h> | |
37 | #include "kfd_priv.h" | |
41a286fa | 38 | #include "kfd_device_queue_manager.h" |
4a488a7a OG |
39 | |
40 | static long kfd_ioctl(struct file *, unsigned int, unsigned long); | |
41 | static int kfd_open(struct inode *, struct file *); | |
19f6d2a6 | 42 | static int kfd_mmap(struct file *, struct vm_area_struct *); |
4a488a7a OG |
43 | |
44 | static const char kfd_dev_name[] = "kfd"; | |
45 | ||
46 | static const struct file_operations kfd_fops = { | |
47 | .owner = THIS_MODULE, | |
48 | .unlocked_ioctl = kfd_ioctl, | |
49 | .compat_ioctl = kfd_ioctl, | |
50 | .open = kfd_open, | |
19f6d2a6 | 51 | .mmap = kfd_mmap, |
4a488a7a OG |
52 | }; |
53 | ||
54 | static int kfd_char_dev_major = -1; | |
55 | static struct class *kfd_class; | |
56 | struct device *kfd_device; | |
57 | ||
58 | int kfd_chardev_init(void) | |
59 | { | |
60 | int err = 0; | |
61 | ||
62 | kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); | |
63 | err = kfd_char_dev_major; | |
64 | if (err < 0) | |
65 | goto err_register_chrdev; | |
66 | ||
67 | kfd_class = class_create(THIS_MODULE, kfd_dev_name); | |
68 | err = PTR_ERR(kfd_class); | |
69 | if (IS_ERR(kfd_class)) | |
70 | goto err_class_create; | |
71 | ||
72 | kfd_device = device_create(kfd_class, NULL, | |
73 | MKDEV(kfd_char_dev_major, 0), | |
74 | NULL, kfd_dev_name); | |
75 | err = PTR_ERR(kfd_device); | |
76 | if (IS_ERR(kfd_device)) | |
77 | goto err_device_create; | |
78 | ||
79 | return 0; | |
80 | ||
81 | err_device_create: | |
82 | class_destroy(kfd_class); | |
83 | err_class_create: | |
84 | unregister_chrdev(kfd_char_dev_major, kfd_dev_name); | |
85 | err_register_chrdev: | |
86 | return err; | |
87 | } | |
88 | ||
89 | void kfd_chardev_exit(void) | |
90 | { | |
91 | device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); | |
92 | class_destroy(kfd_class); | |
93 | unregister_chrdev(kfd_char_dev_major, kfd_dev_name); | |
94 | } | |
95 | ||
96 | struct device *kfd_chardev(void) | |
97 | { | |
98 | return kfd_device; | |
99 | } | |
100 | ||
101 | ||
102 | static int kfd_open(struct inode *inode, struct file *filep) | |
103 | { | |
19f6d2a6 OG |
104 | struct kfd_process *process; |
105 | ||
4a488a7a OG |
106 | if (iminor(inode) != 0) |
107 | return -ENODEV; | |
108 | ||
19f6d2a6 OG |
109 | process = kfd_create_process(current); |
110 | if (IS_ERR(process)) | |
111 | return PTR_ERR(process); | |
112 | ||
113 | process->is_32bit_user_mode = is_compat_task(); | |
114 | ||
115 | dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", | |
116 | process->pasid, process->is_32bit_user_mode); | |
117 | ||
118 | kfd_init_apertures(process); | |
119 | ||
4a488a7a OG |
120 | return 0; |
121 | } | |
122 | ||
123 | static long kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, | |
124 | void __user *arg) | |
125 | { | |
126 | return -ENODEV; | |
127 | } | |
128 | ||
39b027d9 OG |
129 | static int set_queue_properties_from_user(struct queue_properties *q_properties, |
130 | struct kfd_ioctl_create_queue_args *args) | |
131 | { | |
132 | if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { | |
133 | pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); | |
134 | return -EINVAL; | |
135 | } | |
136 | ||
137 | if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { | |
138 | pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); | |
139 | return -EINVAL; | |
140 | } | |
141 | ||
142 | if ((args->ring_base_address) && | |
143 | (!access_ok(VERIFY_WRITE, args->ring_base_address, sizeof(uint64_t)))) { | |
144 | pr_err("kfd: can't access ring base address\n"); | |
145 | return -EFAULT; | |
146 | } | |
147 | ||
148 | if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { | |
149 | pr_err("kfd: ring size must be a power of 2 or 0\n"); | |
150 | return -EINVAL; | |
151 | } | |
152 | ||
153 | if (!access_ok(VERIFY_WRITE, args->read_pointer_address, sizeof(uint32_t))) { | |
154 | pr_err("kfd: can't access read pointer\n"); | |
155 | return -EFAULT; | |
156 | } | |
157 | ||
158 | if (!access_ok(VERIFY_WRITE, args->write_pointer_address, sizeof(uint32_t))) { | |
159 | pr_err("kfd: can't access write pointer\n"); | |
160 | return -EFAULT; | |
161 | } | |
162 | ||
163 | q_properties->is_interop = false; | |
164 | q_properties->queue_percent = args->queue_percentage; | |
165 | q_properties->priority = args->queue_priority; | |
166 | q_properties->queue_address = args->ring_base_address; | |
167 | q_properties->queue_size = args->ring_size; | |
168 | q_properties->read_ptr = (uint32_t *) args->read_pointer_address; | |
169 | q_properties->write_ptr = (uint32_t *) args->write_pointer_address; | |
170 | if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || | |
171 | args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) | |
172 | q_properties->type = KFD_QUEUE_TYPE_COMPUTE; | |
173 | else | |
174 | return -ENOTSUPP; | |
175 | ||
176 | if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) | |
177 | q_properties->format = KFD_QUEUE_FORMAT_AQL; | |
178 | else | |
179 | q_properties->format = KFD_QUEUE_FORMAT_PM4; | |
180 | ||
181 | pr_debug("Queue Percentage (%d, %d)\n", | |
182 | q_properties->queue_percent, args->queue_percentage); | |
183 | ||
184 | pr_debug("Queue Priority (%d, %d)\n", | |
185 | q_properties->priority, args->queue_priority); | |
186 | ||
187 | pr_debug("Queue Address (0x%llX, 0x%llX)\n", | |
188 | q_properties->queue_address, args->ring_base_address); | |
189 | ||
190 | pr_debug("Queue Size (0x%llX, %u)\n", | |
191 | q_properties->queue_size, args->ring_size); | |
192 | ||
193 | pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n", | |
194 | (uint64_t) q_properties->read_ptr, | |
195 | (uint64_t) q_properties->write_ptr); | |
196 | ||
197 | pr_debug("Queue Format (%d)\n", q_properties->format); | |
198 | ||
199 | return 0; | |
200 | } | |
201 | ||
4a488a7a OG |
202 | static long kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, |
203 | void __user *arg) | |
204 | { | |
39b027d9 OG |
205 | struct kfd_ioctl_create_queue_args args; |
206 | struct kfd_dev *dev; | |
207 | int err = 0; | |
208 | unsigned int queue_id; | |
209 | struct kfd_process_device *pdd; | |
210 | struct queue_properties q_properties; | |
211 | ||
212 | memset(&q_properties, 0, sizeof(struct queue_properties)); | |
213 | ||
214 | if (copy_from_user(&args, arg, sizeof(args))) | |
215 | return -EFAULT; | |
216 | ||
217 | pr_debug("kfd: creating queue ioctl\n"); | |
218 | ||
219 | err = set_queue_properties_from_user(&q_properties, &args); | |
220 | if (err) | |
221 | return err; | |
222 | ||
223 | dev = kfd_device_by_id(args.gpu_id); | |
224 | if (dev == NULL) | |
225 | return -EINVAL; | |
226 | ||
227 | mutex_lock(&p->mutex); | |
228 | ||
229 | pdd = kfd_bind_process_to_device(dev, p); | |
230 | if (IS_ERR(pdd) < 0) { | |
231 | err = PTR_ERR(pdd); | |
232 | goto err_bind_process; | |
233 | } | |
234 | ||
235 | pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n", | |
236 | p->pasid, | |
237 | dev->id); | |
238 | ||
239 | err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, 0, | |
240 | KFD_QUEUE_TYPE_COMPUTE, &queue_id); | |
241 | if (err != 0) | |
242 | goto err_create_queue; | |
243 | ||
244 | args.queue_id = queue_id; | |
245 | ||
246 | /* Return gpu_id as doorbell offset for mmap usage */ | |
247 | args.doorbell_offset = args.gpu_id << PAGE_SHIFT; | |
248 | ||
249 | if (copy_to_user(arg, &args, sizeof(args))) { | |
250 | err = -EFAULT; | |
251 | goto err_copy_args_out; | |
252 | } | |
253 | ||
254 | mutex_unlock(&p->mutex); | |
255 | ||
256 | pr_debug("kfd: queue id %d was created successfully\n", args.queue_id); | |
257 | ||
258 | pr_debug("ring buffer address == 0x%016llX\n", | |
259 | args.ring_base_address); | |
260 | ||
261 | pr_debug("read ptr address == 0x%016llX\n", | |
262 | args.read_pointer_address); | |
263 | ||
264 | pr_debug("write ptr address == 0x%016llX\n", | |
265 | args.write_pointer_address); | |
266 | ||
267 | return 0; | |
268 | ||
269 | err_copy_args_out: | |
270 | pqm_destroy_queue(&p->pqm, queue_id); | |
271 | err_create_queue: | |
272 | err_bind_process: | |
273 | mutex_unlock(&p->mutex); | |
274 | return err; | |
4a488a7a OG |
275 | } |
276 | ||
277 | static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, | |
278 | void __user *arg) | |
279 | { | |
39b027d9 OG |
280 | int retval; |
281 | struct kfd_ioctl_destroy_queue_args args; | |
282 | ||
283 | if (copy_from_user(&args, arg, sizeof(args))) | |
284 | return -EFAULT; | |
285 | ||
286 | pr_debug("kfd: destroying queue id %d for PASID %d\n", | |
287 | args.queue_id, | |
288 | p->pasid); | |
289 | ||
290 | mutex_lock(&p->mutex); | |
291 | ||
292 | retval = pqm_destroy_queue(&p->pqm, args.queue_id); | |
293 | ||
294 | mutex_unlock(&p->mutex); | |
295 | return retval; | |
4a488a7a OG |
296 | } |
297 | ||
298 | static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, | |
299 | void __user *arg) | |
300 | { | |
39b027d9 OG |
301 | int retval; |
302 | struct kfd_ioctl_update_queue_args args; | |
303 | struct queue_properties properties; | |
304 | ||
305 | if (copy_from_user(&args, arg, sizeof(args))) | |
306 | return -EFAULT; | |
307 | ||
308 | if (args.queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { | |
309 | pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); | |
310 | return -EINVAL; | |
311 | } | |
312 | ||
313 | if (args.queue_priority > KFD_MAX_QUEUE_PRIORITY) { | |
314 | pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); | |
315 | return -EINVAL; | |
316 | } | |
317 | ||
318 | if ((args.ring_base_address) && | |
319 | (!access_ok(VERIFY_WRITE, args.ring_base_address, sizeof(uint64_t)))) { | |
320 | pr_err("kfd: can't access ring base address\n"); | |
321 | return -EFAULT; | |
322 | } | |
323 | ||
324 | if (!is_power_of_2(args.ring_size) && (args.ring_size != 0)) { | |
325 | pr_err("kfd: ring size must be a power of 2 or 0\n"); | |
326 | return -EINVAL; | |
327 | } | |
328 | ||
329 | properties.queue_address = args.ring_base_address; | |
330 | properties.queue_size = args.ring_size; | |
331 | properties.queue_percent = args.queue_percentage; | |
332 | properties.priority = args.queue_priority; | |
333 | ||
334 | pr_debug("kfd: updating queue id %d for PASID %d\n", | |
335 | args.queue_id, p->pasid); | |
336 | ||
337 | mutex_lock(&p->mutex); | |
338 | ||
339 | retval = pqm_update_queue(&p->pqm, args.queue_id, &properties); | |
340 | ||
341 | mutex_unlock(&p->mutex); | |
342 | ||
343 | return retval; | |
4a488a7a OG |
344 | } |
345 | ||
346 | static long kfd_ioctl_set_memory_policy(struct file *filep, | |
347 | struct kfd_process *p, void __user *arg) | |
348 | { | |
41a286fa AL |
349 | struct kfd_ioctl_set_memory_policy_args args; |
350 | struct kfd_dev *dev; | |
351 | int err = 0; | |
352 | struct kfd_process_device *pdd; | |
353 | enum cache_policy default_policy, alternate_policy; | |
354 | ||
355 | if (copy_from_user(&args, arg, sizeof(args))) | |
356 | return -EFAULT; | |
357 | ||
358 | if (args.default_policy != KFD_IOC_CACHE_POLICY_COHERENT | |
359 | && args.default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | |
360 | return -EINVAL; | |
361 | } | |
362 | ||
363 | if (args.alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT | |
364 | && args.alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { | |
365 | return -EINVAL; | |
366 | } | |
367 | ||
368 | dev = kfd_device_by_id(args.gpu_id); | |
369 | if (dev == NULL) | |
370 | return -EINVAL; | |
371 | ||
372 | mutex_lock(&p->mutex); | |
373 | ||
374 | pdd = kfd_bind_process_to_device(dev, p); | |
375 | if (IS_ERR(pdd) < 0) { | |
376 | err = PTR_ERR(pdd); | |
377 | goto out; | |
378 | } | |
379 | ||
380 | default_policy = (args.default_policy == KFD_IOC_CACHE_POLICY_COHERENT) | |
381 | ? cache_policy_coherent : cache_policy_noncoherent; | |
382 | ||
383 | alternate_policy = | |
384 | (args.alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) | |
385 | ? cache_policy_coherent : cache_policy_noncoherent; | |
386 | ||
387 | if (!dev->dqm->set_cache_memory_policy(dev->dqm, | |
388 | &pdd->qpd, | |
389 | default_policy, | |
390 | alternate_policy, | |
391 | (void __user *)args.alternate_aperture_base, | |
392 | args.alternate_aperture_size)) | |
393 | err = -EINVAL; | |
394 | ||
395 | out: | |
396 | mutex_unlock(&p->mutex); | |
397 | ||
398 | return err; | |
4a488a7a OG |
399 | } |
400 | ||
401 | static long kfd_ioctl_get_clock_counters(struct file *filep, | |
402 | struct kfd_process *p, void __user *arg) | |
403 | { | |
4fac47c8 EP |
404 | struct kfd_ioctl_get_clock_counters_args args; |
405 | struct kfd_dev *dev; | |
406 | struct timespec time; | |
407 | ||
408 | if (copy_from_user(&args, arg, sizeof(args))) | |
409 | return -EFAULT; | |
410 | ||
411 | dev = kfd_device_by_id(args.gpu_id); | |
412 | if (dev == NULL) | |
413 | return -EINVAL; | |
414 | ||
415 | /* Reading GPU clock counter from KGD */ | |
416 | args.gpu_clock_counter = kfd2kgd->get_gpu_clock_counter(dev->kgd); | |
417 | ||
418 | /* No access to rdtsc. Using raw monotonic time */ | |
419 | getrawmonotonic(&time); | |
420 | args.cpu_clock_counter = (uint64_t)timespec_to_ns(&time); | |
421 | ||
422 | get_monotonic_boottime(&time); | |
423 | args.system_clock_counter = (uint64_t)timespec_to_ns(&time); | |
424 | ||
425 | /* Since the counter is in nano-seconds we use 1GHz frequency */ | |
426 | args.system_clock_freq = 1000000000; | |
427 | ||
428 | if (copy_to_user(arg, &args, sizeof(args))) | |
429 | return -EFAULT; | |
430 | ||
431 | return 0; | |
4a488a7a OG |
432 | } |
433 | ||
434 | ||
435 | static int kfd_ioctl_get_process_apertures(struct file *filp, | |
436 | struct kfd_process *p, void __user *arg) | |
437 | { | |
438 | return -ENODEV; | |
439 | } | |
440 | ||
441 | static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) | |
442 | { | |
443 | struct kfd_process *process; | |
444 | long err = -EINVAL; | |
445 | ||
446 | dev_dbg(kfd_device, | |
447 | "ioctl cmd 0x%x (#%d), arg 0x%lx\n", | |
448 | cmd, _IOC_NR(cmd), arg); | |
449 | ||
19f6d2a6 OG |
450 | process = kfd_get_process(current); |
451 | if (IS_ERR(process)) | |
452 | return PTR_ERR(process); | |
4a488a7a OG |
453 | |
454 | switch (cmd) { | |
455 | case KFD_IOC_GET_VERSION: | |
456 | err = kfd_ioctl_get_version(filep, process, (void __user *)arg); | |
457 | break; | |
458 | case KFD_IOC_CREATE_QUEUE: | |
459 | err = kfd_ioctl_create_queue(filep, process, | |
460 | (void __user *)arg); | |
461 | break; | |
462 | ||
463 | case KFD_IOC_DESTROY_QUEUE: | |
464 | err = kfd_ioctl_destroy_queue(filep, process, | |
465 | (void __user *)arg); | |
466 | break; | |
467 | ||
468 | case KFD_IOC_SET_MEMORY_POLICY: | |
469 | err = kfd_ioctl_set_memory_policy(filep, process, | |
470 | (void __user *)arg); | |
471 | break; | |
472 | ||
473 | case KFD_IOC_GET_CLOCK_COUNTERS: | |
474 | err = kfd_ioctl_get_clock_counters(filep, process, | |
475 | (void __user *)arg); | |
476 | break; | |
477 | ||
478 | case KFD_IOC_GET_PROCESS_APERTURES: | |
479 | err = kfd_ioctl_get_process_apertures(filep, process, | |
480 | (void __user *)arg); | |
481 | break; | |
482 | ||
483 | case KFD_IOC_UPDATE_QUEUE: | |
484 | err = kfd_ioctl_update_queue(filep, process, | |
485 | (void __user *)arg); | |
486 | break; | |
487 | ||
488 | default: | |
489 | dev_err(kfd_device, | |
490 | "unknown ioctl cmd 0x%x, arg 0x%lx)\n", | |
491 | cmd, arg); | |
492 | err = -EINVAL; | |
493 | break; | |
494 | } | |
495 | ||
496 | if (err < 0) | |
497 | dev_err(kfd_device, | |
498 | "ioctl error %ld for ioctl cmd 0x%x (#%d)\n", | |
499 | err, cmd, _IOC_NR(cmd)); | |
500 | ||
501 | return err; | |
502 | } | |
19f6d2a6 OG |
503 | |
504 | static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) | |
505 | { | |
506 | struct kfd_process *process; | |
507 | ||
508 | process = kfd_get_process(current); | |
509 | if (IS_ERR(process)) | |
510 | return PTR_ERR(process); | |
511 | ||
512 | return kfd_doorbell_mmap(process, vma); | |
513 | } |