2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/bsearch.h>
24 #include <linux/pci.h>
25 #include <linux/slab.h>
27 #include "kfd_device_queue_manager.h"
28 #include "kfd_pm4_headers_vi.h"
29 #include "cwsr_trap_handler.h"
30 #include "kfd_iommu.h"
31 #include "amdgpu_amdkfd.h"
33 #define MQD_SIZE_ALIGNED 768
36 * kfd_locked is used to lock the kfd driver during suspend or reset
37 * once locked, kfd driver will stop any further GPU execution.
38 * create process (open) will return -EAGAIN.
40 static atomic_t kfd_locked = ATOMIC_INIT(0);
42 #ifdef CONFIG_DRM_AMDGPU_CIK
43 extern const struct kfd2kgd_calls gfx_v7_kfd2kgd;
45 extern const struct kfd2kgd_calls gfx_v8_kfd2kgd;
46 extern const struct kfd2kgd_calls gfx_v9_kfd2kgd;
47 extern const struct kfd2kgd_calls arcturus_kfd2kgd;
48 extern const struct kfd2kgd_calls gfx_v10_kfd2kgd;
50 static const struct kfd2kgd_calls *kfd2kgd_funcs[] = {
51 #ifdef KFD_SUPPORT_IOMMU_V2
52 #ifdef CONFIG_DRM_AMDGPU_CIK
53 [CHIP_KAVERI] = &gfx_v7_kfd2kgd,
55 [CHIP_CARRIZO] = &gfx_v8_kfd2kgd,
56 [CHIP_RAVEN] = &gfx_v9_kfd2kgd,
58 #ifdef CONFIG_DRM_AMDGPU_CIK
59 [CHIP_HAWAII] = &gfx_v7_kfd2kgd,
61 [CHIP_TONGA] = &gfx_v8_kfd2kgd,
62 [CHIP_FIJI] = &gfx_v8_kfd2kgd,
63 [CHIP_POLARIS10] = &gfx_v8_kfd2kgd,
64 [CHIP_POLARIS11] = &gfx_v8_kfd2kgd,
65 [CHIP_POLARIS12] = &gfx_v8_kfd2kgd,
66 [CHIP_VEGAM] = &gfx_v8_kfd2kgd,
67 [CHIP_VEGA10] = &gfx_v9_kfd2kgd,
68 [CHIP_VEGA12] = &gfx_v9_kfd2kgd,
69 [CHIP_VEGA20] = &gfx_v9_kfd2kgd,
70 [CHIP_RENOIR] = &gfx_v9_kfd2kgd,
71 [CHIP_ARCTURUS] = &arcturus_kfd2kgd,
72 [CHIP_NAVI10] = &gfx_v10_kfd2kgd,
73 [CHIP_NAVI12] = &gfx_v10_kfd2kgd,
74 [CHIP_NAVI14] = &gfx_v10_kfd2kgd,
77 #ifdef KFD_SUPPORT_IOMMU_V2
78 static const struct kfd_device_info kaveri_device_info = {
79 .asic_family = CHIP_KAVERI,
80 .asic_name = "kaveri",
82 /* max num of queues for KV.TODO should be a dynamic value */
85 .ih_ring_entry_size = 4 * sizeof(uint32_t),
86 .event_interrupt_class = &event_interrupt_class_cik,
87 .num_of_watch_points = 4,
88 .mqd_size_aligned = MQD_SIZE_ALIGNED,
89 .supports_cwsr = false,
90 .needs_iommu_device = true,
91 .needs_pci_atomics = false,
92 .num_sdma_engines = 2,
93 .num_xgmi_sdma_engines = 0,
94 .num_sdma_queues_per_engine = 2,
97 static const struct kfd_device_info carrizo_device_info = {
98 .asic_family = CHIP_CARRIZO,
99 .asic_name = "carrizo",
100 .max_pasid_bits = 16,
101 /* max num of queues for CZ.TODO should be a dynamic value */
104 .ih_ring_entry_size = 4 * sizeof(uint32_t),
105 .event_interrupt_class = &event_interrupt_class_cik,
106 .num_of_watch_points = 4,
107 .mqd_size_aligned = MQD_SIZE_ALIGNED,
108 .supports_cwsr = true,
109 .needs_iommu_device = true,
110 .needs_pci_atomics = false,
111 .num_sdma_engines = 2,
112 .num_xgmi_sdma_engines = 0,
113 .num_sdma_queues_per_engine = 2,
116 static const struct kfd_device_info raven_device_info = {
117 .asic_family = CHIP_RAVEN,
118 .asic_name = "raven",
119 .max_pasid_bits = 16,
122 .ih_ring_entry_size = 8 * sizeof(uint32_t),
123 .event_interrupt_class = &event_interrupt_class_v9,
124 .num_of_watch_points = 4,
125 .mqd_size_aligned = MQD_SIZE_ALIGNED,
126 .supports_cwsr = true,
127 .needs_iommu_device = true,
128 .needs_pci_atomics = true,
129 .num_sdma_engines = 1,
130 .num_xgmi_sdma_engines = 0,
131 .num_sdma_queues_per_engine = 2,
135 static const struct kfd_device_info hawaii_device_info = {
136 .asic_family = CHIP_HAWAII,
137 .asic_name = "hawaii",
138 .max_pasid_bits = 16,
139 /* max num of queues for KV.TODO should be a dynamic value */
142 .ih_ring_entry_size = 4 * sizeof(uint32_t),
143 .event_interrupt_class = &event_interrupt_class_cik,
144 .num_of_watch_points = 4,
145 .mqd_size_aligned = MQD_SIZE_ALIGNED,
146 .supports_cwsr = false,
147 .needs_iommu_device = false,
148 .needs_pci_atomics = false,
149 .num_sdma_engines = 2,
150 .num_xgmi_sdma_engines = 0,
151 .num_sdma_queues_per_engine = 2,
154 static const struct kfd_device_info tonga_device_info = {
155 .asic_family = CHIP_TONGA,
156 .asic_name = "tonga",
157 .max_pasid_bits = 16,
160 .ih_ring_entry_size = 4 * sizeof(uint32_t),
161 .event_interrupt_class = &event_interrupt_class_cik,
162 .num_of_watch_points = 4,
163 .mqd_size_aligned = MQD_SIZE_ALIGNED,
164 .supports_cwsr = false,
165 .needs_iommu_device = false,
166 .needs_pci_atomics = true,
167 .num_sdma_engines = 2,
168 .num_xgmi_sdma_engines = 0,
169 .num_sdma_queues_per_engine = 2,
172 static const struct kfd_device_info fiji_device_info = {
173 .asic_family = CHIP_FIJI,
175 .max_pasid_bits = 16,
178 .ih_ring_entry_size = 4 * sizeof(uint32_t),
179 .event_interrupt_class = &event_interrupt_class_cik,
180 .num_of_watch_points = 4,
181 .mqd_size_aligned = MQD_SIZE_ALIGNED,
182 .supports_cwsr = true,
183 .needs_iommu_device = false,
184 .needs_pci_atomics = true,
185 .num_sdma_engines = 2,
186 .num_xgmi_sdma_engines = 0,
187 .num_sdma_queues_per_engine = 2,
190 static const struct kfd_device_info fiji_vf_device_info = {
191 .asic_family = CHIP_FIJI,
193 .max_pasid_bits = 16,
196 .ih_ring_entry_size = 4 * sizeof(uint32_t),
197 .event_interrupt_class = &event_interrupt_class_cik,
198 .num_of_watch_points = 4,
199 .mqd_size_aligned = MQD_SIZE_ALIGNED,
200 .supports_cwsr = true,
201 .needs_iommu_device = false,
202 .needs_pci_atomics = false,
203 .num_sdma_engines = 2,
204 .num_xgmi_sdma_engines = 0,
205 .num_sdma_queues_per_engine = 2,
209 static const struct kfd_device_info polaris10_device_info = {
210 .asic_family = CHIP_POLARIS10,
211 .asic_name = "polaris10",
212 .max_pasid_bits = 16,
215 .ih_ring_entry_size = 4 * sizeof(uint32_t),
216 .event_interrupt_class = &event_interrupt_class_cik,
217 .num_of_watch_points = 4,
218 .mqd_size_aligned = MQD_SIZE_ALIGNED,
219 .supports_cwsr = true,
220 .needs_iommu_device = false,
221 .needs_pci_atomics = true,
222 .num_sdma_engines = 2,
223 .num_xgmi_sdma_engines = 0,
224 .num_sdma_queues_per_engine = 2,
227 static const struct kfd_device_info polaris10_vf_device_info = {
228 .asic_family = CHIP_POLARIS10,
229 .asic_name = "polaris10",
230 .max_pasid_bits = 16,
233 .ih_ring_entry_size = 4 * sizeof(uint32_t),
234 .event_interrupt_class = &event_interrupt_class_cik,
235 .num_of_watch_points = 4,
236 .mqd_size_aligned = MQD_SIZE_ALIGNED,
237 .supports_cwsr = true,
238 .needs_iommu_device = false,
239 .needs_pci_atomics = false,
240 .num_sdma_engines = 2,
241 .num_xgmi_sdma_engines = 0,
242 .num_sdma_queues_per_engine = 2,
245 static const struct kfd_device_info polaris11_device_info = {
246 .asic_family = CHIP_POLARIS11,
247 .asic_name = "polaris11",
248 .max_pasid_bits = 16,
251 .ih_ring_entry_size = 4 * sizeof(uint32_t),
252 .event_interrupt_class = &event_interrupt_class_cik,
253 .num_of_watch_points = 4,
254 .mqd_size_aligned = MQD_SIZE_ALIGNED,
255 .supports_cwsr = true,
256 .needs_iommu_device = false,
257 .needs_pci_atomics = true,
258 .num_sdma_engines = 2,
259 .num_xgmi_sdma_engines = 0,
260 .num_sdma_queues_per_engine = 2,
263 static const struct kfd_device_info polaris12_device_info = {
264 .asic_family = CHIP_POLARIS12,
265 .asic_name = "polaris12",
266 .max_pasid_bits = 16,
269 .ih_ring_entry_size = 4 * sizeof(uint32_t),
270 .event_interrupt_class = &event_interrupt_class_cik,
271 .num_of_watch_points = 4,
272 .mqd_size_aligned = MQD_SIZE_ALIGNED,
273 .supports_cwsr = true,
274 .needs_iommu_device = false,
275 .needs_pci_atomics = true,
276 .num_sdma_engines = 2,
277 .num_xgmi_sdma_engines = 0,
278 .num_sdma_queues_per_engine = 2,
281 static const struct kfd_device_info vegam_device_info = {
282 .asic_family = CHIP_VEGAM,
283 .asic_name = "vegam",
284 .max_pasid_bits = 16,
287 .ih_ring_entry_size = 4 * sizeof(uint32_t),
288 .event_interrupt_class = &event_interrupt_class_cik,
289 .num_of_watch_points = 4,
290 .mqd_size_aligned = MQD_SIZE_ALIGNED,
291 .supports_cwsr = true,
292 .needs_iommu_device = false,
293 .needs_pci_atomics = true,
294 .num_sdma_engines = 2,
295 .num_xgmi_sdma_engines = 0,
296 .num_sdma_queues_per_engine = 2,
299 static const struct kfd_device_info vega10_device_info = {
300 .asic_family = CHIP_VEGA10,
301 .asic_name = "vega10",
302 .max_pasid_bits = 16,
305 .ih_ring_entry_size = 8 * sizeof(uint32_t),
306 .event_interrupt_class = &event_interrupt_class_v9,
307 .num_of_watch_points = 4,
308 .mqd_size_aligned = MQD_SIZE_ALIGNED,
309 .supports_cwsr = true,
310 .needs_iommu_device = false,
311 .needs_pci_atomics = false,
312 .num_sdma_engines = 2,
313 .num_xgmi_sdma_engines = 0,
314 .num_sdma_queues_per_engine = 2,
317 static const struct kfd_device_info vega10_vf_device_info = {
318 .asic_family = CHIP_VEGA10,
319 .asic_name = "vega10",
320 .max_pasid_bits = 16,
323 .ih_ring_entry_size = 8 * sizeof(uint32_t),
324 .event_interrupt_class = &event_interrupt_class_v9,
325 .num_of_watch_points = 4,
326 .mqd_size_aligned = MQD_SIZE_ALIGNED,
327 .supports_cwsr = true,
328 .needs_iommu_device = false,
329 .needs_pci_atomics = false,
330 .num_sdma_engines = 2,
331 .num_xgmi_sdma_engines = 0,
332 .num_sdma_queues_per_engine = 2,
335 static const struct kfd_device_info vega12_device_info = {
336 .asic_family = CHIP_VEGA12,
337 .asic_name = "vega12",
338 .max_pasid_bits = 16,
341 .ih_ring_entry_size = 8 * sizeof(uint32_t),
342 .event_interrupt_class = &event_interrupt_class_v9,
343 .num_of_watch_points = 4,
344 .mqd_size_aligned = MQD_SIZE_ALIGNED,
345 .supports_cwsr = true,
346 .needs_iommu_device = false,
347 .needs_pci_atomics = false,
348 .num_sdma_engines = 2,
349 .num_xgmi_sdma_engines = 0,
350 .num_sdma_queues_per_engine = 2,
353 static const struct kfd_device_info vega20_device_info = {
354 .asic_family = CHIP_VEGA20,
355 .asic_name = "vega20",
356 .max_pasid_bits = 16,
359 .ih_ring_entry_size = 8 * sizeof(uint32_t),
360 .event_interrupt_class = &event_interrupt_class_v9,
361 .num_of_watch_points = 4,
362 .mqd_size_aligned = MQD_SIZE_ALIGNED,
363 .supports_cwsr = true,
364 .needs_iommu_device = false,
365 .needs_pci_atomics = false,
366 .num_sdma_engines = 2,
367 .num_xgmi_sdma_engines = 0,
368 .num_sdma_queues_per_engine = 8,
371 static const struct kfd_device_info arcturus_device_info = {
372 .asic_family = CHIP_ARCTURUS,
373 .asic_name = "arcturus",
374 .max_pasid_bits = 16,
377 .ih_ring_entry_size = 8 * sizeof(uint32_t),
378 .event_interrupt_class = &event_interrupt_class_v9,
379 .num_of_watch_points = 4,
380 .mqd_size_aligned = MQD_SIZE_ALIGNED,
381 .supports_cwsr = true,
382 .needs_iommu_device = false,
383 .needs_pci_atomics = false,
384 .num_sdma_engines = 2,
385 .num_xgmi_sdma_engines = 6,
386 .num_sdma_queues_per_engine = 8,
389 static const struct kfd_device_info renoir_device_info = {
390 .asic_family = CHIP_RENOIR,
391 .asic_name = "renoir",
392 .max_pasid_bits = 16,
395 .ih_ring_entry_size = 8 * sizeof(uint32_t),
396 .event_interrupt_class = &event_interrupt_class_v9,
397 .num_of_watch_points = 4,
398 .mqd_size_aligned = MQD_SIZE_ALIGNED,
399 .supports_cwsr = true,
400 .needs_iommu_device = false,
401 .needs_pci_atomics = false,
402 .num_sdma_engines = 1,
403 .num_xgmi_sdma_engines = 0,
404 .num_sdma_queues_per_engine = 2,
407 static const struct kfd_device_info navi10_device_info = {
408 .asic_family = CHIP_NAVI10,
409 .asic_name = "navi10",
410 .max_pasid_bits = 16,
413 .ih_ring_entry_size = 8 * sizeof(uint32_t),
414 .event_interrupt_class = &event_interrupt_class_v9,
415 .num_of_watch_points = 4,
416 .mqd_size_aligned = MQD_SIZE_ALIGNED,
417 .needs_iommu_device = false,
418 .supports_cwsr = true,
419 .needs_pci_atomics = false,
420 .num_sdma_engines = 2,
421 .num_xgmi_sdma_engines = 0,
422 .num_sdma_queues_per_engine = 8,
425 static const struct kfd_device_info navi12_device_info = {
426 .asic_family = CHIP_NAVI12,
427 .asic_name = "navi12",
428 .max_pasid_bits = 16,
431 .ih_ring_entry_size = 8 * sizeof(uint32_t),
432 .event_interrupt_class = &event_interrupt_class_v9,
433 .num_of_watch_points = 4,
434 .mqd_size_aligned = MQD_SIZE_ALIGNED,
435 .needs_iommu_device = false,
436 .supports_cwsr = true,
437 .needs_pci_atomics = false,
438 .num_sdma_engines = 2,
439 .num_xgmi_sdma_engines = 0,
440 .num_sdma_queues_per_engine = 8,
443 static const struct kfd_device_info navi14_device_info = {
444 .asic_family = CHIP_NAVI14,
445 .asic_name = "navi14",
446 .max_pasid_bits = 16,
449 .ih_ring_entry_size = 8 * sizeof(uint32_t),
450 .event_interrupt_class = &event_interrupt_class_v9,
451 .num_of_watch_points = 4,
452 .mqd_size_aligned = MQD_SIZE_ALIGNED,
453 .needs_iommu_device = false,
454 .supports_cwsr = true,
455 .needs_pci_atomics = false,
456 .num_sdma_engines = 2,
457 .num_xgmi_sdma_engines = 0,
458 .num_sdma_queues_per_engine = 8,
461 /* For each entry, [0] is regular and [1] is virtualisation device. */
462 static const struct kfd_device_info *kfd_supported_devices[][2] = {
463 #ifdef KFD_SUPPORT_IOMMU_V2
464 [CHIP_KAVERI] = {&kaveri_device_info, NULL},
465 [CHIP_CARRIZO] = {&carrizo_device_info, NULL},
466 [CHIP_RAVEN] = {&raven_device_info, NULL},
468 [CHIP_HAWAII] = {&hawaii_device_info, NULL},
469 [CHIP_TONGA] = {&tonga_device_info, NULL},
470 [CHIP_FIJI] = {&fiji_device_info, &fiji_vf_device_info},
471 [CHIP_POLARIS10] = {&polaris10_device_info, &polaris10_vf_device_info},
472 [CHIP_POLARIS11] = {&polaris11_device_info, NULL},
473 [CHIP_POLARIS12] = {&polaris12_device_info, NULL},
474 [CHIP_VEGAM] = {&vegam_device_info, NULL},
475 [CHIP_VEGA10] = {&vega10_device_info, &vega10_vf_device_info},
476 [CHIP_VEGA12] = {&vega12_device_info, NULL},
477 [CHIP_VEGA20] = {&vega20_device_info, NULL},
478 [CHIP_RENOIR] = {&renoir_device_info, NULL},
479 [CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info},
480 [CHIP_NAVI10] = {&navi10_device_info, NULL},
481 [CHIP_NAVI12] = {&navi12_device_info, &navi12_device_info},
482 [CHIP_NAVI14] = {&navi14_device_info, NULL},
485 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
486 unsigned int chunk_size);
487 static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
489 static int kfd_resume(struct kfd_dev *kfd);
491 struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
492 struct pci_dev *pdev, unsigned int asic_type, bool vf)
495 const struct kfd_device_info *device_info;
496 const struct kfd2kgd_calls *f2g;
498 if (asic_type >= sizeof(kfd_supported_devices) / (sizeof(void *) * 2)
499 || asic_type >= sizeof(kfd2kgd_funcs) / sizeof(void *)) {
500 dev_err(kfd_device, "asic_type %d out of range\n", asic_type);
501 return NULL; /* asic_type out of range */
504 device_info = kfd_supported_devices[asic_type][vf];
505 f2g = kfd2kgd_funcs[asic_type];
507 if (!device_info || !f2g) {
508 dev_err(kfd_device, "%s %s not supported in kfd\n",
509 amdgpu_asic_name[asic_type], vf ? "VF" : "");
513 kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
517 /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.
518 * 32 and 64-bit requests are possible and must be
521 kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd);
522 if (device_info->needs_pci_atomics &&
523 !kfd->pci_atomic_requested) {
525 "skipped device %x:%x, PCI rejects atomics\n",
526 pdev->vendor, pdev->device);
532 kfd->device_info = device_info;
534 kfd->init_complete = false;
536 atomic_set(&kfd->compute_profile, 0);
538 mutex_init(&kfd->doorbell_mutex);
539 memset(&kfd->doorbell_available_index, 0,
540 sizeof(kfd->doorbell_available_index));
542 atomic_set(&kfd->sram_ecc_flag, 0);
547 static void kfd_cwsr_init(struct kfd_dev *kfd)
549 if (cwsr_enable && kfd->device_info->supports_cwsr) {
550 if (kfd->device_info->asic_family < CHIP_VEGA10) {
551 BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE);
552 kfd->cwsr_isa = cwsr_trap_gfx8_hex;
553 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);
554 } else if (kfd->device_info->asic_family == CHIP_ARCTURUS) {
555 BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > PAGE_SIZE);
556 kfd->cwsr_isa = cwsr_trap_arcturus_hex;
557 kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex);
558 } else if (kfd->device_info->asic_family < CHIP_NAVI10) {
559 BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE);
560 kfd->cwsr_isa = cwsr_trap_gfx9_hex;
561 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex);
563 BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > PAGE_SIZE);
564 kfd->cwsr_isa = cwsr_trap_gfx10_hex;
565 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex);
568 kfd->cwsr_enabled = true;
572 bool kgd2kfd_device_init(struct kfd_dev *kfd,
573 struct drm_device *ddev,
574 const struct kgd2kfd_shared_resources *gpu_resources)
579 kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
581 kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
583 kfd->shared_resources = *gpu_resources;
585 kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1;
586 kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1;
587 kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd
588 - kfd->vm_info.first_vmid_kfd + 1;
590 /* Verify module parameters regarding mapped process number*/
591 if ((hws_max_conc_proc < 0)
592 || (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) {
594 "hws_max_conc_proc %d must be between 0 and %d, use %d instead\n",
595 hws_max_conc_proc, kfd->vm_info.vmid_num_kfd,
596 kfd->vm_info.vmid_num_kfd);
597 kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd;
599 kfd->max_proc_per_quantum = hws_max_conc_proc;
601 /* Allocate global GWS that is shared by all KFD processes */
602 if (hws_gws_support && amdgpu_amdkfd_alloc_gws(kfd->kgd,
603 amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws)) {
604 dev_err(kfd_device, "Could not allocate %d gws\n",
605 amdgpu_amdkfd_get_num_gws(kfd->kgd));
608 /* calculate max size of mqds needed for queues */
609 size = max_num_of_queues_per_device *
610 kfd->device_info->mqd_size_aligned;
613 * calculate max size of runlist packet.
614 * There can be only 2 packets at once
616 size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) +
617 max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)
618 + sizeof(struct pm4_mes_runlist)) * 2;
620 /* Add size of HIQ & DIQ */
621 size += KFD_KERNEL_QUEUE_SIZE * 2;
623 /* add another 512KB for all other allocations on gart (HPD, fences) */
626 if (amdgpu_amdkfd_alloc_gtt_mem(
627 kfd->kgd, size, &kfd->gtt_mem,
628 &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr,
630 dev_err(kfd_device, "Could not allocate %d bytes\n", size);
631 goto alloc_gtt_mem_failure;
634 dev_info(kfd_device, "Allocated %d bytes on gart\n", size);
636 /* Initialize GTT sa with 512 byte chunk size */
637 if (kfd_gtt_sa_init(kfd, size, 512) != 0) {
638 dev_err(kfd_device, "Error initializing gtt sub-allocator\n");
639 goto kfd_gtt_sa_init_error;
642 if (kfd_doorbell_init(kfd)) {
644 "Error initializing doorbell aperture\n");
645 goto kfd_doorbell_error;
648 if (kfd->kfd2kgd->get_hive_id)
649 kfd->hive_id = kfd->kfd2kgd->get_hive_id(kfd->kgd);
651 if (kfd_interrupt_init(kfd)) {
652 dev_err(kfd_device, "Error initializing interrupts\n");
653 goto kfd_interrupt_error;
656 kfd->dqm = device_queue_manager_init(kfd);
658 dev_err(kfd_device, "Error initializing queue manager\n");
659 goto device_queue_manager_error;
662 if (kfd_iommu_device_init(kfd)) {
663 dev_err(kfd_device, "Error initializing iommuv2\n");
664 goto device_iommu_error;
670 goto kfd_resume_error;
674 if (kfd_topology_add_device(kfd)) {
675 dev_err(kfd_device, "Error adding device to topology\n");
676 goto kfd_topology_add_device_error;
679 kfd->init_complete = true;
680 dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor,
683 pr_debug("Starting kfd with the following scheduling policy %d\n",
684 kfd->dqm->sched_policy);
688 kfd_topology_add_device_error:
691 device_queue_manager_uninit(kfd->dqm);
692 device_queue_manager_error:
693 kfd_interrupt_exit(kfd);
695 kfd_doorbell_fini(kfd);
697 kfd_gtt_sa_fini(kfd);
698 kfd_gtt_sa_init_error:
699 amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
700 alloc_gtt_mem_failure:
702 amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
704 "device %x:%x NOT added due to errors\n",
705 kfd->pdev->vendor, kfd->pdev->device);
707 return kfd->init_complete;
710 void kgd2kfd_device_exit(struct kfd_dev *kfd)
712 if (kfd->init_complete) {
713 kgd2kfd_suspend(kfd);
714 device_queue_manager_uninit(kfd->dqm);
715 kfd_interrupt_exit(kfd);
716 kfd_topology_remove_device(kfd);
717 kfd_doorbell_fini(kfd);
718 kfd_gtt_sa_fini(kfd);
719 amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
721 amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
727 int kgd2kfd_pre_reset(struct kfd_dev *kfd)
729 if (!kfd->init_complete)
731 kgd2kfd_suspend(kfd);
733 /* hold dqm->lock to prevent further execution*/
736 kfd_signal_reset_event(kfd);
741 * Fix me. KFD won't be able to resume existing process for now.
742 * We will keep all existing process in a evicted state and
743 * wait the process to be terminated.
746 int kgd2kfd_post_reset(struct kfd_dev *kfd)
750 if (!kfd->init_complete)
753 dqm_unlock(kfd->dqm);
755 ret = kfd_resume(kfd);
758 count = atomic_dec_return(&kfd_locked);
760 atomic_set(&kfd->sram_ecc_flag, 0);
765 bool kfd_is_locked(void)
767 return (atomic_read(&kfd_locked) > 0);
770 void kgd2kfd_suspend(struct kfd_dev *kfd)
772 if (!kfd->init_complete)
775 /* For first KFD device suspend all the KFD processes */
776 if (atomic_inc_return(&kfd_locked) == 1)
777 kfd_suspend_all_processes();
779 kfd->dqm->ops.stop(kfd->dqm);
781 kfd_iommu_suspend(kfd);
784 int kgd2kfd_resume(struct kfd_dev *kfd)
788 if (!kfd->init_complete)
791 ret = kfd_resume(kfd);
795 count = atomic_dec_return(&kfd_locked);
796 WARN_ONCE(count < 0, "KFD suspend / resume ref. error");
798 ret = kfd_resume_all_processes();
803 static int kfd_resume(struct kfd_dev *kfd)
807 err = kfd_iommu_resume(kfd);
810 "Failed to resume IOMMU for device %x:%x\n",
811 kfd->pdev->vendor, kfd->pdev->device);
815 err = kfd->dqm->ops.start(kfd->dqm);
818 "Error starting queue manager for device %x:%x\n",
819 kfd->pdev->vendor, kfd->pdev->device);
820 goto dqm_start_error;
826 kfd_iommu_suspend(kfd);
830 /* This is called directly from KGD at ISR. */
831 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
833 uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE];
834 bool is_patched = false;
837 if (!kfd->init_complete)
840 if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) {
841 dev_err_once(kfd_device, "Ring entry too small\n");
845 spin_lock_irqsave(&kfd->interrupt_lock, flags);
847 if (kfd->interrupts_active
848 && interrupt_is_wanted(kfd, ih_ring_entry,
849 patched_ihre, &is_patched)
850 && enqueue_ih_ring_entry(kfd,
851 is_patched ? patched_ihre : ih_ring_entry))
852 queue_work(kfd->ih_wq, &kfd->interrupt_work);
854 spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
857 int kgd2kfd_quiesce_mm(struct mm_struct *mm)
859 struct kfd_process *p;
862 /* Because we are called from arbitrary context (workqueue) as opposed
863 * to process context, kfd_process could attempt to exit while we are
864 * running so the lookup function increments the process ref count.
866 p = kfd_lookup_process_by_mm(mm);
870 r = kfd_process_evict_queues(p);
872 kfd_unref_process(p);
876 int kgd2kfd_resume_mm(struct mm_struct *mm)
878 struct kfd_process *p;
881 /* Because we are called from arbitrary context (workqueue) as opposed
882 * to process context, kfd_process could attempt to exit while we are
883 * running so the lookup function increments the process ref count.
885 p = kfd_lookup_process_by_mm(mm);
889 r = kfd_process_restore_queues(p);
891 kfd_unref_process(p);
895 /** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will
896 * prepare for safe eviction of KFD BOs that belong to the specified
899 * @mm: mm_struct that identifies the specified KFD process
900 * @fence: eviction fence attached to KFD process BOs
903 int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
904 struct dma_fence *fence)
906 struct kfd_process *p;
907 unsigned long active_time;
908 unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS);
913 if (dma_fence_is_signaled(fence))
916 p = kfd_lookup_process_by_mm(mm);
920 if (fence->seqno == p->last_eviction_seqno)
923 p->last_eviction_seqno = fence->seqno;
925 /* Avoid KFD process starvation. Wait for at least
926 * PROCESS_ACTIVE_TIME_MS before evicting the process again
928 active_time = get_jiffies_64() - p->last_restore_timestamp;
929 if (delay_jiffies > active_time)
930 delay_jiffies -= active_time;
934 /* During process initialization eviction_work.dwork is initialized
935 * to kfd_evict_bo_worker
937 schedule_delayed_work(&p->eviction_work, delay_jiffies);
939 kfd_unref_process(p);
943 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
944 unsigned int chunk_size)
946 unsigned int num_of_longs;
948 if (WARN_ON(buf_size < chunk_size))
950 if (WARN_ON(buf_size == 0))
952 if (WARN_ON(chunk_size == 0))
955 kfd->gtt_sa_chunk_size = chunk_size;
956 kfd->gtt_sa_num_of_chunks = buf_size / chunk_size;
958 num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) /
961 kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL);
963 if (!kfd->gtt_sa_bitmap)
966 pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n",
967 kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap);
969 mutex_init(&kfd->gtt_sa_lock);
975 static void kfd_gtt_sa_fini(struct kfd_dev *kfd)
977 mutex_destroy(&kfd->gtt_sa_lock);
978 kfree(kfd->gtt_sa_bitmap);
981 static inline uint64_t kfd_gtt_sa_calc_gpu_addr(uint64_t start_addr,
982 unsigned int bit_num,
983 unsigned int chunk_size)
985 return start_addr + bit_num * chunk_size;
988 static inline uint32_t *kfd_gtt_sa_calc_cpu_addr(void *start_addr,
989 unsigned int bit_num,
990 unsigned int chunk_size)
992 return (uint32_t *) ((uint64_t) start_addr + bit_num * chunk_size);
995 int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
996 struct kfd_mem_obj **mem_obj)
998 unsigned int found, start_search, cur_size;
1003 if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)
1006 *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
1010 pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);
1014 mutex_lock(&kfd->gtt_sa_lock);
1016 kfd_gtt_restart_search:
1017 /* Find the first chunk that is free */
1018 found = find_next_zero_bit(kfd->gtt_sa_bitmap,
1019 kfd->gtt_sa_num_of_chunks,
1022 pr_debug("Found = %d\n", found);
1024 /* If there wasn't any free chunk, bail out */
1025 if (found == kfd->gtt_sa_num_of_chunks)
1026 goto kfd_gtt_no_free_chunk;
1028 /* Update fields of mem_obj */
1029 (*mem_obj)->range_start = found;
1030 (*mem_obj)->range_end = found;
1031 (*mem_obj)->gpu_addr = kfd_gtt_sa_calc_gpu_addr(
1032 kfd->gtt_start_gpu_addr,
1034 kfd->gtt_sa_chunk_size);
1035 (*mem_obj)->cpu_ptr = kfd_gtt_sa_calc_cpu_addr(
1036 kfd->gtt_start_cpu_ptr,
1038 kfd->gtt_sa_chunk_size);
1040 pr_debug("gpu_addr = %p, cpu_addr = %p\n",
1041 (uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr);
1043 /* If we need only one chunk, mark it as allocated and get out */
1044 if (size <= kfd->gtt_sa_chunk_size) {
1045 pr_debug("Single bit\n");
1046 set_bit(found, kfd->gtt_sa_bitmap);
1050 /* Otherwise, try to see if we have enough contiguous chunks */
1051 cur_size = size - kfd->gtt_sa_chunk_size;
1053 (*mem_obj)->range_end =
1054 find_next_zero_bit(kfd->gtt_sa_bitmap,
1055 kfd->gtt_sa_num_of_chunks, ++found);
1057 * If next free chunk is not contiguous than we need to
1058 * restart our search from the last free chunk we found (which
1059 * wasn't contiguous to the previous ones
1061 if ((*mem_obj)->range_end != found) {
1062 start_search = found;
1063 goto kfd_gtt_restart_search;
1067 * If we reached end of buffer, bail out with error
1069 if (found == kfd->gtt_sa_num_of_chunks)
1070 goto kfd_gtt_no_free_chunk;
1072 /* Check if we don't need another chunk */
1073 if (cur_size <= kfd->gtt_sa_chunk_size)
1076 cur_size -= kfd->gtt_sa_chunk_size;
1078 } while (cur_size > 0);
1080 pr_debug("range_start = %d, range_end = %d\n",
1081 (*mem_obj)->range_start, (*mem_obj)->range_end);
1083 /* Mark the chunks as allocated */
1084 for (found = (*mem_obj)->range_start;
1085 found <= (*mem_obj)->range_end;
1087 set_bit(found, kfd->gtt_sa_bitmap);
1090 mutex_unlock(&kfd->gtt_sa_lock);
1093 kfd_gtt_no_free_chunk:
1094 pr_debug("Allocation failed with mem_obj = %p\n", mem_obj);
1095 mutex_unlock(&kfd->gtt_sa_lock);
1100 int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj)
1104 /* Act like kfree when trying to free a NULL object */
1108 pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n",
1109 mem_obj, mem_obj->range_start, mem_obj->range_end);
1111 mutex_lock(&kfd->gtt_sa_lock);
1113 /* Mark the chunks as free */
1114 for (bit = mem_obj->range_start;
1115 bit <= mem_obj->range_end;
1117 clear_bit(bit, kfd->gtt_sa_bitmap);
1119 mutex_unlock(&kfd->gtt_sa_lock);
1125 void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
1128 atomic_inc(&kfd->sram_ecc_flag);
1131 void kfd_inc_compute_active(struct kfd_dev *kfd)
1133 if (atomic_inc_return(&kfd->compute_profile) == 1)
1134 amdgpu_amdkfd_set_compute_idle(kfd->kgd, false);
1137 void kfd_dec_compute_active(struct kfd_dev *kfd)
1139 int count = atomic_dec_return(&kfd->compute_profile);
1142 amdgpu_amdkfd_set_compute_idle(kfd->kgd, true);
1143 WARN_ONCE(count < 0, "Compute profile ref. count error");
1146 #if defined(CONFIG_DEBUG_FS)
1148 /* This function will send a package to HIQ to hang the HWS
1149 * which will trigger a GPU reset and bring the HWS back to normal state
1151 int kfd_debugfs_hang_hws(struct kfd_dev *dev)
1155 if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) {
1156 pr_err("HWS is not enabled");
1160 r = pm_debugfs_hang_hws(&dev->dqm->packets);
1162 r = dqm_debugfs_execute_queues(dev->dqm);