Commit | Line | Data |
---|---|---|
4a488a7a OG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
4a488a7a OG |
23 | #include <linux/bsearch.h> |
24 | #include <linux/pci.h> | |
25 | #include <linux/slab.h> | |
26 | #include "kfd_priv.h" | |
64c7f8cf | 27 | #include "kfd_device_queue_manager.h" |
507968dd | 28 | #include "kfd_pm4_headers_vi.h" |
0db54b24 | 29 | #include "cwsr_trap_handler.h" |
64d1c3a4 | 30 | #include "kfd_iommu.h" |
5b87245f | 31 | #include "amdgpu_amdkfd.h" |
2c2b0d88 | 32 | #include "kfd_smi_events.h" |
4a488a7a | 33 | |
19f6d2a6 | 34 | #define MQD_SIZE_ALIGNED 768 |
e42051d2 SL |
35 | |
36 | /* | |
37 | * kfd_locked is used to lock the kfd driver during suspend or reset | |
38 | * once locked, kfd driver will stop any further GPU execution. | |
39 | * create process (open) will return -EAGAIN. | |
40 | */ | |
41 | static atomic_t kfd_locked = ATOMIC_INIT(0); | |
19f6d2a6 | 42 | |
a3e520a2 | 43 | #ifdef CONFIG_DRM_AMDGPU_CIK |
e392c887 | 44 | extern const struct kfd2kgd_calls gfx_v7_kfd2kgd; |
a3e520a2 | 45 | #endif |
e392c887 YZ |
46 | extern const struct kfd2kgd_calls gfx_v8_kfd2kgd; |
47 | extern const struct kfd2kgd_calls gfx_v9_kfd2kgd; | |
48 | extern const struct kfd2kgd_calls arcturus_kfd2kgd; | |
49 | extern const struct kfd2kgd_calls gfx_v10_kfd2kgd; | |
3a2f0c81 | 50 | extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd; |
e392c887 YZ |
51 | |
52 | static const struct kfd2kgd_calls *kfd2kgd_funcs[] = { | |
53 | #ifdef KFD_SUPPORT_IOMMU_V2 | |
a3e520a2 | 54 | #ifdef CONFIG_DRM_AMDGPU_CIK |
e392c887 | 55 | [CHIP_KAVERI] = &gfx_v7_kfd2kgd, |
a3e520a2 | 56 | #endif |
e392c887 YZ |
57 | [CHIP_CARRIZO] = &gfx_v8_kfd2kgd, |
58 | [CHIP_RAVEN] = &gfx_v9_kfd2kgd, | |
59 | #endif | |
a3e520a2 | 60 | #ifdef CONFIG_DRM_AMDGPU_CIK |
e392c887 | 61 | [CHIP_HAWAII] = &gfx_v7_kfd2kgd, |
a3e520a2 | 62 | #endif |
e392c887 YZ |
63 | [CHIP_TONGA] = &gfx_v8_kfd2kgd, |
64 | [CHIP_FIJI] = &gfx_v8_kfd2kgd, | |
65 | [CHIP_POLARIS10] = &gfx_v8_kfd2kgd, | |
66 | [CHIP_POLARIS11] = &gfx_v8_kfd2kgd, | |
67 | [CHIP_POLARIS12] = &gfx_v8_kfd2kgd, | |
68 | [CHIP_VEGAM] = &gfx_v8_kfd2kgd, | |
69 | [CHIP_VEGA10] = &gfx_v9_kfd2kgd, | |
70 | [CHIP_VEGA12] = &gfx_v9_kfd2kgd, | |
71 | [CHIP_VEGA20] = &gfx_v9_kfd2kgd, | |
72 | [CHIP_RENOIR] = &gfx_v9_kfd2kgd, | |
73 | [CHIP_ARCTURUS] = &arcturus_kfd2kgd, | |
74 | [CHIP_NAVI10] = &gfx_v10_kfd2kgd, | |
75 | [CHIP_NAVI12] = &gfx_v10_kfd2kgd, | |
76 | [CHIP_NAVI14] = &gfx_v10_kfd2kgd, | |
3a2f0c81 | 77 | [CHIP_SIENNA_CICHLID] = &gfx_v10_3_kfd2kgd, |
09759e13 | 78 | [CHIP_NAVY_FLOUNDER] = &gfx_v10_3_kfd2kgd, |
3a5e715d | 79 | [CHIP_VANGOGH] = &gfx_v10_3_kfd2kgd, |
8f72ce64 | 80 | [CHIP_DIMGREY_CAVEFISH] = &gfx_v10_3_kfd2kgd, |
e392c887 YZ |
81 | }; |
82 | ||
64d1c3a4 | 83 | #ifdef KFD_SUPPORT_IOMMU_V2 |
4a488a7a | 84 | static const struct kfd_device_info kaveri_device_info = { |
0da7558c | 85 | .asic_family = CHIP_KAVERI, |
c181159a | 86 | .asic_name = "kaveri", |
0da7558c | 87 | .max_pasid_bits = 16, |
992839ad YS |
88 | /* max num of queues for KV.TODO should be a dynamic value */ |
89 | .max_no_of_hqd = 24, | |
ada2b29c | 90 | .doorbell_size = 4, |
0da7558c | 91 | .ih_ring_entry_size = 4 * sizeof(uint32_t), |
f3a39818 | 92 | .event_interrupt_class = &event_interrupt_class_cik, |
fbeb661b | 93 | .num_of_watch_points = 4, |
373d7080 FK |
94 | .mqd_size_aligned = MQD_SIZE_ALIGNED, |
95 | .supports_cwsr = false, | |
64d1c3a4 | 96 | .needs_iommu_device = true, |
3ee2d00c | 97 | .needs_pci_atomics = false, |
98bb9222 | 98 | .num_sdma_engines = 2, |
1b4670f6 | 99 | .num_xgmi_sdma_engines = 0, |
d5094189 | 100 | .num_sdma_queues_per_engine = 2, |
0da7558c BG |
101 | }; |
102 | ||
103 | static const struct kfd_device_info carrizo_device_info = { | |
104 | .asic_family = CHIP_CARRIZO, | |
c181159a | 105 | .asic_name = "carrizo", |
4a488a7a | 106 | .max_pasid_bits = 16, |
eaccd6e7 OG |
107 | /* max num of queues for CZ.TODO should be a dynamic value */ |
108 | .max_no_of_hqd = 24, | |
ada2b29c | 109 | .doorbell_size = 4, |
b3f5e6b4 | 110 | .ih_ring_entry_size = 4 * sizeof(uint32_t), |
eaccd6e7 | 111 | .event_interrupt_class = &event_interrupt_class_cik, |
f7c826ad | 112 | .num_of_watch_points = 4, |
373d7080 FK |
113 | .mqd_size_aligned = MQD_SIZE_ALIGNED, |
114 | .supports_cwsr = true, | |
64d1c3a4 | 115 | .needs_iommu_device = true, |
3ee2d00c | 116 | .needs_pci_atomics = false, |
98bb9222 | 117 | .num_sdma_engines = 2, |
1b4670f6 | 118 | .num_xgmi_sdma_engines = 0, |
d5094189 | 119 | .num_sdma_queues_per_engine = 2, |
4a488a7a | 120 | }; |
6127896f | 121 | #endif |
4d663df6 YZ |
122 | |
123 | static const struct kfd_device_info raven_device_info = { | |
124 | .asic_family = CHIP_RAVEN, | |
c181159a | 125 | .asic_name = "raven", |
4d663df6 YZ |
126 | .max_pasid_bits = 16, |
127 | .max_no_of_hqd = 24, | |
128 | .doorbell_size = 8, | |
129 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
130 | .event_interrupt_class = &event_interrupt_class_v9, | |
131 | .num_of_watch_points = 4, | |
132 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
133 | .supports_cwsr = true, | |
134 | .needs_iommu_device = true, | |
135 | .needs_pci_atomics = true, | |
136 | .num_sdma_engines = 1, | |
1b4670f6 | 137 | .num_xgmi_sdma_engines = 0, |
d5094189 | 138 | .num_sdma_queues_per_engine = 2, |
4d663df6 | 139 | }; |
4a488a7a | 140 | |
a3084e6c FK |
141 | static const struct kfd_device_info hawaii_device_info = { |
142 | .asic_family = CHIP_HAWAII, | |
c181159a | 143 | .asic_name = "hawaii", |
a3084e6c FK |
144 | .max_pasid_bits = 16, |
145 | /* max num of queues for KV.TODO should be a dynamic value */ | |
146 | .max_no_of_hqd = 24, | |
ada2b29c | 147 | .doorbell_size = 4, |
a3084e6c FK |
148 | .ih_ring_entry_size = 4 * sizeof(uint32_t), |
149 | .event_interrupt_class = &event_interrupt_class_cik, | |
150 | .num_of_watch_points = 4, | |
151 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
152 | .supports_cwsr = false, | |
64d1c3a4 | 153 | .needs_iommu_device = false, |
a3084e6c | 154 | .needs_pci_atomics = false, |
98bb9222 | 155 | .num_sdma_engines = 2, |
1b4670f6 | 156 | .num_xgmi_sdma_engines = 0, |
d5094189 | 157 | .num_sdma_queues_per_engine = 2, |
a3084e6c FK |
158 | }; |
159 | ||
160 | static const struct kfd_device_info tonga_device_info = { | |
161 | .asic_family = CHIP_TONGA, | |
c181159a | 162 | .asic_name = "tonga", |
a3084e6c FK |
163 | .max_pasid_bits = 16, |
164 | .max_no_of_hqd = 24, | |
ada2b29c | 165 | .doorbell_size = 4, |
a3084e6c FK |
166 | .ih_ring_entry_size = 4 * sizeof(uint32_t), |
167 | .event_interrupt_class = &event_interrupt_class_cik, | |
168 | .num_of_watch_points = 4, | |
169 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
170 | .supports_cwsr = false, | |
64d1c3a4 | 171 | .needs_iommu_device = false, |
a3084e6c | 172 | .needs_pci_atomics = true, |
98bb9222 | 173 | .num_sdma_engines = 2, |
1b4670f6 | 174 | .num_xgmi_sdma_engines = 0, |
d5094189 | 175 | .num_sdma_queues_per_engine = 2, |
a3084e6c FK |
176 | }; |
177 | ||
a3084e6c FK |
178 | static const struct kfd_device_info fiji_device_info = { |
179 | .asic_family = CHIP_FIJI, | |
c181159a | 180 | .asic_name = "fiji", |
a3084e6c FK |
181 | .max_pasid_bits = 16, |
182 | .max_no_of_hqd = 24, | |
ada2b29c | 183 | .doorbell_size = 4, |
a3084e6c FK |
184 | .ih_ring_entry_size = 4 * sizeof(uint32_t), |
185 | .event_interrupt_class = &event_interrupt_class_cik, | |
186 | .num_of_watch_points = 4, | |
187 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
188 | .supports_cwsr = true, | |
64d1c3a4 | 189 | .needs_iommu_device = false, |
a3084e6c | 190 | .needs_pci_atomics = true, |
98bb9222 | 191 | .num_sdma_engines = 2, |
1b4670f6 | 192 | .num_xgmi_sdma_engines = 0, |
d5094189 | 193 | .num_sdma_queues_per_engine = 2, |
a3084e6c FK |
194 | }; |
195 | ||
196 | static const struct kfd_device_info fiji_vf_device_info = { | |
197 | .asic_family = CHIP_FIJI, | |
c181159a | 198 | .asic_name = "fiji", |
a3084e6c FK |
199 | .max_pasid_bits = 16, |
200 | .max_no_of_hqd = 24, | |
ada2b29c | 201 | .doorbell_size = 4, |
a3084e6c FK |
202 | .ih_ring_entry_size = 4 * sizeof(uint32_t), |
203 | .event_interrupt_class = &event_interrupt_class_cik, | |
204 | .num_of_watch_points = 4, | |
205 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
206 | .supports_cwsr = true, | |
64d1c3a4 | 207 | .needs_iommu_device = false, |
a3084e6c | 208 | .needs_pci_atomics = false, |
98bb9222 | 209 | .num_sdma_engines = 2, |
1b4670f6 | 210 | .num_xgmi_sdma_engines = 0, |
d5094189 | 211 | .num_sdma_queues_per_engine = 2, |
a3084e6c FK |
212 | }; |
213 | ||
214 | ||
215 | static const struct kfd_device_info polaris10_device_info = { | |
216 | .asic_family = CHIP_POLARIS10, | |
c181159a | 217 | .asic_name = "polaris10", |
a3084e6c FK |
218 | .max_pasid_bits = 16, |
219 | .max_no_of_hqd = 24, | |
ada2b29c | 220 | .doorbell_size = 4, |
a3084e6c FK |
221 | .ih_ring_entry_size = 4 * sizeof(uint32_t), |
222 | .event_interrupt_class = &event_interrupt_class_cik, | |
223 | .num_of_watch_points = 4, | |
224 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
225 | .supports_cwsr = true, | |
64d1c3a4 | 226 | .needs_iommu_device = false, |
a3084e6c | 227 | .needs_pci_atomics = true, |
98bb9222 | 228 | .num_sdma_engines = 2, |
1b4670f6 | 229 | .num_xgmi_sdma_engines = 0, |
d5094189 | 230 | .num_sdma_queues_per_engine = 2, |
a3084e6c FK |
231 | }; |
232 | ||
233 | static const struct kfd_device_info polaris10_vf_device_info = { | |
234 | .asic_family = CHIP_POLARIS10, | |
c181159a | 235 | .asic_name = "polaris10", |
a3084e6c FK |
236 | .max_pasid_bits = 16, |
237 | .max_no_of_hqd = 24, | |
ada2b29c | 238 | .doorbell_size = 4, |
a3084e6c FK |
239 | .ih_ring_entry_size = 4 * sizeof(uint32_t), |
240 | .event_interrupt_class = &event_interrupt_class_cik, | |
241 | .num_of_watch_points = 4, | |
242 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
243 | .supports_cwsr = true, | |
64d1c3a4 | 244 | .needs_iommu_device = false, |
a3084e6c | 245 | .needs_pci_atomics = false, |
98bb9222 | 246 | .num_sdma_engines = 2, |
1b4670f6 | 247 | .num_xgmi_sdma_engines = 0, |
d5094189 | 248 | .num_sdma_queues_per_engine = 2, |
a3084e6c FK |
249 | }; |
250 | ||
251 | static const struct kfd_device_info polaris11_device_info = { | |
252 | .asic_family = CHIP_POLARIS11, | |
c181159a | 253 | .asic_name = "polaris11", |
a3084e6c FK |
254 | .max_pasid_bits = 16, |
255 | .max_no_of_hqd = 24, | |
ada2b29c | 256 | .doorbell_size = 4, |
a3084e6c FK |
257 | .ih_ring_entry_size = 4 * sizeof(uint32_t), |
258 | .event_interrupt_class = &event_interrupt_class_cik, | |
259 | .num_of_watch_points = 4, | |
260 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
261 | .supports_cwsr = true, | |
64d1c3a4 | 262 | .needs_iommu_device = false, |
a3084e6c | 263 | .needs_pci_atomics = true, |
98bb9222 | 264 | .num_sdma_engines = 2, |
1b4670f6 | 265 | .num_xgmi_sdma_engines = 0, |
d5094189 | 266 | .num_sdma_queues_per_engine = 2, |
a3084e6c FK |
267 | }; |
268 | ||
846a44d7 GB |
269 | static const struct kfd_device_info polaris12_device_info = { |
270 | .asic_family = CHIP_POLARIS12, | |
c181159a | 271 | .asic_name = "polaris12", |
846a44d7 GB |
272 | .max_pasid_bits = 16, |
273 | .max_no_of_hqd = 24, | |
274 | .doorbell_size = 4, | |
275 | .ih_ring_entry_size = 4 * sizeof(uint32_t), | |
276 | .event_interrupt_class = &event_interrupt_class_cik, | |
277 | .num_of_watch_points = 4, | |
278 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
279 | .supports_cwsr = true, | |
280 | .needs_iommu_device = false, | |
281 | .needs_pci_atomics = true, | |
282 | .num_sdma_engines = 2, | |
1b4670f6 | 283 | .num_xgmi_sdma_engines = 0, |
846a44d7 GB |
284 | .num_sdma_queues_per_engine = 2, |
285 | }; | |
286 | ||
ed81cd6e KR |
287 | static const struct kfd_device_info vegam_device_info = { |
288 | .asic_family = CHIP_VEGAM, | |
c181159a | 289 | .asic_name = "vegam", |
ed81cd6e KR |
290 | .max_pasid_bits = 16, |
291 | .max_no_of_hqd = 24, | |
292 | .doorbell_size = 4, | |
293 | .ih_ring_entry_size = 4 * sizeof(uint32_t), | |
294 | .event_interrupt_class = &event_interrupt_class_cik, | |
295 | .num_of_watch_points = 4, | |
296 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
297 | .supports_cwsr = true, | |
298 | .needs_iommu_device = false, | |
299 | .needs_pci_atomics = true, | |
300 | .num_sdma_engines = 2, | |
301 | .num_xgmi_sdma_engines = 0, | |
846a44d7 GB |
302 | .num_sdma_queues_per_engine = 2, |
303 | }; | |
304 | ||
389056e5 FK |
305 | static const struct kfd_device_info vega10_device_info = { |
306 | .asic_family = CHIP_VEGA10, | |
c181159a | 307 | .asic_name = "vega10", |
389056e5 FK |
308 | .max_pasid_bits = 16, |
309 | .max_no_of_hqd = 24, | |
310 | .doorbell_size = 8, | |
311 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
312 | .event_interrupt_class = &event_interrupt_class_v9, | |
313 | .num_of_watch_points = 4, | |
314 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
315 | .supports_cwsr = true, | |
316 | .needs_iommu_device = false, | |
317 | .needs_pci_atomics = false, | |
98bb9222 | 318 | .num_sdma_engines = 2, |
1b4670f6 | 319 | .num_xgmi_sdma_engines = 0, |
d5094189 | 320 | .num_sdma_queues_per_engine = 2, |
389056e5 FK |
321 | }; |
322 | ||
323 | static const struct kfd_device_info vega10_vf_device_info = { | |
324 | .asic_family = CHIP_VEGA10, | |
c181159a | 325 | .asic_name = "vega10", |
389056e5 FK |
326 | .max_pasid_bits = 16, |
327 | .max_no_of_hqd = 24, | |
328 | .doorbell_size = 8, | |
329 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
330 | .event_interrupt_class = &event_interrupt_class_v9, | |
331 | .num_of_watch_points = 4, | |
332 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
333 | .supports_cwsr = true, | |
334 | .needs_iommu_device = false, | |
335 | .needs_pci_atomics = false, | |
98bb9222 | 336 | .num_sdma_engines = 2, |
1b4670f6 | 337 | .num_xgmi_sdma_engines = 0, |
d5094189 | 338 | .num_sdma_queues_per_engine = 2, |
389056e5 FK |
339 | }; |
340 | ||
846a44d7 GB |
341 | static const struct kfd_device_info vega12_device_info = { |
342 | .asic_family = CHIP_VEGA12, | |
c181159a | 343 | .asic_name = "vega12", |
846a44d7 GB |
344 | .max_pasid_bits = 16, |
345 | .max_no_of_hqd = 24, | |
346 | .doorbell_size = 8, | |
347 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
348 | .event_interrupt_class = &event_interrupt_class_v9, | |
349 | .num_of_watch_points = 4, | |
350 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
351 | .supports_cwsr = true, | |
352 | .needs_iommu_device = false, | |
353 | .needs_pci_atomics = false, | |
354 | .num_sdma_engines = 2, | |
1b4670f6 | 355 | .num_xgmi_sdma_engines = 0, |
846a44d7 GB |
356 | .num_sdma_queues_per_engine = 2, |
357 | }; | |
358 | ||
22a3a294 SL |
359 | static const struct kfd_device_info vega20_device_info = { |
360 | .asic_family = CHIP_VEGA20, | |
c181159a | 361 | .asic_name = "vega20", |
22a3a294 SL |
362 | .max_pasid_bits = 16, |
363 | .max_no_of_hqd = 24, | |
364 | .doorbell_size = 8, | |
365 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
366 | .event_interrupt_class = &event_interrupt_class_v9, | |
367 | .num_of_watch_points = 4, | |
368 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
369 | .supports_cwsr = true, | |
370 | .needs_iommu_device = false, | |
006a0b3d | 371 | .needs_pci_atomics = false, |
22a3a294 | 372 | .num_sdma_engines = 2, |
1b4670f6 | 373 | .num_xgmi_sdma_engines = 0, |
22a3a294 SL |
374 | .num_sdma_queues_per_engine = 8, |
375 | }; | |
376 | ||
49adcf8a YZ |
377 | static const struct kfd_device_info arcturus_device_info = { |
378 | .asic_family = CHIP_ARCTURUS, | |
c181159a | 379 | .asic_name = "arcturus", |
49adcf8a YZ |
380 | .max_pasid_bits = 16, |
381 | .max_no_of_hqd = 24, | |
382 | .doorbell_size = 8, | |
383 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
384 | .event_interrupt_class = &event_interrupt_class_v9, | |
385 | .num_of_watch_points = 4, | |
386 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
387 | .supports_cwsr = true, | |
388 | .needs_iommu_device = false, | |
389 | .needs_pci_atomics = false, | |
b6689cf7 OZ |
390 | .num_sdma_engines = 2, |
391 | .num_xgmi_sdma_engines = 6, | |
49adcf8a YZ |
392 | .num_sdma_queues_per_engine = 8, |
393 | }; | |
394 | ||
2b9c2211 HR |
395 | static const struct kfd_device_info renoir_device_info = { |
396 | .asic_family = CHIP_RENOIR, | |
acb9acbe | 397 | .asic_name = "renoir", |
2b9c2211 HR |
398 | .max_pasid_bits = 16, |
399 | .max_no_of_hqd = 24, | |
400 | .doorbell_size = 8, | |
401 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
402 | .event_interrupt_class = &event_interrupt_class_v9, | |
403 | .num_of_watch_points = 4, | |
404 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
405 | .supports_cwsr = true, | |
406 | .needs_iommu_device = false, | |
407 | .needs_pci_atomics = false, | |
408 | .num_sdma_engines = 1, | |
409 | .num_xgmi_sdma_engines = 0, | |
410 | .num_sdma_queues_per_engine = 2, | |
411 | }; | |
412 | ||
14328aa5 PC |
413 | static const struct kfd_device_info navi10_device_info = { |
414 | .asic_family = CHIP_NAVI10, | |
c181159a | 415 | .asic_name = "navi10", |
14328aa5 PC |
416 | .max_pasid_bits = 16, |
417 | .max_no_of_hqd = 24, | |
418 | .doorbell_size = 8, | |
419 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
420 | .event_interrupt_class = &event_interrupt_class_v9, | |
421 | .num_of_watch_points = 4, | |
422 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
423 | .needs_iommu_device = false, | |
424 | .supports_cwsr = true, | |
6cc980e3 | 425 | .needs_pci_atomics = true, |
14328aa5 PC |
426 | .num_sdma_engines = 2, |
427 | .num_xgmi_sdma_engines = 0, | |
428 | .num_sdma_queues_per_engine = 8, | |
429 | }; | |
430 | ||
b77fb9d8 | 431 | static const struct kfd_device_info navi12_device_info = { |
0e94b564 | 432 | .asic_family = CHIP_NAVI12, |
b77fb9d8 | 433 | .asic_name = "navi12", |
434 | .max_pasid_bits = 16, | |
435 | .max_no_of_hqd = 24, | |
436 | .doorbell_size = 8, | |
437 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
438 | .event_interrupt_class = &event_interrupt_class_v9, | |
439 | .num_of_watch_points = 4, | |
440 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
441 | .needs_iommu_device = false, | |
442 | .supports_cwsr = true, | |
6cc980e3 | 443 | .needs_pci_atomics = true, |
b77fb9d8 | 444 | .num_sdma_engines = 2, |
445 | .num_xgmi_sdma_engines = 0, | |
446 | .num_sdma_queues_per_engine = 8, | |
447 | }; | |
448 | ||
8099ae40 YZ |
449 | static const struct kfd_device_info navi14_device_info = { |
450 | .asic_family = CHIP_NAVI14, | |
451 | .asic_name = "navi14", | |
452 | .max_pasid_bits = 16, | |
453 | .max_no_of_hqd = 24, | |
454 | .doorbell_size = 8, | |
455 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
456 | .event_interrupt_class = &event_interrupt_class_v9, | |
457 | .num_of_watch_points = 4, | |
458 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
459 | .needs_iommu_device = false, | |
460 | .supports_cwsr = true, | |
6cc980e3 | 461 | .needs_pci_atomics = true, |
8099ae40 YZ |
462 | .num_sdma_engines = 2, |
463 | .num_xgmi_sdma_engines = 0, | |
464 | .num_sdma_queues_per_engine = 8, | |
465 | }; | |
466 | ||
3a2f0c81 YZ |
467 | static const struct kfd_device_info sienna_cichlid_device_info = { |
468 | .asic_family = CHIP_SIENNA_CICHLID, | |
469 | .asic_name = "sienna_cichlid", | |
470 | .max_pasid_bits = 16, | |
471 | .max_no_of_hqd = 24, | |
472 | .doorbell_size = 8, | |
473 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
474 | .event_interrupt_class = &event_interrupt_class_v9, | |
475 | .num_of_watch_points = 4, | |
476 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
477 | .needs_iommu_device = false, | |
478 | .supports_cwsr = true, | |
6cc980e3 | 479 | .needs_pci_atomics = true, |
3a2f0c81 YZ |
480 | .num_sdma_engines = 4, |
481 | .num_xgmi_sdma_engines = 0, | |
482 | .num_sdma_queues_per_engine = 8, | |
483 | }; | |
484 | ||
de89b2e4 CG |
485 | static const struct kfd_device_info navy_flounder_device_info = { |
486 | .asic_family = CHIP_NAVY_FLOUNDER, | |
487 | .asic_name = "navy_flounder", | |
488 | .max_pasid_bits = 16, | |
489 | .max_no_of_hqd = 24, | |
490 | .doorbell_size = 8, | |
491 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
492 | .event_interrupt_class = &event_interrupt_class_v9, | |
493 | .num_of_watch_points = 4, | |
494 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
495 | .needs_iommu_device = false, | |
496 | .supports_cwsr = true, | |
6cc980e3 | 497 | .needs_pci_atomics = true, |
de89b2e4 CG |
498 | .num_sdma_engines = 2, |
499 | .num_xgmi_sdma_engines = 0, | |
500 | .num_sdma_queues_per_engine = 8, | |
501 | }; | |
502 | ||
3a5e715d HR |
503 | static const struct kfd_device_info vangogh_device_info = { |
504 | .asic_family = CHIP_VANGOGH, | |
505 | .asic_name = "vangogh", | |
506 | .max_pasid_bits = 16, | |
507 | .max_no_of_hqd = 24, | |
508 | .doorbell_size = 8, | |
509 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
510 | .event_interrupt_class = &event_interrupt_class_v9, | |
511 | .num_of_watch_points = 4, | |
512 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
513 | .needs_iommu_device = false, | |
514 | .supports_cwsr = true, | |
515 | .needs_pci_atomics = false, | |
516 | .num_sdma_engines = 1, | |
517 | .num_xgmi_sdma_engines = 0, | |
518 | .num_sdma_queues_per_engine = 2, | |
519 | }; | |
520 | ||
eb5a34d4 CG |
521 | static const struct kfd_device_info dimgrey_cavefish_device_info = { |
522 | .asic_family = CHIP_DIMGREY_CAVEFISH, | |
523 | .asic_name = "dimgrey_cavefish", | |
524 | .max_pasid_bits = 16, | |
525 | .max_no_of_hqd = 24, | |
526 | .doorbell_size = 8, | |
527 | .ih_ring_entry_size = 8 * sizeof(uint32_t), | |
528 | .event_interrupt_class = &event_interrupt_class_v9, | |
529 | .num_of_watch_points = 4, | |
530 | .mqd_size_aligned = MQD_SIZE_ALIGNED, | |
531 | .needs_iommu_device = false, | |
532 | .supports_cwsr = true, | |
6cc980e3 | 533 | .needs_pci_atomics = true, |
eb5a34d4 CG |
534 | .num_sdma_engines = 2, |
535 | .num_xgmi_sdma_engines = 0, | |
536 | .num_sdma_queues_per_engine = 8, | |
537 | }; | |
538 | ||
539 | ||
050091ab YZ |
540 | /* For each entry, [0] is regular and [1] is virtualisation device. */ |
541 | static const struct kfd_device_info *kfd_supported_devices[][2] = { | |
95a5bd1b | 542 | #ifdef KFD_SUPPORT_IOMMU_V2 |
050091ab | 543 | [CHIP_KAVERI] = {&kaveri_device_info, NULL}, |
95a5bd1b | 544 | [CHIP_CARRIZO] = {&carrizo_device_info, NULL}, |
95a5bd1b | 545 | #endif |
2b3bbf23 | 546 | [CHIP_RAVEN] = {&raven_device_info, NULL}, |
050091ab YZ |
547 | [CHIP_HAWAII] = {&hawaii_device_info, NULL}, |
548 | [CHIP_TONGA] = {&tonga_device_info, NULL}, | |
549 | [CHIP_FIJI] = {&fiji_device_info, &fiji_vf_device_info}, | |
050091ab YZ |
550 | [CHIP_POLARIS10] = {&polaris10_device_info, &polaris10_vf_device_info}, |
551 | [CHIP_POLARIS11] = {&polaris11_device_info, NULL}, | |
552 | [CHIP_POLARIS12] = {&polaris12_device_info, NULL}, | |
553 | [CHIP_VEGAM] = {&vegam_device_info, NULL}, | |
554 | [CHIP_VEGA10] = {&vega10_device_info, &vega10_vf_device_info}, | |
555 | [CHIP_VEGA12] = {&vega12_device_info, NULL}, | |
556 | [CHIP_VEGA20] = {&vega20_device_info, NULL}, | |
2b9c2211 | 557 | [CHIP_RENOIR] = {&renoir_device_info, NULL}, |
050091ab YZ |
558 | [CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info}, |
559 | [CHIP_NAVI10] = {&navi10_device_info, NULL}, | |
b77fb9d8 | 560 | [CHIP_NAVI12] = {&navi12_device_info, &navi12_device_info}, |
8099ae40 | 561 | [CHIP_NAVI14] = {&navi14_device_info, NULL}, |
adab4dad | 562 | [CHIP_SIENNA_CICHLID] = {&sienna_cichlid_device_info, &sienna_cichlid_device_info}, |
de89b2e4 | 563 | [CHIP_NAVY_FLOUNDER] = {&navy_flounder_device_info, &navy_flounder_device_info}, |
3a5e715d | 564 | [CHIP_VANGOGH] = {&vangogh_device_info, NULL}, |
eb5a34d4 | 565 | [CHIP_DIMGREY_CAVEFISH] = {&dimgrey_cavefish_device_info, &dimgrey_cavefish_device_info}, |
4a488a7a OG |
566 | }; |
567 | ||
6e81090b OG |
568 | static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, |
569 | unsigned int chunk_size); | |
570 | static void kfd_gtt_sa_fini(struct kfd_dev *kfd); | |
571 | ||
b8935a7c YZ |
572 | static int kfd_resume(struct kfd_dev *kfd); |
573 | ||
050091ab | 574 | struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, |
e392c887 | 575 | struct pci_dev *pdev, unsigned int asic_type, bool vf) |
4a488a7a | 576 | { |
050091ab YZ |
577 | struct kfd_dev *kfd; |
578 | const struct kfd_device_info *device_info; | |
e392c887 | 579 | const struct kfd2kgd_calls *f2g; |
4a488a7a | 580 | |
e392c887 YZ |
581 | if (asic_type >= sizeof(kfd_supported_devices) / (sizeof(void *) * 2) |
582 | || asic_type >= sizeof(kfd2kgd_funcs) / sizeof(void *)) { | |
050091ab YZ |
583 | dev_err(kfd_device, "asic_type %d out of range\n", asic_type); |
584 | return NULL; /* asic_type out of range */ | |
4a488a7a OG |
585 | } |
586 | ||
050091ab | 587 | device_info = kfd_supported_devices[asic_type][vf]; |
e392c887 | 588 | f2g = kfd2kgd_funcs[asic_type]; |
4a488a7a | 589 | |
aa5e899d | 590 | if (!device_info || !f2g) { |
050091ab YZ |
591 | dev_err(kfd_device, "%s %s not supported in kfd\n", |
592 | amdgpu_asic_name[asic_type], vf ? "VF" : ""); | |
4a488a7a | 593 | return NULL; |
4ebc7182 | 594 | } |
4a488a7a | 595 | |
d35f00d8 EH |
596 | kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); |
597 | if (!kfd) | |
598 | return NULL; | |
599 | ||
6106dce9 | 600 | /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps. |
601 | * 32 and 64-bit requests are possible and must be | |
602 | * supported. | |
603 | */ | |
aabf3a95 JX |
604 | kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd); |
605 | if (device_info->needs_pci_atomics && | |
606 | !kfd->pci_atomic_requested) { | |
6106dce9 | 607 | dev_info(kfd_device, |
608 | "skipped device %x:%x, PCI rejects atomics\n", | |
609 | pdev->vendor, pdev->device); | |
d35f00d8 | 610 | kfree(kfd); |
6106dce9 | 611 | return NULL; |
aabf3a95 | 612 | } |
4a488a7a OG |
613 | |
614 | kfd->kgd = kgd; | |
615 | kfd->device_info = device_info; | |
616 | kfd->pdev = pdev; | |
19f6d2a6 | 617 | kfd->init_complete = false; |
cea405b1 | 618 | kfd->kfd2kgd = f2g; |
43d8107f | 619 | atomic_set(&kfd->compute_profile, 0); |
cea405b1 XZ |
620 | |
621 | mutex_init(&kfd->doorbell_mutex); | |
622 | memset(&kfd->doorbell_available_index, 0, | |
623 | sizeof(kfd->doorbell_available_index)); | |
4a488a7a | 624 | |
9b54d201 EH |
625 | atomic_set(&kfd->sram_ecc_flag, 0); |
626 | ||
59d7115d MJ |
627 | ida_init(&kfd->doorbell_ida); |
628 | ||
4a488a7a OG |
629 | return kfd; |
630 | } | |
631 | ||
373d7080 FK |
632 | static void kfd_cwsr_init(struct kfd_dev *kfd) |
633 | { | |
634 | if (cwsr_enable && kfd->device_info->supports_cwsr) { | |
3e76c239 FK |
635 | if (kfd->device_info->asic_family < CHIP_VEGA10) { |
636 | BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE); | |
637 | kfd->cwsr_isa = cwsr_trap_gfx8_hex; | |
638 | kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex); | |
3baa24f0 OZ |
639 | } else if (kfd->device_info->asic_family == CHIP_ARCTURUS) { |
640 | BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > PAGE_SIZE); | |
641 | kfd->cwsr_isa = cwsr_trap_arcturus_hex; | |
642 | kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex); | |
14328aa5 | 643 | } else if (kfd->device_info->asic_family < CHIP_NAVI10) { |
3e76c239 FK |
644 | BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE); |
645 | kfd->cwsr_isa = cwsr_trap_gfx9_hex; | |
646 | kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex); | |
80b6cfed JC |
647 | } else if (kfd->device_info->asic_family < CHIP_SIENNA_CICHLID) { |
648 | BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) > PAGE_SIZE); | |
649 | kfd->cwsr_isa = cwsr_trap_nv1x_hex; | |
650 | kfd->cwsr_isa_size = sizeof(cwsr_trap_nv1x_hex); | |
14328aa5 PC |
651 | } else { |
652 | BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > PAGE_SIZE); | |
653 | kfd->cwsr_isa = cwsr_trap_gfx10_hex; | |
654 | kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex); | |
3e76c239 | 655 | } |
373d7080 | 656 | |
373d7080 FK |
657 | kfd->cwsr_enabled = true; |
658 | } | |
659 | } | |
660 | ||
29633d0e JG |
661 | static int kfd_gws_init(struct kfd_dev *kfd) |
662 | { | |
663 | int ret = 0; | |
664 | ||
665 | if (kfd->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) | |
666 | return 0; | |
667 | ||
668 | if (hws_gws_support | |
fea7d919 JG |
669 | || (kfd->device_info->asic_family == CHIP_VEGA10 |
670 | && kfd->mec2_fw_version >= 0x81b3) | |
671 | || (kfd->device_info->asic_family >= CHIP_VEGA12 | |
29633d0e | 672 | && kfd->device_info->asic_family <= CHIP_RAVEN |
fea7d919 JG |
673 | && kfd->mec2_fw_version >= 0x1b3) |
674 | || (kfd->device_info->asic_family == CHIP_ARCTURUS | |
675 | && kfd->mec2_fw_version >= 0x30)) | |
29633d0e JG |
676 | ret = amdgpu_amdkfd_alloc_gws(kfd->kgd, |
677 | amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws); | |
678 | ||
679 | return ret; | |
680 | } | |
681 | ||
938a0650 AL |
682 | static void kfd_smi_init(struct kfd_dev *dev) { |
683 | INIT_LIST_HEAD(&dev->smi_clients); | |
684 | spin_lock_init(&dev->smi_lock); | |
685 | } | |
686 | ||
4a488a7a | 687 | bool kgd2kfd_device_init(struct kfd_dev *kfd, |
3a0c3423 | 688 | struct drm_device *ddev, |
4a488a7a OG |
689 | const struct kgd2kfd_shared_resources *gpu_resources) |
690 | { | |
19f6d2a6 OG |
691 | unsigned int size; |
692 | ||
3a0c3423 | 693 | kfd->ddev = ddev; |
0da8b10e | 694 | kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, |
5ade6c9c | 695 | KGD_ENGINE_MEC1); |
29633d0e JG |
696 | kfd->mec2_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, |
697 | KGD_ENGINE_MEC2); | |
0da8b10e | 698 | kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, |
5ade6c9c | 699 | KGD_ENGINE_SDMA1); |
4a488a7a OG |
700 | kfd->shared_resources = *gpu_resources; |
701 | ||
44008d7a YZ |
702 | kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1; |
703 | kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1; | |
704 | kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd | |
705 | - kfd->vm_info.first_vmid_kfd + 1; | |
706 | ||
a99c6d4f FK |
707 | /* Verify module parameters regarding mapped process number*/ |
708 | if ((hws_max_conc_proc < 0) | |
709 | || (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) { | |
710 | dev_err(kfd_device, | |
711 | "hws_max_conc_proc %d must be between 0 and %d, use %d instead\n", | |
712 | hws_max_conc_proc, kfd->vm_info.vmid_num_kfd, | |
713 | kfd->vm_info.vmid_num_kfd); | |
714 | kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd; | |
715 | } else | |
716 | kfd->max_proc_per_quantum = hws_max_conc_proc; | |
717 | ||
19f6d2a6 | 718 | /* calculate max size of mqds needed for queues */ |
b8cbab04 OG |
719 | size = max_num_of_queues_per_device * |
720 | kfd->device_info->mqd_size_aligned; | |
19f6d2a6 | 721 | |
e18e794e OG |
722 | /* |
723 | * calculate max size of runlist packet. | |
724 | * There can be only 2 packets at once | |
725 | */ | |
507968dd FK |
726 | size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) + |
727 | max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues) | |
728 | + sizeof(struct pm4_mes_runlist)) * 2; | |
e18e794e OG |
729 | |
730 | /* Add size of HIQ & DIQ */ | |
731 | size += KFD_KERNEL_QUEUE_SIZE * 2; | |
732 | ||
733 | /* add another 512KB for all other allocations on gart (HPD, fences) */ | |
19f6d2a6 OG |
734 | size += 512 * 1024; |
735 | ||
7cd52c91 | 736 | if (amdgpu_amdkfd_alloc_gtt_mem( |
cea405b1 | 737 | kfd->kgd, size, &kfd->gtt_mem, |
15426dbb YZ |
738 | &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr, |
739 | false)) { | |
79775b62 | 740 | dev_err(kfd_device, "Could not allocate %d bytes\n", size); |
e09d4fc8 | 741 | goto alloc_gtt_mem_failure; |
19f6d2a6 OG |
742 | } |
743 | ||
79775b62 | 744 | dev_info(kfd_device, "Allocated %d bytes on gart\n", size); |
e18e794e | 745 | |
73a1da0b OG |
746 | /* Initialize GTT sa with 512 byte chunk size */ |
747 | if (kfd_gtt_sa_init(kfd, size, 512) != 0) { | |
79775b62 | 748 | dev_err(kfd_device, "Error initializing gtt sub-allocator\n"); |
73a1da0b OG |
749 | goto kfd_gtt_sa_init_error; |
750 | } | |
751 | ||
735df2ba FK |
752 | if (kfd_doorbell_init(kfd)) { |
753 | dev_err(kfd_device, | |
754 | "Error initializing doorbell aperture\n"); | |
755 | goto kfd_doorbell_error; | |
756 | } | |
19f6d2a6 | 757 | |
332f6e1e | 758 | kfd->hive_id = amdgpu_amdkfd_get_hive_id(kfd->kgd); |
0c1690e3 | 759 | |
9b498efa AD |
760 | kfd->noretry = amdgpu_amdkfd_get_noretry(kfd->kgd); |
761 | ||
2249d558 | 762 | if (kfd_interrupt_init(kfd)) { |
79775b62 | 763 | dev_err(kfd_device, "Error initializing interrupts\n"); |
2249d558 AL |
764 | goto kfd_interrupt_error; |
765 | } | |
766 | ||
64c7f8cf BG |
767 | kfd->dqm = device_queue_manager_init(kfd); |
768 | if (!kfd->dqm) { | |
79775b62 | 769 | dev_err(kfd_device, "Error initializing queue manager\n"); |
64c7f8cf BG |
770 | goto device_queue_manager_error; |
771 | } | |
772 | ||
29633d0e JG |
773 | /* If supported on this device, allocate global GWS that is shared |
774 | * by all KFD processes | |
775 | */ | |
776 | if (kfd_gws_init(kfd)) { | |
777 | dev_err(kfd_device, "Could not allocate %d gws\n", | |
778 | amdgpu_amdkfd_get_num_gws(kfd->kgd)); | |
779 | goto gws_error; | |
780 | } | |
781 | ||
6127896f HR |
782 | /* If CRAT is broken, won't set iommu enabled */ |
783 | kfd_double_confirm_iommu_support(kfd); | |
784 | ||
64d1c3a4 FK |
785 | if (kfd_iommu_device_init(kfd)) { |
786 | dev_err(kfd_device, "Error initializing iommuv2\n"); | |
787 | goto device_iommu_error; | |
64c7f8cf BG |
788 | } |
789 | ||
373d7080 FK |
790 | kfd_cwsr_init(kfd); |
791 | ||
b8935a7c YZ |
792 | if (kfd_resume(kfd)) |
793 | goto kfd_resume_error; | |
794 | ||
fbeb661b YS |
795 | kfd->dbgmgr = NULL; |
796 | ||
465ab9e0 OZ |
797 | if (kfd_topology_add_device(kfd)) { |
798 | dev_err(kfd_device, "Error adding device to topology\n"); | |
799 | goto kfd_topology_add_device_error; | |
800 | } | |
801 | ||
938a0650 AL |
802 | kfd_smi_init(kfd); |
803 | ||
4a488a7a | 804 | kfd->init_complete = true; |
79775b62 | 805 | dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor, |
4a488a7a OG |
806 | kfd->pdev->device); |
807 | ||
79775b62 | 808 | pr_debug("Starting kfd with the following scheduling policy %d\n", |
d146c5a7 | 809 | kfd->dqm->sched_policy); |
64c7f8cf | 810 | |
19f6d2a6 OG |
811 | goto out; |
812 | ||
465ab9e0 | 813 | kfd_topology_add_device_error: |
b8935a7c | 814 | kfd_resume_error: |
64d1c3a4 | 815 | device_iommu_error: |
29633d0e | 816 | gws_error: |
64c7f8cf BG |
817 | device_queue_manager_uninit(kfd->dqm); |
818 | device_queue_manager_error: | |
2249d558 AL |
819 | kfd_interrupt_exit(kfd); |
820 | kfd_interrupt_error: | |
735df2ba FK |
821 | kfd_doorbell_fini(kfd); |
822 | kfd_doorbell_error: | |
73a1da0b OG |
823 | kfd_gtt_sa_fini(kfd); |
824 | kfd_gtt_sa_init_error: | |
7cd52c91 | 825 | amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); |
e09d4fc8 | 826 | alloc_gtt_mem_failure: |
29633d0e | 827 | if (kfd->gws) |
e09d4fc8 | 828 | amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); |
19f6d2a6 | 829 | dev_err(kfd_device, |
79775b62 | 830 | "device %x:%x NOT added due to errors\n", |
19f6d2a6 OG |
831 | kfd->pdev->vendor, kfd->pdev->device); |
832 | out: | |
833 | return kfd->init_complete; | |
4a488a7a OG |
834 | } |
835 | ||
836 | void kgd2kfd_device_exit(struct kfd_dev *kfd) | |
837 | { | |
b17f068a | 838 | if (kfd->init_complete) { |
9593f4d6 | 839 | kgd2kfd_suspend(kfd, false); |
64c7f8cf | 840 | device_queue_manager_uninit(kfd->dqm); |
2249d558 | 841 | kfd_interrupt_exit(kfd); |
b17f068a | 842 | kfd_topology_remove_device(kfd); |
735df2ba | 843 | kfd_doorbell_fini(kfd); |
59d7115d | 844 | ida_destroy(&kfd->doorbell_ida); |
73a1da0b | 845 | kfd_gtt_sa_fini(kfd); |
7cd52c91 | 846 | amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); |
29633d0e | 847 | if (kfd->gws) |
e09d4fc8 | 848 | amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); |
b17f068a | 849 | } |
5b5c4e40 | 850 | |
4a488a7a OG |
851 | kfree(kfd); |
852 | } | |
853 | ||
e3b7a967 SL |
854 | int kgd2kfd_pre_reset(struct kfd_dev *kfd) |
855 | { | |
e42051d2 SL |
856 | if (!kfd->init_complete) |
857 | return 0; | |
09c34e8d | 858 | |
55977744 MJ |
859 | kfd_smi_event_update_gpu_reset(kfd, false); |
860 | ||
09c34e8d FK |
861 | kfd->dqm->ops.pre_reset(kfd->dqm); |
862 | ||
9593f4d6 | 863 | kgd2kfd_suspend(kfd, false); |
e42051d2 | 864 | |
e42051d2 | 865 | kfd_signal_reset_event(kfd); |
e3b7a967 SL |
866 | return 0; |
867 | } | |
868 | ||
e42051d2 SL |
869 | /* |
870 | * Fix me. KFD won't be able to resume existing process for now. | |
871 | * We will keep all existing process in a evicted state and | |
872 | * wait the process to be terminated. | |
873 | */ | |
874 | ||
e3b7a967 SL |
875 | int kgd2kfd_post_reset(struct kfd_dev *kfd) |
876 | { | |
a1bd079f | 877 | int ret; |
e42051d2 SL |
878 | |
879 | if (!kfd->init_complete) | |
880 | return 0; | |
881 | ||
e42051d2 SL |
882 | ret = kfd_resume(kfd); |
883 | if (ret) | |
884 | return ret; | |
a1bd079f | 885 | atomic_dec(&kfd_locked); |
9b54d201 EH |
886 | |
887 | atomic_set(&kfd->sram_ecc_flag, 0); | |
888 | ||
55977744 MJ |
889 | kfd_smi_event_update_gpu_reset(kfd, true); |
890 | ||
e3b7a967 SL |
891 | return 0; |
892 | } | |
893 | ||
e42051d2 SL |
894 | bool kfd_is_locked(void) |
895 | { | |
896 | return (atomic_read(&kfd_locked) > 0); | |
897 | } | |
898 | ||
9593f4d6 | 899 | void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm) |
4a488a7a | 900 | { |
733fa1f7 YZ |
901 | if (!kfd->init_complete) |
902 | return; | |
903 | ||
9593f4d6 RB |
904 | /* for runtime suspend, skip locking kfd */ |
905 | if (!run_pm) { | |
906 | /* For first KFD device suspend all the KFD processes */ | |
907 | if (atomic_inc_return(&kfd_locked) == 1) | |
908 | kfd_suspend_all_processes(); | |
909 | } | |
26103436 | 910 | |
733fa1f7 | 911 | kfd->dqm->ops.stop(kfd->dqm); |
64d1c3a4 | 912 | kfd_iommu_suspend(kfd); |
4a488a7a OG |
913 | } |
914 | ||
9593f4d6 | 915 | int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm) |
4a488a7a | 916 | { |
26103436 FK |
917 | int ret, count; |
918 | ||
b8935a7c YZ |
919 | if (!kfd->init_complete) |
920 | return 0; | |
b17f068a | 921 | |
26103436 FK |
922 | ret = kfd_resume(kfd); |
923 | if (ret) | |
924 | return ret; | |
925 | ||
9593f4d6 RB |
926 | /* for runtime resume, skip unlocking kfd */ |
927 | if (!run_pm) { | |
928 | count = atomic_dec_return(&kfd_locked); | |
929 | WARN_ONCE(count < 0, "KFD suspend / resume ref. error"); | |
930 | if (count == 0) | |
931 | ret = kfd_resume_all_processes(); | |
932 | } | |
b17f068a | 933 | |
26103436 | 934 | return ret; |
b8935a7c YZ |
935 | } |
936 | ||
937 | static int kfd_resume(struct kfd_dev *kfd) | |
938 | { | |
939 | int err = 0; | |
b8935a7c | 940 | |
64d1c3a4 FK |
941 | err = kfd_iommu_resume(kfd); |
942 | if (err) { | |
943 | dev_err(kfd_device, | |
944 | "Failed to resume IOMMU for device %x:%x\n", | |
945 | kfd->pdev->vendor, kfd->pdev->device); | |
946 | return err; | |
947 | } | |
733fa1f7 | 948 | |
b8935a7c YZ |
949 | err = kfd->dqm->ops.start(kfd->dqm); |
950 | if (err) { | |
951 | dev_err(kfd_device, | |
952 | "Error starting queue manager for device %x:%x\n", | |
953 | kfd->pdev->vendor, kfd->pdev->device); | |
954 | goto dqm_start_error; | |
b17f068a OG |
955 | } |
956 | ||
b8935a7c YZ |
957 | return err; |
958 | ||
959 | dqm_start_error: | |
64d1c3a4 | 960 | kfd_iommu_suspend(kfd); |
b8935a7c | 961 | return err; |
4a488a7a OG |
962 | } |
963 | ||
b3eca59d PY |
964 | static inline void kfd_queue_work(struct workqueue_struct *wq, |
965 | struct work_struct *work) | |
966 | { | |
967 | int cpu, new_cpu; | |
968 | ||
969 | cpu = new_cpu = smp_processor_id(); | |
970 | do { | |
971 | new_cpu = cpumask_next(new_cpu, cpu_online_mask) % nr_cpu_ids; | |
972 | if (cpu_to_node(new_cpu) == numa_node_id()) | |
973 | break; | |
974 | } while (cpu != new_cpu); | |
975 | ||
976 | queue_work_on(new_cpu, wq, work); | |
977 | } | |
978 | ||
b3f5e6b4 AL |
979 | /* This is called directly from KGD at ISR. */ |
980 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) | |
4a488a7a | 981 | { |
58e69886 LX |
982 | uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE]; |
983 | bool is_patched = false; | |
2383a767 | 984 | unsigned long flags; |
58e69886 | 985 | |
2249d558 AL |
986 | if (!kfd->init_complete) |
987 | return; | |
988 | ||
58e69886 LX |
989 | if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) { |
990 | dev_err_once(kfd_device, "Ring entry too small\n"); | |
991 | return; | |
992 | } | |
993 | ||
2383a767 | 994 | spin_lock_irqsave(&kfd->interrupt_lock, flags); |
2249d558 AL |
995 | |
996 | if (kfd->interrupts_active | |
58e69886 LX |
997 | && interrupt_is_wanted(kfd, ih_ring_entry, |
998 | patched_ihre, &is_patched) | |
999 | && enqueue_ih_ring_entry(kfd, | |
1000 | is_patched ? patched_ihre : ih_ring_entry)) | |
b3eca59d | 1001 | kfd_queue_work(kfd->ih_wq, &kfd->interrupt_work); |
2249d558 | 1002 | |
2383a767 | 1003 | spin_unlock_irqrestore(&kfd->interrupt_lock, flags); |
4a488a7a | 1004 | } |
6e81090b | 1005 | |
6b95e797 FK |
1006 | int kgd2kfd_quiesce_mm(struct mm_struct *mm) |
1007 | { | |
1008 | struct kfd_process *p; | |
1009 | int r; | |
1010 | ||
1011 | /* Because we are called from arbitrary context (workqueue) as opposed | |
1012 | * to process context, kfd_process could attempt to exit while we are | |
1013 | * running so the lookup function increments the process ref count. | |
1014 | */ | |
1015 | p = kfd_lookup_process_by_mm(mm); | |
1016 | if (!p) | |
1017 | return -ESRCH; | |
1018 | ||
b2057956 | 1019 | WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); |
6b95e797 FK |
1020 | r = kfd_process_evict_queues(p); |
1021 | ||
1022 | kfd_unref_process(p); | |
1023 | return r; | |
1024 | } | |
1025 | ||
1026 | int kgd2kfd_resume_mm(struct mm_struct *mm) | |
1027 | { | |
1028 | struct kfd_process *p; | |
1029 | int r; | |
1030 | ||
1031 | /* Because we are called from arbitrary context (workqueue) as opposed | |
1032 | * to process context, kfd_process could attempt to exit while we are | |
1033 | * running so the lookup function increments the process ref count. | |
1034 | */ | |
1035 | p = kfd_lookup_process_by_mm(mm); | |
1036 | if (!p) | |
1037 | return -ESRCH; | |
1038 | ||
1039 | r = kfd_process_restore_queues(p); | |
1040 | ||
1041 | kfd_unref_process(p); | |
1042 | return r; | |
1043 | } | |
1044 | ||
26103436 FK |
1045 | /** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will |
1046 | * prepare for safe eviction of KFD BOs that belong to the specified | |
1047 | * process. | |
1048 | * | |
1049 | * @mm: mm_struct that identifies the specified KFD process | |
1050 | * @fence: eviction fence attached to KFD process BOs | |
1051 | * | |
1052 | */ | |
1053 | int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, | |
1054 | struct dma_fence *fence) | |
1055 | { | |
1056 | struct kfd_process *p; | |
1057 | unsigned long active_time; | |
1058 | unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS); | |
1059 | ||
1060 | if (!fence) | |
1061 | return -EINVAL; | |
1062 | ||
1063 | if (dma_fence_is_signaled(fence)) | |
1064 | return 0; | |
1065 | ||
1066 | p = kfd_lookup_process_by_mm(mm); | |
1067 | if (!p) | |
1068 | return -ENODEV; | |
1069 | ||
1070 | if (fence->seqno == p->last_eviction_seqno) | |
1071 | goto out; | |
1072 | ||
1073 | p->last_eviction_seqno = fence->seqno; | |
1074 | ||
1075 | /* Avoid KFD process starvation. Wait for at least | |
1076 | * PROCESS_ACTIVE_TIME_MS before evicting the process again | |
1077 | */ | |
1078 | active_time = get_jiffies_64() - p->last_restore_timestamp; | |
1079 | if (delay_jiffies > active_time) | |
1080 | delay_jiffies -= active_time; | |
1081 | else | |
1082 | delay_jiffies = 0; | |
1083 | ||
1084 | /* During process initialization eviction_work.dwork is initialized | |
1085 | * to kfd_evict_bo_worker | |
1086 | */ | |
b2057956 FK |
1087 | WARN(debug_evictions, "Scheduling eviction of pid %d in %ld jiffies", |
1088 | p->lead_thread->pid, delay_jiffies); | |
26103436 FK |
1089 | schedule_delayed_work(&p->eviction_work, delay_jiffies); |
1090 | out: | |
1091 | kfd_unref_process(p); | |
1092 | return 0; | |
1093 | } | |
1094 | ||
6e81090b OG |
1095 | static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, |
1096 | unsigned int chunk_size) | |
1097 | { | |
8625ff9c | 1098 | unsigned int num_of_longs; |
6e81090b | 1099 | |
32fa8219 FK |
1100 | if (WARN_ON(buf_size < chunk_size)) |
1101 | return -EINVAL; | |
1102 | if (WARN_ON(buf_size == 0)) | |
1103 | return -EINVAL; | |
1104 | if (WARN_ON(chunk_size == 0)) | |
1105 | return -EINVAL; | |
6e81090b OG |
1106 | |
1107 | kfd->gtt_sa_chunk_size = chunk_size; | |
1108 | kfd->gtt_sa_num_of_chunks = buf_size / chunk_size; | |
1109 | ||
8625ff9c FK |
1110 | num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) / |
1111 | BITS_PER_LONG; | |
6e81090b | 1112 | |
8625ff9c | 1113 | kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL); |
6e81090b OG |
1114 | |
1115 | if (!kfd->gtt_sa_bitmap) | |
1116 | return -ENOMEM; | |
1117 | ||
79775b62 | 1118 | pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n", |
6e81090b OG |
1119 | kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap); |
1120 | ||
1121 | mutex_init(&kfd->gtt_sa_lock); | |
1122 | ||
1123 | return 0; | |
1124 | ||
1125 | } | |
1126 | ||
1127 | static void kfd_gtt_sa_fini(struct kfd_dev *kfd) | |
1128 | { | |
1129 | mutex_destroy(&kfd->gtt_sa_lock); | |
1130 | kfree(kfd->gtt_sa_bitmap); | |
1131 | } | |
1132 | ||
1133 | static inline uint64_t kfd_gtt_sa_calc_gpu_addr(uint64_t start_addr, | |
1134 | unsigned int bit_num, | |
1135 | unsigned int chunk_size) | |
1136 | { | |
1137 | return start_addr + bit_num * chunk_size; | |
1138 | } | |
1139 | ||
1140 | static inline uint32_t *kfd_gtt_sa_calc_cpu_addr(void *start_addr, | |
1141 | unsigned int bit_num, | |
1142 | unsigned int chunk_size) | |
1143 | { | |
1144 | return (uint32_t *) ((uint64_t) start_addr + bit_num * chunk_size); | |
1145 | } | |
1146 | ||
1147 | int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, | |
1148 | struct kfd_mem_obj **mem_obj) | |
1149 | { | |
1150 | unsigned int found, start_search, cur_size; | |
1151 | ||
6e81090b OG |
1152 | if (size == 0) |
1153 | return -EINVAL; | |
1154 | ||
1155 | if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size) | |
1156 | return -ENOMEM; | |
1157 | ||
1cd106ec FK |
1158 | *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL); |
1159 | if (!(*mem_obj)) | |
6e81090b OG |
1160 | return -ENOMEM; |
1161 | ||
79775b62 | 1162 | pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size); |
6e81090b OG |
1163 | |
1164 | start_search = 0; | |
1165 | ||
1166 | mutex_lock(&kfd->gtt_sa_lock); | |
1167 | ||
1168 | kfd_gtt_restart_search: | |
1169 | /* Find the first chunk that is free */ | |
1170 | found = find_next_zero_bit(kfd->gtt_sa_bitmap, | |
1171 | kfd->gtt_sa_num_of_chunks, | |
1172 | start_search); | |
1173 | ||
79775b62 | 1174 | pr_debug("Found = %d\n", found); |
6e81090b OG |
1175 | |
1176 | /* If there wasn't any free chunk, bail out */ | |
1177 | if (found == kfd->gtt_sa_num_of_chunks) | |
1178 | goto kfd_gtt_no_free_chunk; | |
1179 | ||
1180 | /* Update fields of mem_obj */ | |
1181 | (*mem_obj)->range_start = found; | |
1182 | (*mem_obj)->range_end = found; | |
1183 | (*mem_obj)->gpu_addr = kfd_gtt_sa_calc_gpu_addr( | |
1184 | kfd->gtt_start_gpu_addr, | |
1185 | found, | |
1186 | kfd->gtt_sa_chunk_size); | |
1187 | (*mem_obj)->cpu_ptr = kfd_gtt_sa_calc_cpu_addr( | |
1188 | kfd->gtt_start_cpu_ptr, | |
1189 | found, | |
1190 | kfd->gtt_sa_chunk_size); | |
1191 | ||
79775b62 | 1192 | pr_debug("gpu_addr = %p, cpu_addr = %p\n", |
6e81090b OG |
1193 | (uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr); |
1194 | ||
1195 | /* If we need only one chunk, mark it as allocated and get out */ | |
1196 | if (size <= kfd->gtt_sa_chunk_size) { | |
79775b62 | 1197 | pr_debug("Single bit\n"); |
6e81090b OG |
1198 | set_bit(found, kfd->gtt_sa_bitmap); |
1199 | goto kfd_gtt_out; | |
1200 | } | |
1201 | ||
1202 | /* Otherwise, try to see if we have enough contiguous chunks */ | |
1203 | cur_size = size - kfd->gtt_sa_chunk_size; | |
1204 | do { | |
1205 | (*mem_obj)->range_end = | |
1206 | find_next_zero_bit(kfd->gtt_sa_bitmap, | |
1207 | kfd->gtt_sa_num_of_chunks, ++found); | |
1208 | /* | |
1209 | * If next free chunk is not contiguous than we need to | |
1210 | * restart our search from the last free chunk we found (which | |
1211 | * wasn't contiguous to the previous ones | |
1212 | */ | |
1213 | if ((*mem_obj)->range_end != found) { | |
1214 | start_search = found; | |
1215 | goto kfd_gtt_restart_search; | |
1216 | } | |
1217 | ||
1218 | /* | |
1219 | * If we reached end of buffer, bail out with error | |
1220 | */ | |
1221 | if (found == kfd->gtt_sa_num_of_chunks) | |
1222 | goto kfd_gtt_no_free_chunk; | |
1223 | ||
1224 | /* Check if we don't need another chunk */ | |
1225 | if (cur_size <= kfd->gtt_sa_chunk_size) | |
1226 | cur_size = 0; | |
1227 | else | |
1228 | cur_size -= kfd->gtt_sa_chunk_size; | |
1229 | ||
1230 | } while (cur_size > 0); | |
1231 | ||
79775b62 | 1232 | pr_debug("range_start = %d, range_end = %d\n", |
6e81090b OG |
1233 | (*mem_obj)->range_start, (*mem_obj)->range_end); |
1234 | ||
1235 | /* Mark the chunks as allocated */ | |
1236 | for (found = (*mem_obj)->range_start; | |
1237 | found <= (*mem_obj)->range_end; | |
1238 | found++) | |
1239 | set_bit(found, kfd->gtt_sa_bitmap); | |
1240 | ||
1241 | kfd_gtt_out: | |
1242 | mutex_unlock(&kfd->gtt_sa_lock); | |
1243 | return 0; | |
1244 | ||
1245 | kfd_gtt_no_free_chunk: | |
3148a6a0 | 1246 | pr_debug("Allocation failed with mem_obj = %p\n", *mem_obj); |
6e81090b | 1247 | mutex_unlock(&kfd->gtt_sa_lock); |
3148a6a0 | 1248 | kfree(*mem_obj); |
6e81090b OG |
1249 | return -ENOMEM; |
1250 | } | |
1251 | ||
1252 | int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj) | |
1253 | { | |
1254 | unsigned int bit; | |
1255 | ||
9216ed29 OG |
1256 | /* Act like kfree when trying to free a NULL object */ |
1257 | if (!mem_obj) | |
1258 | return 0; | |
6e81090b | 1259 | |
79775b62 | 1260 | pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n", |
6e81090b OG |
1261 | mem_obj, mem_obj->range_start, mem_obj->range_end); |
1262 | ||
1263 | mutex_lock(&kfd->gtt_sa_lock); | |
1264 | ||
1265 | /* Mark the chunks as free */ | |
1266 | for (bit = mem_obj->range_start; | |
1267 | bit <= mem_obj->range_end; | |
1268 | bit++) | |
1269 | clear_bit(bit, kfd->gtt_sa_bitmap); | |
1270 | ||
1271 | mutex_unlock(&kfd->gtt_sa_lock); | |
1272 | ||
1273 | kfree(mem_obj); | |
1274 | return 0; | |
1275 | } | |
a29ec470 | 1276 | |
9b54d201 EH |
1277 | void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) |
1278 | { | |
1279 | if (kfd) | |
1280 | atomic_inc(&kfd->sram_ecc_flag); | |
1281 | } | |
1282 | ||
43d8107f HK |
1283 | void kfd_inc_compute_active(struct kfd_dev *kfd) |
1284 | { | |
1285 | if (atomic_inc_return(&kfd->compute_profile) == 1) | |
1286 | amdgpu_amdkfd_set_compute_idle(kfd->kgd, false); | |
1287 | } | |
1288 | ||
1289 | void kfd_dec_compute_active(struct kfd_dev *kfd) | |
1290 | { | |
1291 | int count = atomic_dec_return(&kfd->compute_profile); | |
1292 | ||
1293 | if (count == 0) | |
1294 | amdgpu_amdkfd_set_compute_idle(kfd->kgd, true); | |
1295 | WARN_ONCE(count < 0, "Compute profile ref. count error"); | |
1296 | } | |
1297 | ||
2c2b0d88 MJ |
1298 | void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask) |
1299 | { | |
1300 | if (kfd) | |
1301 | kfd_smi_event_update_thermal_throttling(kfd, throttle_bitmask); | |
1302 | } | |
1303 | ||
a29ec470 SL |
1304 | #if defined(CONFIG_DEBUG_FS) |
1305 | ||
1306 | /* This function will send a package to HIQ to hang the HWS | |
1307 | * which will trigger a GPU reset and bring the HWS back to normal state | |
1308 | */ | |
1309 | int kfd_debugfs_hang_hws(struct kfd_dev *dev) | |
1310 | { | |
1311 | int r = 0; | |
1312 | ||
1313 | if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) { | |
1314 | pr_err("HWS is not enabled"); | |
1315 | return -EINVAL; | |
1316 | } | |
1317 | ||
1318 | r = pm_debugfs_hang_hws(&dev->dqm->packets); | |
1319 | if (!r) | |
1320 | r = dqm_debugfs_execute_queues(dev->dqm); | |
1321 | ||
1322 | return r; | |
1323 | } | |
1324 | ||
1325 | #endif |