Commit | Line | Data |
---|---|---|
edc61147 HZ |
1 | /* |
2 | * Copyright 2019 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | */ | |
23 | ||
b23b2e9e AD |
24 | #include <linux/pci.h> |
25 | ||
edc61147 HZ |
26 | #include "amdgpu.h" |
27 | #include "amdgpu_ih.h" | |
28 | ||
29 | #include "oss/osssys_5_0_0_offset.h" | |
30 | #include "oss/osssys_5_0_0_sh_mask.h" | |
31 | ||
32 | #include "soc15_common.h" | |
33 | #include "navi10_ih.h" | |
34 | ||
022b6518 | 35 | #define MAX_REARM_RETRY 10 |
edc61147 | 36 | |
757b3af8 LG |
37 | #define mmIH_CHICKEN_Sienna_Cichlid 0x018d |
38 | #define mmIH_CHICKEN_Sienna_Cichlid_BASE_IDX 0 | |
39 | ||
edc61147 HZ |
40 | static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev); |
41 | ||
5212d163 HZ |
42 | /** |
43 | * navi10_ih_init_register_offset - Initialize register offset for ih rings | |
44 | * | |
45 | * @adev: amdgpu_device pointer | |
46 | * | |
47 | * Initialize register offset ih rings (NAVI10). | |
48 | */ | |
49 | static void navi10_ih_init_register_offset(struct amdgpu_device *adev) | |
50 | { | |
51 | struct amdgpu_ih_regs *ih_regs; | |
52 | ||
53 | if (adev->irq.ih.ring_size) { | |
54 | ih_regs = &adev->irq.ih.ih_regs; | |
55 | ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE); | |
56 | ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI); | |
57 | ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); | |
58 | ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); | |
59 | ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); | |
60 | ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR); | |
61 | ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO); | |
62 | ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI); | |
63 | ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL; | |
64 | } | |
65 | ||
66 | if (adev->irq.ih1.ring_size) { | |
67 | ih_regs = &adev->irq.ih1.ih_regs; | |
68 | ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1); | |
69 | ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1); | |
70 | ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); | |
71 | ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); | |
72 | ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); | |
73 | ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1); | |
74 | ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1; | |
75 | } | |
76 | ||
77 | if (adev->irq.ih2.ring_size) { | |
78 | ih_regs = &adev->irq.ih2.ih_regs; | |
79 | ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2); | |
80 | ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2); | |
81 | ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); | |
82 | ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); | |
83 | ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); | |
84 | ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2); | |
85 | ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2; | |
86 | } | |
87 | } | |
88 | ||
5ea6f9c2 CG |
89 | /** |
90 | * force_update_wptr_for_self_int - Force update the wptr for self interrupt | |
91 | * | |
92 | * @adev: amdgpu_device pointer | |
93 | * @threshold: threshold to trigger the wptr reporting | |
94 | * @timeout: timeout to trigger the wptr reporting | |
95 | * @enabled: Enable/disable timeout flush mechanism | |
96 | * | |
97 | * threshold input range: 0 ~ 15, default 0, | |
98 | * real_threshold = 2^threshold | |
99 | * timeout input range: 0 ~ 20, default 8, | |
100 | * real_timeout = (2^timeout) * 1024 / (socclk_freq) | |
101 | * | |
102 | * Force update wptr for self interrupt ( >= SIENNA_CICHLID). | |
103 | */ | |
104 | static void | |
105 | force_update_wptr_for_self_int(struct amdgpu_device *adev, | |
106 | u32 threshold, u32 timeout, bool enabled) | |
107 | { | |
108 | u32 ih_cntl, ih_rb_cntl; | |
109 | ||
4e8303cf | 110 | if (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) < IP_VERSION(5, 0, 3)) |
5ea6f9c2 CG |
111 | return; |
112 | ||
113 | ih_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_CNTL2); | |
114 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); | |
115 | ||
116 | ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL2, | |
117 | SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT, timeout); | |
118 | ih_cntl = REG_SET_FIELD(ih_cntl, IH_CNTL2, | |
119 | SELF_IV_FORCE_WPTR_UPDATE_ENABLE, enabled); | |
120 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, | |
121 | RB_USED_INT_THRESHOLD, threshold); | |
122 | ||
2b9ced5a RK |
123 | if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) { |
124 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, ih_rb_cntl)) | |
125 | return; | |
126 | } else { | |
127 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); | |
128 | } | |
129 | ||
5ea6f9c2 CG |
130 | ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); |
131 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, | |
132 | RB_USED_INT_THRESHOLD, threshold); | |
2b9ced5a RK |
133 | if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) { |
134 | if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, ih_rb_cntl)) | |
135 | return; | |
136 | } else { | |
137 | WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); | |
138 | } | |
139 | ||
5ea6f9c2 CG |
140 | WREG32_SOC15(OSSSYS, 0, mmIH_CNTL2, ih_cntl); |
141 | } | |
142 | ||
1ce6940e HZ |
143 | /** |
144 | * navi10_ih_toggle_ring_interrupts - toggle the interrupt ring buffer | |
145 | * | |
146 | * @adev: amdgpu_device pointer | |
147 | * @ih: amdgpu_ih_ring pointet | |
148 | * @enable: true - enable the interrupts, false - disable the interrupts | |
149 | * | |
150 | * Toggle the interrupt ring buffer (NAVI10) | |
151 | */ | |
152 | static int navi10_ih_toggle_ring_interrupts(struct amdgpu_device *adev, | |
153 | struct amdgpu_ih_ring *ih, | |
154 | bool enable) | |
155 | { | |
156 | struct amdgpu_ih_regs *ih_regs; | |
157 | uint32_t tmp; | |
158 | ||
159 | ih_regs = &ih->ih_regs; | |
160 | ||
161 | tmp = RREG32(ih_regs->ih_rb_cntl); | |
162 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); | |
71ee9236 | 163 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_GPU_TS_ENABLE, 1); |
1ce6940e HZ |
164 | /* enable_intr field is only valid in ring0 */ |
165 | if (ih == &adev->irq.ih) | |
166 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); | |
4aa7e6e0 YW |
167 | |
168 | if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) { | |
2b9ced5a | 169 | if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) |
4aa7e6e0 | 170 | return -ETIMEDOUT; |
4aa7e6e0 YW |
171 | } else { |
172 | WREG32(ih_regs->ih_rb_cntl, tmp); | |
173 | } | |
1ce6940e HZ |
174 | |
175 | if (enable) { | |
176 | ih->enabled = true; | |
177 | } else { | |
178 | /* set rptr, wptr to 0 */ | |
179 | WREG32(ih_regs->ih_rb_rptr, 0); | |
180 | WREG32(ih_regs->ih_rb_wptr, 0); | |
181 | ih->enabled = false; | |
182 | ih->rptr = 0; | |
183 | } | |
184 | ||
185 | return 0; | |
186 | } | |
187 | ||
6e7b7c7f HZ |
188 | /** |
189 | * navi10_ih_toggle_interrupts - Toggle all the available interrupt ring buffers | |
190 | * | |
191 | * @adev: amdgpu_device pointer | |
192 | * @enable: enable or disable interrupt ring buffers | |
193 | * | |
194 | * Toggle all the available interrupt ring buffers (NAVI10). | |
195 | */ | |
196 | static int navi10_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable) | |
197 | { | |
198 | struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; | |
199 | int i; | |
200 | int r; | |
201 | ||
202 | for (i = 0; i < ARRAY_SIZE(ih); i++) { | |
203 | if (ih[i]->ring_size) { | |
204 | r = navi10_ih_toggle_ring_interrupts(adev, ih[i], enable); | |
205 | if (r) | |
206 | return r; | |
207 | } | |
208 | } | |
209 | ||
210 | return 0; | |
211 | } | |
212 | ||
edc61147 HZ |
213 | static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) |
214 | { | |
215 | int rb_bufsz = order_base_2(ih->ring_size / 4); | |
216 | ||
217 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
218 | MC_SPACE, ih->use_bus_addr ? 1 : 4); | |
219 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
220 | WPTR_OVERFLOW_CLEAR, 1); | |
221 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
222 | WPTR_OVERFLOW_ENABLE, 1); | |
223 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); | |
224 | /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register | |
225 | * value is written to memory | |
226 | */ | |
227 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, | |
228 | WPTR_WRITEBACK_ENABLE, 1); | |
229 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1); | |
230 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0); | |
231 | ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0); | |
232 | ||
233 | return ih_rb_cntl; | |
234 | } | |
235 | ||
ab518012 AS |
236 | static uint32_t navi10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) |
237 | { | |
238 | u32 ih_doorbell_rtpr = 0; | |
239 | ||
240 | if (ih->use_doorbell) { | |
241 | ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, | |
242 | IH_DOORBELL_RPTR, OFFSET, | |
243 | ih->doorbell_index); | |
244 | ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, | |
245 | IH_DOORBELL_RPTR, | |
246 | ENABLE, 1); | |
247 | } else { | |
248 | ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, | |
249 | IH_DOORBELL_RPTR, | |
250 | ENABLE, 0); | |
251 | } | |
252 | return ih_doorbell_rtpr; | |
253 | } | |
254 | ||
1514cb7d HZ |
255 | /** |
256 | * navi10_ih_enable_ring - enable an ih ring buffer | |
257 | * | |
258 | * @adev: amdgpu_device pointer | |
259 | * @ih: amdgpu_ih_ring pointer | |
260 | * | |
261 | * Enable an ih ring buffer (NAVI10) | |
262 | */ | |
263 | static int navi10_ih_enable_ring(struct amdgpu_device *adev, | |
264 | struct amdgpu_ih_ring *ih) | |
265 | { | |
266 | struct amdgpu_ih_regs *ih_regs; | |
267 | uint32_t tmp; | |
268 | ||
269 | ih_regs = &ih->ih_regs; | |
270 | ||
271 | /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ | |
272 | WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8); | |
273 | WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff); | |
274 | ||
275 | tmp = RREG32(ih_regs->ih_rb_cntl); | |
276 | tmp = navi10_ih_rb_cntl(ih, tmp); | |
277 | if (ih == &adev->irq.ih) | |
278 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); | |
23eb4925 | 279 | if (ih == &adev->irq.ih1) |
1514cb7d | 280 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); |
4aa7e6e0 YW |
281 | |
282 | if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) { | |
283 | if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { | |
284 | DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); | |
285 | return -ETIMEDOUT; | |
286 | } | |
287 | } else { | |
288 | WREG32(ih_regs->ih_rb_cntl, tmp); | |
289 | } | |
1514cb7d HZ |
290 | |
291 | if (ih == &adev->irq.ih) { | |
292 | /* set the ih ring 0 writeback address whether it's enabled or not */ | |
293 | WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr)); | |
294 | WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF); | |
295 | } | |
296 | ||
297 | /* set rptr, wptr to 0 */ | |
298 | WREG32(ih_regs->ih_rb_wptr, 0); | |
299 | WREG32(ih_regs->ih_rb_rptr, 0); | |
300 | ||
301 | WREG32(ih_regs->ih_doorbell_rptr, navi10_ih_doorbell_rptr(ih)); | |
302 | ||
303 | return 0; | |
304 | } | |
305 | ||
edc61147 HZ |
306 | /** |
307 | * navi10_ih_irq_init - init and enable the interrupt ring | |
308 | * | |
309 | * @adev: amdgpu_device pointer | |
310 | * | |
311 | * Allocate a ring buffer for the interrupt controller, | |
312 | * enable the RLC, disable interrupts, enable the IH | |
313 | * ring buffer and enable it (NAVI). | |
314 | * Called at device load and reume. | |
315 | * Returns 0 for success, errors for failure. | |
316 | */ | |
317 | static int navi10_ih_irq_init(struct amdgpu_device *adev) | |
318 | { | |
fc4aa19f HZ |
319 | struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; |
320 | u32 ih_chicken; | |
6e7b7c7f | 321 | int ret; |
fc4aa19f | 322 | int i; |
edc61147 HZ |
323 | |
324 | /* disable irqs */ | |
6e7b7c7f HZ |
325 | ret = navi10_ih_toggle_interrupts(adev, false); |
326 | if (ret) | |
327 | return ret; | |
edc61147 | 328 | |
bebc0762 | 329 | adev->nbio.funcs->ih_control(adev); |
edc61147 | 330 | |
edc61147 | 331 | if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) { |
fc4aa19f | 332 | if (ih[0]->use_bus_addr) { |
4e8303cf | 333 | switch (amdgpu_ip_version(adev, OSSSYS_HWIP, 0)) { |
7c69d615 AD |
334 | case IP_VERSION(5, 0, 3): |
335 | case IP_VERSION(5, 2, 0): | |
336 | case IP_VERSION(5, 2, 1): | |
757b3af8 LG |
337 | ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN_Sienna_Cichlid); |
338 | ih_chicken = REG_SET_FIELD(ih_chicken, | |
339 | IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1); | |
340 | WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN_Sienna_Cichlid, ih_chicken); | |
341 | break; | |
342 | default: | |
343 | ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN); | |
344 | ih_chicken = REG_SET_FIELD(ih_chicken, | |
345 | IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1); | |
346 | WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken); | |
347 | break; | |
348 | } | |
edc61147 HZ |
349 | } |
350 | } | |
351 | ||
fc4aa19f HZ |
352 | for (i = 0; i < ARRAY_SIZE(ih); i++) { |
353 | if (ih[i]->ring_size) { | |
354 | ret = navi10_ih_enable_ring(adev, ih[i]); | |
355 | if (ret) | |
356 | return ret; | |
0ab176e6 | 357 | } |
ab518012 AS |
358 | } |
359 | ||
fc4aa19f HZ |
360 | /* update doorbell range for ih ring 0*/ |
361 | adev->nbio.funcs->ih_doorbell_range(adev, ih[0]->use_doorbell, | |
362 | ih[0]->doorbell_index); | |
ab518012 | 363 | |
edc61147 HZ |
364 | pci_set_master(adev->pdev); |
365 | ||
366 | /* enable interrupts */ | |
6e7b7c7f HZ |
367 | ret = navi10_ih_toggle_interrupts(adev, true); |
368 | if (ret) | |
369 | return ret; | |
5ea6f9c2 CG |
370 | /* enable wptr force update for self int */ |
371 | force_update_wptr_for_self_int(adev, 0, 8, true); | |
edc61147 | 372 | |
7f03b148 HZ |
373 | if (adev->irq.ih_soft.ring_size) |
374 | adev->irq.ih_soft.enabled = true; | |
375 | ||
7eca4006 | 376 | return 0; |
edc61147 HZ |
377 | } |
378 | ||
379 | /** | |
380 | * navi10_ih_irq_disable - disable interrupts | |
381 | * | |
382 | * @adev: amdgpu_device pointer | |
383 | * | |
384 | * Disable interrupts on the hw (NAVI10). | |
385 | */ | |
386 | static void navi10_ih_irq_disable(struct amdgpu_device *adev) | |
387 | { | |
5ea6f9c2 | 388 | force_update_wptr_for_self_int(adev, 0, 8, false); |
6e7b7c7f | 389 | navi10_ih_toggle_interrupts(adev, false); |
edc61147 HZ |
390 | |
391 | /* Wait and acknowledge irq */ | |
392 | mdelay(1); | |
393 | } | |
394 | ||
395 | /** | |
396 | * navi10_ih_get_wptr - get the IH ring buffer wptr | |
397 | * | |
398 | * @adev: amdgpu_device pointer | |
c56fb081 | 399 | * @ih: IH ring buffer to fetch wptr |
edc61147 HZ |
400 | * |
401 | * Get the IH ring buffer wptr from either the register | |
402 | * or the writeback memory buffer (NAVI10). Also check for | |
403 | * ring buffer overflow and deal with it. | |
404 | * Returns the value of the wptr. | |
405 | */ | |
406 | static u32 navi10_ih_get_wptr(struct amdgpu_device *adev, | |
407 | struct amdgpu_ih_ring *ih) | |
408 | { | |
2d2fbf68 HZ |
409 | u32 wptr, tmp; |
410 | struct amdgpu_ih_regs *ih_regs; | |
edc61147 | 411 | |
de8341ee | 412 | if (ih == &adev->irq.ih || ih == &adev->irq.ih_soft) { |
23eb4925 PY |
413 | /* Only ring0 supports writeback. On other rings fall back |
414 | * to register-based code with overflow checking below. | |
de8341ee MJ |
415 | * ih_soft ring doesn't have any backing hardware registers, |
416 | * update wptr and return. | |
23eb4925 PY |
417 | */ |
418 | wptr = le32_to_cpu(*ih->wptr_cpu); | |
edc61147 | 419 | |
23eb4925 PY |
420 | if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) |
421 | goto out; | |
422 | } | |
edc61147 | 423 | |
23eb4925 PY |
424 | ih_regs = &ih->ih_regs; |
425 | ||
426 | /* Double check that the overflow wasn't already cleared. */ | |
2d2fbf68 | 427 | wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); |
edc61147 HZ |
428 | if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) |
429 | goto out; | |
430 | wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); | |
431 | ||
432 | /* When a ring buffer overflow happen start parsing interrupt | |
433 | * from the last not overwritten vector (wptr + 32). Hopefully | |
434 | * this should allow us to catch up. | |
435 | */ | |
436 | tmp = (wptr + 32) & ih->ptr_mask; | |
437 | dev_warn(adev->dev, "IH ring buffer overflow " | |
438 | "(0x%08X, 0x%08X, 0x%08X)\n", | |
439 | wptr, ih->rptr, tmp); | |
440 | ih->rptr = tmp; | |
441 | ||
2d2fbf68 | 442 | tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl); |
edc61147 | 443 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); |
2d2fbf68 | 444 | WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); |
73302562 FV |
445 | |
446 | /* Unset the CLEAR_OVERFLOW bit immediately so new overflows | |
447 | * can be detected. | |
448 | */ | |
449 | tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 0); | |
450 | WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); | |
edc61147 HZ |
451 | out: |
452 | return (wptr & ih->ptr_mask); | |
453 | } | |
454 | ||
022b6518 SD |
455 | /** |
456 | * navi10_ih_irq_rearm - rearm IRQ if lost | |
457 | * | |
458 | * @adev: amdgpu_device pointer | |
c56fb081 | 459 | * @ih: IH ring to match |
022b6518 SD |
460 | * |
461 | */ | |
462 | static void navi10_ih_irq_rearm(struct amdgpu_device *adev, | |
463 | struct amdgpu_ih_ring *ih) | |
464 | { | |
022b6518 SD |
465 | uint32_t v = 0; |
466 | uint32_t i = 0; | |
2d2fbf68 | 467 | struct amdgpu_ih_regs *ih_regs; |
022b6518 | 468 | |
2d2fbf68 | 469 | ih_regs = &ih->ih_regs; |
022b6518 SD |
470 | |
471 | /* Rearm IRQ / re-write doorbell if doorbell write is lost */ | |
472 | for (i = 0; i < MAX_REARM_RETRY; i++) { | |
2d2fbf68 | 473 | v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr); |
022b6518 SD |
474 | if ((v < ih->ring_size) && (v != ih->rptr)) |
475 | WDOORBELL32(ih->doorbell_index, ih->rptr); | |
476 | else | |
477 | break; | |
478 | } | |
479 | } | |
480 | ||
edc61147 HZ |
481 | /** |
482 | * navi10_ih_set_rptr - set the IH ring buffer rptr | |
483 | * | |
484 | * @adev: amdgpu_device pointer | |
485 | * | |
c56fb081 | 486 | * @ih: IH ring buffer to set rptr |
edc61147 HZ |
487 | * Set the IH ring buffer rptr. |
488 | */ | |
489 | static void navi10_ih_set_rptr(struct amdgpu_device *adev, | |
490 | struct amdgpu_ih_ring *ih) | |
491 | { | |
2d2fbf68 HZ |
492 | struct amdgpu_ih_regs *ih_regs; |
493 | ||
de8341ee MJ |
494 | if (ih == &adev->irq.ih_soft) |
495 | return; | |
496 | ||
edc61147 HZ |
497 | if (ih->use_doorbell) { |
498 | /* XXX check if swapping is necessary on BE */ | |
499 | *ih->rptr_cpu = ih->rptr; | |
500 | WDOORBELL32(ih->doorbell_index, ih->rptr); | |
022b6518 SD |
501 | |
502 | if (amdgpu_sriov_vf(adev)) | |
503 | navi10_ih_irq_rearm(adev, ih); | |
2d2fbf68 HZ |
504 | } else { |
505 | ih_regs = &ih->ih_regs; | |
506 | WREG32(ih_regs->ih_rb_rptr, ih->rptr); | |
ab518012 AS |
507 | } |
508 | } | |
509 | ||
510 | /** | |
511 | * navi10_ih_self_irq - dispatch work for ring 1 and 2 | |
512 | * | |
513 | * @adev: amdgpu_device pointer | |
514 | * @source: irq source | |
515 | * @entry: IV with WPTR update | |
516 | * | |
517 | * Update the WPTR from the IV and schedule work to handle the entries. | |
518 | */ | |
519 | static int navi10_ih_self_irq(struct amdgpu_device *adev, | |
520 | struct amdgpu_irq_src *source, | |
521 | struct amdgpu_iv_entry *entry) | |
522 | { | |
ab518012 AS |
523 | switch (entry->ring_id) { |
524 | case 1: | |
ab518012 AS |
525 | schedule_work(&adev->irq.ih1_work); |
526 | break; | |
527 | case 2: | |
ab518012 AS |
528 | schedule_work(&adev->irq.ih2_work); |
529 | break; | |
530 | default: break; | |
531 | } | |
532 | return 0; | |
533 | } | |
534 | ||
535 | static const struct amdgpu_irq_src_funcs navi10_ih_self_irq_funcs = { | |
536 | .process = navi10_ih_self_irq, | |
537 | }; | |
538 | ||
539 | static void navi10_ih_set_self_irq_funcs(struct amdgpu_device *adev) | |
540 | { | |
541 | adev->irq.self_irq.num_types = 0; | |
542 | adev->irq.self_irq.funcs = &navi10_ih_self_irq_funcs; | |
edc61147 HZ |
543 | } |
544 | ||
545 | static int navi10_ih_early_init(void *handle) | |
546 | { | |
547 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
548 | ||
549 | navi10_ih_set_interrupt_funcs(adev); | |
ab518012 | 550 | navi10_ih_set_self_irq_funcs(adev); |
edc61147 HZ |
551 | return 0; |
552 | } | |
553 | ||
554 | static int navi10_ih_sw_init(void *handle) | |
555 | { | |
556 | int r; | |
557 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
558 | bool use_bus_addr; | |
559 | ||
ab518012 AS |
560 | r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0, |
561 | &adev->irq.self_irq); | |
562 | ||
563 | if (r) | |
564 | return r; | |
565 | ||
edc61147 HZ |
566 | /* use gpu virtual address for ih ring |
567 | * until ih_checken is programmed to allow | |
568 | * use bus address for ih ring by psp bl */ | |
bf13cb1f HR |
569 | if ((adev->flags & AMD_IS_APU) || |
570 | (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) | |
571 | use_bus_addr = false; | |
572 | else | |
573 | use_bus_addr = true; | |
bf80d34b | 574 | r = amdgpu_ih_ring_init(adev, &adev->irq.ih, IH_RING_SIZE, use_bus_addr); |
edc61147 HZ |
575 | if (r) |
576 | return r; | |
577 | ||
578 | adev->irq.ih.use_doorbell = true; | |
579 | adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; | |
580 | ||
abb6fccb AS |
581 | adev->irq.ih1.ring_size = 0; |
582 | adev->irq.ih2.ring_size = 0; | |
ab518012 | 583 | |
a362976b HZ |
584 | /* initialize ih control registers offset */ |
585 | navi10_ih_init_register_offset(adev); | |
586 | ||
bf80d34b | 587 | r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, IH_SW_RING_SIZE, true); |
d4581f7d CK |
588 | if (r) |
589 | return r; | |
590 | ||
edc61147 HZ |
591 | r = amdgpu_irq_init(adev); |
592 | ||
593 | return r; | |
594 | } | |
595 | ||
596 | static int navi10_ih_sw_fini(void *handle) | |
597 | { | |
598 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
599 | ||
72c8c97b | 600 | amdgpu_irq_fini_sw(adev); |
edc61147 HZ |
601 | |
602 | return 0; | |
603 | } | |
604 | ||
605 | static int navi10_ih_hw_init(void *handle) | |
606 | { | |
edc61147 HZ |
607 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
608 | ||
364d453f | 609 | return navi10_ih_irq_init(adev); |
edc61147 HZ |
610 | } |
611 | ||
612 | static int navi10_ih_hw_fini(void *handle) | |
613 | { | |
614 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
615 | ||
616 | navi10_ih_irq_disable(adev); | |
617 | ||
618 | return 0; | |
619 | } | |
620 | ||
621 | static int navi10_ih_suspend(void *handle) | |
622 | { | |
623 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
624 | ||
625 | return navi10_ih_hw_fini(adev); | |
626 | } | |
627 | ||
628 | static int navi10_ih_resume(void *handle) | |
629 | { | |
630 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
631 | ||
632 | return navi10_ih_hw_init(adev); | |
633 | } | |
634 | ||
635 | static bool navi10_ih_is_idle(void *handle) | |
636 | { | |
637 | /* todo */ | |
638 | return true; | |
639 | } | |
640 | ||
641 | static int navi10_ih_wait_for_idle(void *handle) | |
642 | { | |
643 | /* todo */ | |
644 | return -ETIMEDOUT; | |
645 | } | |
646 | ||
647 | static int navi10_ih_soft_reset(void *handle) | |
648 | { | |
649 | /* todo */ | |
650 | return 0; | |
651 | } | |
652 | ||
653 | static void navi10_ih_update_clockgating_state(struct amdgpu_device *adev, | |
654 | bool enable) | |
655 | { | |
656 | uint32_t data, def, field_val; | |
657 | ||
658 | if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) { | |
659 | def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL); | |
660 | field_val = enable ? 0 : 1; | |
661 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
662 | DBUS_MUX_CLK_SOFT_OVERRIDE, field_val); | |
663 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
664 | OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val); | |
665 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
666 | LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val); | |
667 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
668 | DYN_CLK_SOFT_OVERRIDE, field_val); | |
669 | data = REG_SET_FIELD(data, IH_CLK_CTRL, | |
670 | REG_CLK_SOFT_OVERRIDE, field_val); | |
671 | if (def != data) | |
672 | WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data); | |
673 | } | |
edc61147 HZ |
674 | } |
675 | ||
676 | static int navi10_ih_set_clockgating_state(void *handle, | |
677 | enum amd_clockgating_state state) | |
678 | { | |
679 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
680 | ||
681 | navi10_ih_update_clockgating_state(adev, | |
a9d4fe2f | 682 | state == AMD_CG_STATE_GATE); |
edc61147 HZ |
683 | return 0; |
684 | } | |
685 | ||
686 | static int navi10_ih_set_powergating_state(void *handle, | |
687 | enum amd_powergating_state state) | |
688 | { | |
689 | return 0; | |
690 | } | |
691 | ||
25faeddc | 692 | static void navi10_ih_get_clockgating_state(void *handle, u64 *flags) |
edc61147 HZ |
693 | { |
694 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; | |
695 | ||
696 | if (!RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL)) | |
697 | *flags |= AMD_CG_SUPPORT_IH_CG; | |
edc61147 HZ |
698 | } |
699 | ||
700 | static const struct amd_ip_funcs navi10_ih_ip_funcs = { | |
701 | .name = "navi10_ih", | |
702 | .early_init = navi10_ih_early_init, | |
703 | .late_init = NULL, | |
704 | .sw_init = navi10_ih_sw_init, | |
705 | .sw_fini = navi10_ih_sw_fini, | |
706 | .hw_init = navi10_ih_hw_init, | |
707 | .hw_fini = navi10_ih_hw_fini, | |
708 | .suspend = navi10_ih_suspend, | |
709 | .resume = navi10_ih_resume, | |
710 | .is_idle = navi10_ih_is_idle, | |
711 | .wait_for_idle = navi10_ih_wait_for_idle, | |
712 | .soft_reset = navi10_ih_soft_reset, | |
713 | .set_clockgating_state = navi10_ih_set_clockgating_state, | |
714 | .set_powergating_state = navi10_ih_set_powergating_state, | |
715 | .get_clockgating_state = navi10_ih_get_clockgating_state, | |
716 | }; | |
717 | ||
718 | static const struct amdgpu_ih_funcs navi10_ih_funcs = { | |
719 | .get_wptr = navi10_ih_get_wptr, | |
40838281 | 720 | .decode_iv = amdgpu_ih_decode_iv_helper, |
3c2d6ea2 | 721 | .decode_iv_ts = amdgpu_ih_decode_iv_ts_helper, |
edc61147 HZ |
722 | .set_rptr = navi10_ih_set_rptr |
723 | }; | |
724 | ||
725 | static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev) | |
726 | { | |
727 | if (adev->irq.ih_funcs == NULL) | |
728 | adev->irq.ih_funcs = &navi10_ih_funcs; | |
729 | } | |
730 | ||
731 | const struct amdgpu_ip_block_version navi10_ih_ip_block = | |
732 | { | |
733 | .type = AMD_IP_BLOCK_TYPE_IH, | |
734 | .major = 5, | |
735 | .minor = 0, | |
736 | .rev = 0, | |
737 | .funcs = &navi10_ih_ip_funcs, | |
738 | }; |