Commit | Line | Data |
---|---|---|
770d13b1 CK |
1 | /* |
2 | * Copyright 2018 Advanced Micro Devices, Inc. | |
3 | * All Rights Reserved. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the | |
7 | * "Software"), to deal in the Software without restriction, including | |
8 | * without limitation the rights to use, copy, modify, merge, publish, | |
9 | * distribute, sub license, and/or sell copies of the Software, and to | |
10 | * permit persons to whom the Software is furnished to do so, subject to | |
11 | * the following conditions: | |
12 | * | |
13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
15 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
16 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
17 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
18 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
19 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
20 | * | |
21 | * The above copyright notice and this permission notice (including the | |
22 | * next paragraph) shall be included in all copies or substantial portions | |
23 | * of the Software. | |
24 | * | |
25 | */ | |
26 | #ifndef __AMDGPU_GMC_H__ | |
27 | #define __AMDGPU_GMC_H__ | |
28 | ||
29 | #include <linux/types.h> | |
30 | ||
31 | #include "amdgpu_irq.h" | |
32 | ||
ad9a5b78 CK |
33 | /* VA hole for 48bit addresses on Vega10 */ |
34 | #define AMDGPU_GMC_HOLE_START 0x0000800000000000ULL | |
35 | #define AMDGPU_GMC_HOLE_END 0xffff800000000000ULL | |
36 | ||
37 | /* | |
38 | * Hardware is programmed as if the hole doesn't exists with start and end | |
39 | * address values. | |
40 | * | |
41 | * This mask is used to remove the upper 16bits of the VA and so come up with | |
42 | * the linear addr value. | |
43 | */ | |
44 | #define AMDGPU_GMC_HOLE_MASK 0x0000ffffffffffffULL | |
45 | ||
c1a8abd9 CK |
46 | /* |
47 | * Ring size as power of two for the log of recent faults. | |
48 | */ | |
49 | #define AMDGPU_GMC_FAULT_RING_ORDER 8 | |
50 | #define AMDGPU_GMC_FAULT_RING_SIZE (1 << AMDGPU_GMC_FAULT_RING_ORDER) | |
51 | ||
52 | /* | |
53 | * Hash size as power of two for the log of recent faults | |
54 | */ | |
55 | #define AMDGPU_GMC_FAULT_HASH_ORDER 8 | |
56 | #define AMDGPU_GMC_FAULT_HASH_SIZE (1 << AMDGPU_GMC_FAULT_HASH_ORDER) | |
57 | ||
58 | /* | |
59 | * Number of IH timestamp ticks until a fault is considered handled | |
60 | */ | |
61 | #define AMDGPU_GMC_FAULT_TIMEOUT 5000ULL | |
62 | ||
770d13b1 CK |
63 | struct firmware; |
64 | ||
c1a8abd9 CK |
65 | /* |
66 | * GMC page fault information | |
67 | */ | |
68 | struct amdgpu_gmc_fault { | |
69 | uint64_t timestamp; | |
70 | uint64_t next:AMDGPU_GMC_FAULT_RING_ORDER; | |
71 | uint64_t key:52; | |
72 | }; | |
73 | ||
770d13b1 CK |
74 | /* |
75 | * VMHUB structures, functions & helpers | |
76 | */ | |
2577db91 HR |
77 | struct amdgpu_vmhub_funcs { |
78 | void (*print_l2_protection_fault_status)(struct amdgpu_device *adev, | |
79 | uint32_t status); | |
caa9f483 | 80 | uint32_t (*get_invalidate_req)(unsigned int vmid, uint32_t flush_type); |
2577db91 HR |
81 | }; |
82 | ||
770d13b1 CK |
83 | struct amdgpu_vmhub { |
84 | uint32_t ctx0_ptb_addr_lo32; | |
85 | uint32_t ctx0_ptb_addr_hi32; | |
6c2c8972 | 86 | uint32_t vm_inv_eng0_sem; |
770d13b1 CK |
87 | uint32_t vm_inv_eng0_req; |
88 | uint32_t vm_inv_eng0_ack; | |
89 | uint32_t vm_context0_cntl; | |
90 | uint32_t vm_l2_pro_fault_status; | |
91 | uint32_t vm_l2_pro_fault_cntl; | |
1f9d56c3 HR |
92 | |
93 | /* | |
94 | * store the register distances between two continuous context domain | |
95 | * and invalidation engine. | |
96 | */ | |
97 | uint32_t ctx_distance; | |
98 | uint32_t ctx_addr_distance; /* include LO32/HI32 */ | |
99 | uint32_t eng_distance; | |
100 | uint32_t eng_addr_distance; /* include LO32/HI32 */ | |
5befb6fc HR |
101 | |
102 | uint32_t vm_cntx_cntl_vm_fault; | |
2577db91 HR |
103 | |
104 | const struct amdgpu_vmhub_funcs *vmhub_funcs; | |
770d13b1 CK |
105 | }; |
106 | ||
107 | /* | |
108 | * GPU MC structures, functions & helpers | |
109 | */ | |
132f34e4 CK |
110 | struct amdgpu_gmc_funcs { |
111 | /* flush the vm tlb via mmio */ | |
3ff98548 OZ |
112 | void (*flush_gpu_tlb)(struct amdgpu_device *adev, uint32_t vmid, |
113 | uint32_t vmhub, uint32_t flush_type); | |
ea930000 AS |
114 | /* flush the vm tlb via pasid */ |
115 | int (*flush_gpu_tlb_pasid)(struct amdgpu_device *adev, uint16_t pasid, | |
116 | uint32_t flush_type, bool all_hub); | |
7ef11047 CK |
117 | /* flush the vm tlb via ring */ |
118 | uint64_t (*emit_flush_gpu_tlb)(struct amdgpu_ring *ring, unsigned vmid, | |
c633c00b CK |
119 | uint64_t pd_addr); |
120 | /* Change the VMID -> PASID mapping */ | |
121 | void (*emit_pasid_mapping)(struct amdgpu_ring *ring, unsigned vmid, | |
122 | unsigned pasid); | |
132f34e4 CK |
123 | /* enable/disable PRT support */ |
124 | void (*set_prt)(struct amdgpu_device *adev, bool enable); | |
71776b6d CK |
125 | /* map mtype to hardware flags */ |
126 | uint64_t (*map_mtype)(struct amdgpu_device *adev, uint32_t flags); | |
132f34e4 CK |
127 | /* get the pde for a given mc addr */ |
128 | void (*get_vm_pde)(struct amdgpu_device *adev, int level, | |
129 | u64 *dst, u64 *flags); | |
cbfae36c CK |
130 | /* get the pte flags to use for a BO VA mapping */ |
131 | void (*get_vm_pte)(struct amdgpu_device *adev, | |
132 | struct amdgpu_bo_va_mapping *mapping, | |
133 | uint64_t *flags); | |
dd285c5d AD |
134 | /* get the amount of memory used by the vbios for pre-OS console */ |
135 | unsigned int (*get_vbios_fb_size)(struct amdgpu_device *adev); | |
132f34e4 CK |
136 | }; |
137 | ||
76a5b367 AD |
138 | struct amdgpu_xgmi { |
139 | /* from psp */ | |
dd3c45d3 | 140 | u64 node_id; |
76a5b367 AD |
141 | u64 hive_id; |
142 | /* fixed per family */ | |
143 | u64 node_segment_size; | |
144 | /* physical node (0-3) */ | |
145 | unsigned physical_node_id; | |
146 | /* number of nodes (0-4) */ | |
147 | unsigned num_physical_nodes; | |
fb30fc59 SL |
148 | /* gpu list in the same hive */ |
149 | struct list_head head; | |
47622ba0 | 150 | bool supported; |
029fbd43 | 151 | struct ras_common_if *ras_if; |
76a5b367 AD |
152 | }; |
153 | ||
770d13b1 | 154 | struct amdgpu_gmc { |
f6baa074 OZ |
155 | /* FB's physical address in MMIO space (for CPU to |
156 | * map FB). This is different compared to the agp/ | |
157 | * gart/vram_start/end field as the later is from | |
158 | * GPU's view and aper_base is from CPU's view. | |
159 | */ | |
770d13b1 CK |
160 | resource_size_t aper_size; |
161 | resource_size_t aper_base; | |
162 | /* for some chips with <= 32MB we need to lie | |
163 | * about vram size near mc fb location */ | |
164 | u64 mc_vram_size; | |
165 | u64 visible_vram_size; | |
f6baa074 OZ |
166 | /* AGP aperture start and end in MC address space |
167 | * Driver find a hole in the MC address space | |
168 | * to place AGP by setting MC_VM_AGP_BOT/TOP registers | |
169 | * Under VMID0, logical address == MC address. AGP | |
170 | * aperture maps to physical bus or IOVA addressed. | |
171 | * AGP aperture is used to simulate FB in ZFB case. | |
172 | * AGP aperture is also used for page table in system | |
173 | * memory (mainly for APU). | |
174 | * | |
175 | */ | |
d76364fc CK |
176 | u64 agp_size; |
177 | u64 agp_start; | |
178 | u64 agp_end; | |
f6baa074 OZ |
179 | /* GART aperture start and end in MC address space |
180 | * Driver find a hole in the MC address space | |
181 | * to place GART by setting VM_CONTEXT0_PAGE_TABLE_START/END_ADDR | |
182 | * registers | |
183 | * Under VMID0, logical address inside GART aperture will | |
184 | * be translated through gpuvm gart page table to access | |
185 | * paged system memory | |
186 | */ | |
770d13b1 CK |
187 | u64 gart_size; |
188 | u64 gart_start; | |
189 | u64 gart_end; | |
f6baa074 OZ |
190 | /* Frame buffer aperture of this GPU device. Different from |
191 | * fb_start (see below), this only covers the local GPU device. | |
192 | * Driver get fb_start from MC_VM_FB_LOCATION_BASE (set by vbios) | |
193 | * and calculate vram_start of this local device by adding an | |
194 | * offset inside the XGMI hive. | |
195 | * Under VMID0, logical address == MC address | |
196 | */ | |
770d13b1 CK |
197 | u64 vram_start; |
198 | u64 vram_end; | |
6fdd68b1 AD |
199 | /* FB region , it's same as local vram region in single GPU, in XGMI |
200 | * configuration, this region covers all GPUs in the same hive , | |
201 | * each GPU in the hive has the same view of this FB region . | |
202 | * GPU0's vram starts at offset (0 * segment size) , | |
203 | * GPU1 starts at offset (1 * segment size), etc. | |
204 | */ | |
205 | u64 fb_start; | |
206 | u64 fb_end; | |
770d13b1 CK |
207 | unsigned vram_width; |
208 | u64 real_vram_size; | |
209 | int vram_mtrr; | |
210 | u64 mc_mask; | |
211 | const struct firmware *fw; /* MC firmware */ | |
212 | uint32_t fw_version; | |
213 | struct amdgpu_irq_src vm_fault; | |
214 | uint32_t vram_type; | |
ad02e08e | 215 | uint8_t vram_vendor; |
770d13b1 CK |
216 | uint32_t srbm_soft_reset; |
217 | bool prt_warning; | |
c2ecd79b | 218 | uint32_t sdpif_register; |
770d13b1 CK |
219 | /* apertures */ |
220 | u64 shared_aperture_start; | |
221 | u64 shared_aperture_end; | |
222 | u64 private_aperture_start; | |
223 | u64 private_aperture_end; | |
224 | /* protects concurrent invalidation */ | |
225 | spinlock_t invalidate_lock; | |
226 | bool translate_further; | |
b97dfa27 | 227 | struct kfd_vm_fault_info *vm_fault_info; |
228 | atomic_t vm_fault_info_updated; | |
132f34e4 | 229 | |
c1a8abd9 CK |
230 | struct amdgpu_gmc_fault fault_ring[AMDGPU_GMC_FAULT_RING_SIZE]; |
231 | struct { | |
232 | uint64_t idx:AMDGPU_GMC_FAULT_RING_ORDER; | |
233 | } fault_hash[AMDGPU_GMC_FAULT_HASH_SIZE]; | |
234 | uint64_t last_fault:AMDGPU_GMC_FAULT_RING_ORDER; | |
235 | ||
c6252390 LT |
236 | bool tmz_enabled; |
237 | ||
132f34e4 | 238 | const struct amdgpu_gmc_funcs *gmc_funcs; |
76a5b367 AD |
239 | |
240 | struct amdgpu_xgmi xgmi; | |
791c4769 | 241 | struct amdgpu_irq_src ecc_irq; |
9b498efa | 242 | int noretry; |
770d13b1 CK |
243 | }; |
244 | ||
3ff98548 | 245 | #define amdgpu_gmc_flush_gpu_tlb(adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type))) |
ea930000 AS |
246 | #define amdgpu_gmc_flush_gpu_tlb_pasid(adev, pasid, type, allhub) \ |
247 | ((adev)->gmc.gmc_funcs->flush_gpu_tlb_pasid \ | |
248 | ((adev), (pasid), (type), (allhub))) | |
c082b998 HR |
249 | #define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr)) |
250 | #define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid)) | |
71776b6d | 251 | #define amdgpu_gmc_map_mtype(adev, flags) (adev)->gmc.gmc_funcs->map_mtype((adev),(flags)) |
c082b998 | 252 | #define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags)) |
cbfae36c | 253 | #define amdgpu_gmc_get_vm_pte(adev, mapping, flags) (adev)->gmc.gmc_funcs->get_vm_pte((adev), (mapping), (flags)) |
dd285c5d | 254 | #define amdgpu_gmc_get_vbios_fb_size(adev) (adev)->gmc.gmc_funcs->get_vbios_fb_size((adev)) |
c082b998 | 255 | |
c8c5e569 AG |
256 | /** |
257 | * amdgpu_gmc_vram_full_visible - Check if full VRAM is visible through the BAR | |
258 | * | |
259 | * @adev: amdgpu_device pointer | |
260 | * | |
261 | * Returns: | |
262 | * True if full VRAM is visible through the BAR | |
263 | */ | |
264 | static inline bool amdgpu_gmc_vram_full_visible(struct amdgpu_gmc *gmc) | |
265 | { | |
266 | WARN_ON(gmc->real_vram_size < gmc->visible_vram_size); | |
267 | ||
268 | return (gmc->real_vram_size == gmc->visible_vram_size); | |
269 | } | |
270 | ||
ad9a5b78 CK |
271 | /** |
272 | * amdgpu_gmc_sign_extend - sign extend the given gmc address | |
273 | * | |
274 | * @addr: address to extend | |
275 | */ | |
276 | static inline uint64_t amdgpu_gmc_sign_extend(uint64_t addr) | |
277 | { | |
278 | if (addr >= AMDGPU_GMC_HOLE_START) | |
279 | addr |= AMDGPU_GMC_HOLE_END; | |
280 | ||
281 | return addr; | |
282 | } | |
283 | ||
24a8d289 CK |
284 | void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level, |
285 | uint64_t *addr, uint64_t *flags); | |
6490bd76 YZ |
286 | int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, |
287 | uint32_t gpu_page_idx, uint64_t addr, | |
288 | uint64_t flags); | |
11c3a249 | 289 | uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo); |
485fc361 | 290 | uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo); |
961c75cf CK |
291 | void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, |
292 | u64 base); | |
293 | void amdgpu_gmc_gart_location(struct amdgpu_device *adev, | |
294 | struct amdgpu_gmc *mc); | |
d76364fc CK |
295 | void amdgpu_gmc_agp_location(struct amdgpu_device *adev, |
296 | struct amdgpu_gmc *mc); | |
c1a8abd9 CK |
297 | bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, |
298 | uint16_t pasid, uint64_t timestamp); | |
ba083492 | 299 | int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev); |
2adf1344 | 300 | void amdgpu_gmc_ras_fini(struct amdgpu_device *adev); |
bdbe90f0 | 301 | int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev); |
11c3a249 | 302 | |
c6252390 | 303 | extern void amdgpu_gmc_tmz_set(struct amdgpu_device *adev); |
9b498efa | 304 | extern void amdgpu_gmc_noretry_set(struct amdgpu_device *adev); |
c6252390 | 305 | |
f2c1b5c1 HR |
306 | extern void |
307 | amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type, | |
308 | bool enable); | |
309 | ||
dd285c5d AD |
310 | void amdgpu_gmc_get_vbios_allocations(struct amdgpu_device *adev); |
311 | ||
770d13b1 | 312 | #endif |