Commit | Line | Data |
---|---|---|
bb990bb0 CK |
1 | /* |
2 | * Copyright 2016 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * Authors: Christian König | |
23 | */ | |
24 | ||
f700b18c CK |
25 | #include <drm/ttm/ttm_range_manager.h> |
26 | ||
bb990bb0 CK |
27 | #include "amdgpu.h" |
28 | ||
d624e1bf | 29 | struct amdgpu_gtt_node { |
d624e1bf | 30 | struct ttm_buffer_object *tbo; |
f700b18c | 31 | struct ttm_range_mgr_node base; |
d624e1bf CK |
32 | }; |
33 | ||
a614b336 CK |
34 | static inline struct amdgpu_gtt_mgr * |
35 | to_gtt_mgr(struct ttm_resource_manager *man) | |
0af135b8 DA |
36 | { |
37 | return container_of(man, struct amdgpu_gtt_mgr, manager); | |
38 | } | |
39 | ||
d624e1bf CK |
40 | static inline struct amdgpu_gtt_node * |
41 | to_amdgpu_gtt_node(struct ttm_resource *res) | |
42 | { | |
cb1c8146 | 43 | return container_of(res, struct amdgpu_gtt_node, base.base); |
d624e1bf | 44 | } |
c1c7ce8f | 45 | |
55c374e9 KR |
46 | /** |
47 | * DOC: mem_info_gtt_total | |
48 | * | |
49 | * The amdgpu driver provides a sysfs API for reporting current total size of | |
50 | * the GTT. | |
51 | * The file mem_info_gtt_total is used for this, and returns the total size of | |
52 | * the GTT block, in bytes | |
53 | */ | |
54 | static ssize_t amdgpu_mem_info_gtt_total_show(struct device *dev, | |
a614b336 CK |
55 | struct device_attribute *attr, |
56 | char *buf) | |
55c374e9 KR |
57 | { |
58 | struct drm_device *ddev = dev_get_drvdata(dev); | |
1348969a | 59 | struct amdgpu_device *adev = drm_to_adev(ddev); |
a614b336 | 60 | struct ttm_resource_manager *man; |
55c374e9 | 61 | |
a614b336 | 62 | man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); |
36000c7a | 63 | return sysfs_emit(buf, "%llu\n", man->size * PAGE_SIZE); |
55c374e9 KR |
64 | } |
65 | ||
66 | /** | |
67 | * DOC: mem_info_gtt_used | |
68 | * | |
69 | * The amdgpu driver provides a sysfs API for reporting current total amount of | |
70 | * used GTT. | |
71 | * The file mem_info_gtt_used is used for this, and returns the current used | |
72 | * size of the GTT block, in bytes | |
73 | */ | |
74 | static ssize_t amdgpu_mem_info_gtt_used_show(struct device *dev, | |
a614b336 CK |
75 | struct device_attribute *attr, |
76 | char *buf) | |
55c374e9 KR |
77 | { |
78 | struct drm_device *ddev = dev_get_drvdata(dev); | |
1348969a | 79 | struct amdgpu_device *adev = drm_to_adev(ddev); |
a614b336 | 80 | struct ttm_resource_manager *man; |
55c374e9 | 81 | |
a614b336 | 82 | man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); |
36000c7a | 83 | return sysfs_emit(buf, "%llu\n", amdgpu_gtt_mgr_usage(man)); |
55c374e9 KR |
84 | } |
85 | ||
86 | static DEVICE_ATTR(mem_info_gtt_total, S_IRUGO, | |
87 | amdgpu_mem_info_gtt_total_show, NULL); | |
88 | static DEVICE_ATTR(mem_info_gtt_used, S_IRUGO, | |
89 | amdgpu_mem_info_gtt_used_show, NULL); | |
90 | ||
35bba831 AG |
91 | static struct attribute *amdgpu_gtt_mgr_attributes[] = { |
92 | &dev_attr_mem_info_gtt_total.attr, | |
93 | &dev_attr_mem_info_gtt_used.attr, | |
94 | NULL | |
95 | }; | |
96 | ||
97 | const struct attribute_group amdgpu_gtt_mgr_attr_group = { | |
98 | .attrs = amdgpu_gtt_mgr_attributes | |
99 | }; | |
100 | ||
98a7f88c | 101 | /** |
3da917b6 | 102 | * amdgpu_gtt_mgr_has_gart_addr - Check if mem has address space |
98a7f88c | 103 | * |
cb1c8146 | 104 | * @res: the mem object to check |
98a7f88c CK |
105 | * |
106 | * Check if a mem object has already address space allocated. | |
107 | */ | |
cb1c8146 | 108 | bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_resource *res) |
98a7f88c | 109 | { |
cb1c8146 | 110 | struct amdgpu_gtt_node *node = to_amdgpu_gtt_node(res); |
d624e1bf | 111 | |
f700b18c | 112 | return drm_mm_node_allocated(&node->base.mm_nodes[0]); |
bb990bb0 CK |
113 | } |
114 | ||
115 | /** | |
116 | * amdgpu_gtt_mgr_new - allocate a new node | |
117 | * | |
118 | * @man: TTM memory type manager | |
119 | * @tbo: TTM BO we need this range for | |
120 | * @place: placement flags and restrictions | |
6333a495 | 121 | * @res: the resulting mem object |
bb990bb0 CK |
122 | * |
123 | * Dummy, allocate the node but no space for it yet. | |
124 | */ | |
9de59bc2 | 125 | static int amdgpu_gtt_mgr_new(struct ttm_resource_manager *man, |
bb990bb0 CK |
126 | struct ttm_buffer_object *tbo, |
127 | const struct ttm_place *place, | |
cb1c8146 | 128 | struct ttm_resource **res) |
bb990bb0 | 129 | { |
0af135b8 | 130 | struct amdgpu_gtt_mgr *mgr = to_gtt_mgr(man); |
cb1c8146 | 131 | uint32_t num_pages = PFN_UP(tbo->base.size); |
c1c7ce8f | 132 | struct amdgpu_gtt_node *node; |
bb990bb0 CK |
133 | int r; |
134 | ||
2b70af79 LY |
135 | if (!(place->flags & TTM_PL_FLAG_TEMPORARY) && |
136 | atomic64_add_return(num_pages, &mgr->used) > man->size) { | |
137 | atomic64_sub(num_pages, &mgr->used); | |
138 | return -ENOSPC; | |
bb990bb0 | 139 | } |
bb990bb0 | 140 | |
f700b18c | 141 | node = kzalloc(struct_size(node, base.mm_nodes, 1), GFP_KERNEL); |
47e50d5c FC |
142 | if (!node) { |
143 | r = -ENOMEM; | |
144 | goto err_out; | |
145 | } | |
bb990bb0 | 146 | |
c1c7ce8f | 147 | node->tbo = tbo; |
f700b18c | 148 | ttm_resource_init(tbo, place, &node->base.base); |
bb990bb0 | 149 | |
d624e1bf CK |
150 | if (place->lpfn) { |
151 | spin_lock(&mgr->lock); | |
f700b18c CK |
152 | r = drm_mm_insert_node_in_range(&mgr->mm, |
153 | &node->base.mm_nodes[0], | |
cb1c8146 CK |
154 | num_pages, tbo->page_alignment, |
155 | 0, place->fpfn, place->lpfn, | |
d624e1bf CK |
156 | DRM_MM_INSERT_BEST); |
157 | spin_unlock(&mgr->lock); | |
158 | if (unlikely(r)) | |
159 | goto err_free; | |
bb990bb0 | 160 | |
cb1c8146 | 161 | node->base.base.start = node->base.mm_nodes[0].start; |
d624e1bf | 162 | } else { |
f700b18c | 163 | node->base.mm_nodes[0].start = 0; |
cb1c8146 CK |
164 | node->base.mm_nodes[0].size = node->base.base.num_pages; |
165 | node->base.base.start = AMDGPU_BO_INVALID_OFFSET; | |
d624e1bf | 166 | } |
bb990bb0 | 167 | |
cb1c8146 | 168 | *res = &node->base.base; |
bb990bb0 | 169 | return 0; |
1e691e24 CK |
170 | |
171 | err_free: | |
de3688e4 | 172 | ttm_resource_fini(man, &node->base.base); |
1e691e24 CK |
173 | kfree(node); |
174 | ||
47e50d5c | 175 | err_out: |
3e640f1b | 176 | if (!(place->flags & TTM_PL_FLAG_TEMPORARY)) |
2b70af79 | 177 | atomic64_sub(num_pages, &mgr->used); |
47e50d5c FC |
178 | |
179 | return r; | |
bb990bb0 CK |
180 | } |
181 | ||
182 | /** | |
183 | * amdgpu_gtt_mgr_del - free ranges | |
184 | * | |
185 | * @man: TTM memory type manager | |
6333a495 | 186 | * @res: TTM memory object |
bb990bb0 CK |
187 | * |
188 | * Free the allocated GTT again. | |
189 | */ | |
9de59bc2 | 190 | static void amdgpu_gtt_mgr_del(struct ttm_resource_manager *man, |
cb1c8146 | 191 | struct ttm_resource *res) |
bb990bb0 | 192 | { |
cb1c8146 | 193 | struct amdgpu_gtt_node *node = to_amdgpu_gtt_node(res); |
0af135b8 | 194 | struct amdgpu_gtt_mgr *mgr = to_gtt_mgr(man); |
bb990bb0 | 195 | |
d624e1bf | 196 | spin_lock(&mgr->lock); |
f700b18c CK |
197 | if (drm_mm_node_allocated(&node->base.mm_nodes[0])) |
198 | drm_mm_remove_node(&node->base.mm_nodes[0]); | |
d624e1bf | 199 | spin_unlock(&mgr->lock); |
3e640f1b LY |
200 | |
201 | if (!(res->placement & TTM_PL_FLAG_TEMPORARY)) | |
2b70af79 | 202 | atomic64_sub(res->num_pages, &mgr->used); |
bb990bb0 | 203 | |
de3688e4 | 204 | ttm_resource_fini(man, res); |
d624e1bf | 205 | kfree(node); |
bb990bb0 CK |
206 | } |
207 | ||
9255d77d CK |
208 | /** |
209 | * amdgpu_gtt_mgr_usage - return usage of GTT domain | |
210 | * | |
211 | * @man: TTM memory type manager | |
212 | * | |
213 | * Return how many bytes are used in the GTT domain | |
214 | */ | |
9de59bc2 | 215 | uint64_t amdgpu_gtt_mgr_usage(struct ttm_resource_manager *man) |
9255d77d | 216 | { |
0af135b8 | 217 | struct amdgpu_gtt_mgr *mgr = to_gtt_mgr(man); |
9255d77d | 218 | |
2b70af79 | 219 | return atomic64_read(&mgr->used) * PAGE_SIZE; |
9255d77d CK |
220 | } |
221 | ||
a614b336 CK |
222 | /** |
223 | * amdgpu_gtt_mgr_recover - re-init gart | |
224 | * | |
225 | * @man: TTM memory type manager | |
226 | * | |
227 | * Re-init the gart for each known BO in the GTT. | |
228 | */ | |
9de59bc2 | 229 | int amdgpu_gtt_mgr_recover(struct ttm_resource_manager *man) |
c1c7ce8f | 230 | { |
0af135b8 | 231 | struct amdgpu_gtt_mgr *mgr = to_gtt_mgr(man); |
19a1d935 | 232 | struct amdgpu_device *adev; |
c1c7ce8f CK |
233 | struct amdgpu_gtt_node *node; |
234 | struct drm_mm_node *mm_node; | |
235 | int r = 0; | |
236 | ||
19a1d935 | 237 | adev = container_of(mgr, typeof(*adev), mman.gtt_mgr); |
c1c7ce8f CK |
238 | spin_lock(&mgr->lock); |
239 | drm_mm_for_each_node(mm_node, &mgr->mm) { | |
f700b18c | 240 | node = container_of(mm_node, typeof(*node), base.mm_nodes[0]); |
c1c7ce8f CK |
241 | r = amdgpu_ttm_recover_gart(node->tbo); |
242 | if (r) | |
243 | break; | |
244 | } | |
245 | spin_unlock(&mgr->lock); | |
246 | ||
19a1d935 ND |
247 | amdgpu_gart_invalidate_tlb(adev); |
248 | ||
c1c7ce8f CK |
249 | return r; |
250 | } | |
251 | ||
bb990bb0 CK |
252 | /** |
253 | * amdgpu_gtt_mgr_debug - dump VRAM table | |
254 | * | |
255 | * @man: TTM memory type manager | |
373533f8 | 256 | * @printer: DRM printer to use |
bb990bb0 CK |
257 | * |
258 | * Dump the table content using printk. | |
259 | */ | |
9de59bc2 | 260 | static void amdgpu_gtt_mgr_debug(struct ttm_resource_manager *man, |
373533f8 | 261 | struct drm_printer *printer) |
bb990bb0 | 262 | { |
0af135b8 | 263 | struct amdgpu_gtt_mgr *mgr = to_gtt_mgr(man); |
bb990bb0 CK |
264 | |
265 | spin_lock(&mgr->lock); | |
373533f8 | 266 | drm_mm_print(&mgr->mm, printer); |
bb990bb0 | 267 | spin_unlock(&mgr->lock); |
97cbb284 | 268 | |
2b70af79 LY |
269 | drm_printf(printer, "man size:%llu pages, gtt used:%llu pages\n", |
270 | man->size, atomic64_read(&mgr->used)); | |
bb990bb0 CK |
271 | } |
272 | ||
9de59bc2 | 273 | static const struct ttm_resource_manager_func amdgpu_gtt_mgr_func = { |
e92ae67d CK |
274 | .alloc = amdgpu_gtt_mgr_new, |
275 | .free = amdgpu_gtt_mgr_del, | |
613e61a0 | 276 | .debug = amdgpu_gtt_mgr_debug |
bb990bb0 | 277 | }; |
a614b336 CK |
278 | |
279 | /** | |
280 | * amdgpu_gtt_mgr_init - init GTT manager and DRM MM | |
281 | * | |
282 | * @adev: amdgpu_device pointer | |
283 | * @gtt_size: maximum size of GTT | |
284 | * | |
285 | * Allocate and initialize the GTT manager. | |
286 | */ | |
287 | int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size) | |
288 | { | |
289 | struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr; | |
290 | struct ttm_resource_manager *man = &mgr->manager; | |
291 | uint64_t start, size; | |
a614b336 CK |
292 | |
293 | man->use_tt = true; | |
294 | man->func = &amdgpu_gtt_mgr_func; | |
295 | ||
3f268ef0 CK |
296 | ttm_resource_manager_init(man, &adev->mman.bdev, |
297 | gtt_size >> PAGE_SHIFT); | |
a614b336 CK |
298 | |
299 | start = AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GTT_NUM_TRANSFER_WINDOWS; | |
300 | size = (adev->gmc.gart_size >> PAGE_SHIFT) - start; | |
301 | drm_mm_init(&mgr->mm, start, size); | |
302 | spin_lock_init(&mgr->lock); | |
2b70af79 | 303 | atomic64_set(&mgr->used, 0); |
a614b336 | 304 | |
a614b336 CK |
305 | ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, &mgr->manager); |
306 | ttm_resource_manager_set_used(man, true); | |
307 | return 0; | |
308 | } | |
309 | ||
310 | /** | |
311 | * amdgpu_gtt_mgr_fini - free and destroy GTT manager | |
312 | * | |
313 | * @adev: amdgpu_device pointer | |
314 | * | |
315 | * Destroy and free the GTT manager, returns -EBUSY if ranges are still | |
316 | * allocated inside it. | |
317 | */ | |
318 | void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev) | |
319 | { | |
320 | struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr; | |
321 | struct ttm_resource_manager *man = &mgr->manager; | |
322 | int ret; | |
323 | ||
324 | ttm_resource_manager_set_used(man, false); | |
325 | ||
326 | ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man); | |
327 | if (ret) | |
328 | return; | |
329 | ||
330 | spin_lock(&mgr->lock); | |
331 | drm_mm_takedown(&mgr->mm); | |
332 | spin_unlock(&mgr->lock); | |
333 | ||
a614b336 CK |
334 | ttm_resource_manager_cleanup(man); |
335 | ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, NULL); | |
336 | } |