Commit | Line | Data |
---|---|---|
232a6eba MA |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2019 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include "intel_memory_region.h" | |
7 | #include "i915_drv.h" | |
8 | ||
da1184cd MA |
9 | /* XXX: Hysterical raisins. BIT(inst) needs to just be (inst) at some point. */ |
10 | #define REGION_MAP(type, inst) \ | |
11 | BIT((type) + INTEL_MEMORY_TYPE_SHIFT) | BIT(inst) | |
12 | ||
13 | const u32 intel_region_map[] = { | |
14 | [INTEL_REGION_SMEM] = REGION_MAP(INTEL_MEMORY_SYSTEM, 0), | |
15 | [INTEL_REGION_LMEM] = REGION_MAP(INTEL_MEMORY_LOCAL, 0), | |
16 | [INTEL_REGION_STOLEN] = REGION_MAP(INTEL_MEMORY_STOLEN, 0), | |
17 | }; | |
18 | ||
50129bca R |
19 | struct intel_memory_region * |
20 | intel_memory_region_by_type(struct drm_i915_private *i915, | |
21 | enum intel_memory_type mem_type) | |
22 | { | |
23 | struct intel_memory_region *mr; | |
24 | int id; | |
25 | ||
26 | for_each_memory_region(mr, i915, id) | |
27 | if (mr->type == mem_type) | |
28 | return mr; | |
29 | ||
30 | return NULL; | |
31 | } | |
32 | ||
232a6eba MA |
33 | static u64 |
34 | intel_memory_region_free_pages(struct intel_memory_region *mem, | |
35 | struct list_head *blocks) | |
36 | { | |
37 | struct i915_buddy_block *block, *on; | |
38 | u64 size = 0; | |
39 | ||
40 | list_for_each_entry_safe(block, on, blocks, link) { | |
41 | size += i915_buddy_block_size(&mem->mm, block); | |
42 | i915_buddy_free(&mem->mm, block); | |
43 | } | |
44 | INIT_LIST_HEAD(blocks); | |
45 | ||
46 | return size; | |
47 | } | |
48 | ||
49 | void | |
50 | __intel_memory_region_put_pages_buddy(struct intel_memory_region *mem, | |
51 | struct list_head *blocks) | |
52 | { | |
53 | mutex_lock(&mem->mm_lock); | |
38f1cb68 | 54 | mem->avail += intel_memory_region_free_pages(mem, blocks); |
232a6eba MA |
55 | mutex_unlock(&mem->mm_lock); |
56 | } | |
57 | ||
58 | void | |
59 | __intel_memory_region_put_block_buddy(struct i915_buddy_block *block) | |
60 | { | |
61 | struct list_head blocks; | |
62 | ||
63 | INIT_LIST_HEAD(&blocks); | |
64 | list_add(&block->link, &blocks); | |
65 | __intel_memory_region_put_pages_buddy(block->private, &blocks); | |
66 | } | |
67 | ||
68 | int | |
69 | __intel_memory_region_get_pages_buddy(struct intel_memory_region *mem, | |
70 | resource_size_t size, | |
71 | unsigned int flags, | |
72 | struct list_head *blocks) | |
73 | { | |
232a6eba | 74 | unsigned int min_order = 0; |
2f0b97ca | 75 | unsigned long n_pages; |
232a6eba MA |
76 | |
77 | GEM_BUG_ON(!IS_ALIGNED(size, mem->mm.chunk_size)); | |
78 | GEM_BUG_ON(!list_empty(blocks)); | |
79 | ||
80 | if (flags & I915_ALLOC_MIN_PAGE_SIZE) { | |
81 | min_order = ilog2(mem->min_page_size) - | |
82 | ilog2(mem->mm.chunk_size); | |
83 | } | |
84 | ||
2f0b97ca MA |
85 | if (flags & I915_ALLOC_CONTIGUOUS) { |
86 | size = roundup_pow_of_two(size); | |
87 | min_order = ilog2(size) - ilog2(mem->mm.chunk_size); | |
88 | } | |
89 | ||
0a9a5532 CW |
90 | if (size > BIT(mem->mm.max_order) * mem->mm.chunk_size) |
91 | return -E2BIG; | |
92 | ||
2f0b97ca MA |
93 | n_pages = size >> ilog2(mem->mm.chunk_size); |
94 | ||
232a6eba MA |
95 | mutex_lock(&mem->mm_lock); |
96 | ||
97 | do { | |
98 | struct i915_buddy_block *block; | |
99 | unsigned int order; | |
100 | ||
101 | order = fls(n_pages) - 1; | |
102 | GEM_BUG_ON(order > mem->mm.max_order); | |
103 | GEM_BUG_ON(order < min_order); | |
104 | ||
105 | do { | |
106 | block = i915_buddy_alloc(&mem->mm, order); | |
107 | if (!IS_ERR(block)) | |
108 | break; | |
109 | ||
110 | if (order-- == min_order) | |
111 | goto err_free_blocks; | |
112 | } while (1); | |
113 | ||
114 | n_pages -= BIT(order); | |
115 | ||
116 | block->private = mem; | |
117 | list_add(&block->link, blocks); | |
118 | ||
119 | if (!n_pages) | |
120 | break; | |
121 | } while (1); | |
122 | ||
38f1cb68 | 123 | mem->avail -= size; |
232a6eba MA |
124 | mutex_unlock(&mem->mm_lock); |
125 | return 0; | |
126 | ||
127 | err_free_blocks: | |
128 | intel_memory_region_free_pages(mem, blocks); | |
129 | mutex_unlock(&mem->mm_lock); | |
130 | return -ENXIO; | |
131 | } | |
132 | ||
133 | struct i915_buddy_block * | |
134 | __intel_memory_region_get_block_buddy(struct intel_memory_region *mem, | |
135 | resource_size_t size, | |
136 | unsigned int flags) | |
137 | { | |
138 | struct i915_buddy_block *block; | |
139 | LIST_HEAD(blocks); | |
140 | int ret; | |
141 | ||
142 | ret = __intel_memory_region_get_pages_buddy(mem, size, flags, &blocks); | |
143 | if (ret) | |
144 | return ERR_PTR(ret); | |
145 | ||
146 | block = list_first_entry(&blocks, typeof(*block), link); | |
147 | list_del_init(&block->link); | |
148 | return block; | |
149 | } | |
150 | ||
151 | int intel_memory_region_init_buddy(struct intel_memory_region *mem) | |
152 | { | |
153 | return i915_buddy_init(&mem->mm, resource_size(&mem->region), | |
154 | PAGE_SIZE); | |
155 | } | |
156 | ||
157 | void intel_memory_region_release_buddy(struct intel_memory_region *mem) | |
158 | { | |
159 | i915_buddy_fini(&mem->mm); | |
160 | } | |
161 | ||
162 | struct intel_memory_region * | |
163 | intel_memory_region_create(struct drm_i915_private *i915, | |
164 | resource_size_t start, | |
165 | resource_size_t size, | |
166 | resource_size_t min_page_size, | |
167 | resource_size_t io_start, | |
168 | const struct intel_memory_region_ops *ops) | |
169 | { | |
170 | struct intel_memory_region *mem; | |
171 | int err; | |
172 | ||
173 | mem = kzalloc(sizeof(*mem), GFP_KERNEL); | |
174 | if (!mem) | |
175 | return ERR_PTR(-ENOMEM); | |
176 | ||
177 | mem->i915 = i915; | |
178 | mem->region = (struct resource)DEFINE_RES_MEM(start, size); | |
179 | mem->io_start = io_start; | |
180 | mem->min_page_size = min_page_size; | |
181 | mem->ops = ops; | |
38f1cb68 LF |
182 | mem->total = size; |
183 | mem->avail = mem->total; | |
232a6eba | 184 | |
7c98501a MA |
185 | mutex_init(&mem->objects.lock); |
186 | INIT_LIST_HEAD(&mem->objects.list); | |
187 | INIT_LIST_HEAD(&mem->objects.purgeable); | |
188 | ||
232a6eba MA |
189 | mutex_init(&mem->mm_lock); |
190 | ||
191 | if (ops->init) { | |
192 | err = ops->init(mem); | |
193 | if (err) | |
194 | goto err_free; | |
195 | } | |
196 | ||
197 | kref_init(&mem->kref); | |
198 | return mem; | |
199 | ||
200 | err_free: | |
201 | kfree(mem); | |
202 | return ERR_PTR(err); | |
203 | } | |
204 | ||
38f1cb68 LF |
205 | void intel_memory_region_set_name(struct intel_memory_region *mem, |
206 | const char *fmt, ...) | |
207 | { | |
208 | va_list ap; | |
209 | ||
210 | va_start(ap, fmt); | |
211 | vsnprintf(mem->name, sizeof(mem->name), fmt, ap); | |
212 | va_end(ap); | |
213 | } | |
214 | ||
232a6eba MA |
215 | static void __intel_memory_region_destroy(struct kref *kref) |
216 | { | |
217 | struct intel_memory_region *mem = | |
218 | container_of(kref, typeof(*mem), kref); | |
219 | ||
220 | if (mem->ops->release) | |
221 | mem->ops->release(mem); | |
222 | ||
223 | mutex_destroy(&mem->mm_lock); | |
7c98501a | 224 | mutex_destroy(&mem->objects.lock); |
232a6eba MA |
225 | kfree(mem); |
226 | } | |
227 | ||
228 | struct intel_memory_region * | |
229 | intel_memory_region_get(struct intel_memory_region *mem) | |
230 | { | |
231 | kref_get(&mem->kref); | |
232 | return mem; | |
233 | } | |
234 | ||
235 | void intel_memory_region_put(struct intel_memory_region *mem) | |
236 | { | |
237 | kref_put(&mem->kref, __intel_memory_region_destroy); | |
238 | } | |
239 | ||
3fc794f2 CW |
240 | /* Global memory region registration -- only slight layer inversions! */ |
241 | ||
242 | int intel_memory_regions_hw_probe(struct drm_i915_private *i915) | |
243 | { | |
244 | int err, i; | |
245 | ||
246 | for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) { | |
247 | struct intel_memory_region *mem = ERR_PTR(-ENODEV); | |
248 | u32 type; | |
249 | ||
250 | if (!HAS_REGION(i915, BIT(i))) | |
251 | continue; | |
252 | ||
253 | type = MEMORY_TYPE_FROM_REGION(intel_region_map[i]); | |
254 | switch (type) { | |
255 | case INTEL_MEMORY_SYSTEM: | |
256 | mem = i915_gem_shmem_setup(i915); | |
257 | break; | |
258 | case INTEL_MEMORY_STOLEN: | |
259 | mem = i915_gem_stolen_setup(i915); | |
260 | break; | |
16292243 MA |
261 | case INTEL_MEMORY_LOCAL: |
262 | mem = intel_setup_fake_lmem(i915); | |
263 | break; | |
3fc794f2 CW |
264 | } |
265 | ||
266 | if (IS_ERR(mem)) { | |
267 | err = PTR_ERR(mem); | |
268 | DRM_ERROR("Failed to setup region(%d) type=%d\n", err, type); | |
269 | goto out_cleanup; | |
270 | } | |
271 | ||
272 | mem->id = intel_region_map[i]; | |
273 | mem->type = type; | |
274 | mem->instance = MEMORY_INSTANCE_FROM_REGION(intel_region_map[i]); | |
275 | ||
276 | i915->mm.regions[i] = mem; | |
277 | } | |
278 | ||
279 | return 0; | |
280 | ||
281 | out_cleanup: | |
282 | intel_memory_regions_driver_release(i915); | |
283 | return err; | |
284 | } | |
285 | ||
286 | void intel_memory_regions_driver_release(struct drm_i915_private *i915) | |
287 | { | |
288 | int i; | |
289 | ||
290 | for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) { | |
291 | struct intel_memory_region *region = | |
292 | fetch_and_zero(&i915->mm.regions[i]); | |
293 | ||
294 | if (region) | |
295 | intel_memory_region_put(region); | |
296 | } | |
297 | } | |
298 | ||
232a6eba MA |
299 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
300 | #include "selftests/intel_memory_region.c" | |
301 | #include "selftests/mock_region.c" | |
302 | #endif |