Commit | Line | Data |
---|---|---|
7c8c6b97 PM |
1 | /* |
2 | * Procedures for maintaining information about logical memory blocks. | |
3 | * | |
4 | * Peter Bergner, IBM Corp. June 2001. | |
5 | * Copyright (C) 2001 Peter Bergner. | |
d9b2b2a2 | 6 | * |
7c8c6b97 PM |
7 | * This program is free software; you can redistribute it and/or |
8 | * modify it under the terms of the GNU General Public License | |
9 | * as published by the Free Software Foundation; either version | |
10 | * 2 of the License, or (at your option) any later version. | |
11 | */ | |
12 | ||
7c8c6b97 PM |
13 | #include <linux/kernel.h> |
14 | #include <linux/init.h> | |
15 | #include <linux/bitops.h> | |
d9b2b2a2 | 16 | #include <linux/lmb.h> |
7c8c6b97 | 17 | |
3b9331da ME |
18 | #define LMB_ALLOC_ANYWHERE 0 |
19 | ||
eb481899 ME |
20 | struct lmb lmb; |
21 | ||
7c8c6b97 PM |
22 | void lmb_dump_all(void) |
23 | { | |
24 | #ifdef DEBUG | |
25 | unsigned long i; | |
26 | ||
300613e5 PM |
27 | pr_debug("lmb_dump_all:\n"); |
28 | pr_debug(" memory.cnt = 0x%lx\n", lmb.memory.cnt); | |
29 | pr_debug(" memory.size = 0x%llx\n", | |
e5f27095 | 30 | (unsigned long long)lmb.memory.size); |
7c8c6b97 | 31 | for (i=0; i < lmb.memory.cnt ;i++) { |
300613e5 | 32 | pr_debug(" memory.region[0x%x].base = 0x%llx\n", |
e5f27095 | 33 | i, (unsigned long long)lmb.memory.region[i].base); |
300613e5 | 34 | pr_debug(" .size = 0x%llx\n", |
e5f27095 | 35 | (unsigned long long)lmb.memory.region[i].size); |
7c8c6b97 PM |
36 | } |
37 | ||
300613e5 PM |
38 | pr_debug(" reserved.cnt = 0x%lx\n", lmb.reserved.cnt); |
39 | pr_debug(" reserved.size = 0x%lx\n", lmb.reserved.size); | |
7c8c6b97 | 40 | for (i=0; i < lmb.reserved.cnt ;i++) { |
300613e5 | 41 | pr_debug(" reserved.region[0x%x].base = 0x%llx\n", |
e5f27095 | 42 | i, (unsigned long long)lmb.reserved.region[i].base); |
300613e5 | 43 | pr_debug(" .size = 0x%llx\n", |
e5f27095 | 44 | (unsigned long long)lmb.reserved.region[i].size); |
7c8c6b97 PM |
45 | } |
46 | #endif /* DEBUG */ | |
47 | } | |
48 | ||
300613e5 PM |
49 | static unsigned long __init lmb_addrs_overlap(u64 base1, u64 size1, |
50 | u64 base2, u64 size2) | |
7c8c6b97 | 51 | { |
300613e5 | 52 | return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); |
7c8c6b97 PM |
53 | } |
54 | ||
e5f27095 BB |
55 | static long __init lmb_addrs_adjacent(u64 base1, u64 size1, |
56 | u64 base2, u64 size2) | |
7c8c6b97 PM |
57 | { |
58 | if (base2 == base1 + size1) | |
59 | return 1; | |
60 | else if (base1 == base2 + size2) | |
61 | return -1; | |
62 | ||
63 | return 0; | |
64 | } | |
65 | ||
66 | static long __init lmb_regions_adjacent(struct lmb_region *rgn, | |
67 | unsigned long r1, unsigned long r2) | |
68 | { | |
e5f27095 BB |
69 | u64 base1 = rgn->region[r1].base; |
70 | u64 size1 = rgn->region[r1].size; | |
71 | u64 base2 = rgn->region[r2].base; | |
72 | u64 size2 = rgn->region[r2].size; | |
7c8c6b97 PM |
73 | |
74 | return lmb_addrs_adjacent(base1, size1, base2, size2); | |
75 | } | |
76 | ||
2babf5c2 | 77 | static void __init lmb_remove_region(struct lmb_region *rgn, unsigned long r) |
7c8c6b97 PM |
78 | { |
79 | unsigned long i; | |
80 | ||
2babf5c2 ME |
81 | for (i = r; i < rgn->cnt - 1; i++) { |
82 | rgn->region[i].base = rgn->region[i + 1].base; | |
83 | rgn->region[i].size = rgn->region[i + 1].size; | |
7c8c6b97 PM |
84 | } |
85 | rgn->cnt--; | |
86 | } | |
87 | ||
2babf5c2 ME |
88 | /* Assumption: base addr of region 1 < base addr of region 2 */ |
89 | static void __init lmb_coalesce_regions(struct lmb_region *rgn, | |
90 | unsigned long r1, unsigned long r2) | |
91 | { | |
92 | rgn->region[r1].size += rgn->region[r2].size; | |
93 | lmb_remove_region(rgn, r2); | |
94 | } | |
95 | ||
7c8c6b97 PM |
96 | void __init lmb_init(void) |
97 | { | |
98 | /* Create a dummy zero size LMB which will get coalesced away later. | |
99 | * This simplifies the lmb_add() code below... | |
100 | */ | |
101 | lmb.memory.region[0].base = 0; | |
102 | lmb.memory.region[0].size = 0; | |
103 | lmb.memory.cnt = 1; | |
104 | ||
105 | /* Ditto. */ | |
106 | lmb.reserved.region[0].base = 0; | |
107 | lmb.reserved.region[0].size = 0; | |
108 | lmb.reserved.cnt = 1; | |
109 | } | |
110 | ||
7c8c6b97 PM |
111 | void __init lmb_analyze(void) |
112 | { | |
113 | int i; | |
114 | ||
115 | lmb.memory.size = 0; | |
116 | ||
117 | for (i = 0; i < lmb.memory.cnt; i++) | |
118 | lmb.memory.size += lmb.memory.region[i].size; | |
119 | } | |
120 | ||
e5f27095 | 121 | static long __init lmb_add_region(struct lmb_region *rgn, u64 base, u64 size) |
7c8c6b97 | 122 | { |
56d6d1a7 MA |
123 | unsigned long coalesced = 0; |
124 | long adjacent, i; | |
7c8c6b97 | 125 | |
27e6672b KG |
126 | if ((rgn->cnt == 1) && (rgn->region[0].size == 0)) { |
127 | rgn->region[0].base = base; | |
128 | rgn->region[0].size = size; | |
129 | return 0; | |
130 | } | |
131 | ||
7c8c6b97 | 132 | /* First try and coalesce this LMB with another. */ |
300613e5 | 133 | for (i = 0; i < rgn->cnt; i++) { |
e5f27095 BB |
134 | u64 rgnbase = rgn->region[i].base; |
135 | u64 rgnsize = rgn->region[i].size; | |
7c8c6b97 | 136 | |
eb6de286 DG |
137 | if ((rgnbase == base) && (rgnsize == size)) |
138 | /* Already have this region, so we're done */ | |
139 | return 0; | |
140 | ||
300613e5 PM |
141 | adjacent = lmb_addrs_adjacent(base, size, rgnbase, rgnsize); |
142 | if (adjacent > 0) { | |
7c8c6b97 PM |
143 | rgn->region[i].base -= size; |
144 | rgn->region[i].size += size; | |
145 | coalesced++; | |
146 | break; | |
300613e5 | 147 | } else if (adjacent < 0) { |
7c8c6b97 PM |
148 | rgn->region[i].size += size; |
149 | coalesced++; | |
150 | break; | |
151 | } | |
152 | } | |
153 | ||
300613e5 | 154 | if ((i < rgn->cnt - 1) && lmb_regions_adjacent(rgn, i, i+1)) { |
7c8c6b97 PM |
155 | lmb_coalesce_regions(rgn, i, i+1); |
156 | coalesced++; | |
157 | } | |
158 | ||
159 | if (coalesced) | |
160 | return coalesced; | |
161 | if (rgn->cnt >= MAX_LMB_REGIONS) | |
162 | return -1; | |
163 | ||
164 | /* Couldn't coalesce the LMB, so add it to the sorted table. */ | |
300613e5 | 165 | for (i = rgn->cnt - 1; i >= 0; i--) { |
7c8c6b97 PM |
166 | if (base < rgn->region[i].base) { |
167 | rgn->region[i+1].base = rgn->region[i].base; | |
168 | rgn->region[i+1].size = rgn->region[i].size; | |
169 | } else { | |
170 | rgn->region[i+1].base = base; | |
171 | rgn->region[i+1].size = size; | |
172 | break; | |
173 | } | |
174 | } | |
74b20dad KG |
175 | |
176 | if (base < rgn->region[0].base) { | |
177 | rgn->region[0].base = base; | |
178 | rgn->region[0].size = size; | |
179 | } | |
7c8c6b97 PM |
180 | rgn->cnt++; |
181 | ||
182 | return 0; | |
183 | } | |
184 | ||
e5f27095 | 185 | long __init lmb_add(u64 base, u64 size) |
7c8c6b97 | 186 | { |
300613e5 | 187 | struct lmb_region *_rgn = &lmb.memory; |
7c8c6b97 PM |
188 | |
189 | /* On pSeries LPAR systems, the first LMB is our RMO region. */ | |
190 | if (base == 0) | |
191 | lmb.rmo_size = size; | |
192 | ||
193 | return lmb_add_region(_rgn, base, size); | |
194 | ||
195 | } | |
196 | ||
e5f27095 | 197 | long __init lmb_reserve(u64 base, u64 size) |
7c8c6b97 | 198 | { |
300613e5 | 199 | struct lmb_region *_rgn = &lmb.reserved; |
7c8c6b97 | 200 | |
8c20fafa ME |
201 | BUG_ON(0 == size); |
202 | ||
7c8c6b97 PM |
203 | return lmb_add_region(_rgn, base, size); |
204 | } | |
205 | ||
300613e5 | 206 | long __init lmb_overlaps_region(struct lmb_region *rgn, u64 base, u64 size) |
7c8c6b97 PM |
207 | { |
208 | unsigned long i; | |
209 | ||
300613e5 | 210 | for (i = 0; i < rgn->cnt; i++) { |
e5f27095 BB |
211 | u64 rgnbase = rgn->region[i].base; |
212 | u64 rgnsize = rgn->region[i].size; | |
300613e5 | 213 | if (lmb_addrs_overlap(base, size, rgnbase, rgnsize)) |
7c8c6b97 | 214 | break; |
7c8c6b97 PM |
215 | } |
216 | ||
217 | return (i < rgn->cnt) ? i : -1; | |
218 | } | |
219 | ||
c50f68c8 DM |
220 | static u64 lmb_align_down(u64 addr, u64 size) |
221 | { | |
222 | return addr & ~(size - 1); | |
223 | } | |
224 | ||
225 | static u64 lmb_align_up(u64 addr, u64 size) | |
226 | { | |
227 | return (addr + (size - 1)) & ~(size - 1); | |
228 | } | |
229 | ||
230 | static u64 __init lmb_alloc_nid_unreserved(u64 start, u64 end, | |
231 | u64 size, u64 align) | |
232 | { | |
233 | u64 base; | |
234 | long j; | |
235 | ||
236 | base = lmb_align_down((end - size), align); | |
237 | while (start <= base && | |
238 | ((j = lmb_overlaps_region(&lmb.reserved, base, size)) >= 0)) | |
239 | base = lmb_align_down(lmb.reserved.region[j].base - size, | |
240 | align); | |
241 | ||
242 | if (base != 0 && start <= base) { | |
243 | if (lmb_add_region(&lmb.reserved, base, | |
244 | lmb_align_up(size, align)) < 0) | |
245 | base = ~(u64)0; | |
246 | return base; | |
247 | } | |
248 | ||
249 | return ~(u64)0; | |
250 | } | |
251 | ||
252 | static u64 __init lmb_alloc_nid_region(struct lmb_property *mp, | |
253 | u64 (*nid_range)(u64, u64, int *), | |
254 | u64 size, u64 align, int nid) | |
255 | { | |
256 | u64 start, end; | |
257 | ||
258 | start = mp->base; | |
259 | end = start + mp->size; | |
260 | ||
261 | start = lmb_align_up(start, align); | |
262 | while (start < end) { | |
263 | u64 this_end; | |
264 | int this_nid; | |
265 | ||
266 | this_end = nid_range(start, end, &this_nid); | |
267 | if (this_nid == nid) { | |
268 | u64 ret = lmb_alloc_nid_unreserved(start, this_end, | |
269 | size, align); | |
270 | if (ret != ~(u64)0) | |
271 | return ret; | |
272 | } | |
273 | start = this_end; | |
274 | } | |
275 | ||
276 | return ~(u64)0; | |
277 | } | |
278 | ||
279 | u64 __init lmb_alloc_nid(u64 size, u64 align, int nid, | |
280 | u64 (*nid_range)(u64 start, u64 end, int *nid)) | |
281 | { | |
282 | struct lmb_region *mem = &lmb.memory; | |
283 | int i; | |
284 | ||
285 | for (i = 0; i < mem->cnt; i++) { | |
286 | u64 ret = lmb_alloc_nid_region(&mem->region[i], | |
287 | nid_range, | |
288 | size, align, nid); | |
289 | if (ret != ~(u64)0) | |
290 | return ret; | |
291 | } | |
292 | ||
293 | return lmb_alloc(size, align); | |
294 | } | |
295 | ||
e5f27095 | 296 | u64 __init lmb_alloc(u64 size, u64 align) |
7c8c6b97 PM |
297 | { |
298 | return lmb_alloc_base(size, align, LMB_ALLOC_ANYWHERE); | |
299 | } | |
300 | ||
e5f27095 | 301 | u64 __init lmb_alloc_base(u64 size, u64 align, u64 max_addr) |
d7a5b2ff | 302 | { |
e5f27095 | 303 | u64 alloc; |
d7a5b2ff ME |
304 | |
305 | alloc = __lmb_alloc_base(size, align, max_addr); | |
306 | ||
2c276603 | 307 | if (alloc == 0) |
e5f27095 BB |
308 | panic("ERROR: Failed to allocate 0x%llx bytes below 0x%llx.\n", |
309 | (unsigned long long) size, (unsigned long long) max_addr); | |
d7a5b2ff ME |
310 | |
311 | return alloc; | |
312 | } | |
313 | ||
e5f27095 | 314 | u64 __init __lmb_alloc_base(u64 size, u64 align, u64 max_addr) |
7c8c6b97 PM |
315 | { |
316 | long i, j; | |
e5f27095 | 317 | u64 base = 0; |
7c8c6b97 | 318 | |
8c20fafa ME |
319 | BUG_ON(0 == size); |
320 | ||
d9b2b2a2 | 321 | /* On some platforms, make sure we allocate lowmem */ |
7c8c6b97 | 322 | if (max_addr == LMB_ALLOC_ANYWHERE) |
d9b2b2a2 DM |
323 | max_addr = LMB_REAL_LIMIT; |
324 | ||
300613e5 | 325 | for (i = lmb.memory.cnt - 1; i >= 0; i--) { |
e5f27095 BB |
326 | u64 lmbbase = lmb.memory.region[i].base; |
327 | u64 lmbsize = lmb.memory.region[i].size; | |
7c8c6b97 PM |
328 | |
329 | if (max_addr == LMB_ALLOC_ANYWHERE) | |
d9b2b2a2 | 330 | base = lmb_align_down(lmbbase + lmbsize - size, align); |
7c8c6b97 PM |
331 | else if (lmbbase < max_addr) { |
332 | base = min(lmbbase + lmbsize, max_addr); | |
d9b2b2a2 | 333 | base = lmb_align_down(base - size, align); |
7c8c6b97 PM |
334 | } else |
335 | continue; | |
336 | ||
300613e5 PM |
337 | while (lmbbase <= base) { |
338 | j = lmb_overlaps_region(&lmb.reserved, base, size); | |
339 | if (j < 0) | |
340 | break; | |
d9b2b2a2 DM |
341 | base = lmb_align_down(lmb.reserved.region[j].base - size, |
342 | align); | |
300613e5 | 343 | } |
7c8c6b97 PM |
344 | |
345 | if ((base != 0) && (lmbbase <= base)) | |
346 | break; | |
347 | } | |
348 | ||
349 | if (i < 0) | |
350 | return 0; | |
351 | ||
eea89e13 DM |
352 | if (lmb_add_region(&lmb.reserved, base, lmb_align_up(size, align)) < 0) |
353 | return 0; | |
7c8c6b97 PM |
354 | |
355 | return base; | |
356 | } | |
357 | ||
358 | /* You must call lmb_analyze() before this. */ | |
e5f27095 | 359 | u64 __init lmb_phys_mem_size(void) |
7c8c6b97 PM |
360 | { |
361 | return lmb.memory.size; | |
362 | } | |
363 | ||
e5f27095 | 364 | u64 __init lmb_end_of_DRAM(void) |
7c8c6b97 PM |
365 | { |
366 | int idx = lmb.memory.cnt - 1; | |
367 | ||
368 | return (lmb.memory.region[idx].base + lmb.memory.region[idx].size); | |
369 | } | |
370 | ||
2babf5c2 | 371 | /* You must call lmb_analyze() after this. */ |
e5f27095 | 372 | void __init lmb_enforce_memory_limit(u64 memory_limit) |
7c8c6b97 | 373 | { |
e5f27095 BB |
374 | unsigned long i; |
375 | u64 limit; | |
2babf5c2 | 376 | struct lmb_property *p; |
7c8c6b97 | 377 | |
300613e5 | 378 | if (!memory_limit) |
7c8c6b97 PM |
379 | return; |
380 | ||
2babf5c2 | 381 | /* Truncate the lmb regions to satisfy the memory limit. */ |
7c8c6b97 PM |
382 | limit = memory_limit; |
383 | for (i = 0; i < lmb.memory.cnt; i++) { | |
384 | if (limit > lmb.memory.region[i].size) { | |
385 | limit -= lmb.memory.region[i].size; | |
386 | continue; | |
387 | } | |
388 | ||
389 | lmb.memory.region[i].size = limit; | |
390 | lmb.memory.cnt = i + 1; | |
391 | break; | |
392 | } | |
2babf5c2 | 393 | |
30f30e13 ME |
394 | if (lmb.memory.region[0].size < lmb.rmo_size) |
395 | lmb.rmo_size = lmb.memory.region[0].size; | |
2babf5c2 ME |
396 | |
397 | /* And truncate any reserves above the limit also. */ | |
398 | for (i = 0; i < lmb.reserved.cnt; i++) { | |
399 | p = &lmb.reserved.region[i]; | |
400 | ||
401 | if (p->base > memory_limit) | |
402 | p->size = 0; | |
403 | else if ((p->base + p->size) > memory_limit) | |
404 | p->size = memory_limit - p->base; | |
405 | ||
406 | if (p->size == 0) { | |
407 | lmb_remove_region(&lmb.reserved, i); | |
408 | i--; | |
409 | } | |
410 | } | |
7c8c6b97 | 411 | } |
f98eeb4e | 412 | |
e5f27095 | 413 | int __init lmb_is_reserved(u64 addr) |
f98eeb4e KG |
414 | { |
415 | int i; | |
416 | ||
417 | for (i = 0; i < lmb.reserved.cnt; i++) { | |
e5f27095 BB |
418 | u64 upper = lmb.reserved.region[i].base + |
419 | lmb.reserved.region[i].size - 1; | |
f98eeb4e KG |
420 | if ((addr >= lmb.reserved.region[i].base) && (addr <= upper)) |
421 | return 1; | |
422 | } | |
423 | return 0; | |
424 | } |