Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
1da177e4 LT |
2 | /* |
3 | * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 | |
4 | */ | |
5 | #ifndef _LINUX_BOOTMEM_H | |
6 | #define _LINUX_BOOTMEM_H | |
7 | ||
1da177e4 | 8 | #include <linux/mmzone.h> |
26f09e9b | 9 | #include <linux/mm_types.h> |
e786e86a | 10 | #include <asm/dma.h> |
2382705f | 11 | #include <asm/processor.h> |
1da177e4 LT |
12 | |
13 | /* | |
14 | * simple boot-time physical memory area allocator. | |
15 | */ | |
16 | ||
17 | extern unsigned long max_low_pfn; | |
18 | extern unsigned long min_low_pfn; | |
19 | ||
20 | /* | |
21 | * highest page | |
22 | */ | |
23 | extern unsigned long max_pfn; | |
8dd33030 IM |
24 | /* |
25 | * highest possible page | |
26 | */ | |
27 | extern unsigned long long max_possible_pfn; | |
1da177e4 | 28 | |
08677214 | 29 | #ifndef CONFIG_NO_BOOTMEM |
a83c0ea7 MR |
30 | /** |
31 | * struct bootmem_data - per-node information used by the bootmem allocator | |
32 | * @node_min_pfn: the starting physical address of the node's memory | |
33 | * @node_low_pfn: the end physical address of the directly addressable memory | |
34 | * @node_bootmem_map: is a bitmap pointer - the bits represent all physical | |
35 | * memory pages (including holes) on the node. | |
36 | * @last_end_off: the offset within the page of the end of the last allocation; | |
37 | * if 0, the page used is full | |
38 | * @hint_idx: the PFN of the page used with the last allocation; | |
39 | * together with using this with the @last_end_offset field, | |
40 | * a test can be made to see if allocations can be merged | |
41 | * with the page used for the last allocation rather than | |
42 | * using up a full new page. | |
43 | * @list: list entry in the linked list ordered by the memory addresses | |
1da177e4 LT |
44 | */ |
45 | typedef struct bootmem_data { | |
3560e249 | 46 | unsigned long node_min_pfn; |
1da177e4 LT |
47 | unsigned long node_low_pfn; |
48 | void *node_bootmem_map; | |
5f2809e6 JW |
49 | unsigned long last_end_off; |
50 | unsigned long hint_idx; | |
679bc9fb | 51 | struct list_head list; |
1da177e4 LT |
52 | } bootmem_data_t; |
53 | ||
b61bfa3c | 54 | extern bootmem_data_t bootmem_node_data[]; |
08677214 | 55 | #endif |
b61bfa3c | 56 | |
f71bf0ca | 57 | extern unsigned long bootmem_bootmap_pages(unsigned long); |
223e8dc9 JW |
58 | |
59 | extern unsigned long init_bootmem_node(pg_data_t *pgdat, | |
60 | unsigned long freepfn, | |
61 | unsigned long startpfn, | |
62 | unsigned long endpfn); | |
f71bf0ca | 63 | extern unsigned long init_bootmem(unsigned long addr, unsigned long memend); |
223e8dc9 | 64 | |
223e8dc9 | 65 | extern unsigned long free_all_bootmem(void); |
f784a3f1 | 66 | extern void reset_node_managed_pages(pg_data_t *pgdat); |
7b4b2a0d | 67 | extern void reset_all_zones_managed_pages(void); |
223e8dc9 JW |
68 | |
69 | extern void free_bootmem_node(pg_data_t *pgdat, | |
70 | unsigned long addr, | |
71 | unsigned long size); | |
81df9bff JK |
72 | extern void free_bootmem(unsigned long physaddr, unsigned long size); |
73 | extern void free_bootmem_late(unsigned long physaddr, unsigned long size); | |
223e8dc9 JW |
74 | |
75 | /* | |
76 | * Flags for reserve_bootmem (also if CONFIG_HAVE_ARCH_BOOTMEM_NODE, | |
77 | * the architecture-specific code should honor this). | |
78 | * | |
1754e44e WSH |
79 | * If flags is BOOTMEM_DEFAULT, then the return value is always 0 (success). |
80 | * If flags contains BOOTMEM_EXCLUSIVE, then -EBUSY is returned if the memory | |
81 | * already was reserved. | |
223e8dc9 JW |
82 | */ |
83 | #define BOOTMEM_DEFAULT 0 | |
84 | #define BOOTMEM_EXCLUSIVE (1<<0) | |
85 | ||
c1329375 TH |
86 | extern int reserve_bootmem(unsigned long addr, |
87 | unsigned long size, | |
88 | int flags); | |
2d0aae41 TH |
89 | extern int reserve_bootmem_node(pg_data_t *pgdat, |
90 | unsigned long physaddr, | |
91 | unsigned long size, | |
92 | int flags); | |
93 | ||
94 | extern void *__alloc_bootmem(unsigned long size, | |
f71bf0ca FBH |
95 | unsigned long align, |
96 | unsigned long goal); | |
2d0aae41 | 97 | extern void *__alloc_bootmem_nopanic(unsigned long size, |
f71bf0ca | 98 | unsigned long align, |
48a27055 | 99 | unsigned long goal) __malloc; |
223e8dc9 JW |
100 | extern void *__alloc_bootmem_node(pg_data_t *pgdat, |
101 | unsigned long size, | |
102 | unsigned long align, | |
48a27055 | 103 | unsigned long goal) __malloc; |
08677214 YL |
104 | void *__alloc_bootmem_node_high(pg_data_t *pgdat, |
105 | unsigned long size, | |
106 | unsigned long align, | |
48a27055 | 107 | unsigned long goal) __malloc; |
223e8dc9 JW |
108 | extern void *__alloc_bootmem_node_nopanic(pg_data_t *pgdat, |
109 | unsigned long size, | |
110 | unsigned long align, | |
48a27055 | 111 | unsigned long goal) __malloc; |
99ab7b19 YL |
112 | void *___alloc_bootmem_node_nopanic(pg_data_t *pgdat, |
113 | unsigned long size, | |
114 | unsigned long align, | |
115 | unsigned long goal, | |
48a27055 | 116 | unsigned long limit) __malloc; |
2d0aae41 TH |
117 | extern void *__alloc_bootmem_low(unsigned long size, |
118 | unsigned long align, | |
48a27055 | 119 | unsigned long goal) __malloc; |
38fa4175 YL |
120 | void *__alloc_bootmem_low_nopanic(unsigned long size, |
121 | unsigned long align, | |
48a27055 | 122 | unsigned long goal) __malloc; |
f71bf0ca FBH |
123 | extern void *__alloc_bootmem_low_node(pg_data_t *pgdat, |
124 | unsigned long size, | |
125 | unsigned long align, | |
48a27055 | 126 | unsigned long goal) __malloc; |
c1329375 | 127 | |
8bba154e YL |
128 | #ifdef CONFIG_NO_BOOTMEM |
129 | /* We are using top down, so it is safe to use 0 here */ | |
130 | #define BOOTMEM_LOW_LIMIT 0 | |
131 | #else | |
132 | #define BOOTMEM_LOW_LIMIT __pa(MAX_DMA_ADDRESS) | |
133 | #endif | |
134 | ||
2382705f | 135 | #ifndef ARCH_LOW_ADDRESS_LIMIT |
136 | #define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL | |
137 | #endif | |
138 | ||
1da177e4 | 139 | #define alloc_bootmem(x) \ |
8bba154e | 140 | __alloc_bootmem(x, SMP_CACHE_BYTES, BOOTMEM_LOW_LIMIT) |
53dde5f3 | 141 | #define alloc_bootmem_align(x, align) \ |
8bba154e | 142 | __alloc_bootmem(x, align, BOOTMEM_LOW_LIMIT) |
74768ed8 | 143 | #define alloc_bootmem_nopanic(x) \ |
8bba154e | 144 | __alloc_bootmem_nopanic(x, SMP_CACHE_BYTES, BOOTMEM_LOW_LIMIT) |
1da177e4 | 145 | #define alloc_bootmem_pages(x) \ |
8bba154e | 146 | __alloc_bootmem(x, PAGE_SIZE, BOOTMEM_LOW_LIMIT) |
74768ed8 | 147 | #define alloc_bootmem_pages_nopanic(x) \ |
8bba154e | 148 | __alloc_bootmem_nopanic(x, PAGE_SIZE, BOOTMEM_LOW_LIMIT) |
1da177e4 | 149 | #define alloc_bootmem_node(pgdat, x) \ |
8bba154e | 150 | __alloc_bootmem_node(pgdat, x, SMP_CACHE_BYTES, BOOTMEM_LOW_LIMIT) |
8f389a99 | 151 | #define alloc_bootmem_node_nopanic(pgdat, x) \ |
8bba154e | 152 | __alloc_bootmem_node_nopanic(pgdat, x, SMP_CACHE_BYTES, BOOTMEM_LOW_LIMIT) |
1da177e4 | 153 | #define alloc_bootmem_pages_node(pgdat, x) \ |
8bba154e | 154 | __alloc_bootmem_node(pgdat, x, PAGE_SIZE, BOOTMEM_LOW_LIMIT) |
2d0aae41 | 155 | #define alloc_bootmem_pages_node_nopanic(pgdat, x) \ |
8bba154e | 156 | __alloc_bootmem_node_nopanic(pgdat, x, PAGE_SIZE, BOOTMEM_LOW_LIMIT) |
2d0aae41 TH |
157 | |
158 | #define alloc_bootmem_low(x) \ | |
159 | __alloc_bootmem_low(x, SMP_CACHE_BYTES, 0) | |
38fa4175 YL |
160 | #define alloc_bootmem_low_pages_nopanic(x) \ |
161 | __alloc_bootmem_low_nopanic(x, PAGE_SIZE, 0) | |
2d0aae41 TH |
162 | #define alloc_bootmem_low_pages(x) \ |
163 | __alloc_bootmem_low(x, PAGE_SIZE, 0) | |
1da177e4 | 164 | #define alloc_bootmem_low_pages_node(pgdat, x) \ |
71fb2e8f | 165 | __alloc_bootmem_low_node(pgdat, x, PAGE_SIZE, 0) |
c6af5e9f | 166 | |
26f09e9b SS |
167 | |
168 | #if defined(CONFIG_HAVE_MEMBLOCK) && defined(CONFIG_NO_BOOTMEM) | |
169 | ||
170 | /* FIXME: use MEMBLOCK_ALLOC_* variants here */ | |
171 | #define BOOTMEM_ALLOC_ACCESSIBLE 0 | |
172 | #define BOOTMEM_ALLOC_ANYWHERE (~(phys_addr_t)0) | |
173 | ||
174 | /* FIXME: Move to memblock.h at a point where we remove nobootmem.c */ | |
ea1f5f37 PT |
175 | void *memblock_virt_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align, |
176 | phys_addr_t min_addr, | |
177 | phys_addr_t max_addr, int nid); | |
26f09e9b SS |
178 | void *memblock_virt_alloc_try_nid_nopanic(phys_addr_t size, |
179 | phys_addr_t align, phys_addr_t min_addr, | |
180 | phys_addr_t max_addr, int nid); | |
181 | void *memblock_virt_alloc_try_nid(phys_addr_t size, phys_addr_t align, | |
182 | phys_addr_t min_addr, phys_addr_t max_addr, int nid); | |
183 | void __memblock_free_early(phys_addr_t base, phys_addr_t size); | |
184 | void __memblock_free_late(phys_addr_t base, phys_addr_t size); | |
185 | ||
186 | static inline void * __init memblock_virt_alloc( | |
187 | phys_addr_t size, phys_addr_t align) | |
188 | { | |
189 | return memblock_virt_alloc_try_nid(size, align, BOOTMEM_LOW_LIMIT, | |
190 | BOOTMEM_ALLOC_ACCESSIBLE, | |
191 | NUMA_NO_NODE); | |
192 | } | |
193 | ||
ea1f5f37 PT |
194 | static inline void * __init memblock_virt_alloc_raw( |
195 | phys_addr_t size, phys_addr_t align) | |
196 | { | |
197 | return memblock_virt_alloc_try_nid_raw(size, align, BOOTMEM_LOW_LIMIT, | |
198 | BOOTMEM_ALLOC_ACCESSIBLE, | |
199 | NUMA_NO_NODE); | |
200 | } | |
201 | ||
26f09e9b SS |
202 | static inline void * __init memblock_virt_alloc_nopanic( |
203 | phys_addr_t size, phys_addr_t align) | |
204 | { | |
205 | return memblock_virt_alloc_try_nid_nopanic(size, align, | |
206 | BOOTMEM_LOW_LIMIT, | |
207 | BOOTMEM_ALLOC_ACCESSIBLE, | |
208 | NUMA_NO_NODE); | |
209 | } | |
210 | ||
ad6492b8 YL |
211 | static inline void * __init memblock_virt_alloc_low( |
212 | phys_addr_t size, phys_addr_t align) | |
213 | { | |
214 | return memblock_virt_alloc_try_nid(size, align, | |
215 | BOOTMEM_LOW_LIMIT, | |
216 | ARCH_LOW_ADDRESS_LIMIT, | |
217 | NUMA_NO_NODE); | |
218 | } | |
219 | static inline void * __init memblock_virt_alloc_low_nopanic( | |
220 | phys_addr_t size, phys_addr_t align) | |
221 | { | |
222 | return memblock_virt_alloc_try_nid_nopanic(size, align, | |
223 | BOOTMEM_LOW_LIMIT, | |
224 | ARCH_LOW_ADDRESS_LIMIT, | |
225 | NUMA_NO_NODE); | |
226 | } | |
227 | ||
26f09e9b SS |
228 | static inline void * __init memblock_virt_alloc_from_nopanic( |
229 | phys_addr_t size, phys_addr_t align, phys_addr_t min_addr) | |
230 | { | |
231 | return memblock_virt_alloc_try_nid_nopanic(size, align, min_addr, | |
232 | BOOTMEM_ALLOC_ACCESSIBLE, | |
233 | NUMA_NO_NODE); | |
234 | } | |
235 | ||
236 | static inline void * __init memblock_virt_alloc_node( | |
237 | phys_addr_t size, int nid) | |
238 | { | |
239 | return memblock_virt_alloc_try_nid(size, 0, BOOTMEM_LOW_LIMIT, | |
240 | BOOTMEM_ALLOC_ACCESSIBLE, nid); | |
241 | } | |
242 | ||
243 | static inline void * __init memblock_virt_alloc_node_nopanic( | |
244 | phys_addr_t size, int nid) | |
245 | { | |
246 | return memblock_virt_alloc_try_nid_nopanic(size, 0, BOOTMEM_LOW_LIMIT, | |
247 | BOOTMEM_ALLOC_ACCESSIBLE, | |
248 | nid); | |
249 | } | |
250 | ||
251 | static inline void __init memblock_free_early( | |
252 | phys_addr_t base, phys_addr_t size) | |
253 | { | |
254 | __memblock_free_early(base, size); | |
255 | } | |
256 | ||
257 | static inline void __init memblock_free_early_nid( | |
258 | phys_addr_t base, phys_addr_t size, int nid) | |
259 | { | |
260 | __memblock_free_early(base, size); | |
261 | } | |
262 | ||
263 | static inline void __init memblock_free_late( | |
264 | phys_addr_t base, phys_addr_t size) | |
265 | { | |
266 | __memblock_free_late(base, size); | |
267 | } | |
268 | ||
269 | #else | |
270 | ||
271 | #define BOOTMEM_ALLOC_ACCESSIBLE 0 | |
272 | ||
273 | ||
274 | /* Fall back to all the existing bootmem APIs */ | |
275 | static inline void * __init memblock_virt_alloc( | |
276 | phys_addr_t size, phys_addr_t align) | |
277 | { | |
278 | if (!align) | |
279 | align = SMP_CACHE_BYTES; | |
280 | return __alloc_bootmem(size, align, BOOTMEM_LOW_LIMIT); | |
281 | } | |
282 | ||
ea1f5f37 PT |
283 | static inline void * __init memblock_virt_alloc_raw( |
284 | phys_addr_t size, phys_addr_t align) | |
285 | { | |
286 | if (!align) | |
287 | align = SMP_CACHE_BYTES; | |
288 | return __alloc_bootmem_nopanic(size, align, BOOTMEM_LOW_LIMIT); | |
289 | } | |
290 | ||
26f09e9b SS |
291 | static inline void * __init memblock_virt_alloc_nopanic( |
292 | phys_addr_t size, phys_addr_t align) | |
293 | { | |
294 | if (!align) | |
295 | align = SMP_CACHE_BYTES; | |
296 | return __alloc_bootmem_nopanic(size, align, BOOTMEM_LOW_LIMIT); | |
297 | } | |
298 | ||
ad6492b8 YL |
299 | static inline void * __init memblock_virt_alloc_low( |
300 | phys_addr_t size, phys_addr_t align) | |
301 | { | |
302 | if (!align) | |
303 | align = SMP_CACHE_BYTES; | |
07bacb38 | 304 | return __alloc_bootmem_low(size, align, 0); |
ad6492b8 YL |
305 | } |
306 | ||
307 | static inline void * __init memblock_virt_alloc_low_nopanic( | |
308 | phys_addr_t size, phys_addr_t align) | |
309 | { | |
310 | if (!align) | |
311 | align = SMP_CACHE_BYTES; | |
07bacb38 | 312 | return __alloc_bootmem_low_nopanic(size, align, 0); |
ad6492b8 YL |
313 | } |
314 | ||
26f09e9b SS |
315 | static inline void * __init memblock_virt_alloc_from_nopanic( |
316 | phys_addr_t size, phys_addr_t align, phys_addr_t min_addr) | |
317 | { | |
318 | return __alloc_bootmem_nopanic(size, align, min_addr); | |
319 | } | |
320 | ||
321 | static inline void * __init memblock_virt_alloc_node( | |
322 | phys_addr_t size, int nid) | |
323 | { | |
324 | return __alloc_bootmem_node(NODE_DATA(nid), size, SMP_CACHE_BYTES, | |
325 | BOOTMEM_LOW_LIMIT); | |
326 | } | |
327 | ||
328 | static inline void * __init memblock_virt_alloc_node_nopanic( | |
329 | phys_addr_t size, int nid) | |
330 | { | |
331 | return __alloc_bootmem_node_nopanic(NODE_DATA(nid), size, | |
332 | SMP_CACHE_BYTES, | |
333 | BOOTMEM_LOW_LIMIT); | |
334 | } | |
335 | ||
336 | static inline void * __init memblock_virt_alloc_try_nid(phys_addr_t size, | |
337 | phys_addr_t align, phys_addr_t min_addr, phys_addr_t max_addr, int nid) | |
338 | { | |
339 | return __alloc_bootmem_node_high(NODE_DATA(nid), size, align, | |
340 | min_addr); | |
341 | } | |
342 | ||
ea1f5f37 PT |
343 | static inline void * __init memblock_virt_alloc_try_nid_raw( |
344 | phys_addr_t size, phys_addr_t align, | |
345 | phys_addr_t min_addr, phys_addr_t max_addr, int nid) | |
346 | { | |
347 | return ___alloc_bootmem_node_nopanic(NODE_DATA(nid), size, align, | |
348 | min_addr, max_addr); | |
349 | } | |
350 | ||
26f09e9b SS |
351 | static inline void * __init memblock_virt_alloc_try_nid_nopanic( |
352 | phys_addr_t size, phys_addr_t align, | |
353 | phys_addr_t min_addr, phys_addr_t max_addr, int nid) | |
354 | { | |
355 | return ___alloc_bootmem_node_nopanic(NODE_DATA(nid), size, align, | |
356 | min_addr, max_addr); | |
357 | } | |
358 | ||
359 | static inline void __init memblock_free_early( | |
360 | phys_addr_t base, phys_addr_t size) | |
361 | { | |
362 | free_bootmem(base, size); | |
363 | } | |
364 | ||
365 | static inline void __init memblock_free_early_nid( | |
366 | phys_addr_t base, phys_addr_t size, int nid) | |
367 | { | |
368 | free_bootmem_node(NODE_DATA(nid), base, size); | |
369 | } | |
370 | ||
371 | static inline void __init memblock_free_late( | |
372 | phys_addr_t base, phys_addr_t size) | |
373 | { | |
374 | free_bootmem_late(base, size); | |
375 | } | |
376 | #endif /* defined(CONFIG_HAVE_MEMBLOCK) && defined(CONFIG_NO_BOOTMEM) */ | |
377 | ||
f71bf0ca FBH |
378 | extern void *alloc_large_system_hash(const char *tablename, |
379 | unsigned long bucketsize, | |
380 | unsigned long numentries, | |
381 | int scale, | |
382 | int flags, | |
383 | unsigned int *_hash_shift, | |
384 | unsigned int *_hash_mask, | |
31fe62b9 TB |
385 | unsigned long low_limit, |
386 | unsigned long high_limit); | |
1da177e4 | 387 | |
04903664 | 388 | #define HASH_EARLY 0x00000001 /* Allocating during early boot? */ |
2c85f51d JB |
389 | #define HASH_SMALL 0x00000002 /* sub-page allocation allowed, min |
390 | * shift passed via *_hash_shift */ | |
3749a8f0 | 391 | #define HASH_ZERO 0x00000004 /* Zero allocated hash table */ |
1da177e4 | 392 | |
c2fdf3a9 AB |
393 | /* Only NUMA needs hash distribution. 64bit NUMA architectures have |
394 | * sufficient vmalloc space. | |
1da177e4 | 395 | */ |
a9919c79 RV |
396 | #ifdef CONFIG_NUMA |
397 | #define HASHDIST_DEFAULT IS_ENABLED(CONFIG_64BIT) | |
398 | extern int hashdist; /* Distribute hashes across NUMA nodes? */ | |
1da177e4 | 399 | #else |
a9919c79 | 400 | #define hashdist (0) |
1da177e4 | 401 | #endif |
1da177e4 LT |
402 | |
403 | ||
404 | #endif /* _LINUX_BOOTMEM_H */ |