Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
b8ef9172 TH |
2 | /* |
3 | * NUMA emulation | |
4 | */ | |
5 | #include <linux/kernel.h> | |
6 | #include <linux/errno.h> | |
7 | #include <linux/topology.h> | |
8 | #include <linux/memblock.h> | |
9 | #include <asm/dma.h> | |
10 | ||
11 | #include "numa_internal.h" | |
12 | ||
148f9bb8 | 13 | static int emu_nid_to_phys[MAX_NUMNODES]; |
b8ef9172 TH |
14 | static char *emu_cmdline __initdata; |
15 | ||
16 | void __init numa_emu_cmdline(char *str) | |
17 | { | |
18 | emu_cmdline = str; | |
19 | } | |
20 | ||
21 | static int __init emu_find_memblk_by_nid(int nid, const struct numa_meminfo *mi) | |
22 | { | |
23 | int i; | |
24 | ||
25 | for (i = 0; i < mi->nr_blks; i++) | |
26 | if (mi->blk[i].nid == nid) | |
27 | return i; | |
28 | return -ENOENT; | |
29 | } | |
30 | ||
e37aade3 | 31 | static u64 __init mem_hole_size(u64 start, u64 end) |
474b881b TH |
32 | { |
33 | unsigned long start_pfn = PFN_UP(start); | |
34 | unsigned long end_pfn = PFN_DOWN(end); | |
35 | ||
36 | if (start_pfn < end_pfn) | |
37 | return PFN_PHYS(absent_pages_in_range(start_pfn, end_pfn)); | |
38 | return 0; | |
39 | } | |
40 | ||
b8ef9172 TH |
41 | /* |
42 | * Sets up nid to range from @start to @end. The return value is -errno if | |
43 | * something went wrong, 0 otherwise. | |
44 | */ | |
45 | static int __init emu_setup_memblk(struct numa_meminfo *ei, | |
46 | struct numa_meminfo *pi, | |
47 | int nid, int phys_blk, u64 size) | |
48 | { | |
49 | struct numa_memblk *eb = &ei->blk[ei->nr_blks]; | |
50 | struct numa_memblk *pb = &pi->blk[phys_blk]; | |
51 | ||
52 | if (ei->nr_blks >= NR_NODE_MEMBLKS) { | |
53 | pr_err("NUMA: Too many emulated memblks, failing emulation\n"); | |
54 | return -EINVAL; | |
55 | } | |
56 | ||
57 | ei->nr_blks++; | |
58 | eb->start = pb->start; | |
59 | eb->end = pb->start + size; | |
60 | eb->nid = nid; | |
61 | ||
62 | if (emu_nid_to_phys[nid] == NUMA_NO_NODE) | |
3b6c62f3 | 63 | emu_nid_to_phys[nid] = pb->nid; |
b8ef9172 TH |
64 | |
65 | pb->start += size; | |
66 | if (pb->start >= pb->end) { | |
67 | WARN_ON_ONCE(pb->start > pb->end); | |
68 | numa_remove_memblk_from(phys_blk, pi); | |
69 | } | |
70 | ||
365811d6 BH |
71 | printk(KERN_INFO "Faking node %d at [mem %#018Lx-%#018Lx] (%LuMB)\n", |
72 | nid, eb->start, eb->end - 1, (eb->end - eb->start) >> 20); | |
b8ef9172 TH |
73 | return 0; |
74 | } | |
75 | ||
76 | /* | |
77 | * Sets up nr_nodes fake nodes interleaved over physical nodes ranging from addr | |
d80a9eb3 WY |
78 | * to max_addr. |
79 | * | |
80 | * Returns zero on success or negative on error. | |
b8ef9172 TH |
81 | */ |
82 | static int __init split_nodes_interleave(struct numa_meminfo *ei, | |
83 | struct numa_meminfo *pi, | |
84 | u64 addr, u64 max_addr, int nr_nodes) | |
85 | { | |
d80a9eb3 | 86 | nodemask_t physnode_mask = numa_nodes_parsed; |
b8ef9172 TH |
87 | u64 size; |
88 | int big; | |
89 | int nid = 0; | |
90 | int i, ret; | |
91 | ||
92 | if (nr_nodes <= 0) | |
93 | return -1; | |
94 | if (nr_nodes > MAX_NUMNODES) { | |
95 | pr_info("numa=fake=%d too large, reducing to %d\n", | |
96 | nr_nodes, MAX_NUMNODES); | |
97 | nr_nodes = MAX_NUMNODES; | |
98 | } | |
99 | ||
1b7e03ef TH |
100 | /* |
101 | * Calculate target node size. x86_32 freaks on __udivdi3() so do | |
102 | * the division in ulong number of pages and convert back. | |
103 | */ | |
474b881b | 104 | size = max_addr - addr - mem_hole_size(addr, max_addr); |
1b7e03ef TH |
105 | size = PFN_PHYS((unsigned long)(size >> PAGE_SHIFT) / nr_nodes); |
106 | ||
b8ef9172 TH |
107 | /* |
108 | * Calculate the number of big nodes that can be allocated as a result | |
109 | * of consolidating the remainder. | |
110 | */ | |
111 | big = ((size & ~FAKE_NODE_MIN_HASH_MASK) * nr_nodes) / | |
112 | FAKE_NODE_MIN_SIZE; | |
113 | ||
114 | size &= FAKE_NODE_MIN_HASH_MASK; | |
115 | if (!size) { | |
116 | pr_err("Not enough memory for each node. " | |
117 | "NUMA emulation disabled.\n"); | |
118 | return -1; | |
119 | } | |
120 | ||
b8ef9172 TH |
121 | /* |
122 | * Continue to fill physical nodes with fake nodes until there is no | |
123 | * memory left on any of them. | |
124 | */ | |
125 | while (nodes_weight(physnode_mask)) { | |
126 | for_each_node_mask(i, physnode_mask) { | |
127 | u64 dma32_end = PFN_PHYS(MAX_DMA32_PFN); | |
128 | u64 start, limit, end; | |
129 | int phys_blk; | |
130 | ||
131 | phys_blk = emu_find_memblk_by_nid(i, pi); | |
132 | if (phys_blk < 0) { | |
133 | node_clear(i, physnode_mask); | |
134 | continue; | |
135 | } | |
136 | start = pi->blk[phys_blk].start; | |
137 | limit = pi->blk[phys_blk].end; | |
138 | end = start + size; | |
139 | ||
140 | if (nid < big) | |
141 | end += FAKE_NODE_MIN_SIZE; | |
142 | ||
143 | /* | |
144 | * Continue to add memory to this fake node if its | |
145 | * non-reserved memory is less than the per-node size. | |
146 | */ | |
474b881b | 147 | while (end - start - mem_hole_size(start, end) < size) { |
b8ef9172 TH |
148 | end += FAKE_NODE_MIN_SIZE; |
149 | if (end > limit) { | |
150 | end = limit; | |
151 | break; | |
152 | } | |
153 | } | |
154 | ||
155 | /* | |
156 | * If there won't be at least FAKE_NODE_MIN_SIZE of | |
157 | * non-reserved memory in ZONE_DMA32 for the next node, | |
158 | * this one must extend to the boundary. | |
159 | */ | |
160 | if (end < dma32_end && dma32_end - end - | |
474b881b | 161 | mem_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE) |
b8ef9172 TH |
162 | end = dma32_end; |
163 | ||
164 | /* | |
165 | * If there won't be enough non-reserved memory for the | |
166 | * next node, this one must extend to the end of the | |
167 | * physical node. | |
168 | */ | |
474b881b | 169 | if (limit - end - mem_hole_size(end, limit) < size) |
b8ef9172 TH |
170 | end = limit; |
171 | ||
172 | ret = emu_setup_memblk(ei, pi, nid++ % nr_nodes, | |
173 | phys_blk, | |
174 | min(end, limit) - start); | |
175 | if (ret < 0) | |
176 | return ret; | |
177 | } | |
178 | } | |
179 | return 0; | |
180 | } | |
181 | ||
182 | /* | |
183 | * Returns the end address of a node so that there is at least `size' amount of | |
184 | * non-reserved memory or `max_addr' is reached. | |
185 | */ | |
186 | static u64 __init find_end_of_node(u64 start, u64 max_addr, u64 size) | |
187 | { | |
188 | u64 end = start + size; | |
189 | ||
474b881b | 190 | while (end - start - mem_hole_size(start, end) < size) { |
b8ef9172 TH |
191 | end += FAKE_NODE_MIN_SIZE; |
192 | if (end > max_addr) { | |
193 | end = max_addr; | |
194 | break; | |
195 | } | |
196 | } | |
197 | return end; | |
198 | } | |
199 | ||
cc9aec03 DW |
200 | static u64 uniform_size(u64 max_addr, u64 base, u64 hole, int nr_nodes) |
201 | { | |
202 | unsigned long max_pfn = PHYS_PFN(max_addr); | |
203 | unsigned long base_pfn = PHYS_PFN(base); | |
204 | unsigned long hole_pfns = PHYS_PFN(hole); | |
205 | ||
206 | return PFN_PHYS((max_pfn - base_pfn - hole_pfns) / nr_nodes); | |
207 | } | |
208 | ||
b8ef9172 TH |
209 | /* |
210 | * Sets up fake nodes of `size' interleaved over physical nodes ranging from | |
d80a9eb3 WY |
211 | * `addr' to `max_addr'. |
212 | * | |
213 | * Returns zero on success or negative on error. | |
b8ef9172 | 214 | */ |
cc9aec03 | 215 | static int __init split_nodes_size_interleave_uniform(struct numa_meminfo *ei, |
b8ef9172 | 216 | struct numa_meminfo *pi, |
cc9aec03 DW |
217 | u64 addr, u64 max_addr, u64 size, |
218 | int nr_nodes, struct numa_memblk *pblk, | |
219 | int nid) | |
b8ef9172 | 220 | { |
d80a9eb3 | 221 | nodemask_t physnode_mask = numa_nodes_parsed; |
cc9aec03 | 222 | int i, ret, uniform = 0; |
b8ef9172 | 223 | u64 min_size; |
b8ef9172 | 224 | |
cc9aec03 | 225 | if ((!size && !nr_nodes) || (nr_nodes && !pblk)) |
b8ef9172 | 226 | return -1; |
cc9aec03 | 227 | |
b8ef9172 | 228 | /* |
cc9aec03 DW |
229 | * In the 'uniform' case split the passed in physical node by |
230 | * nr_nodes, in the non-uniform case, ignore the passed in | |
231 | * physical block and try to create nodes of at least size | |
232 | * @size. | |
233 | * | |
234 | * In the uniform case, split the nodes strictly by physical | |
235 | * capacity, i.e. ignore holes. In the non-uniform case account | |
236 | * for holes and treat @size as a minimum floor. | |
b8ef9172 | 237 | */ |
cc9aec03 DW |
238 | if (!nr_nodes) |
239 | nr_nodes = MAX_NUMNODES; | |
240 | else { | |
241 | nodes_clear(physnode_mask); | |
242 | node_set(pblk->nid, physnode_mask); | |
243 | uniform = 1; | |
244 | } | |
245 | ||
246 | if (uniform) { | |
247 | min_size = uniform_size(max_addr, addr, 0, nr_nodes); | |
248 | size = min_size; | |
249 | } else { | |
250 | /* | |
251 | * The limit on emulated nodes is MAX_NUMNODES, so the | |
252 | * size per node is increased accordingly if the | |
253 | * requested size is too small. This creates a uniform | |
254 | * distribution of node sizes across the entire machine | |
255 | * (but not necessarily over physical nodes). | |
256 | */ | |
257 | min_size = uniform_size(max_addr, addr, | |
258 | mem_hole_size(addr, max_addr), nr_nodes); | |
259 | } | |
260 | min_size = ALIGN(max(min_size, FAKE_NODE_MIN_SIZE), FAKE_NODE_MIN_SIZE); | |
b8ef9172 TH |
261 | if (size < min_size) { |
262 | pr_err("Fake node size %LuMB too small, increasing to %LuMB\n", | |
263 | size >> 20, min_size >> 20); | |
264 | size = min_size; | |
265 | } | |
cc9aec03 | 266 | size = ALIGN_DOWN(size, FAKE_NODE_MIN_SIZE); |
b8ef9172 | 267 | |
b8ef9172 TH |
268 | /* |
269 | * Fill physical nodes with fake nodes of size until there is no memory | |
270 | * left on any of them. | |
271 | */ | |
272 | while (nodes_weight(physnode_mask)) { | |
273 | for_each_node_mask(i, physnode_mask) { | |
1b7e03ef | 274 | u64 dma32_end = PFN_PHYS(MAX_DMA32_PFN); |
b8ef9172 TH |
275 | u64 start, limit, end; |
276 | int phys_blk; | |
277 | ||
278 | phys_blk = emu_find_memblk_by_nid(i, pi); | |
279 | if (phys_blk < 0) { | |
280 | node_clear(i, physnode_mask); | |
281 | continue; | |
282 | } | |
cc9aec03 | 283 | |
b8ef9172 TH |
284 | start = pi->blk[phys_blk].start; |
285 | limit = pi->blk[phys_blk].end; | |
286 | ||
cc9aec03 DW |
287 | if (uniform) |
288 | end = start + size; | |
289 | else | |
290 | end = find_end_of_node(start, limit, size); | |
b8ef9172 TH |
291 | /* |
292 | * If there won't be at least FAKE_NODE_MIN_SIZE of | |
293 | * non-reserved memory in ZONE_DMA32 for the next node, | |
294 | * this one must extend to the boundary. | |
295 | */ | |
296 | if (end < dma32_end && dma32_end - end - | |
474b881b | 297 | mem_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE) |
b8ef9172 TH |
298 | end = dma32_end; |
299 | ||
300 | /* | |
301 | * If there won't be enough non-reserved memory for the | |
302 | * next node, this one must extend to the end of the | |
303 | * physical node. | |
304 | */ | |
cc9aec03 DW |
305 | if ((limit - end - mem_hole_size(end, limit) < size) |
306 | && !uniform) | |
b8ef9172 TH |
307 | end = limit; |
308 | ||
309 | ret = emu_setup_memblk(ei, pi, nid++ % MAX_NUMNODES, | |
310 | phys_blk, | |
311 | min(end, limit) - start); | |
312 | if (ret < 0) | |
313 | return ret; | |
314 | } | |
315 | } | |
cc9aec03 DW |
316 | return nid; |
317 | } | |
318 | ||
319 | static int __init split_nodes_size_interleave(struct numa_meminfo *ei, | |
320 | struct numa_meminfo *pi, | |
321 | u64 addr, u64 max_addr, u64 size) | |
322 | { | |
323 | return split_nodes_size_interleave_uniform(ei, pi, addr, max_addr, size, | |
324 | 0, NULL, NUMA_NO_NODE); | |
b8ef9172 TH |
325 | } |
326 | ||
158f424f WY |
327 | int __init setup_emu2phys_nid(int *dfl_phys_nid) |
328 | { | |
329 | int i, max_emu_nid = 0; | |
330 | ||
331 | *dfl_phys_nid = NUMA_NO_NODE; | |
332 | for (i = 0; i < ARRAY_SIZE(emu_nid_to_phys); i++) { | |
333 | if (emu_nid_to_phys[i] != NUMA_NO_NODE) { | |
334 | max_emu_nid = i; | |
335 | if (*dfl_phys_nid == NUMA_NO_NODE) | |
336 | *dfl_phys_nid = emu_nid_to_phys[i]; | |
337 | } | |
338 | } | |
339 | ||
340 | return max_emu_nid; | |
341 | } | |
342 | ||
90e6b677 TH |
343 | /** |
344 | * numa_emulation - Emulate NUMA nodes | |
345 | * @numa_meminfo: NUMA configuration to massage | |
346 | * @numa_dist_cnt: The size of the physical NUMA distance table | |
347 | * | |
348 | * Emulate NUMA nodes according to the numa=fake kernel parameter. | |
349 | * @numa_meminfo contains the physical memory configuration and is modified | |
350 | * to reflect the emulated configuration on success. @numa_dist_cnt is | |
351 | * used to determine the size of the physical distance table. | |
352 | * | |
353 | * On success, the following modifications are made. | |
354 | * | |
355 | * - @numa_meminfo is updated to reflect the emulated nodes. | |
356 | * | |
357 | * - __apicid_to_node[] is updated such that APIC IDs are mapped to the | |
358 | * emulated nodes. | |
359 | * | |
360 | * - NUMA distance table is rebuilt to represent distances between emulated | |
361 | * nodes. The distances are determined considering how emulated nodes | |
362 | * are mapped to physical nodes and match the actual distances. | |
363 | * | |
364 | * - emu_nid_to_phys[] reflects how emulated nodes are mapped to physical | |
365 | * nodes. This is used by numa_add_cpu() and numa_remove_cpu(). | |
366 | * | |
367 | * If emulation is not enabled or fails, emu_nid_to_phys[] is filled with | |
368 | * identity mapping and no other modification is made. | |
b8ef9172 TH |
369 | */ |
370 | void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt) | |
371 | { | |
372 | static struct numa_meminfo ei __initdata; | |
373 | static struct numa_meminfo pi __initdata; | |
1b7e03ef | 374 | const u64 max_addr = PFN_PHYS(max_pfn); |
b8ef9172 | 375 | u8 *phys_dist = NULL; |
ce003330 | 376 | size_t phys_size = numa_dist_cnt * numa_dist_cnt * sizeof(phys_dist[0]); |
56396e68 | 377 | int max_emu_nid, dfl_phys_nid; |
b8ef9172 TH |
378 | int i, j, ret; |
379 | ||
380 | if (!emu_cmdline) | |
381 | goto no_emu; | |
382 | ||
383 | memset(&ei, 0, sizeof(ei)); | |
384 | pi = *numa_meminfo; | |
385 | ||
386 | for (i = 0; i < MAX_NUMNODES; i++) | |
387 | emu_nid_to_phys[i] = NUMA_NO_NODE; | |
388 | ||
389 | /* | |
390 | * If the numa=fake command-line contains a 'M' or 'G', it represents | |
391 | * the fixed node size. Otherwise, if it is just a single number N, | |
392 | * split the system RAM into N fake nodes. | |
393 | */ | |
cc9aec03 DW |
394 | if (strchr(emu_cmdline, 'U')) { |
395 | nodemask_t physnode_mask = numa_nodes_parsed; | |
396 | unsigned long n; | |
397 | int nid = 0; | |
398 | ||
399 | n = simple_strtoul(emu_cmdline, &emu_cmdline, 0); | |
400 | ret = -1; | |
401 | for_each_node_mask(i, physnode_mask) { | |
402 | ret = split_nodes_size_interleave_uniform(&ei, &pi, | |
403 | pi.blk[i].start, pi.blk[i].end, 0, | |
404 | n, &pi.blk[i], nid); | |
405 | if (ret < 0) | |
406 | break; | |
407 | if (ret < n) { | |
408 | pr_info("%s: phys: %d only got %d of %ld nodes, failing\n", | |
409 | __func__, i, ret, n); | |
410 | ret = -1; | |
411 | break; | |
412 | } | |
413 | nid = ret; | |
414 | } | |
415 | } else if (strchr(emu_cmdline, 'M') || strchr(emu_cmdline, 'G')) { | |
b8ef9172 TH |
416 | u64 size; |
417 | ||
418 | size = memparse(emu_cmdline, &emu_cmdline); | |
419 | ret = split_nodes_size_interleave(&ei, &pi, 0, max_addr, size); | |
420 | } else { | |
421 | unsigned long n; | |
422 | ||
94c0dd32 | 423 | n = simple_strtoul(emu_cmdline, &emu_cmdline, 0); |
b8ef9172 TH |
424 | ret = split_nodes_interleave(&ei, &pi, 0, max_addr, n); |
425 | } | |
94c0dd32 PZ |
426 | if (*emu_cmdline == ':') |
427 | emu_cmdline++; | |
b8ef9172 TH |
428 | |
429 | if (ret < 0) | |
430 | goto no_emu; | |
431 | ||
432 | if (numa_cleanup_meminfo(&ei) < 0) { | |
433 | pr_warning("NUMA: Warning: constructed meminfo invalid, disabling emulation\n"); | |
434 | goto no_emu; | |
435 | } | |
436 | ||
ce003330 | 437 | /* copy the physical distance table */ |
b8ef9172 | 438 | if (numa_dist_cnt) { |
b8ef9172 TH |
439 | u64 phys; |
440 | ||
1b7e03ef | 441 | phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped), |
ce003330 | 442 | phys_size, PAGE_SIZE); |
1f5026a7 | 443 | if (!phys) { |
b8ef9172 TH |
444 | pr_warning("NUMA: Warning: can't allocate copy of distance table, disabling emulation\n"); |
445 | goto no_emu; | |
446 | } | |
24aa0788 | 447 | memblock_reserve(phys, phys_size); |
b8ef9172 TH |
448 | phys_dist = __va(phys); |
449 | ||
450 | for (i = 0; i < numa_dist_cnt; i++) | |
451 | for (j = 0; j < numa_dist_cnt; j++) | |
452 | phys_dist[i * numa_dist_cnt + j] = | |
453 | node_distance(i, j); | |
454 | } | |
455 | ||
56396e68 TH |
456 | /* |
457 | * Determine the max emulated nid and the default phys nid to use | |
458 | * for unmapped nodes. | |
459 | */ | |
158f424f | 460 | max_emu_nid = setup_emu2phys_nid(&dfl_phys_nid); |
078a1989 | 461 | |
b8ef9172 TH |
462 | /* commit */ |
463 | *numa_meminfo = ei; | |
464 | ||
4f167201 WY |
465 | /* Make sure numa_nodes_parsed only contains emulated nodes */ |
466 | nodes_clear(numa_nodes_parsed); | |
467 | for (i = 0; i < ARRAY_SIZE(ei.blk); i++) | |
468 | if (ei.blk[i].start != ei.blk[i].end && | |
469 | ei.blk[i].nid != NUMA_NO_NODE) | |
470 | node_set(ei.blk[i].nid, numa_nodes_parsed); | |
471 | ||
b8ef9172 TH |
472 | /* |
473 | * Transform __apicid_to_node table to use emulated nids by | |
474 | * reverse-mapping phys_nid. The maps should always exist but fall | |
475 | * back to zero just in case. | |
476 | */ | |
477 | for (i = 0; i < ARRAY_SIZE(__apicid_to_node); i++) { | |
478 | if (__apicid_to_node[i] == NUMA_NO_NODE) | |
479 | continue; | |
480 | for (j = 0; j < ARRAY_SIZE(emu_nid_to_phys); j++) | |
481 | if (__apicid_to_node[i] == emu_nid_to_phys[j]) | |
482 | break; | |
483 | __apicid_to_node[i] = j < ARRAY_SIZE(emu_nid_to_phys) ? j : 0; | |
484 | } | |
485 | ||
486 | /* make sure all emulated nodes are mapped to a physical node */ | |
487 | for (i = 0; i < ARRAY_SIZE(emu_nid_to_phys); i++) | |
488 | if (emu_nid_to_phys[i] == NUMA_NO_NODE) | |
078a1989 | 489 | emu_nid_to_phys[i] = dfl_phys_nid; |
b8ef9172 | 490 | |
56396e68 | 491 | /* transform distance table */ |
b8ef9172 | 492 | numa_reset_distance(); |
56396e68 TH |
493 | for (i = 0; i < max_emu_nid + 1; i++) { |
494 | for (j = 0; j < max_emu_nid + 1; j++) { | |
b8ef9172 TH |
495 | int physi = emu_nid_to_phys[i]; |
496 | int physj = emu_nid_to_phys[j]; | |
497 | int dist; | |
498 | ||
94c0dd32 PZ |
499 | if (get_option(&emu_cmdline, &dist) == 2) |
500 | ; | |
501 | else if (physi >= numa_dist_cnt || physj >= numa_dist_cnt) | |
b8ef9172 TH |
502 | dist = physi == physj ? |
503 | LOCAL_DISTANCE : REMOTE_DISTANCE; | |
504 | else | |
505 | dist = phys_dist[physi * numa_dist_cnt + physj]; | |
506 | ||
507 | numa_set_distance(i, j, dist); | |
508 | } | |
509 | } | |
ce003330 YL |
510 | |
511 | /* free the copied physical distance table */ | |
512 | if (phys_dist) | |
24aa0788 | 513 | memblock_free(__pa(phys_dist), phys_size); |
b8ef9172 TH |
514 | return; |
515 | ||
516 | no_emu: | |
517 | /* No emulation. Build identity emu_nid_to_phys[] for numa_add_cpu() */ | |
518 | for (i = 0; i < ARRAY_SIZE(emu_nid_to_phys); i++) | |
519 | emu_nid_to_phys[i] = i; | |
520 | } | |
521 | ||
522 | #ifndef CONFIG_DEBUG_PER_CPU_MAPS | |
148f9bb8 | 523 | void numa_add_cpu(int cpu) |
b8ef9172 TH |
524 | { |
525 | int physnid, nid; | |
526 | ||
51b361b4 | 527 | nid = early_cpu_to_node(cpu); |
b8ef9172 TH |
528 | BUG_ON(nid == NUMA_NO_NODE || !node_online(nid)); |
529 | ||
530 | physnid = emu_nid_to_phys[nid]; | |
531 | ||
532 | /* | |
533 | * Map the cpu to each emulated node that is allocated on the physical | |
534 | * node of the cpu's apic id. | |
535 | */ | |
536 | for_each_online_node(nid) | |
537 | if (emu_nid_to_phys[nid] == physnid) | |
538 | cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); | |
539 | } | |
540 | ||
148f9bb8 | 541 | void numa_remove_cpu(int cpu) |
b8ef9172 TH |
542 | { |
543 | int i; | |
544 | ||
545 | for_each_online_node(i) | |
546 | cpumask_clear_cpu(cpu, node_to_cpumask_map[i]); | |
547 | } | |
548 | #else /* !CONFIG_DEBUG_PER_CPU_MAPS */ | |
148f9bb8 | 549 | static void numa_set_cpumask(int cpu, bool enable) |
b8ef9172 | 550 | { |
7a6c6547 | 551 | int nid, physnid; |
b8ef9172 TH |
552 | |
553 | nid = early_cpu_to_node(cpu); | |
554 | if (nid == NUMA_NO_NODE) { | |
555 | /* early_cpu_to_node() already emits a warning and trace */ | |
556 | return; | |
557 | } | |
558 | ||
559 | physnid = emu_nid_to_phys[nid]; | |
560 | ||
7a6c6547 | 561 | for_each_online_node(nid) { |
b8ef9172 TH |
562 | if (emu_nid_to_phys[nid] != physnid) |
563 | continue; | |
564 | ||
7a6c6547 | 565 | debug_cpumask_set_cpu(cpu, nid, enable); |
b8ef9172 TH |
566 | } |
567 | } | |
568 | ||
148f9bb8 | 569 | void numa_add_cpu(int cpu) |
b8ef9172 | 570 | { |
7a6c6547 | 571 | numa_set_cpumask(cpu, true); |
b8ef9172 TH |
572 | } |
573 | ||
148f9bb8 | 574 | void numa_remove_cpu(int cpu) |
b8ef9172 | 575 | { |
7a6c6547 | 576 | numa_set_cpumask(cpu, false); |
b8ef9172 TH |
577 | } |
578 | #endif /* !CONFIG_DEBUG_PER_CPU_MAPS */ |