Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
3947be19 | 2 | /* |
10fbcf4c | 3 | * Memory subsystem support |
3947be19 DH |
4 | * |
5 | * Written by Matt Tolentino <matthew.e.tolentino@intel.com> | |
6 | * Dave Hansen <haveblue@us.ibm.com> | |
7 | * | |
8 | * This file provides the necessary infrastructure to represent | |
9 | * a SPARSEMEM-memory-model system's physical memory in /sysfs. | |
10 | * All arch-independent code that assumes MEMORY_HOTPLUG requires | |
11 | * SPARSEMEM should be contained here, or in mm/memory_hotplug.c. | |
12 | */ | |
13 | ||
3947be19 DH |
14 | #include <linux/module.h> |
15 | #include <linux/init.h> | |
3947be19 | 16 | #include <linux/topology.h> |
c59ede7b | 17 | #include <linux/capability.h> |
3947be19 DH |
18 | #include <linux/device.h> |
19 | #include <linux/memory.h> | |
3947be19 DH |
20 | #include <linux/memory_hotplug.h> |
21 | #include <linux/mm.h> | |
9f1b16a5 | 22 | #include <linux/stat.h> |
5a0e3ad6 | 23 | #include <linux/slab.h> |
4fb6eabf | 24 | #include <linux/xarray.h> |
9f1b16a5 | 25 | |
60063497 | 26 | #include <linux/atomic.h> |
7c0f6ba6 | 27 | #include <linux/uaccess.h> |
3947be19 DH |
28 | |
29 | #define MEMORY_CLASS_NAME "memory" | |
0c2c99b1 | 30 | |
4dc8207b DH |
31 | static const char *const online_type_to_str[] = { |
32 | [MMOP_OFFLINE] = "offline", | |
33 | [MMOP_ONLINE] = "online", | |
34 | [MMOP_ONLINE_KERNEL] = "online_kernel", | |
35 | [MMOP_ONLINE_MOVABLE] = "online_movable", | |
36 | }; | |
37 | ||
1adf8b46 | 38 | int mhp_online_type_from_str(const char *str) |
4dc8207b DH |
39 | { |
40 | int i; | |
41 | ||
42 | for (i = 0; i < ARRAY_SIZE(online_type_to_str); i++) { | |
43 | if (sysfs_streq(str, online_type_to_str[i])) | |
44 | return i; | |
45 | } | |
46 | return -EINVAL; | |
47 | } | |
48 | ||
7315f0cc GZ |
49 | #define to_memory_block(dev) container_of(dev, struct memory_block, dev) |
50 | ||
0c2c99b1 NF |
51 | static int sections_per_block; |
52 | ||
178bdbed | 53 | static inline unsigned long memory_block_id(unsigned long section_nr) |
0c2c99b1 NF |
54 | { |
55 | return section_nr / sections_per_block; | |
56 | } | |
3947be19 | 57 | |
90ec010f | 58 | static inline unsigned long pfn_to_block_id(unsigned long pfn) |
db051a0d | 59 | { |
178bdbed | 60 | return memory_block_id(pfn_to_section_nr(pfn)); |
db051a0d DH |
61 | } |
62 | ||
ea884641 DH |
63 | static inline unsigned long phys_to_block_id(unsigned long phys) |
64 | { | |
65 | return pfn_to_block_id(PFN_DOWN(phys)); | |
66 | } | |
67 | ||
4960e05e RW |
68 | static int memory_subsys_online(struct device *dev); |
69 | static int memory_subsys_offline(struct device *dev); | |
70 | ||
580fc9c7 | 71 | static const struct bus_type memory_subsys = { |
af5ca3f4 | 72 | .name = MEMORY_CLASS_NAME, |
10fbcf4c | 73 | .dev_name = MEMORY_CLASS_NAME, |
4960e05e RW |
74 | .online = memory_subsys_online, |
75 | .offline = memory_subsys_offline, | |
3947be19 DH |
76 | }; |
77 | ||
4fb6eabf SC |
78 | /* |
79 | * Memory blocks are cached in a local radix tree to avoid | |
80 | * a costly linear search for the corresponding device on | |
81 | * the subsystem bus. | |
82 | */ | |
83 | static DEFINE_XARRAY(memory_blocks); | |
84 | ||
028fc57a DH |
85 | /* |
86 | * Memory groups, indexed by memory group id (mgid). | |
87 | */ | |
88 | static DEFINE_XARRAY_FLAGS(memory_groups, XA_FLAGS_ALLOC); | |
3fcebf90 | 89 | #define MEMORY_GROUP_MARK_DYNAMIC XA_MARK_1 |
028fc57a | 90 | |
e041c683 | 91 | static BLOCKING_NOTIFIER_HEAD(memory_chain); |
3947be19 | 92 | |
98a38ebd | 93 | int register_memory_notifier(struct notifier_block *nb) |
3947be19 | 94 | { |
2aeebca2 | 95 | return blocking_notifier_chain_register(&memory_chain, nb); |
3947be19 | 96 | } |
3c82c30c | 97 | EXPORT_SYMBOL(register_memory_notifier); |
3947be19 | 98 | |
98a38ebd | 99 | void unregister_memory_notifier(struct notifier_block *nb) |
3947be19 | 100 | { |
2aeebca2 | 101 | blocking_notifier_chain_unregister(&memory_chain, nb); |
3947be19 | 102 | } |
3c82c30c | 103 | EXPORT_SYMBOL(unregister_memory_notifier); |
3947be19 | 104 | |
fa7194eb YI |
105 | static void memory_block_release(struct device *dev) |
106 | { | |
7315f0cc | 107 | struct memory_block *mem = to_memory_block(dev); |
1a8c64e1 AK |
108 | /* Verify that the altmap is freed */ |
109 | WARN_ON(mem->altmap); | |
fa7194eb YI |
110 | kfree(mem); |
111 | } | |
112 | ||
0c2c99b1 NF |
113 | unsigned long __weak memory_block_size_bytes(void) |
114 | { | |
115 | return MIN_MEMORY_BLOCK_SIZE; | |
116 | } | |
c221c0b0 | 117 | EXPORT_SYMBOL_GPL(memory_block_size_bytes); |
0c2c99b1 | 118 | |
7c09f428 | 119 | /* Show the memory block ID, relative to the memory block size */ |
3f8e9178 DH |
120 | static ssize_t phys_index_show(struct device *dev, |
121 | struct device_attribute *attr, char *buf) | |
3947be19 | 122 | { |
7315f0cc | 123 | struct memory_block *mem = to_memory_block(dev); |
948b3edb | 124 | |
7c09f428 | 125 | return sysfs_emit(buf, "%08lx\n", memory_block_id(mem->start_section_nr)); |
d3360164 NF |
126 | } |
127 | ||
5c755e9f | 128 | /* |
53cdc1cb DH |
129 | * Legacy interface that we cannot remove. Always indicate "removable" |
130 | * with CONFIG_MEMORY_HOTREMOVE - bad heuristic. | |
5c755e9f | 131 | */ |
3f8e9178 DH |
132 | static ssize_t removable_show(struct device *dev, struct device_attribute *attr, |
133 | char *buf) | |
5c755e9f | 134 | { |
aa838896 | 135 | return sysfs_emit(buf, "%d\n", (int)IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)); |
5c755e9f BP |
136 | } |
137 | ||
3947be19 DH |
138 | /* |
139 | * online, offline, going offline, etc. | |
140 | */ | |
3f8e9178 DH |
141 | static ssize_t state_show(struct device *dev, struct device_attribute *attr, |
142 | char *buf) | |
3947be19 | 143 | { |
7315f0cc | 144 | struct memory_block *mem = to_memory_block(dev); |
973c3911 | 145 | const char *output; |
3947be19 DH |
146 | |
147 | /* | |
148 | * We can probably put these states in a nice little array | |
149 | * so that they're not open-coded | |
150 | */ | |
151 | switch (mem->state) { | |
3d3af6af | 152 | case MEM_ONLINE: |
973c3911 | 153 | output = "online"; |
3d3af6af IC |
154 | break; |
155 | case MEM_OFFLINE: | |
973c3911 | 156 | output = "offline"; |
3d3af6af IC |
157 | break; |
158 | case MEM_GOING_OFFLINE: | |
973c3911 | 159 | output = "going-offline"; |
3d3af6af IC |
160 | break; |
161 | default: | |
3d3af6af | 162 | WARN_ON(1); |
973c3911 | 163 | return sysfs_emit(buf, "ERROR-UNKNOWN-%ld\n", mem->state); |
3947be19 DH |
164 | } |
165 | ||
973c3911 | 166 | return sysfs_emit(buf, "%s\n", output); |
3947be19 DH |
167 | } |
168 | ||
7b78d335 | 169 | int memory_notify(unsigned long val, void *v) |
3947be19 | 170 | { |
e041c683 | 171 | return blocking_notifier_call_chain(&memory_chain, val, v); |
3947be19 DH |
172 | } |
173 | ||
5033091d NH |
174 | #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) |
175 | static unsigned long memblk_nr_poison(struct memory_block *mem); | |
176 | #else | |
177 | static inline unsigned long memblk_nr_poison(struct memory_block *mem) | |
178 | { | |
179 | return 0; | |
180 | } | |
181 | #endif | |
182 | ||
001002e7 SK |
183 | /* |
184 | * Must acquire mem_hotplug_lock in write mode. | |
185 | */ | |
8736cc2d OS |
186 | static int memory_block_online(struct memory_block *mem) |
187 | { | |
188 | unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); | |
189 | unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; | |
1a8c64e1 | 190 | unsigned long nr_vmemmap_pages = 0; |
c5f1e2d1 | 191 | struct memory_notify arg; |
a08a2ae3 OS |
192 | struct zone *zone; |
193 | int ret; | |
194 | ||
5033091d NH |
195 | if (memblk_nr_poison(mem)) |
196 | return -EHWPOISON; | |
197 | ||
445fcf7c DH |
198 | zone = zone_for_pfn_range(mem->online_type, mem->nid, mem->group, |
199 | start_pfn, nr_pages); | |
a08a2ae3 OS |
200 | |
201 | /* | |
202 | * Although vmemmap pages have a different lifecycle than the pages | |
203 | * they describe (they remain until the memory is unplugged), doing | |
204 | * their initialization and accounting at memory onlining/offlining | |
205 | * stage helps to keep accounting easier to follow - e.g vmemmaps | |
206 | * belong to the same zone as the memory they backed. | |
207 | */ | |
1a8c64e1 AK |
208 | if (mem->altmap) |
209 | nr_vmemmap_pages = mem->altmap->free; | |
210 | ||
c5f1e2d1 SK |
211 | arg.altmap_start_pfn = start_pfn; |
212 | arg.altmap_nr_pages = nr_vmemmap_pages; | |
213 | arg.start_pfn = start_pfn + nr_vmemmap_pages; | |
214 | arg.nr_pages = nr_pages - nr_vmemmap_pages; | |
001002e7 | 215 | mem_hotplug_begin(); |
c5f1e2d1 SK |
216 | ret = memory_notify(MEM_PREPARE_ONLINE, &arg); |
217 | ret = notifier_to_errno(ret); | |
218 | if (ret) | |
219 | goto out_notifier; | |
220 | ||
a08a2ae3 | 221 | if (nr_vmemmap_pages) { |
c5f1e2d1 SK |
222 | ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, |
223 | zone, mem->altmap->inaccessible); | |
a08a2ae3 | 224 | if (ret) |
001002e7 | 225 | goto out; |
a08a2ae3 OS |
226 | } |
227 | ||
228 | ret = online_pages(start_pfn + nr_vmemmap_pages, | |
836809ec | 229 | nr_pages - nr_vmemmap_pages, zone, mem->group); |
a08a2ae3 OS |
230 | if (ret) { |
231 | if (nr_vmemmap_pages) | |
232 | mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); | |
001002e7 | 233 | goto out; |
a08a2ae3 OS |
234 | } |
235 | ||
236 | /* | |
237 | * Account once onlining succeeded. If the zone was unpopulated, it is | |
238 | * now already properly populated. | |
239 | */ | |
240 | if (nr_vmemmap_pages) | |
836809ec | 241 | adjust_present_page_count(pfn_to_page(start_pfn), mem->group, |
4b097002 | 242 | nr_vmemmap_pages); |
8736cc2d | 243 | |
395f6081 | 244 | mem->zone = zone; |
c5f1e2d1 SK |
245 | mem_hotplug_done(); |
246 | return ret; | |
001002e7 | 247 | out: |
c5f1e2d1 SK |
248 | memory_notify(MEM_FINISH_OFFLINE, &arg); |
249 | out_notifier: | |
001002e7 | 250 | mem_hotplug_done(); |
a08a2ae3 | 251 | return ret; |
8736cc2d OS |
252 | } |
253 | ||
001002e7 SK |
254 | /* |
255 | * Must acquire mem_hotplug_lock in write mode. | |
256 | */ | |
8736cc2d OS |
257 | static int memory_block_offline(struct memory_block *mem) |
258 | { | |
259 | unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); | |
260 | unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; | |
1a8c64e1 | 261 | unsigned long nr_vmemmap_pages = 0; |
c5f1e2d1 | 262 | struct memory_notify arg; |
a08a2ae3 OS |
263 | int ret; |
264 | ||
395f6081 DH |
265 | if (!mem->zone) |
266 | return -EINVAL; | |
267 | ||
a08a2ae3 OS |
268 | /* |
269 | * Unaccount before offlining, such that unpopulated zone and kthreads | |
270 | * can properly be torn down in offline_pages(). | |
271 | */ | |
1a8c64e1 AK |
272 | if (mem->altmap) |
273 | nr_vmemmap_pages = mem->altmap->free; | |
274 | ||
001002e7 | 275 | mem_hotplug_begin(); |
4b097002 | 276 | if (nr_vmemmap_pages) |
836809ec | 277 | adjust_present_page_count(pfn_to_page(start_pfn), mem->group, |
4b097002 | 278 | -nr_vmemmap_pages); |
8736cc2d | 279 | |
a08a2ae3 | 280 | ret = offline_pages(start_pfn + nr_vmemmap_pages, |
395f6081 | 281 | nr_pages - nr_vmemmap_pages, mem->zone, mem->group); |
a08a2ae3 OS |
282 | if (ret) { |
283 | /* offline_pages() failed. Account back. */ | |
284 | if (nr_vmemmap_pages) | |
4b097002 | 285 | adjust_present_page_count(pfn_to_page(start_pfn), |
836809ec | 286 | mem->group, nr_vmemmap_pages); |
001002e7 | 287 | goto out; |
a08a2ae3 OS |
288 | } |
289 | ||
290 | if (nr_vmemmap_pages) | |
291 | mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); | |
292 | ||
395f6081 | 293 | mem->zone = NULL; |
c5f1e2d1 SK |
294 | arg.altmap_start_pfn = start_pfn; |
295 | arg.altmap_nr_pages = nr_vmemmap_pages; | |
296 | arg.start_pfn = start_pfn + nr_vmemmap_pages; | |
297 | arg.nr_pages = nr_pages - nr_vmemmap_pages; | |
298 | memory_notify(MEM_FINISH_OFFLINE, &arg); | |
001002e7 SK |
299 | out: |
300 | mem_hotplug_done(); | |
a08a2ae3 | 301 | return ret; |
8736cc2d OS |
302 | } |
303 | ||
3947be19 DH |
304 | /* |
305 | * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is | |
306 | * OK to have direct references to sparsemem variables in here. | |
307 | */ | |
308 | static int | |
8736cc2d | 309 | memory_block_action(struct memory_block *mem, unsigned long action) |
3947be19 | 310 | { |
3947be19 | 311 | int ret; |
3947be19 | 312 | |
3947be19 | 313 | switch (action) { |
3d3af6af | 314 | case MEM_ONLINE: |
8736cc2d | 315 | ret = memory_block_online(mem); |
3d3af6af IC |
316 | break; |
317 | case MEM_OFFLINE: | |
8736cc2d | 318 | ret = memory_block_offline(mem); |
3d3af6af IC |
319 | break; |
320 | default: | |
321 | WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: " | |
8736cc2d | 322 | "%ld\n", __func__, mem->start_section_nr, action, action); |
3d3af6af | 323 | ret = -EINVAL; |
3947be19 | 324 | } |
3947be19 DH |
325 | |
326 | return ret; | |
327 | } | |
328 | ||
dc18d706 | 329 | static int memory_block_change_state(struct memory_block *mem, |
fa2be40f | 330 | unsigned long to_state, unsigned long from_state_req) |
3947be19 | 331 | { |
de0ed36a | 332 | int ret = 0; |
0c2c99b1 | 333 | |
4960e05e RW |
334 | if (mem->state != from_state_req) |
335 | return -EINVAL; | |
3947be19 | 336 | |
0c2c99b1 NF |
337 | if (to_state == MEM_OFFLINE) |
338 | mem->state = MEM_GOING_OFFLINE; | |
339 | ||
8736cc2d | 340 | ret = memory_block_action(mem, to_state); |
b2c064b2 | 341 | mem->state = ret ? from_state_req : to_state; |
fa2be40f | 342 | |
4960e05e RW |
343 | return ret; |
344 | } | |
0c2c99b1 | 345 | |
fa2be40f | 346 | /* The device lock serializes operations on memory_subsys_[online|offline] */ |
4960e05e RW |
347 | static int memory_subsys_online(struct device *dev) |
348 | { | |
7315f0cc | 349 | struct memory_block *mem = to_memory_block(dev); |
4960e05e | 350 | int ret; |
3947be19 | 351 | |
fa2be40f SJ |
352 | if (mem->state == MEM_ONLINE) |
353 | return 0; | |
4960e05e | 354 | |
fa2be40f | 355 | /* |
efc978ad DH |
356 | * When called via device_online() without configuring the online_type, |
357 | * we want to default to MMOP_ONLINE. | |
fa2be40f | 358 | */ |
efc978ad | 359 | if (mem->online_type == MMOP_OFFLINE) |
956f8b44 | 360 | mem->online_type = MMOP_ONLINE; |
4960e05e | 361 | |
fa2be40f | 362 | ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE); |
efc978ad | 363 | mem->online_type = MMOP_OFFLINE; |
4960e05e | 364 | |
4960e05e RW |
365 | return ret; |
366 | } | |
367 | ||
4960e05e | 368 | static int memory_subsys_offline(struct device *dev) |
e90bdb7f | 369 | { |
7315f0cc | 370 | struct memory_block *mem = to_memory_block(dev); |
e90bdb7f | 371 | |
fa2be40f SJ |
372 | if (mem->state == MEM_OFFLINE) |
373 | return 0; | |
e90bdb7f | 374 | |
fa2be40f | 375 | return memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE); |
e90bdb7f | 376 | } |
4960e05e | 377 | |
3f8e9178 DH |
378 | static ssize_t state_store(struct device *dev, struct device_attribute *attr, |
379 | const char *buf, size_t count) | |
3947be19 | 380 | { |
1adf8b46 | 381 | const int online_type = mhp_online_type_from_str(buf); |
7315f0cc | 382 | struct memory_block *mem = to_memory_block(dev); |
4dc8207b DH |
383 | int ret; |
384 | ||
385 | if (online_type < 0) | |
386 | return -EINVAL; | |
3947be19 | 387 | |
5e33bc41 RW |
388 | ret = lock_device_hotplug_sysfs(); |
389 | if (ret) | |
390 | return ret; | |
4960e05e | 391 | |
fa2be40f | 392 | switch (online_type) { |
4f7c6b49 TC |
393 | case MMOP_ONLINE_KERNEL: |
394 | case MMOP_ONLINE_MOVABLE: | |
956f8b44 | 395 | case MMOP_ONLINE: |
381eab4a | 396 | /* mem->online_type is protected by device_hotplug_lock */ |
fa2be40f SJ |
397 | mem->online_type = online_type; |
398 | ret = device_online(&mem->dev); | |
399 | break; | |
4f7c6b49 | 400 | case MMOP_OFFLINE: |
fa2be40f SJ |
401 | ret = device_offline(&mem->dev); |
402 | break; | |
403 | default: | |
404 | ret = -EINVAL; /* should never happen */ | |
4960e05e | 405 | } |
4960e05e RW |
406 | |
407 | unlock_device_hotplug(); | |
0c2c99b1 | 408 | |
d66ba15b | 409 | if (ret < 0) |
3947be19 | 410 | return ret; |
d66ba15b RA |
411 | if (ret) |
412 | return -EINVAL; | |
413 | ||
3947be19 DH |
414 | return count; |
415 | } | |
416 | ||
417 | /* | |
e9a2e48e DH |
418 | * Legacy interface that we cannot remove: s390x exposes the storage increment |
419 | * covered by a memory block, allowing for identifying which memory blocks | |
420 | * comprise a storage increment. Since a memory block spans complete | |
421 | * storage increments nowadays, this interface is basically unused. Other | |
422 | * archs never exposed != 0. | |
3947be19 | 423 | */ |
3f8e9178 | 424 | static ssize_t phys_device_show(struct device *dev, |
10fbcf4c | 425 | struct device_attribute *attr, char *buf) |
3947be19 | 426 | { |
7315f0cc | 427 | struct memory_block *mem = to_memory_block(dev); |
e9a2e48e | 428 | unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); |
948b3edb | 429 | |
e9a2e48e DH |
430 | return sysfs_emit(buf, "%d\n", |
431 | arch_get_memory_phys_device(start_pfn)); | |
3947be19 DH |
432 | } |
433 | ||
ed2f2400 | 434 | #ifdef CONFIG_MEMORY_HOTREMOVE |
973c3911 | 435 | static int print_allowed_zone(char *buf, int len, int nid, |
445fcf7c | 436 | struct memory_group *group, |
973c3911 JP |
437 | unsigned long start_pfn, unsigned long nr_pages, |
438 | int online_type, struct zone *default_zone) | |
e5e68930 MH |
439 | { |
440 | struct zone *zone; | |
441 | ||
445fcf7c | 442 | zone = zone_for_pfn_range(online_type, nid, group, start_pfn, nr_pages); |
973c3911 JP |
443 | if (zone == default_zone) |
444 | return 0; | |
948b3edb | 445 | |
973c3911 | 446 | return sysfs_emit_at(buf, len, " %s", zone->name); |
e5e68930 MH |
447 | } |
448 | ||
3f8e9178 | 449 | static ssize_t valid_zones_show(struct device *dev, |
ed2f2400 ZZ |
450 | struct device_attribute *attr, char *buf) |
451 | { | |
452 | struct memory_block *mem = to_memory_block(dev); | |
f1dd2cd1 | 453 | unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); |
ed2f2400 | 454 | unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; |
445fcf7c | 455 | struct memory_group *group = mem->group; |
e5e68930 | 456 | struct zone *default_zone; |
445fcf7c | 457 | int nid = mem->nid; |
973c3911 | 458 | int len = 0; |
ed2f2400 | 459 | |
f1dd2cd1 MH |
460 | /* |
461 | * Check the existing zone. Make sure that we do that only on the | |
462 | * online nodes otherwise the page_zone is not reliable | |
463 | */ | |
464 | if (mem->state == MEM_ONLINE) { | |
4e8346d0 | 465 | /* |
395f6081 DH |
466 | * If !mem->zone, the memory block spans multiple zones and |
467 | * cannot get offlined. | |
4e8346d0 | 468 | */ |
395f6081 | 469 | default_zone = mem->zone; |
92917998 | 470 | if (!default_zone) |
973c3911 JP |
471 | return sysfs_emit(buf, "%s\n", "none"); |
472 | len += sysfs_emit_at(buf, len, "%s", default_zone->name); | |
f1dd2cd1 | 473 | goto out; |
ed2f2400 ZZ |
474 | } |
475 | ||
445fcf7c DH |
476 | default_zone = zone_for_pfn_range(MMOP_ONLINE, nid, group, |
477 | start_pfn, nr_pages); | |
ed2f2400 | 478 | |
973c3911 | 479 | len += sysfs_emit_at(buf, len, "%s", default_zone->name); |
445fcf7c | 480 | len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages, |
973c3911 | 481 | MMOP_ONLINE_KERNEL, default_zone); |
445fcf7c | 482 | len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages, |
973c3911 | 483 | MMOP_ONLINE_MOVABLE, default_zone); |
f1dd2cd1 | 484 | out: |
948b3edb | 485 | len += sysfs_emit_at(buf, len, "\n"); |
973c3911 | 486 | return len; |
ed2f2400 | 487 | } |
3f8e9178 | 488 | static DEVICE_ATTR_RO(valid_zones); |
ed2f2400 ZZ |
489 | #endif |
490 | ||
3f8e9178 DH |
491 | static DEVICE_ATTR_RO(phys_index); |
492 | static DEVICE_ATTR_RW(state); | |
493 | static DEVICE_ATTR_RO(phys_device); | |
494 | static DEVICE_ATTR_RO(removable); | |
3947be19 | 495 | |
3947be19 | 496 | /* |
f915fb7f | 497 | * Show the memory block size (shared by all memory blocks). |
3947be19 | 498 | */ |
3f8e9178 DH |
499 | static ssize_t block_size_bytes_show(struct device *dev, |
500 | struct device_attribute *attr, char *buf) | |
3947be19 | 501 | { |
aa838896 | 502 | return sysfs_emit(buf, "%lx\n", memory_block_size_bytes()); |
3947be19 DH |
503 | } |
504 | ||
3f8e9178 | 505 | static DEVICE_ATTR_RO(block_size_bytes); |
3947be19 | 506 | |
31bc3858 VK |
507 | /* |
508 | * Memory auto online policy. | |
509 | */ | |
510 | ||
3f8e9178 DH |
511 | static ssize_t auto_online_blocks_show(struct device *dev, |
512 | struct device_attribute *attr, char *buf) | |
31bc3858 | 513 | { |
aa838896 | 514 | return sysfs_emit(buf, "%s\n", |
1adf8b46 | 515 | online_type_to_str[mhp_default_online_type]); |
31bc3858 VK |
516 | } |
517 | ||
3f8e9178 DH |
518 | static ssize_t auto_online_blocks_store(struct device *dev, |
519 | struct device_attribute *attr, | |
520 | const char *buf, size_t count) | |
31bc3858 | 521 | { |
1adf8b46 | 522 | const int online_type = mhp_online_type_from_str(buf); |
5f47adf7 DH |
523 | |
524 | if (online_type < 0) | |
31bc3858 VK |
525 | return -EINVAL; |
526 | ||
1adf8b46 | 527 | mhp_default_online_type = online_type; |
31bc3858 VK |
528 | return count; |
529 | } | |
530 | ||
3f8e9178 | 531 | static DEVICE_ATTR_RW(auto_online_blocks); |
31bc3858 | 532 | |
88a6f899 ED |
533 | #ifdef CONFIG_CRASH_HOTPLUG |
534 | #include <linux/kexec.h> | |
535 | static ssize_t crash_hotplug_show(struct device *dev, | |
536 | struct device_attribute *attr, char *buf) | |
537 | { | |
79365026 | 538 | return sysfs_emit(buf, "%d\n", crash_check_hotplug_support()); |
88a6f899 ED |
539 | } |
540 | static DEVICE_ATTR_RO(crash_hotplug); | |
541 | #endif | |
542 | ||
3947be19 DH |
543 | /* |
544 | * Some architectures will have custom drivers to do this, and | |
545 | * will not need to do it from userspace. The fake hot-add code | |
546 | * as well as ppc64 will do all of their discovery in userspace | |
547 | * and will require this interface. | |
548 | */ | |
549 | #ifdef CONFIG_ARCH_MEMORY_PROBE | |
3f8e9178 DH |
550 | static ssize_t probe_store(struct device *dev, struct device_attribute *attr, |
551 | const char *buf, size_t count) | |
3947be19 DH |
552 | { |
553 | u64 phys_addr; | |
cb5490a5 | 554 | int nid, ret; |
61b94fea | 555 | unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block; |
3947be19 | 556 | |
b69deb2b ZZ |
557 | ret = kstrtoull(buf, 0, &phys_addr); |
558 | if (ret) | |
559 | return ret; | |
3947be19 | 560 | |
61b94fea AB |
561 | if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1)) |
562 | return -EINVAL; | |
563 | ||
8df1d0e4 DH |
564 | ret = lock_device_hotplug_sysfs(); |
565 | if (ret) | |
37803841 | 566 | return ret; |
8df1d0e4 | 567 | |
cb5490a5 | 568 | nid = memory_add_physaddr_to_nid(phys_addr); |
8df1d0e4 | 569 | ret = __add_memory(nid, phys_addr, |
b6117199 DH |
570 | MIN_MEMORY_BLOCK_SIZE * sections_per_block, |
571 | MHP_NONE); | |
6add7cd6 | 572 | |
cb5490a5 JA |
573 | if (ret) |
574 | goto out; | |
3947be19 | 575 | |
9f0af69b NK |
576 | ret = count; |
577 | out: | |
8df1d0e4 | 578 | unlock_device_hotplug(); |
9f0af69b | 579 | return ret; |
3947be19 | 580 | } |
3947be19 | 581 | |
3f8e9178 | 582 | static DEVICE_ATTR_WO(probe); |
3947be19 DH |
583 | #endif |
584 | ||
facb6011 AK |
585 | #ifdef CONFIG_MEMORY_FAILURE |
586 | /* | |
587 | * Support for offlining pages of memory | |
588 | */ | |
589 | ||
590 | /* Soft offline a page */ | |
3f8e9178 DH |
591 | static ssize_t soft_offline_page_store(struct device *dev, |
592 | struct device_attribute *attr, | |
593 | const char *buf, size_t count) | |
facb6011 AK |
594 | { |
595 | int ret; | |
596 | u64 pfn; | |
597 | if (!capable(CAP_SYS_ADMIN)) | |
598 | return -EPERM; | |
34da5e67 | 599 | if (kstrtoull(buf, 0, &pfn) < 0) |
facb6011 AK |
600 | return -EINVAL; |
601 | pfn >>= PAGE_SHIFT; | |
feec24a6 | 602 | ret = soft_offline_page(pfn, 0); |
facb6011 AK |
603 | return ret == 0 ? count : ret; |
604 | } | |
605 | ||
606 | /* Forcibly offline a page, including killing processes. */ | |
3f8e9178 DH |
607 | static ssize_t hard_offline_page_store(struct device *dev, |
608 | struct device_attribute *attr, | |
609 | const char *buf, size_t count) | |
facb6011 AK |
610 | { |
611 | int ret; | |
612 | u64 pfn; | |
613 | if (!capable(CAP_SYS_ADMIN)) | |
614 | return -EPERM; | |
34da5e67 | 615 | if (kstrtoull(buf, 0, &pfn) < 0) |
facb6011 AK |
616 | return -EINVAL; |
617 | pfn >>= PAGE_SHIFT; | |
67f22ba7 | 618 | ret = memory_failure(pfn, MF_SW_SIMULATED); |
d1fe111f | 619 | if (ret == -EOPNOTSUPP) |
620 | ret = 0; | |
facb6011 AK |
621 | return ret ? ret : count; |
622 | } | |
623 | ||
3f8e9178 DH |
624 | static DEVICE_ATTR_WO(soft_offline_page); |
625 | static DEVICE_ATTR_WO(hard_offline_page); | |
facb6011 AK |
626 | #endif |
627 | ||
e9a2e48e | 628 | /* See phys_device_show(). */ |
bc32df00 HC |
629 | int __weak arch_get_memory_phys_device(unsigned long start_pfn) |
630 | { | |
631 | return 0; | |
632 | } | |
3947be19 | 633 | |
4fb6eabf SC |
634 | /* |
635 | * A reference for the returned memory block device is acquired. | |
636 | * | |
637 | * Called under device_hotplug_lock. | |
638 | */ | |
dd625285 | 639 | static struct memory_block *find_memory_block_by_id(unsigned long block_id) |
3947be19 | 640 | { |
4fb6eabf | 641 | struct memory_block *mem; |
3947be19 | 642 | |
4fb6eabf SC |
643 | mem = xa_load(&memory_blocks, block_id); |
644 | if (mem) | |
645 | get_device(&mem->dev); | |
646 | return mem; | |
db051a0d DH |
647 | } |
648 | ||
98383031 | 649 | /* |
4fb6eabf | 650 | * Called under device_hotplug_lock. |
98383031 | 651 | */ |
fc1f5e98 | 652 | struct memory_block *find_memory_block(unsigned long section_nr) |
98383031 | 653 | { |
fc1f5e98 | 654 | unsigned long block_id = memory_block_id(section_nr); |
dd625285 DH |
655 | |
656 | return find_memory_block_by_id(block_id); | |
98383031 RH |
657 | } |
658 | ||
96b2c0fc NF |
659 | static struct attribute *memory_memblk_attrs[] = { |
660 | &dev_attr_phys_index.attr, | |
96b2c0fc NF |
661 | &dev_attr_state.attr, |
662 | &dev_attr_phys_device.attr, | |
663 | &dev_attr_removable.attr, | |
ed2f2400 ZZ |
664 | #ifdef CONFIG_MEMORY_HOTREMOVE |
665 | &dev_attr_valid_zones.attr, | |
666 | #endif | |
96b2c0fc NF |
667 | NULL |
668 | }; | |
669 | ||
5a576764 | 670 | static const struct attribute_group memory_memblk_attr_group = { |
96b2c0fc NF |
671 | .attrs = memory_memblk_attrs, |
672 | }; | |
673 | ||
674 | static const struct attribute_group *memory_memblk_attr_groups[] = { | |
675 | &memory_memblk_attr_group, | |
676 | NULL, | |
677 | }; | |
678 | ||
2aa065f7 | 679 | static int __add_memory_block(struct memory_block *memory) |
96b2c0fc | 680 | { |
085aa2de AY |
681 | int ret; |
682 | ||
96b2c0fc NF |
683 | memory->dev.bus = &memory_subsys; |
684 | memory->dev.id = memory->start_section_nr / sections_per_block; | |
685 | memory->dev.release = memory_block_release; | |
686 | memory->dev.groups = memory_memblk_attr_groups; | |
f991fae5 | 687 | memory->dev.offline = memory->state == MEM_OFFLINE; |
96b2c0fc | 688 | |
085aa2de | 689 | ret = device_register(&memory->dev); |
4fb6eabf | 690 | if (ret) { |
085aa2de | 691 | put_device(&memory->dev); |
4fb6eabf SC |
692 | return ret; |
693 | } | |
694 | ret = xa_err(xa_store(&memory_blocks, memory->dev.id, memory, | |
695 | GFP_KERNEL)); | |
f47f758c | 696 | if (ret) |
4fb6eabf | 697 | device_unregister(&memory->dev); |
f47f758c | 698 | |
085aa2de | 699 | return ret; |
96b2c0fc NF |
700 | } |
701 | ||
395f6081 DH |
702 | static struct zone *early_node_zone_for_memory_block(struct memory_block *mem, |
703 | int nid) | |
704 | { | |
705 | const unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); | |
706 | const unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; | |
707 | struct zone *zone, *matching_zone = NULL; | |
708 | pg_data_t *pgdat = NODE_DATA(nid); | |
709 | int i; | |
710 | ||
711 | /* | |
712 | * This logic only works for early memory, when the applicable zones | |
713 | * already span the memory block. We don't expect overlapping zones on | |
714 | * a single node for early memory. So if we're told that some PFNs | |
715 | * of a node fall into this memory block, we can assume that all node | |
716 | * zones that intersect with the memory block are actually applicable. | |
717 | * No need to look at the memmap. | |
718 | */ | |
719 | for (i = 0; i < MAX_NR_ZONES; i++) { | |
720 | zone = pgdat->node_zones + i; | |
721 | if (!populated_zone(zone)) | |
722 | continue; | |
723 | if (!zone_intersects(zone, start_pfn, nr_pages)) | |
724 | continue; | |
725 | if (!matching_zone) { | |
726 | matching_zone = zone; | |
727 | continue; | |
728 | } | |
729 | /* Spans multiple zones ... */ | |
730 | matching_zone = NULL; | |
731 | break; | |
732 | } | |
733 | return matching_zone; | |
734 | } | |
735 | ||
736 | #ifdef CONFIG_NUMA | |
737 | /** | |
738 | * memory_block_add_nid() - Indicate that system RAM falling into this memory | |
739 | * block device (partially) belongs to the given node. | |
740 | * @mem: The memory block device. | |
741 | * @nid: The node id. | |
742 | * @context: The memory initialization context. | |
743 | * | |
744 | * Indicate that system RAM falling into this memory block (partially) belongs | |
745 | * to the given node. If the context indicates ("early") that we are adding the | |
746 | * node during node device subsystem initialization, this will also properly | |
747 | * set/adjust mem->zone based on the zone ranges of the given node. | |
748 | */ | |
749 | void memory_block_add_nid(struct memory_block *mem, int nid, | |
750 | enum meminit_context context) | |
751 | { | |
752 | if (context == MEMINIT_EARLY && mem->nid != nid) { | |
753 | /* | |
754 | * For early memory we have to determine the zone when setting | |
755 | * the node id and handle multiple nodes spanning a single | |
756 | * memory block by indicate via zone == NULL that we're not | |
757 | * dealing with a single zone. So if we're setting the node id | |
758 | * the first time, determine if there is a single zone. If we're | |
759 | * setting the node id a second time to a different node, | |
760 | * invalidate the single detected zone. | |
761 | */ | |
762 | if (mem->nid == NUMA_NO_NODE) | |
763 | mem->zone = early_node_zone_for_memory_block(mem, nid); | |
764 | else | |
765 | mem->zone = NULL; | |
766 | } | |
767 | ||
768 | /* | |
769 | * If this memory block spans multiple nodes, we only indicate | |
770 | * the last processed node. If we span multiple nodes (not applicable | |
771 | * to hotplugged memory), zone == NULL will prohibit memory offlining | |
772 | * and consequently unplug. | |
773 | */ | |
774 | mem->nid = nid; | |
775 | } | |
776 | #endif | |
777 | ||
2aa065f7 | 778 | static int add_memory_block(unsigned long block_id, unsigned long state, |
1a8c64e1 | 779 | struct vmem_altmap *altmap, |
2aa065f7 | 780 | struct memory_group *group) |
e4619c85 | 781 | { |
0c2c99b1 | 782 | struct memory_block *mem; |
e4619c85 NF |
783 | int ret = 0; |
784 | ||
dd625285 | 785 | mem = find_memory_block_by_id(block_id); |
db051a0d DH |
786 | if (mem) { |
787 | put_device(&mem->dev); | |
788 | return -EEXIST; | |
789 | } | |
0c2c99b1 | 790 | mem = kzalloc(sizeof(*mem), GFP_KERNEL); |
e4619c85 NF |
791 | if (!mem) |
792 | return -ENOMEM; | |
793 | ||
18115825 | 794 | mem->start_section_nr = block_id * sections_per_block; |
e4619c85 | 795 | mem->state = state; |
d84f2f5a | 796 | mem->nid = NUMA_NO_NODE; |
1a8c64e1 | 797 | mem->altmap = altmap; |
028fc57a DH |
798 | INIT_LIST_HEAD(&mem->group_next); |
799 | ||
395f6081 DH |
800 | #ifndef CONFIG_NUMA |
801 | if (state == MEM_ONLINE) | |
802 | /* | |
803 | * MEM_ONLINE at this point implies early memory. With NUMA, | |
804 | * we'll determine the zone when setting the node id via | |
805 | * memory_block_add_nid(). Memory hotplug updated the zone | |
806 | * manually when memory onlining/offlining succeeds. | |
807 | */ | |
808 | mem->zone = early_node_zone_for_memory_block(mem, NUMA_NO_NODE); | |
809 | #endif /* CONFIG_NUMA */ | |
810 | ||
2aa065f7 | 811 | ret = __add_memory_block(mem); |
7ea0d2d7 DH |
812 | if (ret) |
813 | return ret; | |
814 | ||
028fc57a DH |
815 | if (group) { |
816 | mem->group = group; | |
817 | list_add(&mem->group_next, &group->memory_blocks); | |
818 | } | |
e4619c85 | 819 | |
7ea0d2d7 | 820 | return 0; |
0c2c99b1 NF |
821 | } |
822 | ||
2aa065f7 | 823 | static int __init add_boot_memory_block(unsigned long base_section_nr) |
0c2c99b1 | 824 | { |
68c3a6ac | 825 | int section_count = 0; |
2491f0a2 | 826 | unsigned long nr; |
0c2c99b1 | 827 | |
2491f0a2 DH |
828 | for (nr = base_section_nr; nr < base_section_nr + sections_per_block; |
829 | nr++) | |
830 | if (present_section_nr(nr)) | |
18115825 | 831 | section_count++; |
e4619c85 | 832 | |
cb5e39b8 SJ |
833 | if (section_count == 0) |
834 | return 0; | |
2aa065f7 | 835 | return add_memory_block(memory_block_id(base_section_nr), |
1a8c64e1 | 836 | MEM_ONLINE, NULL, NULL); |
2aa065f7 DH |
837 | } |
838 | ||
839 | static int add_hotplug_memory_block(unsigned long block_id, | |
1a8c64e1 | 840 | struct vmem_altmap *altmap, |
2aa065f7 DH |
841 | struct memory_group *group) |
842 | { | |
1a8c64e1 | 843 | return add_memory_block(block_id, MEM_OFFLINE, altmap, group); |
e4619c85 NF |
844 | } |
845 | ||
2aa065f7 | 846 | static void remove_memory_block(struct memory_block *memory) |
db051a0d DH |
847 | { |
848 | if (WARN_ON_ONCE(memory->dev.bus != &memory_subsys)) | |
849 | return; | |
850 | ||
4fb6eabf SC |
851 | WARN_ON(xa_erase(&memory_blocks, memory->dev.id) == NULL); |
852 | ||
028fc57a DH |
853 | if (memory->group) { |
854 | list_del(&memory->group_next); | |
855 | memory->group = NULL; | |
856 | } | |
857 | ||
db051a0d DH |
858 | /* drop the ref. we got via find_memory_block() */ |
859 | put_device(&memory->dev); | |
860 | device_unregister(&memory->dev); | |
861 | } | |
862 | ||
4edd7cef | 863 | /* |
db051a0d DH |
864 | * Create memory block devices for the given memory area. Start and size |
865 | * have to be aligned to memory block granularity. Memory block devices | |
866 | * will be initialized as offline. | |
848e19ad DH |
867 | * |
868 | * Called under device_hotplug_lock. | |
4edd7cef | 869 | */ |
a08a2ae3 | 870 | int create_memory_block_devices(unsigned long start, unsigned long size, |
1a8c64e1 | 871 | struct vmem_altmap *altmap, |
028fc57a | 872 | struct memory_group *group) |
4edd7cef | 873 | { |
90ec010f DH |
874 | const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start)); |
875 | unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size)); | |
d7f80530 | 876 | struct memory_block *mem; |
db051a0d DH |
877 | unsigned long block_id; |
878 | int ret = 0; | |
b1eaef3d | 879 | |
db051a0d DH |
880 | if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) || |
881 | !IS_ALIGNED(size, memory_block_size_bytes()))) | |
882 | return -EINVAL; | |
b1eaef3d | 883 | |
db051a0d | 884 | for (block_id = start_block_id; block_id != end_block_id; block_id++) { |
1a8c64e1 | 885 | ret = add_hotplug_memory_block(block_id, altmap, group); |
d7f80530 | 886 | if (ret) |
db051a0d | 887 | break; |
db051a0d DH |
888 | } |
889 | if (ret) { | |
890 | end_block_id = block_id; | |
891 | for (block_id = start_block_id; block_id != end_block_id; | |
892 | block_id++) { | |
dd625285 | 893 | mem = find_memory_block_by_id(block_id); |
848e19ad DH |
894 | if (WARN_ON_ONCE(!mem)) |
895 | continue; | |
2aa065f7 | 896 | remove_memory_block(mem); |
db051a0d | 897 | } |
d7f80530 | 898 | } |
b1eaef3d | 899 | return ret; |
4edd7cef DR |
900 | } |
901 | ||
4c4b7f9b DH |
902 | /* |
903 | * Remove memory block devices for the given memory area. Start and size | |
904 | * have to be aligned to memory block granularity. Memory block devices | |
905 | * have to be offline. | |
848e19ad DH |
906 | * |
907 | * Called under device_hotplug_lock. | |
4c4b7f9b DH |
908 | */ |
909 | void remove_memory_block_devices(unsigned long start, unsigned long size) | |
3947be19 | 910 | { |
90ec010f DH |
911 | const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start)); |
912 | const unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size)); | |
3947be19 | 913 | struct memory_block *mem; |
90ec010f | 914 | unsigned long block_id; |
3947be19 | 915 | |
4c4b7f9b DH |
916 | if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) || |
917 | !IS_ALIGNED(size, memory_block_size_bytes()))) | |
cb7b3a36 DH |
918 | return; |
919 | ||
4c4b7f9b | 920 | for (block_id = start_block_id; block_id != end_block_id; block_id++) { |
dd625285 | 921 | mem = find_memory_block_by_id(block_id); |
4c4b7f9b DH |
922 | if (WARN_ON_ONCE(!mem)) |
923 | continue; | |
5033091d | 924 | num_poisoned_pages_sub(-1UL, memblk_nr_poison(mem)); |
4c4b7f9b | 925 | unregister_memory_block_under_nodes(mem); |
2aa065f7 | 926 | remove_memory_block(mem); |
4c4b7f9b | 927 | } |
3947be19 DH |
928 | } |
929 | ||
96b2c0fc NF |
930 | static struct attribute *memory_root_attrs[] = { |
931 | #ifdef CONFIG_ARCH_MEMORY_PROBE | |
932 | &dev_attr_probe.attr, | |
933 | #endif | |
934 | ||
935 | #ifdef CONFIG_MEMORY_FAILURE | |
936 | &dev_attr_soft_offline_page.attr, | |
937 | &dev_attr_hard_offline_page.attr, | |
938 | #endif | |
939 | ||
940 | &dev_attr_block_size_bytes.attr, | |
31bc3858 | 941 | &dev_attr_auto_online_blocks.attr, |
88a6f899 ED |
942 | #ifdef CONFIG_CRASH_HOTPLUG |
943 | &dev_attr_crash_hotplug.attr, | |
944 | #endif | |
96b2c0fc NF |
945 | NULL |
946 | }; | |
947 | ||
5a576764 | 948 | static const struct attribute_group memory_root_attr_group = { |
96b2c0fc NF |
949 | .attrs = memory_root_attrs, |
950 | }; | |
951 | ||
952 | static const struct attribute_group *memory_root_attr_groups[] = { | |
953 | &memory_root_attr_group, | |
954 | NULL, | |
955 | }; | |
956 | ||
3947be19 | 957 | /* |
848e19ad DH |
958 | * Initialize the sysfs support for memory devices. At the time this function |
959 | * is called, we cannot have concurrent creation/deletion of memory block | |
960 | * devices, the device_hotplug_lock is not needed. | |
3947be19 | 961 | */ |
902ce63b | 962 | void __init memory_dev_init(void) |
3947be19 | 963 | { |
3947be19 | 964 | int ret; |
2491f0a2 | 965 | unsigned long block_sz, nr; |
3947be19 | 966 | |
902ce63b DH |
967 | /* Validate the configured memory block size */ |
968 | block_sz = memory_block_size_bytes(); | |
969 | if (!is_power_of_2(block_sz) || block_sz < MIN_MEMORY_BLOCK_SIZE) | |
970 | panic("Memory block size not suitable: 0x%lx\n", block_sz); | |
971 | sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE; | |
972 | ||
96b2c0fc | 973 | ret = subsys_system_register(&memory_subsys, memory_root_attr_groups); |
28ec24e2 | 974 | if (ret) |
848e19ad | 975 | panic("%s() failed to register subsystem: %d\n", __func__, ret); |
3947be19 DH |
976 | |
977 | /* | |
978 | * Create entries for memory sections that were found | |
979 | * during boot and have been initialized | |
980 | */ | |
2491f0a2 DH |
981 | for (nr = 0; nr <= __highest_present_section_nr; |
982 | nr += sections_per_block) { | |
2aa065f7 | 983 | ret = add_boot_memory_block(nr); |
848e19ad DH |
984 | if (ret) |
985 | panic("%s() failed to add memory block: %d\n", __func__, | |
986 | ret); | |
3947be19 | 987 | } |
3947be19 | 988 | } |
ea884641 DH |
989 | |
990 | /** | |
991 | * walk_memory_blocks - walk through all present memory blocks overlapped | |
992 | * by the range [start, start + size) | |
993 | * | |
994 | * @start: start address of the memory range | |
995 | * @size: size of the memory range | |
996 | * @arg: argument passed to func | |
997 | * @func: callback for each memory section walked | |
998 | * | |
999 | * This function walks through all present memory blocks overlapped by the | |
1000 | * range [start, start + size), calling func on each memory block. | |
1001 | * | |
1002 | * In case func() returns an error, walking is aborted and the error is | |
1003 | * returned. | |
4fb6eabf SC |
1004 | * |
1005 | * Called under device_hotplug_lock. | |
ea884641 DH |
1006 | */ |
1007 | int walk_memory_blocks(unsigned long start, unsigned long size, | |
1008 | void *arg, walk_memory_blocks_func_t func) | |
1009 | { | |
1010 | const unsigned long start_block_id = phys_to_block_id(start); | |
1011 | const unsigned long end_block_id = phys_to_block_id(start + size - 1); | |
1012 | struct memory_block *mem; | |
1013 | unsigned long block_id; | |
1014 | int ret = 0; | |
1015 | ||
dd625285 DH |
1016 | if (!size) |
1017 | return 0; | |
1018 | ||
ea884641 | 1019 | for (block_id = start_block_id; block_id <= end_block_id; block_id++) { |
dd625285 | 1020 | mem = find_memory_block_by_id(block_id); |
ea884641 DH |
1021 | if (!mem) |
1022 | continue; | |
1023 | ||
1024 | ret = func(mem, arg); | |
1025 | put_device(&mem->dev); | |
1026 | if (ret) | |
1027 | break; | |
1028 | } | |
1029 | return ret; | |
1030 | } | |
2c91f8fc DH |
1031 | |
1032 | struct for_each_memory_block_cb_data { | |
1033 | walk_memory_blocks_func_t func; | |
1034 | void *arg; | |
1035 | }; | |
1036 | ||
1037 | static int for_each_memory_block_cb(struct device *dev, void *data) | |
1038 | { | |
1039 | struct memory_block *mem = to_memory_block(dev); | |
1040 | struct for_each_memory_block_cb_data *cb_data = data; | |
1041 | ||
1042 | return cb_data->func(mem, cb_data->arg); | |
1043 | } | |
1044 | ||
1045 | /** | |
1046 | * for_each_memory_block - walk through all present memory blocks | |
1047 | * | |
1048 | * @arg: argument passed to func | |
1049 | * @func: callback for each memory block walked | |
1050 | * | |
1051 | * This function walks through all present memory blocks, calling func on | |
1052 | * each memory block. | |
1053 | * | |
1054 | * In case func() returns an error, walking is aborted and the error is | |
1055 | * returned. | |
1056 | */ | |
1057 | int for_each_memory_block(void *arg, walk_memory_blocks_func_t func) | |
1058 | { | |
1059 | struct for_each_memory_block_cb_data cb_data = { | |
1060 | .func = func, | |
1061 | .arg = arg, | |
1062 | }; | |
1063 | ||
1064 | return bus_for_each_dev(&memory_subsys, NULL, &cb_data, | |
1065 | for_each_memory_block_cb); | |
1066 | } | |
028fc57a DH |
1067 | |
1068 | /* | |
1069 | * This is an internal helper to unify allocation and initialization of | |
1070 | * memory groups. Note that the passed memory group will be copied to a | |
1071 | * dynamically allocated memory group. After this call, the passed | |
1072 | * memory group should no longer be used. | |
1073 | */ | |
1074 | static int memory_group_register(struct memory_group group) | |
1075 | { | |
1076 | struct memory_group *new_group; | |
1077 | uint32_t mgid; | |
1078 | int ret; | |
1079 | ||
1080 | if (!node_possible(group.nid)) | |
1081 | return -EINVAL; | |
1082 | ||
1083 | new_group = kzalloc(sizeof(group), GFP_KERNEL); | |
1084 | if (!new_group) | |
1085 | return -ENOMEM; | |
1086 | *new_group = group; | |
1087 | INIT_LIST_HEAD(&new_group->memory_blocks); | |
1088 | ||
1089 | ret = xa_alloc(&memory_groups, &mgid, new_group, xa_limit_31b, | |
1090 | GFP_KERNEL); | |
1091 | if (ret) { | |
1092 | kfree(new_group); | |
1093 | return ret; | |
3fcebf90 DH |
1094 | } else if (group.is_dynamic) { |
1095 | xa_set_mark(&memory_groups, mgid, MEMORY_GROUP_MARK_DYNAMIC); | |
028fc57a DH |
1096 | } |
1097 | return mgid; | |
1098 | } | |
1099 | ||
1100 | /** | |
1101 | * memory_group_register_static() - Register a static memory group. | |
1102 | * @nid: The node id. | |
1103 | * @max_pages: The maximum number of pages we'll have in this static memory | |
1104 | * group. | |
1105 | * | |
1106 | * Register a new static memory group and return the memory group id. | |
1107 | * All memory in the group belongs to a single unit, such as a DIMM. All | |
1108 | * memory belonging to a static memory group is added in one go to be removed | |
1109 | * in one go -- it's static. | |
1110 | * | |
1111 | * Returns an error if out of memory, if the node id is invalid, if no new | |
1112 | * memory groups can be registered, or if max_pages is invalid (0). Otherwise, | |
1113 | * returns the new memory group id. | |
1114 | */ | |
1115 | int memory_group_register_static(int nid, unsigned long max_pages) | |
1116 | { | |
1117 | struct memory_group group = { | |
1118 | .nid = nid, | |
1119 | .s = { | |
1120 | .max_pages = max_pages, | |
1121 | }, | |
1122 | }; | |
1123 | ||
1124 | if (!max_pages) | |
1125 | return -EINVAL; | |
1126 | return memory_group_register(group); | |
1127 | } | |
1128 | EXPORT_SYMBOL_GPL(memory_group_register_static); | |
1129 | ||
1130 | /** | |
1131 | * memory_group_register_dynamic() - Register a dynamic memory group. | |
1132 | * @nid: The node id. | |
1133 | * @unit_pages: Unit in pages in which is memory added/removed in this dynamic | |
1134 | * memory group. | |
1135 | * | |
1136 | * Register a new dynamic memory group and return the memory group id. | |
1137 | * Memory within a dynamic memory group is added/removed dynamically | |
1138 | * in unit_pages. | |
1139 | * | |
1140 | * Returns an error if out of memory, if the node id is invalid, if no new | |
1141 | * memory groups can be registered, or if unit_pages is invalid (0, not a | |
1142 | * power of two, smaller than a single memory block). Otherwise, returns the | |
1143 | * new memory group id. | |
1144 | */ | |
1145 | int memory_group_register_dynamic(int nid, unsigned long unit_pages) | |
1146 | { | |
1147 | struct memory_group group = { | |
1148 | .nid = nid, | |
1149 | .is_dynamic = true, | |
1150 | .d = { | |
1151 | .unit_pages = unit_pages, | |
1152 | }, | |
1153 | }; | |
1154 | ||
1155 | if (!unit_pages || !is_power_of_2(unit_pages) || | |
1156 | unit_pages < PHYS_PFN(memory_block_size_bytes())) | |
1157 | return -EINVAL; | |
1158 | return memory_group_register(group); | |
1159 | } | |
1160 | EXPORT_SYMBOL_GPL(memory_group_register_dynamic); | |
1161 | ||
1162 | /** | |
1163 | * memory_group_unregister() - Unregister a memory group. | |
1164 | * @mgid: the memory group id | |
1165 | * | |
1166 | * Unregister a memory group. If any memory block still belongs to this | |
1167 | * memory group, unregistering will fail. | |
1168 | * | |
1169 | * Returns -EINVAL if the memory group id is invalid, returns -EBUSY if some | |
1170 | * memory blocks still belong to this memory group and returns 0 if | |
1171 | * unregistering succeeded. | |
1172 | */ | |
1173 | int memory_group_unregister(int mgid) | |
1174 | { | |
1175 | struct memory_group *group; | |
1176 | ||
1177 | if (mgid < 0) | |
1178 | return -EINVAL; | |
1179 | ||
1180 | group = xa_load(&memory_groups, mgid); | |
1181 | if (!group) | |
1182 | return -EINVAL; | |
1183 | if (!list_empty(&group->memory_blocks)) | |
1184 | return -EBUSY; | |
1185 | xa_erase(&memory_groups, mgid); | |
1186 | kfree(group); | |
1187 | return 0; | |
1188 | } | |
1189 | EXPORT_SYMBOL_GPL(memory_group_unregister); | |
1190 | ||
1191 | /* | |
1192 | * This is an internal helper only to be used in core memory hotplug code to | |
1193 | * lookup a memory group. We don't care about locking, as we don't expect a | |
1194 | * memory group to get unregistered while adding memory to it -- because | |
1195 | * the group and the memory is managed by the same driver. | |
1196 | */ | |
1197 | struct memory_group *memory_group_find_by_id(int mgid) | |
1198 | { | |
1199 | return xa_load(&memory_groups, mgid); | |
1200 | } | |
3fcebf90 DH |
1201 | |
1202 | /* | |
1203 | * This is an internal helper only to be used in core memory hotplug code to | |
1204 | * walk all dynamic memory groups excluding a given memory group, either | |
1205 | * belonging to a specific node, or belonging to any node. | |
1206 | */ | |
1207 | int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func, | |
1208 | struct memory_group *excluded, void *arg) | |
1209 | { | |
1210 | struct memory_group *group; | |
1211 | unsigned long index; | |
1212 | int ret = 0; | |
1213 | ||
1214 | xa_for_each_marked(&memory_groups, index, group, | |
1215 | MEMORY_GROUP_MARK_DYNAMIC) { | |
1216 | if (group == excluded) | |
1217 | continue; | |
1218 | #ifdef CONFIG_NUMA | |
1219 | if (nid != NUMA_NO_NODE && group->nid != nid) | |
1220 | continue; | |
1221 | #endif /* CONFIG_NUMA */ | |
1222 | ret = func(group, arg); | |
1223 | if (ret) | |
1224 | break; | |
1225 | } | |
1226 | return ret; | |
1227 | } | |
5033091d NH |
1228 | |
1229 | #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) | |
1230 | void memblk_nr_poison_inc(unsigned long pfn) | |
1231 | { | |
1232 | const unsigned long block_id = pfn_to_block_id(pfn); | |
1233 | struct memory_block *mem = find_memory_block_by_id(block_id); | |
1234 | ||
1235 | if (mem) | |
1236 | atomic_long_inc(&mem->nr_hwpoison); | |
1237 | } | |
1238 | ||
1239 | void memblk_nr_poison_sub(unsigned long pfn, long i) | |
1240 | { | |
1241 | const unsigned long block_id = pfn_to_block_id(pfn); | |
1242 | struct memory_block *mem = find_memory_block_by_id(block_id); | |
1243 | ||
1244 | if (mem) | |
1245 | atomic_long_sub(i, &mem->nr_hwpoison); | |
1246 | } | |
1247 | ||
1248 | static unsigned long memblk_nr_poison(struct memory_block *mem) | |
1249 | { | |
1250 | return atomic_long_read(&mem->nr_hwpoison); | |
1251 | } | |
1252 | #endif |