Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
84af458b BK |
2 | /* |
3 | * Collaborative memory management interface. | |
4 | * | |
5 | * Copyright (C) 2008 IBM Corporation | |
6 | * Author(s): Brian King (brking@linux.vnet.ibm.com), | |
84af458b BK |
7 | */ |
8 | ||
9 | #include <linux/ctype.h> | |
10 | #include <linux/delay.h> | |
11 | #include <linux/errno.h> | |
12 | #include <linux/fs.h> | |
5a0e3ad6 | 13 | #include <linux/gfp.h> |
84af458b BK |
14 | #include <linux/kthread.h> |
15 | #include <linux/module.h> | |
16 | #include <linux/oom.h> | |
fecba962 | 17 | #include <linux/reboot.h> |
84af458b BK |
18 | #include <linux/sched.h> |
19 | #include <linux/stringify.h> | |
20 | #include <linux/swap.h> | |
6c9d2909 | 21 | #include <linux/device.h> |
fe030c9b | 22 | #include <linux/balloon_compaction.h> |
84af458b BK |
23 | #include <asm/firmware.h> |
24 | #include <asm/hvcall.h> | |
25 | #include <asm/mmu.h> | |
7c0f6ba6 | 26 | #include <linux/uaccess.h> |
14b8a76b | 27 | #include <linux/memory.h> |
212bebb4 | 28 | #include <asm/plpar_wrappers.h> |
84af458b | 29 | |
8f272a5d ME |
30 | #include "pseries.h" |
31 | ||
84af458b BK |
32 | #define CMM_DRIVER_VERSION "1.0.0" |
33 | #define CMM_DEFAULT_DELAY 1 | |
14b8a76b | 34 | #define CMM_HOTPLUG_DELAY 5 |
84af458b BK |
35 | #define CMM_DEBUG 0 |
36 | #define CMM_DISABLE 0 | |
37 | #define CMM_OOM_KB 1024 | |
38 | #define CMM_MIN_MEM_MB 256 | |
39 | #define KB2PAGES(_p) ((_p)>>(PAGE_SHIFT-10)) | |
40 | #define PAGES2KB(_p) ((_p)<<(PAGE_SHIFT-10)) | |
7659f5d6 | 41 | |
14b8a76b | 42 | #define CMM_MEM_HOTPLUG_PRI 1 |
84af458b BK |
43 | |
44 | static unsigned int delay = CMM_DEFAULT_DELAY; | |
14b8a76b | 45 | static unsigned int hotplug_delay = CMM_HOTPLUG_DELAY; |
84af458b BK |
46 | static unsigned int oom_kb = CMM_OOM_KB; |
47 | static unsigned int cmm_debug = CMM_DEBUG; | |
48 | static unsigned int cmm_disabled = CMM_DISABLE; | |
49 | static unsigned long min_mem_mb = CMM_MIN_MEM_MB; | |
b1713975 DH |
50 | static bool __read_mostly simulate; |
51 | static unsigned long simulate_loan_target_kb; | |
6c9d2909 | 52 | static struct device cmm_dev; |
84af458b BK |
53 | |
54 | MODULE_AUTHOR("Brian King <brking@linux.vnet.ibm.com>"); | |
55 | MODULE_DESCRIPTION("IBM System p Collaborative Memory Manager"); | |
56 | MODULE_LICENSE("GPL"); | |
57 | MODULE_VERSION(CMM_DRIVER_VERSION); | |
58 | ||
57ad583f | 59 | module_param_named(delay, delay, uint, 0644); |
84af458b BK |
60 | MODULE_PARM_DESC(delay, "Delay (in seconds) between polls to query hypervisor paging requests. " |
61 | "[Default=" __stringify(CMM_DEFAULT_DELAY) "]"); | |
57ad583f | 62 | module_param_named(hotplug_delay, hotplug_delay, uint, 0644); |
b0b5a765 | 63 | MODULE_PARM_DESC(hotplug_delay, "Delay (in seconds) after memory hotplug remove " |
14b8a76b RJ |
64 | "before loaning resumes. " |
65 | "[Default=" __stringify(CMM_HOTPLUG_DELAY) "]"); | |
57ad583f | 66 | module_param_named(oom_kb, oom_kb, uint, 0644); |
84af458b BK |
67 | MODULE_PARM_DESC(oom_kb, "Amount of memory in kb to free on OOM. " |
68 | "[Default=" __stringify(CMM_OOM_KB) "]"); | |
57ad583f | 69 | module_param_named(min_mem_mb, min_mem_mb, ulong, 0644); |
84af458b BK |
70 | MODULE_PARM_DESC(min_mem_mb, "Minimum amount of memory (in MB) to not balloon. " |
71 | "[Default=" __stringify(CMM_MIN_MEM_MB) "]"); | |
57ad583f | 72 | module_param_named(debug, cmm_debug, uint, 0644); |
84af458b BK |
73 | MODULE_PARM_DESC(debug, "Enable module debugging logging. Set to 1 to enable. " |
74 | "[Default=" __stringify(CMM_DEBUG) "]"); | |
b1713975 DH |
75 | module_param_named(simulate, simulate, bool, 0444); |
76 | MODULE_PARM_DESC(simulate, "Enable simulation mode (no communication with hw)."); | |
84af458b | 77 | |
84af458b BK |
78 | #define cmm_dbg(...) if (cmm_debug) { printk(KERN_INFO "cmm: "__VA_ARGS__); } |
79 | ||
1ef2f06b | 80 | static atomic_long_t loaned_pages; |
84af458b BK |
81 | static unsigned long loaned_pages_target; |
82 | static unsigned long oom_freed_pages; | |
83 | ||
14b8a76b RJ |
84 | static DEFINE_MUTEX(hotplug_mutex); |
85 | static int hotplug_occurred; /* protected by the hotplug mutex */ | |
86 | ||
84af458b | 87 | static struct task_struct *cmm_thread_ptr; |
fe030c9b | 88 | static struct balloon_dev_info b_dev_info; |
84af458b | 89 | |
4a1745c5 | 90 | static long plpar_page_set_loaned(struct page *page) |
8f272a5d | 91 | { |
4a1745c5 | 92 | const unsigned long vpa = page_to_phys(page); |
8f272a5d ME |
93 | unsigned long cmo_page_sz = cmo_get_page_size(); |
94 | long rc = 0; | |
95 | int i; | |
96 | ||
b1713975 DH |
97 | if (unlikely(simulate)) |
98 | return 0; | |
99 | ||
8f272a5d ME |
100 | for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz) |
101 | rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED, vpa + i, 0); | |
102 | ||
103 | for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz) | |
104 | plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE, | |
105 | vpa + i - cmo_page_sz, 0); | |
106 | ||
107 | return rc; | |
108 | } | |
109 | ||
4a1745c5 | 110 | static long plpar_page_set_active(struct page *page) |
8f272a5d | 111 | { |
4a1745c5 | 112 | const unsigned long vpa = page_to_phys(page); |
8f272a5d ME |
113 | unsigned long cmo_page_sz = cmo_get_page_size(); |
114 | long rc = 0; | |
115 | int i; | |
116 | ||
b1713975 DH |
117 | if (unlikely(simulate)) |
118 | return 0; | |
119 | ||
8f272a5d ME |
120 | for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz) |
121 | rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE, vpa + i, 0); | |
122 | ||
123 | for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz) | |
124 | plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED, | |
125 | vpa + i - cmo_page_sz, 0); | |
126 | ||
127 | return rc; | |
128 | } | |
129 | ||
84af458b BK |
130 | /** |
131 | * cmm_alloc_pages - Allocate pages and mark them as loaned | |
132 | * @nr: number of pages to allocate | |
133 | * | |
134 | * Return value: | |
135 | * number of pages requested to be allocated which were not | |
136 | **/ | |
137 | static long cmm_alloc_pages(long nr) | |
138 | { | |
4a1745c5 | 139 | struct page *page; |
84af458b BK |
140 | long rc; |
141 | ||
142 | cmm_dbg("Begin request for %ld pages\n", nr); | |
143 | ||
144 | while (nr) { | |
14b8a76b RJ |
145 | /* Exit if a hotplug operation is in progress or occurred */ |
146 | if (mutex_trylock(&hotplug_mutex)) { | |
147 | if (hotplug_occurred) { | |
148 | mutex_unlock(&hotplug_mutex); | |
149 | break; | |
150 | } | |
151 | mutex_unlock(&hotplug_mutex); | |
152 | } else { | |
153 | break; | |
154 | } | |
155 | ||
e8decafe | 156 | page = balloon_page_alloc(); |
4a1745c5 | 157 | if (!page) |
84af458b | 158 | break; |
4a1745c5 DH |
159 | rc = plpar_page_set_loaned(page); |
160 | if (rc) { | |
5df72bf3 | 161 | pr_err("%s: Can not set page to loaned. rc=%ld\n", __func__, rc); |
4a1745c5 | 162 | __free_page(page); |
84af458b BK |
163 | break; |
164 | } | |
165 | ||
fe030c9b | 166 | balloon_page_enqueue(&b_dev_info, page); |
1ef2f06b | 167 | atomic_long_inc(&loaned_pages); |
287b8977 | 168 | adjust_managed_page_count(page, -1); |
84af458b BK |
169 | nr--; |
170 | } | |
171 | ||
172 | cmm_dbg("End request with %ld pages unfulfilled\n", nr); | |
173 | return nr; | |
174 | } | |
175 | ||
176 | /** | |
177 | * cmm_free_pages - Free pages and mark them as active | |
178 | * @nr: number of pages to free | |
179 | * | |
180 | * Return value: | |
181 | * number of pages requested to be freed which were not | |
182 | **/ | |
183 | static long cmm_free_pages(long nr) | |
184 | { | |
fe030c9b | 185 | struct page *page; |
84af458b BK |
186 | |
187 | cmm_dbg("Begin free of %ld pages.\n", nr); | |
fe030c9b DH |
188 | while (nr) { |
189 | page = balloon_page_dequeue(&b_dev_info); | |
190 | if (!page) | |
84af458b | 191 | break; |
4a1745c5 | 192 | plpar_page_set_active(page); |
287b8977 | 193 | adjust_managed_page_count(page, 1); |
4a1745c5 | 194 | __free_page(page); |
1ef2f06b | 195 | atomic_long_dec(&loaned_pages); |
84af458b | 196 | nr--; |
84af458b | 197 | } |
84af458b BK |
198 | cmm_dbg("End request with %ld pages unfulfilled\n", nr); |
199 | return nr; | |
200 | } | |
201 | ||
202 | /** | |
203 | * cmm_oom_notify - OOM notifier | |
204 | * @self: notifier block struct | |
205 | * @dummy: not used | |
206 | * @parm: returned - number of pages freed | |
207 | * | |
208 | * Return value: | |
209 | * NOTIFY_OK | |
210 | **/ | |
211 | static int cmm_oom_notify(struct notifier_block *self, | |
212 | unsigned long dummy, void *parm) | |
213 | { | |
214 | unsigned long *freed = parm; | |
215 | long nr = KB2PAGES(oom_kb); | |
216 | ||
217 | cmm_dbg("OOM processing started\n"); | |
218 | nr = cmm_free_pages(nr); | |
1ef2f06b | 219 | loaned_pages_target = atomic_long_read(&loaned_pages); |
84af458b BK |
220 | *freed += KB2PAGES(oom_kb) - nr; |
221 | oom_freed_pages += KB2PAGES(oom_kb) - nr; | |
222 | cmm_dbg("OOM processing complete\n"); | |
223 | return NOTIFY_OK; | |
224 | } | |
225 | ||
226 | /** | |
227 | * cmm_get_mpp - Read memory performance parameters | |
228 | * | |
229 | * Makes hcall to query the current page loan request from the hypervisor. | |
230 | * | |
231 | * Return value: | |
232 | * nothing | |
233 | **/ | |
234 | static void cmm_get_mpp(void) | |
235 | { | |
1ef2f06b DH |
236 | const long __loaned_pages = atomic_long_read(&loaned_pages); |
237 | const long total_pages = totalram_pages() + __loaned_pages; | |
84af458b BK |
238 | int rc; |
239 | struct hvcall_mpp_data mpp_data; | |
8be8cf5b | 240 | signed long active_pages_target, page_loan_request, target; |
8be8cf5b | 241 | signed long min_mem_pages = (min_mem_mb * 1024 * 1024) / PAGE_SIZE; |
84af458b | 242 | |
b1713975 DH |
243 | if (likely(!simulate)) { |
244 | rc = h_get_mpp(&mpp_data); | |
245 | if (rc != H_SUCCESS) | |
246 | return; | |
247 | page_loan_request = div_s64((s64)mpp_data.loan_request, | |
248 | PAGE_SIZE); | |
249 | target = page_loan_request + __loaned_pages; | |
250 | } else { | |
251 | target = KB2PAGES(simulate_loan_target_kb); | |
252 | page_loan_request = target - __loaned_pages; | |
253 | } | |
8be8cf5b BK |
254 | |
255 | if (target < 0 || total_pages < min_mem_pages) | |
256 | target = 0; | |
257 | ||
258 | if (target > oom_freed_pages) | |
259 | target -= oom_freed_pages; | |
84af458b | 260 | else |
8be8cf5b BK |
261 | target = 0; |
262 | ||
263 | active_pages_target = total_pages - target; | |
264 | ||
265 | if (min_mem_pages > active_pages_target) | |
266 | target = total_pages - min_mem_pages; | |
84af458b | 267 | |
8be8cf5b BK |
268 | if (target < 0) |
269 | target = 0; | |
84af458b | 270 | |
8be8cf5b | 271 | loaned_pages_target = target; |
84af458b BK |
272 | |
273 | cmm_dbg("delta = %ld, loaned = %lu, target = %lu, oom = %lu, totalram = %lu\n", | |
1ef2f06b | 274 | page_loan_request, __loaned_pages, loaned_pages_target, |
ca79b0c2 | 275 | oom_freed_pages, totalram_pages()); |
84af458b BK |
276 | } |
277 | ||
278 | static struct notifier_block cmm_oom_nb = { | |
279 | .notifier_call = cmm_oom_notify | |
280 | }; | |
281 | ||
282 | /** | |
283 | * cmm_thread - CMM task thread | |
284 | * @dummy: not used | |
285 | * | |
286 | * Return value: | |
287 | * 0 | |
288 | **/ | |
289 | static int cmm_thread(void *dummy) | |
290 | { | |
291 | unsigned long timeleft; | |
1ef2f06b | 292 | long __loaned_pages; |
84af458b BK |
293 | |
294 | while (1) { | |
295 | timeleft = msleep_interruptible(delay * 1000); | |
296 | ||
14b8a76b | 297 | if (kthread_should_stop() || timeleft) |
84af458b | 298 | break; |
14b8a76b RJ |
299 | |
300 | if (mutex_trylock(&hotplug_mutex)) { | |
301 | if (hotplug_occurred) { | |
302 | hotplug_occurred = 0; | |
303 | mutex_unlock(&hotplug_mutex); | |
304 | cmm_dbg("Hotplug operation has occurred, " | |
305 | "loaning activity suspended " | |
306 | "for %d seconds.\n", | |
307 | hotplug_delay); | |
308 | timeleft = msleep_interruptible(hotplug_delay * | |
309 | 1000); | |
310 | if (kthread_should_stop() || timeleft) | |
311 | break; | |
312 | continue; | |
313 | } | |
314 | mutex_unlock(&hotplug_mutex); | |
315 | } else { | |
316 | cmm_dbg("Hotplug operation in progress, activity " | |
317 | "suspended\n"); | |
318 | continue; | |
84af458b BK |
319 | } |
320 | ||
321 | cmm_get_mpp(); | |
322 | ||
1ef2f06b DH |
323 | __loaned_pages = atomic_long_read(&loaned_pages); |
324 | if (loaned_pages_target > __loaned_pages) { | |
325 | if (cmm_alloc_pages(loaned_pages_target - __loaned_pages)) | |
326 | loaned_pages_target = __loaned_pages; | |
327 | } else if (loaned_pages_target < __loaned_pages) | |
328 | cmm_free_pages(__loaned_pages - loaned_pages_target); | |
84af458b BK |
329 | } |
330 | return 0; | |
331 | } | |
332 | ||
333 | #define CMM_SHOW(name, format, args...) \ | |
6c9d2909 KS |
334 | static ssize_t show_##name(struct device *dev, \ |
335 | struct device_attribute *attr, \ | |
3cee67f7 | 336 | char *buf) \ |
84af458b BK |
337 | { \ |
338 | return sprintf(buf, format, ##args); \ | |
339 | } \ | |
57ad583f | 340 | static DEVICE_ATTR(name, 0444, show_##name, NULL) |
84af458b | 341 | |
1ef2f06b | 342 | CMM_SHOW(loaned_kb, "%lu\n", PAGES2KB(atomic_long_read(&loaned_pages))); |
84af458b BK |
343 | CMM_SHOW(loaned_target_kb, "%lu\n", PAGES2KB(loaned_pages_target)); |
344 | ||
6c9d2909 KS |
345 | static ssize_t show_oom_pages(struct device *dev, |
346 | struct device_attribute *attr, char *buf) | |
84af458b BK |
347 | { |
348 | return sprintf(buf, "%lu\n", PAGES2KB(oom_freed_pages)); | |
349 | } | |
350 | ||
6c9d2909 KS |
351 | static ssize_t store_oom_pages(struct device *dev, |
352 | struct device_attribute *attr, | |
84af458b BK |
353 | const char *buf, size_t count) |
354 | { | |
355 | unsigned long val = simple_strtoul (buf, NULL, 10); | |
356 | ||
357 | if (!capable(CAP_SYS_ADMIN)) | |
358 | return -EPERM; | |
359 | if (val != 0) | |
360 | return -EBADMSG; | |
361 | ||
362 | oom_freed_pages = 0; | |
363 | return count; | |
364 | } | |
365 | ||
57ad583f | 366 | static DEVICE_ATTR(oom_freed_kb, 0644, |
84af458b BK |
367 | show_oom_pages, store_oom_pages); |
368 | ||
6c9d2909 KS |
369 | static struct device_attribute *cmm_attrs[] = { |
370 | &dev_attr_loaned_kb, | |
371 | &dev_attr_loaned_target_kb, | |
372 | &dev_attr_oom_freed_kb, | |
84af458b BK |
373 | }; |
374 | ||
b1713975 DH |
375 | static DEVICE_ULONG_ATTR(simulate_loan_target_kb, 0644, |
376 | simulate_loan_target_kb); | |
377 | ||
6c9d2909 | 378 | static struct bus_type cmm_subsys = { |
84af458b | 379 | .name = "cmm", |
6c9d2909 | 380 | .dev_name = "cmm", |
84af458b BK |
381 | }; |
382 | ||
7d821274 DH |
383 | static void cmm_release_device(struct device *dev) |
384 | { | |
385 | } | |
386 | ||
84af458b BK |
387 | /** |
388 | * cmm_sysfs_register - Register with sysfs | |
389 | * | |
390 | * Return value: | |
391 | * 0 on success / other on failure | |
392 | **/ | |
6c9d2909 | 393 | static int cmm_sysfs_register(struct device *dev) |
84af458b BK |
394 | { |
395 | int i, rc; | |
396 | ||
6c9d2909 | 397 | if ((rc = subsys_system_register(&cmm_subsys, NULL))) |
84af458b BK |
398 | return rc; |
399 | ||
6c9d2909 KS |
400 | dev->id = 0; |
401 | dev->bus = &cmm_subsys; | |
7d821274 | 402 | dev->release = cmm_release_device; |
84af458b | 403 | |
6c9d2909 KS |
404 | if ((rc = device_register(dev))) |
405 | goto subsys_unregister; | |
84af458b BK |
406 | |
407 | for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++) { | |
6c9d2909 | 408 | if ((rc = device_create_file(dev, cmm_attrs[i]))) |
84af458b BK |
409 | goto fail; |
410 | } | |
411 | ||
b1713975 DH |
412 | if (!simulate) |
413 | return 0; | |
414 | rc = device_create_file(dev, &dev_attr_simulate_loan_target_kb.attr); | |
415 | if (rc) | |
416 | goto fail; | |
84af458b BK |
417 | return 0; |
418 | ||
419 | fail: | |
420 | while (--i >= 0) | |
6c9d2909 KS |
421 | device_remove_file(dev, cmm_attrs[i]); |
422 | device_unregister(dev); | |
423 | subsys_unregister: | |
424 | bus_unregister(&cmm_subsys); | |
84af458b BK |
425 | return rc; |
426 | } | |
427 | ||
428 | /** | |
429 | * cmm_unregister_sysfs - Unregister from sysfs | |
430 | * | |
431 | **/ | |
6c9d2909 | 432 | static void cmm_unregister_sysfs(struct device *dev) |
84af458b BK |
433 | { |
434 | int i; | |
435 | ||
436 | for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++) | |
6c9d2909 KS |
437 | device_remove_file(dev, cmm_attrs[i]); |
438 | device_unregister(dev); | |
439 | bus_unregister(&cmm_subsys); | |
84af458b BK |
440 | } |
441 | ||
fecba962 BK |
442 | /** |
443 | * cmm_reboot_notifier - Make sure pages are not still marked as "loaned" | |
444 | * | |
445 | **/ | |
446 | static int cmm_reboot_notifier(struct notifier_block *nb, | |
447 | unsigned long action, void *unused) | |
448 | { | |
449 | if (action == SYS_RESTART) { | |
450 | if (cmm_thread_ptr) | |
451 | kthread_stop(cmm_thread_ptr); | |
452 | cmm_thread_ptr = NULL; | |
1ef2f06b | 453 | cmm_free_pages(atomic_long_read(&loaned_pages)); |
fecba962 BK |
454 | } |
455 | return NOTIFY_DONE; | |
456 | } | |
457 | ||
458 | static struct notifier_block cmm_reboot_nb = { | |
459 | .notifier_call = cmm_reboot_notifier, | |
460 | }; | |
461 | ||
14b8a76b RJ |
462 | /** |
463 | * cmm_memory_cb - Handle memory hotplug notifier calls | |
464 | * @self: notifier block struct | |
465 | * @action: action to take | |
466 | * @arg: struct memory_notify data for handler | |
467 | * | |
468 | * Return value: | |
469 | * NOTIFY_OK or notifier error based on subfunction return value | |
470 | * | |
471 | **/ | |
472 | static int cmm_memory_cb(struct notifier_block *self, | |
473 | unsigned long action, void *arg) | |
474 | { | |
14b8a76b RJ |
475 | switch (action) { |
476 | case MEM_GOING_OFFLINE: | |
477 | mutex_lock(&hotplug_mutex); | |
478 | hotplug_occurred = 1; | |
14b8a76b RJ |
479 | break; |
480 | case MEM_OFFLINE: | |
481 | case MEM_CANCEL_OFFLINE: | |
482 | mutex_unlock(&hotplug_mutex); | |
483 | cmm_dbg("Memory offline operation complete.\n"); | |
484 | break; | |
485 | case MEM_GOING_ONLINE: | |
486 | case MEM_ONLINE: | |
487 | case MEM_CANCEL_ONLINE: | |
488 | break; | |
489 | } | |
490 | ||
701c3167 | 491 | return NOTIFY_OK; |
14b8a76b RJ |
492 | } |
493 | ||
494 | static struct notifier_block cmm_mem_nb = { | |
495 | .notifier_call = cmm_memory_cb, | |
496 | .priority = CMM_MEM_HOTPLUG_PRI | |
497 | }; | |
498 | ||
fe030c9b | 499 | #ifdef CONFIG_BALLOON_COMPACTION |
fe030c9b DH |
500 | static int cmm_migratepage(struct balloon_dev_info *b_dev_info, |
501 | struct page *newpage, struct page *page, | |
502 | enum migrate_mode mode) | |
503 | { | |
504 | unsigned long flags; | |
505 | ||
506 | /* | |
507 | * loan/"inflate" the newpage first. | |
508 | * | |
509 | * We might race against the cmm_thread who might discover after our | |
510 | * loan request that another page is to be unloaned. However, once | |
511 | * the cmm_thread runs again later, this error will automatically | |
512 | * be corrected. | |
513 | */ | |
514 | if (plpar_page_set_loaned(newpage)) { | |
515 | /* Unlikely, but possible. Tell the caller not to retry now. */ | |
516 | pr_err_ratelimited("%s: Cannot set page to loaned.", __func__); | |
517 | return -EBUSY; | |
518 | } | |
519 | ||
520 | /* balloon page list reference */ | |
521 | get_page(newpage); | |
522 | ||
e352f576 DH |
523 | /* |
524 | * When we migrate a page to a different zone, we have to fixup the | |
525 | * count of both involved zones as we adjusted the managed page count | |
526 | * when inflating. | |
527 | */ | |
528 | if (page_zone(page) != page_zone(newpage)) { | |
529 | adjust_managed_page_count(page, 1); | |
530 | adjust_managed_page_count(newpage, -1); | |
531 | } | |
532 | ||
fe030c9b DH |
533 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
534 | balloon_page_insert(b_dev_info, newpage); | |
535 | balloon_page_delete(page); | |
536 | b_dev_info->isolated_pages--; | |
537 | spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); | |
538 | ||
539 | /* | |
540 | * activate/"deflate" the old page. We ignore any errors just like the | |
541 | * other callers. | |
542 | */ | |
543 | plpar_page_set_active(page); | |
544 | ||
545 | /* balloon page list reference */ | |
546 | put_page(page); | |
547 | ||
548 | return MIGRATEPAGE_SUCCESS; | |
549 | } | |
550 | ||
68f2736a | 551 | static void cmm_balloon_compaction_init(void) |
fe030c9b | 552 | { |
fe030c9b DH |
553 | balloon_devinfo_init(&b_dev_info); |
554 | b_dev_info.migratepage = cmm_migratepage; | |
fe030c9b DH |
555 | } |
556 | #else /* CONFIG_BALLOON_COMPACTION */ | |
68f2736a | 557 | static void cmm_balloon_compaction_init(void) |
fe030c9b DH |
558 | { |
559 | } | |
560 | #endif /* CONFIG_BALLOON_COMPACTION */ | |
561 | ||
84af458b BK |
562 | /** |
563 | * cmm_init - Module initialization | |
564 | * | |
565 | * Return value: | |
566 | * 0 on success / other on failure | |
567 | **/ | |
568 | static int cmm_init(void) | |
569 | { | |
68f7a049 | 570 | int rc; |
84af458b | 571 | |
b1713975 | 572 | if (!firmware_has_feature(FW_FEATURE_CMO) && !simulate) |
84af458b BK |
573 | return -EOPNOTSUPP; |
574 | ||
68f2736a | 575 | cmm_balloon_compaction_init(); |
84af458b | 576 | |
fe030c9b DH |
577 | rc = register_oom_notifier(&cmm_oom_nb); |
578 | if (rc < 0) | |
579 | goto out_balloon_compaction; | |
580 | ||
fecba962 | 581 | if ((rc = register_reboot_notifier(&cmm_reboot_nb))) |
84af458b BK |
582 | goto out_oom_notifier; |
583 | ||
6c9d2909 | 584 | if ((rc = cmm_sysfs_register(&cmm_dev))) |
fecba962 BK |
585 | goto out_reboot_notifier; |
586 | ||
022da223 DH |
587 | rc = register_memory_notifier(&cmm_mem_nb); |
588 | if (rc) | |
589 | goto out_unregister_notifier; | |
590 | ||
84af458b | 591 | if (cmm_disabled) |
68f7a049 | 592 | return 0; |
84af458b BK |
593 | |
594 | cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread"); | |
595 | if (IS_ERR(cmm_thread_ptr)) { | |
596 | rc = PTR_ERR(cmm_thread_ptr); | |
14b8a76b | 597 | goto out_unregister_notifier; |
84af458b BK |
598 | } |
599 | ||
68f7a049 | 600 | return 0; |
14b8a76b RJ |
601 | out_unregister_notifier: |
602 | unregister_memory_notifier(&cmm_mem_nb); | |
6c9d2909 | 603 | cmm_unregister_sysfs(&cmm_dev); |
fecba962 BK |
604 | out_reboot_notifier: |
605 | unregister_reboot_notifier(&cmm_reboot_nb); | |
84af458b BK |
606 | out_oom_notifier: |
607 | unregister_oom_notifier(&cmm_oom_nb); | |
fe030c9b | 608 | out_balloon_compaction: |
84af458b BK |
609 | return rc; |
610 | } | |
611 | ||
612 | /** | |
613 | * cmm_exit - Module exit | |
614 | * | |
615 | * Return value: | |
616 | * nothing | |
617 | **/ | |
618 | static void cmm_exit(void) | |
619 | { | |
620 | if (cmm_thread_ptr) | |
621 | kthread_stop(cmm_thread_ptr); | |
622 | unregister_oom_notifier(&cmm_oom_nb); | |
fecba962 | 623 | unregister_reboot_notifier(&cmm_reboot_nb); |
14b8a76b | 624 | unregister_memory_notifier(&cmm_mem_nb); |
1ef2f06b | 625 | cmm_free_pages(atomic_long_read(&loaned_pages)); |
6c9d2909 | 626 | cmm_unregister_sysfs(&cmm_dev); |
84af458b BK |
627 | } |
628 | ||
629 | /** | |
630 | * cmm_set_disable - Disable/Enable CMM | |
631 | * | |
632 | * Return value: | |
633 | * 0 on success / other on failure | |
634 | **/ | |
e4dca7b7 | 635 | static int cmm_set_disable(const char *val, const struct kernel_param *kp) |
84af458b BK |
636 | { |
637 | int disable = simple_strtoul(val, NULL, 10); | |
638 | ||
639 | if (disable != 0 && disable != 1) | |
640 | return -EINVAL; | |
641 | ||
642 | if (disable && !cmm_disabled) { | |
643 | if (cmm_thread_ptr) | |
644 | kthread_stop(cmm_thread_ptr); | |
645 | cmm_thread_ptr = NULL; | |
1ef2f06b | 646 | cmm_free_pages(atomic_long_read(&loaned_pages)); |
84af458b BK |
647 | } else if (!disable && cmm_disabled) { |
648 | cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread"); | |
649 | if (IS_ERR(cmm_thread_ptr)) | |
650 | return PTR_ERR(cmm_thread_ptr); | |
651 | } | |
652 | ||
653 | cmm_disabled = disable; | |
654 | return 0; | |
655 | } | |
656 | ||
657 | module_param_call(disable, cmm_set_disable, param_get_uint, | |
57ad583f | 658 | &cmm_disabled, 0644); |
84af458b BK |
659 | MODULE_PARM_DESC(disable, "Disable CMM. Set to 1 to disable. " |
660 | "[Default=" __stringify(CMM_DISABLE) "]"); | |
661 | ||
662 | module_init(cmm_init); | |
663 | module_exit(cmm_exit); |