Commit | Line | Data |
---|---|---|
8b4770ec | 1 | // SPDX-License-Identifier: GPL-2.0 |
453dc659 DT |
2 | /* |
3 | * VMware Balloon driver. | |
4 | * | |
8b4770ec | 5 | * Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved. |
453dc659 | 6 | * |
453dc659 DT |
7 | * This is VMware physical memory management driver for Linux. The driver |
8 | * acts like a "balloon" that can be inflated to reclaim physical pages by | |
9 | * reserving them in the guest and invalidating them in the monitor, | |
10 | * freeing up the underlying machine pages so they can be allocated to | |
11 | * other guests. The balloon can also be deflated to allow the guest to | |
12 | * use more physical memory. Higher level policies can control the sizes | |
13 | * of balloons in VMs in order to manage physical memory resources. | |
14 | */ | |
15 | ||
16 | //#define DEBUG | |
17 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
18 | ||
19 | #include <linux/types.h> | |
97976e01 | 20 | #include <linux/io.h> |
453dc659 DT |
21 | #include <linux/kernel.h> |
22 | #include <linux/mm.h> | |
f220a80f | 23 | #include <linux/vmalloc.h> |
453dc659 DT |
24 | #include <linux/sched.h> |
25 | #include <linux/module.h> | |
26 | #include <linux/workqueue.h> | |
27 | #include <linux/debugfs.h> | |
28 | #include <linux/seq_file.h> | |
c7b3690f NA |
29 | #include <linux/rwsem.h> |
30 | #include <linux/slab.h> | |
6e4453b3 | 31 | #include <linux/spinlock.h> |
83a8afa7 | 32 | #include <linux/balloon_compaction.h> |
48e3d668 PM |
33 | #include <linux/vmw_vmci_defs.h> |
34 | #include <linux/vmw_vmci_api.h> | |
a10a5698 | 35 | #include <asm/hypervisor.h> |
453dc659 DT |
36 | |
37 | MODULE_AUTHOR("VMware, Inc."); | |
38 | MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver"); | |
453dc659 DT |
39 | MODULE_ALIAS("dmi:*:svnVMware*:*"); |
40 | MODULE_ALIAS("vmware_vmmemctl"); | |
41 | MODULE_LICENSE("GPL"); | |
42 | ||
5d1a86ec NA |
43 | static bool __read_mostly vmwballoon_shrinker_enable; |
44 | module_param(vmwballoon_shrinker_enable, bool, 0444); | |
45 | MODULE_PARM_DESC(vmwballoon_shrinker_enable, | |
46 | "Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performance."); | |
47 | ||
48 | /* Delay in seconds after shrink before inflation. */ | |
49 | #define VMBALLOON_SHRINK_DELAY (5) | |
50 | ||
51 | /* Maximum number of refused pages we accumulate during inflation cycle */ | |
55adaa49 | 52 | #define VMW_BALLOON_MAX_REFUSED 16 |
453dc659 | 53 | |
83a8afa7 NA |
54 | /* Magic number for the balloon mount-point */ |
55 | #define BALLOON_VMW_MAGIC 0x0ba11007 | |
56 | ||
453dc659 DT |
57 | /* |
58 | * Hypervisor communication port definitions. | |
59 | */ | |
60 | #define VMW_BALLOON_HV_PORT 0x5670 | |
61 | #define VMW_BALLOON_HV_MAGIC 0x456c6d6f | |
453dc659 DT |
62 | #define VMW_BALLOON_GUEST_ID 1 /* Linux */ |
63 | ||
eb79100f XD |
64 | enum vmwballoon_capabilities { |
65 | /* | |
66 | * Bit 0 is reserved and not associated to any capability. | |
67 | */ | |
48e3d668 PM |
68 | VMW_BALLOON_BASIC_CMDS = (1 << 1), |
69 | VMW_BALLOON_BATCHED_CMDS = (1 << 2), | |
70 | VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3), | |
71 | VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4), | |
55398302 | 72 | VMW_BALLOON_64_BIT_TARGET = (1 << 5) |
eb79100f XD |
73 | }; |
74 | ||
55398302 | 75 | #define VMW_BALLOON_CAPABILITIES_COMMON (VMW_BALLOON_BASIC_CMDS \ |
365bd7ef | 76 | | VMW_BALLOON_BATCHED_CMDS \ |
48e3d668 PM |
77 | | VMW_BALLOON_BATCHED_2M_CMDS \ |
78 | | VMW_BALLOON_SIGNALLED_WAKEUP_CMD) | |
365bd7ef | 79 | |
25acbdd7 | 80 | #define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT) |
eb79100f | 81 | |
55398302 XD |
82 | /* |
83 | * 64-bit targets are only supported in 64-bit | |
84 | */ | |
85 | #ifdef CONFIG_64BIT | |
86 | #define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_CAPABILITIES_COMMON \ | |
87 | | VMW_BALLOON_64_BIT_TARGET) | |
88 | #else | |
89 | #define VMW_BALLOON_CAPABILITIES VMW_BALLOON_CAPABILITIES_COMMON | |
90 | #endif | |
91 | ||
c7b3690f NA |
92 | enum vmballoon_page_size_type { |
93 | VMW_BALLOON_4K_PAGE, | |
94 | VMW_BALLOON_2M_PAGE, | |
95 | VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE | |
96 | }; | |
97 | ||
98 | #define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1) | |
99 | ||
6e4453b3 NA |
100 | static const char * const vmballoon_page_size_names[] = { |
101 | [VMW_BALLOON_4K_PAGE] = "4k", | |
102 | [VMW_BALLOON_2M_PAGE] = "2M" | |
103 | }; | |
104 | ||
105 | enum vmballoon_op { | |
106 | VMW_BALLOON_INFLATE, | |
107 | VMW_BALLOON_DEFLATE | |
108 | }; | |
109 | ||
c7b3690f NA |
110 | enum vmballoon_op_stat_type { |
111 | VMW_BALLOON_OP_STAT, | |
112 | VMW_BALLOON_OP_FAIL_STAT | |
113 | }; | |
114 | ||
115 | #define VMW_BALLOON_OP_STAT_TYPES (VMW_BALLOON_OP_FAIL_STAT + 1) | |
116 | ||
117 | /** | |
118 | * enum vmballoon_cmd_type - backdoor commands. | |
119 | * | |
120 | * Availability of the commands is as followed: | |
121 | * | |
122 | * %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and | |
123 | * %VMW_BALLOON_CMD_GUEST_ID are always available. | |
124 | * | |
125 | * If the host reports %VMW_BALLOON_BASIC_CMDS are supported then | |
126 | * %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available. | |
f220a80f | 127 | * |
c7b3690f NA |
128 | * If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then |
129 | * %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands | |
130 | * are available. | |
f220a80f | 131 | * |
c7b3690f NA |
132 | * If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then |
133 | * %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK | |
134 | * are supported. | |
135 | * | |
136 | * If the host reports VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then | |
137 | * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported. | |
138 | * | |
139 | * @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor. | |
140 | * @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size. | |
141 | * @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page. | |
142 | * @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about | |
143 | * to be deflated from the balloon. | |
144 | * @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that | |
145 | * runs in the VM. | |
146 | * @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of | |
147 | * ballooned pages (up to 512). | |
148 | * @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of | |
149 | * pages that are about to be deflated from the | |
150 | * balloon (up to 512). | |
151 | * @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK | |
152 | * for 2MB pages. | |
153 | * @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to | |
154 | * @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB | |
155 | * pages. | |
156 | * @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification | |
157 | * that would be invoked when the balloon | |
158 | * size changes. | |
159 | * @VMW_BALLOON_CMD_LAST: Value of the last command. | |
f220a80f | 160 | */ |
c7b3690f NA |
161 | enum vmballoon_cmd_type { |
162 | VMW_BALLOON_CMD_START, | |
163 | VMW_BALLOON_CMD_GET_TARGET, | |
164 | VMW_BALLOON_CMD_LOCK, | |
165 | VMW_BALLOON_CMD_UNLOCK, | |
166 | VMW_BALLOON_CMD_GUEST_ID, | |
167 | /* No command 5 */ | |
168 | VMW_BALLOON_CMD_BATCHED_LOCK = 6, | |
169 | VMW_BALLOON_CMD_BATCHED_UNLOCK, | |
170 | VMW_BALLOON_CMD_BATCHED_2M_LOCK, | |
171 | VMW_BALLOON_CMD_BATCHED_2M_UNLOCK, | |
172 | VMW_BALLOON_CMD_VMCI_DOORBELL_SET, | |
173 | VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET, | |
174 | }; | |
175 | ||
176 | #define VMW_BALLOON_CMD_NUM (VMW_BALLOON_CMD_LAST + 1) | |
177 | ||
178 | enum vmballoon_error_codes { | |
179 | VMW_BALLOON_SUCCESS, | |
180 | VMW_BALLOON_ERROR_CMD_INVALID, | |
181 | VMW_BALLOON_ERROR_PPN_INVALID, | |
182 | VMW_BALLOON_ERROR_PPN_LOCKED, | |
183 | VMW_BALLOON_ERROR_PPN_UNLOCKED, | |
184 | VMW_BALLOON_ERROR_PPN_PINNED, | |
185 | VMW_BALLOON_ERROR_PPN_NOTNEEDED, | |
186 | VMW_BALLOON_ERROR_RESET, | |
187 | VMW_BALLOON_ERROR_BUSY | |
188 | }; | |
eb79100f XD |
189 | |
190 | #define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000) | |
191 | ||
10a95d5d NA |
192 | #define VMW_BALLOON_CMD_WITH_TARGET_MASK \ |
193 | ((1UL << VMW_BALLOON_CMD_GET_TARGET) | \ | |
194 | (1UL << VMW_BALLOON_CMD_LOCK) | \ | |
195 | (1UL << VMW_BALLOON_CMD_UNLOCK) | \ | |
196 | (1UL << VMW_BALLOON_CMD_BATCHED_LOCK) | \ | |
197 | (1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK) | \ | |
198 | (1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK) | \ | |
199 | (1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK)) | |
200 | ||
68131184 NA |
201 | static const char * const vmballoon_cmd_names[] = { |
202 | [VMW_BALLOON_CMD_START] = "start", | |
203 | [VMW_BALLOON_CMD_GET_TARGET] = "target", | |
204 | [VMW_BALLOON_CMD_LOCK] = "lock", | |
205 | [VMW_BALLOON_CMD_UNLOCK] = "unlock", | |
206 | [VMW_BALLOON_CMD_GUEST_ID] = "guestType", | |
207 | [VMW_BALLOON_CMD_BATCHED_LOCK] = "batchLock", | |
208 | [VMW_BALLOON_CMD_BATCHED_UNLOCK] = "batchUnlock", | |
209 | [VMW_BALLOON_CMD_BATCHED_2M_LOCK] = "2m-lock", | |
210 | [VMW_BALLOON_CMD_BATCHED_2M_UNLOCK] = "2m-unlock", | |
211 | [VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet" | |
212 | }; | |
213 | ||
c7b3690f NA |
214 | enum vmballoon_stat_page { |
215 | VMW_BALLOON_PAGE_STAT_ALLOC, | |
216 | VMW_BALLOON_PAGE_STAT_ALLOC_FAIL, | |
217 | VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC, | |
218 | VMW_BALLOON_PAGE_STAT_REFUSED_FREE, | |
219 | VMW_BALLOON_PAGE_STAT_FREE, | |
220 | VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE | |
453dc659 DT |
221 | }; |
222 | ||
c7b3690f NA |
223 | #define VMW_BALLOON_PAGE_STAT_NUM (VMW_BALLOON_PAGE_STAT_LAST + 1) |
224 | ||
225 | enum vmballoon_stat_general { | |
226 | VMW_BALLOON_STAT_TIMER, | |
227 | VMW_BALLOON_STAT_DOORBELL, | |
8840a6f4 | 228 | VMW_BALLOON_STAT_RESET, |
5d1a86ec NA |
229 | VMW_BALLOON_STAT_SHRINK, |
230 | VMW_BALLOON_STAT_SHRINK_FREE, | |
231 | VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE | |
c7b3690f NA |
232 | }; |
233 | ||
234 | #define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1) | |
235 | ||
df8d0d42 | 236 | static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching); |
c7b3690f | 237 | static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled); |
f220a80f | 238 | |
6e4453b3 | 239 | struct vmballoon_ctl { |
453dc659 | 240 | struct list_head pages; |
453dc659 | 241 | struct list_head refused_pages; |
ae29783d | 242 | struct list_head prealloc_pages; |
55adaa49 | 243 | unsigned int n_refused_pages; |
6e4453b3 NA |
244 | unsigned int n_pages; |
245 | enum vmballoon_page_size_type page_size; | |
246 | enum vmballoon_op op; | |
247 | }; | |
248 | ||
6c948757 NA |
249 | /** |
250 | * struct vmballoon_batch_entry - a batch entry for lock or unlock. | |
251 | * | |
252 | * @status: the status of the operation, which is written by the hypervisor. | |
253 | * @reserved: reserved for future use. Must be set to zero. | |
254 | * @pfn: the physical frame number of the page to be locked or unlocked. | |
255 | */ | |
256 | struct vmballoon_batch_entry { | |
257 | u64 status : 5; | |
258 | u64 reserved : PAGE_SHIFT - 5; | |
259 | u64 pfn : 52; | |
260 | } __packed; | |
261 | ||
365bd7ef | 262 | struct vmballoon { |
6e4453b3 NA |
263 | /** |
264 | * @max_page_size: maximum supported page size for ballooning. | |
265 | * | |
266 | * Protected by @conf_sem | |
267 | */ | |
268 | enum vmballoon_page_size_type max_page_size; | |
269 | ||
270 | /** | |
271 | * @size: balloon actual size in basic page size (frames). | |
272 | * | |
273 | * While we currently do not support size which is bigger than 32-bit, | |
274 | * in preparation for future support, use 64-bits. | |
275 | */ | |
276 | atomic64_t size; | |
453dc659 | 277 | |
6e4453b3 NA |
278 | /** |
279 | * @target: balloon target size in basic page size (frames). | |
280 | * | |
281 | * We do not protect the target under the assumption that setting the | |
282 | * value is always done through a single write. If this assumption ever | |
283 | * breaks, we would have to use X_ONCE for accesses, and suffer the less | |
284 | * optimized code. Although we may read stale target value if multiple | |
285 | * accesses happen at once, the performance impact should be minor. | |
286 | */ | |
287 | unsigned long target; | |
453dc659 | 288 | |
6e4453b3 NA |
289 | /** |
290 | * @reset_required: reset flag | |
291 | * | |
292 | * Setting this flag may introduce races, but the code is expected to | |
293 | * handle them gracefully. In the worst case, another operation will | |
294 | * fail as reset did not take place. Clearing the flag is done while | |
295 | * holding @conf_sem for write. | |
296 | */ | |
453dc659 DT |
297 | bool reset_required; |
298 | ||
6e4453b3 NA |
299 | /** |
300 | * @capabilities: hypervisor balloon capabilities. | |
301 | * | |
302 | * Protected by @conf_sem. | |
303 | */ | |
f220a80f XD |
304 | unsigned long capabilities; |
305 | ||
6c948757 NA |
306 | /** |
307 | * @batch_page: pointer to communication batch page. | |
308 | * | |
309 | * When batching is used, batch_page points to a page, which holds up to | |
310 | * %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking. | |
311 | */ | |
312 | struct vmballoon_batch_entry *batch_page; | |
313 | ||
6e4453b3 NA |
314 | /** |
315 | * @batch_max_pages: maximum pages that can be locked/unlocked. | |
316 | * | |
317 | * Indicates the number of pages that the hypervisor can lock or unlock | |
318 | * at once, according to whether batching is enabled. If batching is | |
319 | * disabled, only a single page can be locked/unlock on each operation. | |
320 | * | |
321 | * Protected by @conf_sem. | |
322 | */ | |
f220a80f | 323 | unsigned int batch_max_pages; |
6e4453b3 NA |
324 | |
325 | /** | |
326 | * @page: page to be locked/unlocked by the hypervisor | |
327 | * | |
328 | * @page is only used when batching is disabled and a single page is | |
329 | * reclaimed on each iteration. | |
330 | * | |
331 | * Protected by @comm_lock. | |
332 | */ | |
f220a80f XD |
333 | struct page *page; |
334 | ||
5d1a86ec NA |
335 | /** |
336 | * @shrink_timeout: timeout until the next inflation. | |
337 | * | |
338 | * After an shrink event, indicates the time in jiffies after which | |
339 | * inflation is allowed again. Can be written concurrently with reads, | |
340 | * so must use READ_ONCE/WRITE_ONCE when accessing. | |
341 | */ | |
342 | unsigned long shrink_timeout; | |
343 | ||
453dc659 | 344 | /* statistics */ |
c7b3690f | 345 | struct vmballoon_stats *stats; |
453dc659 | 346 | |
83a8afa7 NA |
347 | /** |
348 | * @b_dev_info: balloon device information descriptor. | |
349 | */ | |
350 | struct balloon_dev_info b_dev_info; | |
351 | ||
453dc659 | 352 | struct delayed_work dwork; |
48e3d668 | 353 | |
83a8afa7 NA |
354 | /** |
355 | * @huge_pages - list of the inflated 2MB pages. | |
356 | * | |
357 | * Protected by @b_dev_info.pages_lock . | |
358 | */ | |
359 | struct list_head huge_pages; | |
360 | ||
6e4453b3 NA |
361 | /** |
362 | * @vmci_doorbell. | |
363 | * | |
364 | * Protected by @conf_sem. | |
365 | */ | |
48e3d668 | 366 | struct vmci_handle vmci_doorbell; |
c7b3690f NA |
367 | |
368 | /** | |
369 | * @conf_sem: semaphore to protect the configuration and the statistics. | |
370 | */ | |
371 | struct rw_semaphore conf_sem; | |
6e4453b3 NA |
372 | |
373 | /** | |
374 | * @comm_lock: lock to protect the communication with the host. | |
375 | * | |
376 | * Lock ordering: @conf_sem -> @comm_lock . | |
377 | */ | |
378 | spinlock_t comm_lock; | |
5d1a86ec NA |
379 | |
380 | /** | |
381 | * @shrinker: shrinker interface that is used to avoid over-inflation. | |
382 | */ | |
17c4eb03 | 383 | struct shrinker *shrinker; |
453dc659 DT |
384 | }; |
385 | ||
386 | static struct vmballoon balloon; | |
453dc659 | 387 | |
c7b3690f NA |
388 | struct vmballoon_stats { |
389 | /* timer / doorbell operations */ | |
390 | atomic64_t general_stat[VMW_BALLOON_STAT_NUM]; | |
391 | ||
392 | /* allocation statistics for huge and small pages */ | |
393 | atomic64_t | |
394 | page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES]; | |
395 | ||
396 | /* Monitor operations: total operations, and failures */ | |
397 | atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES]; | |
398 | }; | |
399 | ||
400 | static inline bool is_vmballoon_stats_on(void) | |
401 | { | |
402 | return IS_ENABLED(CONFIG_DEBUG_FS) && | |
403 | static_branch_unlikely(&balloon_stat_enabled); | |
404 | } | |
405 | ||
406 | static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op, | |
407 | enum vmballoon_op_stat_type type) | |
408 | { | |
409 | if (is_vmballoon_stats_on()) | |
410 | atomic64_inc(&b->stats->ops[op][type]); | |
411 | } | |
412 | ||
413 | static inline void vmballoon_stats_gen_inc(struct vmballoon *b, | |
414 | enum vmballoon_stat_general stat) | |
415 | { | |
416 | if (is_vmballoon_stats_on()) | |
417 | atomic64_inc(&b->stats->general_stat[stat]); | |
418 | } | |
419 | ||
420 | static inline void vmballoon_stats_gen_add(struct vmballoon *b, | |
421 | enum vmballoon_stat_general stat, | |
422 | unsigned int val) | |
423 | { | |
424 | if (is_vmballoon_stats_on()) | |
425 | atomic64_add(val, &b->stats->general_stat[stat]); | |
426 | } | |
427 | ||
428 | static inline void vmballoon_stats_page_inc(struct vmballoon *b, | |
429 | enum vmballoon_stat_page stat, | |
6e4453b3 | 430 | enum vmballoon_page_size_type size) |
c7b3690f NA |
431 | { |
432 | if (is_vmballoon_stats_on()) | |
6e4453b3 NA |
433 | atomic64_inc(&b->stats->page_stat[stat][size]); |
434 | } | |
435 | ||
436 | static inline void vmballoon_stats_page_add(struct vmballoon *b, | |
437 | enum vmballoon_stat_page stat, | |
438 | enum vmballoon_page_size_type size, | |
439 | unsigned int val) | |
440 | { | |
441 | if (is_vmballoon_stats_on()) | |
442 | atomic64_add(val, &b->stats->page_stat[stat][size]); | |
c7b3690f NA |
443 | } |
444 | ||
10a95d5d NA |
445 | static inline unsigned long |
446 | __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1, | |
447 | unsigned long arg2, unsigned long *result) | |
448 | { | |
449 | unsigned long status, dummy1, dummy2, dummy3, local_result; | |
450 | ||
c7b3690f | 451 | vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT); |
68131184 | 452 | |
10a95d5d NA |
453 | asm volatile ("inl %%dx" : |
454 | "=a"(status), | |
455 | "=c"(dummy1), | |
456 | "=d"(dummy2), | |
457 | "=b"(local_result), | |
458 | "=S"(dummy3) : | |
459 | "0"(VMW_BALLOON_HV_MAGIC), | |
460 | "1"(cmd), | |
461 | "2"(VMW_BALLOON_HV_PORT), | |
462 | "3"(arg1), | |
463 | "4"(arg2) : | |
464 | "memory"); | |
465 | ||
466 | /* update the result if needed */ | |
467 | if (result) | |
468 | *result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 : | |
469 | local_result; | |
470 | ||
471 | /* update target when applicable */ | |
472 | if (status == VMW_BALLOON_SUCCESS && | |
473 | ((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK)) | |
6e4453b3 | 474 | WRITE_ONCE(b->target, local_result); |
10a95d5d | 475 | |
68131184 NA |
476 | if (status != VMW_BALLOON_SUCCESS && |
477 | status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) { | |
c7b3690f | 478 | vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT); |
68131184 NA |
479 | pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n", |
480 | __func__, vmballoon_cmd_names[cmd], arg1, arg2, | |
481 | status); | |
482 | } | |
483 | ||
10a95d5d NA |
484 | /* mark reset required accordingly */ |
485 | if (status == VMW_BALLOON_ERROR_RESET) | |
486 | b->reset_required = true; | |
487 | ||
488 | return status; | |
489 | } | |
490 | ||
491 | static __always_inline unsigned long | |
492 | vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1, | |
493 | unsigned long arg2) | |
494 | { | |
495 | unsigned long dummy; | |
496 | ||
497 | return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy); | |
498 | } | |
499 | ||
453dc659 DT |
500 | /* |
501 | * Send "start" command to the host, communicating supported version | |
502 | * of the protocol. | |
503 | */ | |
22d293ee | 504 | static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps) |
453dc659 | 505 | { |
10a95d5d | 506 | unsigned long status, capabilities; |
453dc659 | 507 | |
10a95d5d NA |
508 | status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0, |
509 | &capabilities); | |
f220a80f XD |
510 | |
511 | switch (status) { | |
512 | case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES: | |
513 | b->capabilities = capabilities; | |
365bd7ef | 514 | break; |
f220a80f XD |
515 | case VMW_BALLOON_SUCCESS: |
516 | b->capabilities = VMW_BALLOON_BASIC_CMDS; | |
365bd7ef PM |
517 | break; |
518 | default: | |
22d293ee | 519 | return -EIO; |
f220a80f | 520 | } |
453dc659 | 521 | |
5081efd1 NA |
522 | /* |
523 | * 2MB pages are only supported with batching. If batching is for some | |
524 | * reason disabled, do not use 2MB pages, since otherwise the legacy | |
525 | * mechanism is used with 2MB pages, causing a failure. | |
526 | */ | |
6e4453b3 | 527 | b->max_page_size = VMW_BALLOON_4K_PAGE; |
5081efd1 NA |
528 | if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) && |
529 | (b->capabilities & VMW_BALLOON_BATCHED_CMDS)) | |
6e4453b3 NA |
530 | b->max_page_size = VMW_BALLOON_2M_PAGE; |
531 | ||
365bd7ef | 532 | |
22d293ee | 533 | return 0; |
453dc659 DT |
534 | } |
535 | ||
22d293ee NA |
536 | /** |
537 | * vmballoon_send_guest_id - communicate guest type to the host. | |
538 | * | |
539 | * @b: pointer to the balloon. | |
540 | * | |
453dc659 DT |
541 | * Communicate guest type to the host so that it can adjust ballooning |
542 | * algorithm to the one most appropriate for the guest. This command | |
543 | * is normally issued after sending "start" command and is part of | |
544 | * standard reset sequence. | |
22d293ee NA |
545 | * |
546 | * Return: zero on success or appropriate error code. | |
453dc659 | 547 | */ |
22d293ee | 548 | static int vmballoon_send_guest_id(struct vmballoon *b) |
453dc659 | 549 | { |
10a95d5d | 550 | unsigned long status; |
453dc659 | 551 | |
10a95d5d NA |
552 | status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID, |
553 | VMW_BALLOON_GUEST_ID, 0); | |
453dc659 | 554 | |
22d293ee | 555 | return status == VMW_BALLOON_SUCCESS ? 0 : -EIO; |
453dc659 DT |
556 | } |
557 | ||
6e4453b3 NA |
558 | /** |
559 | * vmballoon_page_order() - return the order of the page | |
560 | * @page_size: the size of the page. | |
561 | * | |
562 | * Return: the allocation order. | |
563 | */ | |
564 | static inline | |
565 | unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size) | |
365bd7ef | 566 | { |
6e4453b3 NA |
567 | return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0; |
568 | } | |
365bd7ef | 569 | |
6e4453b3 NA |
570 | /** |
571 | * vmballoon_page_in_frames() - returns the number of frames in a page. | |
572 | * @page_size: the size of the page. | |
573 | * | |
574 | * Return: the number of 4k frames. | |
575 | */ | |
576 | static inline unsigned int | |
577 | vmballoon_page_in_frames(enum vmballoon_page_size_type page_size) | |
578 | { | |
579 | return 1 << vmballoon_page_order(page_size); | |
365bd7ef PM |
580 | } |
581 | ||
8165540c DH |
582 | /** |
583 | * vmballoon_mark_page_offline() - mark a page as offline | |
584 | * @page: pointer for the page. | |
585 | * @page_size: the size of the page. | |
586 | */ | |
587 | static void | |
588 | vmballoon_mark_page_offline(struct page *page, | |
589 | enum vmballoon_page_size_type page_size) | |
590 | { | |
591 | int i; | |
592 | ||
593 | for (i = 0; i < vmballoon_page_in_frames(page_size); i++) | |
594 | __SetPageOffline(page + i); | |
595 | } | |
596 | ||
597 | /** | |
598 | * vmballoon_mark_page_online() - mark a page as online | |
599 | * @page: pointer for the page. | |
600 | * @page_size: the size of the page. | |
601 | */ | |
602 | static void | |
603 | vmballoon_mark_page_online(struct page *page, | |
604 | enum vmballoon_page_size_type page_size) | |
605 | { | |
606 | int i; | |
607 | ||
608 | for (i = 0; i < vmballoon_page_in_frames(page_size); i++) | |
609 | __ClearPageOffline(page + i); | |
610 | } | |
611 | ||
0395be3e NA |
612 | /** |
613 | * vmballoon_send_get_target() - Retrieve desired balloon size from the host. | |
614 | * | |
615 | * @b: pointer to the balloon. | |
616 | * | |
617 | * Return: zero on success, EINVAL if limit does not fit in 32-bit, as required | |
618 | * by the host-guest protocol and EIO if an error occurred in communicating with | |
619 | * the host. | |
453dc659 | 620 | */ |
0395be3e | 621 | static int vmballoon_send_get_target(struct vmballoon *b) |
453dc659 DT |
622 | { |
623 | unsigned long status; | |
453dc659 | 624 | unsigned long limit; |
453dc659 | 625 | |
ca79b0c2 | 626 | limit = totalram_pages(); |
453dc659 | 627 | |
55398302 XD |
628 | /* Ensure limit fits in 32-bits if 64-bit targets are not supported */ |
629 | if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) && | |
630 | limit != (u32)limit) | |
0395be3e | 631 | return -EINVAL; |
453dc659 | 632 | |
10a95d5d NA |
633 | status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0); |
634 | ||
0395be3e | 635 | return status == VMW_BALLOON_SUCCESS ? 0 : -EIO; |
453dc659 DT |
636 | } |
637 | ||
6e4453b3 NA |
638 | /** |
639 | * vmballoon_alloc_page_list - allocates a list of pages. | |
640 | * | |
641 | * @b: pointer to the balloon. | |
642 | * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation. | |
643 | * @req_n_pages: the number of requested pages. | |
644 | * | |
645 | * Tries to allocate @req_n_pages. Add them to the list of balloon pages in | |
646 | * @ctl.pages and updates @ctl.n_pages to reflect the number of pages. | |
647 | * | |
648 | * Return: zero on success or error code otherwise. | |
649 | */ | |
650 | static int vmballoon_alloc_page_list(struct vmballoon *b, | |
651 | struct vmballoon_ctl *ctl, | |
652 | unsigned int req_n_pages) | |
365bd7ef | 653 | { |
6e4453b3 NA |
654 | struct page *page; |
655 | unsigned int i; | |
365bd7ef | 656 | |
6e4453b3 | 657 | for (i = 0; i < req_n_pages; i++) { |
ae29783d NA |
658 | /* |
659 | * First check if we happen to have pages that were allocated | |
660 | * before. This happens when 2MB page rejected during inflation | |
661 | * by the hypervisor, and then split into 4KB pages. | |
662 | */ | |
663 | if (!list_empty(&ctl->prealloc_pages)) { | |
664 | page = list_first_entry(&ctl->prealloc_pages, | |
665 | struct page, lru); | |
666 | list_del(&page->lru); | |
667 | } else { | |
668 | if (ctl->page_size == VMW_BALLOON_2M_PAGE) | |
669 | page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN| | |
83a8afa7 | 670 | __GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER); |
ae29783d NA |
671 | else |
672 | page = balloon_page_alloc(); | |
365bd7ef | 673 | |
ae29783d NA |
674 | vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC, |
675 | ctl->page_size); | |
676 | } | |
6e4453b3 NA |
677 | |
678 | if (page) { | |
679 | /* Success. Add the page to the list and continue. */ | |
680 | list_add(&page->lru, &ctl->pages); | |
681 | continue; | |
682 | } | |
683 | ||
684 | /* Allocation failed. Update statistics and stop. */ | |
685 | vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL, | |
686 | ctl->page_size); | |
687 | break; | |
688 | } | |
689 | ||
690 | ctl->n_pages = i; | |
691 | ||
692 | return req_n_pages == ctl->n_pages ? 0 : -ENOMEM; | |
365bd7ef PM |
693 | } |
694 | ||
6e4453b3 NA |
695 | /** |
696 | * vmballoon_handle_one_result - Handle lock/unlock result for a single page. | |
697 | * | |
698 | * @b: pointer for %struct vmballoon. | |
699 | * @page: pointer for the page whose result should be handled. | |
700 | * @page_size: size of the page. | |
701 | * @status: status of the operation as provided by the hypervisor. | |
453dc659 | 702 | */ |
6e4453b3 NA |
703 | static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page, |
704 | enum vmballoon_page_size_type page_size, | |
705 | unsigned long status) | |
453dc659 | 706 | { |
6e4453b3 NA |
707 | /* On success do nothing. The page is already on the balloon list. */ |
708 | if (likely(status == VMW_BALLOON_SUCCESS)) | |
709 | return 0; | |
453dc659 | 710 | |
6e4453b3 NA |
711 | pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__, |
712 | page_to_pfn(page), status, | |
713 | vmballoon_page_size_names[page_size]); | |
714 | ||
715 | /* Error occurred */ | |
716 | vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC, | |
717 | page_size); | |
718 | ||
719 | return -EIO; | |
453dc659 DT |
720 | } |
721 | ||
df8d0d42 NA |
722 | /** |
723 | * vmballoon_status_page - returns the status of (un)lock operation | |
724 | * | |
725 | * @b: pointer to the balloon. | |
726 | * @idx: index for the page for which the operation is performed. | |
727 | * @p: pointer to where the page struct is returned. | |
728 | * | |
729 | * Following a lock or unlock operation, returns the status of the operation for | |
730 | * an individual page. Provides the page that the operation was performed on on | |
731 | * the @page argument. | |
732 | * | |
733 | * Returns: The status of a lock or unlock operation for an individual page. | |
453dc659 | 734 | */ |
df8d0d42 NA |
735 | static unsigned long vmballoon_status_page(struct vmballoon *b, int idx, |
736 | struct page **p) | |
453dc659 | 737 | { |
df8d0d42 NA |
738 | if (static_branch_likely(&vmw_balloon_batching)) { |
739 | /* batching mode */ | |
740 | *p = pfn_to_page(b->batch_page[idx].pfn); | |
741 | return b->batch_page[idx].status; | |
742 | } | |
10a95d5d | 743 | |
df8d0d42 NA |
744 | /* non-batching mode */ |
745 | *p = b->page; | |
453dc659 | 746 | |
df8d0d42 NA |
747 | /* |
748 | * If a failure occurs, the indication will be provided in the status | |
749 | * of the entire operation, which is considered before the individual | |
750 | * page status. So for non-batching mode, the indication is always of | |
751 | * success. | |
752 | */ | |
753 | return VMW_BALLOON_SUCCESS; | |
754 | } | |
453dc659 | 755 | |
df8d0d42 NA |
756 | /** |
757 | * vmballoon_lock_op - notifies the host about inflated/deflated pages. | |
758 | * @b: pointer to the balloon. | |
759 | * @num_pages: number of inflated/deflated pages. | |
6e4453b3 NA |
760 | * @page_size: size of the page. |
761 | * @op: the type of operation (lock or unlock). | |
df8d0d42 NA |
762 | * |
763 | * Notify the host about page(s) that were ballooned (or removed from the | |
764 | * balloon) so that host can use it without fear that guest will need it (or | |
765 | * stop using them since the VM does). Host may reject some pages, we need to | |
766 | * check the return value and maybe submit a different page. The pages that are | |
767 | * inflated/deflated are pointed by @b->page. | |
768 | * | |
769 | * Return: result as provided by the hypervisor. | |
770 | */ | |
771 | static unsigned long vmballoon_lock_op(struct vmballoon *b, | |
772 | unsigned int num_pages, | |
6e4453b3 NA |
773 | enum vmballoon_page_size_type page_size, |
774 | enum vmballoon_op op) | |
df8d0d42 NA |
775 | { |
776 | unsigned long cmd, pfn; | |
777 | ||
6e4453b3 NA |
778 | lockdep_assert_held(&b->comm_lock); |
779 | ||
df8d0d42 | 780 | if (static_branch_likely(&vmw_balloon_batching)) { |
6e4453b3 NA |
781 | if (op == VMW_BALLOON_INFLATE) |
782 | cmd = page_size == VMW_BALLOON_2M_PAGE ? | |
783 | VMW_BALLOON_CMD_BATCHED_2M_LOCK : | |
784 | VMW_BALLOON_CMD_BATCHED_LOCK; | |
df8d0d42 | 785 | else |
6e4453b3 NA |
786 | cmd = page_size == VMW_BALLOON_2M_PAGE ? |
787 | VMW_BALLOON_CMD_BATCHED_2M_UNLOCK : | |
788 | VMW_BALLOON_CMD_BATCHED_UNLOCK; | |
df8d0d42 NA |
789 | |
790 | pfn = PHYS_PFN(virt_to_phys(b->batch_page)); | |
791 | } else { | |
6e4453b3 NA |
792 | cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK : |
793 | VMW_BALLOON_CMD_UNLOCK; | |
df8d0d42 NA |
794 | pfn = page_to_pfn(b->page); |
795 | ||
796 | /* In non-batching mode, PFNs must fit in 32-bit */ | |
797 | if (unlikely(pfn != (u32)pfn)) | |
798 | return VMW_BALLOON_ERROR_PPN_INVALID; | |
ef0f8f11 | 799 | } |
453dc659 | 800 | |
df8d0d42 | 801 | return vmballoon_cmd(b, cmd, pfn, num_pages); |
453dc659 DT |
802 | } |
803 | ||
6e4453b3 NA |
804 | /** |
805 | * vmballoon_add_page - adds a page towards lock/unlock operation. | |
806 | * | |
807 | * @b: pointer to the balloon. | |
808 | * @idx: index of the page to be ballooned in this batch. | |
809 | * @p: pointer to the page that is about to be ballooned. | |
810 | * | |
811 | * Adds the page to be ballooned. Must be called while holding @comm_lock. | |
812 | */ | |
813 | static void vmballoon_add_page(struct vmballoon *b, unsigned int idx, | |
814 | struct page *p) | |
815 | { | |
816 | lockdep_assert_held(&b->comm_lock); | |
817 | ||
818 | if (static_branch_likely(&vmw_balloon_batching)) | |
819 | b->batch_page[idx] = (struct vmballoon_batch_entry) | |
820 | { .pfn = page_to_pfn(p) }; | |
821 | else | |
822 | b->page = p; | |
823 | } | |
824 | ||
825 | /** | |
826 | * vmballoon_lock - lock or unlock a batch of pages. | |
827 | * | |
828 | * @b: pointer to the balloon. | |
829 | * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation. | |
830 | * | |
831 | * Notifies the host of about ballooned pages (after inflation or deflation, | |
832 | * according to @ctl). If the host rejects the page put it on the | |
833 | * @ctl refuse list. These refused page are then released when moving to the | |
834 | * next size of pages. | |
835 | * | |
836 | * Note that we neither free any @page here nor put them back on the ballooned | |
837 | * pages list. Instead we queue it for later processing. We do that for several | |
838 | * reasons. First, we do not want to free the page under the lock. Second, it | |
839 | * allows us to unify the handling of lock and unlock. In the inflate case, the | |
840 | * caller will check if there are too many refused pages and release them. | |
841 | * Although it is not identical to the past behavior, it should not affect | |
842 | * performance. | |
843 | */ | |
844 | static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl) | |
f220a80f | 845 | { |
df8d0d42 | 846 | unsigned long batch_status; |
6e4453b3 NA |
847 | struct page *page; |
848 | unsigned int i, num_pages; | |
849 | ||
850 | num_pages = ctl->n_pages; | |
851 | if (num_pages == 0) | |
852 | return 0; | |
f220a80f | 853 | |
6e4453b3 NA |
854 | /* communication with the host is done under the communication lock */ |
855 | spin_lock(&b->comm_lock); | |
f220a80f | 856 | |
6e4453b3 NA |
857 | i = 0; |
858 | list_for_each_entry(page, &ctl->pages, lru) | |
859 | vmballoon_add_page(b, i++, page); | |
860 | ||
861 | batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size, | |
862 | ctl->op); | |
863 | ||
864 | /* | |
865 | * Iterate over the pages in the provided list. Since we are changing | |
866 | * @ctl->n_pages we are saving the original value in @num_pages and | |
867 | * use this value to bound the loop. | |
868 | */ | |
f220a80f | 869 | for (i = 0; i < num_pages; i++) { |
df8d0d42 | 870 | unsigned long status; |
f220a80f | 871 | |
6e4453b3 | 872 | status = vmballoon_status_page(b, i, &page); |
df8d0d42 NA |
873 | |
874 | /* | |
875 | * Failure of the whole batch overrides a single operation | |
876 | * results. | |
877 | */ | |
878 | if (batch_status != VMW_BALLOON_SUCCESS) | |
879 | status = batch_status; | |
f220a80f | 880 | |
6e4453b3 NA |
881 | /* Continue if no error happened */ |
882 | if (!vmballoon_handle_one_result(b, page, ctl->page_size, | |
883 | status)) | |
df8d0d42 | 884 | continue; |
df8d0d42 | 885 | |
8fa3c61a | 886 | /* |
6e4453b3 NA |
887 | * Error happened. Move the pages to the refused list and update |
888 | * the pages number. | |
8fa3c61a | 889 | */ |
6e4453b3 NA |
890 | list_move(&page->lru, &ctl->refused_pages); |
891 | ctl->n_pages--; | |
892 | ctl->n_refused_pages++; | |
f220a80f XD |
893 | } |
894 | ||
6e4453b3 NA |
895 | spin_unlock(&b->comm_lock); |
896 | ||
df8d0d42 | 897 | return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO; |
f220a80f XD |
898 | } |
899 | ||
6e4453b3 NA |
900 | /** |
901 | * vmballoon_release_page_list() - Releases a page list | |
902 | * | |
903 | * @page_list: list of pages to release. | |
904 | * @n_pages: pointer to the number of pages. | |
905 | * @page_size: whether the pages in the list are 2MB (or else 4KB). | |
906 | * | |
907 | * Releases the list of pages and zeros the number of pages. | |
453dc659 | 908 | */ |
6e4453b3 NA |
909 | static void vmballoon_release_page_list(struct list_head *page_list, |
910 | int *n_pages, | |
911 | enum vmballoon_page_size_type page_size) | |
453dc659 | 912 | { |
6e4453b3 | 913 | struct page *page, *tmp; |
df8d0d42 | 914 | |
6e4453b3 NA |
915 | list_for_each_entry_safe(page, tmp, page_list, lru) { |
916 | list_del(&page->lru); | |
917 | __free_pages(page, vmballoon_page_order(page_size)); | |
f220a80f XD |
918 | } |
919 | ||
ae29783d NA |
920 | if (n_pages) |
921 | *n_pages = 0; | |
f220a80f XD |
922 | } |
923 | ||
6e4453b3 | 924 | |
453dc659 DT |
925 | /* |
926 | * Release pages that were allocated while attempting to inflate the | |
927 | * balloon but were refused by the host for one reason or another. | |
928 | */ | |
365bd7ef | 929 | static void vmballoon_release_refused_pages(struct vmballoon *b, |
6e4453b3 | 930 | struct vmballoon_ctl *ctl) |
453dc659 | 931 | { |
6e4453b3 NA |
932 | vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE, |
933 | ctl->page_size); | |
453dc659 | 934 | |
6e4453b3 NA |
935 | vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages, |
936 | ctl->page_size); | |
f220a80f XD |
937 | } |
938 | ||
8b079cd0 NA |
939 | /** |
940 | * vmballoon_change - retrieve the required balloon change | |
941 | * | |
942 | * @b: pointer for the balloon. | |
943 | * | |
944 | * Return: the required change for the balloon size. A positive number | |
945 | * indicates inflation, a negative number indicates a deflation. | |
946 | */ | |
947 | static int64_t vmballoon_change(struct vmballoon *b) | |
948 | { | |
949 | int64_t size, target; | |
950 | ||
6e4453b3 NA |
951 | size = atomic64_read(&b->size); |
952 | target = READ_ONCE(b->target); | |
8b079cd0 NA |
953 | |
954 | /* | |
955 | * We must cast first because of int sizes | |
956 | * Otherwise we might get huge positives instead of negatives | |
957 | */ | |
958 | ||
959 | if (b->reset_required) | |
960 | return 0; | |
961 | ||
962 | /* consider a 2MB slack on deflate, unless the balloon is emptied */ | |
6e4453b3 NA |
963 | if (target < size && target != 0 && |
964 | size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)) | |
8b079cd0 NA |
965 | return 0; |
966 | ||
5d1a86ec NA |
967 | /* If an out-of-memory recently occurred, inflation is disallowed. */ |
968 | if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout))) | |
969 | return 0; | |
970 | ||
8b079cd0 NA |
971 | return target - size; |
972 | } | |
973 | ||
6e4453b3 NA |
974 | /** |
975 | * vmballoon_enqueue_page_list() - Enqueues list of pages after inflation. | |
976 | * | |
977 | * @b: pointer to balloon. | |
978 | * @pages: list of pages to enqueue. | |
979 | * @n_pages: pointer to number of pages in list. The value is zeroed. | |
980 | * @page_size: whether the pages are 2MB or 4KB pages. | |
981 | * | |
982 | * Enqueues the provides list of pages in the ballooned page list, clears the | |
983 | * list and zeroes the number of pages that was provided. | |
984 | */ | |
985 | static void vmballoon_enqueue_page_list(struct vmballoon *b, | |
986 | struct list_head *pages, | |
987 | unsigned int *n_pages, | |
988 | enum vmballoon_page_size_type page_size) | |
989 | { | |
83a8afa7 | 990 | unsigned long flags; |
468e0ffa | 991 | struct page *page; |
83a8afa7 NA |
992 | |
993 | if (page_size == VMW_BALLOON_4K_PAGE) { | |
994 | balloon_page_list_enqueue(&b->b_dev_info, pages); | |
995 | } else { | |
996 | /* | |
997 | * Keep the huge pages in a local list which is not available | |
998 | * for the balloon compaction mechanism. | |
999 | */ | |
1000 | spin_lock_irqsave(&b->b_dev_info.pages_lock, flags); | |
468e0ffa NA |
1001 | |
1002 | list_for_each_entry(page, pages, lru) { | |
1003 | vmballoon_mark_page_offline(page, VMW_BALLOON_2M_PAGE); | |
1004 | } | |
1005 | ||
83a8afa7 NA |
1006 | list_splice_init(pages, &b->huge_pages); |
1007 | __count_vm_events(BALLOON_INFLATE, *n_pages * | |
1008 | vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)); | |
1009 | spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags); | |
1010 | } | |
6e4453b3 | 1011 | |
6e4453b3 NA |
1012 | *n_pages = 0; |
1013 | } | |
1014 | ||
1015 | /** | |
1016 | * vmballoon_dequeue_page_list() - Dequeues page lists for deflation. | |
1017 | * | |
1018 | * @b: pointer to balloon. | |
1019 | * @pages: list of pages to enqueue. | |
1020 | * @n_pages: pointer to number of pages in list. The value is zeroed. | |
1021 | * @page_size: whether the pages are 2MB or 4KB pages. | |
1022 | * @n_req_pages: the number of requested pages. | |
1023 | * | |
1024 | * Dequeues the number of requested pages from the balloon for deflation. The | |
1025 | * number of dequeued pages may be lower, if not enough pages in the requested | |
1026 | * size are available. | |
1027 | */ | |
1028 | static void vmballoon_dequeue_page_list(struct vmballoon *b, | |
1029 | struct list_head *pages, | |
1030 | unsigned int *n_pages, | |
1031 | enum vmballoon_page_size_type page_size, | |
1032 | unsigned int n_req_pages) | |
1033 | { | |
6e4453b3 NA |
1034 | struct page *page, *tmp; |
1035 | unsigned int i = 0; | |
83a8afa7 | 1036 | unsigned long flags; |
6e4453b3 | 1037 | |
83a8afa7 NA |
1038 | /* In the case of 4k pages, use the compaction infrastructure */ |
1039 | if (page_size == VMW_BALLOON_4K_PAGE) { | |
1040 | *n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages, | |
1041 | n_req_pages); | |
1042 | return; | |
1043 | } | |
1044 | ||
1045 | /* 2MB pages */ | |
1046 | spin_lock_irqsave(&b->b_dev_info.pages_lock, flags); | |
1047 | list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) { | |
468e0ffa NA |
1048 | vmballoon_mark_page_online(page, VMW_BALLOON_2M_PAGE); |
1049 | ||
6e4453b3 NA |
1050 | list_move(&page->lru, pages); |
1051 | if (++i == n_req_pages) | |
1052 | break; | |
1053 | } | |
83a8afa7 NA |
1054 | |
1055 | __count_vm_events(BALLOON_DEFLATE, | |
1056 | i * vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)); | |
1057 | spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags); | |
6e4453b3 NA |
1058 | *n_pages = i; |
1059 | } | |
1060 | ||
ae29783d NA |
1061 | /** |
1062 | * vmballoon_split_refused_pages() - Split the 2MB refused pages to 4k. | |
1063 | * | |
1064 | * If inflation of 2MB pages was denied by the hypervisor, it is likely to be | |
1065 | * due to one or few 4KB pages. These 2MB pages may keep being allocated and | |
1066 | * then being refused. To prevent this case, this function splits the refused | |
1067 | * pages into 4KB pages and adds them into @prealloc_pages list. | |
1068 | * | |
1069 | * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation. | |
1070 | */ | |
1071 | static void vmballoon_split_refused_pages(struct vmballoon_ctl *ctl) | |
1072 | { | |
1073 | struct page *page, *tmp; | |
1074 | unsigned int i, order; | |
1075 | ||
1076 | order = vmballoon_page_order(ctl->page_size); | |
1077 | ||
1078 | list_for_each_entry_safe(page, tmp, &ctl->refused_pages, lru) { | |
1079 | list_del(&page->lru); | |
1080 | split_page(page, order); | |
1081 | for (i = 0; i < (1 << order); i++) | |
1082 | list_add(&page[i].lru, &ctl->prealloc_pages); | |
1083 | } | |
1084 | ctl->n_refused_pages = 0; | |
1085 | } | |
1086 | ||
6e4453b3 NA |
1087 | /** |
1088 | * vmballoon_inflate() - Inflate the balloon towards its target size. | |
1089 | * | |
1090 | * @b: pointer to the balloon. | |
453dc659 DT |
1091 | */ |
1092 | static void vmballoon_inflate(struct vmballoon *b) | |
1093 | { | |
6e4453b3 NA |
1094 | int64_t to_inflate_frames; |
1095 | struct vmballoon_ctl ctl = { | |
1096 | .pages = LIST_HEAD_INIT(ctl.pages), | |
1097 | .refused_pages = LIST_HEAD_INIT(ctl.refused_pages), | |
ae29783d | 1098 | .prealloc_pages = LIST_HEAD_INIT(ctl.prealloc_pages), |
6e4453b3 NA |
1099 | .page_size = b->max_page_size, |
1100 | .op = VMW_BALLOON_INFLATE | |
1101 | }; | |
453dc659 | 1102 | |
6e4453b3 NA |
1103 | while ((to_inflate_frames = vmballoon_change(b)) > 0) { |
1104 | unsigned int to_inflate_pages, page_in_frames; | |
1105 | int alloc_error, lock_error = 0; | |
453dc659 | 1106 | |
6e4453b3 NA |
1107 | VM_BUG_ON(!list_empty(&ctl.pages)); |
1108 | VM_BUG_ON(ctl.n_pages != 0); | |
453dc659 | 1109 | |
6e4453b3 | 1110 | page_in_frames = vmballoon_page_in_frames(ctl.page_size); |
453dc659 | 1111 | |
6e4453b3 NA |
1112 | to_inflate_pages = min_t(unsigned long, b->batch_max_pages, |
1113 | DIV_ROUND_UP_ULL(to_inflate_frames, | |
1114 | page_in_frames)); | |
1115 | ||
1116 | /* Start by allocating */ | |
1117 | alloc_error = vmballoon_alloc_page_list(b, &ctl, | |
1118 | to_inflate_pages); | |
1119 | ||
1120 | /* Actually lock the pages by telling the hypervisor */ | |
1121 | lock_error = vmballoon_lock(b, &ctl); | |
1122 | ||
1123 | /* | |
1124 | * If an error indicates that something serious went wrong, | |
1125 | * stop the inflation. | |
1126 | */ | |
1127 | if (lock_error) | |
622074a9 | 1128 | break; |
453dc659 | 1129 | |
6e4453b3 NA |
1130 | /* Update the balloon size */ |
1131 | atomic64_add(ctl.n_pages * page_in_frames, &b->size); | |
8fa3c61a | 1132 | |
6e4453b3 NA |
1133 | vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages, |
1134 | ctl.page_size); | |
10a95d5d | 1135 | |
6e4453b3 NA |
1136 | /* |
1137 | * If allocation failed or the number of refused pages exceeds | |
1138 | * the maximum allowed, move to the next page size. | |
1139 | */ | |
1140 | if (alloc_error || | |
1141 | ctl.n_refused_pages >= VMW_BALLOON_MAX_REFUSED) { | |
1142 | if (ctl.page_size == VMW_BALLOON_4K_PAGE) | |
1143 | break; | |
8fa3c61a NA |
1144 | |
1145 | /* | |
ae29783d NA |
1146 | * Split the refused pages to 4k. This will also empty |
1147 | * the refused pages list. | |
8fa3c61a | 1148 | */ |
ae29783d | 1149 | vmballoon_split_refused_pages(&ctl); |
6e4453b3 | 1150 | ctl.page_size--; |
f220a80f | 1151 | } |
ef0f8f11 | 1152 | |
33d268ed | 1153 | cond_resched(); |
453dc659 DT |
1154 | } |
1155 | ||
6e4453b3 NA |
1156 | /* |
1157 | * Release pages that were allocated while attempting to inflate the | |
1158 | * balloon but were refused by the host for one reason or another, | |
1159 | * and update the statistics. | |
1160 | */ | |
1161 | if (ctl.n_refused_pages != 0) | |
1162 | vmballoon_release_refused_pages(b, &ctl); | |
ae29783d NA |
1163 | |
1164 | vmballoon_release_page_list(&ctl.prealloc_pages, NULL, ctl.page_size); | |
453dc659 DT |
1165 | } |
1166 | ||
6e4453b3 NA |
1167 | /** |
1168 | * vmballoon_deflate() - Decrease the size of the balloon. | |
1169 | * | |
1170 | * @b: pointer to the balloon | |
1171 | * @n_frames: the number of frames to deflate. If zero, automatically | |
1172 | * calculated according to the target size. | |
1173 | * @coordinated: whether to coordinate with the host | |
1174 | * | |
453dc659 | 1175 | * Decrease the size of the balloon allowing guest to use more memory. |
6e4453b3 NA |
1176 | * |
1177 | * Return: The number of deflated frames (i.e., basic page size units) | |
453dc659 | 1178 | */ |
6e4453b3 NA |
1179 | static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames, |
1180 | bool coordinated) | |
453dc659 | 1181 | { |
6e4453b3 NA |
1182 | unsigned long deflated_frames = 0; |
1183 | unsigned long tried_frames = 0; | |
1184 | struct vmballoon_ctl ctl = { | |
1185 | .pages = LIST_HEAD_INIT(ctl.pages), | |
1186 | .refused_pages = LIST_HEAD_INIT(ctl.refused_pages), | |
1187 | .page_size = VMW_BALLOON_4K_PAGE, | |
1188 | .op = VMW_BALLOON_DEFLATE | |
1189 | }; | |
453dc659 | 1190 | |
453dc659 | 1191 | /* free pages to reach target */ |
6e4453b3 NA |
1192 | while (true) { |
1193 | unsigned int to_deflate_pages, n_unlocked_frames; | |
1194 | unsigned int page_in_frames; | |
1195 | int64_t to_deflate_frames; | |
1196 | bool deflated_all; | |
1197 | ||
1198 | page_in_frames = vmballoon_page_in_frames(ctl.page_size); | |
1199 | ||
1200 | VM_BUG_ON(!list_empty(&ctl.pages)); | |
1201 | VM_BUG_ON(ctl.n_pages); | |
1202 | VM_BUG_ON(!list_empty(&ctl.refused_pages)); | |
1203 | VM_BUG_ON(ctl.n_refused_pages); | |
1204 | ||
1205 | /* | |
1206 | * If we were requested a specific number of frames, we try to | |
1207 | * deflate this number of frames. Otherwise, deflation is | |
1208 | * performed according to the target and balloon size. | |
1209 | */ | |
1210 | to_deflate_frames = n_frames ? n_frames - tried_frames : | |
1211 | -vmballoon_change(b); | |
1212 | ||
1213 | /* break if no work to do */ | |
1214 | if (to_deflate_frames <= 0) | |
1215 | break; | |
1216 | ||
1217 | /* | |
1218 | * Calculate the number of frames based on current page size, | |
1219 | * but limit the deflated frames to a single chunk | |
1220 | */ | |
1221 | to_deflate_pages = min_t(unsigned long, b->batch_max_pages, | |
1222 | DIV_ROUND_UP_ULL(to_deflate_frames, | |
1223 | page_in_frames)); | |
1224 | ||
1225 | /* First take the pages from the balloon pages. */ | |
1226 | vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages, | |
1227 | ctl.page_size, to_deflate_pages); | |
f220a80f | 1228 | |
6e4453b3 NA |
1229 | /* |
1230 | * Before pages are moving to the refused list, count their | |
1231 | * frames as frames that we tried to deflate. | |
1232 | */ | |
1233 | tried_frames += ctl.n_pages * page_in_frames; | |
1234 | ||
1235 | /* | |
1236 | * Unlock the pages by communicating with the hypervisor if the | |
1237 | * communication is coordinated (i.e., not pop). We ignore the | |
1238 | * return code. Instead we check if all the pages we manage to | |
1239 | * unlock all the pages. If we failed, we will move to the next | |
1240 | * page size, and would eventually try again later. | |
1241 | */ | |
1242 | if (coordinated) | |
1243 | vmballoon_lock(b, &ctl); | |
1244 | ||
1245 | /* | |
1246 | * Check if we deflated enough. We will move to the next page | |
1247 | * size if we did not manage to do so. This calculation takes | |
1248 | * place now, as once the pages are released, the number of | |
1249 | * pages is zeroed. | |
1250 | */ | |
1251 | deflated_all = (ctl.n_pages == to_deflate_pages); | |
33d268ed | 1252 | |
6e4453b3 NA |
1253 | /* Update local and global counters */ |
1254 | n_unlocked_frames = ctl.n_pages * page_in_frames; | |
1255 | atomic64_sub(n_unlocked_frames, &b->size); | |
1256 | deflated_frames += n_unlocked_frames; | |
453dc659 | 1257 | |
6e4453b3 NA |
1258 | vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE, |
1259 | ctl.page_size, ctl.n_pages); | |
33d268ed | 1260 | |
6e4453b3 NA |
1261 | /* free the ballooned pages */ |
1262 | vmballoon_release_page_list(&ctl.pages, &ctl.n_pages, | |
1263 | ctl.page_size); | |
1264 | ||
1265 | /* Return the refused pages to the ballooned list. */ | |
1266 | vmballoon_enqueue_page_list(b, &ctl.refused_pages, | |
1267 | &ctl.n_refused_pages, | |
1268 | ctl.page_size); | |
1269 | ||
1270 | /* If we failed to unlock all the pages, move to next size. */ | |
1271 | if (!deflated_all) { | |
1272 | if (ctl.page_size == b->max_page_size) | |
1273 | break; | |
1274 | ctl.page_size++; | |
365bd7ef | 1275 | } |
453dc659 | 1276 | |
6e4453b3 | 1277 | cond_resched(); |
365bd7ef | 1278 | } |
6e4453b3 NA |
1279 | |
1280 | return deflated_frames; | |
f220a80f XD |
1281 | } |
1282 | ||
df8d0d42 NA |
1283 | /** |
1284 | * vmballoon_deinit_batching - disables batching mode. | |
1285 | * | |
1286 | * @b: pointer to &struct vmballoon. | |
1287 | * | |
1288 | * Disables batching, by deallocating the page for communication with the | |
1289 | * hypervisor and disabling the static key to indicate that batching is off. | |
1290 | */ | |
1291 | static void vmballoon_deinit_batching(struct vmballoon *b) | |
1292 | { | |
1293 | free_page((unsigned long)b->batch_page); | |
1294 | b->batch_page = NULL; | |
1295 | static_branch_disable(&vmw_balloon_batching); | |
1296 | b->batch_max_pages = 1; | |
1297 | } | |
f220a80f | 1298 | |
df8d0d42 NA |
1299 | /** |
1300 | * vmballoon_init_batching - enable batching mode. | |
1301 | * | |
1302 | * @b: pointer to &struct vmballoon. | |
1303 | * | |
1304 | * Enables batching, by allocating a page for communication with the hypervisor | |
1305 | * and enabling the static_key to use batching. | |
1306 | * | |
1307 | * Return: zero on success or an appropriate error-code. | |
1308 | */ | |
1309 | static int vmballoon_init_batching(struct vmballoon *b) | |
f220a80f | 1310 | { |
b23220fe | 1311 | struct page *page; |
f220a80f | 1312 | |
b23220fe GK |
1313 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); |
1314 | if (!page) | |
df8d0d42 | 1315 | return -ENOMEM; |
f220a80f | 1316 | |
b23220fe | 1317 | b->batch_page = page_address(page); |
df8d0d42 NA |
1318 | b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry); |
1319 | ||
1320 | static_branch_enable(&vmw_balloon_batching); | |
1321 | ||
1322 | return 0; | |
f220a80f XD |
1323 | } |
1324 | ||
48e3d668 PM |
1325 | /* |
1326 | * Receive notification and resize balloon | |
1327 | */ | |
1328 | static void vmballoon_doorbell(void *client_data) | |
1329 | { | |
1330 | struct vmballoon *b = client_data; | |
1331 | ||
c7b3690f | 1332 | vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL); |
48e3d668 PM |
1333 | |
1334 | mod_delayed_work(system_freezable_wq, &b->dwork, 0); | |
1335 | } | |
1336 | ||
1337 | /* | |
1338 | * Clean up vmci doorbell | |
1339 | */ | |
1340 | static void vmballoon_vmci_cleanup(struct vmballoon *b) | |
1341 | { | |
10a95d5d NA |
1342 | vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET, |
1343 | VMCI_INVALID_ID, VMCI_INVALID_ID); | |
48e3d668 | 1344 | |
48e3d668 PM |
1345 | if (!vmci_handle_is_invalid(b->vmci_doorbell)) { |
1346 | vmci_doorbell_destroy(b->vmci_doorbell); | |
1347 | b->vmci_doorbell = VMCI_INVALID_HANDLE; | |
1348 | } | |
1349 | } | |
1350 | ||
22d293ee NA |
1351 | /** |
1352 | * vmballoon_vmci_init - Initialize vmci doorbell. | |
1353 | * | |
1354 | * @b: pointer to the balloon. | |
1355 | * | |
1356 | * Return: zero on success or when wakeup command not supported. Error-code | |
1357 | * otherwise. | |
1358 | * | |
1359 | * Initialize vmci doorbell, to get notified as soon as balloon changes. | |
48e3d668 PM |
1360 | */ |
1361 | static int vmballoon_vmci_init(struct vmballoon *b) | |
1362 | { | |
10a95d5d | 1363 | unsigned long error; |
48e3d668 | 1364 | |
ce664331 NA |
1365 | if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0) |
1366 | return 0; | |
48e3d668 | 1367 | |
ce664331 NA |
1368 | error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB, |
1369 | VMCI_PRIVILEGE_FLAG_RESTRICTED, | |
1370 | vmballoon_doorbell, b); | |
48e3d668 | 1371 | |
ce664331 NA |
1372 | if (error != VMCI_SUCCESS) |
1373 | goto fail; | |
1374 | ||
10a95d5d NA |
1375 | error = __vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET, |
1376 | b->vmci_doorbell.context, | |
1377 | b->vmci_doorbell.resource, NULL); | |
ce664331 | 1378 | |
ce664331 NA |
1379 | if (error != VMW_BALLOON_SUCCESS) |
1380 | goto fail; | |
48e3d668 PM |
1381 | |
1382 | return 0; | |
ce664331 NA |
1383 | fail: |
1384 | vmballoon_vmci_cleanup(b); | |
1385 | return -EIO; | |
48e3d668 PM |
1386 | } |
1387 | ||
6e4453b3 NA |
1388 | /** |
1389 | * vmballoon_pop - Quickly release all pages allocate for the balloon. | |
1390 | * | |
1391 | * @b: pointer to the balloon. | |
1392 | * | |
1393 | * This function is called when host decides to "reset" balloon for one reason | |
1394 | * or another. Unlike normal "deflate" we do not (shall not) notify host of the | |
1395 | * pages being released. | |
1396 | */ | |
1397 | static void vmballoon_pop(struct vmballoon *b) | |
1398 | { | |
1399 | unsigned long size; | |
1400 | ||
1401 | while ((size = atomic64_read(&b->size))) | |
1402 | vmballoon_deflate(b, size, false); | |
1403 | } | |
1404 | ||
f220a80f XD |
1405 | /* |
1406 | * Perform standard reset sequence by popping the balloon (in case it | |
1407 | * is not empty) and then restarting protocol. This operation normally | |
1408 | * happens when host responds with VMW_BALLOON_ERROR_RESET to a command. | |
1409 | */ | |
1410 | static void vmballoon_reset(struct vmballoon *b) | |
1411 | { | |
48e3d668 PM |
1412 | int error; |
1413 | ||
c7b3690f NA |
1414 | down_write(&b->conf_sem); |
1415 | ||
48e3d668 PM |
1416 | vmballoon_vmci_cleanup(b); |
1417 | ||
f220a80f XD |
1418 | /* free all pages, skipping monitor unlock */ |
1419 | vmballoon_pop(b); | |
1420 | ||
22d293ee | 1421 | if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES)) |
d04071a5 | 1422 | goto unlock; |
f220a80f XD |
1423 | |
1424 | if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) { | |
df8d0d42 | 1425 | if (vmballoon_init_batching(b)) { |
f220a80f XD |
1426 | /* |
1427 | * We failed to initialize batching, inform the monitor | |
1428 | * about it by sending a null capability. | |
1429 | * | |
1430 | * The guest will retry in one second. | |
1431 | */ | |
1432 | vmballoon_send_start(b, 0); | |
d04071a5 | 1433 | goto unlock; |
f220a80f XD |
1434 | } |
1435 | } else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) { | |
df8d0d42 | 1436 | vmballoon_deinit_batching(b); |
f220a80f XD |
1437 | } |
1438 | ||
8840a6f4 | 1439 | vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET); |
f220a80f | 1440 | b->reset_required = false; |
48e3d668 PM |
1441 | |
1442 | error = vmballoon_vmci_init(b); | |
1443 | if (error) | |
fd7e92d5 | 1444 | pr_err_once("failed to initialize vmci doorbell\n"); |
48e3d668 | 1445 | |
22d293ee | 1446 | if (vmballoon_send_guest_id(b)) |
fd7e92d5 | 1447 | pr_err_once("failed to send guest ID to the host\n"); |
c7b3690f | 1448 | |
d04071a5 | 1449 | unlock: |
c7b3690f | 1450 | up_write(&b->conf_sem); |
453dc659 DT |
1451 | } |
1452 | ||
8b079cd0 NA |
1453 | /** |
1454 | * vmballoon_work - periodic balloon worker for reset, inflation and deflation. | |
1455 | * | |
1456 | * @work: pointer to the &work_struct which is provided by the workqueue. | |
1457 | * | |
1458 | * Resets the protocol if needed, gets the new size and adjusts balloon as | |
1459 | * needed. Repeat in 1 sec. | |
453dc659 DT |
1460 | */ |
1461 | static void vmballoon_work(struct work_struct *work) | |
1462 | { | |
1463 | struct delayed_work *dwork = to_delayed_work(work); | |
1464 | struct vmballoon *b = container_of(dwork, struct vmballoon, dwork); | |
8b079cd0 | 1465 | int64_t change = 0; |
453dc659 | 1466 | |
453dc659 DT |
1467 | if (b->reset_required) |
1468 | vmballoon_reset(b); | |
1469 | ||
c7b3690f NA |
1470 | down_read(&b->conf_sem); |
1471 | ||
1472 | /* | |
1473 | * Update the stats while holding the semaphore to ensure that | |
1474 | * @stats_enabled is consistent with whether the stats are actually | |
1475 | * enabled | |
1476 | */ | |
1477 | vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER); | |
1478 | ||
0395be3e | 1479 | if (!vmballoon_send_get_target(b)) |
8b079cd0 NA |
1480 | change = vmballoon_change(b); |
1481 | ||
1482 | if (change != 0) { | |
6e4453b3 NA |
1483 | pr_debug("%s - size: %llu, target %lu\n", __func__, |
1484 | atomic64_read(&b->size), READ_ONCE(b->target)); | |
453dc659 | 1485 | |
8b079cd0 | 1486 | if (change > 0) |
453dc659 | 1487 | vmballoon_inflate(b); |
8b079cd0 | 1488 | else /* (change < 0) */ |
6e4453b3 | 1489 | vmballoon_deflate(b, 0, true); |
453dc659 DT |
1490 | } |
1491 | ||
c7b3690f NA |
1492 | up_read(&b->conf_sem); |
1493 | ||
beda94da DT |
1494 | /* |
1495 | * We are using a freezable workqueue so that balloon operations are | |
1496 | * stopped while the system transitions to/from sleep/hibernation. | |
1497 | */ | |
1498 | queue_delayed_work(system_freezable_wq, | |
1499 | dwork, round_jiffies_relative(HZ)); | |
c7b3690f | 1500 | |
453dc659 DT |
1501 | } |
1502 | ||
5d1a86ec NA |
1503 | /** |
1504 | * vmballoon_shrinker_scan() - deflate the balloon due to memory pressure. | |
1505 | * @shrinker: pointer to the balloon shrinker. | |
1506 | * @sc: page reclaim information. | |
1507 | * | |
1508 | * Returns: number of pages that were freed during deflation. | |
1509 | */ | |
1510 | static unsigned long vmballoon_shrinker_scan(struct shrinker *shrinker, | |
1511 | struct shrink_control *sc) | |
1512 | { | |
1513 | struct vmballoon *b = &balloon; | |
1514 | unsigned long deflated_frames; | |
1515 | ||
1516 | pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size)); | |
1517 | ||
1518 | vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK); | |
1519 | ||
1520 | /* | |
1521 | * If the lock is also contended for read, we cannot easily reclaim and | |
1522 | * we bail out. | |
1523 | */ | |
1524 | if (!down_read_trylock(&b->conf_sem)) | |
1525 | return 0; | |
1526 | ||
1527 | deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true); | |
1528 | ||
1529 | vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE, | |
1530 | deflated_frames); | |
1531 | ||
1532 | /* | |
1533 | * Delay future inflation for some time to mitigate the situations in | |
1534 | * which balloon continuously grows and shrinks. Use WRITE_ONCE() since | |
1535 | * the access is asynchronous. | |
1536 | */ | |
1537 | WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY); | |
1538 | ||
1539 | up_read(&b->conf_sem); | |
1540 | ||
1541 | return deflated_frames; | |
1542 | } | |
1543 | ||
1544 | /** | |
1545 | * vmballoon_shrinker_count() - return the number of ballooned pages. | |
1546 | * @shrinker: pointer to the balloon shrinker. | |
1547 | * @sc: page reclaim information. | |
1548 | * | |
1549 | * Returns: number of 4k pages that are allocated for the balloon and can | |
1550 | * therefore be reclaimed under pressure. | |
1551 | */ | |
1552 | static unsigned long vmballoon_shrinker_count(struct shrinker *shrinker, | |
1553 | struct shrink_control *sc) | |
1554 | { | |
1555 | struct vmballoon *b = &balloon; | |
1556 | ||
1557 | return atomic64_read(&b->size); | |
1558 | } | |
1559 | ||
1560 | static void vmballoon_unregister_shrinker(struct vmballoon *b) | |
1561 | { | |
17c4eb03 QZ |
1562 | shrinker_free(b->shrinker); |
1563 | b->shrinker = NULL; | |
5d1a86ec NA |
1564 | } |
1565 | ||
1566 | static int vmballoon_register_shrinker(struct vmballoon *b) | |
1567 | { | |
5d1a86ec NA |
1568 | /* Do nothing if the shrinker is not enabled */ |
1569 | if (!vmwballoon_shrinker_enable) | |
1570 | return 0; | |
1571 | ||
17c4eb03 QZ |
1572 | b->shrinker = shrinker_alloc(0, "vmw-balloon"); |
1573 | if (!b->shrinker) | |
1574 | return -ENOMEM; | |
5d1a86ec | 1575 | |
17c4eb03 QZ |
1576 | b->shrinker->scan_objects = vmballoon_shrinker_scan; |
1577 | b->shrinker->count_objects = vmballoon_shrinker_count; | |
1578 | b->shrinker->private_data = b; | |
5d1a86ec | 1579 | |
17c4eb03 | 1580 | shrinker_register(b->shrinker); |
5d1a86ec | 1581 | |
17c4eb03 | 1582 | return 0; |
5d1a86ec NA |
1583 | } |
1584 | ||
453dc659 DT |
1585 | /* |
1586 | * DEBUGFS Interface | |
1587 | */ | |
1588 | #ifdef CONFIG_DEBUG_FS | |
1589 | ||
c7b3690f NA |
1590 | static const char * const vmballoon_stat_page_names[] = { |
1591 | [VMW_BALLOON_PAGE_STAT_ALLOC] = "alloc", | |
1592 | [VMW_BALLOON_PAGE_STAT_ALLOC_FAIL] = "allocFail", | |
1593 | [VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC] = "errAlloc", | |
1594 | [VMW_BALLOON_PAGE_STAT_REFUSED_FREE] = "errFree", | |
1595 | [VMW_BALLOON_PAGE_STAT_FREE] = "free" | |
1596 | }; | |
1597 | ||
1598 | static const char * const vmballoon_stat_names[] = { | |
1599 | [VMW_BALLOON_STAT_TIMER] = "timer", | |
8840a6f4 NA |
1600 | [VMW_BALLOON_STAT_DOORBELL] = "doorbell", |
1601 | [VMW_BALLOON_STAT_RESET] = "reset", | |
5d1a86ec NA |
1602 | [VMW_BALLOON_STAT_SHRINK] = "shrink", |
1603 | [VMW_BALLOON_STAT_SHRINK_FREE] = "shrinkFree" | |
c7b3690f NA |
1604 | }; |
1605 | ||
c7b3690f NA |
1606 | static int vmballoon_enable_stats(struct vmballoon *b) |
1607 | { | |
1608 | int r = 0; | |
1609 | ||
1610 | down_write(&b->conf_sem); | |
1611 | ||
1612 | /* did we somehow race with another reader which enabled stats? */ | |
1613 | if (b->stats) | |
1614 | goto out; | |
1615 | ||
1616 | b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL); | |
1617 | ||
1618 | if (!b->stats) { | |
1619 | /* allocation failed */ | |
1620 | r = -ENOMEM; | |
1621 | goto out; | |
1622 | } | |
1623 | static_key_enable(&balloon_stat_enabled.key); | |
1624 | out: | |
1625 | up_write(&b->conf_sem); | |
1626 | return r; | |
1627 | } | |
1628 | ||
1629 | /** | |
1630 | * vmballoon_debug_show - shows statistics of balloon operations. | |
1631 | * @f: pointer to the &struct seq_file. | |
1632 | * @offset: ignored. | |
1633 | * | |
1634 | * Provides the statistics that can be accessed in vmmemctl in the debugfs. | |
1635 | * To avoid the overhead - mainly that of memory - of collecting the statistics, | |
1636 | * we only collect statistics after the first time the counters are read. | |
1637 | * | |
1638 | * Return: zero on success or an error code. | |
1639 | */ | |
453dc659 DT |
1640 | static int vmballoon_debug_show(struct seq_file *f, void *offset) |
1641 | { | |
1642 | struct vmballoon *b = f->private; | |
c7b3690f NA |
1643 | int i, j; |
1644 | ||
1645 | /* enables stats if they are disabled */ | |
1646 | if (!b->stats) { | |
1647 | int r = vmballoon_enable_stats(b); | |
1648 | ||
1649 | if (r) | |
1650 | return r; | |
1651 | } | |
453dc659 | 1652 | |
b36e89da | 1653 | /* format capabilities info */ |
6e4453b3 | 1654 | seq_printf(f, "%-22s: %#16x\n", "balloon capabilities", |
c7b3690f | 1655 | VMW_BALLOON_CAPABILITIES); |
6e4453b3 | 1656 | seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities); |
c7b3690f NA |
1657 | seq_printf(f, "%-22s: %16s\n", "is resetting", |
1658 | b->reset_required ? "y" : "n"); | |
b36e89da | 1659 | |
453dc659 | 1660 | /* format size info */ |
6e4453b3 NA |
1661 | seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target)); |
1662 | seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size)); | |
453dc659 | 1663 | |
68131184 NA |
1664 | for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) { |
1665 | if (vmballoon_cmd_names[i] == NULL) | |
1666 | continue; | |
1667 | ||
c7b3690f NA |
1668 | seq_printf(f, "%-22s: %16llu (%llu failed)\n", |
1669 | vmballoon_cmd_names[i], | |
1670 | atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]), | |
1671 | atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT])); | |
68131184 NA |
1672 | } |
1673 | ||
c7b3690f NA |
1674 | for (i = 0; i < VMW_BALLOON_STAT_NUM; i++) |
1675 | seq_printf(f, "%-22s: %16llu\n", | |
1676 | vmballoon_stat_names[i], | |
1677 | atomic64_read(&b->stats->general_stat[i])); | |
1678 | ||
1679 | for (i = 0; i < VMW_BALLOON_PAGE_STAT_NUM; i++) { | |
1680 | for (j = 0; j < VMW_BALLOON_NUM_PAGE_SIZES; j++) | |
1681 | seq_printf(f, "%-18s(%s): %16llu\n", | |
1682 | vmballoon_stat_page_names[i], | |
1683 | vmballoon_page_size_names[j], | |
1684 | atomic64_read(&b->stats->page_stat[i][j])); | |
1685 | } | |
453dc659 DT |
1686 | |
1687 | return 0; | |
1688 | } | |
1689 | ||
2796b43f | 1690 | DEFINE_SHOW_ATTRIBUTE(vmballoon_debug); |
453dc659 | 1691 | |
225afca6 | 1692 | static void __init vmballoon_debugfs_init(struct vmballoon *b) |
453dc659 | 1693 | { |
37b85bd5 GKH |
1694 | debugfs_create_file("vmmemctl", S_IRUGO, NULL, b, |
1695 | &vmballoon_debug_fops); | |
453dc659 DT |
1696 | } |
1697 | ||
1698 | static void __exit vmballoon_debugfs_exit(struct vmballoon *b) | |
1699 | { | |
c7b3690f | 1700 | static_key_disable(&balloon_stat_enabled.key); |
209cdbd0 | 1701 | debugfs_lookup_and_remove("vmmemctl", NULL); |
c7b3690f NA |
1702 | kfree(b->stats); |
1703 | b->stats = NULL; | |
453dc659 DT |
1704 | } |
1705 | ||
1706 | #else | |
1707 | ||
225afca6 | 1708 | static inline void vmballoon_debugfs_init(struct vmballoon *b) |
453dc659 | 1709 | { |
453dc659 DT |
1710 | } |
1711 | ||
1712 | static inline void vmballoon_debugfs_exit(struct vmballoon *b) | |
1713 | { | |
1714 | } | |
1715 | ||
1716 | #endif /* CONFIG_DEBUG_FS */ | |
1717 | ||
83a8afa7 NA |
1718 | |
1719 | #ifdef CONFIG_BALLOON_COMPACTION | |
83a8afa7 NA |
1720 | /** |
1721 | * vmballoon_migratepage() - migrates a balloon page. | |
1722 | * @b_dev_info: balloon device information descriptor. | |
1723 | * @newpage: the page to which @page should be migrated. | |
1724 | * @page: a ballooned page that should be migrated. | |
1725 | * @mode: migration mode, ignored. | |
1726 | * | |
1727 | * This function is really open-coded, but that is according to the interface | |
1728 | * that balloon_compaction provides. | |
1729 | * | |
1730 | * Return: zero on success, -EAGAIN when migration cannot be performed | |
1731 | * momentarily, and -EBUSY if migration failed and should be retried | |
1732 | * with that specific page. | |
1733 | */ | |
1734 | static int vmballoon_migratepage(struct balloon_dev_info *b_dev_info, | |
1735 | struct page *newpage, struct page *page, | |
1736 | enum migrate_mode mode) | |
1737 | { | |
1738 | unsigned long status, flags; | |
1739 | struct vmballoon *b; | |
1740 | int ret; | |
1741 | ||
1742 | b = container_of(b_dev_info, struct vmballoon, b_dev_info); | |
1743 | ||
1744 | /* | |
1745 | * If the semaphore is taken, there is ongoing configuration change | |
1746 | * (i.e., balloon reset), so try again. | |
1747 | */ | |
1748 | if (!down_read_trylock(&b->conf_sem)) | |
1749 | return -EAGAIN; | |
1750 | ||
1751 | spin_lock(&b->comm_lock); | |
1752 | /* | |
1753 | * We must start by deflating and not inflating, as otherwise the | |
1754 | * hypervisor may tell us that it has enough memory and the new page is | |
1755 | * not needed. Since the old page is isolated, we cannot use the list | |
1756 | * interface to unlock it, as the LRU field is used for isolation. | |
1757 | * Instead, we use the native interface directly. | |
1758 | */ | |
1759 | vmballoon_add_page(b, 0, page); | |
1760 | status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE, | |
1761 | VMW_BALLOON_DEFLATE); | |
1762 | ||
1763 | if (status == VMW_BALLOON_SUCCESS) | |
1764 | status = vmballoon_status_page(b, 0, &page); | |
1765 | ||
1766 | /* | |
1767 | * If a failure happened, let the migration mechanism know that it | |
1768 | * should not retry. | |
1769 | */ | |
1770 | if (status != VMW_BALLOON_SUCCESS) { | |
1771 | spin_unlock(&b->comm_lock); | |
1772 | ret = -EBUSY; | |
1773 | goto out_unlock; | |
1774 | } | |
1775 | ||
1776 | /* | |
1777 | * The page is isolated, so it is safe to delete it without holding | |
1778 | * @pages_lock . We keep holding @comm_lock since we will need it in a | |
1779 | * second. | |
1780 | */ | |
1781 | balloon_page_delete(page); | |
1782 | ||
1783 | put_page(page); | |
1784 | ||
1785 | /* Inflate */ | |
1786 | vmballoon_add_page(b, 0, newpage); | |
1787 | status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE, | |
1788 | VMW_BALLOON_INFLATE); | |
1789 | ||
1790 | if (status == VMW_BALLOON_SUCCESS) | |
1791 | status = vmballoon_status_page(b, 0, &newpage); | |
1792 | ||
1793 | spin_unlock(&b->comm_lock); | |
1794 | ||
1795 | if (status != VMW_BALLOON_SUCCESS) { | |
1796 | /* | |
1797 | * A failure happened. While we can deflate the page we just | |
1798 | * inflated, this deflation can also encounter an error. Instead | |
1799 | * we will decrease the size of the balloon to reflect the | |
1800 | * change and report failure. | |
1801 | */ | |
1802 | atomic64_dec(&b->size); | |
1803 | ret = -EBUSY; | |
1804 | } else { | |
1805 | /* | |
1806 | * Success. Take a reference for the page, and we will add it to | |
1807 | * the list after acquiring the lock. | |
1808 | */ | |
1809 | get_page(newpage); | |
1810 | ret = MIGRATEPAGE_SUCCESS; | |
1811 | } | |
1812 | ||
1813 | /* Update the balloon list under the @pages_lock */ | |
1814 | spin_lock_irqsave(&b->b_dev_info.pages_lock, flags); | |
1815 | ||
1816 | /* | |
1817 | * On inflation success, we already took a reference for the @newpage. | |
1818 | * If we succeed just insert it to the list and update the statistics | |
1819 | * under the lock. | |
1820 | */ | |
1821 | if (ret == MIGRATEPAGE_SUCCESS) { | |
1822 | balloon_page_insert(&b->b_dev_info, newpage); | |
1823 | __count_vm_event(BALLOON_MIGRATE); | |
1824 | } | |
1825 | ||
1826 | /* | |
1827 | * We deflated successfully, so regardless to the inflation success, we | |
1828 | * need to reduce the number of isolated_pages. | |
1829 | */ | |
1830 | b->b_dev_info.isolated_pages--; | |
1831 | spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags); | |
1832 | ||
1833 | out_unlock: | |
1834 | up_read(&b->conf_sem); | |
1835 | return ret; | |
1836 | } | |
1837 | ||
83a8afa7 NA |
1838 | /** |
1839 | * vmballoon_compaction_init() - initialized compaction for the balloon. | |
1840 | * | |
1841 | * @b: pointer to the balloon. | |
1842 | * | |
1843 | * If during the initialization a failure occurred, this function does not | |
1844 | * perform cleanup. The caller must call vmballoon_compaction_deinit() in this | |
1845 | * case. | |
1846 | * | |
1847 | * Return: zero on success or error code on failure. | |
1848 | */ | |
68f2736a | 1849 | static __init void vmballoon_compaction_init(struct vmballoon *b) |
83a8afa7 | 1850 | { |
83a8afa7 | 1851 | b->b_dev_info.migratepage = vmballoon_migratepage; |
83a8afa7 NA |
1852 | } |
1853 | ||
1854 | #else /* CONFIG_BALLOON_COMPACTION */ | |
68f2736a | 1855 | static inline void vmballoon_compaction_init(struct vmballoon *b) |
83a8afa7 | 1856 | { |
83a8afa7 | 1857 | } |
83a8afa7 NA |
1858 | #endif /* CONFIG_BALLOON_COMPACTION */ |
1859 | ||
453dc659 DT |
1860 | static int __init vmballoon_init(void) |
1861 | { | |
1862 | int error; | |
6e4453b3 | 1863 | |
453dc659 DT |
1864 | /* |
1865 | * Check if we are running on VMware's hypervisor and bail out | |
1866 | * if we are not. | |
1867 | */ | |
03b2a320 | 1868 | if (x86_hyper_type != X86_HYPER_VMWARE) |
453dc659 DT |
1869 | return -ENODEV; |
1870 | ||
453dc659 DT |
1871 | INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work); |
1872 | ||
5d1a86ec NA |
1873 | error = vmballoon_register_shrinker(&balloon); |
1874 | if (error) | |
17c4eb03 | 1875 | return error; |
5d1a86ec | 1876 | |
83a8afa7 NA |
1877 | /* |
1878 | * Initialization of compaction must be done after the call to | |
1879 | * balloon_devinfo_init() . | |
1880 | */ | |
1881 | balloon_devinfo_init(&balloon.b_dev_info); | |
68f2736a | 1882 | vmballoon_compaction_init(&balloon); |
83a8afa7 NA |
1883 | |
1884 | INIT_LIST_HEAD(&balloon.huge_pages); | |
6e4453b3 | 1885 | spin_lock_init(&balloon.comm_lock); |
c7b3690f | 1886 | init_rwsem(&balloon.conf_sem); |
48e3d668 | 1887 | balloon.vmci_doorbell = VMCI_INVALID_HANDLE; |
d7568c13 PM |
1888 | balloon.batch_page = NULL; |
1889 | balloon.page = NULL; | |
1890 | balloon.reset_required = true; | |
1891 | ||
beda94da | 1892 | queue_delayed_work(system_freezable_wq, &balloon.dwork, 0); |
453dc659 | 1893 | |
f632a817 LT |
1894 | vmballoon_debugfs_init(&balloon); |
1895 | ||
453dc659 | 1896 | return 0; |
453dc659 | 1897 | } |
c3cc1b0f NA |
1898 | |
1899 | /* | |
1900 | * Using late_initcall() instead of module_init() allows the balloon to use the | |
1901 | * VMCI doorbell even when the balloon is built into the kernel. Otherwise the | |
1902 | * VMCI is probed only after the balloon is initialized. If the balloon is used | |
1903 | * as a module, late_initcall() is equivalent to module_init(). | |
1904 | */ | |
1905 | late_initcall(vmballoon_init); | |
453dc659 DT |
1906 | |
1907 | static void __exit vmballoon_exit(void) | |
1908 | { | |
5d1a86ec | 1909 | vmballoon_unregister_shrinker(&balloon); |
48e3d668 | 1910 | vmballoon_vmci_cleanup(&balloon); |
453dc659 | 1911 | cancel_delayed_work_sync(&balloon.dwork); |
453dc659 DT |
1912 | |
1913 | vmballoon_debugfs_exit(&balloon); | |
1914 | ||
1915 | /* | |
1916 | * Deallocate all reserved memory, and reset connection with monitor. | |
1917 | * Reset connection before deallocating memory to avoid potential for | |
1918 | * additional spurious resets from guest touching deallocated pages. | |
1919 | */ | |
d7568c13 | 1920 | vmballoon_send_start(&balloon, 0); |
453dc659 DT |
1921 | vmballoon_pop(&balloon); |
1922 | } | |
1923 | module_exit(vmballoon_exit); |