1 // SPDX-License-Identifier: GPL-2.0
3 * VMware Balloon driver.
5 * Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved.
7 * This is VMware physical memory management driver for Linux. The driver
8 * acts like a "balloon" that can be inflated to reclaim physical pages by
9 * reserving them in the guest and invalidating them in the monitor,
10 * freeing up the underlying machine pages so they can be allocated to
11 * other guests. The balloon can also be deflated to allow the guest to
12 * use more physical memory. Higher level policies can control the sizes
13 * of balloons in VMs in order to manage physical memory resources.
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19 #include <linux/types.h>
20 #include <linux/kernel.h>
22 #include <linux/vmalloc.h>
23 #include <linux/sched.h>
24 #include <linux/module.h>
25 #include <linux/workqueue.h>
26 #include <linux/debugfs.h>
27 #include <linux/seq_file.h>
28 #include <linux/rwsem.h>
29 #include <linux/slab.h>
30 #include <linux/spinlock.h>
31 #include <linux/vmw_vmci_defs.h>
32 #include <linux/vmw_vmci_api.h>
33 #include <asm/hypervisor.h>
35 MODULE_AUTHOR("VMware, Inc.");
36 MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver");
37 MODULE_VERSION("1.5.0.0-k");
38 MODULE_ALIAS("dmi:*:svnVMware*:*");
39 MODULE_ALIAS("vmware_vmmemctl");
40 MODULE_LICENSE("GPL");
43 * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We don't allow wait
44 * (__GFP_RECLAIM) for huge page allocations. Use __GFP_NOWARN, to suppress page
45 * allocation failure warnings. Disallow access to emergency low-memory pools.
47 #define VMW_HUGE_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \
51 * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We allow lightweight
52 * reclamation (__GFP_NORETRY). Use __GFP_NOWARN, to suppress page allocation
53 * failure warnings. Disallow access to emergency low-memory pools.
55 #define VMW_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \
56 __GFP_NOMEMALLOC|__GFP_NORETRY)
58 /* Maximum number of refused pages we accumulate during inflation cycle */
59 #define VMW_BALLOON_MAX_REFUSED 16
62 * Hypervisor communication port definitions.
64 #define VMW_BALLOON_HV_PORT 0x5670
65 #define VMW_BALLOON_HV_MAGIC 0x456c6d6f
66 #define VMW_BALLOON_GUEST_ID 1 /* Linux */
68 enum vmwballoon_capabilities {
70 * Bit 0 is reserved and not associated to any capability.
72 VMW_BALLOON_BASIC_CMDS = (1 << 1),
73 VMW_BALLOON_BATCHED_CMDS = (1 << 2),
74 VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3),
75 VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4),
78 #define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_BASIC_CMDS \
79 | VMW_BALLOON_BATCHED_CMDS \
80 | VMW_BALLOON_BATCHED_2M_CMDS \
81 | VMW_BALLOON_SIGNALLED_WAKEUP_CMD)
83 #define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT)
85 enum vmballoon_page_size_type {
88 VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE
91 #define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1)
93 static const char * const vmballoon_page_size_names[] = {
94 [VMW_BALLOON_4K_PAGE] = "4k",
95 [VMW_BALLOON_2M_PAGE] = "2M"
103 enum vmballoon_op_stat_type {
105 VMW_BALLOON_OP_FAIL_STAT
108 #define VMW_BALLOON_OP_STAT_TYPES (VMW_BALLOON_OP_FAIL_STAT + 1)
111 * enum vmballoon_cmd_type - backdoor commands.
113 * Availability of the commands is as followed:
115 * %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and
116 * %VMW_BALLOON_CMD_GUEST_ID are always available.
118 * If the host reports %VMW_BALLOON_BASIC_CMDS are supported then
119 * %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available.
121 * If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then
122 * %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands
125 * If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then
126 * %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK
129 * If the host reports VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then
130 * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported.
132 * @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor.
133 * @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size.
134 * @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page.
135 * @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about
136 * to be deflated from the balloon.
137 * @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that
139 * @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of
140 * ballooned pages (up to 512).
141 * @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of
142 * pages that are about to be deflated from the
143 * balloon (up to 512).
144 * @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK
146 * @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to
147 * @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB
149 * @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification
150 * that would be invoked when the balloon
152 * @VMW_BALLOON_CMD_LAST: Value of the last command.
154 enum vmballoon_cmd_type {
155 VMW_BALLOON_CMD_START,
156 VMW_BALLOON_CMD_GET_TARGET,
157 VMW_BALLOON_CMD_LOCK,
158 VMW_BALLOON_CMD_UNLOCK,
159 VMW_BALLOON_CMD_GUEST_ID,
161 VMW_BALLOON_CMD_BATCHED_LOCK = 6,
162 VMW_BALLOON_CMD_BATCHED_UNLOCK,
163 VMW_BALLOON_CMD_BATCHED_2M_LOCK,
164 VMW_BALLOON_CMD_BATCHED_2M_UNLOCK,
165 VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
166 VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
169 #define VMW_BALLOON_CMD_NUM (VMW_BALLOON_CMD_LAST + 1)
171 enum vmballoon_error_codes {
173 VMW_BALLOON_ERROR_CMD_INVALID,
174 VMW_BALLOON_ERROR_PPN_INVALID,
175 VMW_BALLOON_ERROR_PPN_LOCKED,
176 VMW_BALLOON_ERROR_PPN_UNLOCKED,
177 VMW_BALLOON_ERROR_PPN_PINNED,
178 VMW_BALLOON_ERROR_PPN_NOTNEEDED,
179 VMW_BALLOON_ERROR_RESET,
180 VMW_BALLOON_ERROR_BUSY
183 #define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000)
185 #define VMW_BALLOON_CMD_WITH_TARGET_MASK \
186 ((1UL << VMW_BALLOON_CMD_GET_TARGET) | \
187 (1UL << VMW_BALLOON_CMD_LOCK) | \
188 (1UL << VMW_BALLOON_CMD_UNLOCK) | \
189 (1UL << VMW_BALLOON_CMD_BATCHED_LOCK) | \
190 (1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK) | \
191 (1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK) | \
192 (1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK))
194 static const char * const vmballoon_cmd_names[] = {
195 [VMW_BALLOON_CMD_START] = "start",
196 [VMW_BALLOON_CMD_GET_TARGET] = "target",
197 [VMW_BALLOON_CMD_LOCK] = "lock",
198 [VMW_BALLOON_CMD_UNLOCK] = "unlock",
199 [VMW_BALLOON_CMD_GUEST_ID] = "guestType",
200 [VMW_BALLOON_CMD_BATCHED_LOCK] = "batchLock",
201 [VMW_BALLOON_CMD_BATCHED_UNLOCK] = "batchUnlock",
202 [VMW_BALLOON_CMD_BATCHED_2M_LOCK] = "2m-lock",
203 [VMW_BALLOON_CMD_BATCHED_2M_UNLOCK] = "2m-unlock",
204 [VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet"
207 enum vmballoon_stat_page {
208 VMW_BALLOON_PAGE_STAT_ALLOC,
209 VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,
210 VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
211 VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
212 VMW_BALLOON_PAGE_STAT_FREE,
213 VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE
216 #define VMW_BALLOON_PAGE_STAT_NUM (VMW_BALLOON_PAGE_STAT_LAST + 1)
218 enum vmballoon_stat_general {
219 VMW_BALLOON_STAT_TIMER,
220 VMW_BALLOON_STAT_DOORBELL,
221 VMW_BALLOON_STAT_RESET,
222 VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_RESET
225 #define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1)
228 static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);
229 static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
231 struct vmballoon_ctl {
232 struct list_head pages;
233 struct list_head refused_pages;
234 unsigned int n_refused_pages;
235 unsigned int n_pages;
236 enum vmballoon_page_size_type page_size;
237 enum vmballoon_op op;
240 struct vmballoon_page_size {
241 /* list of reserved physical pages */
242 struct list_head pages;
246 * struct vmballoon_batch_entry - a batch entry for lock or unlock.
248 * @status: the status of the operation, which is written by the hypervisor.
249 * @reserved: reserved for future use. Must be set to zero.
250 * @pfn: the physical frame number of the page to be locked or unlocked.
252 struct vmballoon_batch_entry {
254 u64 reserved : PAGE_SHIFT - 5;
259 struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES];
262 * @max_page_size: maximum supported page size for ballooning.
264 * Protected by @conf_sem
266 enum vmballoon_page_size_type max_page_size;
269 * @size: balloon actual size in basic page size (frames).
271 * While we currently do not support size which is bigger than 32-bit,
272 * in preparation for future support, use 64-bits.
277 * @target: balloon target size in basic page size (frames).
279 * We do not protect the target under the assumption that setting the
280 * value is always done through a single write. If this assumption ever
281 * breaks, we would have to use X_ONCE for accesses, and suffer the less
282 * optimized code. Although we may read stale target value if multiple
283 * accesses happen at once, the performance impact should be minor.
285 unsigned long target;
288 * @reset_required: reset flag
290 * Setting this flag may introduce races, but the code is expected to
291 * handle them gracefully. In the worst case, another operation will
292 * fail as reset did not take place. Clearing the flag is done while
293 * holding @conf_sem for write.
298 * @capabilities: hypervisor balloon capabilities.
300 * Protected by @conf_sem.
302 unsigned long capabilities;
305 * @batch_page: pointer to communication batch page.
307 * When batching is used, batch_page points to a page, which holds up to
308 * %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking.
310 struct vmballoon_batch_entry *batch_page;
313 * @batch_max_pages: maximum pages that can be locked/unlocked.
315 * Indicates the number of pages that the hypervisor can lock or unlock
316 * at once, according to whether batching is enabled. If batching is
317 * disabled, only a single page can be locked/unlock on each operation.
319 * Protected by @conf_sem.
321 unsigned int batch_max_pages;
324 * @page: page to be locked/unlocked by the hypervisor
326 * @page is only used when batching is disabled and a single page is
327 * reclaimed on each iteration.
329 * Protected by @comm_lock.
334 struct vmballoon_stats *stats;
336 #ifdef CONFIG_DEBUG_FS
337 /* debugfs file exporting statistics */
338 struct dentry *dbg_entry;
341 struct delayed_work dwork;
346 * Protected by @conf_sem.
348 struct vmci_handle vmci_doorbell;
351 * @conf_sem: semaphore to protect the configuration and the statistics.
353 struct rw_semaphore conf_sem;
356 * @comm_lock: lock to protect the communication with the host.
358 * Lock ordering: @conf_sem -> @comm_lock .
360 spinlock_t comm_lock;
363 static struct vmballoon balloon;
365 struct vmballoon_stats {
366 /* timer / doorbell operations */
367 atomic64_t general_stat[VMW_BALLOON_STAT_NUM];
369 /* allocation statistics for huge and small pages */
371 page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES];
373 /* Monitor operations: total operations, and failures */
374 atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES];
377 static inline bool is_vmballoon_stats_on(void)
379 return IS_ENABLED(CONFIG_DEBUG_FS) &&
380 static_branch_unlikely(&balloon_stat_enabled);
383 static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,
384 enum vmballoon_op_stat_type type)
386 if (is_vmballoon_stats_on())
387 atomic64_inc(&b->stats->ops[op][type]);
390 static inline void vmballoon_stats_gen_inc(struct vmballoon *b,
391 enum vmballoon_stat_general stat)
393 if (is_vmballoon_stats_on())
394 atomic64_inc(&b->stats->general_stat[stat]);
397 static inline void vmballoon_stats_gen_add(struct vmballoon *b,
398 enum vmballoon_stat_general stat,
401 if (is_vmballoon_stats_on())
402 atomic64_add(val, &b->stats->general_stat[stat]);
405 static inline void vmballoon_stats_page_inc(struct vmballoon *b,
406 enum vmballoon_stat_page stat,
407 enum vmballoon_page_size_type size)
409 if (is_vmballoon_stats_on())
410 atomic64_inc(&b->stats->page_stat[stat][size]);
413 static inline void vmballoon_stats_page_add(struct vmballoon *b,
414 enum vmballoon_stat_page stat,
415 enum vmballoon_page_size_type size,
418 if (is_vmballoon_stats_on())
419 atomic64_add(val, &b->stats->page_stat[stat][size]);
422 static inline unsigned long
423 __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
424 unsigned long arg2, unsigned long *result)
426 unsigned long status, dummy1, dummy2, dummy3, local_result;
428 vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT);
430 asm volatile ("inl %%dx" :
436 "0"(VMW_BALLOON_HV_MAGIC),
438 "2"(VMW_BALLOON_HV_PORT),
443 /* update the result if needed */
445 *result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 :
448 /* update target when applicable */
449 if (status == VMW_BALLOON_SUCCESS &&
450 ((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))
451 WRITE_ONCE(b->target, local_result);
453 if (status != VMW_BALLOON_SUCCESS &&
454 status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {
455 vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT);
456 pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n",
457 __func__, vmballoon_cmd_names[cmd], arg1, arg2,
461 /* mark reset required accordingly */
462 if (status == VMW_BALLOON_ERROR_RESET)
463 b->reset_required = true;
468 static __always_inline unsigned long
469 vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
474 return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy);
478 * Send "start" command to the host, communicating supported version
481 static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
483 unsigned long status, capabilities;
485 status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,
489 case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES:
490 b->capabilities = capabilities;
492 case VMW_BALLOON_SUCCESS:
493 b->capabilities = VMW_BALLOON_BASIC_CMDS;
500 * 2MB pages are only supported with batching. If batching is for some
501 * reason disabled, do not use 2MB pages, since otherwise the legacy
502 * mechanism is used with 2MB pages, causing a failure.
504 b->max_page_size = VMW_BALLOON_4K_PAGE;
505 if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&
506 (b->capabilities & VMW_BALLOON_BATCHED_CMDS))
507 b->max_page_size = VMW_BALLOON_2M_PAGE;
514 * vmballoon_send_guest_id - communicate guest type to the host.
516 * @b: pointer to the balloon.
518 * Communicate guest type to the host so that it can adjust ballooning
519 * algorithm to the one most appropriate for the guest. This command
520 * is normally issued after sending "start" command and is part of
521 * standard reset sequence.
523 * Return: zero on success or appropriate error code.
525 static int vmballoon_send_guest_id(struct vmballoon *b)
527 unsigned long status;
529 status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,
530 VMW_BALLOON_GUEST_ID, 0);
532 return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
536 * vmballoon_page_order() - return the order of the page
537 * @page_size: the size of the page.
539 * Return: the allocation order.
542 unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size)
544 return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0;
548 * vmballoon_page_in_frames() - returns the number of frames in a page.
549 * @page_size: the size of the page.
551 * Return: the number of 4k frames.
553 static inline unsigned int
554 vmballoon_page_in_frames(enum vmballoon_page_size_type page_size)
556 return 1 << vmballoon_page_order(page_size);
560 * vmballoon_mark_page_offline() - mark a page as offline
561 * @page: pointer for the page.
562 * @page_size: the size of the page.
565 vmballoon_mark_page_offline(struct page *page,
566 enum vmballoon_page_size_type page_size)
570 for (i = 0; i < vmballoon_page_in_frames(page_size); i++)
571 __SetPageOffline(page + i);
575 * vmballoon_mark_page_online() - mark a page as online
576 * @page: pointer for the page.
577 * @page_size: the size of the page.
580 vmballoon_mark_page_online(struct page *page,
581 enum vmballoon_page_size_type page_size)
585 for (i = 0; i < vmballoon_page_in_frames(page_size); i++)
586 __ClearPageOffline(page + i);
590 * vmballoon_send_get_target() - Retrieve desired balloon size from the host.
592 * @b: pointer to the balloon.
594 * Return: zero on success, EINVAL if limit does not fit in 32-bit, as required
595 * by the host-guest protocol and EIO if an error occurred in communicating with
598 static int vmballoon_send_get_target(struct vmballoon *b)
600 unsigned long status;
603 limit = totalram_pages();
605 /* Ensure limit fits in 32-bits */
606 if (limit != (u32)limit)
609 status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);
611 return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
615 * vmballoon_alloc_page_list - allocates a list of pages.
617 * @b: pointer to the balloon.
618 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
619 * @req_n_pages: the number of requested pages.
621 * Tries to allocate @req_n_pages. Add them to the list of balloon pages in
622 * @ctl.pages and updates @ctl.n_pages to reflect the number of pages.
624 * Return: zero on success or error code otherwise.
626 static int vmballoon_alloc_page_list(struct vmballoon *b,
627 struct vmballoon_ctl *ctl,
628 unsigned int req_n_pages)
633 for (i = 0; i < req_n_pages; i++) {
634 if (ctl->page_size == VMW_BALLOON_2M_PAGE)
635 page = alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS,
636 VMW_BALLOON_2M_ORDER);
638 page = alloc_page(VMW_PAGE_ALLOC_FLAGS);
640 /* Update statistics */
641 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,
645 vmballoon_mark_page_offline(page, ctl->page_size);
646 /* Success. Add the page to the list and continue. */
647 list_add(&page->lru, &ctl->pages);
651 /* Allocation failed. Update statistics and stop. */
652 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,
659 return req_n_pages == ctl->n_pages ? 0 : -ENOMEM;
663 * vmballoon_handle_one_result - Handle lock/unlock result for a single page.
665 * @b: pointer for %struct vmballoon.
666 * @page: pointer for the page whose result should be handled.
667 * @page_size: size of the page.
668 * @status: status of the operation as provided by the hypervisor.
670 static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,
671 enum vmballoon_page_size_type page_size,
672 unsigned long status)
674 /* On success do nothing. The page is already on the balloon list. */
675 if (likely(status == VMW_BALLOON_SUCCESS))
678 pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__,
679 page_to_pfn(page), status,
680 vmballoon_page_size_names[page_size]);
683 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
690 * vmballoon_status_page - returns the status of (un)lock operation
692 * @b: pointer to the balloon.
693 * @idx: index for the page for which the operation is performed.
694 * @p: pointer to where the page struct is returned.
696 * Following a lock or unlock operation, returns the status of the operation for
697 * an individual page. Provides the page that the operation was performed on on
698 * the @page argument.
700 * Returns: The status of a lock or unlock operation for an individual page.
702 static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,
705 if (static_branch_likely(&vmw_balloon_batching)) {
707 *p = pfn_to_page(b->batch_page[idx].pfn);
708 return b->batch_page[idx].status;
711 /* non-batching mode */
715 * If a failure occurs, the indication will be provided in the status
716 * of the entire operation, which is considered before the individual
717 * page status. So for non-batching mode, the indication is always of
720 return VMW_BALLOON_SUCCESS;
724 * vmballoon_lock_op - notifies the host about inflated/deflated pages.
725 * @b: pointer to the balloon.
726 * @num_pages: number of inflated/deflated pages.
727 * @page_size: size of the page.
728 * @op: the type of operation (lock or unlock).
730 * Notify the host about page(s) that were ballooned (or removed from the
731 * balloon) so that host can use it without fear that guest will need it (or
732 * stop using them since the VM does). Host may reject some pages, we need to
733 * check the return value and maybe submit a different page. The pages that are
734 * inflated/deflated are pointed by @b->page.
736 * Return: result as provided by the hypervisor.
738 static unsigned long vmballoon_lock_op(struct vmballoon *b,
739 unsigned int num_pages,
740 enum vmballoon_page_size_type page_size,
741 enum vmballoon_op op)
743 unsigned long cmd, pfn;
745 lockdep_assert_held(&b->comm_lock);
747 if (static_branch_likely(&vmw_balloon_batching)) {
748 if (op == VMW_BALLOON_INFLATE)
749 cmd = page_size == VMW_BALLOON_2M_PAGE ?
750 VMW_BALLOON_CMD_BATCHED_2M_LOCK :
751 VMW_BALLOON_CMD_BATCHED_LOCK;
753 cmd = page_size == VMW_BALLOON_2M_PAGE ?
754 VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
755 VMW_BALLOON_CMD_BATCHED_UNLOCK;
757 pfn = PHYS_PFN(virt_to_phys(b->batch_page));
759 cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK :
760 VMW_BALLOON_CMD_UNLOCK;
761 pfn = page_to_pfn(b->page);
763 /* In non-batching mode, PFNs must fit in 32-bit */
764 if (unlikely(pfn != (u32)pfn))
765 return VMW_BALLOON_ERROR_PPN_INVALID;
768 return vmballoon_cmd(b, cmd, pfn, num_pages);
772 * vmballoon_add_page - adds a page towards lock/unlock operation.
774 * @b: pointer to the balloon.
775 * @idx: index of the page to be ballooned in this batch.
776 * @p: pointer to the page that is about to be ballooned.
778 * Adds the page to be ballooned. Must be called while holding @comm_lock.
780 static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,
783 lockdep_assert_held(&b->comm_lock);
785 if (static_branch_likely(&vmw_balloon_batching))
786 b->batch_page[idx] = (struct vmballoon_batch_entry)
787 { .pfn = page_to_pfn(p) };
793 * vmballoon_lock - lock or unlock a batch of pages.
795 * @b: pointer to the balloon.
796 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
798 * Notifies the host of about ballooned pages (after inflation or deflation,
799 * according to @ctl). If the host rejects the page put it on the
800 * @ctl refuse list. These refused page are then released when moving to the
801 * next size of pages.
803 * Note that we neither free any @page here nor put them back on the ballooned
804 * pages list. Instead we queue it for later processing. We do that for several
805 * reasons. First, we do not want to free the page under the lock. Second, it
806 * allows us to unify the handling of lock and unlock. In the inflate case, the
807 * caller will check if there are too many refused pages and release them.
808 * Although it is not identical to the past behavior, it should not affect
811 static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)
813 unsigned long batch_status;
815 unsigned int i, num_pages;
817 num_pages = ctl->n_pages;
821 /* communication with the host is done under the communication lock */
822 spin_lock(&b->comm_lock);
825 list_for_each_entry(page, &ctl->pages, lru)
826 vmballoon_add_page(b, i++, page);
828 batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,
832 * Iterate over the pages in the provided list. Since we are changing
833 * @ctl->n_pages we are saving the original value in @num_pages and
834 * use this value to bound the loop.
836 for (i = 0; i < num_pages; i++) {
837 unsigned long status;
839 status = vmballoon_status_page(b, i, &page);
842 * Failure of the whole batch overrides a single operation
845 if (batch_status != VMW_BALLOON_SUCCESS)
846 status = batch_status;
848 /* Continue if no error happened */
849 if (!vmballoon_handle_one_result(b, page, ctl->page_size,
854 * Error happened. Move the pages to the refused list and update
857 list_move(&page->lru, &ctl->refused_pages);
859 ctl->n_refused_pages++;
862 spin_unlock(&b->comm_lock);
864 return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
868 * vmballoon_release_page_list() - Releases a page list
870 * @page_list: list of pages to release.
871 * @n_pages: pointer to the number of pages.
872 * @page_size: whether the pages in the list are 2MB (or else 4KB).
874 * Releases the list of pages and zeros the number of pages.
876 static void vmballoon_release_page_list(struct list_head *page_list,
878 enum vmballoon_page_size_type page_size)
880 struct page *page, *tmp;
882 list_for_each_entry_safe(page, tmp, page_list, lru) {
883 list_del(&page->lru);
884 vmballoon_mark_page_online(page, page_size);
885 __free_pages(page, vmballoon_page_order(page_size));
893 * Release pages that were allocated while attempting to inflate the
894 * balloon but were refused by the host for one reason or another.
896 static void vmballoon_release_refused_pages(struct vmballoon *b,
897 struct vmballoon_ctl *ctl)
899 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
902 vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages,
907 * vmballoon_change - retrieve the required balloon change
909 * @b: pointer for the balloon.
911 * Return: the required change for the balloon size. A positive number
912 * indicates inflation, a negative number indicates a deflation.
914 static int64_t vmballoon_change(struct vmballoon *b)
916 int64_t size, target;
918 size = atomic64_read(&b->size);
919 target = READ_ONCE(b->target);
922 * We must cast first because of int sizes
923 * Otherwise we might get huge positives instead of negatives
926 if (b->reset_required)
929 /* consider a 2MB slack on deflate, unless the balloon is emptied */
930 if (target < size && target != 0 &&
931 size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE))
934 return target - size;
938 * vmballoon_enqueue_page_list() - Enqueues list of pages after inflation.
940 * @b: pointer to balloon.
941 * @pages: list of pages to enqueue.
942 * @n_pages: pointer to number of pages in list. The value is zeroed.
943 * @page_size: whether the pages are 2MB or 4KB pages.
945 * Enqueues the provides list of pages in the ballooned page list, clears the
946 * list and zeroes the number of pages that was provided.
948 static void vmballoon_enqueue_page_list(struct vmballoon *b,
949 struct list_head *pages,
950 unsigned int *n_pages,
951 enum vmballoon_page_size_type page_size)
953 struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
955 list_splice_init(pages, &page_size_info->pages);
960 * vmballoon_dequeue_page_list() - Dequeues page lists for deflation.
962 * @b: pointer to balloon.
963 * @pages: list of pages to enqueue.
964 * @n_pages: pointer to number of pages in list. The value is zeroed.
965 * @page_size: whether the pages are 2MB or 4KB pages.
966 * @n_req_pages: the number of requested pages.
968 * Dequeues the number of requested pages from the balloon for deflation. The
969 * number of dequeued pages may be lower, if not enough pages in the requested
970 * size are available.
972 static void vmballoon_dequeue_page_list(struct vmballoon *b,
973 struct list_head *pages,
974 unsigned int *n_pages,
975 enum vmballoon_page_size_type page_size,
976 unsigned int n_req_pages)
978 struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size];
979 struct page *page, *tmp;
982 list_for_each_entry_safe(page, tmp, &page_size_info->pages, lru) {
983 list_move(&page->lru, pages);
984 if (++i == n_req_pages)
991 * vmballoon_inflate() - Inflate the balloon towards its target size.
993 * @b: pointer to the balloon.
995 static void vmballoon_inflate(struct vmballoon *b)
997 int64_t to_inflate_frames;
998 struct vmballoon_ctl ctl = {
999 .pages = LIST_HEAD_INIT(ctl.pages),
1000 .refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
1001 .page_size = b->max_page_size,
1002 .op = VMW_BALLOON_INFLATE
1005 while ((to_inflate_frames = vmballoon_change(b)) > 0) {
1006 unsigned int to_inflate_pages, page_in_frames;
1007 int alloc_error, lock_error = 0;
1009 VM_BUG_ON(!list_empty(&ctl.pages));
1010 VM_BUG_ON(ctl.n_pages != 0);
1012 page_in_frames = vmballoon_page_in_frames(ctl.page_size);
1014 to_inflate_pages = min_t(unsigned long, b->batch_max_pages,
1015 DIV_ROUND_UP_ULL(to_inflate_frames,
1018 /* Start by allocating */
1019 alloc_error = vmballoon_alloc_page_list(b, &ctl,
1022 /* Actually lock the pages by telling the hypervisor */
1023 lock_error = vmballoon_lock(b, &ctl);
1026 * If an error indicates that something serious went wrong,
1027 * stop the inflation.
1032 /* Update the balloon size */
1033 atomic64_add(ctl.n_pages * page_in_frames, &b->size);
1035 vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages,
1039 * If allocation failed or the number of refused pages exceeds
1040 * the maximum allowed, move to the next page size.
1043 ctl.n_refused_pages >= VMW_BALLOON_MAX_REFUSED) {
1044 if (ctl.page_size == VMW_BALLOON_4K_PAGE)
1048 * Ignore errors from locking as we now switch to 4k
1049 * pages and we might get different errors.
1051 vmballoon_release_refused_pages(b, &ctl);
1059 * Release pages that were allocated while attempting to inflate the
1060 * balloon but were refused by the host for one reason or another,
1061 * and update the statistics.
1063 if (ctl.n_refused_pages != 0)
1064 vmballoon_release_refused_pages(b, &ctl);
1068 * vmballoon_deflate() - Decrease the size of the balloon.
1070 * @b: pointer to the balloon
1071 * @n_frames: the number of frames to deflate. If zero, automatically
1072 * calculated according to the target size.
1073 * @coordinated: whether to coordinate with the host
1075 * Decrease the size of the balloon allowing guest to use more memory.
1077 * Return: The number of deflated frames (i.e., basic page size units)
1079 static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames,
1082 unsigned long deflated_frames = 0;
1083 unsigned long tried_frames = 0;
1084 struct vmballoon_ctl ctl = {
1085 .pages = LIST_HEAD_INIT(ctl.pages),
1086 .refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
1087 .page_size = VMW_BALLOON_4K_PAGE,
1088 .op = VMW_BALLOON_DEFLATE
1091 /* free pages to reach target */
1093 unsigned int to_deflate_pages, n_unlocked_frames;
1094 unsigned int page_in_frames;
1095 int64_t to_deflate_frames;
1098 page_in_frames = vmballoon_page_in_frames(ctl.page_size);
1100 VM_BUG_ON(!list_empty(&ctl.pages));
1101 VM_BUG_ON(ctl.n_pages);
1102 VM_BUG_ON(!list_empty(&ctl.refused_pages));
1103 VM_BUG_ON(ctl.n_refused_pages);
1106 * If we were requested a specific number of frames, we try to
1107 * deflate this number of frames. Otherwise, deflation is
1108 * performed according to the target and balloon size.
1110 to_deflate_frames = n_frames ? n_frames - tried_frames :
1111 -vmballoon_change(b);
1113 /* break if no work to do */
1114 if (to_deflate_frames <= 0)
1118 * Calculate the number of frames based on current page size,
1119 * but limit the deflated frames to a single chunk
1121 to_deflate_pages = min_t(unsigned long, b->batch_max_pages,
1122 DIV_ROUND_UP_ULL(to_deflate_frames,
1125 /* First take the pages from the balloon pages. */
1126 vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages,
1127 ctl.page_size, to_deflate_pages);
1130 * Before pages are moving to the refused list, count their
1131 * frames as frames that we tried to deflate.
1133 tried_frames += ctl.n_pages * page_in_frames;
1136 * Unlock the pages by communicating with the hypervisor if the
1137 * communication is coordinated (i.e., not pop). We ignore the
1138 * return code. Instead we check if all the pages we manage to
1139 * unlock all the pages. If we failed, we will move to the next
1140 * page size, and would eventually try again later.
1143 vmballoon_lock(b, &ctl);
1146 * Check if we deflated enough. We will move to the next page
1147 * size if we did not manage to do so. This calculation takes
1148 * place now, as once the pages are released, the number of
1151 deflated_all = (ctl.n_pages == to_deflate_pages);
1153 /* Update local and global counters */
1154 n_unlocked_frames = ctl.n_pages * page_in_frames;
1155 atomic64_sub(n_unlocked_frames, &b->size);
1156 deflated_frames += n_unlocked_frames;
1158 vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE,
1159 ctl.page_size, ctl.n_pages);
1161 /* free the ballooned pages */
1162 vmballoon_release_page_list(&ctl.pages, &ctl.n_pages,
1165 /* Return the refused pages to the ballooned list. */
1166 vmballoon_enqueue_page_list(b, &ctl.refused_pages,
1167 &ctl.n_refused_pages,
1170 /* If we failed to unlock all the pages, move to next size. */
1171 if (!deflated_all) {
1172 if (ctl.page_size == b->max_page_size)
1180 return deflated_frames;
1184 * vmballoon_deinit_batching - disables batching mode.
1186 * @b: pointer to &struct vmballoon.
1188 * Disables batching, by deallocating the page for communication with the
1189 * hypervisor and disabling the static key to indicate that batching is off.
1191 static void vmballoon_deinit_batching(struct vmballoon *b)
1193 free_page((unsigned long)b->batch_page);
1194 b->batch_page = NULL;
1195 static_branch_disable(&vmw_balloon_batching);
1196 b->batch_max_pages = 1;
1200 * vmballoon_init_batching - enable batching mode.
1202 * @b: pointer to &struct vmballoon.
1204 * Enables batching, by allocating a page for communication with the hypervisor
1205 * and enabling the static_key to use batching.
1207 * Return: zero on success or an appropriate error-code.
1209 static int vmballoon_init_batching(struct vmballoon *b)
1213 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1217 b->batch_page = page_address(page);
1218 b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry);
1220 static_branch_enable(&vmw_balloon_batching);
1226 * Receive notification and resize balloon
1228 static void vmballoon_doorbell(void *client_data)
1230 struct vmballoon *b = client_data;
1232 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL);
1234 mod_delayed_work(system_freezable_wq, &b->dwork, 0);
1238 * Clean up vmci doorbell
1240 static void vmballoon_vmci_cleanup(struct vmballoon *b)
1242 vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
1243 VMCI_INVALID_ID, VMCI_INVALID_ID);
1245 if (!vmci_handle_is_invalid(b->vmci_doorbell)) {
1246 vmci_doorbell_destroy(b->vmci_doorbell);
1247 b->vmci_doorbell = VMCI_INVALID_HANDLE;
1252 * vmballoon_vmci_init - Initialize vmci doorbell.
1254 * @b: pointer to the balloon.
1256 * Return: zero on success or when wakeup command not supported. Error-code
1259 * Initialize vmci doorbell, to get notified as soon as balloon changes.
1261 static int vmballoon_vmci_init(struct vmballoon *b)
1263 unsigned long error;
1265 if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)
1268 error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB,
1269 VMCI_PRIVILEGE_FLAG_RESTRICTED,
1270 vmballoon_doorbell, b);
1272 if (error != VMCI_SUCCESS)
1275 error = __vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
1276 b->vmci_doorbell.context,
1277 b->vmci_doorbell.resource, NULL);
1279 if (error != VMW_BALLOON_SUCCESS)
1284 vmballoon_vmci_cleanup(b);
1289 * vmballoon_pop - Quickly release all pages allocate for the balloon.
1291 * @b: pointer to the balloon.
1293 * This function is called when host decides to "reset" balloon for one reason
1294 * or another. Unlike normal "deflate" we do not (shall not) notify host of the
1295 * pages being released.
1297 static void vmballoon_pop(struct vmballoon *b)
1301 while ((size = atomic64_read(&b->size)))
1302 vmballoon_deflate(b, size, false);
1306 * Perform standard reset sequence by popping the balloon (in case it
1307 * is not empty) and then restarting protocol. This operation normally
1308 * happens when host responds with VMW_BALLOON_ERROR_RESET to a command.
1310 static void vmballoon_reset(struct vmballoon *b)
1314 down_write(&b->conf_sem);
1316 vmballoon_vmci_cleanup(b);
1318 /* free all pages, skipping monitor unlock */
1321 if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))
1324 if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {
1325 if (vmballoon_init_batching(b)) {
1327 * We failed to initialize batching, inform the monitor
1328 * about it by sending a null capability.
1330 * The guest will retry in one second.
1332 vmballoon_send_start(b, 0);
1335 } else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {
1336 vmballoon_deinit_batching(b);
1339 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET);
1340 b->reset_required = false;
1342 error = vmballoon_vmci_init(b);
1344 pr_err("failed to initialize vmci doorbell\n");
1346 if (vmballoon_send_guest_id(b))
1347 pr_err("failed to send guest ID to the host\n");
1349 up_write(&b->conf_sem);
1353 * vmballoon_work - periodic balloon worker for reset, inflation and deflation.
1355 * @work: pointer to the &work_struct which is provided by the workqueue.
1357 * Resets the protocol if needed, gets the new size and adjusts balloon as
1358 * needed. Repeat in 1 sec.
1360 static void vmballoon_work(struct work_struct *work)
1362 struct delayed_work *dwork = to_delayed_work(work);
1363 struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);
1366 if (b->reset_required)
1369 down_read(&b->conf_sem);
1372 * Update the stats while holding the semaphore to ensure that
1373 * @stats_enabled is consistent with whether the stats are actually
1376 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER);
1378 if (!vmballoon_send_get_target(b))
1379 change = vmballoon_change(b);
1382 pr_debug("%s - size: %llu, target %lu\n", __func__,
1383 atomic64_read(&b->size), READ_ONCE(b->target));
1386 vmballoon_inflate(b);
1387 else /* (change < 0) */
1388 vmballoon_deflate(b, 0, true);
1391 up_read(&b->conf_sem);
1394 * We are using a freezable workqueue so that balloon operations are
1395 * stopped while the system transitions to/from sleep/hibernation.
1397 queue_delayed_work(system_freezable_wq,
1398 dwork, round_jiffies_relative(HZ));
1405 #ifdef CONFIG_DEBUG_FS
1407 static const char * const vmballoon_stat_page_names[] = {
1408 [VMW_BALLOON_PAGE_STAT_ALLOC] = "alloc",
1409 [VMW_BALLOON_PAGE_STAT_ALLOC_FAIL] = "allocFail",
1410 [VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC] = "errAlloc",
1411 [VMW_BALLOON_PAGE_STAT_REFUSED_FREE] = "errFree",
1412 [VMW_BALLOON_PAGE_STAT_FREE] = "free"
1415 static const char * const vmballoon_stat_names[] = {
1416 [VMW_BALLOON_STAT_TIMER] = "timer",
1417 [VMW_BALLOON_STAT_DOORBELL] = "doorbell",
1418 [VMW_BALLOON_STAT_RESET] = "reset",
1421 static int vmballoon_enable_stats(struct vmballoon *b)
1425 down_write(&b->conf_sem);
1427 /* did we somehow race with another reader which enabled stats? */
1431 b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL);
1434 /* allocation failed */
1438 static_key_enable(&balloon_stat_enabled.key);
1440 up_write(&b->conf_sem);
1445 * vmballoon_debug_show - shows statistics of balloon operations.
1446 * @f: pointer to the &struct seq_file.
1449 * Provides the statistics that can be accessed in vmmemctl in the debugfs.
1450 * To avoid the overhead - mainly that of memory - of collecting the statistics,
1451 * we only collect statistics after the first time the counters are read.
1453 * Return: zero on success or an error code.
1455 static int vmballoon_debug_show(struct seq_file *f, void *offset)
1457 struct vmballoon *b = f->private;
1460 /* enables stats if they are disabled */
1462 int r = vmballoon_enable_stats(b);
1468 /* format capabilities info */
1469 seq_printf(f, "%-22s: %#16x\n", "balloon capabilities",
1470 VMW_BALLOON_CAPABILITIES);
1471 seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities);
1472 seq_printf(f, "%-22s: %16s\n", "is resetting",
1473 b->reset_required ? "y" : "n");
1475 /* format size info */
1476 seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target));
1477 seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size));
1479 for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) {
1480 if (vmballoon_cmd_names[i] == NULL)
1483 seq_printf(f, "%-22s: %16llu (%llu failed)\n",
1484 vmballoon_cmd_names[i],
1485 atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]),
1486 atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT]));
1489 for (i = 0; i < VMW_BALLOON_STAT_NUM; i++)
1490 seq_printf(f, "%-22s: %16llu\n",
1491 vmballoon_stat_names[i],
1492 atomic64_read(&b->stats->general_stat[i]));
1494 for (i = 0; i < VMW_BALLOON_PAGE_STAT_NUM; i++) {
1495 for (j = 0; j < VMW_BALLOON_NUM_PAGE_SIZES; j++)
1496 seq_printf(f, "%-18s(%s): %16llu\n",
1497 vmballoon_stat_page_names[i],
1498 vmballoon_page_size_names[j],
1499 atomic64_read(&b->stats->page_stat[i][j]));
1505 DEFINE_SHOW_ATTRIBUTE(vmballoon_debug);
1507 static int __init vmballoon_debugfs_init(struct vmballoon *b)
1511 b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b,
1512 &vmballoon_debug_fops);
1513 if (IS_ERR(b->dbg_entry)) {
1514 error = PTR_ERR(b->dbg_entry);
1515 pr_err("failed to create debugfs entry, error: %d\n", error);
1522 static void __exit vmballoon_debugfs_exit(struct vmballoon *b)
1524 static_key_disable(&balloon_stat_enabled.key);
1525 debugfs_remove(b->dbg_entry);
1532 static inline int vmballoon_debugfs_init(struct vmballoon *b)
1537 static inline void vmballoon_debugfs_exit(struct vmballoon *b)
1541 #endif /* CONFIG_DEBUG_FS */
1543 static int __init vmballoon_init(void)
1545 enum vmballoon_page_size_type page_size;
1549 * Check if we are running on VMware's hypervisor and bail out
1552 if (x86_hyper_type != X86_HYPER_VMWARE)
1555 for (page_size = VMW_BALLOON_4K_PAGE;
1556 page_size <= VMW_BALLOON_LAST_SIZE; page_size++)
1557 INIT_LIST_HEAD(&balloon.page_sizes[page_size].pages);
1560 INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
1562 error = vmballoon_debugfs_init(&balloon);
1566 spin_lock_init(&balloon.comm_lock);
1567 init_rwsem(&balloon.conf_sem);
1568 balloon.vmci_doorbell = VMCI_INVALID_HANDLE;
1569 balloon.batch_page = NULL;
1570 balloon.page = NULL;
1571 balloon.reset_required = true;
1573 queue_delayed_work(system_freezable_wq, &balloon.dwork, 0);
1579 * Using late_initcall() instead of module_init() allows the balloon to use the
1580 * VMCI doorbell even when the balloon is built into the kernel. Otherwise the
1581 * VMCI is probed only after the balloon is initialized. If the balloon is used
1582 * as a module, late_initcall() is equivalent to module_init().
1584 late_initcall(vmballoon_init);
1586 static void __exit vmballoon_exit(void)
1588 vmballoon_vmci_cleanup(&balloon);
1589 cancel_delayed_work_sync(&balloon.dwork);
1591 vmballoon_debugfs_exit(&balloon);
1594 * Deallocate all reserved memory, and reset connection with monitor.
1595 * Reset connection before deallocating memory to avoid potential for
1596 * additional spurious resets from guest touching deallocated pages.
1598 vmballoon_send_start(&balloon, 0);
1599 vmballoon_pop(&balloon);
1601 module_exit(vmballoon_exit);