Commit | Line | Data |
---|---|---|
8b4770ec | 1 | // SPDX-License-Identifier: GPL-2.0 |
453dc659 DT |
2 | /* |
3 | * VMware Balloon driver. | |
4 | * | |
8b4770ec | 5 | * Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved. |
453dc659 | 6 | * |
453dc659 DT |
7 | * This is VMware physical memory management driver for Linux. The driver |
8 | * acts like a "balloon" that can be inflated to reclaim physical pages by | |
9 | * reserving them in the guest and invalidating them in the monitor, | |
10 | * freeing up the underlying machine pages so they can be allocated to | |
11 | * other guests. The balloon can also be deflated to allow the guest to | |
12 | * use more physical memory. Higher level policies can control the sizes | |
13 | * of balloons in VMs in order to manage physical memory resources. | |
14 | */ | |
15 | ||
16 | //#define DEBUG | |
17 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
18 | ||
19 | #include <linux/types.h> | |
20 | #include <linux/kernel.h> | |
21 | #include <linux/mm.h> | |
f220a80f | 22 | #include <linux/vmalloc.h> |
453dc659 DT |
23 | #include <linux/sched.h> |
24 | #include <linux/module.h> | |
25 | #include <linux/workqueue.h> | |
26 | #include <linux/debugfs.h> | |
27 | #include <linux/seq_file.h> | |
c7b3690f NA |
28 | #include <linux/rwsem.h> |
29 | #include <linux/slab.h> | |
6e4453b3 | 30 | #include <linux/spinlock.h> |
48e3d668 PM |
31 | #include <linux/vmw_vmci_defs.h> |
32 | #include <linux/vmw_vmci_api.h> | |
a10a5698 | 33 | #include <asm/hypervisor.h> |
453dc659 DT |
34 | |
35 | MODULE_AUTHOR("VMware, Inc."); | |
36 | MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver"); | |
453dc659 DT |
37 | MODULE_ALIAS("dmi:*:svnVMware*:*"); |
38 | MODULE_ALIAS("vmware_vmmemctl"); | |
39 | MODULE_LICENSE("GPL"); | |
40 | ||
453dc659 | 41 | /* |
622074a9 NA |
42 | * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We don't allow wait |
43 | * (__GFP_RECLAIM) for huge page allocations. Use __GFP_NOWARN, to suppress page | |
44 | * allocation failure warnings. Disallow access to emergency low-memory pools. | |
453dc659 | 45 | */ |
622074a9 NA |
46 | #define VMW_HUGE_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \ |
47 | __GFP_NOMEMALLOC) | |
453dc659 DT |
48 | |
49 | /* | |
622074a9 NA |
50 | * Use __GFP_HIGHMEM to allow pages from HIGHMEM zone. We allow lightweight |
51 | * reclamation (__GFP_NORETRY). Use __GFP_NOWARN, to suppress page allocation | |
52 | * failure warnings. Disallow access to emergency low-memory pools. | |
453dc659 | 53 | */ |
622074a9 NA |
54 | #define VMW_PAGE_ALLOC_FLAGS (__GFP_HIGHMEM|__GFP_NOWARN| \ |
55 | __GFP_NOMEMALLOC|__GFP_NORETRY) | |
453dc659 | 56 | |
55adaa49 DT |
57 | /* Maximum number of refused pages we accumulate during inflation cycle */ |
58 | #define VMW_BALLOON_MAX_REFUSED 16 | |
453dc659 DT |
59 | |
60 | /* | |
61 | * Hypervisor communication port definitions. | |
62 | */ | |
63 | #define VMW_BALLOON_HV_PORT 0x5670 | |
64 | #define VMW_BALLOON_HV_MAGIC 0x456c6d6f | |
453dc659 DT |
65 | #define VMW_BALLOON_GUEST_ID 1 /* Linux */ |
66 | ||
eb79100f XD |
67 | enum vmwballoon_capabilities { |
68 | /* | |
69 | * Bit 0 is reserved and not associated to any capability. | |
70 | */ | |
48e3d668 PM |
71 | VMW_BALLOON_BASIC_CMDS = (1 << 1), |
72 | VMW_BALLOON_BATCHED_CMDS = (1 << 2), | |
73 | VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3), | |
74 | VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4), | |
55398302 | 75 | VMW_BALLOON_64_BIT_TARGET = (1 << 5) |
eb79100f XD |
76 | }; |
77 | ||
55398302 | 78 | #define VMW_BALLOON_CAPABILITIES_COMMON (VMW_BALLOON_BASIC_CMDS \ |
365bd7ef | 79 | | VMW_BALLOON_BATCHED_CMDS \ |
48e3d668 PM |
80 | | VMW_BALLOON_BATCHED_2M_CMDS \ |
81 | | VMW_BALLOON_SIGNALLED_WAKEUP_CMD) | |
365bd7ef | 82 | |
25acbdd7 | 83 | #define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT) |
eb79100f | 84 | |
55398302 XD |
85 | /* |
86 | * 64-bit targets are only supported in 64-bit | |
87 | */ | |
88 | #ifdef CONFIG_64BIT | |
89 | #define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_CAPABILITIES_COMMON \ | |
90 | | VMW_BALLOON_64_BIT_TARGET) | |
91 | #else | |
92 | #define VMW_BALLOON_CAPABILITIES VMW_BALLOON_CAPABILITIES_COMMON | |
93 | #endif | |
94 | ||
c7b3690f NA |
95 | enum vmballoon_page_size_type { |
96 | VMW_BALLOON_4K_PAGE, | |
97 | VMW_BALLOON_2M_PAGE, | |
98 | VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE | |
99 | }; | |
100 | ||
101 | #define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1) | |
102 | ||
6e4453b3 NA |
103 | static const char * const vmballoon_page_size_names[] = { |
104 | [VMW_BALLOON_4K_PAGE] = "4k", | |
105 | [VMW_BALLOON_2M_PAGE] = "2M" | |
106 | }; | |
107 | ||
108 | enum vmballoon_op { | |
109 | VMW_BALLOON_INFLATE, | |
110 | VMW_BALLOON_DEFLATE | |
111 | }; | |
112 | ||
c7b3690f NA |
113 | enum vmballoon_op_stat_type { |
114 | VMW_BALLOON_OP_STAT, | |
115 | VMW_BALLOON_OP_FAIL_STAT | |
116 | }; | |
117 | ||
118 | #define VMW_BALLOON_OP_STAT_TYPES (VMW_BALLOON_OP_FAIL_STAT + 1) | |
119 | ||
120 | /** | |
121 | * enum vmballoon_cmd_type - backdoor commands. | |
122 | * | |
123 | * Availability of the commands is as followed: | |
124 | * | |
125 | * %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and | |
126 | * %VMW_BALLOON_CMD_GUEST_ID are always available. | |
127 | * | |
128 | * If the host reports %VMW_BALLOON_BASIC_CMDS are supported then | |
129 | * %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available. | |
f220a80f | 130 | * |
c7b3690f NA |
131 | * If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then |
132 | * %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands | |
133 | * are available. | |
f220a80f | 134 | * |
c7b3690f NA |
135 | * If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then |
136 | * %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK | |
137 | * are supported. | |
138 | * | |
139 | * If the host reports VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then | |
140 | * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported. | |
141 | * | |
142 | * @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor. | |
143 | * @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size. | |
144 | * @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page. | |
145 | * @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about | |
146 | * to be deflated from the balloon. | |
147 | * @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that | |
148 | * runs in the VM. | |
149 | * @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of | |
150 | * ballooned pages (up to 512). | |
151 | * @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of | |
152 | * pages that are about to be deflated from the | |
153 | * balloon (up to 512). | |
154 | * @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK | |
155 | * for 2MB pages. | |
156 | * @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to | |
157 | * @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB | |
158 | * pages. | |
159 | * @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification | |
160 | * that would be invoked when the balloon | |
161 | * size changes. | |
162 | * @VMW_BALLOON_CMD_LAST: Value of the last command. | |
f220a80f | 163 | */ |
c7b3690f NA |
164 | enum vmballoon_cmd_type { |
165 | VMW_BALLOON_CMD_START, | |
166 | VMW_BALLOON_CMD_GET_TARGET, | |
167 | VMW_BALLOON_CMD_LOCK, | |
168 | VMW_BALLOON_CMD_UNLOCK, | |
169 | VMW_BALLOON_CMD_GUEST_ID, | |
170 | /* No command 5 */ | |
171 | VMW_BALLOON_CMD_BATCHED_LOCK = 6, | |
172 | VMW_BALLOON_CMD_BATCHED_UNLOCK, | |
173 | VMW_BALLOON_CMD_BATCHED_2M_LOCK, | |
174 | VMW_BALLOON_CMD_BATCHED_2M_UNLOCK, | |
175 | VMW_BALLOON_CMD_VMCI_DOORBELL_SET, | |
176 | VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET, | |
177 | }; | |
178 | ||
179 | #define VMW_BALLOON_CMD_NUM (VMW_BALLOON_CMD_LAST + 1) | |
180 | ||
181 | enum vmballoon_error_codes { | |
182 | VMW_BALLOON_SUCCESS, | |
183 | VMW_BALLOON_ERROR_CMD_INVALID, | |
184 | VMW_BALLOON_ERROR_PPN_INVALID, | |
185 | VMW_BALLOON_ERROR_PPN_LOCKED, | |
186 | VMW_BALLOON_ERROR_PPN_UNLOCKED, | |
187 | VMW_BALLOON_ERROR_PPN_PINNED, | |
188 | VMW_BALLOON_ERROR_PPN_NOTNEEDED, | |
189 | VMW_BALLOON_ERROR_RESET, | |
190 | VMW_BALLOON_ERROR_BUSY | |
191 | }; | |
eb79100f XD |
192 | |
193 | #define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000) | |
194 | ||
10a95d5d NA |
195 | #define VMW_BALLOON_CMD_WITH_TARGET_MASK \ |
196 | ((1UL << VMW_BALLOON_CMD_GET_TARGET) | \ | |
197 | (1UL << VMW_BALLOON_CMD_LOCK) | \ | |
198 | (1UL << VMW_BALLOON_CMD_UNLOCK) | \ | |
199 | (1UL << VMW_BALLOON_CMD_BATCHED_LOCK) | \ | |
200 | (1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK) | \ | |
201 | (1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK) | \ | |
202 | (1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK)) | |
203 | ||
68131184 NA |
204 | static const char * const vmballoon_cmd_names[] = { |
205 | [VMW_BALLOON_CMD_START] = "start", | |
206 | [VMW_BALLOON_CMD_GET_TARGET] = "target", | |
207 | [VMW_BALLOON_CMD_LOCK] = "lock", | |
208 | [VMW_BALLOON_CMD_UNLOCK] = "unlock", | |
209 | [VMW_BALLOON_CMD_GUEST_ID] = "guestType", | |
210 | [VMW_BALLOON_CMD_BATCHED_LOCK] = "batchLock", | |
211 | [VMW_BALLOON_CMD_BATCHED_UNLOCK] = "batchUnlock", | |
212 | [VMW_BALLOON_CMD_BATCHED_2M_LOCK] = "2m-lock", | |
213 | [VMW_BALLOON_CMD_BATCHED_2M_UNLOCK] = "2m-unlock", | |
214 | [VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet" | |
215 | }; | |
216 | ||
c7b3690f NA |
217 | enum vmballoon_stat_page { |
218 | VMW_BALLOON_PAGE_STAT_ALLOC, | |
219 | VMW_BALLOON_PAGE_STAT_ALLOC_FAIL, | |
220 | VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC, | |
221 | VMW_BALLOON_PAGE_STAT_REFUSED_FREE, | |
222 | VMW_BALLOON_PAGE_STAT_FREE, | |
223 | VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE | |
453dc659 DT |
224 | }; |
225 | ||
c7b3690f NA |
226 | #define VMW_BALLOON_PAGE_STAT_NUM (VMW_BALLOON_PAGE_STAT_LAST + 1) |
227 | ||
228 | enum vmballoon_stat_general { | |
229 | VMW_BALLOON_STAT_TIMER, | |
230 | VMW_BALLOON_STAT_DOORBELL, | |
8840a6f4 NA |
231 | VMW_BALLOON_STAT_RESET, |
232 | VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_RESET | |
c7b3690f NA |
233 | }; |
234 | ||
235 | #define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1) | |
236 | ||
453dc659 | 237 | |
df8d0d42 | 238 | static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching); |
c7b3690f | 239 | static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled); |
f220a80f | 240 | |
6e4453b3 | 241 | struct vmballoon_ctl { |
453dc659 | 242 | struct list_head pages; |
453dc659 | 243 | struct list_head refused_pages; |
55adaa49 | 244 | unsigned int n_refused_pages; |
6e4453b3 NA |
245 | unsigned int n_pages; |
246 | enum vmballoon_page_size_type page_size; | |
247 | enum vmballoon_op op; | |
248 | }; | |
249 | ||
250 | struct vmballoon_page_size { | |
251 | /* list of reserved physical pages */ | |
252 | struct list_head pages; | |
365bd7ef PM |
253 | }; |
254 | ||
6c948757 NA |
255 | /** |
256 | * struct vmballoon_batch_entry - a batch entry for lock or unlock. | |
257 | * | |
258 | * @status: the status of the operation, which is written by the hypervisor. | |
259 | * @reserved: reserved for future use. Must be set to zero. | |
260 | * @pfn: the physical frame number of the page to be locked or unlocked. | |
261 | */ | |
262 | struct vmballoon_batch_entry { | |
263 | u64 status : 5; | |
264 | u64 reserved : PAGE_SHIFT - 5; | |
265 | u64 pfn : 52; | |
266 | } __packed; | |
267 | ||
365bd7ef PM |
268 | struct vmballoon { |
269 | struct vmballoon_page_size page_sizes[VMW_BALLOON_NUM_PAGE_SIZES]; | |
270 | ||
6e4453b3 NA |
271 | /** |
272 | * @max_page_size: maximum supported page size for ballooning. | |
273 | * | |
274 | * Protected by @conf_sem | |
275 | */ | |
276 | enum vmballoon_page_size_type max_page_size; | |
277 | ||
278 | /** | |
279 | * @size: balloon actual size in basic page size (frames). | |
280 | * | |
281 | * While we currently do not support size which is bigger than 32-bit, | |
282 | * in preparation for future support, use 64-bits. | |
283 | */ | |
284 | atomic64_t size; | |
453dc659 | 285 | |
6e4453b3 NA |
286 | /** |
287 | * @target: balloon target size in basic page size (frames). | |
288 | * | |
289 | * We do not protect the target under the assumption that setting the | |
290 | * value is always done through a single write. If this assumption ever | |
291 | * breaks, we would have to use X_ONCE for accesses, and suffer the less | |
292 | * optimized code. Although we may read stale target value if multiple | |
293 | * accesses happen at once, the performance impact should be minor. | |
294 | */ | |
295 | unsigned long target; | |
453dc659 | 296 | |
6e4453b3 NA |
297 | /** |
298 | * @reset_required: reset flag | |
299 | * | |
300 | * Setting this flag may introduce races, but the code is expected to | |
301 | * handle them gracefully. In the worst case, another operation will | |
302 | * fail as reset did not take place. Clearing the flag is done while | |
303 | * holding @conf_sem for write. | |
304 | */ | |
453dc659 DT |
305 | bool reset_required; |
306 | ||
6e4453b3 NA |
307 | /** |
308 | * @capabilities: hypervisor balloon capabilities. | |
309 | * | |
310 | * Protected by @conf_sem. | |
311 | */ | |
f220a80f XD |
312 | unsigned long capabilities; |
313 | ||
6c948757 NA |
314 | /** |
315 | * @batch_page: pointer to communication batch page. | |
316 | * | |
317 | * When batching is used, batch_page points to a page, which holds up to | |
318 | * %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking. | |
319 | */ | |
320 | struct vmballoon_batch_entry *batch_page; | |
321 | ||
6e4453b3 NA |
322 | /** |
323 | * @batch_max_pages: maximum pages that can be locked/unlocked. | |
324 | * | |
325 | * Indicates the number of pages that the hypervisor can lock or unlock | |
326 | * at once, according to whether batching is enabled. If batching is | |
327 | * disabled, only a single page can be locked/unlock on each operation. | |
328 | * | |
329 | * Protected by @conf_sem. | |
330 | */ | |
f220a80f | 331 | unsigned int batch_max_pages; |
6e4453b3 NA |
332 | |
333 | /** | |
334 | * @page: page to be locked/unlocked by the hypervisor | |
335 | * | |
336 | * @page is only used when batching is disabled and a single page is | |
337 | * reclaimed on each iteration. | |
338 | * | |
339 | * Protected by @comm_lock. | |
340 | */ | |
f220a80f XD |
341 | struct page *page; |
342 | ||
453dc659 | 343 | /* statistics */ |
c7b3690f | 344 | struct vmballoon_stats *stats; |
453dc659 | 345 | |
c7b3690f | 346 | #ifdef CONFIG_DEBUG_FS |
453dc659 DT |
347 | /* debugfs file exporting statistics */ |
348 | struct dentry *dbg_entry; | |
349 | #endif | |
350 | ||
453dc659 | 351 | struct delayed_work dwork; |
48e3d668 | 352 | |
6e4453b3 NA |
353 | /** |
354 | * @vmci_doorbell. | |
355 | * | |
356 | * Protected by @conf_sem. | |
357 | */ | |
48e3d668 | 358 | struct vmci_handle vmci_doorbell; |
c7b3690f NA |
359 | |
360 | /** | |
361 | * @conf_sem: semaphore to protect the configuration and the statistics. | |
362 | */ | |
363 | struct rw_semaphore conf_sem; | |
6e4453b3 NA |
364 | |
365 | /** | |
366 | * @comm_lock: lock to protect the communication with the host. | |
367 | * | |
368 | * Lock ordering: @conf_sem -> @comm_lock . | |
369 | */ | |
370 | spinlock_t comm_lock; | |
453dc659 DT |
371 | }; |
372 | ||
373 | static struct vmballoon balloon; | |
453dc659 | 374 | |
c7b3690f NA |
375 | struct vmballoon_stats { |
376 | /* timer / doorbell operations */ | |
377 | atomic64_t general_stat[VMW_BALLOON_STAT_NUM]; | |
378 | ||
379 | /* allocation statistics for huge and small pages */ | |
380 | atomic64_t | |
381 | page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES]; | |
382 | ||
383 | /* Monitor operations: total operations, and failures */ | |
384 | atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES]; | |
385 | }; | |
386 | ||
387 | static inline bool is_vmballoon_stats_on(void) | |
388 | { | |
389 | return IS_ENABLED(CONFIG_DEBUG_FS) && | |
390 | static_branch_unlikely(&balloon_stat_enabled); | |
391 | } | |
392 | ||
393 | static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op, | |
394 | enum vmballoon_op_stat_type type) | |
395 | { | |
396 | if (is_vmballoon_stats_on()) | |
397 | atomic64_inc(&b->stats->ops[op][type]); | |
398 | } | |
399 | ||
400 | static inline void vmballoon_stats_gen_inc(struct vmballoon *b, | |
401 | enum vmballoon_stat_general stat) | |
402 | { | |
403 | if (is_vmballoon_stats_on()) | |
404 | atomic64_inc(&b->stats->general_stat[stat]); | |
405 | } | |
406 | ||
407 | static inline void vmballoon_stats_gen_add(struct vmballoon *b, | |
408 | enum vmballoon_stat_general stat, | |
409 | unsigned int val) | |
410 | { | |
411 | if (is_vmballoon_stats_on()) | |
412 | atomic64_add(val, &b->stats->general_stat[stat]); | |
413 | } | |
414 | ||
415 | static inline void vmballoon_stats_page_inc(struct vmballoon *b, | |
416 | enum vmballoon_stat_page stat, | |
6e4453b3 | 417 | enum vmballoon_page_size_type size) |
c7b3690f NA |
418 | { |
419 | if (is_vmballoon_stats_on()) | |
6e4453b3 NA |
420 | atomic64_inc(&b->stats->page_stat[stat][size]); |
421 | } | |
422 | ||
423 | static inline void vmballoon_stats_page_add(struct vmballoon *b, | |
424 | enum vmballoon_stat_page stat, | |
425 | enum vmballoon_page_size_type size, | |
426 | unsigned int val) | |
427 | { | |
428 | if (is_vmballoon_stats_on()) | |
429 | atomic64_add(val, &b->stats->page_stat[stat][size]); | |
c7b3690f NA |
430 | } |
431 | ||
10a95d5d NA |
432 | static inline unsigned long |
433 | __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1, | |
434 | unsigned long arg2, unsigned long *result) | |
435 | { | |
436 | unsigned long status, dummy1, dummy2, dummy3, local_result; | |
437 | ||
c7b3690f | 438 | vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT); |
68131184 | 439 | |
10a95d5d NA |
440 | asm volatile ("inl %%dx" : |
441 | "=a"(status), | |
442 | "=c"(dummy1), | |
443 | "=d"(dummy2), | |
444 | "=b"(local_result), | |
445 | "=S"(dummy3) : | |
446 | "0"(VMW_BALLOON_HV_MAGIC), | |
447 | "1"(cmd), | |
448 | "2"(VMW_BALLOON_HV_PORT), | |
449 | "3"(arg1), | |
450 | "4"(arg2) : | |
451 | "memory"); | |
452 | ||
453 | /* update the result if needed */ | |
454 | if (result) | |
455 | *result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 : | |
456 | local_result; | |
457 | ||
458 | /* update target when applicable */ | |
459 | if (status == VMW_BALLOON_SUCCESS && | |
460 | ((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK)) | |
6e4453b3 | 461 | WRITE_ONCE(b->target, local_result); |
10a95d5d | 462 | |
68131184 NA |
463 | if (status != VMW_BALLOON_SUCCESS && |
464 | status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) { | |
c7b3690f | 465 | vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT); |
68131184 NA |
466 | pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n", |
467 | __func__, vmballoon_cmd_names[cmd], arg1, arg2, | |
468 | status); | |
469 | } | |
470 | ||
10a95d5d NA |
471 | /* mark reset required accordingly */ |
472 | if (status == VMW_BALLOON_ERROR_RESET) | |
473 | b->reset_required = true; | |
474 | ||
475 | return status; | |
476 | } | |
477 | ||
478 | static __always_inline unsigned long | |
479 | vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1, | |
480 | unsigned long arg2) | |
481 | { | |
482 | unsigned long dummy; | |
483 | ||
484 | return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy); | |
485 | } | |
486 | ||
453dc659 DT |
487 | /* |
488 | * Send "start" command to the host, communicating supported version | |
489 | * of the protocol. | |
490 | */ | |
22d293ee | 491 | static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps) |
453dc659 | 492 | { |
10a95d5d | 493 | unsigned long status, capabilities; |
453dc659 | 494 | |
10a95d5d NA |
495 | status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0, |
496 | &capabilities); | |
f220a80f XD |
497 | |
498 | switch (status) { | |
499 | case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES: | |
500 | b->capabilities = capabilities; | |
365bd7ef | 501 | break; |
f220a80f XD |
502 | case VMW_BALLOON_SUCCESS: |
503 | b->capabilities = VMW_BALLOON_BASIC_CMDS; | |
365bd7ef PM |
504 | break; |
505 | default: | |
22d293ee | 506 | return -EIO; |
f220a80f | 507 | } |
453dc659 | 508 | |
5081efd1 NA |
509 | /* |
510 | * 2MB pages are only supported with batching. If batching is for some | |
511 | * reason disabled, do not use 2MB pages, since otherwise the legacy | |
512 | * mechanism is used with 2MB pages, causing a failure. | |
513 | */ | |
6e4453b3 | 514 | b->max_page_size = VMW_BALLOON_4K_PAGE; |
5081efd1 NA |
515 | if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) && |
516 | (b->capabilities & VMW_BALLOON_BATCHED_CMDS)) | |
6e4453b3 NA |
517 | b->max_page_size = VMW_BALLOON_2M_PAGE; |
518 | ||
365bd7ef | 519 | |
22d293ee | 520 | return 0; |
453dc659 DT |
521 | } |
522 | ||
22d293ee NA |
523 | /** |
524 | * vmballoon_send_guest_id - communicate guest type to the host. | |
525 | * | |
526 | * @b: pointer to the balloon. | |
527 | * | |
453dc659 DT |
528 | * Communicate guest type to the host so that it can adjust ballooning |
529 | * algorithm to the one most appropriate for the guest. This command | |
530 | * is normally issued after sending "start" command and is part of | |
531 | * standard reset sequence. | |
22d293ee NA |
532 | * |
533 | * Return: zero on success or appropriate error code. | |
453dc659 | 534 | */ |
22d293ee | 535 | static int vmballoon_send_guest_id(struct vmballoon *b) |
453dc659 | 536 | { |
10a95d5d | 537 | unsigned long status; |
453dc659 | 538 | |
10a95d5d NA |
539 | status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID, |
540 | VMW_BALLOON_GUEST_ID, 0); | |
453dc659 | 541 | |
22d293ee | 542 | return status == VMW_BALLOON_SUCCESS ? 0 : -EIO; |
453dc659 DT |
543 | } |
544 | ||
6e4453b3 NA |
545 | /** |
546 | * vmballoon_page_order() - return the order of the page | |
547 | * @page_size: the size of the page. | |
548 | * | |
549 | * Return: the allocation order. | |
550 | */ | |
551 | static inline | |
552 | unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size) | |
365bd7ef | 553 | { |
6e4453b3 NA |
554 | return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0; |
555 | } | |
365bd7ef | 556 | |
6e4453b3 NA |
557 | /** |
558 | * vmballoon_page_in_frames() - returns the number of frames in a page. | |
559 | * @page_size: the size of the page. | |
560 | * | |
561 | * Return: the number of 4k frames. | |
562 | */ | |
563 | static inline unsigned int | |
564 | vmballoon_page_in_frames(enum vmballoon_page_size_type page_size) | |
565 | { | |
566 | return 1 << vmballoon_page_order(page_size); | |
365bd7ef PM |
567 | } |
568 | ||
0395be3e NA |
569 | /** |
570 | * vmballoon_send_get_target() - Retrieve desired balloon size from the host. | |
571 | * | |
572 | * @b: pointer to the balloon. | |
573 | * | |
574 | * Return: zero on success, EINVAL if limit does not fit in 32-bit, as required | |
575 | * by the host-guest protocol and EIO if an error occurred in communicating with | |
576 | * the host. | |
453dc659 | 577 | */ |
0395be3e | 578 | static int vmballoon_send_get_target(struct vmballoon *b) |
453dc659 DT |
579 | { |
580 | unsigned long status; | |
453dc659 | 581 | unsigned long limit; |
453dc659 | 582 | |
ca79b0c2 | 583 | limit = totalram_pages(); |
453dc659 | 584 | |
55398302 XD |
585 | /* Ensure limit fits in 32-bits if 64-bit targets are not supported */ |
586 | if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) && | |
587 | limit != (u32)limit) | |
0395be3e | 588 | return -EINVAL; |
453dc659 | 589 | |
10a95d5d NA |
590 | status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0); |
591 | ||
0395be3e | 592 | return status == VMW_BALLOON_SUCCESS ? 0 : -EIO; |
453dc659 DT |
593 | } |
594 | ||
6e4453b3 NA |
595 | /** |
596 | * vmballoon_alloc_page_list - allocates a list of pages. | |
597 | * | |
598 | * @b: pointer to the balloon. | |
599 | * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation. | |
600 | * @req_n_pages: the number of requested pages. | |
601 | * | |
602 | * Tries to allocate @req_n_pages. Add them to the list of balloon pages in | |
603 | * @ctl.pages and updates @ctl.n_pages to reflect the number of pages. | |
604 | * | |
605 | * Return: zero on success or error code otherwise. | |
606 | */ | |
607 | static int vmballoon_alloc_page_list(struct vmballoon *b, | |
608 | struct vmballoon_ctl *ctl, | |
609 | unsigned int req_n_pages) | |
365bd7ef | 610 | { |
6e4453b3 NA |
611 | struct page *page; |
612 | unsigned int i; | |
365bd7ef | 613 | |
6e4453b3 NA |
614 | for (i = 0; i < req_n_pages; i++) { |
615 | if (ctl->page_size == VMW_BALLOON_2M_PAGE) | |
616 | page = alloc_pages(VMW_HUGE_PAGE_ALLOC_FLAGS, | |
617 | VMW_BALLOON_2M_ORDER); | |
618 | else | |
619 | page = alloc_page(VMW_PAGE_ALLOC_FLAGS); | |
365bd7ef | 620 | |
6e4453b3 NA |
621 | /* Update statistics */ |
622 | vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC, | |
623 | ctl->page_size); | |
624 | ||
625 | if (page) { | |
626 | /* Success. Add the page to the list and continue. */ | |
627 | list_add(&page->lru, &ctl->pages); | |
628 | continue; | |
629 | } | |
630 | ||
631 | /* Allocation failed. Update statistics and stop. */ | |
632 | vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL, | |
633 | ctl->page_size); | |
634 | break; | |
635 | } | |
636 | ||
637 | ctl->n_pages = i; | |
638 | ||
639 | return req_n_pages == ctl->n_pages ? 0 : -ENOMEM; | |
365bd7ef PM |
640 | } |
641 | ||
6e4453b3 NA |
642 | /** |
643 | * vmballoon_handle_one_result - Handle lock/unlock result for a single page. | |
644 | * | |
645 | * @b: pointer for %struct vmballoon. | |
646 | * @page: pointer for the page whose result should be handled. | |
647 | * @page_size: size of the page. | |
648 | * @status: status of the operation as provided by the hypervisor. | |
453dc659 | 649 | */ |
6e4453b3 NA |
650 | static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page, |
651 | enum vmballoon_page_size_type page_size, | |
652 | unsigned long status) | |
453dc659 | 653 | { |
6e4453b3 NA |
654 | /* On success do nothing. The page is already on the balloon list. */ |
655 | if (likely(status == VMW_BALLOON_SUCCESS)) | |
656 | return 0; | |
453dc659 | 657 | |
6e4453b3 NA |
658 | pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__, |
659 | page_to_pfn(page), status, | |
660 | vmballoon_page_size_names[page_size]); | |
661 | ||
662 | /* Error occurred */ | |
663 | vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC, | |
664 | page_size); | |
665 | ||
666 | return -EIO; | |
453dc659 DT |
667 | } |
668 | ||
df8d0d42 NA |
669 | /** |
670 | * vmballoon_status_page - returns the status of (un)lock operation | |
671 | * | |
672 | * @b: pointer to the balloon. | |
673 | * @idx: index for the page for which the operation is performed. | |
674 | * @p: pointer to where the page struct is returned. | |
675 | * | |
676 | * Following a lock or unlock operation, returns the status of the operation for | |
677 | * an individual page. Provides the page that the operation was performed on on | |
678 | * the @page argument. | |
679 | * | |
680 | * Returns: The status of a lock or unlock operation for an individual page. | |
453dc659 | 681 | */ |
df8d0d42 NA |
682 | static unsigned long vmballoon_status_page(struct vmballoon *b, int idx, |
683 | struct page **p) | |
453dc659 | 684 | { |
df8d0d42 NA |
685 | if (static_branch_likely(&vmw_balloon_batching)) { |
686 | /* batching mode */ | |
687 | *p = pfn_to_page(b->batch_page[idx].pfn); | |
688 | return b->batch_page[idx].status; | |
689 | } | |
10a95d5d | 690 | |
df8d0d42 NA |
691 | /* non-batching mode */ |
692 | *p = b->page; | |
453dc659 | 693 | |
df8d0d42 NA |
694 | /* |
695 | * If a failure occurs, the indication will be provided in the status | |
696 | * of the entire operation, which is considered before the individual | |
697 | * page status. So for non-batching mode, the indication is always of | |
698 | * success. | |
699 | */ | |
700 | return VMW_BALLOON_SUCCESS; | |
701 | } | |
453dc659 | 702 | |
df8d0d42 NA |
703 | /** |
704 | * vmballoon_lock_op - notifies the host about inflated/deflated pages. | |
705 | * @b: pointer to the balloon. | |
706 | * @num_pages: number of inflated/deflated pages. | |
6e4453b3 NA |
707 | * @page_size: size of the page. |
708 | * @op: the type of operation (lock or unlock). | |
df8d0d42 NA |
709 | * |
710 | * Notify the host about page(s) that were ballooned (or removed from the | |
711 | * balloon) so that host can use it without fear that guest will need it (or | |
712 | * stop using them since the VM does). Host may reject some pages, we need to | |
713 | * check the return value and maybe submit a different page. The pages that are | |
714 | * inflated/deflated are pointed by @b->page. | |
715 | * | |
716 | * Return: result as provided by the hypervisor. | |
717 | */ | |
718 | static unsigned long vmballoon_lock_op(struct vmballoon *b, | |
719 | unsigned int num_pages, | |
6e4453b3 NA |
720 | enum vmballoon_page_size_type page_size, |
721 | enum vmballoon_op op) | |
df8d0d42 NA |
722 | { |
723 | unsigned long cmd, pfn; | |
724 | ||
6e4453b3 NA |
725 | lockdep_assert_held(&b->comm_lock); |
726 | ||
df8d0d42 | 727 | if (static_branch_likely(&vmw_balloon_batching)) { |
6e4453b3 NA |
728 | if (op == VMW_BALLOON_INFLATE) |
729 | cmd = page_size == VMW_BALLOON_2M_PAGE ? | |
730 | VMW_BALLOON_CMD_BATCHED_2M_LOCK : | |
731 | VMW_BALLOON_CMD_BATCHED_LOCK; | |
df8d0d42 | 732 | else |
6e4453b3 NA |
733 | cmd = page_size == VMW_BALLOON_2M_PAGE ? |
734 | VMW_BALLOON_CMD_BATCHED_2M_UNLOCK : | |
735 | VMW_BALLOON_CMD_BATCHED_UNLOCK; | |
df8d0d42 NA |
736 | |
737 | pfn = PHYS_PFN(virt_to_phys(b->batch_page)); | |
738 | } else { | |
6e4453b3 NA |
739 | cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK : |
740 | VMW_BALLOON_CMD_UNLOCK; | |
df8d0d42 NA |
741 | pfn = page_to_pfn(b->page); |
742 | ||
743 | /* In non-batching mode, PFNs must fit in 32-bit */ | |
744 | if (unlikely(pfn != (u32)pfn)) | |
745 | return VMW_BALLOON_ERROR_PPN_INVALID; | |
ef0f8f11 | 746 | } |
453dc659 | 747 | |
df8d0d42 | 748 | return vmballoon_cmd(b, cmd, pfn, num_pages); |
453dc659 DT |
749 | } |
750 | ||
6e4453b3 NA |
751 | /** |
752 | * vmballoon_add_page - adds a page towards lock/unlock operation. | |
753 | * | |
754 | * @b: pointer to the balloon. | |
755 | * @idx: index of the page to be ballooned in this batch. | |
756 | * @p: pointer to the page that is about to be ballooned. | |
757 | * | |
758 | * Adds the page to be ballooned. Must be called while holding @comm_lock. | |
759 | */ | |
760 | static void vmballoon_add_page(struct vmballoon *b, unsigned int idx, | |
761 | struct page *p) | |
762 | { | |
763 | lockdep_assert_held(&b->comm_lock); | |
764 | ||
765 | if (static_branch_likely(&vmw_balloon_batching)) | |
766 | b->batch_page[idx] = (struct vmballoon_batch_entry) | |
767 | { .pfn = page_to_pfn(p) }; | |
768 | else | |
769 | b->page = p; | |
770 | } | |
771 | ||
772 | /** | |
773 | * vmballoon_lock - lock or unlock a batch of pages. | |
774 | * | |
775 | * @b: pointer to the balloon. | |
776 | * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation. | |
777 | * | |
778 | * Notifies the host of about ballooned pages (after inflation or deflation, | |
779 | * according to @ctl). If the host rejects the page put it on the | |
780 | * @ctl refuse list. These refused page are then released when moving to the | |
781 | * next size of pages. | |
782 | * | |
783 | * Note that we neither free any @page here nor put them back on the ballooned | |
784 | * pages list. Instead we queue it for later processing. We do that for several | |
785 | * reasons. First, we do not want to free the page under the lock. Second, it | |
786 | * allows us to unify the handling of lock and unlock. In the inflate case, the | |
787 | * caller will check if there are too many refused pages and release them. | |
788 | * Although it is not identical to the past behavior, it should not affect | |
789 | * performance. | |
790 | */ | |
791 | static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl) | |
f220a80f | 792 | { |
df8d0d42 | 793 | unsigned long batch_status; |
6e4453b3 NA |
794 | struct page *page; |
795 | unsigned int i, num_pages; | |
796 | ||
797 | num_pages = ctl->n_pages; | |
798 | if (num_pages == 0) | |
799 | return 0; | |
f220a80f | 800 | |
6e4453b3 NA |
801 | /* communication with the host is done under the communication lock */ |
802 | spin_lock(&b->comm_lock); | |
f220a80f | 803 | |
6e4453b3 NA |
804 | i = 0; |
805 | list_for_each_entry(page, &ctl->pages, lru) | |
806 | vmballoon_add_page(b, i++, page); | |
807 | ||
808 | batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size, | |
809 | ctl->op); | |
810 | ||
811 | /* | |
812 | * Iterate over the pages in the provided list. Since we are changing | |
813 | * @ctl->n_pages we are saving the original value in @num_pages and | |
814 | * use this value to bound the loop. | |
815 | */ | |
f220a80f | 816 | for (i = 0; i < num_pages; i++) { |
df8d0d42 | 817 | unsigned long status; |
f220a80f | 818 | |
6e4453b3 | 819 | status = vmballoon_status_page(b, i, &page); |
df8d0d42 NA |
820 | |
821 | /* | |
822 | * Failure of the whole batch overrides a single operation | |
823 | * results. | |
824 | */ | |
825 | if (batch_status != VMW_BALLOON_SUCCESS) | |
826 | status = batch_status; | |
f220a80f | 827 | |
6e4453b3 NA |
828 | /* Continue if no error happened */ |
829 | if (!vmballoon_handle_one_result(b, page, ctl->page_size, | |
830 | status)) | |
df8d0d42 | 831 | continue; |
df8d0d42 | 832 | |
8fa3c61a | 833 | /* |
6e4453b3 NA |
834 | * Error happened. Move the pages to the refused list and update |
835 | * the pages number. | |
8fa3c61a | 836 | */ |
6e4453b3 NA |
837 | list_move(&page->lru, &ctl->refused_pages); |
838 | ctl->n_pages--; | |
839 | ctl->n_refused_pages++; | |
f220a80f XD |
840 | } |
841 | ||
6e4453b3 NA |
842 | spin_unlock(&b->comm_lock); |
843 | ||
df8d0d42 | 844 | return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO; |
f220a80f XD |
845 | } |
846 | ||
6e4453b3 NA |
847 | /** |
848 | * vmballoon_release_page_list() - Releases a page list | |
849 | * | |
850 | * @page_list: list of pages to release. | |
851 | * @n_pages: pointer to the number of pages. | |
852 | * @page_size: whether the pages in the list are 2MB (or else 4KB). | |
853 | * | |
854 | * Releases the list of pages and zeros the number of pages. | |
453dc659 | 855 | */ |
6e4453b3 NA |
856 | static void vmballoon_release_page_list(struct list_head *page_list, |
857 | int *n_pages, | |
858 | enum vmballoon_page_size_type page_size) | |
453dc659 | 859 | { |
6e4453b3 | 860 | struct page *page, *tmp; |
df8d0d42 | 861 | |
6e4453b3 NA |
862 | list_for_each_entry_safe(page, tmp, page_list, lru) { |
863 | list_del(&page->lru); | |
864 | __free_pages(page, vmballoon_page_order(page_size)); | |
f220a80f XD |
865 | } |
866 | ||
6e4453b3 | 867 | *n_pages = 0; |
f220a80f XD |
868 | } |
869 | ||
6e4453b3 | 870 | |
453dc659 DT |
871 | /* |
872 | * Release pages that were allocated while attempting to inflate the | |
873 | * balloon but were refused by the host for one reason or another. | |
874 | */ | |
365bd7ef | 875 | static void vmballoon_release_refused_pages(struct vmballoon *b, |
6e4453b3 | 876 | struct vmballoon_ctl *ctl) |
453dc659 | 877 | { |
6e4453b3 NA |
878 | vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE, |
879 | ctl->page_size); | |
453dc659 | 880 | |
6e4453b3 NA |
881 | vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages, |
882 | ctl->page_size); | |
f220a80f XD |
883 | } |
884 | ||
8b079cd0 NA |
885 | /** |
886 | * vmballoon_change - retrieve the required balloon change | |
887 | * | |
888 | * @b: pointer for the balloon. | |
889 | * | |
890 | * Return: the required change for the balloon size. A positive number | |
891 | * indicates inflation, a negative number indicates a deflation. | |
892 | */ | |
893 | static int64_t vmballoon_change(struct vmballoon *b) | |
894 | { | |
895 | int64_t size, target; | |
896 | ||
6e4453b3 NA |
897 | size = atomic64_read(&b->size); |
898 | target = READ_ONCE(b->target); | |
8b079cd0 NA |
899 | |
900 | /* | |
901 | * We must cast first because of int sizes | |
902 | * Otherwise we might get huge positives instead of negatives | |
903 | */ | |
904 | ||
905 | if (b->reset_required) | |
906 | return 0; | |
907 | ||
908 | /* consider a 2MB slack on deflate, unless the balloon is emptied */ | |
6e4453b3 NA |
909 | if (target < size && target != 0 && |
910 | size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE)) | |
8b079cd0 NA |
911 | return 0; |
912 | ||
913 | return target - size; | |
914 | } | |
915 | ||
6e4453b3 NA |
916 | /** |
917 | * vmballoon_enqueue_page_list() - Enqueues list of pages after inflation. | |
918 | * | |
919 | * @b: pointer to balloon. | |
920 | * @pages: list of pages to enqueue. | |
921 | * @n_pages: pointer to number of pages in list. The value is zeroed. | |
922 | * @page_size: whether the pages are 2MB or 4KB pages. | |
923 | * | |
924 | * Enqueues the provides list of pages in the ballooned page list, clears the | |
925 | * list and zeroes the number of pages that was provided. | |
926 | */ | |
927 | static void vmballoon_enqueue_page_list(struct vmballoon *b, | |
928 | struct list_head *pages, | |
929 | unsigned int *n_pages, | |
930 | enum vmballoon_page_size_type page_size) | |
931 | { | |
932 | struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size]; | |
933 | ||
934 | list_splice_init(pages, &page_size_info->pages); | |
935 | *n_pages = 0; | |
936 | } | |
937 | ||
938 | /** | |
939 | * vmballoon_dequeue_page_list() - Dequeues page lists for deflation. | |
940 | * | |
941 | * @b: pointer to balloon. | |
942 | * @pages: list of pages to enqueue. | |
943 | * @n_pages: pointer to number of pages in list. The value is zeroed. | |
944 | * @page_size: whether the pages are 2MB or 4KB pages. | |
945 | * @n_req_pages: the number of requested pages. | |
946 | * | |
947 | * Dequeues the number of requested pages from the balloon for deflation. The | |
948 | * number of dequeued pages may be lower, if not enough pages in the requested | |
949 | * size are available. | |
950 | */ | |
951 | static void vmballoon_dequeue_page_list(struct vmballoon *b, | |
952 | struct list_head *pages, | |
953 | unsigned int *n_pages, | |
954 | enum vmballoon_page_size_type page_size, | |
955 | unsigned int n_req_pages) | |
956 | { | |
957 | struct vmballoon_page_size *page_size_info = &b->page_sizes[page_size]; | |
958 | struct page *page, *tmp; | |
959 | unsigned int i = 0; | |
960 | ||
961 | list_for_each_entry_safe(page, tmp, &page_size_info->pages, lru) { | |
962 | list_move(&page->lru, pages); | |
963 | if (++i == n_req_pages) | |
964 | break; | |
965 | } | |
966 | *n_pages = i; | |
967 | } | |
968 | ||
969 | /** | |
970 | * vmballoon_inflate() - Inflate the balloon towards its target size. | |
971 | * | |
972 | * @b: pointer to the balloon. | |
453dc659 DT |
973 | */ |
974 | static void vmballoon_inflate(struct vmballoon *b) | |
975 | { | |
6e4453b3 NA |
976 | int64_t to_inflate_frames; |
977 | struct vmballoon_ctl ctl = { | |
978 | .pages = LIST_HEAD_INIT(ctl.pages), | |
979 | .refused_pages = LIST_HEAD_INIT(ctl.refused_pages), | |
980 | .page_size = b->max_page_size, | |
981 | .op = VMW_BALLOON_INFLATE | |
982 | }; | |
453dc659 | 983 | |
6e4453b3 NA |
984 | while ((to_inflate_frames = vmballoon_change(b)) > 0) { |
985 | unsigned int to_inflate_pages, page_in_frames; | |
986 | int alloc_error, lock_error = 0; | |
453dc659 | 987 | |
6e4453b3 NA |
988 | VM_BUG_ON(!list_empty(&ctl.pages)); |
989 | VM_BUG_ON(ctl.n_pages != 0); | |
453dc659 | 990 | |
6e4453b3 | 991 | page_in_frames = vmballoon_page_in_frames(ctl.page_size); |
453dc659 | 992 | |
6e4453b3 NA |
993 | to_inflate_pages = min_t(unsigned long, b->batch_max_pages, |
994 | DIV_ROUND_UP_ULL(to_inflate_frames, | |
995 | page_in_frames)); | |
996 | ||
997 | /* Start by allocating */ | |
998 | alloc_error = vmballoon_alloc_page_list(b, &ctl, | |
999 | to_inflate_pages); | |
1000 | ||
1001 | /* Actually lock the pages by telling the hypervisor */ | |
1002 | lock_error = vmballoon_lock(b, &ctl); | |
1003 | ||
1004 | /* | |
1005 | * If an error indicates that something serious went wrong, | |
1006 | * stop the inflation. | |
1007 | */ | |
1008 | if (lock_error) | |
622074a9 | 1009 | break; |
453dc659 | 1010 | |
6e4453b3 NA |
1011 | /* Update the balloon size */ |
1012 | atomic64_add(ctl.n_pages * page_in_frames, &b->size); | |
8fa3c61a | 1013 | |
6e4453b3 NA |
1014 | vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages, |
1015 | ctl.page_size); | |
10a95d5d | 1016 | |
6e4453b3 NA |
1017 | /* |
1018 | * If allocation failed or the number of refused pages exceeds | |
1019 | * the maximum allowed, move to the next page size. | |
1020 | */ | |
1021 | if (alloc_error || | |
1022 | ctl.n_refused_pages >= VMW_BALLOON_MAX_REFUSED) { | |
1023 | if (ctl.page_size == VMW_BALLOON_4K_PAGE) | |
1024 | break; | |
8fa3c61a NA |
1025 | |
1026 | /* | |
6e4453b3 NA |
1027 | * Ignore errors from locking as we now switch to 4k |
1028 | * pages and we might get different errors. | |
8fa3c61a | 1029 | */ |
6e4453b3 NA |
1030 | vmballoon_release_refused_pages(b, &ctl); |
1031 | ctl.page_size--; | |
f220a80f | 1032 | } |
ef0f8f11 | 1033 | |
33d268ed | 1034 | cond_resched(); |
453dc659 DT |
1035 | } |
1036 | ||
6e4453b3 NA |
1037 | /* |
1038 | * Release pages that were allocated while attempting to inflate the | |
1039 | * balloon but were refused by the host for one reason or another, | |
1040 | * and update the statistics. | |
1041 | */ | |
1042 | if (ctl.n_refused_pages != 0) | |
1043 | vmballoon_release_refused_pages(b, &ctl); | |
453dc659 DT |
1044 | } |
1045 | ||
6e4453b3 NA |
1046 | /** |
1047 | * vmballoon_deflate() - Decrease the size of the balloon. | |
1048 | * | |
1049 | * @b: pointer to the balloon | |
1050 | * @n_frames: the number of frames to deflate. If zero, automatically | |
1051 | * calculated according to the target size. | |
1052 | * @coordinated: whether to coordinate with the host | |
1053 | * | |
453dc659 | 1054 | * Decrease the size of the balloon allowing guest to use more memory. |
6e4453b3 NA |
1055 | * |
1056 | * Return: The number of deflated frames (i.e., basic page size units) | |
453dc659 | 1057 | */ |
6e4453b3 NA |
1058 | static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames, |
1059 | bool coordinated) | |
453dc659 | 1060 | { |
6e4453b3 NA |
1061 | unsigned long deflated_frames = 0; |
1062 | unsigned long tried_frames = 0; | |
1063 | struct vmballoon_ctl ctl = { | |
1064 | .pages = LIST_HEAD_INIT(ctl.pages), | |
1065 | .refused_pages = LIST_HEAD_INIT(ctl.refused_pages), | |
1066 | .page_size = VMW_BALLOON_4K_PAGE, | |
1067 | .op = VMW_BALLOON_DEFLATE | |
1068 | }; | |
453dc659 | 1069 | |
453dc659 | 1070 | /* free pages to reach target */ |
6e4453b3 NA |
1071 | while (true) { |
1072 | unsigned int to_deflate_pages, n_unlocked_frames; | |
1073 | unsigned int page_in_frames; | |
1074 | int64_t to_deflate_frames; | |
1075 | bool deflated_all; | |
1076 | ||
1077 | page_in_frames = vmballoon_page_in_frames(ctl.page_size); | |
1078 | ||
1079 | VM_BUG_ON(!list_empty(&ctl.pages)); | |
1080 | VM_BUG_ON(ctl.n_pages); | |
1081 | VM_BUG_ON(!list_empty(&ctl.refused_pages)); | |
1082 | VM_BUG_ON(ctl.n_refused_pages); | |
1083 | ||
1084 | /* | |
1085 | * If we were requested a specific number of frames, we try to | |
1086 | * deflate this number of frames. Otherwise, deflation is | |
1087 | * performed according to the target and balloon size. | |
1088 | */ | |
1089 | to_deflate_frames = n_frames ? n_frames - tried_frames : | |
1090 | -vmballoon_change(b); | |
1091 | ||
1092 | /* break if no work to do */ | |
1093 | if (to_deflate_frames <= 0) | |
1094 | break; | |
1095 | ||
1096 | /* | |
1097 | * Calculate the number of frames based on current page size, | |
1098 | * but limit the deflated frames to a single chunk | |
1099 | */ | |
1100 | to_deflate_pages = min_t(unsigned long, b->batch_max_pages, | |
1101 | DIV_ROUND_UP_ULL(to_deflate_frames, | |
1102 | page_in_frames)); | |
1103 | ||
1104 | /* First take the pages from the balloon pages. */ | |
1105 | vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages, | |
1106 | ctl.page_size, to_deflate_pages); | |
f220a80f | 1107 | |
6e4453b3 NA |
1108 | /* |
1109 | * Before pages are moving to the refused list, count their | |
1110 | * frames as frames that we tried to deflate. | |
1111 | */ | |
1112 | tried_frames += ctl.n_pages * page_in_frames; | |
1113 | ||
1114 | /* | |
1115 | * Unlock the pages by communicating with the hypervisor if the | |
1116 | * communication is coordinated (i.e., not pop). We ignore the | |
1117 | * return code. Instead we check if all the pages we manage to | |
1118 | * unlock all the pages. If we failed, we will move to the next | |
1119 | * page size, and would eventually try again later. | |
1120 | */ | |
1121 | if (coordinated) | |
1122 | vmballoon_lock(b, &ctl); | |
1123 | ||
1124 | /* | |
1125 | * Check if we deflated enough. We will move to the next page | |
1126 | * size if we did not manage to do so. This calculation takes | |
1127 | * place now, as once the pages are released, the number of | |
1128 | * pages is zeroed. | |
1129 | */ | |
1130 | deflated_all = (ctl.n_pages == to_deflate_pages); | |
33d268ed | 1131 | |
6e4453b3 NA |
1132 | /* Update local and global counters */ |
1133 | n_unlocked_frames = ctl.n_pages * page_in_frames; | |
1134 | atomic64_sub(n_unlocked_frames, &b->size); | |
1135 | deflated_frames += n_unlocked_frames; | |
453dc659 | 1136 | |
6e4453b3 NA |
1137 | vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE, |
1138 | ctl.page_size, ctl.n_pages); | |
33d268ed | 1139 | |
6e4453b3 NA |
1140 | /* free the ballooned pages */ |
1141 | vmballoon_release_page_list(&ctl.pages, &ctl.n_pages, | |
1142 | ctl.page_size); | |
1143 | ||
1144 | /* Return the refused pages to the ballooned list. */ | |
1145 | vmballoon_enqueue_page_list(b, &ctl.refused_pages, | |
1146 | &ctl.n_refused_pages, | |
1147 | ctl.page_size); | |
1148 | ||
1149 | /* If we failed to unlock all the pages, move to next size. */ | |
1150 | if (!deflated_all) { | |
1151 | if (ctl.page_size == b->max_page_size) | |
1152 | break; | |
1153 | ctl.page_size++; | |
365bd7ef | 1154 | } |
453dc659 | 1155 | |
6e4453b3 | 1156 | cond_resched(); |
365bd7ef | 1157 | } |
6e4453b3 NA |
1158 | |
1159 | return deflated_frames; | |
f220a80f XD |
1160 | } |
1161 | ||
df8d0d42 NA |
1162 | /** |
1163 | * vmballoon_deinit_batching - disables batching mode. | |
1164 | * | |
1165 | * @b: pointer to &struct vmballoon. | |
1166 | * | |
1167 | * Disables batching, by deallocating the page for communication with the | |
1168 | * hypervisor and disabling the static key to indicate that batching is off. | |
1169 | */ | |
1170 | static void vmballoon_deinit_batching(struct vmballoon *b) | |
1171 | { | |
1172 | free_page((unsigned long)b->batch_page); | |
1173 | b->batch_page = NULL; | |
1174 | static_branch_disable(&vmw_balloon_batching); | |
1175 | b->batch_max_pages = 1; | |
1176 | } | |
f220a80f | 1177 | |
df8d0d42 NA |
1178 | /** |
1179 | * vmballoon_init_batching - enable batching mode. | |
1180 | * | |
1181 | * @b: pointer to &struct vmballoon. | |
1182 | * | |
1183 | * Enables batching, by allocating a page for communication with the hypervisor | |
1184 | * and enabling the static_key to use batching. | |
1185 | * | |
1186 | * Return: zero on success or an appropriate error-code. | |
1187 | */ | |
1188 | static int vmballoon_init_batching(struct vmballoon *b) | |
f220a80f | 1189 | { |
b23220fe | 1190 | struct page *page; |
f220a80f | 1191 | |
b23220fe GK |
1192 | page = alloc_page(GFP_KERNEL | __GFP_ZERO); |
1193 | if (!page) | |
df8d0d42 | 1194 | return -ENOMEM; |
f220a80f | 1195 | |
b23220fe | 1196 | b->batch_page = page_address(page); |
df8d0d42 NA |
1197 | b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry); |
1198 | ||
1199 | static_branch_enable(&vmw_balloon_batching); | |
1200 | ||
1201 | return 0; | |
f220a80f XD |
1202 | } |
1203 | ||
48e3d668 PM |
1204 | /* |
1205 | * Receive notification and resize balloon | |
1206 | */ | |
1207 | static void vmballoon_doorbell(void *client_data) | |
1208 | { | |
1209 | struct vmballoon *b = client_data; | |
1210 | ||
c7b3690f | 1211 | vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL); |
48e3d668 PM |
1212 | |
1213 | mod_delayed_work(system_freezable_wq, &b->dwork, 0); | |
1214 | } | |
1215 | ||
1216 | /* | |
1217 | * Clean up vmci doorbell | |
1218 | */ | |
1219 | static void vmballoon_vmci_cleanup(struct vmballoon *b) | |
1220 | { | |
10a95d5d NA |
1221 | vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET, |
1222 | VMCI_INVALID_ID, VMCI_INVALID_ID); | |
48e3d668 | 1223 | |
48e3d668 PM |
1224 | if (!vmci_handle_is_invalid(b->vmci_doorbell)) { |
1225 | vmci_doorbell_destroy(b->vmci_doorbell); | |
1226 | b->vmci_doorbell = VMCI_INVALID_HANDLE; | |
1227 | } | |
1228 | } | |
1229 | ||
22d293ee NA |
1230 | /** |
1231 | * vmballoon_vmci_init - Initialize vmci doorbell. | |
1232 | * | |
1233 | * @b: pointer to the balloon. | |
1234 | * | |
1235 | * Return: zero on success or when wakeup command not supported. Error-code | |
1236 | * otherwise. | |
1237 | * | |
1238 | * Initialize vmci doorbell, to get notified as soon as balloon changes. | |
48e3d668 PM |
1239 | */ |
1240 | static int vmballoon_vmci_init(struct vmballoon *b) | |
1241 | { | |
10a95d5d | 1242 | unsigned long error; |
48e3d668 | 1243 | |
ce664331 NA |
1244 | if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0) |
1245 | return 0; | |
48e3d668 | 1246 | |
ce664331 NA |
1247 | error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB, |
1248 | VMCI_PRIVILEGE_FLAG_RESTRICTED, | |
1249 | vmballoon_doorbell, b); | |
48e3d668 | 1250 | |
ce664331 NA |
1251 | if (error != VMCI_SUCCESS) |
1252 | goto fail; | |
1253 | ||
10a95d5d NA |
1254 | error = __vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET, |
1255 | b->vmci_doorbell.context, | |
1256 | b->vmci_doorbell.resource, NULL); | |
ce664331 | 1257 | |
ce664331 NA |
1258 | if (error != VMW_BALLOON_SUCCESS) |
1259 | goto fail; | |
48e3d668 PM |
1260 | |
1261 | return 0; | |
ce664331 NA |
1262 | fail: |
1263 | vmballoon_vmci_cleanup(b); | |
1264 | return -EIO; | |
48e3d668 PM |
1265 | } |
1266 | ||
6e4453b3 NA |
1267 | /** |
1268 | * vmballoon_pop - Quickly release all pages allocate for the balloon. | |
1269 | * | |
1270 | * @b: pointer to the balloon. | |
1271 | * | |
1272 | * This function is called when host decides to "reset" balloon for one reason | |
1273 | * or another. Unlike normal "deflate" we do not (shall not) notify host of the | |
1274 | * pages being released. | |
1275 | */ | |
1276 | static void vmballoon_pop(struct vmballoon *b) | |
1277 | { | |
1278 | unsigned long size; | |
1279 | ||
1280 | while ((size = atomic64_read(&b->size))) | |
1281 | vmballoon_deflate(b, size, false); | |
1282 | } | |
1283 | ||
f220a80f XD |
1284 | /* |
1285 | * Perform standard reset sequence by popping the balloon (in case it | |
1286 | * is not empty) and then restarting protocol. This operation normally | |
1287 | * happens when host responds with VMW_BALLOON_ERROR_RESET to a command. | |
1288 | */ | |
1289 | static void vmballoon_reset(struct vmballoon *b) | |
1290 | { | |
48e3d668 PM |
1291 | int error; |
1292 | ||
c7b3690f NA |
1293 | down_write(&b->conf_sem); |
1294 | ||
48e3d668 PM |
1295 | vmballoon_vmci_cleanup(b); |
1296 | ||
f220a80f XD |
1297 | /* free all pages, skipping monitor unlock */ |
1298 | vmballoon_pop(b); | |
1299 | ||
22d293ee | 1300 | if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES)) |
f220a80f XD |
1301 | return; |
1302 | ||
1303 | if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) { | |
df8d0d42 | 1304 | if (vmballoon_init_batching(b)) { |
f220a80f XD |
1305 | /* |
1306 | * We failed to initialize batching, inform the monitor | |
1307 | * about it by sending a null capability. | |
1308 | * | |
1309 | * The guest will retry in one second. | |
1310 | */ | |
1311 | vmballoon_send_start(b, 0); | |
1312 | return; | |
1313 | } | |
1314 | } else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) { | |
df8d0d42 | 1315 | vmballoon_deinit_batching(b); |
f220a80f XD |
1316 | } |
1317 | ||
8840a6f4 | 1318 | vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET); |
f220a80f | 1319 | b->reset_required = false; |
48e3d668 PM |
1320 | |
1321 | error = vmballoon_vmci_init(b); | |
1322 | if (error) | |
1323 | pr_err("failed to initialize vmci doorbell\n"); | |
1324 | ||
22d293ee | 1325 | if (vmballoon_send_guest_id(b)) |
f220a80f | 1326 | pr_err("failed to send guest ID to the host\n"); |
c7b3690f NA |
1327 | |
1328 | up_write(&b->conf_sem); | |
453dc659 DT |
1329 | } |
1330 | ||
8b079cd0 NA |
1331 | /** |
1332 | * vmballoon_work - periodic balloon worker for reset, inflation and deflation. | |
1333 | * | |
1334 | * @work: pointer to the &work_struct which is provided by the workqueue. | |
1335 | * | |
1336 | * Resets the protocol if needed, gets the new size and adjusts balloon as | |
1337 | * needed. Repeat in 1 sec. | |
453dc659 DT |
1338 | */ |
1339 | static void vmballoon_work(struct work_struct *work) | |
1340 | { | |
1341 | struct delayed_work *dwork = to_delayed_work(work); | |
1342 | struct vmballoon *b = container_of(dwork, struct vmballoon, dwork); | |
8b079cd0 | 1343 | int64_t change = 0; |
453dc659 | 1344 | |
453dc659 DT |
1345 | if (b->reset_required) |
1346 | vmballoon_reset(b); | |
1347 | ||
c7b3690f NA |
1348 | down_read(&b->conf_sem); |
1349 | ||
1350 | /* | |
1351 | * Update the stats while holding the semaphore to ensure that | |
1352 | * @stats_enabled is consistent with whether the stats are actually | |
1353 | * enabled | |
1354 | */ | |
1355 | vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER); | |
1356 | ||
0395be3e | 1357 | if (!vmballoon_send_get_target(b)) |
8b079cd0 NA |
1358 | change = vmballoon_change(b); |
1359 | ||
1360 | if (change != 0) { | |
6e4453b3 NA |
1361 | pr_debug("%s - size: %llu, target %lu\n", __func__, |
1362 | atomic64_read(&b->size), READ_ONCE(b->target)); | |
453dc659 | 1363 | |
8b079cd0 | 1364 | if (change > 0) |
453dc659 | 1365 | vmballoon_inflate(b); |
8b079cd0 | 1366 | else /* (change < 0) */ |
6e4453b3 | 1367 | vmballoon_deflate(b, 0, true); |
453dc659 DT |
1368 | } |
1369 | ||
c7b3690f NA |
1370 | up_read(&b->conf_sem); |
1371 | ||
beda94da DT |
1372 | /* |
1373 | * We are using a freezable workqueue so that balloon operations are | |
1374 | * stopped while the system transitions to/from sleep/hibernation. | |
1375 | */ | |
1376 | queue_delayed_work(system_freezable_wq, | |
1377 | dwork, round_jiffies_relative(HZ)); | |
c7b3690f | 1378 | |
453dc659 DT |
1379 | } |
1380 | ||
1381 | /* | |
1382 | * DEBUGFS Interface | |
1383 | */ | |
1384 | #ifdef CONFIG_DEBUG_FS | |
1385 | ||
c7b3690f NA |
1386 | static const char * const vmballoon_stat_page_names[] = { |
1387 | [VMW_BALLOON_PAGE_STAT_ALLOC] = "alloc", | |
1388 | [VMW_BALLOON_PAGE_STAT_ALLOC_FAIL] = "allocFail", | |
1389 | [VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC] = "errAlloc", | |
1390 | [VMW_BALLOON_PAGE_STAT_REFUSED_FREE] = "errFree", | |
1391 | [VMW_BALLOON_PAGE_STAT_FREE] = "free" | |
1392 | }; | |
1393 | ||
1394 | static const char * const vmballoon_stat_names[] = { | |
1395 | [VMW_BALLOON_STAT_TIMER] = "timer", | |
8840a6f4 NA |
1396 | [VMW_BALLOON_STAT_DOORBELL] = "doorbell", |
1397 | [VMW_BALLOON_STAT_RESET] = "reset", | |
c7b3690f NA |
1398 | }; |
1399 | ||
c7b3690f NA |
1400 | static int vmballoon_enable_stats(struct vmballoon *b) |
1401 | { | |
1402 | int r = 0; | |
1403 | ||
1404 | down_write(&b->conf_sem); | |
1405 | ||
1406 | /* did we somehow race with another reader which enabled stats? */ | |
1407 | if (b->stats) | |
1408 | goto out; | |
1409 | ||
1410 | b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL); | |
1411 | ||
1412 | if (!b->stats) { | |
1413 | /* allocation failed */ | |
1414 | r = -ENOMEM; | |
1415 | goto out; | |
1416 | } | |
1417 | static_key_enable(&balloon_stat_enabled.key); | |
1418 | out: | |
1419 | up_write(&b->conf_sem); | |
1420 | return r; | |
1421 | } | |
1422 | ||
1423 | /** | |
1424 | * vmballoon_debug_show - shows statistics of balloon operations. | |
1425 | * @f: pointer to the &struct seq_file. | |
1426 | * @offset: ignored. | |
1427 | * | |
1428 | * Provides the statistics that can be accessed in vmmemctl in the debugfs. | |
1429 | * To avoid the overhead - mainly that of memory - of collecting the statistics, | |
1430 | * we only collect statistics after the first time the counters are read. | |
1431 | * | |
1432 | * Return: zero on success or an error code. | |
1433 | */ | |
453dc659 DT |
1434 | static int vmballoon_debug_show(struct seq_file *f, void *offset) |
1435 | { | |
1436 | struct vmballoon *b = f->private; | |
c7b3690f NA |
1437 | int i, j; |
1438 | ||
1439 | /* enables stats if they are disabled */ | |
1440 | if (!b->stats) { | |
1441 | int r = vmballoon_enable_stats(b); | |
1442 | ||
1443 | if (r) | |
1444 | return r; | |
1445 | } | |
453dc659 | 1446 | |
b36e89da | 1447 | /* format capabilities info */ |
6e4453b3 | 1448 | seq_printf(f, "%-22s: %#16x\n", "balloon capabilities", |
c7b3690f | 1449 | VMW_BALLOON_CAPABILITIES); |
6e4453b3 | 1450 | seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities); |
c7b3690f NA |
1451 | seq_printf(f, "%-22s: %16s\n", "is resetting", |
1452 | b->reset_required ? "y" : "n"); | |
b36e89da | 1453 | |
453dc659 | 1454 | /* format size info */ |
6e4453b3 NA |
1455 | seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target)); |
1456 | seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size)); | |
453dc659 | 1457 | |
68131184 NA |
1458 | for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) { |
1459 | if (vmballoon_cmd_names[i] == NULL) | |
1460 | continue; | |
1461 | ||
c7b3690f NA |
1462 | seq_printf(f, "%-22s: %16llu (%llu failed)\n", |
1463 | vmballoon_cmd_names[i], | |
1464 | atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]), | |
1465 | atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT])); | |
68131184 NA |
1466 | } |
1467 | ||
c7b3690f NA |
1468 | for (i = 0; i < VMW_BALLOON_STAT_NUM; i++) |
1469 | seq_printf(f, "%-22s: %16llu\n", | |
1470 | vmballoon_stat_names[i], | |
1471 | atomic64_read(&b->stats->general_stat[i])); | |
1472 | ||
1473 | for (i = 0; i < VMW_BALLOON_PAGE_STAT_NUM; i++) { | |
1474 | for (j = 0; j < VMW_BALLOON_NUM_PAGE_SIZES; j++) | |
1475 | seq_printf(f, "%-18s(%s): %16llu\n", | |
1476 | vmballoon_stat_page_names[i], | |
1477 | vmballoon_page_size_names[j], | |
1478 | atomic64_read(&b->stats->page_stat[i][j])); | |
1479 | } | |
453dc659 DT |
1480 | |
1481 | return 0; | |
1482 | } | |
1483 | ||
2796b43f | 1484 | DEFINE_SHOW_ATTRIBUTE(vmballoon_debug); |
453dc659 DT |
1485 | |
1486 | static int __init vmballoon_debugfs_init(struct vmballoon *b) | |
1487 | { | |
1488 | int error; | |
1489 | ||
1490 | b->dbg_entry = debugfs_create_file("vmmemctl", S_IRUGO, NULL, b, | |
1491 | &vmballoon_debug_fops); | |
1492 | if (IS_ERR(b->dbg_entry)) { | |
1493 | error = PTR_ERR(b->dbg_entry); | |
1494 | pr_err("failed to create debugfs entry, error: %d\n", error); | |
1495 | return error; | |
1496 | } | |
1497 | ||
1498 | return 0; | |
1499 | } | |
1500 | ||
1501 | static void __exit vmballoon_debugfs_exit(struct vmballoon *b) | |
1502 | { | |
c7b3690f | 1503 | static_key_disable(&balloon_stat_enabled.key); |
453dc659 | 1504 | debugfs_remove(b->dbg_entry); |
c7b3690f NA |
1505 | kfree(b->stats); |
1506 | b->stats = NULL; | |
453dc659 DT |
1507 | } |
1508 | ||
1509 | #else | |
1510 | ||
1511 | static inline int vmballoon_debugfs_init(struct vmballoon *b) | |
1512 | { | |
1513 | return 0; | |
1514 | } | |
1515 | ||
1516 | static inline void vmballoon_debugfs_exit(struct vmballoon *b) | |
1517 | { | |
1518 | } | |
1519 | ||
1520 | #endif /* CONFIG_DEBUG_FS */ | |
1521 | ||
1522 | static int __init vmballoon_init(void) | |
1523 | { | |
6e4453b3 | 1524 | enum vmballoon_page_size_type page_size; |
453dc659 | 1525 | int error; |
6e4453b3 | 1526 | |
453dc659 DT |
1527 | /* |
1528 | * Check if we are running on VMware's hypervisor and bail out | |
1529 | * if we are not. | |
1530 | */ | |
03b2a320 | 1531 | if (x86_hyper_type != X86_HYPER_VMWARE) |
453dc659 DT |
1532 | return -ENODEV; |
1533 | ||
6e4453b3 NA |
1534 | for (page_size = VMW_BALLOON_4K_PAGE; |
1535 | page_size <= VMW_BALLOON_LAST_SIZE; page_size++) | |
1536 | INIT_LIST_HEAD(&balloon.page_sizes[page_size].pages); | |
1537 | ||
453dc659 | 1538 | |
453dc659 DT |
1539 | INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work); |
1540 | ||
453dc659 DT |
1541 | error = vmballoon_debugfs_init(&balloon); |
1542 | if (error) | |
beda94da | 1543 | return error; |
453dc659 | 1544 | |
6e4453b3 | 1545 | spin_lock_init(&balloon.comm_lock); |
c7b3690f | 1546 | init_rwsem(&balloon.conf_sem); |
48e3d668 | 1547 | balloon.vmci_doorbell = VMCI_INVALID_HANDLE; |
d7568c13 PM |
1548 | balloon.batch_page = NULL; |
1549 | balloon.page = NULL; | |
1550 | balloon.reset_required = true; | |
1551 | ||
beda94da | 1552 | queue_delayed_work(system_freezable_wq, &balloon.dwork, 0); |
453dc659 DT |
1553 | |
1554 | return 0; | |
453dc659 | 1555 | } |
c3cc1b0f NA |
1556 | |
1557 | /* | |
1558 | * Using late_initcall() instead of module_init() allows the balloon to use the | |
1559 | * VMCI doorbell even when the balloon is built into the kernel. Otherwise the | |
1560 | * VMCI is probed only after the balloon is initialized. If the balloon is used | |
1561 | * as a module, late_initcall() is equivalent to module_init(). | |
1562 | */ | |
1563 | late_initcall(vmballoon_init); | |
453dc659 DT |
1564 | |
1565 | static void __exit vmballoon_exit(void) | |
1566 | { | |
48e3d668 | 1567 | vmballoon_vmci_cleanup(&balloon); |
453dc659 | 1568 | cancel_delayed_work_sync(&balloon.dwork); |
453dc659 DT |
1569 | |
1570 | vmballoon_debugfs_exit(&balloon); | |
1571 | ||
1572 | /* | |
1573 | * Deallocate all reserved memory, and reset connection with monitor. | |
1574 | * Reset connection before deallocating memory to avoid potential for | |
1575 | * additional spurious resets from guest touching deallocated pages. | |
1576 | */ | |
d7568c13 | 1577 | vmballoon_send_start(&balloon, 0); |
453dc659 DT |
1578 | vmballoon_pop(&balloon); |
1579 | } | |
1580 | module_exit(vmballoon_exit); |