Commit | Line | Data |
---|---|---|
18468d93 RA |
1 | /* |
2 | * mm/balloon_compaction.c | |
3 | * | |
4 | * Common interface for making balloon pages movable by compaction. | |
5 | * | |
6 | * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <aquini@redhat.com> | |
7 | */ | |
8 | #include <linux/mm.h> | |
9 | #include <linux/slab.h> | |
10 | #include <linux/export.h> | |
11 | #include <linux/balloon_compaction.h> | |
12 | ||
13 | /* | |
14 | * balloon_devinfo_alloc - allocates a balloon device information descriptor. | |
15 | * @balloon_dev_descriptor: pointer to reference the balloon device which | |
16 | * this struct balloon_dev_info will be servicing. | |
17 | * | |
18 | * Driver must call it to properly allocate and initialize an instance of | |
19 | * struct balloon_dev_info which will be used to reference a balloon device | |
20 | * as well as to keep track of the balloon device page list. | |
21 | */ | |
22 | struct balloon_dev_info *balloon_devinfo_alloc(void *balloon_dev_descriptor) | |
23 | { | |
24 | struct balloon_dev_info *b_dev_info; | |
25 | b_dev_info = kmalloc(sizeof(*b_dev_info), GFP_KERNEL); | |
26 | if (!b_dev_info) | |
27 | return ERR_PTR(-ENOMEM); | |
28 | ||
29 | b_dev_info->balloon_device = balloon_dev_descriptor; | |
30 | b_dev_info->mapping = NULL; | |
31 | b_dev_info->isolated_pages = 0; | |
32 | spin_lock_init(&b_dev_info->pages_lock); | |
33 | INIT_LIST_HEAD(&b_dev_info->pages); | |
34 | ||
35 | return b_dev_info; | |
36 | } | |
37 | EXPORT_SYMBOL_GPL(balloon_devinfo_alloc); | |
38 | ||
39 | /* | |
40 | * balloon_page_enqueue - allocates a new page and inserts it into the balloon | |
41 | * page list. | |
42 | * @b_dev_info: balloon device decriptor where we will insert a new page to | |
43 | * | |
44 | * Driver must call it to properly allocate a new enlisted balloon page | |
45 | * before definetively removing it from the guest system. | |
46 | * This function returns the page address for the recently enqueued page or | |
47 | * NULL in the case we fail to allocate a new page this turn. | |
48 | */ | |
49 | struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info) | |
50 | { | |
51 | unsigned long flags; | |
52 | struct page *page = alloc_page(balloon_mapping_gfp_mask() | | |
53 | __GFP_NOMEMALLOC | __GFP_NORETRY); | |
54 | if (!page) | |
55 | return NULL; | |
56 | ||
57 | /* | |
58 | * Block others from accessing the 'page' when we get around to | |
59 | * establishing additional references. We should be the only one | |
60 | * holding a reference to the 'page' at this point. | |
61 | */ | |
62 | BUG_ON(!trylock_page(page)); | |
63 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); | |
64 | balloon_page_insert(page, b_dev_info->mapping, &b_dev_info->pages); | |
65 | spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); | |
66 | unlock_page(page); | |
67 | return page; | |
68 | } | |
69 | EXPORT_SYMBOL_GPL(balloon_page_enqueue); | |
70 | ||
71 | /* | |
72 | * balloon_page_dequeue - removes a page from balloon's page list and returns | |
73 | * the its address to allow the driver release the page. | |
74 | * @b_dev_info: balloon device decriptor where we will grab a page from. | |
75 | * | |
76 | * Driver must call it to properly de-allocate a previous enlisted balloon page | |
77 | * before definetively releasing it back to the guest system. | |
78 | * This function returns the page address for the recently dequeued page or | |
79 | * NULL in the case we find balloon's page list temporarily empty due to | |
80 | * compaction isolated pages. | |
81 | */ | |
82 | struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info) | |
83 | { | |
84 | struct page *page, *tmp; | |
85 | unsigned long flags; | |
86 | bool dequeued_page; | |
87 | ||
88 | dequeued_page = false; | |
89 | list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) { | |
90 | /* | |
91 | * Block others from accessing the 'page' while we get around | |
92 | * establishing additional references and preparing the 'page' | |
93 | * to be released by the balloon driver. | |
94 | */ | |
95 | if (trylock_page(page)) { | |
d6d86c0a KK |
96 | if (!PagePrivate(page)) { |
97 | /* raced with isolation */ | |
98 | unlock_page(page); | |
99 | continue; | |
100 | } | |
18468d93 | 101 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
18468d93 RA |
102 | balloon_page_delete(page); |
103 | spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); | |
104 | unlock_page(page); | |
105 | dequeued_page = true; | |
106 | break; | |
107 | } | |
108 | } | |
109 | ||
110 | if (!dequeued_page) { | |
111 | /* | |
112 | * If we are unable to dequeue a balloon page because the page | |
113 | * list is empty and there is no isolated pages, then something | |
114 | * went out of track and some balloon pages are lost. | |
115 | * BUG() here, otherwise the balloon driver may get stuck into | |
116 | * an infinite loop while attempting to release all its pages. | |
117 | */ | |
118 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); | |
119 | if (unlikely(list_empty(&b_dev_info->pages) && | |
120 | !b_dev_info->isolated_pages)) | |
121 | BUG(); | |
122 | spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); | |
123 | page = NULL; | |
124 | } | |
125 | return page; | |
126 | } | |
127 | EXPORT_SYMBOL_GPL(balloon_page_dequeue); | |
128 | ||
129 | #ifdef CONFIG_BALLOON_COMPACTION | |
130 | /* | |
131 | * balloon_mapping_alloc - allocates a special ->mapping for ballooned pages. | |
132 | * @b_dev_info: holds the balloon device information descriptor. | |
133 | * @a_ops: balloon_mapping address_space_operations descriptor. | |
134 | * | |
135 | * Driver must call it to properly allocate and initialize an instance of | |
136 | * struct address_space which will be used as the special page->mapping for | |
137 | * balloon device enlisted page instances. | |
138 | */ | |
139 | struct address_space *balloon_mapping_alloc(struct balloon_dev_info *b_dev_info, | |
140 | const struct address_space_operations *a_ops) | |
141 | { | |
142 | struct address_space *mapping; | |
143 | ||
144 | mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); | |
145 | if (!mapping) | |
146 | return ERR_PTR(-ENOMEM); | |
147 | ||
148 | /* | |
149 | * Give a clean 'zeroed' status to all elements of this special | |
150 | * balloon page->mapping struct address_space instance. | |
151 | */ | |
152 | address_space_init_once(mapping); | |
153 | ||
154 | /* | |
155 | * Set mapping->flags appropriately, to allow balloon pages | |
156 | * ->mapping identification. | |
157 | */ | |
158 | mapping_set_balloon(mapping); | |
159 | mapping_set_gfp_mask(mapping, balloon_mapping_gfp_mask()); | |
160 | ||
161 | /* balloon's page->mapping->a_ops callback descriptor */ | |
162 | mapping->a_ops = a_ops; | |
163 | ||
164 | /* | |
165 | * Establish a pointer reference back to the balloon device descriptor | |
166 | * this particular page->mapping will be servicing. | |
167 | * This is used by compaction / migration procedures to identify and | |
168 | * access the balloon device pageset while isolating / migrating pages. | |
169 | * | |
170 | * As some balloon drivers can register multiple balloon devices | |
171 | * for a single guest, this also helps compaction / migration to | |
172 | * properly deal with multiple balloon pagesets, when required. | |
173 | */ | |
174 | mapping->private_data = b_dev_info; | |
175 | b_dev_info->mapping = mapping; | |
176 | ||
177 | return mapping; | |
178 | } | |
179 | EXPORT_SYMBOL_GPL(balloon_mapping_alloc); | |
180 | ||
181 | static inline void __isolate_balloon_page(struct page *page) | |
182 | { | |
183 | struct balloon_dev_info *b_dev_info = page->mapping->private_data; | |
184 | unsigned long flags; | |
d6d86c0a | 185 | |
18468d93 | 186 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
d6d86c0a | 187 | ClearPagePrivate(page); |
18468d93 RA |
188 | list_del(&page->lru); |
189 | b_dev_info->isolated_pages++; | |
190 | spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); | |
191 | } | |
192 | ||
193 | static inline void __putback_balloon_page(struct page *page) | |
194 | { | |
195 | struct balloon_dev_info *b_dev_info = page->mapping->private_data; | |
196 | unsigned long flags; | |
d6d86c0a | 197 | |
18468d93 | 198 | spin_lock_irqsave(&b_dev_info->pages_lock, flags); |
d6d86c0a | 199 | SetPagePrivate(page); |
18468d93 RA |
200 | list_add(&page->lru, &b_dev_info->pages); |
201 | b_dev_info->isolated_pages--; | |
202 | spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); | |
203 | } | |
204 | ||
205 | static inline int __migrate_balloon_page(struct address_space *mapping, | |
206 | struct page *newpage, struct page *page, enum migrate_mode mode) | |
207 | { | |
208 | return page->mapping->a_ops->migratepage(mapping, newpage, page, mode); | |
209 | } | |
210 | ||
211 | /* __isolate_lru_page() counterpart for a ballooned page */ | |
212 | bool balloon_page_isolate(struct page *page) | |
213 | { | |
214 | /* | |
215 | * Avoid burning cycles with pages that are yet under __free_pages(), | |
216 | * or just got freed under us. | |
217 | * | |
218 | * In case we 'win' a race for a balloon page being freed under us and | |
219 | * raise its refcount preventing __free_pages() from doing its job | |
220 | * the put_page() at the end of this block will take care of | |
221 | * release this page, thus avoiding a nasty leakage. | |
222 | */ | |
223 | if (likely(get_page_unless_zero(page))) { | |
224 | /* | |
225 | * As balloon pages are not isolated from LRU lists, concurrent | |
226 | * compaction threads can race against page migration functions | |
227 | * as well as race against the balloon driver releasing a page. | |
228 | * | |
229 | * In order to avoid having an already isolated balloon page | |
230 | * being (wrongly) re-isolated while it is under migration, | |
231 | * or to avoid attempting to isolate pages being released by | |
232 | * the balloon driver, lets be sure we have the page lock | |
233 | * before proceeding with the balloon page isolation steps. | |
234 | */ | |
235 | if (likely(trylock_page(page))) { | |
236 | /* | |
d6d86c0a | 237 | * A ballooned page, by default, has PagePrivate set. |
18468d93 | 238 | * Prevent concurrent compaction threads from isolating |
d6d86c0a | 239 | * an already isolated balloon page by clearing it. |
18468d93 | 240 | */ |
d6d86c0a | 241 | if (balloon_page_movable(page)) { |
18468d93 RA |
242 | __isolate_balloon_page(page); |
243 | unlock_page(page); | |
244 | return true; | |
245 | } | |
246 | unlock_page(page); | |
247 | } | |
248 | put_page(page); | |
249 | } | |
250 | return false; | |
251 | } | |
252 | ||
253 | /* putback_lru_page() counterpart for a ballooned page */ | |
254 | void balloon_page_putback(struct page *page) | |
255 | { | |
256 | /* | |
257 | * 'lock_page()' stabilizes the page and prevents races against | |
258 | * concurrent isolation threads attempting to re-isolate it. | |
259 | */ | |
260 | lock_page(page); | |
261 | ||
262 | if (__is_movable_balloon_page(page)) { | |
263 | __putback_balloon_page(page); | |
264 | /* drop the extra ref count taken for page isolation */ | |
265 | put_page(page); | |
266 | } else { | |
267 | WARN_ON(1); | |
f0b791a3 | 268 | dump_page(page, "not movable balloon page"); |
18468d93 RA |
269 | } |
270 | unlock_page(page); | |
271 | } | |
272 | ||
273 | /* move_to_new_page() counterpart for a ballooned page */ | |
274 | int balloon_page_migrate(struct page *newpage, | |
275 | struct page *page, enum migrate_mode mode) | |
276 | { | |
277 | struct address_space *mapping; | |
278 | int rc = -EAGAIN; | |
279 | ||
280 | /* | |
281 | * Block others from accessing the 'newpage' when we get around to | |
282 | * establishing additional references. We should be the only one | |
283 | * holding a reference to the 'newpage' at this point. | |
284 | */ | |
285 | BUG_ON(!trylock_page(newpage)); | |
286 | ||
287 | if (WARN_ON(!__is_movable_balloon_page(page))) { | |
f0b791a3 | 288 | dump_page(page, "not movable balloon page"); |
18468d93 RA |
289 | unlock_page(newpage); |
290 | return rc; | |
291 | } | |
292 | ||
293 | mapping = page->mapping; | |
294 | if (mapping) | |
295 | rc = __migrate_balloon_page(mapping, newpage, page, mode); | |
296 | ||
297 | unlock_page(newpage); | |
298 | return rc; | |
299 | } | |
300 | #endif /* CONFIG_BALLOON_COMPACTION */ |