Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
a38e4082 DC |
2 | /* |
3 | * Copyright (c) 2013 Red Hat, Inc. and Parallels Inc. All rights reserved. | |
4 | * Authors: David Chinner and Glauber Costa | |
5 | * | |
6 | * Generic LRU infrastructure | |
7 | */ | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/module.h> | |
3b1d58a4 | 10 | #include <linux/mm.h> |
a38e4082 | 11 | #include <linux/list_lru.h> |
5ca302c8 | 12 | #include <linux/slab.h> |
c0a5b560 | 13 | #include <linux/mutex.h> |
60d3fd32 | 14 | #include <linux/memcontrol.h> |
4d96ba35 | 15 | #include "slab.h" |
88f2ef73 | 16 | #include "internal.h" |
c0a5b560 | 17 | |
84c07d11 | 18 | #ifdef CONFIG_MEMCG_KMEM |
3eef1127 | 19 | static LIST_HEAD(memcg_list_lrus); |
c0a5b560 VD |
20 | static DEFINE_MUTEX(list_lrus_mutex); |
21 | ||
3eef1127 MS |
22 | static inline bool list_lru_memcg_aware(struct list_lru *lru) |
23 | { | |
24 | return lru->memcg_aware; | |
25 | } | |
26 | ||
c0a5b560 VD |
27 | static void list_lru_register(struct list_lru *lru) |
28 | { | |
3eef1127 MS |
29 | if (!list_lru_memcg_aware(lru)) |
30 | return; | |
31 | ||
c0a5b560 | 32 | mutex_lock(&list_lrus_mutex); |
3eef1127 | 33 | list_add(&lru->list, &memcg_list_lrus); |
c0a5b560 VD |
34 | mutex_unlock(&list_lrus_mutex); |
35 | } | |
36 | ||
37 | static void list_lru_unregister(struct list_lru *lru) | |
38 | { | |
3eef1127 MS |
39 | if (!list_lru_memcg_aware(lru)) |
40 | return; | |
41 | ||
c0a5b560 VD |
42 | mutex_lock(&list_lrus_mutex); |
43 | list_del(&lru->list); | |
44 | mutex_unlock(&list_lrus_mutex); | |
45 | } | |
c0a5b560 | 46 | |
fae91d6d KT |
47 | static int lru_shrinker_id(struct list_lru *lru) |
48 | { | |
49 | return lru->shrinker_id; | |
50 | } | |
51 | ||
60d3fd32 | 52 | static inline struct list_lru_one * |
6a6b7b77 | 53 | list_lru_from_memcg_idx(struct list_lru *lru, int nid, int idx) |
60d3fd32 | 54 | { |
bbca91cc | 55 | if (list_lru_memcg_aware(lru) && idx >= 0) { |
d7011070 | 56 | struct list_lru_memcg *mlru = xa_load(&lru->xa, idx); |
5abc1e37 | 57 | |
5abc1e37 MS |
58 | return mlru ? &mlru->node[nid] : NULL; |
59 | } | |
bbca91cc | 60 | return &lru->node[nid].lru; |
60d3fd32 | 61 | } |
60d3fd32 | 62 | #else |
e0295238 KT |
63 | static void list_lru_register(struct list_lru *lru) |
64 | { | |
65 | } | |
66 | ||
67 | static void list_lru_unregister(struct list_lru *lru) | |
68 | { | |
69 | } | |
70 | ||
fae91d6d KT |
71 | static int lru_shrinker_id(struct list_lru *lru) |
72 | { | |
73 | return -1; | |
74 | } | |
75 | ||
60d3fd32 VD |
76 | static inline bool list_lru_memcg_aware(struct list_lru *lru) |
77 | { | |
78 | return false; | |
79 | } | |
80 | ||
81 | static inline struct list_lru_one * | |
6a6b7b77 | 82 | list_lru_from_memcg_idx(struct list_lru *lru, int nid, int idx) |
60d3fd32 | 83 | { |
6a6b7b77 | 84 | return &lru->node[nid].lru; |
60d3fd32 | 85 | } |
84c07d11 | 86 | #endif /* CONFIG_MEMCG_KMEM */ |
60d3fd32 | 87 | |
0a97c01c NP |
88 | bool list_lru_add(struct list_lru *lru, struct list_head *item, int nid, |
89 | struct mem_cgroup *memcg) | |
a38e4082 | 90 | { |
3b1d58a4 | 91 | struct list_lru_node *nlru = &lru->node[nid]; |
60d3fd32 | 92 | struct list_lru_one *l; |
3b1d58a4 DC |
93 | |
94 | spin_lock(&nlru->lock); | |
a38e4082 | 95 | if (list_empty(item)) { |
0a97c01c | 96 | l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); |
60d3fd32 | 97 | list_add_tail(item, &l->list); |
fae91d6d KT |
98 | /* Set shrinker bit if the first element was added */ |
99 | if (!l->nr_items++) | |
0a97c01c | 100 | set_shrinker_bit(memcg, nid, lru_shrinker_id(lru)); |
2c80cd57 | 101 | nlru->nr_items++; |
3b1d58a4 | 102 | spin_unlock(&nlru->lock); |
a38e4082 DC |
103 | return true; |
104 | } | |
3b1d58a4 | 105 | spin_unlock(&nlru->lock); |
a38e4082 DC |
106 | return false; |
107 | } | |
108 | EXPORT_SYMBOL_GPL(list_lru_add); | |
109 | ||
0a97c01c | 110 | bool list_lru_add_obj(struct list_lru *lru, struct list_head *item) |
a38e4082 | 111 | { |
3b1d58a4 | 112 | int nid = page_to_nid(virt_to_page(item)); |
0a97c01c NP |
113 | struct mem_cgroup *memcg = list_lru_memcg_aware(lru) ? |
114 | mem_cgroup_from_slab_obj(item) : NULL; | |
115 | ||
116 | return list_lru_add(lru, item, nid, memcg); | |
117 | } | |
118 | EXPORT_SYMBOL_GPL(list_lru_add_obj); | |
119 | ||
120 | bool list_lru_del(struct list_lru *lru, struct list_head *item, int nid, | |
121 | struct mem_cgroup *memcg) | |
122 | { | |
3b1d58a4 | 123 | struct list_lru_node *nlru = &lru->node[nid]; |
60d3fd32 | 124 | struct list_lru_one *l; |
3b1d58a4 DC |
125 | |
126 | spin_lock(&nlru->lock); | |
a38e4082 | 127 | if (!list_empty(item)) { |
0a97c01c | 128 | l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); |
a38e4082 | 129 | list_del_init(item); |
60d3fd32 | 130 | l->nr_items--; |
2c80cd57 | 131 | nlru->nr_items--; |
3b1d58a4 | 132 | spin_unlock(&nlru->lock); |
a38e4082 DC |
133 | return true; |
134 | } | |
3b1d58a4 | 135 | spin_unlock(&nlru->lock); |
a38e4082 DC |
136 | return false; |
137 | } | |
138 | EXPORT_SYMBOL_GPL(list_lru_del); | |
139 | ||
0a97c01c NP |
140 | bool list_lru_del_obj(struct list_lru *lru, struct list_head *item) |
141 | { | |
142 | int nid = page_to_nid(virt_to_page(item)); | |
143 | struct mem_cgroup *memcg = list_lru_memcg_aware(lru) ? | |
144 | mem_cgroup_from_slab_obj(item) : NULL; | |
145 | ||
146 | return list_lru_del(lru, item, nid, memcg); | |
147 | } | |
148 | EXPORT_SYMBOL_GPL(list_lru_del_obj); | |
149 | ||
3f97b163 VD |
150 | void list_lru_isolate(struct list_lru_one *list, struct list_head *item) |
151 | { | |
152 | list_del_init(item); | |
153 | list->nr_items--; | |
154 | } | |
155 | EXPORT_SYMBOL_GPL(list_lru_isolate); | |
156 | ||
157 | void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item, | |
158 | struct list_head *head) | |
159 | { | |
160 | list_move(item, head); | |
161 | list->nr_items--; | |
162 | } | |
163 | EXPORT_SYMBOL_GPL(list_lru_isolate_move); | |
164 | ||
0a97c01c NP |
165 | void list_lru_putback(struct list_lru *lru, struct list_head *item, int nid, |
166 | struct mem_cgroup *memcg) | |
167 | { | |
168 | struct list_lru_one *list = | |
169 | list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); | |
170 | ||
171 | if (list_empty(item)) { | |
172 | list_add_tail(item, &list->list); | |
173 | if (!list->nr_items++) | |
174 | set_shrinker_bit(memcg, nid, lru_shrinker_id(lru)); | |
175 | } | |
176 | } | |
177 | EXPORT_SYMBOL_GPL(list_lru_putback); | |
178 | ||
930eaac5 AM |
179 | unsigned long list_lru_count_one(struct list_lru *lru, |
180 | int nid, struct mem_cgroup *memcg) | |
a38e4082 | 181 | { |
60d3fd32 | 182 | struct list_lru_one *l; |
41d17431 | 183 | long count; |
3b1d58a4 | 184 | |
0c7c1bed | 185 | rcu_read_lock(); |
7c52f65d | 186 | l = list_lru_from_memcg_idx(lru, nid, memcg_kmem_id(memcg)); |
5abc1e37 | 187 | count = l ? READ_ONCE(l->nr_items) : 0; |
0c7c1bed | 188 | rcu_read_unlock(); |
3b1d58a4 | 189 | |
41d17431 MS |
190 | if (unlikely(count < 0)) |
191 | count = 0; | |
192 | ||
3b1d58a4 DC |
193 | return count; |
194 | } | |
60d3fd32 VD |
195 | EXPORT_SYMBOL_GPL(list_lru_count_one); |
196 | ||
197 | unsigned long list_lru_count_node(struct list_lru *lru, int nid) | |
198 | { | |
2c80cd57 | 199 | struct list_lru_node *nlru; |
60d3fd32 | 200 | |
2c80cd57 ST |
201 | nlru = &lru->node[nid]; |
202 | return nlru->nr_items; | |
60d3fd32 | 203 | } |
6a4f496f | 204 | EXPORT_SYMBOL_GPL(list_lru_count_node); |
3b1d58a4 | 205 | |
60d3fd32 | 206 | static unsigned long |
6a6b7b77 | 207 | __list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx, |
60d3fd32 VD |
208 | list_lru_walk_cb isolate, void *cb_arg, |
209 | unsigned long *nr_to_walk) | |
3b1d58a4 | 210 | { |
6a6b7b77 | 211 | struct list_lru_node *nlru = &lru->node[nid]; |
60d3fd32 | 212 | struct list_lru_one *l; |
a38e4082 | 213 | struct list_head *item, *n; |
3b1d58a4 | 214 | unsigned long isolated = 0; |
a38e4082 | 215 | |
a38e4082 | 216 | restart: |
5abc1e37 MS |
217 | l = list_lru_from_memcg_idx(lru, nid, memcg_idx); |
218 | if (!l) | |
219 | goto out; | |
220 | ||
60d3fd32 | 221 | list_for_each_safe(item, n, &l->list) { |
a38e4082 | 222 | enum lru_status ret; |
5cedf721 DC |
223 | |
224 | /* | |
225 | * decrement nr_to_walk first so that we don't livelock if we | |
3dc5f032 | 226 | * get stuck on large numbers of LRU_RETRY items |
5cedf721 | 227 | */ |
c56b097a | 228 | if (!*nr_to_walk) |
5cedf721 | 229 | break; |
c56b097a | 230 | --*nr_to_walk; |
5cedf721 | 231 | |
3f97b163 | 232 | ret = isolate(item, l, &nlru->lock, cb_arg); |
a38e4082 | 233 | switch (ret) { |
449dd698 JW |
234 | case LRU_REMOVED_RETRY: |
235 | assert_spin_locked(&nlru->lock); | |
e4a9bc58 | 236 | fallthrough; |
a38e4082 | 237 | case LRU_REMOVED: |
3b1d58a4 | 238 | isolated++; |
2c80cd57 | 239 | nlru->nr_items--; |
449dd698 JW |
240 | /* |
241 | * If the lru lock has been dropped, our list | |
242 | * traversal is now invalid and so we have to | |
243 | * restart from scratch. | |
244 | */ | |
245 | if (ret == LRU_REMOVED_RETRY) | |
246 | goto restart; | |
a38e4082 DC |
247 | break; |
248 | case LRU_ROTATE: | |
60d3fd32 | 249 | list_move_tail(item, &l->list); |
a38e4082 DC |
250 | break; |
251 | case LRU_SKIP: | |
252 | break; | |
253 | case LRU_RETRY: | |
5cedf721 DC |
254 | /* |
255 | * The lru lock has been dropped, our list traversal is | |
256 | * now invalid and so we have to restart from scratch. | |
257 | */ | |
449dd698 | 258 | assert_spin_locked(&nlru->lock); |
a38e4082 DC |
259 | goto restart; |
260 | default: | |
261 | BUG(); | |
262 | } | |
a38e4082 | 263 | } |
5abc1e37 | 264 | out: |
3b1d58a4 DC |
265 | return isolated; |
266 | } | |
60d3fd32 VD |
267 | |
268 | unsigned long | |
269 | list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg, | |
270 | list_lru_walk_cb isolate, void *cb_arg, | |
271 | unsigned long *nr_to_walk) | |
272 | { | |
6cfe57a9 SAS |
273 | struct list_lru_node *nlru = &lru->node[nid]; |
274 | unsigned long ret; | |
275 | ||
276 | spin_lock(&nlru->lock); | |
7c52f65d | 277 | ret = __list_lru_walk_one(lru, nid, memcg_kmem_id(memcg), isolate, |
6a6b7b77 | 278 | cb_arg, nr_to_walk); |
6cfe57a9 SAS |
279 | spin_unlock(&nlru->lock); |
280 | return ret; | |
60d3fd32 VD |
281 | } |
282 | EXPORT_SYMBOL_GPL(list_lru_walk_one); | |
283 | ||
6b51e881 SAS |
284 | unsigned long |
285 | list_lru_walk_one_irq(struct list_lru *lru, int nid, struct mem_cgroup *memcg, | |
286 | list_lru_walk_cb isolate, void *cb_arg, | |
287 | unsigned long *nr_to_walk) | |
288 | { | |
289 | struct list_lru_node *nlru = &lru->node[nid]; | |
290 | unsigned long ret; | |
291 | ||
292 | spin_lock_irq(&nlru->lock); | |
7c52f65d | 293 | ret = __list_lru_walk_one(lru, nid, memcg_kmem_id(memcg), isolate, |
6a6b7b77 | 294 | cb_arg, nr_to_walk); |
6b51e881 SAS |
295 | spin_unlock_irq(&nlru->lock); |
296 | return ret; | |
297 | } | |
298 | ||
60d3fd32 VD |
299 | unsigned long list_lru_walk_node(struct list_lru *lru, int nid, |
300 | list_lru_walk_cb isolate, void *cb_arg, | |
301 | unsigned long *nr_to_walk) | |
302 | { | |
303 | long isolated = 0; | |
60d3fd32 | 304 | |
87a5ffc1 SAS |
305 | isolated += list_lru_walk_one(lru, nid, NULL, isolate, cb_arg, |
306 | nr_to_walk); | |
bbca91cc MS |
307 | |
308 | #ifdef CONFIG_MEMCG_KMEM | |
60d3fd32 | 309 | if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) { |
d7011070 | 310 | struct list_lru_memcg *mlru; |
bbca91cc MS |
311 | unsigned long index; |
312 | ||
313 | xa_for_each(&lru->xa, index, mlru) { | |
6cfe57a9 SAS |
314 | struct list_lru_node *nlru = &lru->node[nid]; |
315 | ||
316 | spin_lock(&nlru->lock); | |
bbca91cc | 317 | isolated += __list_lru_walk_one(lru, nid, index, |
6e018968 SAS |
318 | isolate, cb_arg, |
319 | nr_to_walk); | |
6cfe57a9 SAS |
320 | spin_unlock(&nlru->lock); |
321 | ||
60d3fd32 VD |
322 | if (*nr_to_walk <= 0) |
323 | break; | |
324 | } | |
325 | } | |
bbca91cc MS |
326 | #endif |
327 | ||
60d3fd32 VD |
328 | return isolated; |
329 | } | |
3b1d58a4 DC |
330 | EXPORT_SYMBOL_GPL(list_lru_walk_node); |
331 | ||
60d3fd32 VD |
332 | static void init_one_lru(struct list_lru_one *l) |
333 | { | |
334 | INIT_LIST_HEAD(&l->list); | |
335 | l->nr_items = 0; | |
336 | } | |
337 | ||
84c07d11 | 338 | #ifdef CONFIG_MEMCG_KMEM |
d7011070 | 339 | static struct list_lru_memcg *memcg_init_list_lru_one(gfp_t gfp) |
88f2ef73 MS |
340 | { |
341 | int nid; | |
d7011070 | 342 | struct list_lru_memcg *mlru; |
88f2ef73 MS |
343 | |
344 | mlru = kmalloc(struct_size(mlru, node, nr_node_ids), gfp); | |
345 | if (!mlru) | |
346 | return NULL; | |
347 | ||
348 | for_each_node(nid) | |
349 | init_one_lru(&mlru->node[nid]); | |
350 | ||
351 | return mlru; | |
352 | } | |
353 | ||
5abc1e37 | 354 | static void memcg_list_lru_free(struct list_lru *lru, int src_idx) |
60d3fd32 | 355 | { |
d7011070 | 356 | struct list_lru_memcg *mlru = xa_erase_irq(&lru->xa, src_idx); |
5abc1e37 MS |
357 | |
358 | /* | |
359 | * The __list_lru_walk_one() can walk the list of this node. | |
360 | * We need kvfree_rcu() here. And the walking of the list | |
361 | * is under lru->node[nid]->lock, which can serve as a RCU | |
362 | * read-side critical section. | |
363 | */ | |
364 | if (mlru) | |
365 | kvfree_rcu(mlru, rcu); | |
60d3fd32 VD |
366 | } |
367 | ||
bbca91cc | 368 | static inline void memcg_init_list_lru(struct list_lru *lru, bool memcg_aware) |
60d3fd32 | 369 | { |
bbca91cc MS |
370 | if (memcg_aware) |
371 | xa_init_flags(&lru->xa, XA_FLAGS_LOCK_IRQ); | |
6a6b7b77 | 372 | lru->memcg_aware = memcg_aware; |
60d3fd32 VD |
373 | } |
374 | ||
6a6b7b77 | 375 | static void memcg_destroy_list_lru(struct list_lru *lru) |
60d3fd32 | 376 | { |
bbca91cc | 377 | XA_STATE(xas, &lru->xa, 0); |
d7011070 | 378 | struct list_lru_memcg *mlru; |
6a6b7b77 MS |
379 | |
380 | if (!list_lru_memcg_aware(lru)) | |
381 | return; | |
382 | ||
bbca91cc MS |
383 | xas_lock_irq(&xas); |
384 | xas_for_each(&xas, mlru, ULONG_MAX) { | |
385 | kfree(mlru); | |
386 | xas_store(&xas, NULL); | |
60d3fd32 | 387 | } |
bbca91cc | 388 | xas_unlock_irq(&xas); |
60d3fd32 | 389 | } |
2788cf0c | 390 | |
1f391eb2 MS |
391 | static void memcg_reparent_list_lru_node(struct list_lru *lru, int nid, |
392 | int src_idx, struct mem_cgroup *dst_memcg) | |
2788cf0c | 393 | { |
3b82c4dc | 394 | struct list_lru_node *nlru = &lru->node[nid]; |
9bec5c35 | 395 | int dst_idx = dst_memcg->kmemcg_id; |
2788cf0c VD |
396 | struct list_lru_one *src, *dst; |
397 | ||
398 | /* | |
399 | * Since list_lru_{add,del} may be called under an IRQ-safe lock, | |
400 | * we have to use IRQ-safe primitives here to avoid deadlock. | |
401 | */ | |
402 | spin_lock_irq(&nlru->lock); | |
403 | ||
6a6b7b77 | 404 | src = list_lru_from_memcg_idx(lru, nid, src_idx); |
5abc1e37 MS |
405 | if (!src) |
406 | goto out; | |
6a6b7b77 | 407 | dst = list_lru_from_memcg_idx(lru, nid, dst_idx); |
2788cf0c VD |
408 | |
409 | list_splice_init(&src->list, &dst->list); | |
8199be00 YS |
410 | |
411 | if (src->nr_items) { | |
412 | dst->nr_items += src->nr_items; | |
2bfd3637 | 413 | set_shrinker_bit(dst_memcg, nid, lru_shrinker_id(lru)); |
8199be00 YS |
414 | src->nr_items = 0; |
415 | } | |
5abc1e37 | 416 | out: |
2788cf0c VD |
417 | spin_unlock_irq(&nlru->lock); |
418 | } | |
419 | ||
1f391eb2 MS |
420 | static void memcg_reparent_list_lru(struct list_lru *lru, |
421 | int src_idx, struct mem_cgroup *dst_memcg) | |
2788cf0c VD |
422 | { |
423 | int i; | |
424 | ||
145949a1 | 425 | for_each_node(i) |
1f391eb2 | 426 | memcg_reparent_list_lru_node(lru, i, src_idx, dst_memcg); |
5abc1e37 MS |
427 | |
428 | memcg_list_lru_free(lru, src_idx); | |
2788cf0c VD |
429 | } |
430 | ||
1f391eb2 | 431 | void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *parent) |
2788cf0c | 432 | { |
5abc1e37 | 433 | struct cgroup_subsys_state *css; |
2788cf0c | 434 | struct list_lru *lru; |
1f391eb2 | 435 | int src_idx = memcg->kmemcg_id; |
5abc1e37 MS |
436 | |
437 | /* | |
438 | * Change kmemcg_id of this cgroup and all its descendants to the | |
439 | * parent's id, and then move all entries from this cgroup's list_lrus | |
440 | * to ones of the parent. | |
441 | * | |
442 | * After we have finished, all list_lrus corresponding to this cgroup | |
443 | * are guaranteed to remain empty. So we can safely free this cgroup's | |
444 | * list lrus in memcg_list_lru_free(). | |
445 | * | |
446 | * Changing ->kmemcg_id to the parent can prevent memcg_list_lru_alloc() | |
447 | * from allocating list lrus for this cgroup after memcg_list_lru_free() | |
448 | * call. | |
449 | */ | |
450 | rcu_read_lock(); | |
1f391eb2 MS |
451 | css_for_each_descendant_pre(css, &memcg->css) { |
452 | struct mem_cgroup *child; | |
5abc1e37 | 453 | |
1f391eb2 | 454 | child = mem_cgroup_from_css(css); |
bbca91cc | 455 | WRITE_ONCE(child->kmemcg_id, parent->kmemcg_id); |
5abc1e37 MS |
456 | } |
457 | rcu_read_unlock(); | |
2788cf0c VD |
458 | |
459 | mutex_lock(&list_lrus_mutex); | |
3eef1127 | 460 | list_for_each_entry(lru, &memcg_list_lrus, list) |
1f391eb2 | 461 | memcg_reparent_list_lru(lru, src_idx, parent); |
2788cf0c VD |
462 | mutex_unlock(&list_lrus_mutex); |
463 | } | |
88f2ef73 | 464 | |
bbca91cc MS |
465 | static inline bool memcg_list_lru_allocated(struct mem_cgroup *memcg, |
466 | struct list_lru *lru) | |
88f2ef73 | 467 | { |
bbca91cc | 468 | int idx = memcg->kmemcg_id; |
88f2ef73 | 469 | |
bbca91cc | 470 | return idx < 0 || xa_load(&lru->xa, idx); |
88f2ef73 MS |
471 | } |
472 | ||
473 | int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru, | |
474 | gfp_t gfp) | |
475 | { | |
476 | int i; | |
477 | unsigned long flags; | |
88f2ef73 | 478 | struct list_lru_memcg_table { |
d7011070 | 479 | struct list_lru_memcg *mlru; |
88f2ef73 MS |
480 | struct mem_cgroup *memcg; |
481 | } *table; | |
bbca91cc | 482 | XA_STATE(xas, &lru->xa, 0); |
88f2ef73 MS |
483 | |
484 | if (!list_lru_memcg_aware(lru) || memcg_list_lru_allocated(memcg, lru)) | |
485 | return 0; | |
486 | ||
487 | gfp &= GFP_RECLAIM_MASK; | |
488 | table = kmalloc_array(memcg->css.cgroup->level, sizeof(*table), gfp); | |
489 | if (!table) | |
490 | return -ENOMEM; | |
491 | ||
492 | /* | |
493 | * Because the list_lru can be reparented to the parent cgroup's | |
494 | * list_lru, we should make sure that this cgroup and all its | |
d7011070 | 495 | * ancestors have allocated list_lru_memcg. |
88f2ef73 MS |
496 | */ |
497 | for (i = 0; memcg; memcg = parent_mem_cgroup(memcg), i++) { | |
498 | if (memcg_list_lru_allocated(memcg, lru)) | |
499 | break; | |
500 | ||
501 | table[i].memcg = memcg; | |
502 | table[i].mlru = memcg_init_list_lru_one(gfp); | |
503 | if (!table[i].mlru) { | |
504 | while (i--) | |
505 | kfree(table[i].mlru); | |
506 | kfree(table); | |
507 | return -ENOMEM; | |
508 | } | |
509 | } | |
510 | ||
bbca91cc | 511 | xas_lock_irqsave(&xas, flags); |
88f2ef73 | 512 | while (i--) { |
bbca91cc | 513 | int index = READ_ONCE(table[i].memcg->kmemcg_id); |
d7011070 | 514 | struct list_lru_memcg *mlru = table[i].mlru; |
88f2ef73 | 515 | |
bbca91cc MS |
516 | xas_set(&xas, index); |
517 | retry: | |
518 | if (unlikely(index < 0 || xas_error(&xas) || xas_load(&xas))) { | |
5abc1e37 | 519 | kfree(mlru); |
bbca91cc MS |
520 | } else { |
521 | xas_store(&xas, mlru); | |
522 | if (xas_error(&xas) == -ENOMEM) { | |
523 | xas_unlock_irqrestore(&xas, flags); | |
524 | if (xas_nomem(&xas, gfp)) | |
525 | xas_set_err(&xas, 0); | |
526 | xas_lock_irqsave(&xas, flags); | |
527 | /* | |
528 | * The xas lock has been released, this memcg | |
529 | * can be reparented before us. So reload | |
530 | * memcg id. More details see the comments | |
531 | * in memcg_reparent_list_lrus(). | |
532 | */ | |
533 | index = READ_ONCE(table[i].memcg->kmemcg_id); | |
534 | if (index < 0) | |
535 | xas_set_err(&xas, 0); | |
536 | else if (!xas_error(&xas) && index != xas.xa_index) | |
537 | xas_set(&xas, index); | |
538 | goto retry; | |
539 | } | |
540 | } | |
88f2ef73 | 541 | } |
bbca91cc MS |
542 | /* xas_nomem() is used to free memory instead of memory allocation. */ |
543 | if (xas.xa_alloc) | |
544 | xas_nomem(&xas, gfp); | |
545 | xas_unlock_irqrestore(&xas, flags); | |
88f2ef73 MS |
546 | kfree(table); |
547 | ||
bbca91cc | 548 | return xas_error(&xas); |
88f2ef73 | 549 | } |
60d3fd32 | 550 | #else |
bbca91cc | 551 | static inline void memcg_init_list_lru(struct list_lru *lru, bool memcg_aware) |
60d3fd32 | 552 | { |
60d3fd32 VD |
553 | } |
554 | ||
555 | static void memcg_destroy_list_lru(struct list_lru *lru) | |
556 | { | |
557 | } | |
84c07d11 | 558 | #endif /* CONFIG_MEMCG_KMEM */ |
60d3fd32 VD |
559 | |
560 | int __list_lru_init(struct list_lru *lru, bool memcg_aware, | |
c92e8e10 | 561 | struct lock_class_key *key, struct shrinker *shrinker) |
a38e4082 | 562 | { |
3b1d58a4 | 563 | int i; |
60d3fd32 | 564 | |
c92e8e10 KT |
565 | #ifdef CONFIG_MEMCG_KMEM |
566 | if (shrinker) | |
567 | lru->shrinker_id = shrinker->id; | |
568 | else | |
569 | lru->shrinker_id = -1; | |
570 | #endif | |
5ca302c8 | 571 | |
b9726c26 | 572 | lru->node = kcalloc(nr_node_ids, sizeof(*lru->node), GFP_KERNEL); |
5ca302c8 | 573 | if (!lru->node) |
bbca91cc | 574 | return -ENOMEM; |
a38e4082 | 575 | |
145949a1 | 576 | for_each_node(i) { |
3b1d58a4 | 577 | spin_lock_init(&lru->node[i].lock); |
449dd698 JW |
578 | if (key) |
579 | lockdep_set_class(&lru->node[i].lock, key); | |
60d3fd32 VD |
580 | init_one_lru(&lru->node[i].lru); |
581 | } | |
582 | ||
bbca91cc | 583 | memcg_init_list_lru(lru, memcg_aware); |
c0a5b560 | 584 | list_lru_register(lru); |
bbca91cc MS |
585 | |
586 | return 0; | |
a38e4082 | 587 | } |
60d3fd32 | 588 | EXPORT_SYMBOL_GPL(__list_lru_init); |
5ca302c8 GC |
589 | |
590 | void list_lru_destroy(struct list_lru *lru) | |
591 | { | |
c0a5b560 VD |
592 | /* Already destroyed or not yet initialized? */ |
593 | if (!lru->node) | |
594 | return; | |
60d3fd32 | 595 | |
c0a5b560 | 596 | list_lru_unregister(lru); |
60d3fd32 VD |
597 | |
598 | memcg_destroy_list_lru(lru); | |
5ca302c8 | 599 | kfree(lru->node); |
c0a5b560 | 600 | lru->node = NULL; |
60d3fd32 | 601 | |
c92e8e10 KT |
602 | #ifdef CONFIG_MEMCG_KMEM |
603 | lru->shrinker_id = -1; | |
604 | #endif | |
5ca302c8 GC |
605 | } |
606 | EXPORT_SYMBOL_GPL(list_lru_destroy); |