Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
3e32cb2e JW |
2 | /* |
3 | * Lockless hierarchical page accounting & limiting | |
4 | * | |
5 | * Copyright (C) 2014 Red Hat, Inc., Johannes Weiner | |
6 | */ | |
7 | ||
8 | #include <linux/page_counter.h> | |
9 | #include <linux/atomic.h> | |
10 | #include <linux/kernel.h> | |
11 | #include <linux/string.h> | |
12 | #include <linux/sched.h> | |
13 | #include <linux/bug.h> | |
14 | #include <asm/page.h> | |
15 | ||
bf8d5d52 RG |
16 | static void propagate_protected_usage(struct page_counter *c, |
17 | unsigned long usage) | |
23067153 | 18 | { |
bf8d5d52 | 19 | unsigned long protected, old_protected; |
23067153 RG |
20 | long delta; |
21 | ||
22 | if (!c->parent) | |
23 | return; | |
24 | ||
cfdab60b SB |
25 | protected = min(usage, READ_ONCE(c->min)); |
26 | old_protected = atomic_long_read(&c->min_usage); | |
27 | if (protected != old_protected) { | |
bf8d5d52 RG |
28 | old_protected = atomic_long_xchg(&c->min_usage, protected); |
29 | delta = protected - old_protected; | |
30 | if (delta) | |
31 | atomic_long_add(delta, &c->parent->children_min_usage); | |
32 | } | |
23067153 | 33 | |
cfdab60b SB |
34 | protected = min(usage, READ_ONCE(c->low)); |
35 | old_protected = atomic_long_read(&c->low_usage); | |
36 | if (protected != old_protected) { | |
bf8d5d52 RG |
37 | old_protected = atomic_long_xchg(&c->low_usage, protected); |
38 | delta = protected - old_protected; | |
39 | if (delta) | |
40 | atomic_long_add(delta, &c->parent->children_low_usage); | |
41 | } | |
23067153 RG |
42 | } |
43 | ||
3e32cb2e JW |
44 | /** |
45 | * page_counter_cancel - take pages out of the local counter | |
46 | * @counter: counter | |
47 | * @nr_pages: number of pages to cancel | |
3e32cb2e | 48 | */ |
64f21993 | 49 | void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e JW |
50 | { |
51 | long new; | |
52 | ||
bbec2e15 | 53 | new = atomic_long_sub_return(nr_pages, &counter->usage); |
3e32cb2e | 54 | /* More uncharges than charges? */ |
9317d0ff JW |
55 | if (WARN_ONCE(new < 0, "page_counter underflow: %ld nr_pages=%lu\n", |
56 | new, nr_pages)) { | |
57 | new = 0; | |
58 | atomic_long_set(&counter->usage, new); | |
59 | } | |
60 | propagate_protected_usage(counter, new); | |
3e32cb2e JW |
61 | } |
62 | ||
63 | /** | |
64 | * page_counter_charge - hierarchically charge pages | |
65 | * @counter: counter | |
66 | * @nr_pages: number of pages to charge | |
67 | * | |
68 | * NOTE: This does not consider any configured counter limits. | |
69 | */ | |
70 | void page_counter_charge(struct page_counter *counter, unsigned long nr_pages) | |
71 | { | |
72 | struct page_counter *c; | |
73 | ||
74 | for (c = counter; c; c = c->parent) { | |
75 | long new; | |
76 | ||
bbec2e15 | 77 | new = atomic_long_add_return(nr_pages, &c->usage); |
a6f23d14 | 78 | propagate_protected_usage(c, new); |
3e32cb2e JW |
79 | /* |
80 | * This is indeed racy, but we can live with some | |
81 | * inaccuracy in the watermark. | |
82 | */ | |
6e4bd50f QC |
83 | if (new > READ_ONCE(c->watermark)) |
84 | WRITE_ONCE(c->watermark, new); | |
3e32cb2e JW |
85 | } |
86 | } | |
87 | ||
88 | /** | |
89 | * page_counter_try_charge - try to hierarchically charge pages | |
90 | * @counter: counter | |
91 | * @nr_pages: number of pages to charge | |
92 | * @fail: points first counter to hit its limit, if any | |
93 | * | |
6071ca52 JW |
94 | * Returns %true on success, or %false and @fail if the counter or one |
95 | * of its ancestors has hit its configured limit. | |
3e32cb2e | 96 | */ |
6071ca52 JW |
97 | bool page_counter_try_charge(struct page_counter *counter, |
98 | unsigned long nr_pages, | |
99 | struct page_counter **fail) | |
3e32cb2e JW |
100 | { |
101 | struct page_counter *c; | |
102 | ||
103 | for (c = counter; c; c = c->parent) { | |
104 | long new; | |
105 | /* | |
106 | * Charge speculatively to avoid an expensive CAS. If | |
107 | * a bigger charge fails, it might falsely lock out a | |
108 | * racing smaller charge and send it into reclaim | |
109 | * early, but the error is limited to the difference | |
110 | * between the two sizes, which is less than 2M/4M in | |
111 | * case of a THP locking out a regular page charge. | |
112 | * | |
113 | * The atomic_long_add_return() implies a full memory | |
114 | * barrier between incrementing the count and reading | |
d437024e | 115 | * the limit. When racing with page_counter_set_max(), |
3e32cb2e JW |
116 | * we either see the new limit or the setter sees the |
117 | * counter has changed and retries. | |
118 | */ | |
bbec2e15 RG |
119 | new = atomic_long_add_return(nr_pages, &c->usage); |
120 | if (new > c->max) { | |
121 | atomic_long_sub(nr_pages, &c->usage); | |
3e32cb2e JW |
122 | /* |
123 | * This is racy, but we can live with some | |
6e4bd50f QC |
124 | * inaccuracy in the failcnt which is only used |
125 | * to report stats. | |
3e32cb2e | 126 | */ |
6e4bd50f | 127 | data_race(c->failcnt++); |
3e32cb2e JW |
128 | *fail = c; |
129 | goto failed; | |
130 | } | |
a6f23d14 | 131 | propagate_protected_usage(c, new); |
3e32cb2e JW |
132 | /* |
133 | * Just like with failcnt, we can live with some | |
134 | * inaccuracy in the watermark. | |
135 | */ | |
6e4bd50f QC |
136 | if (new > READ_ONCE(c->watermark)) |
137 | WRITE_ONCE(c->watermark, new); | |
3e32cb2e | 138 | } |
6071ca52 | 139 | return true; |
3e32cb2e JW |
140 | |
141 | failed: | |
142 | for (c = counter; c != *fail; c = c->parent) | |
143 | page_counter_cancel(c, nr_pages); | |
144 | ||
6071ca52 | 145 | return false; |
3e32cb2e JW |
146 | } |
147 | ||
148 | /** | |
149 | * page_counter_uncharge - hierarchically uncharge pages | |
150 | * @counter: counter | |
151 | * @nr_pages: number of pages to uncharge | |
3e32cb2e | 152 | */ |
64f21993 | 153 | void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e JW |
154 | { |
155 | struct page_counter *c; | |
3e32cb2e | 156 | |
64f21993 JW |
157 | for (c = counter; c; c = c->parent) |
158 | page_counter_cancel(c, nr_pages); | |
3e32cb2e JW |
159 | } |
160 | ||
161 | /** | |
bbec2e15 | 162 | * page_counter_set_max - set the maximum number of pages allowed |
3e32cb2e | 163 | * @counter: counter |
bbec2e15 | 164 | * @nr_pages: limit to set |
3e32cb2e JW |
165 | * |
166 | * Returns 0 on success, -EBUSY if the current number of pages on the | |
167 | * counter already exceeds the specified limit. | |
168 | * | |
169 | * The caller must serialize invocations on the same counter. | |
170 | */ | |
bbec2e15 | 171 | int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e JW |
172 | { |
173 | for (;;) { | |
174 | unsigned long old; | |
bbec2e15 | 175 | long usage; |
3e32cb2e JW |
176 | |
177 | /* | |
178 | * Update the limit while making sure that it's not | |
179 | * below the concurrently-changing counter value. | |
180 | * | |
181 | * The xchg implies two full memory barriers before | |
182 | * and after, so the read-swap-read is ordered and | |
183 | * ensures coherency with page_counter_try_charge(): | |
184 | * that function modifies the count before checking | |
185 | * the limit, so if it sees the old limit, we see the | |
186 | * modified counter and retry. | |
187 | */ | |
13064781 | 188 | usage = page_counter_read(counter); |
3e32cb2e | 189 | |
bbec2e15 | 190 | if (usage > nr_pages) |
3e32cb2e JW |
191 | return -EBUSY; |
192 | ||
bbec2e15 | 193 | old = xchg(&counter->max, nr_pages); |
3e32cb2e | 194 | |
32d77270 | 195 | if (page_counter_read(counter) <= usage || nr_pages >= old) |
3e32cb2e JW |
196 | return 0; |
197 | ||
bbec2e15 | 198 | counter->max = old; |
3e32cb2e JW |
199 | cond_resched(); |
200 | } | |
201 | } | |
202 | ||
bf8d5d52 RG |
203 | /** |
204 | * page_counter_set_min - set the amount of protected memory | |
205 | * @counter: counter | |
206 | * @nr_pages: value to set | |
207 | * | |
208 | * The caller must serialize invocations on the same counter. | |
209 | */ | |
210 | void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages) | |
211 | { | |
212 | struct page_counter *c; | |
213 | ||
c3d53200 | 214 | WRITE_ONCE(counter->min, nr_pages); |
bf8d5d52 RG |
215 | |
216 | for (c = counter; c; c = c->parent) | |
217 | propagate_protected_usage(c, atomic_long_read(&c->usage)); | |
218 | } | |
219 | ||
23067153 RG |
220 | /** |
221 | * page_counter_set_low - set the amount of protected memory | |
222 | * @counter: counter | |
223 | * @nr_pages: value to set | |
224 | * | |
225 | * The caller must serialize invocations on the same counter. | |
226 | */ | |
227 | void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages) | |
228 | { | |
229 | struct page_counter *c; | |
230 | ||
f86b810c | 231 | WRITE_ONCE(counter->low, nr_pages); |
23067153 RG |
232 | |
233 | for (c = counter; c; c = c->parent) | |
bf8d5d52 | 234 | propagate_protected_usage(c, atomic_long_read(&c->usage)); |
23067153 RG |
235 | } |
236 | ||
3e32cb2e JW |
237 | /** |
238 | * page_counter_memparse - memparse() for page counter limits | |
239 | * @buf: string to parse | |
650c5e56 | 240 | * @max: string meaning maximum possible value |
3e32cb2e JW |
241 | * @nr_pages: returns the result in number of pages |
242 | * | |
243 | * Returns -EINVAL, or 0 and @nr_pages on success. @nr_pages will be | |
244 | * limited to %PAGE_COUNTER_MAX. | |
245 | */ | |
650c5e56 JW |
246 | int page_counter_memparse(const char *buf, const char *max, |
247 | unsigned long *nr_pages) | |
3e32cb2e | 248 | { |
3e32cb2e JW |
249 | char *end; |
250 | u64 bytes; | |
251 | ||
650c5e56 | 252 | if (!strcmp(buf, max)) { |
3e32cb2e JW |
253 | *nr_pages = PAGE_COUNTER_MAX; |
254 | return 0; | |
255 | } | |
256 | ||
257 | bytes = memparse(buf, &end); | |
258 | if (*end != '\0') | |
259 | return -EINVAL; | |
260 | ||
261 | *nr_pages = min(bytes / PAGE_SIZE, (u64)PAGE_COUNTER_MAX); | |
262 | ||
263 | return 0; | |
264 | } |