Commit | Line | Data |
---|---|---|
43b0536c SP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * DAMON-based page reclamation | |
4 | * | |
5 | * Author: SeongJae Park <sj@kernel.org> | |
6 | */ | |
7 | ||
8 | #define pr_fmt(fmt) "damon-reclaim: " fmt | |
9 | ||
10 | #include <linux/damon.h> | |
e6aff38b | 11 | #include <linux/kstrtox.h> |
43b0536c | 12 | #include <linux/module.h> |
43b0536c | 13 | |
fdfc119c SP |
14 | #include "modules-common.h" |
15 | ||
43b0536c SP |
16 | #ifdef MODULE_PARAM_PREFIX |
17 | #undef MODULE_PARAM_PREFIX | |
18 | #endif | |
19 | #define MODULE_PARAM_PREFIX "damon_reclaim." | |
20 | ||
21 | /* | |
22 | * Enable or disable DAMON_RECLAIM. | |
23 | * | |
24 | * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. | |
25 | * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could | |
26 | * do no real monitoring and reclamation due to the watermarks-based activation | |
27 | * condition. Refer to below descriptions for the watermarks parameter for | |
28 | * this. | |
29 | */ | |
30 | static bool enabled __read_mostly; | |
43b0536c | 31 | |
e035c280 SP |
32 | /* |
33 | * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``. | |
34 | * | |
35 | * Input parameters that updated while DAMON_RECLAIM is running are not applied | |
36 | * by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values | |
37 | * of parametrs except ``enabled`` again. Once the re-reading is done, this | |
38 | * parameter is set as ``N``. If invalid parameters are found while the | |
39 | * re-reading, DAMON_RECLAIM will be disabled. | |
40 | */ | |
41 | static bool commit_inputs __read_mostly; | |
42 | module_param(commit_inputs, bool, 0600); | |
43 | ||
43b0536c SP |
44 | /* |
45 | * Time threshold for cold memory regions identification in microseconds. | |
46 | * | |
47 | * If a memory region is not accessed for this or longer time, DAMON_RECLAIM | |
48 | * identifies the region as cold, and reclaims. 120 seconds by default. | |
49 | */ | |
50 | static unsigned long min_age __read_mostly = 120000000; | |
51 | module_param(min_age, ulong, 0600); | |
52 | ||
a9d57c73 SP |
53 | static struct damos_quota damon_reclaim_quota = { |
54 | /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */ | |
55 | .ms = 10, | |
56 | .sz = 128 * 1024 * 1024, | |
57 | .reset_interval = 1000, | |
58 | /* Within the quota, page out older regions first. */ | |
59 | .weight_sz = 0, | |
60 | .weight_nr_accesses = 0, | |
61 | .weight_age = 1 | |
62 | }; | |
63 | DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota); | |
43b0536c | 64 | |
81f8f57f | 65 | static struct damos_watermarks damon_reclaim_wmarks = { |
34f47ea6 SP |
66 | .metric = DAMOS_WMARK_FREE_MEM_RATE, |
67 | .interval = 5000000, /* 5 seconds */ | |
68 | .high = 500, /* 50 percent */ | |
69 | .mid = 400, /* 40 percent */ | |
70 | .low = 200, /* 20 percent */ | |
71 | }; | |
72 | DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks); | |
43b0536c | 73 | |
8c341ae3 | 74 | static struct damon_attrs damon_reclaim_mon_attrs = { |
fdfc119c SP |
75 | .sample_interval = 5000, /* 5 ms */ |
76 | .aggr_interval = 100000, /* 100 ms */ | |
8c341ae3 SP |
77 | .ops_update_interval = 0, |
78 | .min_nr_regions = 10, | |
79 | .max_nr_regions = 1000, | |
80 | }; | |
fdfc119c | 81 | DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs); |
43b0536c SP |
82 | |
83 | /* | |
84 | * Start of the target memory region in physical address. | |
85 | * | |
86 | * The start physical address of memory region that DAMON_RECLAIM will do work | |
87 | * against. By default, biggest System RAM is used as the region. | |
88 | */ | |
89 | static unsigned long monitor_region_start __read_mostly; | |
90 | module_param(monitor_region_start, ulong, 0600); | |
91 | ||
92 | /* | |
93 | * End of the target memory region in physical address. | |
94 | * | |
95 | * The end physical address of memory region that DAMON_RECLAIM will do work | |
96 | * against. By default, biggest System RAM is used as the region. | |
97 | */ | |
98 | static unsigned long monitor_region_end __read_mostly; | |
99 | module_param(monitor_region_end, ulong, 0600); | |
100 | ||
66d9faec SP |
101 | /* |
102 | * Skip anonymous pages reclamation. | |
103 | * | |
104 | * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous | |
105 | * pages. By default, ``N``. | |
106 | */ | |
107 | static bool skip_anon __read_mostly; | |
108 | module_param(skip_anon, bool, 0600); | |
109 | ||
43b0536c SP |
110 | /* |
111 | * PID of the DAMON thread | |
112 | * | |
113 | * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. | |
114 | * Else, -1. | |
115 | */ | |
116 | static int kdamond_pid __read_mostly = -1; | |
117 | module_param(kdamond_pid, int, 0400); | |
118 | ||
b71f3ea8 SP |
119 | static struct damos_stat damon_reclaim_stat; |
120 | DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, | |
121 | reclaim_tried_regions, reclaimed_regions, quota_exceeds); | |
60e52e7c | 122 | |
43b0536c SP |
123 | static struct damon_ctx *ctx; |
124 | static struct damon_target *target; | |
125 | ||
43b0536c SP |
126 | static struct damos *damon_reclaim_new_scheme(void) |
127 | { | |
f5a79d7c YD |
128 | struct damos_access_pattern pattern = { |
129 | /* Find regions having PAGE_SIZE or larger size */ | |
130 | .min_sz_region = PAGE_SIZE, | |
131 | .max_sz_region = ULONG_MAX, | |
132 | /* and not accessed at all */ | |
133 | .min_nr_accesses = 0, | |
134 | .max_nr_accesses = 0, | |
135 | /* for min_age or more micro-seconds */ | |
8c341ae3 SP |
136 | .min_age_region = min_age / |
137 | damon_reclaim_mon_attrs.aggr_interval, | |
f5a79d7c YD |
138 | .max_age_region = UINT_MAX, |
139 | }; | |
f5a79d7c YD |
140 | |
141 | return damon_new_scheme( | |
142 | &pattern, | |
43b0536c SP |
143 | /* page out those, as soon as found */ |
144 | DAMOS_PAGEOUT, | |
42f994b7 SP |
145 | /* for each aggregation interval */ |
146 | 0, | |
43b0536c | 147 | /* under the quota. */ |
a9d57c73 | 148 | &damon_reclaim_quota, |
43b0536c | 149 | /* (De)activate this according to the watermarks. */ |
34f47ea6 | 150 | &damon_reclaim_wmarks); |
43b0536c SP |
151 | } |
152 | ||
1b0ca4e4 SP |
153 | static void damon_reclaim_copy_quota_status(struct damos_quota *dst, |
154 | struct damos_quota *src) | |
155 | { | |
156 | dst->total_charged_sz = src->total_charged_sz; | |
157 | dst->total_charged_ns = src->total_charged_ns; | |
158 | dst->charged_sz = src->charged_sz; | |
159 | dst->charged_from = src->charged_from; | |
160 | dst->charge_target_from = src->charge_target_from; | |
161 | dst->charge_addr_from = src->charge_addr_from; | |
162 | } | |
163 | ||
e035c280 | 164 | static int damon_reclaim_apply_parameters(void) |
43b0536c | 165 | { |
1b0ca4e4 | 166 | struct damos *scheme, *old_scheme; |
66d9faec | 167 | struct damos_filter *filter; |
e035c280 | 168 | int err = 0; |
43b0536c | 169 | |
8c341ae3 | 170 | err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs); |
43b0536c SP |
171 | if (err) |
172 | return err; | |
173 | ||
e035c280 SP |
174 | /* Will be freed by next 'damon_set_schemes()' below */ |
175 | scheme = damon_reclaim_new_scheme(); | |
176 | if (!scheme) | |
177 | return -ENOMEM; | |
1b0ca4e4 SP |
178 | if (!list_empty(&ctx->schemes)) { |
179 | damon_for_each_scheme(old_scheme, ctx) | |
180 | damon_reclaim_copy_quota_status(&scheme->quota, | |
181 | &old_scheme->quota); | |
182 | } | |
66d9faec SP |
183 | if (skip_anon) { |
184 | filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true); | |
185 | if (!filter) { | |
186 | /* Will be freed by next 'damon_set_schemes()' below */ | |
187 | damon_destroy_scheme(scheme); | |
188 | return -ENOMEM; | |
189 | } | |
190 | damos_add_filter(scheme, filter); | |
191 | } | |
cc713520 | 192 | damon_set_schemes(ctx, &scheme, 1); |
e035c280 | 193 | |
233f0b31 KX |
194 | return damon_set_region_biggest_system_ram_default(target, |
195 | &monitor_region_start, | |
196 | &monitor_region_end); | |
e035c280 | 197 | } |
43b0536c | 198 | |
e035c280 SP |
199 | static int damon_reclaim_turn(bool on) |
200 | { | |
201 | int err; | |
202 | ||
203 | if (!on) { | |
204 | err = damon_stop(&ctx, 1); | |
205 | if (!err) | |
206 | kdamond_pid = -1; | |
207 | return err; | |
43b0536c | 208 | } |
e035c280 SP |
209 | |
210 | err = damon_reclaim_apply_parameters(); | |
43b0536c | 211 | if (err) |
e035c280 | 212 | return err; |
43b0536c | 213 | |
8b9b0d33 | 214 | err = damon_start(&ctx, 1, true); |
e035c280 SP |
215 | if (err) |
216 | return err; | |
217 | kdamond_pid = ctx->kdamond->pid; | |
218 | return 0; | |
43b0536c SP |
219 | } |
220 | ||
d79905c7 | 221 | static int damon_reclaim_enabled_store(const char *val, |
059342d1 HT |
222 | const struct kernel_param *kp) |
223 | { | |
04e98764 SP |
224 | bool is_enabled = enabled; |
225 | bool enable; | |
226 | int err; | |
059342d1 | 227 | |
e6aff38b | 228 | err = kstrtobool(val, &enable); |
04e98764 SP |
229 | if (err) |
230 | return err; | |
059342d1 | 231 | |
04e98764 SP |
232 | if (is_enabled == enable) |
233 | return 0; | |
29492829 | 234 | |
04e98764 SP |
235 | /* Called before init function. The function will handle this. */ |
236 | if (!ctx) | |
237 | goto set_param_out; | |
238 | ||
239 | err = damon_reclaim_turn(enable); | |
240 | if (err) | |
241 | return err; | |
242 | ||
243 | set_param_out: | |
244 | enabled = enable; | |
245 | return err; | |
059342d1 HT |
246 | } |
247 | ||
248 | static const struct kernel_param_ops enabled_param_ops = { | |
d79905c7 | 249 | .set = damon_reclaim_enabled_store, |
059342d1 HT |
250 | .get = param_get_bool, |
251 | }; | |
252 | ||
253 | module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); | |
254 | MODULE_PARM_DESC(enabled, | |
255 | "Enable or disable DAMON_RECLAIM (default: disabled)"); | |
256 | ||
f25ab3bd SP |
257 | static int damon_reclaim_handle_commit_inputs(void) |
258 | { | |
259 | int err; | |
260 | ||
261 | if (!commit_inputs) | |
262 | return 0; | |
263 | ||
264 | err = damon_reclaim_apply_parameters(); | |
265 | commit_inputs = false; | |
266 | return err; | |
267 | } | |
268 | ||
60e52e7c SP |
269 | static int damon_reclaim_after_aggregation(struct damon_ctx *c) |
270 | { | |
271 | struct damos *s; | |
272 | ||
273 | /* update the stats parameter */ | |
b71f3ea8 SP |
274 | damon_for_each_scheme(s, c) |
275 | damon_reclaim_stat = s->stat; | |
e035c280 | 276 | |
f25ab3bd | 277 | return damon_reclaim_handle_commit_inputs(); |
e035c280 SP |
278 | } |
279 | ||
280 | static int damon_reclaim_after_wmarks_check(struct damon_ctx *c) | |
281 | { | |
f25ab3bd | 282 | return damon_reclaim_handle_commit_inputs(); |
60e52e7c SP |
283 | } |
284 | ||
43b0536c SP |
285 | static int __init damon_reclaim_init(void) |
286 | { | |
7ae2c17f | 287 | int err = damon_modules_new_paddr_ctx_target(&ctx, &target); |
43b0536c | 288 | |
7ae2c17f SP |
289 | if (err) |
290 | return err; | |
4d69c345 | 291 | |
e035c280 | 292 | ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check; |
60e52e7c | 293 | ctx->callback.after_aggregation = damon_reclaim_after_aggregation; |
43b0536c | 294 | |
04e98764 SP |
295 | /* 'enabled' has set before this function, probably via command line */ |
296 | if (enabled) | |
297 | err = damon_reclaim_turn(true); | |
29492829 | 298 | |
04e98764 | 299 | return err; |
43b0536c SP |
300 | } |
301 | ||
302 | module_init(damon_reclaim_init); |