Commit | Line | Data |
---|---|---|
43b0536c SP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * DAMON-based page reclamation | |
4 | * | |
5 | * Author: SeongJae Park <sj@kernel.org> | |
6 | */ | |
7 | ||
8 | #define pr_fmt(fmt) "damon-reclaim: " fmt | |
9 | ||
10 | #include <linux/damon.h> | |
e6aff38b | 11 | #include <linux/kstrtox.h> |
43b0536c | 12 | #include <linux/module.h> |
43b0536c | 13 | |
fdfc119c SP |
14 | #include "modules-common.h" |
15 | ||
43b0536c SP |
16 | #ifdef MODULE_PARAM_PREFIX |
17 | #undef MODULE_PARAM_PREFIX | |
18 | #endif | |
19 | #define MODULE_PARAM_PREFIX "damon_reclaim." | |
20 | ||
21 | /* | |
22 | * Enable or disable DAMON_RECLAIM. | |
23 | * | |
24 | * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. | |
25 | * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could | |
26 | * do no real monitoring and reclamation due to the watermarks-based activation | |
27 | * condition. Refer to below descriptions for the watermarks parameter for | |
28 | * this. | |
29 | */ | |
30 | static bool enabled __read_mostly; | |
43b0536c | 31 | |
e035c280 SP |
32 | /* |
33 | * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``. | |
34 | * | |
35 | * Input parameters that updated while DAMON_RECLAIM is running are not applied | |
36 | * by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values | |
37 | * of parametrs except ``enabled`` again. Once the re-reading is done, this | |
38 | * parameter is set as ``N``. If invalid parameters are found while the | |
39 | * re-reading, DAMON_RECLAIM will be disabled. | |
40 | */ | |
41 | static bool commit_inputs __read_mostly; | |
42 | module_param(commit_inputs, bool, 0600); | |
43 | ||
43b0536c SP |
44 | /* |
45 | * Time threshold for cold memory regions identification in microseconds. | |
46 | * | |
47 | * If a memory region is not accessed for this or longer time, DAMON_RECLAIM | |
48 | * identifies the region as cold, and reclaims. 120 seconds by default. | |
49 | */ | |
50 | static unsigned long min_age __read_mostly = 120000000; | |
51 | module_param(min_age, ulong, 0600); | |
52 | ||
a9d57c73 SP |
53 | static struct damos_quota damon_reclaim_quota = { |
54 | /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */ | |
55 | .ms = 10, | |
56 | .sz = 128 * 1024 * 1024, | |
57 | .reset_interval = 1000, | |
58 | /* Within the quota, page out older regions first. */ | |
59 | .weight_sz = 0, | |
60 | .weight_nr_accesses = 0, | |
61 | .weight_age = 1 | |
62 | }; | |
63 | DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota); | |
43b0536c | 64 | |
81f8f57f | 65 | static struct damos_watermarks damon_reclaim_wmarks = { |
34f47ea6 SP |
66 | .metric = DAMOS_WMARK_FREE_MEM_RATE, |
67 | .interval = 5000000, /* 5 seconds */ | |
68 | .high = 500, /* 50 percent */ | |
69 | .mid = 400, /* 40 percent */ | |
70 | .low = 200, /* 20 percent */ | |
71 | }; | |
72 | DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks); | |
43b0536c | 73 | |
8c341ae3 | 74 | static struct damon_attrs damon_reclaim_mon_attrs = { |
fdfc119c SP |
75 | .sample_interval = 5000, /* 5 ms */ |
76 | .aggr_interval = 100000, /* 100 ms */ | |
8c341ae3 SP |
77 | .ops_update_interval = 0, |
78 | .min_nr_regions = 10, | |
79 | .max_nr_regions = 1000, | |
80 | }; | |
fdfc119c | 81 | DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs); |
43b0536c SP |
82 | |
83 | /* | |
84 | * Start of the target memory region in physical address. | |
85 | * | |
86 | * The start physical address of memory region that DAMON_RECLAIM will do work | |
87 | * against. By default, biggest System RAM is used as the region. | |
88 | */ | |
89 | static unsigned long monitor_region_start __read_mostly; | |
90 | module_param(monitor_region_start, ulong, 0600); | |
91 | ||
92 | /* | |
93 | * End of the target memory region in physical address. | |
94 | * | |
95 | * The end physical address of memory region that DAMON_RECLAIM will do work | |
96 | * against. By default, biggest System RAM is used as the region. | |
97 | */ | |
98 | static unsigned long monitor_region_end __read_mostly; | |
99 | module_param(monitor_region_end, ulong, 0600); | |
100 | ||
66d9faec SP |
101 | /* |
102 | * Skip anonymous pages reclamation. | |
103 | * | |
104 | * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous | |
105 | * pages. By default, ``N``. | |
106 | */ | |
107 | static bool skip_anon __read_mostly; | |
108 | module_param(skip_anon, bool, 0600); | |
109 | ||
43b0536c SP |
110 | /* |
111 | * PID of the DAMON thread | |
112 | * | |
113 | * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. | |
114 | * Else, -1. | |
115 | */ | |
116 | static int kdamond_pid __read_mostly = -1; | |
117 | module_param(kdamond_pid, int, 0400); | |
118 | ||
b71f3ea8 SP |
119 | static struct damos_stat damon_reclaim_stat; |
120 | DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, | |
121 | reclaim_tried_regions, reclaimed_regions, quota_exceeds); | |
60e52e7c | 122 | |
43b0536c SP |
123 | static struct damon_ctx *ctx; |
124 | static struct damon_target *target; | |
125 | ||
43b0536c SP |
126 | static struct damos *damon_reclaim_new_scheme(void) |
127 | { | |
f5a79d7c YD |
128 | struct damos_access_pattern pattern = { |
129 | /* Find regions having PAGE_SIZE or larger size */ | |
130 | .min_sz_region = PAGE_SIZE, | |
131 | .max_sz_region = ULONG_MAX, | |
132 | /* and not accessed at all */ | |
133 | .min_nr_accesses = 0, | |
134 | .max_nr_accesses = 0, | |
135 | /* for min_age or more micro-seconds */ | |
8c341ae3 SP |
136 | .min_age_region = min_age / |
137 | damon_reclaim_mon_attrs.aggr_interval, | |
f5a79d7c YD |
138 | .max_age_region = UINT_MAX, |
139 | }; | |
f5a79d7c YD |
140 | |
141 | return damon_new_scheme( | |
142 | &pattern, | |
43b0536c SP |
143 | /* page out those, as soon as found */ |
144 | DAMOS_PAGEOUT, | |
42f994b7 SP |
145 | /* for each aggregation interval */ |
146 | 0, | |
43b0536c | 147 | /* under the quota. */ |
a9d57c73 | 148 | &damon_reclaim_quota, |
43b0536c | 149 | /* (De)activate this according to the watermarks. */ |
34f47ea6 | 150 | &damon_reclaim_wmarks); |
43b0536c SP |
151 | } |
152 | ||
e035c280 | 153 | static int damon_reclaim_apply_parameters(void) |
43b0536c | 154 | { |
43b0536c | 155 | struct damos *scheme; |
66d9faec | 156 | struct damos_filter *filter; |
e035c280 | 157 | int err = 0; |
43b0536c | 158 | |
8c341ae3 | 159 | err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs); |
43b0536c SP |
160 | if (err) |
161 | return err; | |
162 | ||
e035c280 SP |
163 | /* Will be freed by next 'damon_set_schemes()' below */ |
164 | scheme = damon_reclaim_new_scheme(); | |
165 | if (!scheme) | |
166 | return -ENOMEM; | |
66d9faec SP |
167 | if (skip_anon) { |
168 | filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true); | |
169 | if (!filter) { | |
170 | /* Will be freed by next 'damon_set_schemes()' below */ | |
171 | damon_destroy_scheme(scheme); | |
172 | return -ENOMEM; | |
173 | } | |
174 | damos_add_filter(scheme, filter); | |
175 | } | |
cc713520 | 176 | damon_set_schemes(ctx, &scheme, 1); |
e035c280 | 177 | |
233f0b31 KX |
178 | return damon_set_region_biggest_system_ram_default(target, |
179 | &monitor_region_start, | |
180 | &monitor_region_end); | |
e035c280 | 181 | } |
43b0536c | 182 | |
e035c280 SP |
183 | static int damon_reclaim_turn(bool on) |
184 | { | |
185 | int err; | |
186 | ||
187 | if (!on) { | |
188 | err = damon_stop(&ctx, 1); | |
189 | if (!err) | |
190 | kdamond_pid = -1; | |
191 | return err; | |
43b0536c | 192 | } |
e035c280 SP |
193 | |
194 | err = damon_reclaim_apply_parameters(); | |
43b0536c | 195 | if (err) |
e035c280 | 196 | return err; |
43b0536c | 197 | |
8b9b0d33 | 198 | err = damon_start(&ctx, 1, true); |
e035c280 SP |
199 | if (err) |
200 | return err; | |
201 | kdamond_pid = ctx->kdamond->pid; | |
202 | return 0; | |
43b0536c SP |
203 | } |
204 | ||
d79905c7 | 205 | static int damon_reclaim_enabled_store(const char *val, |
059342d1 HT |
206 | const struct kernel_param *kp) |
207 | { | |
04e98764 SP |
208 | bool is_enabled = enabled; |
209 | bool enable; | |
210 | int err; | |
059342d1 | 211 | |
e6aff38b | 212 | err = kstrtobool(val, &enable); |
04e98764 SP |
213 | if (err) |
214 | return err; | |
059342d1 | 215 | |
04e98764 SP |
216 | if (is_enabled == enable) |
217 | return 0; | |
29492829 | 218 | |
04e98764 SP |
219 | /* Called before init function. The function will handle this. */ |
220 | if (!ctx) | |
221 | goto set_param_out; | |
222 | ||
223 | err = damon_reclaim_turn(enable); | |
224 | if (err) | |
225 | return err; | |
226 | ||
227 | set_param_out: | |
228 | enabled = enable; | |
229 | return err; | |
059342d1 HT |
230 | } |
231 | ||
232 | static const struct kernel_param_ops enabled_param_ops = { | |
d79905c7 | 233 | .set = damon_reclaim_enabled_store, |
059342d1 HT |
234 | .get = param_get_bool, |
235 | }; | |
236 | ||
237 | module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); | |
238 | MODULE_PARM_DESC(enabled, | |
239 | "Enable or disable DAMON_RECLAIM (default: disabled)"); | |
240 | ||
f25ab3bd SP |
241 | static int damon_reclaim_handle_commit_inputs(void) |
242 | { | |
243 | int err; | |
244 | ||
245 | if (!commit_inputs) | |
246 | return 0; | |
247 | ||
248 | err = damon_reclaim_apply_parameters(); | |
249 | commit_inputs = false; | |
250 | return err; | |
251 | } | |
252 | ||
60e52e7c SP |
253 | static int damon_reclaim_after_aggregation(struct damon_ctx *c) |
254 | { | |
255 | struct damos *s; | |
256 | ||
257 | /* update the stats parameter */ | |
b71f3ea8 SP |
258 | damon_for_each_scheme(s, c) |
259 | damon_reclaim_stat = s->stat; | |
e035c280 | 260 | |
f25ab3bd | 261 | return damon_reclaim_handle_commit_inputs(); |
e035c280 SP |
262 | } |
263 | ||
264 | static int damon_reclaim_after_wmarks_check(struct damon_ctx *c) | |
265 | { | |
f25ab3bd | 266 | return damon_reclaim_handle_commit_inputs(); |
60e52e7c SP |
267 | } |
268 | ||
43b0536c SP |
269 | static int __init damon_reclaim_init(void) |
270 | { | |
7ae2c17f | 271 | int err = damon_modules_new_paddr_ctx_target(&ctx, &target); |
43b0536c | 272 | |
7ae2c17f SP |
273 | if (err) |
274 | return err; | |
4d69c345 | 275 | |
e035c280 | 276 | ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check; |
60e52e7c | 277 | ctx->callback.after_aggregation = damon_reclaim_after_aggregation; |
43b0536c | 278 | |
04e98764 SP |
279 | /* 'enabled' has set before this function, probably via command line */ |
280 | if (enabled) | |
281 | err = damon_reclaim_turn(true); | |
29492829 | 282 | |
04e98764 | 283 | return err; |
43b0536c SP |
284 | } |
285 | ||
286 | module_init(damon_reclaim_init); |