Commit | Line | Data |
---|---|---|
43b0536c SP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * DAMON-based page reclamation | |
4 | * | |
5 | * Author: SeongJae Park <sj@kernel.org> | |
6 | */ | |
7 | ||
8 | #define pr_fmt(fmt) "damon-reclaim: " fmt | |
9 | ||
10 | #include <linux/damon.h> | |
11 | #include <linux/ioport.h> | |
12 | #include <linux/module.h> | |
13 | #include <linux/sched.h> | |
14 | #include <linux/workqueue.h> | |
15 | ||
fdfc119c SP |
16 | #include "modules-common.h" |
17 | ||
43b0536c SP |
18 | #ifdef MODULE_PARAM_PREFIX |
19 | #undef MODULE_PARAM_PREFIX | |
20 | #endif | |
21 | #define MODULE_PARAM_PREFIX "damon_reclaim." | |
22 | ||
23 | /* | |
24 | * Enable or disable DAMON_RECLAIM. | |
25 | * | |
26 | * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. | |
27 | * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could | |
28 | * do no real monitoring and reclamation due to the watermarks-based activation | |
29 | * condition. Refer to below descriptions for the watermarks parameter for | |
30 | * this. | |
31 | */ | |
32 | static bool enabled __read_mostly; | |
43b0536c | 33 | |
e035c280 SP |
34 | /* |
35 | * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``. | |
36 | * | |
37 | * Input parameters that updated while DAMON_RECLAIM is running are not applied | |
38 | * by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values | |
39 | * of parametrs except ``enabled`` again. Once the re-reading is done, this | |
40 | * parameter is set as ``N``. If invalid parameters are found while the | |
41 | * re-reading, DAMON_RECLAIM will be disabled. | |
42 | */ | |
43 | static bool commit_inputs __read_mostly; | |
44 | module_param(commit_inputs, bool, 0600); | |
45 | ||
43b0536c SP |
46 | /* |
47 | * Time threshold for cold memory regions identification in microseconds. | |
48 | * | |
49 | * If a memory region is not accessed for this or longer time, DAMON_RECLAIM | |
50 | * identifies the region as cold, and reclaims. 120 seconds by default. | |
51 | */ | |
52 | static unsigned long min_age __read_mostly = 120000000; | |
53 | module_param(min_age, ulong, 0600); | |
54 | ||
a9d57c73 SP |
55 | static struct damos_quota damon_reclaim_quota = { |
56 | /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */ | |
57 | .ms = 10, | |
58 | .sz = 128 * 1024 * 1024, | |
59 | .reset_interval = 1000, | |
60 | /* Within the quota, page out older regions first. */ | |
61 | .weight_sz = 0, | |
62 | .weight_nr_accesses = 0, | |
63 | .weight_age = 1 | |
64 | }; | |
65 | DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota); | |
43b0536c | 66 | |
34f47ea6 SP |
67 | struct damos_watermarks damon_reclaim_wmarks = { |
68 | .metric = DAMOS_WMARK_FREE_MEM_RATE, | |
69 | .interval = 5000000, /* 5 seconds */ | |
70 | .high = 500, /* 50 percent */ | |
71 | .mid = 400, /* 40 percent */ | |
72 | .low = 200, /* 20 percent */ | |
73 | }; | |
74 | DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks); | |
43b0536c | 75 | |
8c341ae3 | 76 | static struct damon_attrs damon_reclaim_mon_attrs = { |
fdfc119c SP |
77 | .sample_interval = 5000, /* 5 ms */ |
78 | .aggr_interval = 100000, /* 100 ms */ | |
8c341ae3 SP |
79 | .ops_update_interval = 0, |
80 | .min_nr_regions = 10, | |
81 | .max_nr_regions = 1000, | |
82 | }; | |
fdfc119c | 83 | DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs); |
43b0536c SP |
84 | |
85 | /* | |
86 | * Start of the target memory region in physical address. | |
87 | * | |
88 | * The start physical address of memory region that DAMON_RECLAIM will do work | |
89 | * against. By default, biggest System RAM is used as the region. | |
90 | */ | |
91 | static unsigned long monitor_region_start __read_mostly; | |
92 | module_param(monitor_region_start, ulong, 0600); | |
93 | ||
94 | /* | |
95 | * End of the target memory region in physical address. | |
96 | * | |
97 | * The end physical address of memory region that DAMON_RECLAIM will do work | |
98 | * against. By default, biggest System RAM is used as the region. | |
99 | */ | |
100 | static unsigned long monitor_region_end __read_mostly; | |
101 | module_param(monitor_region_end, ulong, 0600); | |
102 | ||
103 | /* | |
104 | * PID of the DAMON thread | |
105 | * | |
106 | * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. | |
107 | * Else, -1. | |
108 | */ | |
109 | static int kdamond_pid __read_mostly = -1; | |
110 | module_param(kdamond_pid, int, 0400); | |
111 | ||
b71f3ea8 SP |
112 | static struct damos_stat damon_reclaim_stat; |
113 | DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, | |
114 | reclaim_tried_regions, reclaimed_regions, quota_exceeds); | |
60e52e7c | 115 | |
43b0536c SP |
116 | static struct damon_ctx *ctx; |
117 | static struct damon_target *target; | |
118 | ||
43b0536c SP |
119 | static struct damos *damon_reclaim_new_scheme(void) |
120 | { | |
f5a79d7c YD |
121 | struct damos_access_pattern pattern = { |
122 | /* Find regions having PAGE_SIZE or larger size */ | |
123 | .min_sz_region = PAGE_SIZE, | |
124 | .max_sz_region = ULONG_MAX, | |
125 | /* and not accessed at all */ | |
126 | .min_nr_accesses = 0, | |
127 | .max_nr_accesses = 0, | |
128 | /* for min_age or more micro-seconds */ | |
8c341ae3 SP |
129 | .min_age_region = min_age / |
130 | damon_reclaim_mon_attrs.aggr_interval, | |
f5a79d7c YD |
131 | .max_age_region = UINT_MAX, |
132 | }; | |
f5a79d7c YD |
133 | |
134 | return damon_new_scheme( | |
135 | &pattern, | |
43b0536c SP |
136 | /* page out those, as soon as found */ |
137 | DAMOS_PAGEOUT, | |
138 | /* under the quota. */ | |
a9d57c73 | 139 | &damon_reclaim_quota, |
43b0536c | 140 | /* (De)activate this according to the watermarks. */ |
34f47ea6 | 141 | &damon_reclaim_wmarks); |
43b0536c SP |
142 | } |
143 | ||
e035c280 | 144 | static int damon_reclaim_apply_parameters(void) |
43b0536c | 145 | { |
43b0536c | 146 | struct damos *scheme; |
e035c280 SP |
147 | struct damon_addr_range addr_range; |
148 | int err = 0; | |
43b0536c | 149 | |
8c341ae3 | 150 | err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs); |
43b0536c SP |
151 | if (err) |
152 | return err; | |
153 | ||
e035c280 SP |
154 | /* Will be freed by next 'damon_set_schemes()' below */ |
155 | scheme = damon_reclaim_new_scheme(); | |
156 | if (!scheme) | |
157 | return -ENOMEM; | |
158 | err = damon_set_schemes(ctx, &scheme, 1); | |
159 | if (err) | |
160 | return err; | |
161 | ||
43b0536c SP |
162 | if (monitor_region_start > monitor_region_end) |
163 | return -EINVAL; | |
164 | if (!monitor_region_start && !monitor_region_end && | |
0d83b2d8 XH |
165 | !damon_find_biggest_system_ram(&monitor_region_start, |
166 | &monitor_region_end)) | |
43b0536c | 167 | return -EINVAL; |
e035c280 SP |
168 | addr_range.start = monitor_region_start; |
169 | addr_range.end = monitor_region_end; | |
170 | return damon_set_regions(target, &addr_range, 1); | |
171 | } | |
43b0536c | 172 | |
e035c280 SP |
173 | static int damon_reclaim_turn(bool on) |
174 | { | |
175 | int err; | |
176 | ||
177 | if (!on) { | |
178 | err = damon_stop(&ctx, 1); | |
179 | if (!err) | |
180 | kdamond_pid = -1; | |
181 | return err; | |
43b0536c | 182 | } |
e035c280 SP |
183 | |
184 | err = damon_reclaim_apply_parameters(); | |
43b0536c | 185 | if (err) |
e035c280 | 186 | return err; |
43b0536c | 187 | |
8b9b0d33 | 188 | err = damon_start(&ctx, 1, true); |
e035c280 SP |
189 | if (err) |
190 | return err; | |
191 | kdamond_pid = ctx->kdamond->pid; | |
192 | return 0; | |
43b0536c SP |
193 | } |
194 | ||
43b0536c SP |
195 | static struct delayed_work damon_reclaim_timer; |
196 | static void damon_reclaim_timer_fn(struct work_struct *work) | |
197 | { | |
198 | static bool last_enabled; | |
199 | bool now_enabled; | |
200 | ||
201 | now_enabled = enabled; | |
202 | if (last_enabled != now_enabled) { | |
203 | if (!damon_reclaim_turn(now_enabled)) | |
204 | last_enabled = now_enabled; | |
205 | else | |
206 | enabled = last_enabled; | |
207 | } | |
43b0536c SP |
208 | } |
209 | static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn); | |
210 | ||
29492829 SP |
211 | static bool damon_reclaim_initialized; |
212 | ||
d79905c7 | 213 | static int damon_reclaim_enabled_store(const char *val, |
059342d1 HT |
214 | const struct kernel_param *kp) |
215 | { | |
216 | int rc = param_set_bool(val, kp); | |
217 | ||
218 | if (rc < 0) | |
219 | return rc; | |
220 | ||
29492829 SP |
221 | /* system_wq might not initialized yet */ |
222 | if (!damon_reclaim_initialized) | |
223 | return rc; | |
224 | ||
f943e7e3 | 225 | schedule_delayed_work(&damon_reclaim_timer, 0); |
059342d1 HT |
226 | return 0; |
227 | } | |
228 | ||
229 | static const struct kernel_param_ops enabled_param_ops = { | |
d79905c7 | 230 | .set = damon_reclaim_enabled_store, |
059342d1 HT |
231 | .get = param_get_bool, |
232 | }; | |
233 | ||
234 | module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); | |
235 | MODULE_PARM_DESC(enabled, | |
236 | "Enable or disable DAMON_RECLAIM (default: disabled)"); | |
237 | ||
f25ab3bd SP |
238 | static int damon_reclaim_handle_commit_inputs(void) |
239 | { | |
240 | int err; | |
241 | ||
242 | if (!commit_inputs) | |
243 | return 0; | |
244 | ||
245 | err = damon_reclaim_apply_parameters(); | |
246 | commit_inputs = false; | |
247 | return err; | |
248 | } | |
249 | ||
60e52e7c SP |
250 | static int damon_reclaim_after_aggregation(struct damon_ctx *c) |
251 | { | |
252 | struct damos *s; | |
253 | ||
254 | /* update the stats parameter */ | |
b71f3ea8 SP |
255 | damon_for_each_scheme(s, c) |
256 | damon_reclaim_stat = s->stat; | |
e035c280 | 257 | |
f25ab3bd | 258 | return damon_reclaim_handle_commit_inputs(); |
e035c280 SP |
259 | } |
260 | ||
261 | static int damon_reclaim_after_wmarks_check(struct damon_ctx *c) | |
262 | { | |
f25ab3bd | 263 | return damon_reclaim_handle_commit_inputs(); |
60e52e7c SP |
264 | } |
265 | ||
43b0536c SP |
266 | static int __init damon_reclaim_init(void) |
267 | { | |
268 | ctx = damon_new_ctx(); | |
269 | if (!ctx) | |
270 | return -ENOMEM; | |
271 | ||
188043c7 JN |
272 | if (damon_select_ops(ctx, DAMON_OPS_PADDR)) { |
273 | damon_destroy_ctx(ctx); | |
4d69c345 | 274 | return -EINVAL; |
188043c7 | 275 | } |
4d69c345 | 276 | |
e035c280 | 277 | ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check; |
60e52e7c | 278 | ctx->callback.after_aggregation = damon_reclaim_after_aggregation; |
43b0536c | 279 | |
1971bd63 | 280 | target = damon_new_target(); |
43b0536c SP |
281 | if (!target) { |
282 | damon_destroy_ctx(ctx); | |
283 | return -ENOMEM; | |
284 | } | |
285 | damon_add_target(ctx, target); | |
286 | ||
287 | schedule_delayed_work(&damon_reclaim_timer, 0); | |
29492829 SP |
288 | |
289 | damon_reclaim_initialized = true; | |
43b0536c SP |
290 | return 0; |
291 | } | |
292 | ||
293 | module_init(damon_reclaim_init); |