[linux-block.git] / mm / damon / reclaim.c

// SPDX-License-Identifier: GPL-2.0
/*
 * DAMON-based page reclamation
 *
 * Author: SeongJae Park <sj@kernel.org>
 */

#define pr_fmt(fmt) "damon-reclaim: " fmt

#include <linux/damon.h>
#include <linux/ioport.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/workqueue.h>

#ifdef MODULE_PARAM_PREFIX
#undef MODULE_PARAM_PREFIX
#endif
#define MODULE_PARAM_PREFIX "damon_reclaim."

/*
 * Enable or disable DAMON_RECLAIM.
 *
 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
 * Setting it as ``N`` disables DAMON_RECLAIM.  Note that DAMON_RECLAIM could
 * do no real monitoring and reclamation due to the watermarks-based activation
 * condition.  Refer to below descriptions for the watermarks parameter for
 * this.
 */
static bool enabled __read_mostly;

/*
 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
 *
 * Input parameters that updated while DAMON_RECLAIM is running are not applied
 * by default.  Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
 * of parametrs except ``enabled`` again.  Once the re-reading is done, this
 * parameter is set as ``N``.  If invalid parameters are found while the
 * re-reading, DAMON_RECLAIM will be disabled.
 */
static bool commit_inputs __read_mostly;
module_param(commit_inputs, bool, 0600);

/*
 * Time threshold for cold memory regions identification in microseconds.
 *
 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM
 * identifies the region as cold, and reclaims.  120 seconds by default.
 */
static unsigned long min_age __read_mostly = 120000000;
module_param(min_age, ulong, 0600);

/*
 * Limit of time for trying the reclamation in milliseconds.
 *
 * DAMON_RECLAIM tries to use only up to this time within a time window
 * (quota_reset_interval_ms) for trying reclamation of cold pages.  This can be
 * used for limiting CPU consumption of DAMON_RECLAIM.  If the value is zero,
 * the limit is disabled.
 *
 * 10 ms by default.
 */
static unsigned long quota_ms __read_mostly = 10;
module_param(quota_ms, ulong, 0600);

/*
 * Limit of size of memory for the reclamation in bytes.
 *
 * DAMON_RECLAIM charges amount of memory which it tried to reclaim within a
 * time window (quota_reset_interval_ms) and makes no more than this limit is
 * tried.  This can be used for limiting consumption of CPU and IO.  If this
 * value is zero, the limit is disabled.
 *
 * 128 MiB by default.
 */
static unsigned long quota_sz __read_mostly = 128 * 1024 * 1024;
module_param(quota_sz, ulong, 0600);

/*
 * The time/size quota charge reset interval in milliseconds.
 *
 * The charge reset interval for the quota of time (quota_ms) and size
 * (quota_sz).  That is, DAMON_RECLAIM does not try reclamation for more than
 * quota_ms milliseconds or quota_sz bytes within quota_reset_interval_ms
 * milliseconds.
 *
 * 1 second by default.
 */
static unsigned long quota_reset_interval_ms __read_mostly = 1000;
module_param(quota_reset_interval_ms, ulong, 0600);

/*
 * The watermarks check time interval in microseconds.
 *
 * Minimal time to wait before checking the watermarks, when DAMON_RECLAIM is
 * enabled but inactive due to its watermarks rule.  5 seconds by default.
 */
static unsigned long wmarks_interval __read_mostly = 5000000;
module_param(wmarks_interval, ulong, 0600);

/*
 * Free memory rate (per thousand) for the high watermark.
 *
 * If free memory of the system in bytes per thousand bytes is higher than
 * this, DAMON_RECLAIM becomes inactive, so it does nothing but periodically
 * checks the watermarks.  500 (50%) by default.
 */
static unsigned long wmarks_high __read_mostly = 500;
module_param(wmarks_high, ulong, 0600);

/*
 * Free memory rate (per thousand) for the middle watermark.
 *
 * If free memory of the system in bytes per thousand bytes is between this and
 * the low watermark, DAMON_RECLAIM becomes active, so starts the monitoring
 * and the reclaiming.  400 (40%) by default.
 */
static unsigned long wmarks_mid __read_mostly = 400;
module_param(wmarks_mid, ulong, 0600);

/*
 * Free memory rate (per thousand) for the low watermark.
 *
 * If free memory of the system in bytes per thousand bytes is lower than this,
 * DAMON_RECLAIM becomes inactive, so it does nothing but periodically checks
 * the watermarks.  In the case, the system falls back to the LRU-based page
 * granularity reclamation logic.  200 (20%) by default.
 */
static unsigned long wmarks_low __read_mostly = 200;
module_param(wmarks_low, ulong, 0600);

/*
 * Sampling interval for the monitoring in microseconds.
 *
 * The sampling interval of DAMON for the cold memory monitoring.  Please refer
 * to the DAMON documentation for more detail.  5 ms by default.
 */
static unsigned long sample_interval __read_mostly = 5000;
module_param(sample_interval, ulong, 0600);

/*
 * Aggregation interval for the monitoring in microseconds.
 *
 * The aggregation interval of DAMON for the cold memory monitoring.  Please
 * refer to the DAMON documentation for more detail.  100 ms by default.
 */
static unsigned long aggr_interval __read_mostly = 100000;
module_param(aggr_interval, ulong, 0600);

/*
 * Minimum number of monitoring regions.
 *
 * The minimal number of monitoring regions of DAMON for the cold memory
 * monitoring.  This can be used to set lower-bound of the monitoring quality.
 * But, setting this too high could result in increased monitoring overhead.
 * Please refer to the DAMON documentation for more detail.  10 by default.
 */
static unsigned long min_nr_regions __read_mostly = 10;
module_param(min_nr_regions, ulong, 0600);

/*
 * Maximum number of monitoring regions.
 *
 * The maximum number of monitoring regions of DAMON for the cold memory
 * monitoring.  This can be used to set upper-bound of the monitoring overhead.
 * However, setting this too low could result in bad monitoring quality.
 * Please refer to the DAMON documentation for more detail.  1000 by default.
 */
static unsigned long max_nr_regions __read_mostly = 1000;
module_param(max_nr_regions, ulong, 0600);

/*
 * Start of the target memory region in physical address.
 *
 * The start physical address of memory region that DAMON_RECLAIM will do work
 * against.  By default, biggest System RAM is used as the region.
 */
static unsigned long monitor_region_start __read_mostly;
module_param(monitor_region_start, ulong, 0600);

/*
 * End of the target memory region in physical address.
 *
 * The end physical address of memory region that DAMON_RECLAIM will do work
 * against.  By default, biggest System RAM is used as the region.
 */
static unsigned long monitor_region_end __read_mostly;
module_param(monitor_region_end, ulong, 0600);

/*
 * PID of the DAMON thread
 *
 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
 * Else, -1.
 */
static int kdamond_pid __read_mostly = -1;
module_param(kdamond_pid, int, 0400);

/*
 * Number of memory regions that tried to be reclaimed.
 */
static unsigned long nr_reclaim_tried_regions __read_mostly;
module_param(nr_reclaim_tried_regions, ulong, 0400);

/*
 * Total bytes of memory regions that tried to be reclaimed.
 */
static unsigned long bytes_reclaim_tried_regions __read_mostly;
module_param(bytes_reclaim_tried_regions, ulong, 0400);

/*
 * Number of memory regions that successfully be reclaimed.
 */
static unsigned long nr_reclaimed_regions __read_mostly;
module_param(nr_reclaimed_regions, ulong, 0400);

/*
 * Total bytes of memory regions that successfully be reclaimed.
 */
static unsigned long bytes_reclaimed_regions __read_mostly;
module_param(bytes_reclaimed_regions, ulong, 0400);

/*
 * Number of times that the time/space quota limits have exceeded
 */
static unsigned long nr_quota_exceeds __read_mostly;
module_param(nr_quota_exceeds, ulong, 0400);

static struct damon_ctx *ctx;
static struct damon_target *target;

struct damon_reclaim_ram_walk_arg {
	unsigned long start;
	unsigned long end;
};

static int walk_system_ram(struct resource *res, void *arg)
{
	struct damon_reclaim_ram_walk_arg *a = arg;

	if (a->end - a->start < resource_size(res)) {
		a->start = res->start;
		a->end = res->end;
	}
	return 0;
}

/*
 * Find biggest 'System RAM' resource and store its start and end address in
 * @start and @end, respectively.  If no System RAM is found, returns false.
 */
static bool get_monitoring_region(unsigned long *start, unsigned long *end)
{
	struct damon_reclaim_ram_walk_arg arg = {};

	walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram);
	if (arg.end <= arg.start)
		return false;

	*start = arg.start;
	*end = arg.end;
	return true;
}

static struct damos *damon_reclaim_new_scheme(void)
{
	struct damos_watermarks wmarks = {
		.metric = DAMOS_WMARK_FREE_MEM_RATE,
		.interval = wmarks_interval,
		.high = wmarks_high,
		.mid = wmarks_mid,
		.low = wmarks_low,
	};
	struct damos_quota quota = {
		/*
		 * Do not try reclamation for more than quota_ms milliseconds
		 * or quota_sz bytes within quota_reset_interval_ms.
		 */
		.ms = quota_ms,
		.sz = quota_sz,
		.reset_interval = quota_reset_interval_ms,
		/* Within the quota, page out older regions first. */
		.weight_sz = 0,
		.weight_nr_accesses = 0,
		.weight_age = 1
	};
	struct damos *scheme = damon_new_scheme(
			/* Find regions having PAGE_SIZE or larger size */
			PAGE_SIZE, ULONG_MAX,
			/* and not accessed at all */
			0, 0,
			/* for min_age or more micro-seconds, and */
			min_age / aggr_interval, UINT_MAX,
			/* page out those, as soon as found */
			DAMOS_PAGEOUT,
			/* under the quota. */
			&quota,
			/* (De)activate this according to the watermarks. */
			&wmarks);

	return scheme;
}

static int damon_reclaim_apply_parameters(void)
{
	struct damos *scheme;
	struct damon_addr_range addr_range;
	int err = 0;

	err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0,
			min_nr_regions, max_nr_regions);
	if (err)
		return err;

	/* Will be freed by next 'damon_set_schemes()' below */
	scheme = damon_reclaim_new_scheme();
	if (!scheme)
		return -ENOMEM;
	err = damon_set_schemes(ctx, &scheme, 1);
	if (err)
		return err;

	if (monitor_region_start > monitor_region_end)
		return -EINVAL;
	if (!monitor_region_start && !monitor_region_end &&
			!get_monitoring_region(&monitor_region_start,
				&monitor_region_end))
		return -EINVAL;
	addr_range.start = monitor_region_start;
	addr_range.end = monitor_region_end;
	return damon_set_regions(target, &addr_range, 1);
}

static int damon_reclaim_turn(bool on)
{
	int err;

	if (!on) {
		err = damon_stop(&ctx, 1);
		if (!err)
			kdamond_pid = -1;
		return err;
	}

	err = damon_reclaim_apply_parameters();
	if (err)
		return err;

	err = damon_start(&ctx, 1, true);
	if (err)
		return err;
	kdamond_pid = ctx->kdamond->pid;
	return 0;
}

static struct delayed_work damon_reclaim_timer;
static void damon_reclaim_timer_fn(struct work_struct *work)
{
	static bool last_enabled;
	bool now_enabled;

	now_enabled = enabled;
	if (last_enabled != now_enabled) {
		if (!damon_reclaim_turn(now_enabled))
			last_enabled = now_enabled;
		else
			enabled = last_enabled;
	}
}
static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn);

static bool damon_reclaim_initialized;

static int damon_reclaim_enabled_store(const char *val,
		const struct kernel_param *kp)
{
	int rc = param_set_bool(val, kp);

	if (rc < 0)
		return rc;

	/* system_wq might not initialized yet */
	if (!damon_reclaim_initialized)
		return rc;

	schedule_delayed_work(&damon_reclaim_timer, 0);
	return 0;
}

static const struct kernel_param_ops enabled_param_ops = {
	.set = damon_reclaim_enabled_store,
	.get = param_get_bool,
};

module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
MODULE_PARM_DESC(enabled,
	"Enable or disable DAMON_RECLAIM (default: disabled)");

static int damon_reclaim_handle_commit_inputs(void)
{
	int err;

	if (!commit_inputs)
		return 0;

	err = damon_reclaim_apply_parameters();
	commit_inputs = false;
	return err;
}

static int damon_reclaim_after_aggregation(struct damon_ctx *c)
{
	struct damos *s;

	/* update the stats parameter */
	damon_for_each_scheme(s, c) {
		nr_reclaim_tried_regions = s->stat.nr_tried;
		bytes_reclaim_tried_regions = s->stat.sz_tried;
		nr_reclaimed_regions = s->stat.nr_applied;
		bytes_reclaimed_regions = s->stat.sz_applied;
		nr_quota_exceeds = s->stat.qt_exceeds;
	}

	return damon_reclaim_handle_commit_inputs();
}

static int damon_reclaim_after_wmarks_check(struct damon_ctx *c)
{
	return damon_reclaim_handle_commit_inputs();
}

static int __init damon_reclaim_init(void)
{
	ctx = damon_new_ctx();
	if (!ctx)
		return -ENOMEM;

	if (damon_select_ops(ctx, DAMON_OPS_PADDR)) {
		damon_destroy_ctx(ctx);
		return -EINVAL;
	}

	ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check;
	ctx->callback.after_aggregation = damon_reclaim_after_aggregation;

	target = damon_new_target();
	if (!target) {
		damon_destroy_ctx(ctx);
		return -ENOMEM;
	}
	damon_add_target(ctx, target);

	schedule_delayed_work(&damon_reclaim_timer, 0);

	damon_reclaim_initialized = true;
	return 0;
}

module_init(damon_reclaim_init);
Commit	Line	Data
43b0536c SP	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* DAMON-based page reclamation
	4	*
	5	* Author: SeongJae Park <sj@kernel.org>
	6	*/
	7
	8	#define pr_fmt(fmt) "damon-reclaim: " fmt
	9
	10	#include <linux/damon.h>
	11	#include <linux/ioport.h>
	12	#include <linux/module.h>
	13	#include <linux/sched.h>
	14	#include <linux/workqueue.h>
	15
	16	#ifdef MODULE_PARAM_PREFIX
	17	#undef MODULE_PARAM_PREFIX
	18	#endif
	19	#define MODULE_PARAM_PREFIX "damon_reclaim."
	20
	21	/*
	22	* Enable or disable DAMON_RECLAIM.
	23	*
	24	* You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
	25	* Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could
	26	* do no real monitoring and reclamation due to the watermarks-based activation
	27	* condition. Refer to below descriptions for the watermarks parameter for
	28	* this.
	29	*/
	30	static bool enabled __read_mostly;
43b0536c	31
e035c280 SP	32	/*
	33	* Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
	34	*
	35	* Input parameters that updated while DAMON_RECLAIM is running are not applied
	36	* by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
	37	* of parametrs except ``enabled`` again. Once the re-reading is done, this
	38	* parameter is set as ``N``. If invalid parameters are found while the
	39	* re-reading, DAMON_RECLAIM will be disabled.
	40	*/
	41	static bool commit_inputs __read_mostly;
	42	module_param(commit_inputs, bool, 0600);
	43
43b0536c SP	44	/*
	45	* Time threshold for cold memory regions identification in microseconds.
	46	*
	47	* If a memory region is not accessed for this or longer time, DAMON_RECLAIM
	48	* identifies the region as cold, and reclaims. 120 seconds by default.
	49	*/
	50	static unsigned long min_age __read_mostly = 120000000;
	51	module_param(min_age, ulong, 0600);
	52
	53	/*
	54	* Limit of time for trying the reclamation in milliseconds.
	55	*
	56	* DAMON_RECLAIM tries to use only up to this time within a time window
	57	* (quota_reset_interval_ms) for trying reclamation of cold pages. This can be
	58	* used for limiting CPU consumption of DAMON_RECLAIM. If the value is zero,
	59	* the limit is disabled.
	60	*
	61	* 10 ms by default.
	62	*/
	63	static unsigned long quota_ms __read_mostly = 10;
	64	module_param(quota_ms, ulong, 0600);
	65
	66	/*
	67	* Limit of size of memory for the reclamation in bytes.
	68	*
	69	* DAMON_RECLAIM charges amount of memory which it tried to reclaim within a
	70	* time window (quota_reset_interval_ms) and makes no more than this limit is
	71	* tried. This can be used for limiting consumption of CPU and IO. If this
	72	* value is zero, the limit is disabled.
	73	*
	74	* 128 MiB by default.
	75	*/
	76	static unsigned long quota_sz __read_mostly = 128 * 1024 * 1024;
	77	module_param(quota_sz, ulong, 0600);
	78
	79	/*
	80	* The time/size quota charge reset interval in milliseconds.
	81	*
	82	* The charge reset interval for the quota of time (quota_ms) and size
	83	* (quota_sz). That is, DAMON_RECLAIM does not try reclamation for more than
	84	* quota_ms milliseconds or quota_sz bytes within quota_reset_interval_ms
	85	* milliseconds.
	86	*
	87	* 1 second by default.
	88	*/
	89	static unsigned long quota_reset_interval_ms __read_mostly = 1000;
	90	module_param(quota_reset_interval_ms, ulong, 0600);
	91
	92	/*
	93	* The watermarks check time interval in microseconds.
	94	*
	95	* Minimal time to wait before checking the watermarks, when DAMON_RECLAIM is
	96	* enabled but inactive due to its watermarks rule. 5 seconds by default.
	97	*/
	98	static unsigned long wmarks_interval __read_mostly = 5000000;
	99	module_param(wmarks_interval, ulong, 0600);
	100
	101	/*
	102	* Free memory rate (per thousand) for the high watermark.
	103	*
	104	* If free memory of the system in bytes per thousand bytes is higher than
	105	* this, DAMON_RECLAIM becomes inactive, so it does nothing but periodically
	106	* checks the watermarks. 500 (50%) by default.
	107	*/
108	static unsigned long wmarks_high __read_mostly = 500;
109	module_param(wmarks_high, ulong, 0600);
110
111	/*
112	* Free memory rate (per thousand) for the middle watermark.
113	*
114	* If free memory of the system in bytes per thousand bytes is between this and
115	* the low watermark, DAMON_RECLAIM becomes active, so starts the monitoring
116	* and the reclaiming. 400 (40%) by default.
117	*/
118	static unsigned long wmarks_mid __read_mostly = 400;
119	module_param(wmarks_mid, ulong, 0600);
120
121	/*
122	* Free memory rate (per thousand) for the low watermark.
123	*
124	* If free memory of the system in bytes per thousand bytes is lower than this,
125	* DAMON_RECLAIM becomes inactive, so it does nothing but periodically checks
126	* the watermarks. In the case, the system falls back to the LRU-based page
127	* granularity reclamation logic. 200 (20%) by default.
128	*/
129	static unsigned long wmarks_low __read_mostly = 200;
130	module_param(wmarks_low, ulong, 0600);
131
132	/*
133	* Sampling interval for the monitoring in microseconds.
134	*
135	* The sampling interval of DAMON for the cold memory monitoring. Please refer
136	* to the DAMON documentation for more detail. 5 ms by default.
137	*/
138	static unsigned long sample_interval __read_mostly = 5000;
139	module_param(sample_interval, ulong, 0600);
140
141	/*
142	* Aggregation interval for the monitoring in microseconds.
143	*
144	* The aggregation interval of DAMON for the cold memory monitoring. Please
145	* refer to the DAMON documentation for more detail. 100 ms by default.
146	*/
147	static unsigned long aggr_interval __read_mostly = 100000;
148	module_param(aggr_interval, ulong, 0600);
149
150	/*
151	* Minimum number of monitoring regions.
152	*
153	* The minimal number of monitoring regions of DAMON for the cold memory
154	* monitoring. This can be used to set lower-bound of the monitoring quality.
155	* But, setting this too high could result in increased monitoring overhead.
156	* Please refer to the DAMON documentation for more detail. 10 by default.
157	*/
158	static unsigned long min_nr_regions __read_mostly = 10;
159	module_param(min_nr_regions, ulong, 0600);
160
161	/*
162	* Maximum number of monitoring regions.
163	*
164	* The maximum number of monitoring regions of DAMON for the cold memory
165	* monitoring. This can be used to set upper-bound of the monitoring overhead.
166	* However, setting this too low could result in bad monitoring quality.
167	* Please refer to the DAMON documentation for more detail. 1000 by default.
168	*/
169	static unsigned long max_nr_regions __read_mostly = 1000;
170	module_param(max_nr_regions, ulong, 0600);
171
172	/*
173	* Start of the target memory region in physical address.
174	*
175	* The start physical address of memory region that DAMON_RECLAIM will do work
176	* against. By default, biggest System RAM is used as the region.
177	*/
178	static unsigned long monitor_region_start __read_mostly;
179	module_param(monitor_region_start, ulong, 0600);
180
181	/*
182	* End of the target memory region in physical address.
183	*
184	* The end physical address of memory region that DAMON_RECLAIM will do work
185	* against. By default, biggest System RAM is used as the region.
186	*/
187	static unsigned long monitor_region_end __read_mostly;
188	module_param(monitor_region_end, ulong, 0600);
189
190	/*
191	* PID of the DAMON thread
192	*
193	* If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
194	* Else, -1.
195	*/
196	static int kdamond_pid __read_mostly = -1;
197	module_param(kdamond_pid, int, 0400);
198
60e52e7c SP	199	/*
	200	* Number of memory regions that tried to be reclaimed.
	201	*/
	202	static unsigned long nr_reclaim_tried_regions __read_mostly;
	203	module_param(nr_reclaim_tried_regions, ulong, 0400);
	204
	205	/*
	206	* Total bytes of memory regions that tried to be reclaimed.
	207	*/
	208	static unsigned long bytes_reclaim_tried_regions __read_mostly;
	209	module_param(bytes_reclaim_tried_regions, ulong, 0400);
	210
	211	/*
	212	* Number of memory regions that successfully be reclaimed.
	213	*/
	214	static unsigned long nr_reclaimed_regions __read_mostly;
	215	module_param(nr_reclaimed_regions, ulong, 0400);
	216
	217	/*
	218	* Total bytes of memory regions that successfully be reclaimed.
	219	*/
	220	static unsigned long bytes_reclaimed_regions __read_mostly;
	221	module_param(bytes_reclaimed_regions, ulong, 0400);
	222
	223	/*
	224	* Number of times that the time/space quota limits have exceeded
	225	*/
	226	static unsigned long nr_quota_exceeds __read_mostly;
	227	module_param(nr_quota_exceeds, ulong, 0400);
	228
43b0536c SP	229	static struct damon_ctx *ctx;
	230	static struct damon_target *target;
	231
	232	struct damon_reclaim_ram_walk_arg {
	233	unsigned long start;
	234	unsigned long end;
	235	};
	236
	237	static int walk_system_ram(struct resource res, void arg)
	238	{
	239	struct damon_reclaim_ram_walk_arg *a = arg;
	240
2e14a8d3	241	if (a->end - a->start < resource_size(res)) {
43b0536c SP	242	a->start = res->start;
	243	a->end = res->end;
	244	}
	245	return 0;
	246	}
	247
	248	/*
	249	* Find biggest 'System RAM' resource and store its start and end address in
	250	* @start and @end, respectively. If no System RAM is found, returns false.
	251	*/
	252	static bool get_monitoring_region(unsigned long start, unsigned long end)
	253	{
	254	struct damon_reclaim_ram_walk_arg arg = {};
	255
	256	walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram);
	257	if (arg.end <= arg.start)
	258	return false;
	259
	260	*start = arg.start;
	261	*end = arg.end;
	262	return true;
	263	}
	264
	265	static struct damos *damon_reclaim_new_scheme(void)
	266	{
	267	struct damos_watermarks wmarks = {
	268	.metric = DAMOS_WMARK_FREE_MEM_RATE,
	269	.interval = wmarks_interval,
	270	.high = wmarks_high,
	271	.mid = wmarks_mid,
	272	.low = wmarks_low,
	273	};
	274	struct damos_quota quota = {
	275	/*
	276	* Do not try reclamation for more than quota_ms milliseconds
	277	* or quota_sz bytes within quota_reset_interval_ms.
	278	*/
	279	.ms = quota_ms,
	280	.sz = quota_sz,
	281	.reset_interval = quota_reset_interval_ms,
	282	/* Within the quota, page out older regions first. */
	283	.weight_sz = 0,
	284	.weight_nr_accesses = 0,
	285	.weight_age = 1
	286	};
	287	struct damos *scheme = damon_new_scheme(
	288	/* Find regions having PAGE_SIZE or larger size */
	289	PAGE_SIZE, ULONG_MAX,
	290	/* and not accessed at all */
	291	0, 0,
	292	/* for min_age or more micro-seconds, and */
	293	min_age / aggr_interval, UINT_MAX,
	294	/* page out those, as soon as found */
	295	DAMOS_PAGEOUT,
	296	/* under the quota. */
	297	&quota,
	298	/* (De)activate this according to the watermarks. */
	299	&wmarks);
	300
	301	return scheme;
	302	}
	303
e035c280	304	static int damon_reclaim_apply_parameters(void)
43b0536c	305	{
43b0536c	306	struct damos *scheme;
e035c280 SP	307	struct damon_addr_range addr_range;
e035c280 SP	308	int err = 0;
43b0536c SP	309
	310	err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0,
	311	min_nr_regions, max_nr_regions);
	312	if (err)
	313	return err;
	314
e035c280 SP	315	/* Will be freed by next 'damon_set_schemes()' below */
	316	scheme = damon_reclaim_new_scheme();
	317	if (!scheme)
	318	return -ENOMEM;
	319	err = damon_set_schemes(ctx, &scheme, 1);
	320	if (err)
	321	return err;
	322
43b0536c SP	323	if (monitor_region_start > monitor_region_end)
	324	return -EINVAL;
	325	if (!monitor_region_start && !monitor_region_end &&
	326	!get_monitoring_region(&monitor_region_start,
	327	&monitor_region_end))
	328	return -EINVAL;
e035c280 SP	329	addr_range.start = monitor_region_start;
	330	addr_range.end = monitor_region_end;
	331	return damon_set_regions(target, &addr_range, 1);
	332	}
43b0536c	333
e035c280 SP	334	static int damon_reclaim_turn(bool on)
	335	{
	336	int err;
	337
	338	if (!on) {
	339	err = damon_stop(&ctx, 1);
	340	if (!err)
	341	kdamond_pid = -1;
	342	return err;
43b0536c	343	}
e035c280 SP	344
e035c280 SP	345	err = damon_reclaim_apply_parameters();
43b0536c	346	if (err)
e035c280	347	return err;
43b0536c	348
8b9b0d33	349	err = damon_start(&ctx, 1, true);
e035c280 SP	350	if (err)
	351	return err;
	352	kdamond_pid = ctx->kdamond->pid;
	353	return 0;
43b0536c SP	354	}
43b0536c SP	355
43b0536c SP	356	static struct delayed_work damon_reclaim_timer;
	357	static void damon_reclaim_timer_fn(struct work_struct *work)
	358	{
	359	static bool last_enabled;
	360	bool now_enabled;
	361
	362	now_enabled = enabled;
	363	if (last_enabled != now_enabled) {
	364	if (!damon_reclaim_turn(now_enabled))
	365	last_enabled = now_enabled;
	366	else
	367	enabled = last_enabled;
	368	}
43b0536c SP	369	}
	370	static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn);
	371
29492829 SP	372	static bool damon_reclaim_initialized;
29492829 SP	373
d79905c7	374	static int damon_reclaim_enabled_store(const char *val,
059342d1 HT	375	const struct kernel_param *kp)
	376	{
	377	int rc = param_set_bool(val, kp);
	378
	379	if (rc < 0)
	380	return rc;
	381
29492829 SP	382	/* system_wq might not initialized yet */
	383	if (!damon_reclaim_initialized)
	384	return rc;
	385
f943e7e3	386	schedule_delayed_work(&damon_reclaim_timer, 0);
059342d1 HT	387	return 0;
	388	}
	389
	390	static const struct kernel_param_ops enabled_param_ops = {
d79905c7	391	.set = damon_reclaim_enabled_store,
059342d1 HT	392	.get = param_get_bool,
	393	};
	394
	395	module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
	396	MODULE_PARM_DESC(enabled,
	397	"Enable or disable DAMON_RECLAIM (default: disabled)");
	398
f25ab3bd SP	399	static int damon_reclaim_handle_commit_inputs(void)
	400	{
	401	int err;
	402
	403	if (!commit_inputs)
	404	return 0;
	405
	406	err = damon_reclaim_apply_parameters();
	407	commit_inputs = false;
	408	return err;
	409	}
	410
60e52e7c SP	411	static int damon_reclaim_after_aggregation(struct damon_ctx *c)
	412	{
	413	struct damos *s;
	414
	415	/* update the stats parameter */
	416	damon_for_each_scheme(s, c) {
	417	nr_reclaim_tried_regions = s->stat.nr_tried;
	418	bytes_reclaim_tried_regions = s->stat.sz_tried;
	419	nr_reclaimed_regions = s->stat.nr_applied;
	420	bytes_reclaimed_regions = s->stat.sz_applied;
	421	nr_quota_exceeds = s->stat.qt_exceeds;
	422	}
e035c280	423
f25ab3bd	424	return damon_reclaim_handle_commit_inputs();
e035c280 SP	425	}
	426
	427	static int damon_reclaim_after_wmarks_check(struct damon_ctx *c)
	428	{
f25ab3bd	429	return damon_reclaim_handle_commit_inputs();
60e52e7c SP	430	}
60e52e7c SP	431
43b0536c SP	432	static int __init damon_reclaim_init(void)
	433	{
	434	ctx = damon_new_ctx();
	435	if (!ctx)
	436	return -ENOMEM;
	437
188043c7 JN	438	if (damon_select_ops(ctx, DAMON_OPS_PADDR)) {
188043c7 JN	439	damon_destroy_ctx(ctx);
4d69c345	440	return -EINVAL;
188043c7	441	}
4d69c345	442
e035c280	443	ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check;
60e52e7c	444	ctx->callback.after_aggregation = damon_reclaim_after_aggregation;
43b0536c	445
1971bd63	446	target = damon_new_target();
43b0536c SP	447	if (!target) {
	448	damon_destroy_ctx(ctx);
	449	return -ENOMEM;
	450	}
	451	damon_add_target(ctx, target);
	452
	453	schedule_delayed_work(&damon_reclaim_timer, 0);
29492829 SP	454
29492829 SP	455	damon_reclaim_initialized = true;
43b0536c SP	456	return 0;
	457	}
	458
	459	module_init(damon_reclaim_init);