[linux-block.git] / mm / damon / reclaim.c

// SPDX-License-Identifier: GPL-2.0
/*
 * DAMON-based page reclamation
 *
 * Author: SeongJae Park <sj@kernel.org>
 */

#define pr_fmt(fmt) "damon-reclaim: " fmt

#include <linux/damon.h>
#include <linux/module.h>

#include "modules-common.h"

#ifdef MODULE_PARAM_PREFIX
#undef MODULE_PARAM_PREFIX
#endif
#define MODULE_PARAM_PREFIX "damon_reclaim."

/*
 * Enable or disable DAMON_RECLAIM.
 *
 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
 * Setting it as ``N`` disables DAMON_RECLAIM.  Note that DAMON_RECLAIM could
 * do no real monitoring and reclamation due to the watermarks-based activation
 * condition.  Refer to below descriptions for the watermarks parameter for
 * this.
 */
static bool enabled __read_mostly;

/*
 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
 *
 * Input parameters that updated while DAMON_RECLAIM is running are not applied
 * by default.  Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
 * of parametrs except ``enabled`` again.  Once the re-reading is done, this
 * parameter is set as ``N``.  If invalid parameters are found while the
 * re-reading, DAMON_RECLAIM will be disabled.
 */
static bool commit_inputs __read_mostly;
module_param(commit_inputs, bool, 0600);

/*
 * Time threshold for cold memory regions identification in microseconds.
 *
 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM
 * identifies the region as cold, and reclaims.  120 seconds by default.
 */
static unsigned long min_age __read_mostly = 120000000;
module_param(min_age, ulong, 0600);

static struct damos_quota damon_reclaim_quota = {
	/* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */
	.ms = 10,
	.sz = 128 * 1024 * 1024,
	.reset_interval = 1000,
	/* Within the quota, page out older regions first. */
	.weight_sz = 0,
	.weight_nr_accesses = 0,
	.weight_age = 1
};
DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota);

static struct damos_watermarks damon_reclaim_wmarks = {
	.metric = DAMOS_WMARK_FREE_MEM_RATE,
	.interval = 5000000,	/* 5 seconds */
	.high = 500,		/* 50 percent */
	.mid = 400,		/* 40 percent */
	.low = 200,		/* 20 percent */
};
DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks);

static struct damon_attrs damon_reclaim_mon_attrs = {
	.sample_interval = 5000,	/* 5 ms */
	.aggr_interval = 100000,	/* 100 ms */
	.ops_update_interval = 0,
	.min_nr_regions = 10,
	.max_nr_regions = 1000,
};
DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs);

/*
 * Start of the target memory region in physical address.
 *
 * The start physical address of memory region that DAMON_RECLAIM will do work
 * against.  By default, biggest System RAM is used as the region.
 */
static unsigned long monitor_region_start __read_mostly;
module_param(monitor_region_start, ulong, 0600);

/*
 * End of the target memory region in physical address.
 *
 * The end physical address of memory region that DAMON_RECLAIM will do work
 * against.  By default, biggest System RAM is used as the region.
 */
static unsigned long monitor_region_end __read_mostly;
module_param(monitor_region_end, ulong, 0600);

/*
 * PID of the DAMON thread
 *
 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
 * Else, -1.
 */
static int kdamond_pid __read_mostly = -1;
module_param(kdamond_pid, int, 0400);

static struct damos_stat damon_reclaim_stat;
DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
		reclaim_tried_regions, reclaimed_regions, quota_exceeds);

static struct damon_ctx *ctx;
static struct damon_target *target;

static struct damos *damon_reclaim_new_scheme(void)
{
	struct damos_access_pattern pattern = {
		/* Find regions having PAGE_SIZE or larger size */
		.min_sz_region = PAGE_SIZE,
		.max_sz_region = ULONG_MAX,
		/* and not accessed at all */
		.min_nr_accesses = 0,
		.max_nr_accesses = 0,
		/* for min_age or more micro-seconds */
		.min_age_region = min_age /
			damon_reclaim_mon_attrs.aggr_interval,
		.max_age_region = UINT_MAX,
	};

	return damon_new_scheme(
			&pattern,
			/* page out those, as soon as found */
			DAMOS_PAGEOUT,
			/* under the quota. */
			&damon_reclaim_quota,
			/* (De)activate this according to the watermarks. */
			&damon_reclaim_wmarks);
}

static int damon_reclaim_apply_parameters(void)
{
	struct damos *scheme;
	int err = 0;

	err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs);
	if (err)
		return err;

	/* Will be freed by next 'damon_set_schemes()' below */
	scheme = damon_reclaim_new_scheme();
	if (!scheme)
		return -ENOMEM;
	damon_set_schemes(ctx, &scheme, 1);

	return damon_set_region_biggest_system_ram_default(target,
					&monitor_region_start,
					&monitor_region_end);
}

static int damon_reclaim_turn(bool on)
{
	int err;

	if (!on) {
		err = damon_stop(&ctx, 1);
		if (!err)
			kdamond_pid = -1;
		return err;
	}

	err = damon_reclaim_apply_parameters();
	if (err)
		return err;

	err = damon_start(&ctx, 1, true);
	if (err)
		return err;
	kdamond_pid = ctx->kdamond->pid;
	return 0;
}

static int damon_reclaim_enabled_store(const char *val,
		const struct kernel_param *kp)
{
	bool is_enabled = enabled;
	bool enable;
	int err;

	err = strtobool(val, &enable);
	if (err)
		return err;

	if (is_enabled == enable)
		return 0;

	/* Called before init function.  The function will handle this. */
	if (!ctx)
		goto set_param_out;

	err = damon_reclaim_turn(enable);
	if (err)
		return err;

set_param_out:
	enabled = enable;
	return err;
}

static const struct kernel_param_ops enabled_param_ops = {
	.set = damon_reclaim_enabled_store,
	.get = param_get_bool,
};

module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
MODULE_PARM_DESC(enabled,
	"Enable or disable DAMON_RECLAIM (default: disabled)");

static int damon_reclaim_handle_commit_inputs(void)
{
	int err;

	if (!commit_inputs)
		return 0;

	err = damon_reclaim_apply_parameters();
	commit_inputs = false;
	return err;
}

static int damon_reclaim_after_aggregation(struct damon_ctx *c)
{
	struct damos *s;

	/* update the stats parameter */
	damon_for_each_scheme(s, c)
		damon_reclaim_stat = s->stat;

	return damon_reclaim_handle_commit_inputs();
}

static int damon_reclaim_after_wmarks_check(struct damon_ctx *c)
{
	return damon_reclaim_handle_commit_inputs();
}

static int __init damon_reclaim_init(void)
{
	int err = damon_modules_new_paddr_ctx_target(&ctx, &target);

	if (err)
		return err;

	ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check;
	ctx->callback.after_aggregation = damon_reclaim_after_aggregation;

	/* 'enabled' has set before this function, probably via command line */
	if (enabled)
		err = damon_reclaim_turn(true);

	return err;
}

module_init(damon_reclaim_init);
Commit	Line	Data
43b0536c SP	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* DAMON-based page reclamation
	4	*
	5	* Author: SeongJae Park <sj@kernel.org>
	6	*/
	7
	8	#define pr_fmt(fmt) "damon-reclaim: " fmt
	9
	10	#include <linux/damon.h>
43b0536c	11	#include <linux/module.h>
43b0536c	12
fdfc119c SP	13	#include "modules-common.h"
fdfc119c SP	14
43b0536c SP	15	#ifdef MODULE_PARAM_PREFIX
	16	#undef MODULE_PARAM_PREFIX
	17	#endif
	18	#define MODULE_PARAM_PREFIX "damon_reclaim."
	19
	20	/*
	21	* Enable or disable DAMON_RECLAIM.
	22	*
	23	* You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
	24	* Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could
	25	* do no real monitoring and reclamation due to the watermarks-based activation
	26	* condition. Refer to below descriptions for the watermarks parameter for
	27	* this.
	28	*/
	29	static bool enabled __read_mostly;
43b0536c	30
e035c280 SP	31	/*
	32	* Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
	33	*
	34	* Input parameters that updated while DAMON_RECLAIM is running are not applied
	35	* by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
	36	* of parametrs except ``enabled`` again. Once the re-reading is done, this
	37	* parameter is set as ``N``. If invalid parameters are found while the
	38	* re-reading, DAMON_RECLAIM will be disabled.
	39	*/
	40	static bool commit_inputs __read_mostly;
	41	module_param(commit_inputs, bool, 0600);
	42
43b0536c SP	43	/*
	44	* Time threshold for cold memory regions identification in microseconds.
	45	*
	46	* If a memory region is not accessed for this or longer time, DAMON_RECLAIM
	47	* identifies the region as cold, and reclaims. 120 seconds by default.
	48	*/
	49	static unsigned long min_age __read_mostly = 120000000;
	50	module_param(min_age, ulong, 0600);
	51
a9d57c73 SP	52	static struct damos_quota damon_reclaim_quota = {
	53	/* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */
	54	.ms = 10,
	55	.sz = 128 * 1024 * 1024,
	56	.reset_interval = 1000,
	57	/* Within the quota, page out older regions first. */
	58	.weight_sz = 0,
	59	.weight_nr_accesses = 0,
	60	.weight_age = 1
	61	};
	62	DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota);
43b0536c	63
81f8f57f	64	static struct damos_watermarks damon_reclaim_wmarks = {
34f47ea6 SP	65	.metric = DAMOS_WMARK_FREE_MEM_RATE,
	66	.interval = 5000000, /* 5 seconds */
	67	.high = 500, /* 50 percent */
	68	.mid = 400, /* 40 percent */
	69	.low = 200, /* 20 percent */
	70	};
	71	DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks);
43b0536c	72
8c341ae3	73	static struct damon_attrs damon_reclaim_mon_attrs = {
fdfc119c SP	74	.sample_interval = 5000, /* 5 ms */
fdfc119c SP	75	.aggr_interval = 100000, /* 100 ms */
8c341ae3 SP	76	.ops_update_interval = 0,
	77	.min_nr_regions = 10,
	78	.max_nr_regions = 1000,
	79	};
fdfc119c	80	DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs);
43b0536c SP	81
	82	/*
	83	* Start of the target memory region in physical address.
	84	*
	85	* The start physical address of memory region that DAMON_RECLAIM will do work
	86	* against. By default, biggest System RAM is used as the region.
	87	*/
	88	static unsigned long monitor_region_start __read_mostly;
	89	module_param(monitor_region_start, ulong, 0600);
	90
	91	/*
	92	* End of the target memory region in physical address.
	93	*
	94	* The end physical address of memory region that DAMON_RECLAIM will do work
	95	* against. By default, biggest System RAM is used as the region.
	96	*/
	97	static unsigned long monitor_region_end __read_mostly;
	98	module_param(monitor_region_end, ulong, 0600);
	99
	100	/*
	101	* PID of the DAMON thread
	102	*
	103	* If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
	104	* Else, -1.
	105	*/
	106	static int kdamond_pid __read_mostly = -1;
	107	module_param(kdamond_pid, int, 0400);
	108
b71f3ea8 SP	109	static struct damos_stat damon_reclaim_stat;
	110	DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
	111	reclaim_tried_regions, reclaimed_regions, quota_exceeds);
60e52e7c	112
43b0536c SP	113	static struct damon_ctx *ctx;
	114	static struct damon_target *target;
	115
43b0536c SP	116	static struct damos *damon_reclaim_new_scheme(void)
43b0536c SP	117	{
f5a79d7c YD	118	struct damos_access_pattern pattern = {
	119	/* Find regions having PAGE_SIZE or larger size */
	120	.min_sz_region = PAGE_SIZE,
	121	.max_sz_region = ULONG_MAX,
	122	/* and not accessed at all */
	123	.min_nr_accesses = 0,
	124	.max_nr_accesses = 0,
	125	/* for min_age or more micro-seconds */
8c341ae3 SP	126	.min_age_region = min_age /
8c341ae3 SP	127	damon_reclaim_mon_attrs.aggr_interval,
f5a79d7c YD	128	.max_age_region = UINT_MAX,
f5a79d7c YD	129	};
f5a79d7c YD	130
	131	return damon_new_scheme(
	132	&pattern,
43b0536c SP	133	/* page out those, as soon as found */
	134	DAMOS_PAGEOUT,
	135	/* under the quota. */
a9d57c73	136	&damon_reclaim_quota,
43b0536c	137	/* (De)activate this according to the watermarks. */
34f47ea6	138	&damon_reclaim_wmarks);
43b0536c SP	139	}
43b0536c SP	140
e035c280	141	static int damon_reclaim_apply_parameters(void)
43b0536c	142	{
43b0536c	143	struct damos *scheme;
e035c280	144	int err = 0;
43b0536c	145
8c341ae3	146	err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs);
43b0536c SP	147	if (err)
	148	return err;
	149
e035c280 SP	150	/* Will be freed by next 'damon_set_schemes()' below */
	151	scheme = damon_reclaim_new_scheme();
	152	if (!scheme)
	153	return -ENOMEM;
cc713520	154	damon_set_schemes(ctx, &scheme, 1);
e035c280	155
233f0b31 KX	156	return damon_set_region_biggest_system_ram_default(target,
	157	&monitor_region_start,
	158	&monitor_region_end);
e035c280	159	}
43b0536c	160
e035c280 SP	161	static int damon_reclaim_turn(bool on)
	162	{
	163	int err;
	164
	165	if (!on) {
	166	err = damon_stop(&ctx, 1);
	167	if (!err)
	168	kdamond_pid = -1;
	169	return err;
43b0536c	170	}
e035c280 SP	171
e035c280 SP	172	err = damon_reclaim_apply_parameters();
43b0536c	173	if (err)
e035c280	174	return err;
43b0536c	175
8b9b0d33	176	err = damon_start(&ctx, 1, true);
e035c280 SP	177	if (err)
	178	return err;
	179	kdamond_pid = ctx->kdamond->pid;
	180	return 0;
43b0536c SP	181	}
43b0536c SP	182
d79905c7	183	static int damon_reclaim_enabled_store(const char *val,
059342d1 HT	184	const struct kernel_param *kp)
059342d1 HT	185	{
04e98764 SP	186	bool is_enabled = enabled;
	187	bool enable;
	188	int err;
059342d1	189
04e98764 SP	190	err = strtobool(val, &enable);
	191	if (err)
	192	return err;
059342d1	193
04e98764 SP	194	if (is_enabled == enable)
04e98764 SP	195	return 0;
29492829	196
04e98764 SP	197	/* Called before init function. The function will handle this. */
	198	if (!ctx)
	199	goto set_param_out;
	200
	201	err = damon_reclaim_turn(enable);
	202	if (err)
	203	return err;
	204
	205	set_param_out:
	206	enabled = enable;
	207	return err;
059342d1 HT	208	}
	209
	210	static const struct kernel_param_ops enabled_param_ops = {
d79905c7	211	.set = damon_reclaim_enabled_store,
059342d1 HT	212	.get = param_get_bool,
	213	};
	214
	215	module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
	216	MODULE_PARM_DESC(enabled,
	217	"Enable or disable DAMON_RECLAIM (default: disabled)");
	218
f25ab3bd SP	219	static int damon_reclaim_handle_commit_inputs(void)
	220	{
	221	int err;
	222
	223	if (!commit_inputs)
	224	return 0;
	225
	226	err = damon_reclaim_apply_parameters();
	227	commit_inputs = false;
	228	return err;
	229	}
	230
60e52e7c SP	231	static int damon_reclaim_after_aggregation(struct damon_ctx *c)
	232	{
	233	struct damos *s;
	234
	235	/* update the stats parameter */
b71f3ea8 SP	236	damon_for_each_scheme(s, c)
b71f3ea8 SP	237	damon_reclaim_stat = s->stat;
e035c280	238
f25ab3bd	239	return damon_reclaim_handle_commit_inputs();
e035c280 SP	240	}
	241
	242	static int damon_reclaim_after_wmarks_check(struct damon_ctx *c)
	243	{
f25ab3bd	244	return damon_reclaim_handle_commit_inputs();
60e52e7c SP	245	}
60e52e7c SP	246
43b0536c SP	247	static int __init damon_reclaim_init(void)
43b0536c SP	248	{
7ae2c17f	249	int err = damon_modules_new_paddr_ctx_target(&ctx, &target);
43b0536c	250
7ae2c17f SP	251	if (err)
7ae2c17f SP	252	return err;
4d69c345	253
e035c280	254	ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check;
60e52e7c	255	ctx->callback.after_aggregation = damon_reclaim_after_aggregation;
43b0536c	256
04e98764 SP	257	/* 'enabled' has set before this function, probably via command line */
	258	if (enabled)
	259	err = damon_reclaim_turn(true);
29492829	260
04e98764	261	return err;
43b0536c SP	262	}
	263
	264	module_init(damon_reclaim_init);