[linux-block.git] / mm / memory-tiers.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/types.h>
#include <linux/nodemask.h>
#include <linux/slab.h>
#include <linux/lockdep.h>
#include <linux/memory-tiers.h>

struct memory_tier {
	/* hierarchy of memory tiers */
	struct list_head list;
	/* list of all memory types part of this tier */
	struct list_head memory_types;
	/*
	 * start value of abstract distance. memory tier maps
	 * an abstract distance  range,
	 * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE
	 */
	int adistance_start;
};

struct memory_dev_type {
	/* list of memory types that are part of same tier as this type */
	struct list_head tier_sibiling;
	/* abstract distance for this specific memory type */
	int adistance;
	/* Nodes of same abstract distance */
	nodemask_t nodes;
	struct memory_tier *memtier;
};

static DEFINE_MUTEX(memory_tier_lock);
static LIST_HEAD(memory_tiers);
static struct memory_dev_type *node_memory_types[MAX_NUMNODES];
/*
 * For now we can have 4 faster memory tiers with smaller adistance
 * than default DRAM tier.
 */
static struct memory_dev_type default_dram_type  = {
	.adistance = MEMTIER_ADISTANCE_DRAM,
	.tier_sibiling = LIST_HEAD_INIT(default_dram_type.tier_sibiling),
};

static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtype)
{
	bool found_slot = false;
	struct memory_tier *memtier, *new_memtier;
	int adistance = memtype->adistance;
	unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE;

	lockdep_assert_held_once(&memory_tier_lock);

	/*
	 * If the memtype is already part of a memory tier,
	 * just return that.
	 */
	if (memtype->memtier)
		return memtype->memtier;

	adistance = round_down(adistance, memtier_adistance_chunk_size);
	list_for_each_entry(memtier, &memory_tiers, list) {
		if (adistance == memtier->adistance_start) {
			memtype->memtier = memtier;
			list_add(&memtype->tier_sibiling, &memtier->memory_types);
			return memtier;
		} else if (adistance < memtier->adistance_start) {
			found_slot = true;
			break;
		}
	}

	new_memtier = kmalloc(sizeof(struct memory_tier), GFP_KERNEL);
	if (!new_memtier)
		return ERR_PTR(-ENOMEM);

	new_memtier->adistance_start = adistance;
	INIT_LIST_HEAD(&new_memtier->list);
	INIT_LIST_HEAD(&new_memtier->memory_types);
	if (found_slot)
		list_add_tail(&new_memtier->list, &memtier->list);
	else
		list_add_tail(&new_memtier->list, &memory_tiers);
	memtype->memtier = new_memtier;
	list_add(&memtype->tier_sibiling, &new_memtier->memory_types);
	return new_memtier;
}

static struct memory_tier *set_node_memory_tier(int node)
{
	struct memory_tier *memtier;
	struct memory_dev_type *memtype;

	lockdep_assert_held_once(&memory_tier_lock);

	if (!node_state(node, N_MEMORY))
		return ERR_PTR(-EINVAL);

	if (!node_memory_types[node])
		node_memory_types[node] = &default_dram_type;

	memtype = node_memory_types[node];
	node_set(node, memtype->nodes);
	memtier = find_create_memory_tier(memtype);
	return memtier;
}

static int __init memory_tier_init(void)
{
	int node;
	struct memory_tier *memtier;

	mutex_lock(&memory_tier_lock);
	/*
	 * Look at all the existing N_MEMORY nodes and add them to
	 * default memory tier or to a tier if we already have memory
	 * types assigned.
	 */
	for_each_node_state(node, N_MEMORY) {
		memtier = set_node_memory_tier(node);
		if (IS_ERR(memtier))
			/*
			 * Continue with memtiers we are able to setup
			 */
			break;
	}
	mutex_unlock(&memory_tier_lock);

	return 0;
}
subsys_initcall(memory_tier_init);
Commit	Line	Data
992bf775 AK	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/types.h>
	3	#include <linux/nodemask.h>
	4	#include <linux/slab.h>
	5	#include <linux/lockdep.h>
	6	#include <linux/memory-tiers.h>
	7
	8	struct memory_tier {
	9	/* hierarchy of memory tiers */
	10	struct list_head list;
	11	/* list of all memory types part of this tier */
	12	struct list_head memory_types;
	13	/*
	14	* start value of abstract distance. memory tier maps
	15	* an abstract distance range,
	16	* adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE
	17	*/
	18	int adistance_start;
	19	};
	20
	21	struct memory_dev_type {
	22	/* list of memory types that are part of same tier as this type */
	23	struct list_head tier_sibiling;
	24	/* abstract distance for this specific memory type */
	25	int adistance;
	26	/* Nodes of same abstract distance */
	27	nodemask_t nodes;
	28	struct memory_tier *memtier;
	29	};
	30
	31	static DEFINE_MUTEX(memory_tier_lock);
	32	static LIST_HEAD(memory_tiers);
	33	static struct memory_dev_type *node_memory_types[MAX_NUMNODES];
	34	/*
	35	* For now we can have 4 faster memory tiers with smaller adistance
	36	* than default DRAM tier.
	37	*/
	38	static struct memory_dev_type default_dram_type = {
	39	.adistance = MEMTIER_ADISTANCE_DRAM,
	40	.tier_sibiling = LIST_HEAD_INIT(default_dram_type.tier_sibiling),
	41	};
	42
	43	static struct memory_tier find_create_memory_tier(struct memory_dev_type memtype)
	44	{
	45	bool found_slot = false;
	46	struct memory_tier memtier, new_memtier;
	47	int adistance = memtype->adistance;
	48	unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE;
	49
	50	lockdep_assert_held_once(&memory_tier_lock);
	51
	52	/*
	53	* If the memtype is already part of a memory tier,
	54	* just return that.
	55	*/
	56	if (memtype->memtier)
	57	return memtype->memtier;
	58
	59	adistance = round_down(adistance, memtier_adistance_chunk_size);
	60	list_for_each_entry(memtier, &memory_tiers, list) {
	61	if (adistance == memtier->adistance_start) {
	62	memtype->memtier = memtier;
	63	list_add(&memtype->tier_sibiling, &memtier->memory_types);
	64	return memtier;
65	} else if (adistance < memtier->adistance_start) {
66	found_slot = true;
67	break;
68	}
69	}
70
71	new_memtier = kmalloc(sizeof(struct memory_tier), GFP_KERNEL);
72	if (!new_memtier)
73	return ERR_PTR(-ENOMEM);
74
75	new_memtier->adistance_start = adistance;
76	INIT_LIST_HEAD(&new_memtier->list);
77	INIT_LIST_HEAD(&new_memtier->memory_types);
78	if (found_slot)
79	list_add_tail(&new_memtier->list, &memtier->list);
80	else
81	list_add_tail(&new_memtier->list, &memory_tiers);
82	memtype->memtier = new_memtier;
83	list_add(&memtype->tier_sibiling, &new_memtier->memory_types);
84	return new_memtier;
85	}
86
87	static struct memory_tier *set_node_memory_tier(int node)
88	{
89	struct memory_tier *memtier;
90	struct memory_dev_type *memtype;
91
92	lockdep_assert_held_once(&memory_tier_lock);
93
94	if (!node_state(node, N_MEMORY))
95	return ERR_PTR(-EINVAL);
96
97	if (!node_memory_types[node])
98	node_memory_types[node] = &default_dram_type;
99
100	memtype = node_memory_types[node];
101	node_set(node, memtype->nodes);
102	memtier = find_create_memory_tier(memtype);
103	return memtier;
104	}
105
106	static int __init memory_tier_init(void)
107	{
108	int node;
109	struct memory_tier *memtier;
110
111	mutex_lock(&memory_tier_lock);
112	/*
113	* Look at all the existing N_MEMORY nodes and add them to
114	* default memory tier or to a tier if we already have memory
115	* types assigned.
116	*/
117	for_each_node_state(node, N_MEMORY) {
118	memtier = set_node_memory_tier(node);
119	if (IS_ERR(memtier))
120	/*
121	* Continue with memtiers we are able to setup
122	*/
123	break;
124	}
125	mutex_unlock(&memory_tier_lock);
126
127	return 0;
128	}
129	subsys_initcall(memory_tier_init);