Commit | Line | Data |
---|---|---|
c221c0b0 DH |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2016-2019 Intel Corporation. All rights reserved. */ | |
3 | #include <linux/memremap.h> | |
4 | #include <linux/pagemap.h> | |
5 | #include <linux/memory.h> | |
6 | #include <linux/module.h> | |
7 | #include <linux/device.h> | |
8 | #include <linux/pfn_t.h> | |
9 | #include <linux/slab.h> | |
10 | #include <linux/dax.h> | |
11 | #include <linux/fs.h> | |
12 | #include <linux/mm.h> | |
13 | #include <linux/mman.h> | |
7b88bda3 | 14 | #include <linux/memory-tiers.h> |
4eca0ef4 | 15 | #include <linux/memory_hotplug.h> |
c221c0b0 DH |
16 | #include "dax-private.h" |
17 | #include "bus.h" | |
18 | ||
7b88bda3 AK |
19 | /* |
20 | * Default abstract distance assigned to the NUMA node onlined | |
21 | * by DAX/kmem if the low level platform driver didn't initialize | |
22 | * one for this NUMA node. | |
23 | */ | |
24 | #define MEMTIER_DEFAULT_DAX_ADISTANCE (MEMTIER_ADISTANCE_DRAM * 5) | |
25 | ||
8a725e46 DH |
26 | /* Memory resource name used for add_memory_driver_managed(). */ |
27 | static const char *kmem_name; | |
28 | /* Set if any memory will remain added when the driver will be unloaded. */ | |
29 | static bool any_hotremove_failed; | |
30 | ||
60e93dc0 | 31 | static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) |
59bc8d10 | 32 | { |
60e93dc0 DW |
33 | struct dev_dax_range *dax_range = &dev_dax->ranges[i]; |
34 | struct range *range = &dax_range->range; | |
59bc8d10 DW |
35 | |
36 | /* memory-block align the hotplug range */ | |
60e93dc0 DW |
37 | r->start = ALIGN(range->start, memory_block_size_bytes()); |
38 | r->end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()) - 1; | |
39 | if (r->start >= r->end) { | |
40 | r->start = range->start; | |
41 | r->end = range->end; | |
42 | return -ENOSPC; | |
43 | } | |
44 | return 0; | |
59bc8d10 DW |
45 | } |
46 | ||
a455aa72 DW |
47 | struct dax_kmem_data { |
48 | const char *res_name; | |
eedf634a | 49 | int mgid; |
a455aa72 DW |
50 | struct resource *res[]; |
51 | }; | |
52 | ||
6bc2cfdf HY |
53 | static DEFINE_MUTEX(kmem_memory_type_lock); |
54 | static LIST_HEAD(kmem_memory_types); | |
55 | ||
56 | static struct memory_dev_type *kmem_find_alloc_memory_type(int adist) | |
57 | { | |
58 | bool found = false; | |
59 | struct memory_dev_type *mtype; | |
60 | ||
61 | mutex_lock(&kmem_memory_type_lock); | |
62 | list_for_each_entry(mtype, &kmem_memory_types, list) { | |
63 | if (mtype->adistance == adist) { | |
64 | found = true; | |
65 | break; | |
66 | } | |
67 | } | |
68 | if (!found) { | |
69 | mtype = alloc_memory_type(adist); | |
70 | if (!IS_ERR(mtype)) | |
71 | list_add(&mtype->list, &kmem_memory_types); | |
72 | } | |
73 | mutex_unlock(&kmem_memory_type_lock); | |
74 | ||
75 | return mtype; | |
76 | } | |
77 | ||
78 | static void kmem_put_memory_types(void) | |
79 | { | |
80 | struct memory_dev_type *mtype, *mtn; | |
81 | ||
82 | mutex_lock(&kmem_memory_type_lock); | |
83 | list_for_each_entry_safe(mtype, mtn, &kmem_memory_types, list) { | |
84 | list_del(&mtype->list); | |
85 | put_memory_type(mtype); | |
86 | } | |
87 | mutex_unlock(&kmem_memory_type_lock); | |
88 | } | |
89 | ||
f11cf813 | 90 | static int dev_dax_kmem_probe(struct dev_dax *dev_dax) |
c221c0b0 | 91 | { |
f11cf813 | 92 | struct device *dev = &dev_dax->dev; |
eedf634a | 93 | unsigned long total_len = 0; |
a455aa72 | 94 | struct dax_kmem_data *data; |
6bc2cfdf | 95 | struct memory_dev_type *mtype; |
eedf634a | 96 | int i, rc, mapped = 0; |
4eca0ef4 | 97 | mhp_t mhp_flags; |
c221c0b0 | 98 | int numa_node; |
6bc2cfdf | 99 | int adist = MEMTIER_DEFAULT_DAX_ADISTANCE; |
c221c0b0 DH |
100 | |
101 | /* | |
102 | * Ensure good NUMA information for the persistent memory. | |
103 | * Without this check, there is a risk that slow memory | |
104 | * could be mixed in a node with faster memory, causing | |
105 | * unavoidable performance issues. | |
106 | */ | |
107 | numa_node = dev_dax->target_node; | |
108 | if (numa_node < 0) { | |
f5516ec5 DW |
109 | dev_warn(dev, "rejecting DAX region with invalid node: %d\n", |
110 | numa_node); | |
c221c0b0 DH |
111 | return -EINVAL; |
112 | } | |
113 | ||
6bc2cfdf HY |
114 | mt_calc_adistance(numa_node, &adist); |
115 | mtype = kmem_find_alloc_memory_type(adist); | |
116 | if (IS_ERR(mtype)) | |
117 | return PTR_ERR(mtype); | |
118 | ||
eedf634a DH |
119 | for (i = 0; i < dev_dax->nr_range; i++) { |
120 | struct range range; | |
121 | ||
122 | rc = dax_kmem_range(dev_dax, i, &range); | |
123 | if (rc) { | |
124 | dev_info(dev, "mapping%d: %#llx-%#llx too small after alignment\n", | |
125 | i, range.start, range.end); | |
126 | continue; | |
127 | } | |
128 | total_len += range_len(&range); | |
129 | } | |
130 | ||
131 | if (!total_len) { | |
132 | dev_warn(dev, "rejecting DAX region without any memory after alignment\n"); | |
133 | return -EINVAL; | |
134 | } | |
135 | ||
6bc2cfdf | 136 | init_node_memory_type(numa_node, mtype); |
7b88bda3 AK |
137 | |
138 | rc = -ENOMEM; | |
7d18dd75 | 139 | data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL); |
a455aa72 | 140 | if (!data) |
7b88bda3 | 141 | goto err_dax_kmem_data; |
60858c00 | 142 | |
a455aa72 DW |
143 | data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); |
144 | if (!data->res_name) | |
145 | goto err_res_name; | |
146 | ||
46e66dab | 147 | rc = memory_group_register_static(numa_node, PFN_UP(total_len)); |
eedf634a DH |
148 | if (rc < 0) |
149 | goto err_reg_mgid; | |
150 | data->mgid = rc; | |
151 | ||
60e93dc0 DW |
152 | for (i = 0; i < dev_dax->nr_range; i++) { |
153 | struct resource *res; | |
154 | struct range range; | |
c221c0b0 | 155 | |
60e93dc0 | 156 | rc = dax_kmem_range(dev_dax, i, &range); |
eedf634a | 157 | if (rc) |
60e93dc0 | 158 | continue; |
c221c0b0 | 159 | |
60e93dc0 | 160 | /* Region is permanently reserved if hotremove fails. */ |
a455aa72 | 161 | res = request_mem_region(range.start, range_len(&range), data->res_name); |
60e93dc0 DW |
162 | if (!res) { |
163 | dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", | |
164 | i, range.start, range.end); | |
165 | /* | |
166 | * Once some memory has been onlined we can't | |
167 | * assume that it can be un-onlined safely. | |
168 | */ | |
169 | if (mapped) | |
170 | continue; | |
a455aa72 DW |
171 | rc = -EBUSY; |
172 | goto err_request_mem; | |
60e93dc0 | 173 | } |
a455aa72 | 174 | data->res[i] = res; |
60e93dc0 DW |
175 | |
176 | /* | |
177 | * Set flags appropriate for System RAM. Leave ..._BUSY clear | |
178 | * so that add_memory() can add a child resource. Do not | |
179 | * inherit flags from the parent since it may set new flags | |
180 | * unknown to us that will break add_memory() below. | |
181 | */ | |
182 | res->flags = IORESOURCE_SYSTEM_RAM; | |
183 | ||
4eca0ef4 VV |
184 | mhp_flags = MHP_NID_IS_MGID; |
185 | if (dev_dax->memmap_on_memory) | |
186 | mhp_flags |= MHP_MEMMAP_ON_MEMORY; | |
187 | ||
60e93dc0 DW |
188 | /* |
189 | * Ensure that future kexec'd kernels will not treat | |
190 | * this as RAM automatically. | |
191 | */ | |
eedf634a | 192 | rc = add_memory_driver_managed(data->mgid, range.start, |
4eca0ef4 | 193 | range_len(&range), kmem_name, mhp_flags); |
60e93dc0 DW |
194 | |
195 | if (rc) { | |
196 | dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", | |
197 | i, range.start, range.end); | |
e686c325 | 198 | remove_resource(res); |
a455aa72 DW |
199 | kfree(res); |
200 | data->res[i] = NULL; | |
60e93dc0 DW |
201 | if (mapped) |
202 | continue; | |
a455aa72 | 203 | goto err_request_mem; |
60e93dc0 DW |
204 | } |
205 | mapped++; | |
31e4ca92 | 206 | } |
7e6b431a | 207 | |
a455aa72 | 208 | dev_set_drvdata(dev, data); |
c221c0b0 DH |
209 | |
210 | return 0; | |
a455aa72 DW |
211 | |
212 | err_request_mem: | |
eedf634a DH |
213 | memory_group_unregister(data->mgid); |
214 | err_reg_mgid: | |
a455aa72 DW |
215 | kfree(data->res_name); |
216 | err_res_name: | |
217 | kfree(data); | |
7b88bda3 | 218 | err_dax_kmem_data: |
6bc2cfdf | 219 | clear_node_memory_type(numa_node, mtype); |
a455aa72 | 220 | return rc; |
c221c0b0 DH |
221 | } |
222 | ||
9f960da7 | 223 | #ifdef CONFIG_MEMORY_HOTREMOVE |
0d519e0d | 224 | static void dev_dax_kmem_remove(struct dev_dax *dev_dax) |
9f960da7 | 225 | { |
60e93dc0 | 226 | int i, success = 0; |
7b88bda3 | 227 | int node = dev_dax->target_node; |
f11cf813 | 228 | struct device *dev = &dev_dax->dev; |
a455aa72 | 229 | struct dax_kmem_data *data = dev_get_drvdata(dev); |
9f960da7 PT |
230 | |
231 | /* | |
232 | * We have one shot for removing memory, if some memory blocks were not | |
233 | * offline prior to calling this function remove_memory() will fail, and | |
234 | * there is no way to hotremove this memory until reboot because device | |
235 | * unbind will succeed even if we return failure. | |
236 | */ | |
60e93dc0 DW |
237 | for (i = 0; i < dev_dax->nr_range; i++) { |
238 | struct range range; | |
239 | int rc; | |
240 | ||
241 | rc = dax_kmem_range(dev_dax, i, &range); | |
242 | if (rc) | |
243 | continue; | |
244 | ||
e1c158e4 | 245 | rc = remove_memory(range.start, range_len(&range)); |
60e93dc0 | 246 | if (rc == 0) { |
e686c325 | 247 | remove_resource(data->res[i]); |
a455aa72 DW |
248 | kfree(data->res[i]); |
249 | data->res[i] = NULL; | |
60e93dc0 DW |
250 | success++; |
251 | continue; | |
252 | } | |
8a725e46 | 253 | any_hotremove_failed = true; |
60e93dc0 DW |
254 | dev_err(dev, |
255 | "mapping%d: %#llx-%#llx cannot be hotremoved until the next reboot\n", | |
256 | i, range.start, range.end); | |
9f960da7 PT |
257 | } |
258 | ||
60e93dc0 | 259 | if (success >= dev_dax->nr_range) { |
eedf634a | 260 | memory_group_unregister(data->mgid); |
a455aa72 DW |
261 | kfree(data->res_name); |
262 | kfree(data); | |
60e93dc0 | 263 | dev_set_drvdata(dev, NULL); |
7b88bda3 AK |
264 | /* |
265 | * Clear the memtype association on successful unplug. | |
266 | * If not, we have memory blocks left which can be | |
267 | * offlined/onlined later. We need to keep memory_dev_type | |
268 | * for that. This implies this reference will be around | |
269 | * till next reboot. | |
270 | */ | |
6bc2cfdf | 271 | clear_node_memory_type(node, NULL); |
60e93dc0 | 272 | } |
9f960da7 PT |
273 | } |
274 | #else | |
0d519e0d | 275 | static void dev_dax_kmem_remove(struct dev_dax *dev_dax) |
c221c0b0 DH |
276 | { |
277 | /* | |
9f960da7 PT |
278 | * Without hotremove purposely leak the request_mem_region() for the |
279 | * device-dax range and return '0' to ->remove() attempts. The removal | |
280 | * of the device from the driver always succeeds, but the region is | |
281 | * permanently pinned as reserved by the unreleased | |
c221c0b0 DH |
282 | * request_mem_region(). |
283 | */ | |
8a725e46 | 284 | any_hotremove_failed = true; |
c221c0b0 | 285 | } |
9f960da7 | 286 | #endif /* CONFIG_MEMORY_HOTREMOVE */ |
c221c0b0 DH |
287 | |
288 | static struct dax_device_driver device_dax_kmem_driver = { | |
f11cf813 DW |
289 | .probe = dev_dax_kmem_probe, |
290 | .remove = dev_dax_kmem_remove, | |
e9ee9fe3 | 291 | .type = DAXDRV_KMEM_TYPE, |
c221c0b0 DH |
292 | }; |
293 | ||
294 | static int __init dax_kmem_init(void) | |
295 | { | |
8a725e46 DH |
296 | int rc; |
297 | ||
298 | /* Resource name is permanently allocated if any hotremove fails. */ | |
299 | kmem_name = kstrdup_const("System RAM (kmem)", GFP_KERNEL); | |
300 | if (!kmem_name) | |
301 | return -ENOMEM; | |
302 | ||
303 | rc = dax_driver_register(&device_dax_kmem_driver); | |
304 | if (rc) | |
7b88bda3 AK |
305 | goto error_dax_driver; |
306 | ||
307 | return rc; | |
308 | ||
309 | error_dax_driver: | |
6bc2cfdf | 310 | kmem_put_memory_types(); |
7b88bda3 | 311 | kfree_const(kmem_name); |
8a725e46 | 312 | return rc; |
c221c0b0 DH |
313 | } |
314 | ||
315 | static void __exit dax_kmem_exit(void) | |
316 | { | |
317 | dax_driver_unregister(&device_dax_kmem_driver); | |
8a725e46 DH |
318 | if (!any_hotremove_failed) |
319 | kfree_const(kmem_name); | |
6bc2cfdf | 320 | kmem_put_memory_types(); |
c221c0b0 DH |
321 | } |
322 | ||
323 | MODULE_AUTHOR("Intel Corporation"); | |
324 | MODULE_LICENSE("GPL v2"); | |
325 | module_init(dax_kmem_init); | |
326 | module_exit(dax_kmem_exit); | |
327 | MODULE_ALIAS_DAX_DEVICE(0); |