Commit | Line | Data |
---|---|---|
c221c0b0 DH |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2016-2019 Intel Corporation. All rights reserved. */ | |
3 | #include <linux/memremap.h> | |
4 | #include <linux/pagemap.h> | |
5 | #include <linux/memory.h> | |
6 | #include <linux/module.h> | |
7 | #include <linux/device.h> | |
8 | #include <linux/pfn_t.h> | |
9 | #include <linux/slab.h> | |
10 | #include <linux/dax.h> | |
11 | #include <linux/fs.h> | |
12 | #include <linux/mm.h> | |
13 | #include <linux/mman.h> | |
7b88bda3 | 14 | #include <linux/memory-tiers.h> |
c221c0b0 DH |
15 | #include "dax-private.h" |
16 | #include "bus.h" | |
17 | ||
7b88bda3 AK |
18 | /* |
19 | * Default abstract distance assigned to the NUMA node onlined | |
20 | * by DAX/kmem if the low level platform driver didn't initialize | |
21 | * one for this NUMA node. | |
22 | */ | |
23 | #define MEMTIER_DEFAULT_DAX_ADISTANCE (MEMTIER_ADISTANCE_DRAM * 5) | |
24 | ||
8a725e46 DH |
25 | /* Memory resource name used for add_memory_driver_managed(). */ |
26 | static const char *kmem_name; | |
27 | /* Set if any memory will remain added when the driver will be unloaded. */ | |
28 | static bool any_hotremove_failed; | |
29 | ||
60e93dc0 | 30 | static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) |
59bc8d10 | 31 | { |
60e93dc0 DW |
32 | struct dev_dax_range *dax_range = &dev_dax->ranges[i]; |
33 | struct range *range = &dax_range->range; | |
59bc8d10 DW |
34 | |
35 | /* memory-block align the hotplug range */ | |
60e93dc0 DW |
36 | r->start = ALIGN(range->start, memory_block_size_bytes()); |
37 | r->end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()) - 1; | |
38 | if (r->start >= r->end) { | |
39 | r->start = range->start; | |
40 | r->end = range->end; | |
41 | return -ENOSPC; | |
42 | } | |
43 | return 0; | |
59bc8d10 DW |
44 | } |
45 | ||
a455aa72 DW |
46 | struct dax_kmem_data { |
47 | const char *res_name; | |
eedf634a | 48 | int mgid; |
a455aa72 DW |
49 | struct resource *res[]; |
50 | }; | |
51 | ||
7b88bda3 | 52 | static struct memory_dev_type *dax_slowmem_type; |
f11cf813 | 53 | static int dev_dax_kmem_probe(struct dev_dax *dev_dax) |
c221c0b0 | 54 | { |
f11cf813 | 55 | struct device *dev = &dev_dax->dev; |
eedf634a | 56 | unsigned long total_len = 0; |
a455aa72 | 57 | struct dax_kmem_data *data; |
eedf634a | 58 | int i, rc, mapped = 0; |
c221c0b0 | 59 | int numa_node; |
c221c0b0 DH |
60 | |
61 | /* | |
62 | * Ensure good NUMA information for the persistent memory. | |
63 | * Without this check, there is a risk that slow memory | |
64 | * could be mixed in a node with faster memory, causing | |
65 | * unavoidable performance issues. | |
66 | */ | |
67 | numa_node = dev_dax->target_node; | |
68 | if (numa_node < 0) { | |
f5516ec5 DW |
69 | dev_warn(dev, "rejecting DAX region with invalid node: %d\n", |
70 | numa_node); | |
c221c0b0 DH |
71 | return -EINVAL; |
72 | } | |
73 | ||
eedf634a DH |
74 | for (i = 0; i < dev_dax->nr_range; i++) { |
75 | struct range range; | |
76 | ||
77 | rc = dax_kmem_range(dev_dax, i, &range); | |
78 | if (rc) { | |
79 | dev_info(dev, "mapping%d: %#llx-%#llx too small after alignment\n", | |
80 | i, range.start, range.end); | |
81 | continue; | |
82 | } | |
83 | total_len += range_len(&range); | |
84 | } | |
85 | ||
86 | if (!total_len) { | |
87 | dev_warn(dev, "rejecting DAX region without any memory after alignment\n"); | |
88 | return -EINVAL; | |
89 | } | |
90 | ||
7b88bda3 AK |
91 | init_node_memory_type(numa_node, dax_slowmem_type); |
92 | ||
93 | rc = -ENOMEM; | |
7d18dd75 | 94 | data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL); |
a455aa72 | 95 | if (!data) |
7b88bda3 | 96 | goto err_dax_kmem_data; |
60858c00 | 97 | |
a455aa72 DW |
98 | data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); |
99 | if (!data->res_name) | |
100 | goto err_res_name; | |
101 | ||
eedf634a DH |
102 | rc = memory_group_register_static(numa_node, total_len); |
103 | if (rc < 0) | |
104 | goto err_reg_mgid; | |
105 | data->mgid = rc; | |
106 | ||
60e93dc0 DW |
107 | for (i = 0; i < dev_dax->nr_range; i++) { |
108 | struct resource *res; | |
109 | struct range range; | |
c221c0b0 | 110 | |
60e93dc0 | 111 | rc = dax_kmem_range(dev_dax, i, &range); |
eedf634a | 112 | if (rc) |
60e93dc0 | 113 | continue; |
c221c0b0 | 114 | |
60e93dc0 | 115 | /* Region is permanently reserved if hotremove fails. */ |
a455aa72 | 116 | res = request_mem_region(range.start, range_len(&range), data->res_name); |
60e93dc0 DW |
117 | if (!res) { |
118 | dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", | |
119 | i, range.start, range.end); | |
120 | /* | |
121 | * Once some memory has been onlined we can't | |
122 | * assume that it can be un-onlined safely. | |
123 | */ | |
124 | if (mapped) | |
125 | continue; | |
a455aa72 DW |
126 | rc = -EBUSY; |
127 | goto err_request_mem; | |
60e93dc0 | 128 | } |
a455aa72 | 129 | data->res[i] = res; |
60e93dc0 DW |
130 | |
131 | /* | |
132 | * Set flags appropriate for System RAM. Leave ..._BUSY clear | |
133 | * so that add_memory() can add a child resource. Do not | |
134 | * inherit flags from the parent since it may set new flags | |
135 | * unknown to us that will break add_memory() below. | |
136 | */ | |
137 | res->flags = IORESOURCE_SYSTEM_RAM; | |
138 | ||
139 | /* | |
140 | * Ensure that future kexec'd kernels will not treat | |
141 | * this as RAM automatically. | |
142 | */ | |
eedf634a DH |
143 | rc = add_memory_driver_managed(data->mgid, range.start, |
144 | range_len(&range), kmem_name, MHP_NID_IS_MGID); | |
60e93dc0 DW |
145 | |
146 | if (rc) { | |
147 | dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", | |
148 | i, range.start, range.end); | |
e686c325 | 149 | remove_resource(res); |
a455aa72 DW |
150 | kfree(res); |
151 | data->res[i] = NULL; | |
60e93dc0 DW |
152 | if (mapped) |
153 | continue; | |
a455aa72 | 154 | goto err_request_mem; |
60e93dc0 DW |
155 | } |
156 | mapped++; | |
31e4ca92 | 157 | } |
7e6b431a | 158 | |
a455aa72 | 159 | dev_set_drvdata(dev, data); |
c221c0b0 DH |
160 | |
161 | return 0; | |
a455aa72 DW |
162 | |
163 | err_request_mem: | |
eedf634a DH |
164 | memory_group_unregister(data->mgid); |
165 | err_reg_mgid: | |
a455aa72 DW |
166 | kfree(data->res_name); |
167 | err_res_name: | |
168 | kfree(data); | |
7b88bda3 AK |
169 | err_dax_kmem_data: |
170 | clear_node_memory_type(numa_node, dax_slowmem_type); | |
a455aa72 | 171 | return rc; |
c221c0b0 DH |
172 | } |
173 | ||
9f960da7 | 174 | #ifdef CONFIG_MEMORY_HOTREMOVE |
0d519e0d | 175 | static void dev_dax_kmem_remove(struct dev_dax *dev_dax) |
9f960da7 | 176 | { |
60e93dc0 | 177 | int i, success = 0; |
7b88bda3 | 178 | int node = dev_dax->target_node; |
f11cf813 | 179 | struct device *dev = &dev_dax->dev; |
a455aa72 | 180 | struct dax_kmem_data *data = dev_get_drvdata(dev); |
9f960da7 PT |
181 | |
182 | /* | |
183 | * We have one shot for removing memory, if some memory blocks were not | |
184 | * offline prior to calling this function remove_memory() will fail, and | |
185 | * there is no way to hotremove this memory until reboot because device | |
186 | * unbind will succeed even if we return failure. | |
187 | */ | |
60e93dc0 DW |
188 | for (i = 0; i < dev_dax->nr_range; i++) { |
189 | struct range range; | |
190 | int rc; | |
191 | ||
192 | rc = dax_kmem_range(dev_dax, i, &range); | |
193 | if (rc) | |
194 | continue; | |
195 | ||
e1c158e4 | 196 | rc = remove_memory(range.start, range_len(&range)); |
60e93dc0 | 197 | if (rc == 0) { |
e686c325 | 198 | remove_resource(data->res[i]); |
a455aa72 DW |
199 | kfree(data->res[i]); |
200 | data->res[i] = NULL; | |
60e93dc0 DW |
201 | success++; |
202 | continue; | |
203 | } | |
8a725e46 | 204 | any_hotremove_failed = true; |
60e93dc0 DW |
205 | dev_err(dev, |
206 | "mapping%d: %#llx-%#llx cannot be hotremoved until the next reboot\n", | |
207 | i, range.start, range.end); | |
9f960da7 PT |
208 | } |
209 | ||
60e93dc0 | 210 | if (success >= dev_dax->nr_range) { |
eedf634a | 211 | memory_group_unregister(data->mgid); |
a455aa72 DW |
212 | kfree(data->res_name); |
213 | kfree(data); | |
60e93dc0 | 214 | dev_set_drvdata(dev, NULL); |
7b88bda3 AK |
215 | /* |
216 | * Clear the memtype association on successful unplug. | |
217 | * If not, we have memory blocks left which can be | |
218 | * offlined/onlined later. We need to keep memory_dev_type | |
219 | * for that. This implies this reference will be around | |
220 | * till next reboot. | |
221 | */ | |
222 | clear_node_memory_type(node, dax_slowmem_type); | |
60e93dc0 | 223 | } |
9f960da7 PT |
224 | } |
225 | #else | |
0d519e0d | 226 | static void dev_dax_kmem_remove(struct dev_dax *dev_dax) |
c221c0b0 DH |
227 | { |
228 | /* | |
9f960da7 PT |
229 | * Without hotremove purposely leak the request_mem_region() for the |
230 | * device-dax range and return '0' to ->remove() attempts. The removal | |
231 | * of the device from the driver always succeeds, but the region is | |
232 | * permanently pinned as reserved by the unreleased | |
c221c0b0 DH |
233 | * request_mem_region(). |
234 | */ | |
8a725e46 | 235 | any_hotremove_failed = true; |
c221c0b0 | 236 | } |
9f960da7 | 237 | #endif /* CONFIG_MEMORY_HOTREMOVE */ |
c221c0b0 DH |
238 | |
239 | static struct dax_device_driver device_dax_kmem_driver = { | |
f11cf813 DW |
240 | .probe = dev_dax_kmem_probe, |
241 | .remove = dev_dax_kmem_remove, | |
e9ee9fe3 | 242 | .type = DAXDRV_KMEM_TYPE, |
c221c0b0 DH |
243 | }; |
244 | ||
245 | static int __init dax_kmem_init(void) | |
246 | { | |
8a725e46 DH |
247 | int rc; |
248 | ||
249 | /* Resource name is permanently allocated if any hotremove fails. */ | |
250 | kmem_name = kstrdup_const("System RAM (kmem)", GFP_KERNEL); | |
251 | if (!kmem_name) | |
252 | return -ENOMEM; | |
253 | ||
7b88bda3 AK |
254 | dax_slowmem_type = alloc_memory_type(MEMTIER_DEFAULT_DAX_ADISTANCE); |
255 | if (IS_ERR(dax_slowmem_type)) { | |
256 | rc = PTR_ERR(dax_slowmem_type); | |
257 | goto err_dax_slowmem_type; | |
258 | } | |
259 | ||
8a725e46 DH |
260 | rc = dax_driver_register(&device_dax_kmem_driver); |
261 | if (rc) | |
7b88bda3 AK |
262 | goto error_dax_driver; |
263 | ||
264 | return rc; | |
265 | ||
266 | error_dax_driver: | |
267 | destroy_memory_type(dax_slowmem_type); | |
268 | err_dax_slowmem_type: | |
269 | kfree_const(kmem_name); | |
8a725e46 | 270 | return rc; |
c221c0b0 DH |
271 | } |
272 | ||
273 | static void __exit dax_kmem_exit(void) | |
274 | { | |
275 | dax_driver_unregister(&device_dax_kmem_driver); | |
8a725e46 DH |
276 | if (!any_hotremove_failed) |
277 | kfree_const(kmem_name); | |
7b88bda3 | 278 | destroy_memory_type(dax_slowmem_type); |
c221c0b0 DH |
279 | } |
280 | ||
281 | MODULE_AUTHOR("Intel Corporation"); | |
282 | MODULE_LICENSE("GPL v2"); | |
283 | module_init(dax_kmem_init); | |
284 | module_exit(dax_kmem_exit); | |
285 | MODULE_ALIAS_DAX_DEVICE(0); |