Commit | Line | Data |
---|---|---|
ab68f262 DW |
1 | /* |
2 | * Copyright(c) 2016 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of version 2 of the GNU General Public License as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | */ | |
13 | #include <linux/pagemap.h> | |
14 | #include <linux/module.h> | |
15 | #include <linux/device.h> | |
3bc52c45 | 16 | #include <linux/mount.h> |
ab68f262 | 17 | #include <linux/pfn_t.h> |
3bc52c45 | 18 | #include <linux/hash.h> |
ba09c01d | 19 | #include <linux/cdev.h> |
ab68f262 DW |
20 | #include <linux/slab.h> |
21 | #include <linux/dax.h> | |
22 | #include <linux/fs.h> | |
23 | #include <linux/mm.h> | |
ccdb07f6 | 24 | #include "dax.h" |
ab68f262 | 25 | |
ba09c01d | 26 | static dev_t dax_devt; |
ab68f262 DW |
27 | static struct class *dax_class; |
28 | static DEFINE_IDA(dax_minor_ida); | |
ba09c01d DW |
29 | static int nr_dax = CONFIG_NR_DEV_DAX; |
30 | module_param(nr_dax, int, S_IRUGO); | |
3bc52c45 DW |
31 | static struct vfsmount *dax_mnt; |
32 | static struct kmem_cache *dax_cache __read_mostly; | |
33 | static struct super_block *dax_superblock __read_mostly; | |
ba09c01d | 34 | MODULE_PARM_DESC(nr_dax, "max number of device-dax instances"); |
ab68f262 DW |
35 | |
36 | /** | |
37 | * struct dax_region - mapping infrastructure for dax devices | |
38 | * @id: kernel-wide unique region for a memory range | |
39 | * @base: linear address corresponding to @res | |
40 | * @kref: to pin while other agents have a need to do lookups | |
41 | * @dev: parent device backing this region | |
42 | * @align: allocation and mapping alignment for child dax devices | |
43 | * @res: physical address range of the region | |
44 | * @pfn_flags: identify whether the pfns are paged back or not | |
45 | */ | |
46 | struct dax_region { | |
47 | int id; | |
48 | struct ida ida; | |
49 | void *base; | |
50 | struct kref kref; | |
51 | struct device *dev; | |
52 | unsigned int align; | |
53 | struct resource res; | |
54 | unsigned long pfn_flags; | |
55 | }; | |
56 | ||
57 | /** | |
58 | * struct dax_dev - subdivision of a dax region | |
59 | * @region - parent region | |
60 | * @dev - device backing the character device | |
ba09c01d | 61 | * @cdev - core chardev data |
dee41079 | 62 | * @alive - !alive + rcu grace period == no new mappings can be established |
ab68f262 DW |
63 | * @id - child id in the region |
64 | * @num_resources - number of physical address extents in this device | |
65 | * @res - array of physical address ranges | |
66 | */ | |
67 | struct dax_dev { | |
68 | struct dax_region *region; | |
3bc52c45 | 69 | struct inode *inode; |
ebd84d72 | 70 | struct device dev; |
ba09c01d | 71 | struct cdev cdev; |
dee41079 | 72 | bool alive; |
ab68f262 DW |
73 | int id; |
74 | int num_resources; | |
75 | struct resource res[0]; | |
76 | }; | |
77 | ||
3bc52c45 DW |
78 | static struct inode *dax_alloc_inode(struct super_block *sb) |
79 | { | |
80 | return kmem_cache_alloc(dax_cache, GFP_KERNEL); | |
81 | } | |
82 | ||
83 | static void dax_i_callback(struct rcu_head *head) | |
84 | { | |
85 | struct inode *inode = container_of(head, struct inode, i_rcu); | |
86 | ||
87 | kmem_cache_free(dax_cache, inode); | |
88 | } | |
89 | ||
90 | static void dax_destroy_inode(struct inode *inode) | |
91 | { | |
92 | call_rcu(&inode->i_rcu, dax_i_callback); | |
93 | } | |
94 | ||
95 | static const struct super_operations dax_sops = { | |
96 | .statfs = simple_statfs, | |
97 | .alloc_inode = dax_alloc_inode, | |
98 | .destroy_inode = dax_destroy_inode, | |
99 | .drop_inode = generic_delete_inode, | |
100 | }; | |
101 | ||
102 | static struct dentry *dax_mount(struct file_system_type *fs_type, | |
103 | int flags, const char *dev_name, void *data) | |
104 | { | |
105 | return mount_pseudo(fs_type, "dax:", &dax_sops, NULL, DAXFS_MAGIC); | |
106 | } | |
107 | ||
108 | static struct file_system_type dax_type = { | |
109 | .name = "dax", | |
110 | .mount = dax_mount, | |
111 | .kill_sb = kill_anon_super, | |
112 | }; | |
113 | ||
114 | static int dax_test(struct inode *inode, void *data) | |
115 | { | |
116 | return inode->i_cdev == data; | |
117 | } | |
118 | ||
119 | static int dax_set(struct inode *inode, void *data) | |
120 | { | |
121 | inode->i_cdev = data; | |
122 | return 0; | |
123 | } | |
124 | ||
125 | static struct inode *dax_inode_get(struct cdev *cdev, dev_t devt) | |
126 | { | |
127 | struct inode *inode; | |
128 | ||
129 | inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31), | |
130 | dax_test, dax_set, cdev); | |
131 | ||
132 | if (!inode) | |
133 | return NULL; | |
134 | ||
135 | if (inode->i_state & I_NEW) { | |
136 | inode->i_mode = S_IFCHR; | |
137 | inode->i_flags = S_DAX; | |
138 | inode->i_rdev = devt; | |
139 | mapping_set_gfp_mask(&inode->i_data, GFP_USER); | |
140 | unlock_new_inode(inode); | |
141 | } | |
142 | return inode; | |
143 | } | |
144 | ||
145 | static void init_once(void *inode) | |
146 | { | |
147 | inode_init_once(inode); | |
148 | } | |
149 | ||
150 | static int dax_inode_init(void) | |
151 | { | |
152 | int rc; | |
153 | ||
154 | dax_cache = kmem_cache_create("dax_cache", sizeof(struct inode), 0, | |
155 | (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| | |
156 | SLAB_MEM_SPREAD|SLAB_ACCOUNT), | |
157 | init_once); | |
158 | if (!dax_cache) | |
159 | return -ENOMEM; | |
160 | ||
161 | rc = register_filesystem(&dax_type); | |
162 | if (rc) | |
163 | goto err_register_fs; | |
164 | ||
165 | dax_mnt = kern_mount(&dax_type); | |
166 | if (IS_ERR(dax_mnt)) { | |
167 | rc = PTR_ERR(dax_mnt); | |
168 | goto err_mount; | |
169 | } | |
170 | dax_superblock = dax_mnt->mnt_sb; | |
171 | ||
172 | return 0; | |
173 | ||
174 | err_mount: | |
175 | unregister_filesystem(&dax_type); | |
176 | err_register_fs: | |
177 | kmem_cache_destroy(dax_cache); | |
178 | ||
179 | return rc; | |
180 | } | |
181 | ||
182 | static void dax_inode_exit(void) | |
183 | { | |
184 | kern_unmount(dax_mnt); | |
185 | unregister_filesystem(&dax_type); | |
186 | kmem_cache_destroy(dax_cache); | |
187 | } | |
188 | ||
ab68f262 DW |
189 | static void dax_region_free(struct kref *kref) |
190 | { | |
191 | struct dax_region *dax_region; | |
192 | ||
193 | dax_region = container_of(kref, struct dax_region, kref); | |
194 | kfree(dax_region); | |
195 | } | |
196 | ||
197 | void dax_region_put(struct dax_region *dax_region) | |
198 | { | |
199 | kref_put(&dax_region->kref, dax_region_free); | |
200 | } | |
201 | EXPORT_SYMBOL_GPL(dax_region_put); | |
202 | ||
ab68f262 DW |
203 | struct dax_region *alloc_dax_region(struct device *parent, int region_id, |
204 | struct resource *res, unsigned int align, void *addr, | |
205 | unsigned long pfn_flags) | |
206 | { | |
207 | struct dax_region *dax_region; | |
208 | ||
209 | dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL); | |
210 | ||
211 | if (!dax_region) | |
212 | return NULL; | |
213 | ||
214 | memcpy(&dax_region->res, res, sizeof(*res)); | |
215 | dax_region->pfn_flags = pfn_flags; | |
216 | kref_init(&dax_region->kref); | |
217 | dax_region->id = region_id; | |
218 | ida_init(&dax_region->ida); | |
219 | dax_region->align = align; | |
220 | dax_region->dev = parent; | |
221 | dax_region->base = addr; | |
222 | ||
223 | return dax_region; | |
224 | } | |
225 | EXPORT_SYMBOL_GPL(alloc_dax_region); | |
226 | ||
ebd84d72 DW |
227 | static struct dax_dev *to_dax_dev(struct device *dev) |
228 | { | |
229 | return container_of(dev, struct dax_dev, dev); | |
230 | } | |
231 | ||
ab68f262 DW |
232 | static ssize_t size_show(struct device *dev, |
233 | struct device_attribute *attr, char *buf) | |
234 | { | |
ebd84d72 | 235 | struct dax_dev *dax_dev = to_dax_dev(dev); |
ab68f262 DW |
236 | unsigned long long size = 0; |
237 | int i; | |
238 | ||
239 | for (i = 0; i < dax_dev->num_resources; i++) | |
240 | size += resource_size(&dax_dev->res[i]); | |
241 | ||
242 | return sprintf(buf, "%llu\n", size); | |
243 | } | |
244 | static DEVICE_ATTR_RO(size); | |
245 | ||
246 | static struct attribute *dax_device_attributes[] = { | |
247 | &dev_attr_size.attr, | |
248 | NULL, | |
249 | }; | |
250 | ||
251 | static const struct attribute_group dax_device_attribute_group = { | |
252 | .attrs = dax_device_attributes, | |
253 | }; | |
254 | ||
255 | static const struct attribute_group *dax_attribute_groups[] = { | |
256 | &dax_device_attribute_group, | |
257 | NULL, | |
258 | }; | |
259 | ||
dee41079 DW |
260 | static int check_vma(struct dax_dev *dax_dev, struct vm_area_struct *vma, |
261 | const char *func) | |
262 | { | |
263 | struct dax_region *dax_region = dax_dev->region; | |
ebd84d72 | 264 | struct device *dev = &dax_dev->dev; |
dee41079 DW |
265 | unsigned long mask; |
266 | ||
267 | if (!dax_dev->alive) | |
268 | return -ENXIO; | |
269 | ||
270 | /* prevent private / writable mappings from being established */ | |
271 | if ((vma->vm_flags & (VM_NORESERVE|VM_SHARED|VM_WRITE)) == VM_WRITE) { | |
272 | dev_info(dev, "%s: %s: fail, attempted private mapping\n", | |
273 | current->comm, func); | |
274 | return -EINVAL; | |
275 | } | |
276 | ||
277 | mask = dax_region->align - 1; | |
278 | if (vma->vm_start & mask || vma->vm_end & mask) { | |
279 | dev_info(dev, "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n", | |
280 | current->comm, func, vma->vm_start, vma->vm_end, | |
281 | mask); | |
282 | return -EINVAL; | |
283 | } | |
284 | ||
285 | if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) == PFN_DEV | |
286 | && (vma->vm_flags & VM_DONTCOPY) == 0) { | |
287 | dev_info(dev, "%s: %s: fail, dax range requires MADV_DONTFORK\n", | |
288 | current->comm, func); | |
289 | return -EINVAL; | |
290 | } | |
291 | ||
292 | if (!vma_is_dax(vma)) { | |
293 | dev_info(dev, "%s: %s: fail, vma is not DAX capable\n", | |
294 | current->comm, func); | |
295 | return -EINVAL; | |
296 | } | |
297 | ||
298 | return 0; | |
299 | } | |
300 | ||
301 | static phys_addr_t pgoff_to_phys(struct dax_dev *dax_dev, pgoff_t pgoff, | |
302 | unsigned long size) | |
303 | { | |
304 | struct resource *res; | |
305 | phys_addr_t phys; | |
306 | int i; | |
307 | ||
308 | for (i = 0; i < dax_dev->num_resources; i++) { | |
309 | res = &dax_dev->res[i]; | |
310 | phys = pgoff * PAGE_SIZE + res->start; | |
311 | if (phys >= res->start && phys <= res->end) | |
312 | break; | |
313 | pgoff -= PHYS_PFN(resource_size(res)); | |
314 | } | |
315 | ||
316 | if (i < dax_dev->num_resources) { | |
317 | res = &dax_dev->res[i]; | |
318 | if (phys + size - 1 <= res->end) | |
319 | return phys; | |
320 | } | |
321 | ||
322 | return -1; | |
323 | } | |
324 | ||
325 | static int __dax_dev_fault(struct dax_dev *dax_dev, struct vm_area_struct *vma, | |
326 | struct vm_fault *vmf) | |
327 | { | |
328 | unsigned long vaddr = (unsigned long) vmf->virtual_address; | |
ebd84d72 | 329 | struct device *dev = &dax_dev->dev; |
dee41079 DW |
330 | struct dax_region *dax_region; |
331 | int rc = VM_FAULT_SIGBUS; | |
332 | phys_addr_t phys; | |
333 | pfn_t pfn; | |
334 | ||
335 | if (check_vma(dax_dev, vma, __func__)) | |
336 | return VM_FAULT_SIGBUS; | |
337 | ||
338 | dax_region = dax_dev->region; | |
339 | if (dax_region->align > PAGE_SIZE) { | |
340 | dev_dbg(dev, "%s: alignment > fault size\n", __func__); | |
341 | return VM_FAULT_SIGBUS; | |
342 | } | |
343 | ||
344 | phys = pgoff_to_phys(dax_dev, vmf->pgoff, PAGE_SIZE); | |
345 | if (phys == -1) { | |
346 | dev_dbg(dev, "%s: phys_to_pgoff(%#lx) failed\n", __func__, | |
347 | vmf->pgoff); | |
348 | return VM_FAULT_SIGBUS; | |
349 | } | |
350 | ||
351 | pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); | |
352 | ||
353 | rc = vm_insert_mixed(vma, vaddr, pfn); | |
354 | ||
355 | if (rc == -ENOMEM) | |
356 | return VM_FAULT_OOM; | |
357 | if (rc < 0 && rc != -EBUSY) | |
358 | return VM_FAULT_SIGBUS; | |
359 | ||
360 | return VM_FAULT_NOPAGE; | |
361 | } | |
362 | ||
363 | static int dax_dev_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |
364 | { | |
365 | int rc; | |
366 | struct file *filp = vma->vm_file; | |
367 | struct dax_dev *dax_dev = filp->private_data; | |
368 | ||
ebd84d72 | 369 | dev_dbg(&dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__, |
dee41079 DW |
370 | current->comm, (vmf->flags & FAULT_FLAG_WRITE) |
371 | ? "write" : "read", vma->vm_start, vma->vm_end); | |
372 | rcu_read_lock(); | |
373 | rc = __dax_dev_fault(dax_dev, vma, vmf); | |
374 | rcu_read_unlock(); | |
375 | ||
376 | return rc; | |
377 | } | |
378 | ||
379 | static int __dax_dev_pmd_fault(struct dax_dev *dax_dev, | |
380 | struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, | |
381 | unsigned int flags) | |
382 | { | |
383 | unsigned long pmd_addr = addr & PMD_MASK; | |
ebd84d72 | 384 | struct device *dev = &dax_dev->dev; |
dee41079 DW |
385 | struct dax_region *dax_region; |
386 | phys_addr_t phys; | |
387 | pgoff_t pgoff; | |
388 | pfn_t pfn; | |
389 | ||
390 | if (check_vma(dax_dev, vma, __func__)) | |
391 | return VM_FAULT_SIGBUS; | |
392 | ||
393 | dax_region = dax_dev->region; | |
394 | if (dax_region->align > PMD_SIZE) { | |
395 | dev_dbg(dev, "%s: alignment > fault size\n", __func__); | |
396 | return VM_FAULT_SIGBUS; | |
397 | } | |
398 | ||
399 | /* dax pmd mappings require pfn_t_devmap() */ | |
400 | if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) { | |
401 | dev_dbg(dev, "%s: alignment > fault size\n", __func__); | |
402 | return VM_FAULT_SIGBUS; | |
403 | } | |
404 | ||
405 | pgoff = linear_page_index(vma, pmd_addr); | |
406 | phys = pgoff_to_phys(dax_dev, pgoff, PAGE_SIZE); | |
407 | if (phys == -1) { | |
408 | dev_dbg(dev, "%s: phys_to_pgoff(%#lx) failed\n", __func__, | |
409 | pgoff); | |
410 | return VM_FAULT_SIGBUS; | |
411 | } | |
412 | ||
413 | pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); | |
414 | ||
415 | return vmf_insert_pfn_pmd(vma, addr, pmd, pfn, | |
416 | flags & FAULT_FLAG_WRITE); | |
417 | } | |
418 | ||
419 | static int dax_dev_pmd_fault(struct vm_area_struct *vma, unsigned long addr, | |
420 | pmd_t *pmd, unsigned int flags) | |
421 | { | |
422 | int rc; | |
423 | struct file *filp = vma->vm_file; | |
424 | struct dax_dev *dax_dev = filp->private_data; | |
425 | ||
ebd84d72 | 426 | dev_dbg(&dax_dev->dev, "%s: %s: %s (%#lx - %#lx)\n", __func__, |
dee41079 DW |
427 | current->comm, (flags & FAULT_FLAG_WRITE) |
428 | ? "write" : "read", vma->vm_start, vma->vm_end); | |
429 | ||
430 | rcu_read_lock(); | |
431 | rc = __dax_dev_pmd_fault(dax_dev, vma, addr, pmd, flags); | |
432 | rcu_read_unlock(); | |
433 | ||
434 | return rc; | |
435 | } | |
436 | ||
dee41079 DW |
437 | static const struct vm_operations_struct dax_dev_vm_ops = { |
438 | .fault = dax_dev_fault, | |
439 | .pmd_fault = dax_dev_pmd_fault, | |
dee41079 DW |
440 | }; |
441 | ||
af69f51e | 442 | static int dax_mmap(struct file *filp, struct vm_area_struct *vma) |
dee41079 DW |
443 | { |
444 | struct dax_dev *dax_dev = filp->private_data; | |
445 | int rc; | |
446 | ||
ebd84d72 | 447 | dev_dbg(&dax_dev->dev, "%s\n", __func__); |
dee41079 DW |
448 | |
449 | rc = check_vma(dax_dev, vma, __func__); | |
450 | if (rc) | |
451 | return rc; | |
452 | ||
dee41079 DW |
453 | vma->vm_ops = &dax_dev_vm_ops; |
454 | vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE; | |
455 | return 0; | |
043a9255 DW |
456 | } |
457 | ||
458 | /* return an unmapped area aligned to the dax region specified alignment */ | |
af69f51e | 459 | static unsigned long dax_get_unmapped_area(struct file *filp, |
043a9255 DW |
460 | unsigned long addr, unsigned long len, unsigned long pgoff, |
461 | unsigned long flags) | |
462 | { | |
463 | unsigned long off, off_end, off_align, len_align, addr_align, align; | |
464 | struct dax_dev *dax_dev = filp ? filp->private_data : NULL; | |
465 | struct dax_region *dax_region; | |
466 | ||
467 | if (!dax_dev || addr) | |
468 | goto out; | |
469 | ||
470 | dax_region = dax_dev->region; | |
471 | align = dax_region->align; | |
472 | off = pgoff << PAGE_SHIFT; | |
473 | off_end = off + len; | |
474 | off_align = round_up(off, align); | |
475 | ||
476 | if ((off_end <= off_align) || ((off_end - off_align) < align)) | |
477 | goto out; | |
478 | ||
479 | len_align = len + align; | |
480 | if ((off + len_align) < off) | |
481 | goto out; | |
482 | ||
483 | addr_align = current->mm->get_unmapped_area(filp, addr, len_align, | |
484 | pgoff, flags); | |
485 | if (!IS_ERR_VALUE(addr_align)) { | |
486 | addr_align += (off - addr_align) & (align - 1); | |
487 | return addr_align; | |
488 | } | |
489 | out: | |
490 | return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); | |
491 | } | |
492 | ||
af69f51e | 493 | static int dax_open(struct inode *inode, struct file *filp) |
043a9255 | 494 | { |
ba09c01d | 495 | struct dax_dev *dax_dev; |
043a9255 | 496 | |
ba09c01d DW |
497 | dax_dev = container_of(inode->i_cdev, struct dax_dev, cdev); |
498 | dev_dbg(&dax_dev->dev, "%s\n", __func__); | |
3bc52c45 DW |
499 | inode->i_mapping = dax_dev->inode->i_mapping; |
500 | inode->i_mapping->host = dax_dev->inode; | |
501 | filp->f_mapping = inode->i_mapping; | |
ebd84d72 DW |
502 | filp->private_data = dax_dev; |
503 | inode->i_flags = S_DAX; | |
043a9255 | 504 | |
043a9255 DW |
505 | return 0; |
506 | } | |
dee41079 | 507 | |
af69f51e | 508 | static int dax_release(struct inode *inode, struct file *filp) |
043a9255 DW |
509 | { |
510 | struct dax_dev *dax_dev = filp->private_data; | |
043a9255 | 511 | |
ba09c01d | 512 | dev_dbg(&dax_dev->dev, "%s\n", __func__); |
043a9255 | 513 | return 0; |
dee41079 DW |
514 | } |
515 | ||
ab68f262 DW |
516 | static const struct file_operations dax_fops = { |
517 | .llseek = noop_llseek, | |
518 | .owner = THIS_MODULE, | |
af69f51e DW |
519 | .open = dax_open, |
520 | .release = dax_release, | |
521 | .get_unmapped_area = dax_get_unmapped_area, | |
522 | .mmap = dax_mmap, | |
ab68f262 DW |
523 | }; |
524 | ||
ebd84d72 | 525 | static void dax_dev_release(struct device *dev) |
043a9255 | 526 | { |
ebd84d72 | 527 | struct dax_dev *dax_dev = to_dax_dev(dev); |
043a9255 DW |
528 | struct dax_region *dax_region = dax_dev->region; |
529 | ||
ebd84d72 DW |
530 | ida_simple_remove(&dax_region->ida, dax_dev->id); |
531 | ida_simple_remove(&dax_minor_ida, MINOR(dev->devt)); | |
532 | dax_region_put(dax_region); | |
3bc52c45 | 533 | iput(dax_dev->inode); |
ebd84d72 DW |
534 | kfree(dax_dev); |
535 | } | |
536 | ||
537 | static void unregister_dax_dev(void *dev) | |
538 | { | |
539 | struct dax_dev *dax_dev = to_dax_dev(dev); | |
ba09c01d | 540 | struct cdev *cdev = &dax_dev->cdev; |
ebd84d72 | 541 | |
043a9255 DW |
542 | dev_dbg(dev, "%s\n", __func__); |
543 | ||
544 | /* | |
545 | * Note, rcu is not protecting the liveness of dax_dev, rcu is | |
546 | * ensuring that any fault handlers that might have seen | |
547 | * dax_dev->alive == true, have completed. Any fault handlers | |
548 | * that start after synchronize_rcu() has started will abort | |
549 | * upon seeing dax_dev->alive == false. | |
550 | */ | |
551 | dax_dev->alive = false; | |
552 | synchronize_rcu(); | |
9dc1e492 | 553 | unmap_mapping_range(dax_dev->inode->i_mapping, 0, 0, 1); |
ba09c01d | 554 | cdev_del(cdev); |
043a9255 | 555 | device_unregister(dev); |
043a9255 DW |
556 | } |
557 | ||
558 | int devm_create_dax_dev(struct dax_region *dax_region, struct resource *res, | |
559 | int count) | |
560 | { | |
561 | struct device *parent = dax_region->dev; | |
562 | struct dax_dev *dax_dev; | |
563 | struct device *dev; | |
ba09c01d | 564 | struct cdev *cdev; |
043a9255 DW |
565 | int rc, minor; |
566 | dev_t dev_t; | |
567 | ||
568 | dax_dev = kzalloc(sizeof(*dax_dev) + sizeof(*res) * count, GFP_KERNEL); | |
569 | if (!dax_dev) | |
570 | return -ENOMEM; | |
043a9255 DW |
571 | |
572 | dax_dev->id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL); | |
573 | if (dax_dev->id < 0) { | |
574 | rc = dax_dev->id; | |
575 | goto err_id; | |
576 | } | |
577 | ||
578 | minor = ida_simple_get(&dax_minor_ida, 0, 0, GFP_KERNEL); | |
579 | if (minor < 0) { | |
580 | rc = minor; | |
581 | goto err_minor; | |
582 | } | |
583 | ||
3bc52c45 DW |
584 | dax_dev->inode = dax_inode_get(&dax_dev->cdev, dev_t); |
585 | if (!dax_dev->inode) { | |
586 | rc = -ENOMEM; | |
587 | goto err_inode; | |
588 | } | |
589 | ||
ba09c01d DW |
590 | /* device_initialize() so cdev can reference kobj parent */ |
591 | dev_t = MKDEV(MAJOR(dax_devt), minor); | |
ebd84d72 DW |
592 | dev = &dax_dev->dev; |
593 | device_initialize(dev); | |
ba09c01d DW |
594 | |
595 | cdev = &dax_dev->cdev; | |
596 | cdev_init(cdev, &dax_fops); | |
597 | cdev->owner = parent->driver->owner; | |
598 | cdev->kobj.parent = &dev->kobj; | |
599 | rc = cdev_add(&dax_dev->cdev, dev_t, 1); | |
600 | if (rc) | |
601 | goto err_cdev; | |
602 | ||
603 | /* from here on we're committed to teardown via dax_dev_release() */ | |
604 | memcpy(dax_dev->res, res, sizeof(*res) * count); | |
605 | dax_dev->num_resources = count; | |
606 | dax_dev->alive = true; | |
607 | dax_dev->region = dax_region; | |
608 | kref_get(&dax_region->kref); | |
609 | ||
ebd84d72 DW |
610 | dev->devt = dev_t; |
611 | dev->class = dax_class; | |
612 | dev->parent = parent; | |
613 | dev->groups = dax_attribute_groups; | |
614 | dev->release = dax_dev_release; | |
615 | dev_set_name(dev, "dax%d.%d", dax_region->id, dax_dev->id); | |
616 | rc = device_add(dev); | |
617 | if (rc) { | |
618 | put_device(dev); | |
043a9255 | 619 | return rc; |
ebd84d72 | 620 | } |
043a9255 | 621 | |
ebd84d72 | 622 | return devm_add_action_or_reset(dax_region->dev, unregister_dax_dev, dev); |
043a9255 | 623 | |
ba09c01d | 624 | err_cdev: |
3bc52c45 DW |
625 | iput(dax_dev->inode); |
626 | err_inode: | |
ba09c01d | 627 | ida_simple_remove(&dax_minor_ida, minor); |
043a9255 DW |
628 | err_minor: |
629 | ida_simple_remove(&dax_region->ida, dax_dev->id); | |
630 | err_id: | |
ebd84d72 | 631 | kfree(dax_dev); |
043a9255 DW |
632 | |
633 | return rc; | |
634 | } | |
635 | EXPORT_SYMBOL_GPL(devm_create_dax_dev); | |
636 | ||
ab68f262 DW |
637 | static int __init dax_init(void) |
638 | { | |
639 | int rc; | |
640 | ||
3bc52c45 DW |
641 | rc = dax_inode_init(); |
642 | if (rc) | |
643 | return rc; | |
644 | ||
ba09c01d DW |
645 | nr_dax = max(nr_dax, 256); |
646 | rc = alloc_chrdev_region(&dax_devt, 0, nr_dax, "dax"); | |
647 | if (rc) | |
3bc52c45 | 648 | goto err_chrdev; |
ab68f262 DW |
649 | |
650 | dax_class = class_create(THIS_MODULE, "dax"); | |
3bc52c45 DW |
651 | if (IS_ERR(dax_class)) { |
652 | rc = PTR_ERR(dax_class); | |
653 | goto err_class; | |
654 | } | |
ab68f262 | 655 | |
3bc52c45 DW |
656 | return 0; |
657 | ||
658 | err_class: | |
659 | unregister_chrdev_region(dax_devt, nr_dax); | |
660 | err_chrdev: | |
661 | dax_inode_exit(); | |
662 | return rc; | |
ab68f262 DW |
663 | } |
664 | ||
665 | static void __exit dax_exit(void) | |
666 | { | |
667 | class_destroy(dax_class); | |
ba09c01d | 668 | unregister_chrdev_region(dax_devt, nr_dax); |
ab68f262 | 669 | ida_destroy(&dax_minor_ida); |
3bc52c45 | 670 | dax_inode_exit(); |
ab68f262 DW |
671 | } |
672 | ||
673 | MODULE_AUTHOR("Intel Corporation"); | |
674 | MODULE_LICENSE("GPL v2"); | |
675 | subsys_initcall(dax_init); | |
676 | module_exit(dax_exit); |