Commit | Line | Data |
---|---|---|
51cf784c DW |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */ | |
89ec9f2c | 3 | #include <linux/memremap.h> |
ab68f262 DW |
4 | #include <linux/pagemap.h> |
5 | #include <linux/module.h> | |
6 | #include <linux/device.h> | |
7 | #include <linux/pfn_t.h> | |
ba09c01d | 8 | #include <linux/cdev.h> |
ab68f262 DW |
9 | #include <linux/slab.h> |
10 | #include <linux/dax.h> | |
11 | #include <linux/fs.h> | |
12 | #include <linux/mm.h> | |
ef842302 | 13 | #include <linux/mman.h> |
efebc711 | 14 | #include "dax-private.h" |
51cf784c | 15 | #include "bus.h" |
ab68f262 | 16 | |
5f0694b3 | 17 | static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, |
dee41079 DW |
18 | const char *func) |
19 | { | |
5f0694b3 | 20 | struct device *dev = &dev_dax->dev; |
dee41079 DW |
21 | unsigned long mask; |
22 | ||
7b6be844 | 23 | if (!dax_alive(dev_dax->dax_dev)) |
dee41079 DW |
24 | return -ENXIO; |
25 | ||
4cb19355 | 26 | /* prevent private mappings from being established */ |
325896ff | 27 | if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) { |
5a14e91d JM |
28 | dev_info_ratelimited(dev, |
29 | "%s: %s: fail, attempted private mapping\n", | |
dee41079 DW |
30 | current->comm, func); |
31 | return -EINVAL; | |
32 | } | |
33 | ||
33cf94d7 | 34 | mask = dev_dax->align - 1; |
dee41079 | 35 | if (vma->vm_start & mask || vma->vm_end & mask) { |
5a14e91d JM |
36 | dev_info_ratelimited(dev, |
37 | "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n", | |
dee41079 DW |
38 | current->comm, func, vma->vm_start, vma->vm_end, |
39 | mask); | |
40 | return -EINVAL; | |
41 | } | |
42 | ||
dee41079 | 43 | if (!vma_is_dax(vma)) { |
5a14e91d JM |
44 | dev_info_ratelimited(dev, |
45 | "%s: %s: fail, vma is not DAX capable\n", | |
dee41079 DW |
46 | current->comm, func); |
47 | return -EINVAL; | |
48 | } | |
49 | ||
50 | return 0; | |
51 | } | |
52 | ||
efebc711 | 53 | /* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */ |
73616367 | 54 | __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, |
dee41079 DW |
55 | unsigned long size) |
56 | { | |
60e93dc0 DW |
57 | int i; |
58 | ||
59 | for (i = 0; i < dev_dax->nr_range; i++) { | |
60 | struct dev_dax_range *dax_range = &dev_dax->ranges[i]; | |
61 | struct range *range = &dax_range->range; | |
62 | unsigned long long pgoff_end; | |
63 | phys_addr_t phys; | |
64 | ||
65 | pgoff_end = dax_range->pgoff + PHYS_PFN(range_len(range)) - 1; | |
66 | if (pgoff < dax_range->pgoff || pgoff > pgoff_end) | |
67 | continue; | |
68 | phys = PFN_PHYS(pgoff - dax_range->pgoff) + range->start; | |
f5516ec5 | 69 | if (phys + size - 1 <= range->end) |
dee41079 | 70 | return phys; |
60e93dc0 | 71 | break; |
dee41079 | 72 | } |
dee41079 DW |
73 | return -1; |
74 | } | |
75 | ||
a0fb038e JM |
76 | static void dax_set_mapping(struct vm_fault *vmf, pfn_t pfn, |
77 | unsigned long fault_size) | |
78 | { | |
79 | unsigned long i, nr_pages = fault_size / PAGE_SIZE; | |
80 | struct file *filp = vmf->vma->vm_file; | |
14606001 | 81 | struct dev_dax *dev_dax = filp->private_data; |
a0fb038e JM |
82 | pgoff_t pgoff; |
83 | ||
14606001 JM |
84 | /* mapping is only set on the head */ |
85 | if (dev_dax->pgmap->vmemmap_shift) | |
86 | nr_pages = 1; | |
87 | ||
a0fb038e JM |
88 | pgoff = linear_page_index(vmf->vma, |
89 | ALIGN(vmf->address, fault_size)); | |
90 | ||
91 | for (i = 0; i < nr_pages; i++) { | |
92 | struct page *page = pfn_to_page(pfn_t_to_pfn(pfn) + i); | |
93 | ||
14606001 | 94 | page = compound_head(page); |
a0fb038e JM |
95 | if (page->mapping) |
96 | continue; | |
97 | ||
98 | page->mapping = filp->f_mapping; | |
99 | page->index = pgoff + i; | |
100 | } | |
101 | } | |
102 | ||
226ab561 | 103 | static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax, |
6ec228b6 | 104 | struct vm_fault *vmf) |
dee41079 | 105 | { |
5f0694b3 | 106 | struct device *dev = &dev_dax->dev; |
dee41079 | 107 | phys_addr_t phys; |
6ec228b6 | 108 | pfn_t pfn; |
0134ed4f | 109 | unsigned int fault_size = PAGE_SIZE; |
dee41079 | 110 | |
5f0694b3 | 111 | if (check_vma(dev_dax, vmf->vma, __func__)) |
dee41079 DW |
112 | return VM_FAULT_SIGBUS; |
113 | ||
33cf94d7 | 114 | if (dev_dax->align > PAGE_SIZE) { |
6daaca52 | 115 | dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", |
33cf94d7 | 116 | dev_dax->align, fault_size); |
dee41079 DW |
117 | return VM_FAULT_SIGBUS; |
118 | } | |
119 | ||
33cf94d7 | 120 | if (fault_size != dev_dax->align) |
0134ed4f DJ |
121 | return VM_FAULT_SIGBUS; |
122 | ||
73616367 | 123 | phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE); |
dee41079 | 124 | if (phys == -1) { |
6daaca52 | 125 | dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff); |
dee41079 DW |
126 | return VM_FAULT_SIGBUS; |
127 | } | |
128 | ||
6ec228b6 | 129 | pfn = phys_to_pfn_t(phys, PFN_DEV|PFN_MAP); |
dee41079 | 130 | |
6ec228b6 | 131 | dax_set_mapping(vmf, pfn, fault_size); |
0e7325f0 | 132 | |
6ec228b6 | 133 | return vmf_insert_mixed(vmf->vma, vmf->address, pfn); |
dee41079 DW |
134 | } |
135 | ||
226ab561 | 136 | static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax, |
6ec228b6 | 137 | struct vm_fault *vmf) |
dee41079 | 138 | { |
d8a849e1 | 139 | unsigned long pmd_addr = vmf->address & PMD_MASK; |
5f0694b3 | 140 | struct device *dev = &dev_dax->dev; |
dee41079 DW |
141 | phys_addr_t phys; |
142 | pgoff_t pgoff; | |
6ec228b6 | 143 | pfn_t pfn; |
0134ed4f | 144 | unsigned int fault_size = PMD_SIZE; |
dee41079 | 145 | |
5f0694b3 | 146 | if (check_vma(dev_dax, vmf->vma, __func__)) |
dee41079 DW |
147 | return VM_FAULT_SIGBUS; |
148 | ||
33cf94d7 | 149 | if (dev_dax->align > PMD_SIZE) { |
6daaca52 | 150 | dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", |
33cf94d7 | 151 | dev_dax->align, fault_size); |
dee41079 DW |
152 | return VM_FAULT_SIGBUS; |
153 | } | |
154 | ||
33cf94d7 | 155 | if (fault_size < dev_dax->align) |
0134ed4f | 156 | return VM_FAULT_SIGBUS; |
33cf94d7 | 157 | else if (fault_size > dev_dax->align) |
0134ed4f DJ |
158 | return VM_FAULT_FALLBACK; |
159 | ||
160 | /* if we are outside of the VMA */ | |
161 | if (pmd_addr < vmf->vma->vm_start || | |
162 | (pmd_addr + PMD_SIZE) > vmf->vma->vm_end) | |
163 | return VM_FAULT_SIGBUS; | |
164 | ||
f4200391 | 165 | pgoff = linear_page_index(vmf->vma, pmd_addr); |
73616367 | 166 | phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE); |
dee41079 | 167 | if (phys == -1) { |
6daaca52 | 168 | dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); |
dee41079 DW |
169 | return VM_FAULT_SIGBUS; |
170 | } | |
171 | ||
6ec228b6 | 172 | pfn = phys_to_pfn_t(phys, PFN_DEV|PFN_MAP); |
dee41079 | 173 | |
6ec228b6 | 174 | dax_set_mapping(vmf, pfn, fault_size); |
0e7325f0 | 175 | |
6ec228b6 | 176 | return vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); |
dee41079 DW |
177 | } |
178 | ||
9557feee | 179 | #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD |
226ab561 | 180 | static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, |
6ec228b6 | 181 | struct vm_fault *vmf) |
9557feee DJ |
182 | { |
183 | unsigned long pud_addr = vmf->address & PUD_MASK; | |
5f0694b3 | 184 | struct device *dev = &dev_dax->dev; |
9557feee DJ |
185 | phys_addr_t phys; |
186 | pgoff_t pgoff; | |
6ec228b6 | 187 | pfn_t pfn; |
70b085b0 DJ |
188 | unsigned int fault_size = PUD_SIZE; |
189 | ||
9557feee | 190 | |
5f0694b3 | 191 | if (check_vma(dev_dax, vmf->vma, __func__)) |
9557feee DJ |
192 | return VM_FAULT_SIGBUS; |
193 | ||
33cf94d7 | 194 | if (dev_dax->align > PUD_SIZE) { |
6daaca52 | 195 | dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", |
33cf94d7 | 196 | dev_dax->align, fault_size); |
9557feee DJ |
197 | return VM_FAULT_SIGBUS; |
198 | } | |
199 | ||
33cf94d7 | 200 | if (fault_size < dev_dax->align) |
70b085b0 | 201 | return VM_FAULT_SIGBUS; |
33cf94d7 | 202 | else if (fault_size > dev_dax->align) |
70b085b0 DJ |
203 | return VM_FAULT_FALLBACK; |
204 | ||
205 | /* if we are outside of the VMA */ | |
206 | if (pud_addr < vmf->vma->vm_start || | |
207 | (pud_addr + PUD_SIZE) > vmf->vma->vm_end) | |
208 | return VM_FAULT_SIGBUS; | |
209 | ||
9557feee | 210 | pgoff = linear_page_index(vmf->vma, pud_addr); |
73616367 | 211 | phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE); |
9557feee | 212 | if (phys == -1) { |
6daaca52 | 213 | dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); |
9557feee DJ |
214 | return VM_FAULT_SIGBUS; |
215 | } | |
216 | ||
6ec228b6 | 217 | pfn = phys_to_pfn_t(phys, PFN_DEV|PFN_MAP); |
9557feee | 218 | |
6ec228b6 | 219 | dax_set_mapping(vmf, pfn, fault_size); |
0e7325f0 | 220 | |
6ec228b6 | 221 | return vmf_insert_pfn_pud(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); |
9557feee DJ |
222 | } |
223 | #else | |
226ab561 | 224 | static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, |
6ec228b6 | 225 | struct vm_fault *vmf) |
9557feee DJ |
226 | { |
227 | return VM_FAULT_FALLBACK; | |
228 | } | |
229 | #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ | |
230 | ||
226ab561 | 231 | static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf, |
c791ace1 | 232 | enum page_entry_size pe_size) |
dee41079 | 233 | { |
f4200391 | 234 | struct file *filp = vmf->vma->vm_file; |
36bdac1e SJ |
235 | vm_fault_t rc = VM_FAULT_SIGBUS; |
236 | int id; | |
5f0694b3 | 237 | struct dev_dax *dev_dax = filp->private_data; |
dee41079 | 238 | |
6daaca52 DW |
239 | dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm, |
240 | (vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read", | |
76202620 | 241 | vmf->vma->vm_start, vmf->vma->vm_end, pe_size); |
dee41079 | 242 | |
7b6be844 | 243 | id = dax_read_lock(); |
c791ace1 DJ |
244 | switch (pe_size) { |
245 | case PE_SIZE_PTE: | |
6ec228b6 | 246 | rc = __dev_dax_pte_fault(dev_dax, vmf); |
a2d58167 | 247 | break; |
c791ace1 | 248 | case PE_SIZE_PMD: |
6ec228b6 | 249 | rc = __dev_dax_pmd_fault(dev_dax, vmf); |
9557feee | 250 | break; |
c791ace1 | 251 | case PE_SIZE_PUD: |
6ec228b6 | 252 | rc = __dev_dax_pud_fault(dev_dax, vmf); |
a2d58167 DJ |
253 | break; |
254 | default: | |
54eafcc9 | 255 | rc = VM_FAULT_SIGBUS; |
a2d58167 | 256 | } |
2232c638 | 257 | |
7b6be844 | 258 | dax_read_unlock(id); |
dee41079 DW |
259 | |
260 | return rc; | |
261 | } | |
262 | ||
226ab561 | 263 | static vm_fault_t dev_dax_fault(struct vm_fault *vmf) |
c791ace1 | 264 | { |
5f0694b3 | 265 | return dev_dax_huge_fault(vmf, PE_SIZE_PTE); |
c791ace1 DJ |
266 | } |
267 | ||
dd3b614f | 268 | static int dev_dax_may_split(struct vm_area_struct *vma, unsigned long addr) |
9702cffd DW |
269 | { |
270 | struct file *filp = vma->vm_file; | |
271 | struct dev_dax *dev_dax = filp->private_data; | |
9702cffd | 272 | |
33cf94d7 | 273 | if (!IS_ALIGNED(addr, dev_dax->align)) |
9702cffd DW |
274 | return -EINVAL; |
275 | return 0; | |
276 | } | |
277 | ||
c1d53b92 DW |
278 | static unsigned long dev_dax_pagesize(struct vm_area_struct *vma) |
279 | { | |
280 | struct file *filp = vma->vm_file; | |
281 | struct dev_dax *dev_dax = filp->private_data; | |
c1d53b92 | 282 | |
33cf94d7 | 283 | return dev_dax->align; |
c1d53b92 DW |
284 | } |
285 | ||
5f0694b3 DW |
286 | static const struct vm_operations_struct dax_vm_ops = { |
287 | .fault = dev_dax_fault, | |
288 | .huge_fault = dev_dax_huge_fault, | |
dd3b614f | 289 | .may_split = dev_dax_may_split, |
c1d53b92 | 290 | .pagesize = dev_dax_pagesize, |
dee41079 DW |
291 | }; |
292 | ||
af69f51e | 293 | static int dax_mmap(struct file *filp, struct vm_area_struct *vma) |
dee41079 | 294 | { |
5f0694b3 | 295 | struct dev_dax *dev_dax = filp->private_data; |
7b6be844 | 296 | int rc, id; |
dee41079 | 297 | |
6daaca52 | 298 | dev_dbg(&dev_dax->dev, "trace\n"); |
dee41079 | 299 | |
7b6be844 DW |
300 | /* |
301 | * We lock to check dax_dev liveness and will re-check at | |
302 | * fault time. | |
303 | */ | |
304 | id = dax_read_lock(); | |
5f0694b3 | 305 | rc = check_vma(dev_dax, vma, __func__); |
7b6be844 | 306 | dax_read_unlock(id); |
dee41079 DW |
307 | if (rc) |
308 | return rc; | |
309 | ||
5f0694b3 | 310 | vma->vm_ops = &dax_vm_ops; |
1c71222e | 311 | vm_flags_set(vma, VM_HUGEPAGE); |
dee41079 | 312 | return 0; |
043a9255 DW |
313 | } |
314 | ||
315 | /* return an unmapped area aligned to the dax region specified alignment */ | |
af69f51e | 316 | static unsigned long dax_get_unmapped_area(struct file *filp, |
043a9255 DW |
317 | unsigned long addr, unsigned long len, unsigned long pgoff, |
318 | unsigned long flags) | |
319 | { | |
320 | unsigned long off, off_end, off_align, len_align, addr_align, align; | |
5f0694b3 | 321 | struct dev_dax *dev_dax = filp ? filp->private_data : NULL; |
043a9255 | 322 | |
5f0694b3 | 323 | if (!dev_dax || addr) |
043a9255 DW |
324 | goto out; |
325 | ||
33cf94d7 | 326 | align = dev_dax->align; |
043a9255 DW |
327 | off = pgoff << PAGE_SHIFT; |
328 | off_end = off + len; | |
329 | off_align = round_up(off, align); | |
330 | ||
331 | if ((off_end <= off_align) || ((off_end - off_align) < align)) | |
332 | goto out; | |
333 | ||
334 | len_align = len + align; | |
335 | if ((off + len_align) < off) | |
336 | goto out; | |
dee41079 | 337 | |
043a9255 DW |
338 | addr_align = current->mm->get_unmapped_area(filp, addr, len_align, |
339 | pgoff, flags); | |
340 | if (!IS_ERR_VALUE(addr_align)) { | |
341 | addr_align += (off - addr_align) & (align - 1); | |
342 | return addr_align; | |
343 | } | |
344 | out: | |
345 | return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); | |
346 | } | |
347 | ||
41c9b1be | 348 | static const struct address_space_operations dev_dax_aops = { |
46de8b97 | 349 | .dirty_folio = noop_dirty_folio, |
41c9b1be DJ |
350 | }; |
351 | ||
af69f51e | 352 | static int dax_open(struct inode *inode, struct file *filp) |
043a9255 | 353 | { |
7b6be844 DW |
354 | struct dax_device *dax_dev = inode_dax(inode); |
355 | struct inode *__dax_inode = dax_inode(dax_dev); | |
356 | struct dev_dax *dev_dax = dax_get_private(dax_dev); | |
043a9255 | 357 | |
6daaca52 | 358 | dev_dbg(&dev_dax->dev, "trace\n"); |
7b6be844 DW |
359 | inode->i_mapping = __dax_inode->i_mapping; |
360 | inode->i_mapping->host = __dax_inode; | |
41c9b1be | 361 | inode->i_mapping->a_ops = &dev_dax_aops; |
3bc52c45 | 362 | filp->f_mapping = inode->i_mapping; |
5660e13d | 363 | filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping); |
735e4ae5 | 364 | filp->f_sb_err = file_sample_sb_err(filp); |
5f0694b3 | 365 | filp->private_data = dev_dax; |
ebd84d72 | 366 | inode->i_flags = S_DAX; |
043a9255 | 367 | |
043a9255 DW |
368 | return 0; |
369 | } | |
dee41079 | 370 | |
af69f51e | 371 | static int dax_release(struct inode *inode, struct file *filp) |
043a9255 | 372 | { |
5f0694b3 | 373 | struct dev_dax *dev_dax = filp->private_data; |
043a9255 | 374 | |
6daaca52 | 375 | dev_dbg(&dev_dax->dev, "trace\n"); |
043a9255 | 376 | return 0; |
dee41079 DW |
377 | } |
378 | ||
ab68f262 DW |
379 | static const struct file_operations dax_fops = { |
380 | .llseek = noop_llseek, | |
381 | .owner = THIS_MODULE, | |
af69f51e DW |
382 | .open = dax_open, |
383 | .release = dax_release, | |
384 | .get_unmapped_area = dax_get_unmapped_area, | |
385 | .mmap = dax_mmap, | |
ef842302 | 386 | .mmap_supported_flags = MAP_SYNC, |
ab68f262 DW |
387 | }; |
388 | ||
9567da0b | 389 | static void dev_dax_cdev_del(void *cdev) |
043a9255 | 390 | { |
9567da0b DW |
391 | cdev_del(cdev); |
392 | } | |
043a9255 | 393 | |
9567da0b DW |
394 | static void dev_dax_kill(void *dev_dax) |
395 | { | |
396 | kill_dev_dax(dev_dax); | |
ebd84d72 DW |
397 | } |
398 | ||
2d515352 | 399 | static int dev_dax_probe(struct dev_dax *dev_dax) |
043a9255 | 400 | { |
9567da0b | 401 | struct dax_device *dax_dev = dev_dax->dax_dev; |
f11cf813 | 402 | struct device *dev = &dev_dax->dev; |
f5516ec5 | 403 | struct dev_pagemap *pgmap; |
7b6be844 | 404 | struct inode *inode; |
ba09c01d | 405 | struct cdev *cdev; |
89ec9f2c | 406 | void *addr; |
60e93dc0 | 407 | int rc, i; |
89ec9f2c | 408 | |
fc65c4eb JM |
409 | if (static_dev_dax(dev_dax)) { |
410 | if (dev_dax->nr_range > 1) { | |
411 | dev_warn(dev, | |
412 | "static pgmap / multi-range device conflict\n"); | |
413 | return -EINVAL; | |
414 | } | |
60e93dc0 | 415 | |
fc65c4eb JM |
416 | pgmap = dev_dax->pgmap; |
417 | } else { | |
418 | if (dev_dax->pgmap) { | |
419 | dev_warn(dev, | |
420 | "dynamic-dax with pre-populated page map\n"); | |
421 | return -EINVAL; | |
422 | } | |
60e93dc0 | 423 | |
09b80137 JM |
424 | pgmap = devm_kzalloc(dev, |
425 | struct_size(pgmap, ranges, dev_dax->nr_range - 1), | |
426 | GFP_KERNEL); | |
f5516ec5 DW |
427 | if (!pgmap) |
428 | return -ENOMEM; | |
fc65c4eb | 429 | |
60e93dc0 | 430 | pgmap->nr_range = dev_dax->nr_range; |
fc65c4eb JM |
431 | dev_dax->pgmap = pgmap; |
432 | ||
433 | for (i = 0; i < dev_dax->nr_range; i++) { | |
434 | struct range *range = &dev_dax->ranges[i].range; | |
435 | pgmap->ranges[i] = *range; | |
436 | } | |
60e93dc0 DW |
437 | } |
438 | ||
439 | for (i = 0; i < dev_dax->nr_range; i++) { | |
440 | struct range *range = &dev_dax->ranges[i].range; | |
441 | ||
442 | if (!devm_request_mem_region(dev, range->start, | |
443 | range_len(range), dev_name(dev))) { | |
444 | dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n", | |
445 | i, range->start, range->end); | |
446 | return -EBUSY; | |
447 | } | |
f5516ec5 | 448 | } |
60e93dc0 | 449 | |
f5516ec5 | 450 | pgmap->type = MEMORY_DEVICE_GENERIC; |
14606001 JM |
451 | if (dev_dax->align > PAGE_SIZE) |
452 | pgmap->vmemmap_shift = | |
453 | order_base_2(dev_dax->align >> PAGE_SHIFT); | |
f5516ec5 | 454 | addr = devm_memremap_pages(dev, pgmap); |
50f44ee7 | 455 | if (IS_ERR(addr)) |
89ec9f2c | 456 | return PTR_ERR(addr); |
89ec9f2c | 457 | |
7b6be844 DW |
458 | inode = dax_inode(dax_dev); |
459 | cdev = inode->i_cdev; | |
ba09c01d | 460 | cdev_init(cdev, &dax_fops); |
83762cb5 | 461 | cdev->owner = dev->driver->owner; |
9567da0b DW |
462 | cdev_set_parent(cdev, &dev->kobj); |
463 | rc = cdev_add(cdev, dev->devt, 1); | |
d76911ee | 464 | if (rc) |
9567da0b | 465 | return rc; |
d76911ee | 466 | |
9567da0b DW |
467 | rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev); |
468 | if (rc) | |
469 | return rc; | |
043a9255 | 470 | |
9567da0b DW |
471 | run_dax(dax_dev); |
472 | return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax); | |
473 | } | |
043a9255 | 474 | |
d200781e | 475 | static struct dax_device_driver device_dax_driver = { |
f11cf813 | 476 | .probe = dev_dax_probe, |
e9ee9fe3 | 477 | .type = DAXDRV_DEVICE_TYPE, |
9567da0b | 478 | }; |
043a9255 | 479 | |
ab68f262 DW |
480 | static int __init dax_init(void) |
481 | { | |
9567da0b | 482 | return dax_driver_register(&device_dax_driver); |
ab68f262 DW |
483 | } |
484 | ||
485 | static void __exit dax_exit(void) | |
486 | { | |
d200781e | 487 | dax_driver_unregister(&device_dax_driver); |
ab68f262 DW |
488 | } |
489 | ||
490 | MODULE_AUTHOR("Intel Corporation"); | |
491 | MODULE_LICENSE("GPL v2"); | |
9567da0b | 492 | module_init(dax_init); |
ab68f262 | 493 | module_exit(dax_exit); |
9567da0b | 494 | MODULE_ALIAS_DAX_DEVICE(0); |