Commit | Line | Data |
---|---|---|
51cf784c DW |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */ | |
89ec9f2c | 3 | #include <linux/memremap.h> |
ab68f262 DW |
4 | #include <linux/pagemap.h> |
5 | #include <linux/module.h> | |
6 | #include <linux/device.h> | |
7 | #include <linux/pfn_t.h> | |
ba09c01d | 8 | #include <linux/cdev.h> |
ab68f262 DW |
9 | #include <linux/slab.h> |
10 | #include <linux/dax.h> | |
11 | #include <linux/fs.h> | |
12 | #include <linux/mm.h> | |
ef842302 | 13 | #include <linux/mman.h> |
efebc711 | 14 | #include "dax-private.h" |
51cf784c | 15 | #include "bus.h" |
ab68f262 | 16 | |
89ec9f2c DW |
17 | static struct dev_dax *ref_to_dev_dax(struct percpu_ref *ref) |
18 | { | |
19 | return container_of(ref, struct dev_dax, ref); | |
20 | } | |
21 | ||
22 | static void dev_dax_percpu_release(struct percpu_ref *ref) | |
23 | { | |
24 | struct dev_dax *dev_dax = ref_to_dev_dax(ref); | |
25 | ||
26 | dev_dbg(&dev_dax->dev, "%s\n", __func__); | |
27 | complete(&dev_dax->cmp); | |
28 | } | |
29 | ||
30 | static void dev_dax_percpu_exit(void *data) | |
31 | { | |
32 | struct percpu_ref *ref = data; | |
33 | struct dev_dax *dev_dax = ref_to_dev_dax(ref); | |
34 | ||
35 | dev_dbg(&dev_dax->dev, "%s\n", __func__); | |
36 | wait_for_completion(&dev_dax->cmp); | |
37 | percpu_ref_exit(ref); | |
38 | } | |
39 | ||
40 | static void dev_dax_percpu_kill(struct percpu_ref *data) | |
41 | { | |
42 | struct percpu_ref *ref = data; | |
43 | struct dev_dax *dev_dax = ref_to_dev_dax(ref); | |
44 | ||
45 | dev_dbg(&dev_dax->dev, "%s\n", __func__); | |
46 | percpu_ref_kill(ref); | |
47 | } | |
48 | ||
5f0694b3 | 49 | static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, |
dee41079 DW |
50 | const char *func) |
51 | { | |
5f0694b3 DW |
52 | struct dax_region *dax_region = dev_dax->region; |
53 | struct device *dev = &dev_dax->dev; | |
dee41079 DW |
54 | unsigned long mask; |
55 | ||
7b6be844 | 56 | if (!dax_alive(dev_dax->dax_dev)) |
dee41079 DW |
57 | return -ENXIO; |
58 | ||
4cb19355 | 59 | /* prevent private mappings from being established */ |
325896ff | 60 | if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) { |
5a14e91d JM |
61 | dev_info_ratelimited(dev, |
62 | "%s: %s: fail, attempted private mapping\n", | |
dee41079 DW |
63 | current->comm, func); |
64 | return -EINVAL; | |
65 | } | |
66 | ||
67 | mask = dax_region->align - 1; | |
68 | if (vma->vm_start & mask || vma->vm_end & mask) { | |
5a14e91d JM |
69 | dev_info_ratelimited(dev, |
70 | "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n", | |
dee41079 DW |
71 | current->comm, func, vma->vm_start, vma->vm_end, |
72 | mask); | |
73 | return -EINVAL; | |
74 | } | |
75 | ||
76 | if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) == PFN_DEV | |
77 | && (vma->vm_flags & VM_DONTCOPY) == 0) { | |
5a14e91d JM |
78 | dev_info_ratelimited(dev, |
79 | "%s: %s: fail, dax range requires MADV_DONTFORK\n", | |
dee41079 DW |
80 | current->comm, func); |
81 | return -EINVAL; | |
82 | } | |
83 | ||
84 | if (!vma_is_dax(vma)) { | |
5a14e91d JM |
85 | dev_info_ratelimited(dev, |
86 | "%s: %s: fail, vma is not DAX capable\n", | |
dee41079 DW |
87 | current->comm, func); |
88 | return -EINVAL; | |
89 | } | |
90 | ||
91 | return 0; | |
92 | } | |
93 | ||
efebc711 | 94 | /* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */ |
73616367 | 95 | __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, |
dee41079 DW |
96 | unsigned long size) |
97 | { | |
753a0850 DW |
98 | struct resource *res = &dev_dax->region->res; |
99 | phys_addr_t phys; | |
dee41079 | 100 | |
753a0850 DW |
101 | phys = pgoff * PAGE_SIZE + res->start; |
102 | if (phys >= res->start && phys <= res->end) { | |
dee41079 DW |
103 | if (phys + size - 1 <= res->end) |
104 | return phys; | |
105 | } | |
106 | ||
107 | return -1; | |
108 | } | |
109 | ||
226ab561 | 110 | static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax, |
2232c638 | 111 | struct vm_fault *vmf, pfn_t *pfn) |
dee41079 | 112 | { |
5f0694b3 | 113 | struct device *dev = &dev_dax->dev; |
dee41079 | 114 | struct dax_region *dax_region; |
dee41079 | 115 | phys_addr_t phys; |
0134ed4f | 116 | unsigned int fault_size = PAGE_SIZE; |
dee41079 | 117 | |
5f0694b3 | 118 | if (check_vma(dev_dax, vmf->vma, __func__)) |
dee41079 DW |
119 | return VM_FAULT_SIGBUS; |
120 | ||
5f0694b3 | 121 | dax_region = dev_dax->region; |
dee41079 | 122 | if (dax_region->align > PAGE_SIZE) { |
6daaca52 DW |
123 | dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", |
124 | dax_region->align, fault_size); | |
dee41079 DW |
125 | return VM_FAULT_SIGBUS; |
126 | } | |
127 | ||
0134ed4f DJ |
128 | if (fault_size != dax_region->align) |
129 | return VM_FAULT_SIGBUS; | |
130 | ||
73616367 | 131 | phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE); |
dee41079 | 132 | if (phys == -1) { |
6daaca52 | 133 | dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff); |
dee41079 DW |
134 | return VM_FAULT_SIGBUS; |
135 | } | |
136 | ||
2232c638 | 137 | *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); |
dee41079 | 138 | |
2232c638 | 139 | return vmf_insert_mixed(vmf->vma, vmf->address, *pfn); |
dee41079 DW |
140 | } |
141 | ||
226ab561 | 142 | static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax, |
2232c638 | 143 | struct vm_fault *vmf, pfn_t *pfn) |
dee41079 | 144 | { |
d8a849e1 | 145 | unsigned long pmd_addr = vmf->address & PMD_MASK; |
5f0694b3 | 146 | struct device *dev = &dev_dax->dev; |
dee41079 DW |
147 | struct dax_region *dax_region; |
148 | phys_addr_t phys; | |
149 | pgoff_t pgoff; | |
0134ed4f | 150 | unsigned int fault_size = PMD_SIZE; |
dee41079 | 151 | |
5f0694b3 | 152 | if (check_vma(dev_dax, vmf->vma, __func__)) |
dee41079 DW |
153 | return VM_FAULT_SIGBUS; |
154 | ||
5f0694b3 | 155 | dax_region = dev_dax->region; |
dee41079 | 156 | if (dax_region->align > PMD_SIZE) { |
6daaca52 DW |
157 | dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", |
158 | dax_region->align, fault_size); | |
dee41079 DW |
159 | return VM_FAULT_SIGBUS; |
160 | } | |
161 | ||
162 | /* dax pmd mappings require pfn_t_devmap() */ | |
163 | if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) { | |
6daaca52 | 164 | dev_dbg(dev, "region lacks devmap flags\n"); |
dee41079 DW |
165 | return VM_FAULT_SIGBUS; |
166 | } | |
167 | ||
0134ed4f DJ |
168 | if (fault_size < dax_region->align) |
169 | return VM_FAULT_SIGBUS; | |
170 | else if (fault_size > dax_region->align) | |
171 | return VM_FAULT_FALLBACK; | |
172 | ||
173 | /* if we are outside of the VMA */ | |
174 | if (pmd_addr < vmf->vma->vm_start || | |
175 | (pmd_addr + PMD_SIZE) > vmf->vma->vm_end) | |
176 | return VM_FAULT_SIGBUS; | |
177 | ||
f4200391 | 178 | pgoff = linear_page_index(vmf->vma, pmd_addr); |
73616367 | 179 | phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE); |
dee41079 | 180 | if (phys == -1) { |
6daaca52 | 181 | dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); |
dee41079 DW |
182 | return VM_FAULT_SIGBUS; |
183 | } | |
184 | ||
2232c638 | 185 | *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); |
dee41079 | 186 | |
fce86ff5 | 187 | return vmf_insert_pfn_pmd(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE); |
dee41079 DW |
188 | } |
189 | ||
9557feee | 190 | #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD |
226ab561 | 191 | static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, |
2232c638 | 192 | struct vm_fault *vmf, pfn_t *pfn) |
9557feee DJ |
193 | { |
194 | unsigned long pud_addr = vmf->address & PUD_MASK; | |
5f0694b3 | 195 | struct device *dev = &dev_dax->dev; |
9557feee DJ |
196 | struct dax_region *dax_region; |
197 | phys_addr_t phys; | |
198 | pgoff_t pgoff; | |
70b085b0 DJ |
199 | unsigned int fault_size = PUD_SIZE; |
200 | ||
9557feee | 201 | |
5f0694b3 | 202 | if (check_vma(dev_dax, vmf->vma, __func__)) |
9557feee DJ |
203 | return VM_FAULT_SIGBUS; |
204 | ||
5f0694b3 | 205 | dax_region = dev_dax->region; |
9557feee | 206 | if (dax_region->align > PUD_SIZE) { |
6daaca52 DW |
207 | dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", |
208 | dax_region->align, fault_size); | |
9557feee DJ |
209 | return VM_FAULT_SIGBUS; |
210 | } | |
211 | ||
212 | /* dax pud mappings require pfn_t_devmap() */ | |
213 | if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) { | |
6daaca52 | 214 | dev_dbg(dev, "region lacks devmap flags\n"); |
9557feee DJ |
215 | return VM_FAULT_SIGBUS; |
216 | } | |
217 | ||
70b085b0 DJ |
218 | if (fault_size < dax_region->align) |
219 | return VM_FAULT_SIGBUS; | |
220 | else if (fault_size > dax_region->align) | |
221 | return VM_FAULT_FALLBACK; | |
222 | ||
223 | /* if we are outside of the VMA */ | |
224 | if (pud_addr < vmf->vma->vm_start || | |
225 | (pud_addr + PUD_SIZE) > vmf->vma->vm_end) | |
226 | return VM_FAULT_SIGBUS; | |
227 | ||
9557feee | 228 | pgoff = linear_page_index(vmf->vma, pud_addr); |
73616367 | 229 | phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE); |
9557feee | 230 | if (phys == -1) { |
6daaca52 | 231 | dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); |
9557feee DJ |
232 | return VM_FAULT_SIGBUS; |
233 | } | |
234 | ||
2232c638 | 235 | *pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); |
9557feee | 236 | |
fce86ff5 | 237 | return vmf_insert_pfn_pud(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE); |
9557feee DJ |
238 | } |
239 | #else | |
226ab561 | 240 | static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, |
2232c638 | 241 | struct vm_fault *vmf, pfn_t *pfn) |
9557feee DJ |
242 | { |
243 | return VM_FAULT_FALLBACK; | |
244 | } | |
245 | #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ | |
246 | ||
226ab561 | 247 | static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf, |
c791ace1 | 248 | enum page_entry_size pe_size) |
dee41079 | 249 | { |
f4200391 | 250 | struct file *filp = vmf->vma->vm_file; |
2232c638 | 251 | unsigned long fault_size; |
36bdac1e SJ |
252 | vm_fault_t rc = VM_FAULT_SIGBUS; |
253 | int id; | |
2232c638 | 254 | pfn_t pfn; |
5f0694b3 | 255 | struct dev_dax *dev_dax = filp->private_data; |
dee41079 | 256 | |
6daaca52 DW |
257 | dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm, |
258 | (vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read", | |
76202620 | 259 | vmf->vma->vm_start, vmf->vma->vm_end, pe_size); |
dee41079 | 260 | |
7b6be844 | 261 | id = dax_read_lock(); |
c791ace1 DJ |
262 | switch (pe_size) { |
263 | case PE_SIZE_PTE: | |
2232c638 DW |
264 | fault_size = PAGE_SIZE; |
265 | rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn); | |
a2d58167 | 266 | break; |
c791ace1 | 267 | case PE_SIZE_PMD: |
2232c638 DW |
268 | fault_size = PMD_SIZE; |
269 | rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn); | |
9557feee | 270 | break; |
c791ace1 | 271 | case PE_SIZE_PUD: |
2232c638 DW |
272 | fault_size = PUD_SIZE; |
273 | rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn); | |
a2d58167 DJ |
274 | break; |
275 | default: | |
54eafcc9 | 276 | rc = VM_FAULT_SIGBUS; |
a2d58167 | 277 | } |
2232c638 DW |
278 | |
279 | if (rc == VM_FAULT_NOPAGE) { | |
280 | unsigned long i; | |
35de2995 | 281 | pgoff_t pgoff; |
2232c638 DW |
282 | |
283 | /* | |
284 | * In the device-dax case the only possibility for a | |
285 | * VM_FAULT_NOPAGE result is when device-dax capacity is | |
286 | * mapped. No need to consider the zero page, or racing | |
287 | * conflicting mappings. | |
288 | */ | |
35de2995 DW |
289 | pgoff = linear_page_index(vmf->vma, vmf->address |
290 | & ~(fault_size - 1)); | |
2232c638 DW |
291 | for (i = 0; i < fault_size / PAGE_SIZE; i++) { |
292 | struct page *page; | |
293 | ||
294 | page = pfn_to_page(pfn_t_to_pfn(pfn) + i); | |
295 | if (page->mapping) | |
296 | continue; | |
297 | page->mapping = filp->f_mapping; | |
35de2995 | 298 | page->index = pgoff + i; |
2232c638 DW |
299 | } |
300 | } | |
7b6be844 | 301 | dax_read_unlock(id); |
dee41079 DW |
302 | |
303 | return rc; | |
304 | } | |
305 | ||
226ab561 | 306 | static vm_fault_t dev_dax_fault(struct vm_fault *vmf) |
c791ace1 | 307 | { |
5f0694b3 | 308 | return dev_dax_huge_fault(vmf, PE_SIZE_PTE); |
c791ace1 DJ |
309 | } |
310 | ||
9702cffd DW |
311 | static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr) |
312 | { | |
313 | struct file *filp = vma->vm_file; | |
314 | struct dev_dax *dev_dax = filp->private_data; | |
315 | struct dax_region *dax_region = dev_dax->region; | |
316 | ||
317 | if (!IS_ALIGNED(addr, dax_region->align)) | |
318 | return -EINVAL; | |
319 | return 0; | |
320 | } | |
321 | ||
c1d53b92 DW |
322 | static unsigned long dev_dax_pagesize(struct vm_area_struct *vma) |
323 | { | |
324 | struct file *filp = vma->vm_file; | |
325 | struct dev_dax *dev_dax = filp->private_data; | |
326 | struct dax_region *dax_region = dev_dax->region; | |
327 | ||
328 | return dax_region->align; | |
329 | } | |
330 | ||
5f0694b3 DW |
331 | static const struct vm_operations_struct dax_vm_ops = { |
332 | .fault = dev_dax_fault, | |
333 | .huge_fault = dev_dax_huge_fault, | |
9702cffd | 334 | .split = dev_dax_split, |
c1d53b92 | 335 | .pagesize = dev_dax_pagesize, |
dee41079 DW |
336 | }; |
337 | ||
af69f51e | 338 | static int dax_mmap(struct file *filp, struct vm_area_struct *vma) |
dee41079 | 339 | { |
5f0694b3 | 340 | struct dev_dax *dev_dax = filp->private_data; |
7b6be844 | 341 | int rc, id; |
dee41079 | 342 | |
6daaca52 | 343 | dev_dbg(&dev_dax->dev, "trace\n"); |
dee41079 | 344 | |
7b6be844 DW |
345 | /* |
346 | * We lock to check dax_dev liveness and will re-check at | |
347 | * fault time. | |
348 | */ | |
349 | id = dax_read_lock(); | |
5f0694b3 | 350 | rc = check_vma(dev_dax, vma, __func__); |
7b6be844 | 351 | dax_read_unlock(id); |
dee41079 DW |
352 | if (rc) |
353 | return rc; | |
354 | ||
5f0694b3 | 355 | vma->vm_ops = &dax_vm_ops; |
e1fb4a08 | 356 | vma->vm_flags |= VM_HUGEPAGE; |
dee41079 | 357 | return 0; |
043a9255 DW |
358 | } |
359 | ||
360 | /* return an unmapped area aligned to the dax region specified alignment */ | |
af69f51e | 361 | static unsigned long dax_get_unmapped_area(struct file *filp, |
043a9255 DW |
362 | unsigned long addr, unsigned long len, unsigned long pgoff, |
363 | unsigned long flags) | |
364 | { | |
365 | unsigned long off, off_end, off_align, len_align, addr_align, align; | |
5f0694b3 | 366 | struct dev_dax *dev_dax = filp ? filp->private_data : NULL; |
043a9255 DW |
367 | struct dax_region *dax_region; |
368 | ||
5f0694b3 | 369 | if (!dev_dax || addr) |
043a9255 DW |
370 | goto out; |
371 | ||
5f0694b3 | 372 | dax_region = dev_dax->region; |
043a9255 DW |
373 | align = dax_region->align; |
374 | off = pgoff << PAGE_SHIFT; | |
375 | off_end = off + len; | |
376 | off_align = round_up(off, align); | |
377 | ||
378 | if ((off_end <= off_align) || ((off_end - off_align) < align)) | |
379 | goto out; | |
380 | ||
381 | len_align = len + align; | |
382 | if ((off + len_align) < off) | |
383 | goto out; | |
dee41079 | 384 | |
043a9255 DW |
385 | addr_align = current->mm->get_unmapped_area(filp, addr, len_align, |
386 | pgoff, flags); | |
387 | if (!IS_ERR_VALUE(addr_align)) { | |
388 | addr_align += (off - addr_align) & (align - 1); | |
389 | return addr_align; | |
390 | } | |
391 | out: | |
392 | return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); | |
393 | } | |
394 | ||
41c9b1be DJ |
395 | static const struct address_space_operations dev_dax_aops = { |
396 | .set_page_dirty = noop_set_page_dirty, | |
397 | .invalidatepage = noop_invalidatepage, | |
398 | }; | |
399 | ||
af69f51e | 400 | static int dax_open(struct inode *inode, struct file *filp) |
043a9255 | 401 | { |
7b6be844 DW |
402 | struct dax_device *dax_dev = inode_dax(inode); |
403 | struct inode *__dax_inode = dax_inode(dax_dev); | |
404 | struct dev_dax *dev_dax = dax_get_private(dax_dev); | |
043a9255 | 405 | |
6daaca52 | 406 | dev_dbg(&dev_dax->dev, "trace\n"); |
7b6be844 DW |
407 | inode->i_mapping = __dax_inode->i_mapping; |
408 | inode->i_mapping->host = __dax_inode; | |
41c9b1be | 409 | inode->i_mapping->a_ops = &dev_dax_aops; |
3bc52c45 | 410 | filp->f_mapping = inode->i_mapping; |
5660e13d | 411 | filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping); |
5f0694b3 | 412 | filp->private_data = dev_dax; |
ebd84d72 | 413 | inode->i_flags = S_DAX; |
043a9255 | 414 | |
043a9255 DW |
415 | return 0; |
416 | } | |
dee41079 | 417 | |
af69f51e | 418 | static int dax_release(struct inode *inode, struct file *filp) |
043a9255 | 419 | { |
5f0694b3 | 420 | struct dev_dax *dev_dax = filp->private_data; |
043a9255 | 421 | |
6daaca52 | 422 | dev_dbg(&dev_dax->dev, "trace\n"); |
043a9255 | 423 | return 0; |
dee41079 DW |
424 | } |
425 | ||
ab68f262 DW |
426 | static const struct file_operations dax_fops = { |
427 | .llseek = noop_llseek, | |
428 | .owner = THIS_MODULE, | |
af69f51e DW |
429 | .open = dax_open, |
430 | .release = dax_release, | |
431 | .get_unmapped_area = dax_get_unmapped_area, | |
432 | .mmap = dax_mmap, | |
ef842302 | 433 | .mmap_supported_flags = MAP_SYNC, |
ab68f262 DW |
434 | }; |
435 | ||
9567da0b | 436 | static void dev_dax_cdev_del(void *cdev) |
043a9255 | 437 | { |
9567da0b DW |
438 | cdev_del(cdev); |
439 | } | |
043a9255 | 440 | |
9567da0b DW |
441 | static void dev_dax_kill(void *dev_dax) |
442 | { | |
443 | kill_dev_dax(dev_dax); | |
ebd84d72 DW |
444 | } |
445 | ||
730926c3 | 446 | int dev_dax_probe(struct device *dev) |
043a9255 | 447 | { |
9567da0b DW |
448 | struct dev_dax *dev_dax = to_dev_dax(dev); |
449 | struct dax_device *dax_dev = dev_dax->dax_dev; | |
89ec9f2c | 450 | struct resource *res = &dev_dax->region->res; |
7b6be844 | 451 | struct inode *inode; |
ba09c01d | 452 | struct cdev *cdev; |
89ec9f2c | 453 | void *addr; |
753a0850 | 454 | int rc; |
43fe51e1 | 455 | |
89ec9f2c DW |
456 | /* 1:1 map region resource range to device-dax instance range */ |
457 | if (!devm_request_mem_region(dev, res->start, resource_size(res), | |
458 | dev_name(dev))) { | |
459 | dev_warn(dev, "could not reserve region %pR\n", res); | |
460 | return -EBUSY; | |
461 | } | |
462 | ||
463 | init_completion(&dev_dax->cmp); | |
464 | rc = percpu_ref_init(&dev_dax->ref, dev_dax_percpu_release, 0, | |
465 | GFP_KERNEL); | |
466 | if (rc) | |
467 | return rc; | |
468 | ||
469 | rc = devm_add_action_or_reset(dev, dev_dax_percpu_exit, &dev_dax->ref); | |
470 | if (rc) | |
471 | return rc; | |
472 | ||
473 | dev_dax->pgmap.ref = &dev_dax->ref; | |
474 | dev_dax->pgmap.kill = dev_dax_percpu_kill; | |
475 | addr = devm_memremap_pages(dev, &dev_dax->pgmap); | |
476 | if (IS_ERR(addr)) { | |
477 | devm_remove_action(dev, dev_dax_percpu_exit, &dev_dax->ref); | |
478 | percpu_ref_exit(&dev_dax->ref); | |
479 | return PTR_ERR(addr); | |
480 | } | |
481 | ||
7b6be844 DW |
482 | inode = dax_inode(dax_dev); |
483 | cdev = inode->i_cdev; | |
ba09c01d | 484 | cdev_init(cdev, &dax_fops); |
730926c3 DW |
485 | if (dev->class) { |
486 | /* for the CONFIG_DEV_DAX_PMEM_COMPAT case */ | |
487 | cdev->owner = dev->parent->driver->owner; | |
488 | } else | |
489 | cdev->owner = dev->driver->owner; | |
9567da0b DW |
490 | cdev_set_parent(cdev, &dev->kobj); |
491 | rc = cdev_add(cdev, dev->devt, 1); | |
d76911ee | 492 | if (rc) |
9567da0b | 493 | return rc; |
d76911ee | 494 | |
9567da0b DW |
495 | rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev); |
496 | if (rc) | |
497 | return rc; | |
043a9255 | 498 | |
9567da0b DW |
499 | run_dax(dax_dev); |
500 | return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax); | |
501 | } | |
730926c3 | 502 | EXPORT_SYMBOL_GPL(dev_dax_probe); |
043a9255 | 503 | |
9567da0b DW |
504 | static int dev_dax_remove(struct device *dev) |
505 | { | |
506 | /* all probe actions are unwound by devm */ | |
507 | return 0; | |
043a9255 | 508 | } |
9567da0b | 509 | |
d200781e DW |
510 | static struct dax_device_driver device_dax_driver = { |
511 | .drv = { | |
512 | .probe = dev_dax_probe, | |
513 | .remove = dev_dax_remove, | |
514 | }, | |
515 | .match_always = 1, | |
9567da0b | 516 | }; |
043a9255 | 517 | |
ab68f262 DW |
518 | static int __init dax_init(void) |
519 | { | |
9567da0b | 520 | return dax_driver_register(&device_dax_driver); |
ab68f262 DW |
521 | } |
522 | ||
523 | static void __exit dax_exit(void) | |
524 | { | |
d200781e | 525 | dax_driver_unregister(&device_dax_driver); |
ab68f262 DW |
526 | } |
527 | ||
528 | MODULE_AUTHOR("Intel Corporation"); | |
529 | MODULE_LICENSE("GPL v2"); | |
9567da0b | 530 | module_init(dax_init); |
ab68f262 | 531 | module_exit(dax_exit); |
9567da0b | 532 | MODULE_ALIAS_DAX_DEVICE(0); |