Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
8025e5dd JK |
2 | #include <linux/kernel.h> |
3 | #include <linux/errno.h> | |
4 | #include <linux/err.h> | |
5 | #include <linux/mm.h> | |
6 | #include <linux/slab.h> | |
7 | #include <linux/vmalloc.h> | |
8 | #include <linux/pagemap.h> | |
9 | #include <linux/sched.h> | |
10 | ||
61f9ec1d | 11 | /** |
8025e5dd JK |
12 | * get_vaddr_frames() - map virtual addresses to pfns |
13 | * @start: starting user address | |
14 | * @nr_frames: number of pages / pfns from start to map | |
7f23b350 | 15 | * @gup_flags: flags modifying lookup behaviour |
8025e5dd JK |
16 | * @vec: structure which receives pages / pfns of the addresses mapped. |
17 | * It should have space for at least nr_frames entries. | |
18 | * | |
19 | * This function maps virtual addresses from @start and fills @vec structure | |
20 | * with page frame numbers or page pointers to corresponding pages (choice | |
21 | * depends on the type of the vma underlying the virtual address). If @start | |
22 | * belongs to a normal vma, the function grabs reference to each of the pages | |
23 | * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't | |
24 | * touch page structures and the caller must make sure pfns aren't reused for | |
25 | * anything else while he is using them. | |
26 | * | |
27 | * The function returns number of pages mapped which may be less than | |
28 | * @nr_frames. In particular we stop mapping if there are more vmas of | |
29 | * different type underlying the specified range of virtual addresses. | |
30 | * When the function isn't able to map a single page, it returns error. | |
31 | * | |
32 | * This function takes care of grabbing mmap_sem as necessary. | |
33 | */ | |
34 | int get_vaddr_frames(unsigned long start, unsigned int nr_frames, | |
7f23b350 | 35 | unsigned int gup_flags, struct frame_vector *vec) |
8025e5dd JK |
36 | { |
37 | struct mm_struct *mm = current->mm; | |
38 | struct vm_area_struct *vma; | |
39 | int ret = 0; | |
40 | int err; | |
41 | int locked; | |
42 | ||
43 | if (nr_frames == 0) | |
44 | return 0; | |
45 | ||
46 | if (WARN_ON_ONCE(nr_frames > vec->nr_allocated)) | |
47 | nr_frames = vec->nr_allocated; | |
48 | ||
49 | down_read(&mm->mmap_sem); | |
50 | locked = 1; | |
51 | vma = find_vma_intersection(mm, start, start + 1); | |
52 | if (!vma) { | |
53 | ret = -EFAULT; | |
54 | goto out; | |
55 | } | |
56 | if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) { | |
57 | vec->got_ref = true; | |
58 | vec->is_pfns = false; | |
d4edcf0d | 59 | ret = get_user_pages_locked(start, nr_frames, |
3b913179 | 60 | gup_flags, (struct page **)(vec->ptrs), &locked); |
8025e5dd JK |
61 | goto out; |
62 | } | |
63 | ||
64 | vec->got_ref = false; | |
65 | vec->is_pfns = true; | |
66 | do { | |
67 | unsigned long *nums = frame_vector_pfns(vec); | |
68 | ||
69 | while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) { | |
70 | err = follow_pfn(vma, start, &nums[ret]); | |
71 | if (err) { | |
72 | if (ret == 0) | |
73 | ret = err; | |
74 | goto out; | |
75 | } | |
76 | start += PAGE_SIZE; | |
77 | ret++; | |
78 | } | |
79 | /* | |
80 | * We stop if we have enough pages or if VMA doesn't completely | |
81 | * cover the tail page. | |
82 | */ | |
83 | if (ret >= nr_frames || start < vma->vm_end) | |
84 | break; | |
85 | vma = find_vma_intersection(mm, start, start + 1); | |
86 | } while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP)); | |
87 | out: | |
88 | if (locked) | |
89 | up_read(&mm->mmap_sem); | |
90 | if (!ret) | |
91 | ret = -EFAULT; | |
92 | if (ret > 0) | |
93 | vec->nr_frames = ret; | |
94 | return ret; | |
95 | } | |
96 | EXPORT_SYMBOL(get_vaddr_frames); | |
97 | ||
98 | /** | |
99 | * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired | |
100 | * them | |
101 | * @vec: frame vector to put | |
102 | * | |
103 | * Drop references to pages if get_vaddr_frames() acquired them. We also | |
104 | * invalidate the frame vector so that it is prepared for the next call into | |
105 | * get_vaddr_frames(). | |
106 | */ | |
107 | void put_vaddr_frames(struct frame_vector *vec) | |
108 | { | |
109 | int i; | |
110 | struct page **pages; | |
111 | ||
112 | if (!vec->got_ref) | |
113 | goto out; | |
114 | pages = frame_vector_pages(vec); | |
115 | /* | |
116 | * frame_vector_pages() might needed to do a conversion when | |
117 | * get_vaddr_frames() got pages but vec was later converted to pfns. | |
118 | * But it shouldn't really fail to convert pfns back... | |
119 | */ | |
120 | if (WARN_ON(IS_ERR(pages))) | |
121 | goto out; | |
122 | for (i = 0; i < vec->nr_frames; i++) | |
123 | put_page(pages[i]); | |
124 | vec->got_ref = false; | |
125 | out: | |
126 | vec->nr_frames = 0; | |
127 | } | |
128 | EXPORT_SYMBOL(put_vaddr_frames); | |
129 | ||
130 | /** | |
131 | * frame_vector_to_pages - convert frame vector to contain page pointers | |
132 | * @vec: frame vector to convert | |
133 | * | |
134 | * Convert @vec to contain array of page pointers. If the conversion is | |
135 | * successful, return 0. Otherwise return an error. Note that we do not grab | |
136 | * page references for the page structures. | |
137 | */ | |
138 | int frame_vector_to_pages(struct frame_vector *vec) | |
139 | { | |
140 | int i; | |
141 | unsigned long *nums; | |
142 | struct page **pages; | |
143 | ||
144 | if (!vec->is_pfns) | |
145 | return 0; | |
146 | nums = frame_vector_pfns(vec); | |
147 | for (i = 0; i < vec->nr_frames; i++) | |
148 | if (!pfn_valid(nums[i])) | |
149 | return -EINVAL; | |
150 | pages = (struct page **)nums; | |
151 | for (i = 0; i < vec->nr_frames; i++) | |
152 | pages[i] = pfn_to_page(nums[i]); | |
153 | vec->is_pfns = false; | |
154 | return 0; | |
155 | } | |
156 | EXPORT_SYMBOL(frame_vector_to_pages); | |
157 | ||
158 | /** | |
159 | * frame_vector_to_pfns - convert frame vector to contain pfns | |
160 | * @vec: frame vector to convert | |
161 | * | |
162 | * Convert @vec to contain array of pfns. | |
163 | */ | |
164 | void frame_vector_to_pfns(struct frame_vector *vec) | |
165 | { | |
166 | int i; | |
167 | unsigned long *nums; | |
168 | struct page **pages; | |
169 | ||
170 | if (vec->is_pfns) | |
171 | return; | |
172 | pages = (struct page **)(vec->ptrs); | |
173 | nums = (unsigned long *)pages; | |
174 | for (i = 0; i < vec->nr_frames; i++) | |
175 | nums[i] = page_to_pfn(pages[i]); | |
176 | vec->is_pfns = true; | |
177 | } | |
178 | EXPORT_SYMBOL(frame_vector_to_pfns); | |
179 | ||
180 | /** | |
181 | * frame_vector_create() - allocate & initialize structure for pinned pfns | |
182 | * @nr_frames: number of pfns slots we should reserve | |
183 | * | |
184 | * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns | |
185 | * pfns. | |
186 | */ | |
187 | struct frame_vector *frame_vector_create(unsigned int nr_frames) | |
188 | { | |
189 | struct frame_vector *vec; | |
190 | int size = sizeof(struct frame_vector) + sizeof(void *) * nr_frames; | |
191 | ||
192 | if (WARN_ON_ONCE(nr_frames == 0)) | |
193 | return NULL; | |
194 | /* | |
195 | * This is absurdly high. It's here just to avoid strange effects when | |
196 | * arithmetics overflows. | |
197 | */ | |
198 | if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2)) | |
199 | return NULL; | |
200 | /* | |
201 | * Avoid higher order allocations, use vmalloc instead. It should | |
202 | * be rare anyway. | |
203 | */ | |
752ade68 | 204 | vec = kvmalloc(size, GFP_KERNEL); |
8025e5dd JK |
205 | if (!vec) |
206 | return NULL; | |
207 | vec->nr_allocated = nr_frames; | |
208 | vec->nr_frames = 0; | |
209 | return vec; | |
210 | } | |
211 | EXPORT_SYMBOL(frame_vector_create); | |
212 | ||
213 | /** | |
214 | * frame_vector_destroy() - free memory allocated to carry frame vector | |
215 | * @vec: Frame vector to free | |
216 | * | |
217 | * Free structure allocated by frame_vector_create() to carry frames. | |
218 | */ | |
219 | void frame_vector_destroy(struct frame_vector *vec) | |
220 | { | |
221 | /* Make sure put_vaddr_frames() got called properly... */ | |
222 | VM_BUG_ON(vec->nr_frames > 0); | |
223 | kvfree(vec); | |
224 | } | |
225 | EXPORT_SYMBOL(frame_vector_destroy); |