Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
8025e5dd JK |
2 | #include <linux/kernel.h> |
3 | #include <linux/errno.h> | |
4 | #include <linux/err.h> | |
5 | #include <linux/mm.h> | |
6 | #include <linux/slab.h> | |
7 | #include <linux/vmalloc.h> | |
8 | #include <linux/pagemap.h> | |
9 | #include <linux/sched.h> | |
10 | ||
eb83b8e3 DV |
11 | #include <media/frame_vector.h> |
12 | ||
61f9ec1d | 13 | /** |
8025e5dd JK |
14 | * get_vaddr_frames() - map virtual addresses to pfns |
15 | * @start: starting user address | |
16 | * @nr_frames: number of pages / pfns from start to map | |
e2fc6edd | 17 | * @write: the mapped address has write permission |
8025e5dd JK |
18 | * @vec: structure which receives pages / pfns of the addresses mapped. |
19 | * It should have space for at least nr_frames entries. | |
20 | * | |
21 | * This function maps virtual addresses from @start and fills @vec structure | |
22 | * with page frame numbers or page pointers to corresponding pages (choice | |
23 | * depends on the type of the vma underlying the virtual address). If @start | |
24 | * belongs to a normal vma, the function grabs reference to each of the pages | |
25 | * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't | |
26 | * touch page structures and the caller must make sure pfns aren't reused for | |
27 | * anything else while he is using them. | |
28 | * | |
29 | * The function returns number of pages mapped which may be less than | |
30 | * @nr_frames. In particular we stop mapping if there are more vmas of | |
31 | * different type underlying the specified range of virtual addresses. | |
32 | * When the function isn't able to map a single page, it returns error. | |
33 | * | |
735de5ca HV |
34 | * Note that get_vaddr_frames() cannot follow VM_IO mappings. It used |
35 | * to be able to do that, but that could (racily) return non-refcounted | |
36 | * pfns. | |
37 | * | |
c1e8d7c6 | 38 | * This function takes care of grabbing mmap_lock as necessary. |
8025e5dd | 39 | */ |
e2fc6edd | 40 | int get_vaddr_frames(unsigned long start, unsigned int nr_frames, bool write, |
04769cb1 | 41 | struct frame_vector *vec) |
8025e5dd | 42 | { |
6647e76a | 43 | int ret; |
e2ca6ba6 | 44 | unsigned int gup_flags = FOLL_LONGTERM; |
8025e5dd JK |
45 | |
46 | if (nr_frames == 0) | |
47 | return 0; | |
48 | ||
49 | if (WARN_ON_ONCE(nr_frames > vec->nr_allocated)) | |
50 | nr_frames = vec->nr_allocated; | |
51 | ||
5d65e7a7 AK |
52 | start = untagged_addr(start); |
53 | ||
e2fc6edd HV |
54 | if (write) |
55 | gup_flags |= FOLL_WRITE; | |
56 | ||
57 | ret = pin_user_pages_fast(start, nr_frames, gup_flags, | |
04769cb1 | 58 | (struct page **)(vec->ptrs)); |
6647e76a LT |
59 | vec->got_ref = true; |
60 | vec->is_pfns = false; | |
61 | vec->nr_frames = ret; | |
8025e5dd | 62 | |
6647e76a LT |
63 | if (likely(ret > 0)) |
64 | return ret; | |
8025e5dd | 65 | |
6647e76a LT |
66 | vec->nr_frames = 0; |
67 | return ret ? ret : -EFAULT; | |
8025e5dd JK |
68 | } |
69 | EXPORT_SYMBOL(get_vaddr_frames); | |
70 | ||
71 | /** | |
72 | * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired | |
73 | * them | |
74 | * @vec: frame vector to put | |
75 | * | |
76 | * Drop references to pages if get_vaddr_frames() acquired them. We also | |
77 | * invalidate the frame vector so that it is prepared for the next call into | |
78 | * get_vaddr_frames(). | |
79 | */ | |
80 | void put_vaddr_frames(struct frame_vector *vec) | |
81 | { | |
8025e5dd JK |
82 | struct page **pages; |
83 | ||
84 | if (!vec->got_ref) | |
85 | goto out; | |
86 | pages = frame_vector_pages(vec); | |
87 | /* | |
88 | * frame_vector_pages() might needed to do a conversion when | |
89 | * get_vaddr_frames() got pages but vec was later converted to pfns. | |
90 | * But it shouldn't really fail to convert pfns back... | |
91 | */ | |
92 | if (WARN_ON(IS_ERR(pages))) | |
93 | goto out; | |
55a650c3 JH |
94 | |
95 | unpin_user_pages(pages, vec->nr_frames); | |
8025e5dd JK |
96 | vec->got_ref = false; |
97 | out: | |
98 | vec->nr_frames = 0; | |
99 | } | |
100 | EXPORT_SYMBOL(put_vaddr_frames); | |
101 | ||
102 | /** | |
103 | * frame_vector_to_pages - convert frame vector to contain page pointers | |
104 | * @vec: frame vector to convert | |
105 | * | |
106 | * Convert @vec to contain array of page pointers. If the conversion is | |
107 | * successful, return 0. Otherwise return an error. Note that we do not grab | |
108 | * page references for the page structures. | |
109 | */ | |
110 | int frame_vector_to_pages(struct frame_vector *vec) | |
111 | { | |
112 | int i; | |
113 | unsigned long *nums; | |
114 | struct page **pages; | |
115 | ||
116 | if (!vec->is_pfns) | |
117 | return 0; | |
118 | nums = frame_vector_pfns(vec); | |
119 | for (i = 0; i < vec->nr_frames; i++) | |
120 | if (!pfn_valid(nums[i])) | |
121 | return -EINVAL; | |
122 | pages = (struct page **)nums; | |
123 | for (i = 0; i < vec->nr_frames; i++) | |
124 | pages[i] = pfn_to_page(nums[i]); | |
125 | vec->is_pfns = false; | |
126 | return 0; | |
127 | } | |
128 | EXPORT_SYMBOL(frame_vector_to_pages); | |
129 | ||
130 | /** | |
131 | * frame_vector_to_pfns - convert frame vector to contain pfns | |
132 | * @vec: frame vector to convert | |
133 | * | |
134 | * Convert @vec to contain array of pfns. | |
135 | */ | |
136 | void frame_vector_to_pfns(struct frame_vector *vec) | |
137 | { | |
138 | int i; | |
139 | unsigned long *nums; | |
140 | struct page **pages; | |
141 | ||
142 | if (vec->is_pfns) | |
143 | return; | |
144 | pages = (struct page **)(vec->ptrs); | |
145 | nums = (unsigned long *)pages; | |
146 | for (i = 0; i < vec->nr_frames; i++) | |
147 | nums[i] = page_to_pfn(pages[i]); | |
148 | vec->is_pfns = true; | |
149 | } | |
150 | EXPORT_SYMBOL(frame_vector_to_pfns); | |
151 | ||
152 | /** | |
153 | * frame_vector_create() - allocate & initialize structure for pinned pfns | |
154 | * @nr_frames: number of pfns slots we should reserve | |
155 | * | |
156 | * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns | |
157 | * pfns. | |
158 | */ | |
159 | struct frame_vector *frame_vector_create(unsigned int nr_frames) | |
160 | { | |
161 | struct frame_vector *vec; | |
162 | int size = sizeof(struct frame_vector) + sizeof(void *) * nr_frames; | |
163 | ||
164 | if (WARN_ON_ONCE(nr_frames == 0)) | |
165 | return NULL; | |
166 | /* | |
167 | * This is absurdly high. It's here just to avoid strange effects when | |
168 | * arithmetics overflows. | |
169 | */ | |
170 | if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2)) | |
171 | return NULL; | |
172 | /* | |
173 | * Avoid higher order allocations, use vmalloc instead. It should | |
174 | * be rare anyway. | |
175 | */ | |
752ade68 | 176 | vec = kvmalloc(size, GFP_KERNEL); |
8025e5dd JK |
177 | if (!vec) |
178 | return NULL; | |
179 | vec->nr_allocated = nr_frames; | |
180 | vec->nr_frames = 0; | |
181 | return vec; | |
182 | } | |
183 | EXPORT_SYMBOL(frame_vector_create); | |
184 | ||
185 | /** | |
186 | * frame_vector_destroy() - free memory allocated to carry frame vector | |
187 | * @vec: Frame vector to free | |
188 | * | |
189 | * Free structure allocated by frame_vector_create() to carry frames. | |
190 | */ | |
191 | void frame_vector_destroy(struct frame_vector *vec) | |
192 | { | |
193 | /* Make sure put_vaddr_frames() got called properly... */ | |
194 | VM_BUG_ON(vec->nr_frames > 0); | |
195 | kvfree(vec); | |
196 | } | |
197 | EXPORT_SYMBOL(frame_vector_destroy); |