Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
3d14c5d2 YS |
2 | #include <linux/ceph/ceph_debug.h> |
3 | ||
4 | #include <linux/module.h> | |
5 | #include <linux/sched.h> | |
6 | #include <linux/slab.h> | |
7 | #include <linux/file.h> | |
8 | #include <linux/namei.h> | |
9 | #include <linux/writeback.h> | |
10 | ||
11 | #include <linux/ceph/libceph.h> | |
12 | ||
13 | /* | |
14 | * build a vector of user pages | |
15 | */ | |
b324814e | 16 | struct page **ceph_get_direct_page_vector(const void __user *data, |
b6aa5901 | 17 | int num_pages, bool write_page) |
3d14c5d2 YS |
18 | { |
19 | struct page **pages; | |
38815b78 SW |
20 | int got = 0; |
21 | int rc = 0; | |
3d14c5d2 | 22 | |
6da2ec56 | 23 | pages = kmalloc_array(num_pages, sizeof(*pages), GFP_NOFS); |
3d14c5d2 YS |
24 | if (!pages) |
25 | return ERR_PTR(-ENOMEM); | |
26 | ||
38815b78 | 27 | while (got < num_pages) { |
77478715 | 28 | rc = get_user_pages_fast( |
38815b78 | 29 | (unsigned long)data + ((unsigned long)got * PAGE_SIZE), |
73b0140b | 30 | num_pages - got, write_page ? FOLL_WRITE : 0, pages + got); |
38815b78 SW |
31 | if (rc < 0) |
32 | break; | |
33 | BUG_ON(rc == 0); | |
34 | got += rc; | |
35 | } | |
38815b78 | 36 | if (rc < 0) |
3d14c5d2 YS |
37 | goto fail; |
38 | return pages; | |
39 | ||
40 | fail: | |
38815b78 | 41 | ceph_put_page_vector(pages, got, false); |
3d14c5d2 YS |
42 | return ERR_PTR(rc); |
43 | } | |
44 | EXPORT_SYMBOL(ceph_get_direct_page_vector); | |
45 | ||
b6aa5901 | 46 | void ceph_put_page_vector(struct page **pages, int num_pages, bool dirty) |
3d14c5d2 YS |
47 | { |
48 | int i; | |
49 | ||
b6aa5901 HC |
50 | for (i = 0; i < num_pages; i++) { |
51 | if (dirty) | |
52 | set_page_dirty_lock(pages[i]); | |
3d14c5d2 | 53 | put_page(pages[i]); |
b6aa5901 | 54 | } |
b01da6a0 | 55 | kvfree(pages); |
3d14c5d2 YS |
56 | } |
57 | EXPORT_SYMBOL(ceph_put_page_vector); | |
58 | ||
59 | void ceph_release_page_vector(struct page **pages, int num_pages) | |
60 | { | |
61 | int i; | |
62 | ||
63 | for (i = 0; i < num_pages; i++) | |
64 | __free_pages(pages[i], 0); | |
65 | kfree(pages); | |
66 | } | |
67 | EXPORT_SYMBOL(ceph_release_page_vector); | |
68 | ||
69 | /* | |
70 | * allocate a vector new pages | |
71 | */ | |
72 | struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags) | |
73 | { | |
74 | struct page **pages; | |
75 | int i; | |
76 | ||
6da2ec56 | 77 | pages = kmalloc_array(num_pages, sizeof(*pages), flags); |
3d14c5d2 YS |
78 | if (!pages) |
79 | return ERR_PTR(-ENOMEM); | |
80 | for (i = 0; i < num_pages; i++) { | |
81 | pages[i] = __page_cache_alloc(flags); | |
82 | if (pages[i] == NULL) { | |
83 | ceph_release_page_vector(pages, i); | |
84 | return ERR_PTR(-ENOMEM); | |
85 | } | |
86 | } | |
87 | return pages; | |
88 | } | |
89 | EXPORT_SYMBOL(ceph_alloc_page_vector); | |
90 | ||
91 | /* | |
92 | * copy user data into a page vector | |
93 | */ | |
94 | int ceph_copy_user_to_page_vector(struct page **pages, | |
b324814e | 95 | const void __user *data, |
3d14c5d2 YS |
96 | loff_t off, size_t len) |
97 | { | |
98 | int i = 0; | |
09cbfeaf | 99 | int po = off & ~PAGE_MASK; |
3d14c5d2 YS |
100 | int left = len; |
101 | int l, bad; | |
102 | ||
103 | while (left > 0) { | |
09cbfeaf | 104 | l = min_t(int, PAGE_SIZE-po, left); |
3d14c5d2 YS |
105 | bad = copy_from_user(page_address(pages[i]) + po, data, l); |
106 | if (bad == l) | |
107 | return -EFAULT; | |
108 | data += l - bad; | |
109 | left -= l - bad; | |
110 | po += l - bad; | |
09cbfeaf | 111 | if (po == PAGE_SIZE) { |
3d14c5d2 YS |
112 | po = 0; |
113 | i++; | |
114 | } | |
115 | } | |
116 | return len; | |
117 | } | |
118 | EXPORT_SYMBOL(ceph_copy_user_to_page_vector); | |
119 | ||
903bb32e | 120 | void ceph_copy_to_page_vector(struct page **pages, |
b324814e | 121 | const void *data, |
3d14c5d2 YS |
122 | loff_t off, size_t len) |
123 | { | |
124 | int i = 0; | |
09cbfeaf | 125 | size_t po = off & ~PAGE_MASK; |
3d14c5d2 | 126 | size_t left = len; |
3d14c5d2 YS |
127 | |
128 | while (left > 0) { | |
09cbfeaf | 129 | size_t l = min_t(size_t, PAGE_SIZE-po, left); |
903bb32e | 130 | |
3d14c5d2 YS |
131 | memcpy(page_address(pages[i]) + po, data, l); |
132 | data += l; | |
133 | left -= l; | |
134 | po += l; | |
09cbfeaf | 135 | if (po == PAGE_SIZE) { |
3d14c5d2 YS |
136 | po = 0; |
137 | i++; | |
138 | } | |
139 | } | |
3d14c5d2 YS |
140 | } |
141 | EXPORT_SYMBOL(ceph_copy_to_page_vector); | |
142 | ||
903bb32e | 143 | void ceph_copy_from_page_vector(struct page **pages, |
b324814e | 144 | void *data, |
3d14c5d2 YS |
145 | loff_t off, size_t len) |
146 | { | |
147 | int i = 0; | |
09cbfeaf | 148 | size_t po = off & ~PAGE_MASK; |
3d14c5d2 | 149 | size_t left = len; |
3d14c5d2 YS |
150 | |
151 | while (left > 0) { | |
09cbfeaf | 152 | size_t l = min_t(size_t, PAGE_SIZE-po, left); |
903bb32e | 153 | |
3d14c5d2 YS |
154 | memcpy(data, page_address(pages[i]) + po, l); |
155 | data += l; | |
156 | left -= l; | |
157 | po += l; | |
09cbfeaf | 158 | if (po == PAGE_SIZE) { |
3d14c5d2 YS |
159 | po = 0; |
160 | i++; | |
161 | } | |
162 | } | |
3d14c5d2 YS |
163 | } |
164 | EXPORT_SYMBOL(ceph_copy_from_page_vector); | |
165 | ||
3d14c5d2 YS |
166 | /* |
167 | * Zero an extent within a page vector. Offset is relative to the | |
168 | * start of the first page. | |
169 | */ | |
170 | void ceph_zero_page_vector_range(int off, int len, struct page **pages) | |
171 | { | |
09cbfeaf | 172 | int i = off >> PAGE_SHIFT; |
3d14c5d2 | 173 | |
09cbfeaf | 174 | off &= ~PAGE_MASK; |
3d14c5d2 YS |
175 | |
176 | dout("zero_page_vector_page %u~%u\n", off, len); | |
177 | ||
178 | /* leading partial page? */ | |
179 | if (off) { | |
09cbfeaf | 180 | int end = min((int)PAGE_SIZE, off + len); |
3d14c5d2 YS |
181 | dout("zeroing %d %p head from %d\n", i, pages[i], |
182 | (int)off); | |
183 | zero_user_segment(pages[i], off, end); | |
184 | len -= (end - off); | |
185 | i++; | |
186 | } | |
09cbfeaf | 187 | while (len >= PAGE_SIZE) { |
3d14c5d2 | 188 | dout("zeroing %d %p len=%d\n", i, pages[i], len); |
09cbfeaf KS |
189 | zero_user_segment(pages[i], 0, PAGE_SIZE); |
190 | len -= PAGE_SIZE; | |
3d14c5d2 YS |
191 | i++; |
192 | } | |
193 | /* trailing partial page? */ | |
194 | if (len) { | |
195 | dout("zeroing %d %p tail to %d\n", i, pages[i], (int)len); | |
196 | zero_user_segment(pages[i], 0, len); | |
197 | } | |
198 | } | |
199 | EXPORT_SYMBOL(ceph_zero_page_vector_range); |