iov_iter_advance(): don't modify ->iov_offset for ITER_DISCARD
[linux-block.git] / include / linux / uio.h
CommitLineData
2874c5fd 1/* SPDX-License-Identifier: GPL-2.0-or-later */
1da177e4
LT
2/*
3 * Berkeley style UIO structures - Alan Cox 1994.
1da177e4 4 */
607ca46e
DH
5#ifndef __LINUX_UIO_H
6#define __LINUX_UIO_H
1da177e4 7
92236878 8#include <linux/kernel.h>
aa28de27 9#include <linux/thread_info.h>
607ca46e 10#include <uapi/linux/uio.h>
1da177e4 11
92236878 12struct page;
241699cd 13struct pipe_inode_info;
812ed032
JS
14
15struct kvec {
16 void *iov_base; /* and that should *never* hold a userland pointer */
17 size_t iov_len;
18};
19
00e23707 20enum iter_type {
875f1d07
JA
21 /* iter types */
22 ITER_IOVEC = 4,
23 ITER_KVEC = 8,
24 ITER_BVEC = 16,
25 ITER_PIPE = 32,
26 ITER_DISCARD = 64,
7ff50620 27 ITER_XARRAY = 128,
62a8067a
AV
28};
29
92236878 30struct iov_iter {
875f1d07
JA
31 /*
32 * Bit 0 is the read/write bit, set if we're writing.
33 * Bit 1 is the BVEC_FLAG_NO_REF bit, set if type is a bvec and
34 * the caller isn't expecting to drop a page reference when done.
35 */
aa563d7b 36 unsigned int type;
92236878
KO
37 size_t iov_offset;
38 size_t count;
62a8067a
AV
39 union {
40 const struct iovec *iov;
a280455f 41 const struct kvec *kvec;
62a8067a 42 const struct bio_vec *bvec;
7ff50620 43 struct xarray *xarray;
241699cd
AV
44 struct pipe_inode_info *pipe;
45 };
46 union {
47 unsigned long nr_segs;
27c0e374 48 struct {
8cefc107
DH
49 unsigned int head;
50 unsigned int start_head;
27c0e374 51 };
7ff50620 52 loff_t xarray_start;
62a8067a 53 };
92236878
KO
54};
55
00e23707
DH
56static inline enum iter_type iov_iter_type(const struct iov_iter *i)
57{
b6207430 58 return i->type & ~(READ | WRITE);
00e23707
DH
59}
60
61static inline bool iter_is_iovec(const struct iov_iter *i)
62{
63 return iov_iter_type(i) == ITER_IOVEC;
64}
65
66static inline bool iov_iter_is_kvec(const struct iov_iter *i)
67{
68 return iov_iter_type(i) == ITER_KVEC;
69}
70
71static inline bool iov_iter_is_bvec(const struct iov_iter *i)
72{
73 return iov_iter_type(i) == ITER_BVEC;
74}
75
76static inline bool iov_iter_is_pipe(const struct iov_iter *i)
77{
78 return iov_iter_type(i) == ITER_PIPE;
79}
80
9ea9ce04
DH
81static inline bool iov_iter_is_discard(const struct iov_iter *i)
82{
83 return iov_iter_type(i) == ITER_DISCARD;
84}
85
7ff50620
DH
86static inline bool iov_iter_is_xarray(const struct iov_iter *i)
87{
88 return iov_iter_type(i) == ITER_XARRAY;
89}
90
00e23707
DH
91static inline unsigned char iov_iter_rw(const struct iov_iter *i)
92{
93 return i->type & (READ | WRITE);
94}
95
1da177e4
LT
96/*
97 * Total number of bytes covered by an iovec.
98 *
99 * NOTE that it is not safe to use this function until all the iovec's
100 * segment lengths have been validated. Because the individual lengths can
101 * overflow a size_t when added together.
102 */
103static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs)
104{
105 unsigned long seg;
106 size_t ret = 0;
107
108 for (seg = 0; seg < nr_segs; seg++)
109 ret += iov[seg].iov_len;
110 return ret;
111}
112
92236878
KO
113static inline struct iovec iov_iter_iovec(const struct iov_iter *iter)
114{
115 return (struct iovec) {
116 .iov_base = iter->iov->iov_base + iter->iov_offset,
117 .iov_len = min(iter->count,
118 iter->iov->iov_len - iter->iov_offset),
119 };
120}
121
92236878
KO
122size_t iov_iter_copy_from_user_atomic(struct page *page,
123 struct iov_iter *i, unsigned long offset, size_t bytes);
92236878 124void iov_iter_advance(struct iov_iter *i, size_t bytes);
27c0e374 125void iov_iter_revert(struct iov_iter *i, size_t bytes);
92236878
KO
126int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes);
127size_t iov_iter_single_seg_count(const struct iov_iter *i);
6e58e79d
AV
128size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
129 struct iov_iter *i);
f0d1bec9
AV
130size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
131 struct iov_iter *i);
aa28de27
AV
132
133size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
134size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);
aa28de27 135size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);
aa28de27
AV
136
137static __always_inline __must_check
138size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
139{
140 if (unlikely(!check_copy_size(addr, bytes, true)))
c43aeb19 141 return 0;
aa28de27
AV
142 else
143 return _copy_to_iter(addr, bytes, i);
144}
145
146static __always_inline __must_check
147size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
148{
149 if (unlikely(!check_copy_size(addr, bytes, false)))
c43aeb19 150 return 0;
aa28de27
AV
151 else
152 return _copy_from_iter(addr, bytes, i);
153}
154
155static __always_inline __must_check
156bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
157{
4b6c132b
AV
158 size_t copied = copy_from_iter(addr, bytes, i);
159 if (likely(copied == bytes))
160 return true;
161 iov_iter_revert(i, copied);
162 return false;
aa28de27
AV
163}
164
165static __always_inline __must_check
166size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
167{
168 if (unlikely(!check_copy_size(addr, bytes, false)))
c43aeb19 169 return 0;
aa28de27
AV
170 else
171 return _copy_from_iter_nocache(addr, bytes, i);
172}
173
174static __always_inline __must_check
175bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
176{
4b6c132b
AV
177 size_t copied = copy_from_iter_nocache(addr, bytes, i);
178 if (likely(copied == bytes))
179 return true;
180 iov_iter_revert(i, copied);
181 return false;
aa28de27
AV
182}
183
0aed55af
DW
184#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
185/*
186 * Note, users like pmem that depend on the stricter semantics of
187 * copy_from_iter_flushcache() than copy_from_iter_nocache() must check for
188 * IS_ENABLED(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) before assuming that the
189 * destination is flushed from the cache on return.
190 */
6a37e940 191size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i);
0aed55af 192#else
6a37e940
LT
193#define _copy_from_iter_flushcache _copy_from_iter_nocache
194#endif
195
ec6347bb
DW
196#ifdef CONFIG_ARCH_HAS_COPY_MC
197size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
8780356e 198#else
ec6347bb 199#define _copy_mc_to_iter _copy_to_iter
8780356e
DW
200#endif
201
6a37e940
LT
202static __always_inline __must_check
203size_t copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
0aed55af 204{
6a37e940 205 if (unlikely(!check_copy_size(addr, bytes, false)))
c43aeb19 206 return 0;
6a37e940
LT
207 else
208 return _copy_from_iter_flushcache(addr, bytes, i);
0aed55af 209}
6a37e940 210
8780356e 211static __always_inline __must_check
ec6347bb 212size_t copy_mc_to_iter(void *addr, size_t bytes, struct iov_iter *i)
8780356e 213{
dfb06cba 214 if (unlikely(!check_copy_size(addr, bytes, true)))
8780356e
DW
215 return 0;
216 else
ec6347bb 217 return _copy_mc_to_iter(addr, bytes, i);
8780356e
DW
218}
219
c35e0248 220size_t iov_iter_zero(size_t bytes, struct iov_iter *);
886a3911 221unsigned long iov_iter_alignment(const struct iov_iter *i);
357f435d 222unsigned long iov_iter_gap_alignment(const struct iov_iter *i);
aa563d7b 223void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov,
71d8e532 224 unsigned long nr_segs, size_t count);
aa563d7b 225void iov_iter_kvec(struct iov_iter *i, unsigned int direction, const struct kvec *kvec,
05afcb77 226 unsigned long nr_segs, size_t count);
aa563d7b 227void iov_iter_bvec(struct iov_iter *i, unsigned int direction, const struct bio_vec *bvec,
abb78f87 228 unsigned long nr_segs, size_t count);
aa563d7b 229void iov_iter_pipe(struct iov_iter *i, unsigned int direction, struct pipe_inode_info *pipe,
241699cd 230 size_t count);
9ea9ce04 231void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count);
7ff50620
DH
232void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray,
233 loff_t start, size_t count);
7b2c99d1 234ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages,
2c80929c 235 size_t maxsize, unsigned maxpages, size_t *start);
91f79c43
AV
236ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages,
237 size_t maxsize, size_t *start);
f67da30c 238int iov_iter_npages(const struct iov_iter *i, int maxpages);
92236878 239
4b8164b9
AV
240const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags);
241
b57332b4 242static inline size_t iov_iter_count(const struct iov_iter *i)
92236878
KO
243{
244 return i->count;
245}
246
0b86dbf6
AV
247/*
248 * Cap the iov_iter by given limit; note that the second argument is
249 * *not* the new size - it's upper limit for such. Passing it a value
250 * greater than the amount of data in iov_iter is fine - it'll just do
251 * nothing in that case.
252 */
253static inline void iov_iter_truncate(struct iov_iter *i, u64 count)
0c949334 254{
0b86dbf6
AV
255 /*
256 * count doesn't have to fit in size_t - comparison extends both
257 * operands to u64 here and any value that would be truncated by
258 * conversion in assignement is by definition greater than all
259 * values of size_t, including old i->count.
260 */
0c949334
AV
261 if (i->count > count)
262 i->count = count;
263}
264
b42b15fd
AV
265/*
266 * reexpand a previously truncated iterator; count must be no more than how much
267 * we had shrunk it.
268 */
269static inline void iov_iter_reexpand(struct iov_iter *i, size_t count)
270{
271 i->count = count;
272}
52cbd23a
WB
273
274struct csum_state {
275 __wsum csum;
276 size_t off;
277};
278
279size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csstate, struct iov_iter *i);
a604ec7e 280size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
4b6c132b
AV
281
282static __always_inline __must_check
283bool csum_and_copy_from_iter_full(void *addr, size_t bytes,
284 __wsum *csum, struct iov_iter *i)
285{
286 size_t copied = csum_and_copy_from_iter(addr, bytes, csum, i);
287 if (likely(copied == bytes))
288 return true;
289 iov_iter_revert(i, copied);
290 return false;
291}
d05f4435
SG
292size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
293 struct iov_iter *i);
b42b15fd 294
bfdc5970
CH
295struct iovec *iovec_from_user(const struct iovec __user *uvector,
296 unsigned long nr_segs, unsigned long fast_segs,
297 struct iovec *fast_iov, bool compat);
298ssize_t import_iovec(int type, const struct iovec __user *uvec,
299 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
300 struct iov_iter *i);
301ssize_t __import_iovec(int type, const struct iovec __user *uvec,
302 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
303 struct iov_iter *i, bool compat);
bc917be8
AV
304int import_single_range(int type, void __user *buf, size_t len,
305 struct iovec *iov, struct iov_iter *i);
306
812ed032 307#endif