Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
f87d0fbb RR |
2 | /* |
3 | * Linux host-side vring helpers; for when the kernel needs to access | |
4 | * someone else's vring. | |
5 | * | |
6 | * Copyright IBM Corporation, 2013. | |
7 | * Parts taken from drivers/vhost/vhost.c Copyright 2009 Red Hat, Inc. | |
8 | * | |
f87d0fbb RR |
9 | * Written by: Rusty Russell <rusty@rustcorp.com.au> |
10 | */ | |
11 | #ifndef _LINUX_VRINGH_H | |
12 | #define _LINUX_VRINGH_H | |
13 | #include <uapi/linux/virtio_ring.h> | |
b9f7ac8c | 14 | #include <linux/virtio_byteorder.h> |
f87d0fbb RR |
15 | #include <linux/uio.h> |
16 | #include <linux/slab.h> | |
ea2f6af1 | 17 | #include <linux/spinlock.h> |
3302363a | 18 | #if IS_REACHABLE(CONFIG_VHOST_IOTLB) |
9ad9c49c JW |
19 | #include <linux/dma-direction.h> |
20 | #include <linux/vhost_iotlb.h> | |
3302363a | 21 | #endif |
f87d0fbb RR |
22 | #include <asm/barrier.h> |
23 | ||
24 | /* virtio_ring with information needed for host access. */ | |
25 | struct vringh { | |
b9f7ac8c MT |
26 | /* Everything is little endian */ |
27 | bool little_endian; | |
28 | ||
f87d0fbb RR |
29 | /* Guest publishes used event idx (note: we always do). */ |
30 | bool event_indices; | |
31 | ||
32 | /* Can we get away with weak barriers? */ | |
33 | bool weak_barriers; | |
34 | ||
35 | /* Last available index we saw (ie. where we're up to). */ | |
36 | u16 last_avail_idx; | |
37 | ||
38 | /* Last index we used. */ | |
39 | u16 last_used_idx; | |
40 | ||
41 | /* How many descriptors we've completed since last need_notify(). */ | |
42 | u32 completed; | |
43 | ||
44 | /* The vring (note: it may contain user pointers!) */ | |
45 | struct vring vring; | |
3beee86a | 46 | |
9ad9c49c JW |
47 | /* IOTLB for this vring */ |
48 | struct vhost_iotlb *iotlb; | |
49 | ||
f53d9910 SG |
50 | /* spinlock to synchronize IOTLB accesses */ |
51 | spinlock_t *iotlb_lock; | |
52 | ||
3beee86a SB |
53 | /* The function to call to notify the guest about added buffers */ |
54 | void (*notify)(struct vringh *); | |
55 | }; | |
56 | ||
57 | /** | |
58 | * struct vringh_config_ops - ops for creating a host vring from a virtio driver | |
59 | * @find_vrhs: find the host vrings and instantiate them | |
60 | * vdev: the virtio_device | |
61 | * nhvrs: the number of host vrings to find | |
62 | * hvrs: on success, includes new host vrings | |
63 | * callbacks: array of driver callbacks, for each host vring | |
64 | * include a NULL entry for vqs that do not need a callback | |
65 | * Returns 0 on success or error status | |
66 | * @del_vrhs: free the host vrings found by find_vrhs(). | |
67 | */ | |
68 | struct virtio_device; | |
69 | typedef void vrh_callback_t(struct virtio_device *, struct vringh *); | |
70 | struct vringh_config_ops { | |
71 | int (*find_vrhs)(struct virtio_device *vdev, unsigned nhvrs, | |
72 | struct vringh *vrhs[], vrh_callback_t *callbacks[]); | |
73 | void (*del_vrhs)(struct virtio_device *vdev); | |
f87d0fbb RR |
74 | }; |
75 | ||
76 | /* The memory the vring can access, and what offset to apply. */ | |
77 | struct vringh_range { | |
78 | u64 start, end_incl; | |
79 | u64 offset; | |
80 | }; | |
81 | ||
82 | /** | |
83 | * struct vringh_iov - iovec mangler. | |
84 | * | |
85 | * Mangles iovec in place, and restores it. | |
86 | * Remaining data is iov + i, of used - i elements. | |
87 | */ | |
88 | struct vringh_iov { | |
89 | struct iovec *iov; | |
90 | size_t consumed; /* Within iov[i] */ | |
91 | unsigned i, used, max_num; | |
92 | }; | |
93 | ||
94 | /** | |
95 | * struct vringh_iov - kvec mangler. | |
96 | * | |
97 | * Mangles kvec in place, and restores it. | |
98 | * Remaining data is iov + i, of used - i elements. | |
99 | */ | |
100 | struct vringh_kiov { | |
101 | struct kvec *iov; | |
102 | size_t consumed; /* Within iov[i] */ | |
103 | unsigned i, used, max_num; | |
104 | }; | |
105 | ||
106 | /* Flag on max_num to indicate we're kmalloced. */ | |
107 | #define VRINGH_IOV_ALLOCATED 0x8000000 | |
108 | ||
109 | /* Helpers for userspace vrings. */ | |
b97a8a90 | 110 | int vringh_init_user(struct vringh *vrh, u64 features, |
f87d0fbb | 111 | unsigned int num, bool weak_barriers, |
a865e420 MT |
112 | vring_desc_t __user *desc, |
113 | vring_avail_t __user *avail, | |
114 | vring_used_t __user *used); | |
f87d0fbb RR |
115 | |
116 | static inline void vringh_iov_init(struct vringh_iov *iov, | |
117 | struct iovec *iovec, unsigned num) | |
118 | { | |
119 | iov->used = iov->i = 0; | |
120 | iov->consumed = 0; | |
121 | iov->max_num = num; | |
122 | iov->iov = iovec; | |
123 | } | |
124 | ||
125 | static inline void vringh_iov_reset(struct vringh_iov *iov) | |
126 | { | |
127 | iov->iov[iov->i].iov_len += iov->consumed; | |
128 | iov->iov[iov->i].iov_base -= iov->consumed; | |
129 | iov->consumed = 0; | |
130 | iov->i = 0; | |
131 | } | |
132 | ||
133 | static inline void vringh_iov_cleanup(struct vringh_iov *iov) | |
134 | { | |
135 | if (iov->max_num & VRINGH_IOV_ALLOCATED) | |
136 | kfree(iov->iov); | |
137 | iov->max_num = iov->used = iov->i = iov->consumed = 0; | |
138 | iov->iov = NULL; | |
139 | } | |
140 | ||
141 | /* Convert a descriptor into iovecs. */ | |
142 | int vringh_getdesc_user(struct vringh *vrh, | |
143 | struct vringh_iov *riov, | |
144 | struct vringh_iov *wiov, | |
145 | bool (*getrange)(struct vringh *vrh, | |
146 | u64 addr, struct vringh_range *r), | |
147 | u16 *head); | |
148 | ||
149 | /* Copy bytes from readable vsg, consuming it (and incrementing wiov->i). */ | |
150 | ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len); | |
151 | ||
152 | /* Copy bytes into writable vsg, consuming it (and incrementing wiov->i). */ | |
153 | ssize_t vringh_iov_push_user(struct vringh_iov *wiov, | |
154 | const void *src, size_t len); | |
155 | ||
156 | /* Mark a descriptor as used. */ | |
157 | int vringh_complete_user(struct vringh *vrh, u16 head, u32 len); | |
158 | int vringh_complete_multi_user(struct vringh *vrh, | |
159 | const struct vring_used_elem used[], | |
160 | unsigned num_used); | |
161 | ||
162 | /* Pretend we've never seen descriptor (for easy error handling). */ | |
163 | void vringh_abandon_user(struct vringh *vrh, unsigned int num); | |
164 | ||
165 | /* Do we need to fire the eventfd to notify the other side? */ | |
166 | int vringh_need_notify_user(struct vringh *vrh); | |
167 | ||
168 | bool vringh_notify_enable_user(struct vringh *vrh); | |
169 | void vringh_notify_disable_user(struct vringh *vrh); | |
170 | ||
171 | /* Helpers for kernelspace vrings. */ | |
b97a8a90 | 172 | int vringh_init_kern(struct vringh *vrh, u64 features, |
f87d0fbb RR |
173 | unsigned int num, bool weak_barriers, |
174 | struct vring_desc *desc, | |
175 | struct vring_avail *avail, | |
176 | struct vring_used *used); | |
177 | ||
178 | static inline void vringh_kiov_init(struct vringh_kiov *kiov, | |
179 | struct kvec *kvec, unsigned num) | |
180 | { | |
181 | kiov->used = kiov->i = 0; | |
182 | kiov->consumed = 0; | |
183 | kiov->max_num = num; | |
184 | kiov->iov = kvec; | |
185 | } | |
186 | ||
187 | static inline void vringh_kiov_reset(struct vringh_kiov *kiov) | |
188 | { | |
189 | kiov->iov[kiov->i].iov_len += kiov->consumed; | |
190 | kiov->iov[kiov->i].iov_base -= kiov->consumed; | |
191 | kiov->consumed = 0; | |
192 | kiov->i = 0; | |
193 | } | |
194 | ||
195 | static inline void vringh_kiov_cleanup(struct vringh_kiov *kiov) | |
196 | { | |
197 | if (kiov->max_num & VRINGH_IOV_ALLOCATED) | |
198 | kfree(kiov->iov); | |
199 | kiov->max_num = kiov->used = kiov->i = kiov->consumed = 0; | |
200 | kiov->iov = NULL; | |
201 | } | |
202 | ||
14c9ac05 SG |
203 | static inline size_t vringh_kiov_length(struct vringh_kiov *kiov) |
204 | { | |
205 | size_t len = 0; | |
206 | int i; | |
207 | ||
208 | for (i = kiov->i; i < kiov->used; i++) | |
209 | len += kiov->iov[i].iov_len; | |
210 | ||
211 | return len; | |
212 | } | |
213 | ||
b8c06ad4 SG |
214 | void vringh_kiov_advance(struct vringh_kiov *kiov, size_t len); |
215 | ||
f87d0fbb RR |
216 | int vringh_getdesc_kern(struct vringh *vrh, |
217 | struct vringh_kiov *riov, | |
218 | struct vringh_kiov *wiov, | |
219 | u16 *head, | |
220 | gfp_t gfp); | |
221 | ||
222 | ssize_t vringh_iov_pull_kern(struct vringh_kiov *riov, void *dst, size_t len); | |
223 | ssize_t vringh_iov_push_kern(struct vringh_kiov *wiov, | |
224 | const void *src, size_t len); | |
225 | void vringh_abandon_kern(struct vringh *vrh, unsigned int num); | |
226 | int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len); | |
227 | ||
228 | bool vringh_notify_enable_kern(struct vringh *vrh); | |
229 | void vringh_notify_disable_kern(struct vringh *vrh); | |
230 | ||
231 | int vringh_need_notify_kern(struct vringh *vrh); | |
232 | ||
3beee86a SB |
233 | /* Notify the guest about buffers added to the used ring */ |
234 | static inline void vringh_notify(struct vringh *vrh) | |
235 | { | |
236 | if (vrh->notify) | |
237 | vrh->notify(vrh); | |
238 | } | |
239 | ||
5da7b160 GK |
240 | static inline bool vringh_is_little_endian(const struct vringh *vrh) |
241 | { | |
7d824109 GK |
242 | return vrh->little_endian || |
243 | virtio_legacy_is_little_endian(); | |
5da7b160 GK |
244 | } |
245 | ||
b9f7ac8c MT |
246 | static inline u16 vringh16_to_cpu(const struct vringh *vrh, __virtio16 val) |
247 | { | |
5da7b160 | 248 | return __virtio16_to_cpu(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
249 | } |
250 | ||
251 | static inline __virtio16 cpu_to_vringh16(const struct vringh *vrh, u16 val) | |
252 | { | |
5da7b160 | 253 | return __cpu_to_virtio16(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
254 | } |
255 | ||
256 | static inline u32 vringh32_to_cpu(const struct vringh *vrh, __virtio32 val) | |
257 | { | |
5da7b160 | 258 | return __virtio32_to_cpu(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
259 | } |
260 | ||
261 | static inline __virtio32 cpu_to_vringh32(const struct vringh *vrh, u32 val) | |
262 | { | |
5da7b160 | 263 | return __cpu_to_virtio32(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
264 | } |
265 | ||
266 | static inline u64 vringh64_to_cpu(const struct vringh *vrh, __virtio64 val) | |
267 | { | |
5da7b160 | 268 | return __virtio64_to_cpu(vringh_is_little_endian(vrh), val); |
b9f7ac8c MT |
269 | } |
270 | ||
271 | static inline __virtio64 cpu_to_vringh64(const struct vringh *vrh, u64 val) | |
272 | { | |
5da7b160 | 273 | return __cpu_to_virtio64(vringh_is_little_endian(vrh), val); |
b9f7ac8c | 274 | } |
9ad9c49c | 275 | |
3302363a MT |
276 | #if IS_REACHABLE(CONFIG_VHOST_IOTLB) |
277 | ||
f53d9910 SG |
278 | void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb, |
279 | spinlock_t *iotlb_lock); | |
9ad9c49c JW |
280 | |
281 | int vringh_init_iotlb(struct vringh *vrh, u64 features, | |
282 | unsigned int num, bool weak_barriers, | |
283 | struct vring_desc *desc, | |
284 | struct vring_avail *avail, | |
285 | struct vring_used *used); | |
286 | ||
287 | int vringh_getdesc_iotlb(struct vringh *vrh, | |
288 | struct vringh_kiov *riov, | |
289 | struct vringh_kiov *wiov, | |
290 | u16 *head, | |
291 | gfp_t gfp); | |
292 | ||
293 | ssize_t vringh_iov_pull_iotlb(struct vringh *vrh, | |
294 | struct vringh_kiov *riov, | |
295 | void *dst, size_t len); | |
296 | ssize_t vringh_iov_push_iotlb(struct vringh *vrh, | |
297 | struct vringh_kiov *wiov, | |
298 | const void *src, size_t len); | |
299 | ||
300 | void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num); | |
301 | ||
302 | int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len); | |
303 | ||
304 | bool vringh_notify_enable_iotlb(struct vringh *vrh); | |
305 | void vringh_notify_disable_iotlb(struct vringh *vrh); | |
306 | ||
307 | int vringh_need_notify_iotlb(struct vringh *vrh); | |
308 | ||
3302363a MT |
309 | #endif /* CONFIG_VHOST_IOTLB */ |
310 | ||
f87d0fbb | 311 | #endif /* _LINUX_VRINGH_H */ |