Commit | Line | Data |
---|---|---|
7a338472 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
71ccc212 MT |
2 | /* Copyright (C) 2009 Red Hat, Inc. |
3 | * Author: Michael S. Tsirkin <mst@redhat.com> | |
4 | * | |
71ccc212 MT |
5 | * test virtio server in host kernel. |
6 | */ | |
7 | ||
8 | #include <linux/compat.h> | |
9 | #include <linux/eventfd.h> | |
10 | #include <linux/vhost.h> | |
11 | #include <linux/miscdevice.h> | |
12 | #include <linux/module.h> | |
13 | #include <linux/mutex.h> | |
14 | #include <linux/workqueue.h> | |
71ccc212 MT |
15 | #include <linux/file.h> |
16 | #include <linux/slab.h> | |
17 | ||
18 | #include "test.h" | |
6ac1afbf | 19 | #include "vhost.h" |
71ccc212 MT |
20 | |
21 | /* Max number of bytes transferred before requeueing the job. | |
22 | * Using this limit prevents one virtqueue from starving others. */ | |
23 | #define VHOST_TEST_WEIGHT 0x80000 | |
24 | ||
264b563b TB |
25 | /* Max number of packets transferred before requeueing the job. |
26 | * Using this limit prevents one virtqueue from starving others with | |
27 | * pkts. | |
28 | */ | |
29 | #define VHOST_TEST_PKT_WEIGHT 256 | |
30 | ||
71ccc212 MT |
31 | enum { |
32 | VHOST_TEST_VQ = 0, | |
33 | VHOST_TEST_VQ_MAX = 1, | |
34 | }; | |
35 | ||
36 | struct vhost_test { | |
37 | struct vhost_dev dev; | |
38 | struct vhost_virtqueue vqs[VHOST_TEST_VQ_MAX]; | |
39 | }; | |
40 | ||
41 | /* Expects to be always run from workqueue - which acts as | |
42 | * read-size critical section for our kind of RCU. */ | |
43 | static void handle_vq(struct vhost_test *n) | |
44 | { | |
09a34c84 | 45 | struct vhost_virtqueue *vq = &n->vqs[VHOST_TEST_VQ]; |
71ccc212 MT |
46 | unsigned out, in; |
47 | int head; | |
48 | size_t len, total_len = 0; | |
49 | void *private; | |
50 | ||
09a34c84 | 51 | mutex_lock(&vq->mutex); |
247643f8 | 52 | private = vhost_vq_get_backend(vq); |
09a34c84 MT |
53 | if (!private) { |
54 | mutex_unlock(&vq->mutex); | |
71ccc212 | 55 | return; |
09a34c84 | 56 | } |
71ccc212 | 57 | |
8ea8cf89 | 58 | vhost_disable_notify(&n->dev, vq); |
71ccc212 MT |
59 | |
60 | for (;;) { | |
47283bef | 61 | head = vhost_get_vq_desc(vq, vq->iov, |
71ccc212 MT |
62 | ARRAY_SIZE(vq->iov), |
63 | &out, &in, | |
64 | NULL, NULL); | |
65 | /* On error, stop handling until the next kick. */ | |
66 | if (unlikely(head < 0)) | |
67 | break; | |
68 | /* Nothing new? Wait for eventfd to tell us they refilled. */ | |
69 | if (head == vq->num) { | |
8ea8cf89 MT |
70 | if (unlikely(vhost_enable_notify(&n->dev, vq))) { |
71 | vhost_disable_notify(&n->dev, vq); | |
71ccc212 MT |
72 | continue; |
73 | } | |
74 | break; | |
75 | } | |
76 | if (in) { | |
77 | vq_err(vq, "Unexpected descriptor format for TX: " | |
78 | "out %d, int %d\n", out, in); | |
79 | break; | |
80 | } | |
81 | len = iov_length(vq->iov, out); | |
82 | /* Sanity check */ | |
83 | if (!len) { | |
84 | vq_err(vq, "Unexpected 0 len for TX\n"); | |
85 | break; | |
86 | } | |
87 | vhost_add_used_and_signal(&n->dev, vq, head, 0); | |
88 | total_len += len; | |
264b563b | 89 | if (unlikely(vhost_exceeds_weight(vq, 0, total_len))) |
71ccc212 | 90 | break; |
71ccc212 MT |
91 | } |
92 | ||
93 | mutex_unlock(&vq->mutex); | |
94 | } | |
95 | ||
96 | static void handle_vq_kick(struct vhost_work *work) | |
97 | { | |
98 | struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, | |
99 | poll.work); | |
100 | struct vhost_test *n = container_of(vq->dev, struct vhost_test, dev); | |
101 | ||
102 | handle_vq(n); | |
103 | } | |
104 | ||
105 | static int vhost_test_open(struct inode *inode, struct file *f) | |
106 | { | |
107 | struct vhost_test *n = kmalloc(sizeof *n, GFP_KERNEL); | |
108 | struct vhost_dev *dev; | |
09a34c84 | 109 | struct vhost_virtqueue **vqs; |
71ccc212 MT |
110 | |
111 | if (!n) | |
112 | return -ENOMEM; | |
6da2ec56 | 113 | vqs = kmalloc_array(VHOST_TEST_VQ_MAX, sizeof(*vqs), GFP_KERNEL); |
09a34c84 MT |
114 | if (!vqs) { |
115 | kfree(n); | |
116 | return -ENOMEM; | |
117 | } | |
71ccc212 MT |
118 | |
119 | dev = &n->dev; | |
09a34c84 | 120 | vqs[VHOST_TEST_VQ] = &n->vqs[VHOST_TEST_VQ]; |
71ccc212 | 121 | n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick; |
264b563b | 122 | vhost_dev_init(dev, vqs, VHOST_TEST_VQ_MAX, UIO_MAXIOV, |
044e4b09 | 123 | VHOST_TEST_PKT_WEIGHT, VHOST_TEST_WEIGHT, true, NULL); |
71ccc212 MT |
124 | |
125 | f->private_data = n; | |
126 | ||
127 | return 0; | |
128 | } | |
129 | ||
130 | static void *vhost_test_stop_vq(struct vhost_test *n, | |
131 | struct vhost_virtqueue *vq) | |
132 | { | |
133 | void *private; | |
134 | ||
135 | mutex_lock(&vq->mutex); | |
247643f8 EP |
136 | private = vhost_vq_get_backend(vq); |
137 | vhost_vq_set_backend(vq, NULL); | |
71ccc212 MT |
138 | mutex_unlock(&vq->mutex); |
139 | return private; | |
140 | } | |
141 | ||
142 | static void vhost_test_stop(struct vhost_test *n, void **privatep) | |
143 | { | |
144 | *privatep = vhost_test_stop_vq(n, n->vqs + VHOST_TEST_VQ); | |
145 | } | |
146 | ||
71ccc212 MT |
147 | static void vhost_test_flush(struct vhost_test *n) |
148 | { | |
b2ffa407 | 149 | vhost_dev_flush(&n->dev); |
71ccc212 MT |
150 | } |
151 | ||
152 | static int vhost_test_release(struct inode *inode, struct file *f) | |
153 | { | |
154 | struct vhost_test *n = f->private_data; | |
155 | void *private; | |
156 | ||
157 | vhost_test_stop(n, &private); | |
158 | vhost_test_flush(n); | |
245cdd9f | 159 | vhost_dev_stop(&n->dev); |
f6f93f75 | 160 | vhost_dev_cleanup(&n->dev); |
08006392 | 161 | kfree(n->dev.vqs); |
71ccc212 MT |
162 | kfree(n); |
163 | return 0; | |
164 | } | |
165 | ||
166 | static long vhost_test_run(struct vhost_test *n, int test) | |
167 | { | |
168 | void *priv, *oldpriv; | |
169 | struct vhost_virtqueue *vq; | |
170 | int r, index; | |
171 | ||
172 | if (test < 0 || test > 1) | |
173 | return -EINVAL; | |
174 | ||
175 | mutex_lock(&n->dev.mutex); | |
176 | r = vhost_dev_check_owner(&n->dev); | |
177 | if (r) | |
178 | goto err; | |
179 | ||
180 | for (index = 0; index < n->dev.nvqs; ++index) { | |
181 | /* Verify that ring has been setup correctly. */ | |
182 | if (!vhost_vq_access_ok(&n->vqs[index])) { | |
183 | r = -EFAULT; | |
184 | goto err; | |
185 | } | |
186 | } | |
187 | ||
188 | for (index = 0; index < n->dev.nvqs; ++index) { | |
189 | vq = n->vqs + index; | |
190 | mutex_lock(&vq->mutex); | |
191 | priv = test ? n : NULL; | |
192 | ||
193 | /* start polling new socket */ | |
247643f8 EP |
194 | oldpriv = vhost_vq_get_backend(vq); |
195 | vhost_vq_set_backend(vq, priv); | |
71ccc212 | 196 | |
80f7d030 | 197 | r = vhost_vq_init_access(&n->vqs[index]); |
f59281da | 198 | |
71ccc212 MT |
199 | mutex_unlock(&vq->mutex); |
200 | ||
f59281da JW |
201 | if (r) |
202 | goto err; | |
203 | ||
71ccc212 | 204 | if (oldpriv) { |
c5514758 | 205 | vhost_test_flush(n); |
71ccc212 MT |
206 | } |
207 | } | |
208 | ||
209 | mutex_unlock(&n->dev.mutex); | |
210 | return 0; | |
211 | ||
212 | err: | |
213 | mutex_unlock(&n->dev.mutex); | |
214 | return r; | |
215 | } | |
216 | ||
217 | static long vhost_test_reset_owner(struct vhost_test *n) | |
218 | { | |
219 | void *priv = NULL; | |
220 | long err; | |
3302363a | 221 | struct vhost_iotlb *umem; |
150b9e51 | 222 | |
71ccc212 MT |
223 | mutex_lock(&n->dev.mutex); |
224 | err = vhost_dev_check_owner(&n->dev); | |
225 | if (err) | |
226 | goto done; | |
446374d7 MT |
227 | umem = vhost_dev_reset_owner_prepare(); |
228 | if (!umem) { | |
150b9e51 MT |
229 | err = -ENOMEM; |
230 | goto done; | |
231 | } | |
71ccc212 MT |
232 | vhost_test_stop(n, &priv); |
233 | vhost_test_flush(n); | |
245cdd9f | 234 | vhost_dev_stop(&n->dev); |
446374d7 | 235 | vhost_dev_reset_owner(&n->dev, umem); |
71ccc212 MT |
236 | done: |
237 | mutex_unlock(&n->dev.mutex); | |
238 | return err; | |
239 | } | |
240 | ||
241 | static int vhost_test_set_features(struct vhost_test *n, u64 features) | |
242 | { | |
ea16c514 MT |
243 | struct vhost_virtqueue *vq; |
244 | ||
71ccc212 MT |
245 | mutex_lock(&n->dev.mutex); |
246 | if ((features & (1 << VHOST_F_LOG_ALL)) && | |
247 | !vhost_log_access_ok(&n->dev)) { | |
248 | mutex_unlock(&n->dev.mutex); | |
249 | return -EFAULT; | |
250 | } | |
ea16c514 MT |
251 | vq = &n->vqs[VHOST_TEST_VQ]; |
252 | mutex_lock(&vq->mutex); | |
253 | vq->acked_features = features; | |
254 | mutex_unlock(&vq->mutex); | |
71ccc212 MT |
255 | mutex_unlock(&n->dev.mutex); |
256 | return 0; | |
257 | } | |
258 | ||
264ee5aa EP |
259 | static long vhost_test_set_backend(struct vhost_test *n, unsigned index, int fd) |
260 | { | |
261 | static void *backend; | |
262 | ||
263 | const bool enable = fd != -1; | |
264 | struct vhost_virtqueue *vq; | |
265 | int r; | |
266 | ||
267 | mutex_lock(&n->dev.mutex); | |
268 | r = vhost_dev_check_owner(&n->dev); | |
269 | if (r) | |
270 | goto err; | |
271 | ||
272 | if (index >= VHOST_TEST_VQ_MAX) { | |
273 | r = -ENOBUFS; | |
274 | goto err; | |
275 | } | |
276 | vq = &n->vqs[index]; | |
277 | mutex_lock(&vq->mutex); | |
278 | ||
279 | /* Verify that ring has been setup correctly. */ | |
280 | if (!vhost_vq_access_ok(vq)) { | |
281 | r = -EFAULT; | |
282 | goto err_vq; | |
283 | } | |
284 | if (!enable) { | |
285 | vhost_poll_stop(&vq->poll); | |
286 | backend = vhost_vq_get_backend(vq); | |
287 | vhost_vq_set_backend(vq, NULL); | |
288 | } else { | |
289 | vhost_vq_set_backend(vq, backend); | |
290 | r = vhost_vq_init_access(vq); | |
291 | if (r == 0) | |
292 | r = vhost_poll_start(&vq->poll, vq->kick); | |
293 | } | |
294 | ||
295 | mutex_unlock(&vq->mutex); | |
296 | ||
297 | if (enable) { | |
c5514758 | 298 | vhost_test_flush(n); |
264ee5aa EP |
299 | } |
300 | ||
301 | mutex_unlock(&n->dev.mutex); | |
302 | return 0; | |
303 | ||
304 | err_vq: | |
305 | mutex_unlock(&vq->mutex); | |
306 | err: | |
307 | mutex_unlock(&n->dev.mutex); | |
308 | return r; | |
309 | } | |
310 | ||
71ccc212 MT |
311 | static long vhost_test_ioctl(struct file *f, unsigned int ioctl, |
312 | unsigned long arg) | |
313 | { | |
264ee5aa | 314 | struct vhost_vring_file backend; |
71ccc212 MT |
315 | struct vhost_test *n = f->private_data; |
316 | void __user *argp = (void __user *)arg; | |
317 | u64 __user *featurep = argp; | |
318 | int test; | |
319 | u64 features; | |
320 | int r; | |
321 | switch (ioctl) { | |
322 | case VHOST_TEST_RUN: | |
323 | if (copy_from_user(&test, argp, sizeof test)) | |
324 | return -EFAULT; | |
325 | return vhost_test_run(n, test); | |
264ee5aa EP |
326 | case VHOST_TEST_SET_BACKEND: |
327 | if (copy_from_user(&backend, argp, sizeof backend)) | |
328 | return -EFAULT; | |
329 | return vhost_test_set_backend(n, backend.index, backend.fd); | |
71ccc212 | 330 | case VHOST_GET_FEATURES: |
09a34c84 | 331 | features = VHOST_FEATURES; |
71ccc212 MT |
332 | if (copy_to_user(featurep, &features, sizeof features)) |
333 | return -EFAULT; | |
334 | return 0; | |
335 | case VHOST_SET_FEATURES: | |
336 | if (copy_from_user(&features, featurep, sizeof features)) | |
337 | return -EFAULT; | |
09a34c84 | 338 | if (features & ~VHOST_FEATURES) |
71ccc212 MT |
339 | return -EOPNOTSUPP; |
340 | return vhost_test_set_features(n, features); | |
341 | case VHOST_RESET_OWNER: | |
342 | return vhost_test_reset_owner(n); | |
343 | default: | |
344 | mutex_lock(&n->dev.mutex); | |
73640c99 MT |
345 | r = vhost_dev_ioctl(&n->dev, ioctl, argp); |
346 | if (r == -ENOIOCTLCMD) | |
347 | r = vhost_vring_ioctl(&n->dev, ioctl, argp); | |
71ccc212 MT |
348 | vhost_test_flush(n); |
349 | mutex_unlock(&n->dev.mutex); | |
350 | return r; | |
351 | } | |
352 | } | |
353 | ||
71ccc212 MT |
354 | static const struct file_operations vhost_test_fops = { |
355 | .owner = THIS_MODULE, | |
356 | .release = vhost_test_release, | |
357 | .unlocked_ioctl = vhost_test_ioctl, | |
407e9ef7 | 358 | .compat_ioctl = compat_ptr_ioctl, |
71ccc212 MT |
359 | .open = vhost_test_open, |
360 | .llseek = noop_llseek, | |
361 | }; | |
362 | ||
363 | static struct miscdevice vhost_test_misc = { | |
364 | MISC_DYNAMIC_MINOR, | |
365 | "vhost-test", | |
366 | &vhost_test_fops, | |
367 | }; | |
ca75d601 | 368 | module_misc_device(vhost_test_misc); |
71ccc212 MT |
369 | |
370 | MODULE_VERSION("0.0.1"); | |
371 | MODULE_LICENSE("GPL v2"); | |
372 | MODULE_AUTHOR("Michael S. Tsirkin"); | |
373 | MODULE_DESCRIPTION("Host kernel side for virtio simulator"); |