Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
4e53f78e MT |
2 | #define _GNU_SOURCE |
3 | #include <getopt.h> | |
633fae33 | 4 | #include <limits.h> |
4e53f78e MT |
5 | #include <string.h> |
6 | #include <poll.h> | |
7 | #include <sys/eventfd.h> | |
8 | #include <stdlib.h> | |
9 | #include <assert.h> | |
10 | #include <unistd.h> | |
11 | #include <sys/ioctl.h> | |
12 | #include <sys/stat.h> | |
13 | #include <sys/types.h> | |
14 | #include <fcntl.h> | |
61d0b5a4 | 15 | #include <stdbool.h> |
2d7ce0e8 | 16 | #include <linux/virtio_types.h> |
4e53f78e MT |
17 | #include <linux/vhost.h> |
18 | #include <linux/virtio.h> | |
19 | #include <linux/virtio_ring.h> | |
20 | #include "../../drivers/vhost/test.h" | |
21 | ||
7add78b2 EP |
22 | #define RANDOM_BATCH -1 |
23 | ||
61d0b5a4 RR |
24 | /* Unused */ |
25 | void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end; | |
26 | ||
4e53f78e MT |
27 | struct vq_info { |
28 | int kick; | |
29 | int call; | |
30 | int num; | |
31 | int idx; | |
32 | void *ring; | |
33 | /* copy used for control */ | |
34 | struct vring vring; | |
35 | struct virtqueue *vq; | |
36 | }; | |
37 | ||
38 | struct vdev_info { | |
39 | struct virtio_device vdev; | |
40 | int control; | |
41 | struct pollfd fds[1]; | |
42 | struct vq_info vqs[1]; | |
43 | int nvqs; | |
44 | void *buf; | |
45 | size_t buf_size; | |
46 | struct vhost_memory *mem; | |
47 | }; | |
48 | ||
264ee5aa EP |
49 | static const struct vhost_vring_file no_backend = { .fd = -1 }, |
50 | backend = { .fd = 1 }; | |
1d8bf5c3 | 51 | static const struct vhost_vring_state null_state = {}; |
264ee5aa | 52 | |
46f9c2b9 | 53 | bool vq_notify(struct virtqueue *vq) |
4e53f78e MT |
54 | { |
55 | struct vq_info *info = vq->priv; | |
56 | unsigned long long v = 1; | |
57 | int r; | |
58 | r = write(info->kick, &v, sizeof v); | |
59 | assert(r == sizeof v); | |
46f9c2b9 | 60 | return true; |
4e53f78e MT |
61 | } |
62 | ||
63 | void vq_callback(struct virtqueue *vq) | |
64 | { | |
65 | } | |
66 | ||
67 | ||
68 | void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info) | |
69 | { | |
70 | struct vhost_vring_state state = { .index = info->idx }; | |
71 | struct vhost_vring_file file = { .index = info->idx }; | |
e16e12be | 72 | unsigned long long features = dev->vdev.features; |
4e53f78e MT |
73 | struct vhost_vring_addr addr = { |
74 | .index = info->idx, | |
75 | .desc_user_addr = (uint64_t)(unsigned long)info->vring.desc, | |
76 | .avail_user_addr = (uint64_t)(unsigned long)info->vring.avail, | |
77 | .used_user_addr = (uint64_t)(unsigned long)info->vring.used, | |
78 | }; | |
79 | int r; | |
80 | r = ioctl(dev->control, VHOST_SET_FEATURES, &features); | |
81 | assert(r >= 0); | |
82 | state.num = info->vring.num; | |
83 | r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state); | |
84 | assert(r >= 0); | |
85 | state.num = 0; | |
86 | r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state); | |
87 | assert(r >= 0); | |
88 | r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr); | |
89 | assert(r >= 0); | |
90 | file.fd = info->kick; | |
91 | r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file); | |
92 | assert(r >= 0); | |
93 | file.fd = info->call; | |
94 | r = ioctl(dev->control, VHOST_SET_VRING_CALL, &file); | |
95 | assert(r >= 0); | |
96 | } | |
97 | ||
67412392 EP |
98 | static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev) |
99 | { | |
100 | if (info->vq) | |
101 | vring_del_virtqueue(info->vq); | |
102 | ||
103 | memset(info->ring, 0, vring_size(num, 4096)); | |
104 | vring_init(&info->vring, num, info->ring, 4096); | |
105 | info->vq = __vring_new_virtqueue(info->idx, info->vring, vdev, true, | |
106 | false, vq_notify, vq_callback, "test"); | |
107 | assert(info->vq); | |
108 | info->vq->priv = info; | |
109 | } | |
110 | ||
4e53f78e MT |
111 | static void vq_info_add(struct vdev_info *dev, int num) |
112 | { | |
113 | struct vq_info *info = &dev->vqs[dev->nvqs]; | |
114 | int r; | |
115 | info->idx = dev->nvqs; | |
116 | info->kick = eventfd(0, EFD_NONBLOCK); | |
117 | info->call = eventfd(0, EFD_NONBLOCK); | |
118 | r = posix_memalign(&info->ring, 4096, vring_size(num, 4096)); | |
119 | assert(r >= 0); | |
67412392 | 120 | vq_reset(info, num, &dev->vdev); |
4e53f78e MT |
121 | vhost_vq_setup(dev, info); |
122 | dev->fds[info->idx].fd = info->call; | |
123 | dev->fds[info->idx].events = POLLIN; | |
124 | dev->nvqs++; | |
125 | } | |
126 | ||
127 | static void vdev_info_init(struct vdev_info* dev, unsigned long long features) | |
128 | { | |
129 | int r; | |
130 | memset(dev, 0, sizeof *dev); | |
e16e12be | 131 | dev->vdev.features = features; |
cb91909e | 132 | INIT_LIST_HEAD(&dev->vdev.vqs); |
4e53f78e MT |
133 | dev->buf_size = 1024; |
134 | dev->buf = malloc(dev->buf_size); | |
135 | assert(dev->buf); | |
136 | dev->control = open("/dev/vhost-test", O_RDWR); | |
137 | assert(dev->control >= 0); | |
138 | r = ioctl(dev->control, VHOST_SET_OWNER, NULL); | |
139 | assert(r >= 0); | |
140 | dev->mem = malloc(offsetof(struct vhost_memory, regions) + | |
141 | sizeof dev->mem->regions[0]); | |
142 | assert(dev->mem); | |
143 | memset(dev->mem, 0, offsetof(struct vhost_memory, regions) + | |
144 | sizeof dev->mem->regions[0]); | |
145 | dev->mem->nregions = 1; | |
146 | dev->mem->regions[0].guest_phys_addr = (long)dev->buf; | |
147 | dev->mem->regions[0].userspace_addr = (long)dev->buf; | |
148 | dev->mem->regions[0].memory_size = dev->buf_size; | |
149 | r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem); | |
150 | assert(r >= 0); | |
151 | } | |
152 | ||
153 | /* TODO: this is pretty bad: we get a cache line bounce | |
154 | * for the wait queue on poll and another one on read, | |
155 | * plus the read which is there just to clear the | |
156 | * current state. */ | |
157 | static void wait_for_interrupt(struct vdev_info *dev) | |
158 | { | |
159 | int i; | |
160 | unsigned long long val; | |
161 | poll(dev->fds, dev->nvqs, -1); | |
162 | for (i = 0; i < dev->nvqs; ++i) | |
163 | if (dev->fds[i].revents & POLLIN) { | |
164 | read(dev->fds[i].fd, &val, sizeof val); | |
165 | } | |
166 | } | |
167 | ||
64d09888 | 168 | static void run_test(struct vdev_info *dev, struct vq_info *vq, |
264ee5aa | 169 | bool delayed, int batch, int reset_n, int bufs) |
4e53f78e MT |
170 | { |
171 | struct scatterlist sl; | |
264ee5aa | 172 | long started = 0, completed = 0, next_reset = reset_n; |
633fae33 | 173 | long completed_before, started_before; |
4e53f78e MT |
174 | int r, test = 1; |
175 | unsigned len; | |
176 | long long spurious = 0; | |
7add78b2 | 177 | const bool random_batch = batch == RANDOM_BATCH; |
1d8bf5c3 | 178 | |
4e53f78e MT |
179 | r = ioctl(dev->control, VHOST_TEST_RUN, &test); |
180 | assert(r >= 0); | |
1d8bf5c3 EP |
181 | if (!reset_n) { |
182 | next_reset = INT_MAX; | |
183 | } | |
184 | ||
4e53f78e MT |
185 | for (;;) { |
186 | virtqueue_disable_cb(vq->vq); | |
187 | completed_before = completed; | |
633fae33 | 188 | started_before = started; |
4e53f78e | 189 | do { |
1d8bf5c3 | 190 | const bool reset = completed > next_reset; |
7add78b2 EP |
191 | if (random_batch) |
192 | batch = (random() % vq->vring.num) + 1; | |
193 | ||
633fae33 EP |
194 | while (started < bufs && |
195 | (started - completed) < batch) { | |
4e53f78e | 196 | sg_init_one(&sl, dev->buf, dev->buf_size); |
cf994e0a RR |
197 | r = virtqueue_add_outbuf(vq->vq, &sl, 1, |
198 | dev->buf + started, | |
199 | GFP_ATOMIC); | |
633fae33 EP |
200 | if (unlikely(r != 0)) { |
201 | if (r == -ENOSPC && | |
202 | started > started_before) | |
203 | r = 0; | |
204 | else | |
53c18c99 | 205 | r = -1; |
633fae33 | 206 | break; |
4e53f78e | 207 | } |
633fae33 EP |
208 | |
209 | ++started; | |
210 | ||
211 | if (unlikely(!virtqueue_kick(vq->vq))) { | |
212 | r = -1; | |
213 | break; | |
214 | } | |
215 | } | |
216 | ||
217 | if (started >= bufs) | |
4e53f78e MT |
218 | r = -1; |
219 | ||
264ee5aa EP |
220 | if (reset) { |
221 | r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, | |
222 | &no_backend); | |
223 | assert(!r); | |
224 | } | |
225 | ||
4e53f78e | 226 | /* Flush out completed bufs if any */ |
633fae33 | 227 | while (virtqueue_get_buf(vq->vq, &len)) { |
4e53f78e MT |
228 | ++completed; |
229 | r = 0; | |
230 | } | |
231 | ||
264ee5aa | 232 | if (reset) { |
1d8bf5c3 EP |
233 | struct vhost_vring_state s = { .index = 0 }; |
234 | ||
235 | vq_reset(vq, vq->vring.num, &dev->vdev); | |
236 | ||
237 | r = ioctl(dev->control, VHOST_GET_VRING_BASE, | |
238 | &s); | |
239 | assert(!r); | |
240 | ||
241 | s.num = 0; | |
242 | r = ioctl(dev->control, VHOST_SET_VRING_BASE, | |
243 | &null_state); | |
244 | assert(!r); | |
245 | ||
264ee5aa EP |
246 | r = ioctl(dev->control, VHOST_TEST_SET_BACKEND, |
247 | &backend); | |
248 | assert(!r); | |
249 | ||
1d8bf5c3 | 250 | started = completed; |
264ee5aa EP |
251 | while (completed > next_reset) |
252 | next_reset += completed; | |
253 | } | |
de929b04 | 254 | } while (r == 0); |
633fae33 | 255 | if (completed == completed_before && started == started_before) |
4e53f78e MT |
256 | ++spurious; |
257 | assert(completed <= bufs); | |
258 | assert(started <= bufs); | |
259 | if (completed == bufs) | |
260 | break; | |
64d09888 MT |
261 | if (delayed) { |
262 | if (virtqueue_enable_cb_delayed(vq->vq)) | |
263 | wait_for_interrupt(dev); | |
264 | } else { | |
265 | if (virtqueue_enable_cb(vq->vq)) | |
266 | wait_for_interrupt(dev); | |
4e53f78e MT |
267 | } |
268 | } | |
269 | test = 0; | |
270 | r = ioctl(dev->control, VHOST_TEST_RUN, &test); | |
271 | assert(r >= 0); | |
1d8bf5c3 EP |
272 | fprintf(stderr, |
273 | "spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n", | |
274 | spurious, started, completed); | |
4e53f78e MT |
275 | } |
276 | ||
277 | const char optstring[] = "h"; | |
278 | const struct option longopts[] = { | |
279 | { | |
280 | .name = "help", | |
281 | .val = 'h', | |
282 | }, | |
4423fe40 MT |
283 | { |
284 | .name = "event-idx", | |
285 | .val = 'E', | |
286 | }, | |
287 | { | |
288 | .name = "no-event-idx", | |
289 | .val = 'e', | |
290 | }, | |
4e53f78e MT |
291 | { |
292 | .name = "indirect", | |
293 | .val = 'I', | |
294 | }, | |
295 | { | |
296 | .name = "no-indirect", | |
297 | .val = 'i', | |
298 | }, | |
43b09122 MT |
299 | { |
300 | .name = "virtio-1", | |
301 | .val = '1', | |
302 | }, | |
303 | { | |
304 | .name = "no-virtio-1", | |
305 | .val = '0', | |
306 | }, | |
64d09888 MT |
307 | { |
308 | .name = "delayed-interrupt", | |
309 | .val = 'D', | |
310 | }, | |
311 | { | |
312 | .name = "no-delayed-interrupt", | |
313 | .val = 'd', | |
314 | }, | |
633fae33 EP |
315 | { |
316 | .name = "batch", | |
317 | .val = 'b', | |
318 | .has_arg = required_argument, | |
319 | }, | |
264ee5aa EP |
320 | { |
321 | .name = "reset", | |
322 | .val = 'r', | |
323 | .has_arg = optional_argument, | |
324 | }, | |
4e53f78e MT |
325 | { |
326 | } | |
327 | }; | |
328 | ||
4a7d6455 | 329 | static void help(void) |
4e53f78e | 330 | { |
4423fe40 MT |
331 | fprintf(stderr, "Usage: virtio_test [--help]" |
332 | " [--no-indirect]" | |
333 | " [--no-event-idx]" | |
43b09122 | 334 | " [--no-virtio-1]" |
64d09888 | 335 | " [--delayed-interrupt]" |
7add78b2 | 336 | " [--batch=random/N]" |
264ee5aa | 337 | " [--reset=N]" |
4423fe40 | 338 | "\n"); |
4e53f78e MT |
339 | } |
340 | ||
341 | int main(int argc, char **argv) | |
342 | { | |
343 | struct vdev_info dev; | |
4423fe40 | 344 | unsigned long long features = (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | |
43b09122 | 345 | (1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_F_VERSION_1); |
264ee5aa | 346 | long batch = 1, reset = 0; |
4e53f78e | 347 | int o; |
64d09888 | 348 | bool delayed = false; |
4e53f78e MT |
349 | |
350 | for (;;) { | |
351 | o = getopt_long(argc, argv, optstring, longopts, NULL); | |
352 | switch (o) { | |
353 | case -1: | |
354 | goto done; | |
355 | case '?': | |
356 | help(); | |
357 | exit(2); | |
4423fe40 MT |
358 | case 'e': |
359 | features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX); | |
360 | break; | |
4e53f78e MT |
361 | case 'h': |
362 | help(); | |
363 | goto done; | |
364 | case 'i': | |
365 | features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC); | |
366 | break; | |
43b09122 MT |
367 | case '0': |
368 | features &= ~(1ULL << VIRTIO_F_VERSION_1); | |
369 | break; | |
64d09888 MT |
370 | case 'D': |
371 | delayed = true; | |
372 | break; | |
633fae33 | 373 | case 'b': |
7add78b2 EP |
374 | if (0 == strcmp(optarg, "random")) { |
375 | batch = RANDOM_BATCH; | |
376 | } else { | |
377 | batch = strtol(optarg, NULL, 10); | |
378 | assert(batch > 0); | |
379 | assert(batch < (long)INT_MAX + 1); | |
380 | } | |
633fae33 | 381 | break; |
264ee5aa EP |
382 | case 'r': |
383 | if (!optarg) { | |
384 | reset = 1; | |
385 | } else { | |
386 | reset = strtol(optarg, NULL, 10); | |
387 | assert(reset > 0); | |
388 | assert(reset < (long)INT_MAX + 1); | |
389 | } | |
390 | break; | |
4e53f78e MT |
391 | default: |
392 | assert(0); | |
393 | break; | |
394 | } | |
395 | } | |
396 | ||
397 | done: | |
398 | vdev_info_init(&dev, features); | |
399 | vq_info_add(&dev, 256); | |
264ee5aa | 400 | run_test(&dev, &dev.vqs[0], delayed, batch, reset, 0x100000); |
4e53f78e MT |
401 | return 0; |
402 | } |