Commit | Line | Data |
---|---|---|
7a338472 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
0ea9e1d3 AH |
2 | /* |
3 | * virtio transport for vsock | |
4 | * | |
5 | * Copyright (C) 2013-2015 Red Hat, Inc. | |
6 | * Author: Asias He <asias@redhat.com> | |
7 | * Stefan Hajnoczi <stefanha@redhat.com> | |
8 | * | |
9 | * Some of the code is take from Gerd Hoffmann <kraxel@redhat.com>'s | |
10 | * early virtio-vsock proof-of-concept bits. | |
0ea9e1d3 AH |
11 | */ |
12 | #include <linux/spinlock.h> | |
13 | #include <linux/module.h> | |
14 | #include <linux/list.h> | |
15 | #include <linux/atomic.h> | |
16 | #include <linux/virtio.h> | |
17 | #include <linux/virtio_ids.h> | |
18 | #include <linux/virtio_config.h> | |
19 | #include <linux/virtio_vsock.h> | |
20 | #include <net/sock.h> | |
21 | #include <linux/mutex.h> | |
22 | #include <net/af_vsock.h> | |
23 | ||
24 | static struct workqueue_struct *virtio_vsock_workqueue; | |
f961134a | 25 | static struct virtio_vsock __rcu *the_virtio_vsock; |
0ea9e1d3 | 26 | static DEFINE_MUTEX(the_virtio_vsock_mutex); /* protects the_virtio_vsock */ |
8e6ed963 | 27 | static struct virtio_transport virtio_transport; /* forward declaration */ |
0ea9e1d3 AH |
28 | |
29 | struct virtio_vsock { | |
30 | struct virtio_device *vdev; | |
31 | struct virtqueue *vqs[VSOCK_VQ_MAX]; | |
32 | ||
33 | /* Virtqueue processing is deferred to a workqueue */ | |
34 | struct work_struct tx_work; | |
35 | struct work_struct rx_work; | |
36 | struct work_struct event_work; | |
37 | ||
38 | /* The following fields are protected by tx_lock. vqs[VSOCK_VQ_TX] | |
39 | * must be accessed with tx_lock held. | |
40 | */ | |
41 | struct mutex tx_lock; | |
b917507e | 42 | bool tx_run; |
0ea9e1d3 AH |
43 | |
44 | struct work_struct send_pkt_work; | |
71dc9ec9 | 45 | struct sk_buff_head send_pkt_queue; |
0ea9e1d3 AH |
46 | |
47 | atomic_t queued_replies; | |
48 | ||
49 | /* The following fields are protected by rx_lock. vqs[VSOCK_VQ_RX] | |
50 | * must be accessed with rx_lock held. | |
51 | */ | |
52 | struct mutex rx_lock; | |
b917507e | 53 | bool rx_run; |
0ea9e1d3 AH |
54 | int rx_buf_nr; |
55 | int rx_buf_max_nr; | |
56 | ||
57 | /* The following fields are protected by event_lock. | |
58 | * vqs[VSOCK_VQ_EVENT] must be accessed with event_lock held. | |
59 | */ | |
60 | struct mutex event_lock; | |
b917507e | 61 | bool event_run; |
0ea9e1d3 AH |
62 | struct virtio_vsock_event event_list[8]; |
63 | ||
64 | u32 guest_cid; | |
53efbba1 | 65 | bool seqpacket_allow; |
64c99d2d AK |
66 | |
67 | /* These fields are used only in tx path in function | |
68 | * 'virtio_transport_send_pkt_work()', so to save | |
69 | * stack space in it, place both of them here. Each | |
70 | * pointer from 'out_sgs' points to the corresponding | |
71 | * element in 'out_bufs' - this is initialized in | |
72 | * 'virtio_vsock_probe()'. Both fields are protected | |
73 | * by 'tx_lock'. +1 is needed for packet header. | |
74 | */ | |
75 | struct scatterlist *out_sgs[MAX_SKB_FRAGS + 1]; | |
76 | struct scatterlist out_bufs[MAX_SKB_FRAGS + 1]; | |
0ea9e1d3 AH |
77 | }; |
78 | ||
0ea9e1d3 AH |
79 | static u32 virtio_transport_get_local_cid(void) |
80 | { | |
0deab087 SG |
81 | struct virtio_vsock *vsock; |
82 | u32 ret; | |
0ea9e1d3 | 83 | |
0deab087 SG |
84 | rcu_read_lock(); |
85 | vsock = rcu_dereference(the_virtio_vsock); | |
86 | if (!vsock) { | |
87 | ret = VMADDR_CID_ANY; | |
88 | goto out_rcu; | |
89 | } | |
22b5c0b6 | 90 | |
0deab087 SG |
91 | ret = vsock->guest_cid; |
92 | out_rcu: | |
93 | rcu_read_unlock(); | |
94 | return ret; | |
0ea9e1d3 AH |
95 | } |
96 | ||
97 | static void | |
98 | virtio_transport_send_pkt_work(struct work_struct *work) | |
99 | { | |
100 | struct virtio_vsock *vsock = | |
101 | container_of(work, struct virtio_vsock, send_pkt_work); | |
102 | struct virtqueue *vq; | |
103 | bool added = false; | |
104 | bool restart_rx = false; | |
105 | ||
106 | mutex_lock(&vsock->tx_lock); | |
107 | ||
b917507e SG |
108 | if (!vsock->tx_run) |
109 | goto out; | |
110 | ||
0ea9e1d3 AH |
111 | vq = vsock->vqs[VSOCK_VQ_TX]; |
112 | ||
0ea9e1d3 | 113 | for (;;) { |
0ea9e1d3 | 114 | int ret, in_sg = 0, out_sg = 0; |
64c99d2d | 115 | struct scatterlist **sgs; |
71dc9ec9 | 116 | struct sk_buff *skb; |
0ea9e1d3 AH |
117 | bool reply; |
118 | ||
71dc9ec9 BE |
119 | skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue); |
120 | if (!skb) | |
0ea9e1d3 | 121 | break; |
0ea9e1d3 | 122 | |
71dc9ec9 BE |
123 | virtio_transport_deliver_tap_pkt(skb); |
124 | reply = virtio_vsock_skb_reply(skb); | |
64c99d2d AK |
125 | sgs = vsock->out_sgs; |
126 | sg_init_one(sgs[out_sg], virtio_vsock_hdr(skb), | |
127 | sizeof(*virtio_vsock_hdr(skb))); | |
128 | out_sg++; | |
129 | ||
130 | if (!skb_is_nonlinear(skb)) { | |
131 | if (skb->len > 0) { | |
132 | sg_init_one(sgs[out_sg], skb->data, skb->len); | |
133 | out_sg++; | |
134 | } | |
135 | } else { | |
136 | struct skb_shared_info *si; | |
137 | int i; | |
138 | ||
139 | /* If skb is nonlinear, then its buffer must contain | |
140 | * only header and nothing more. Data is stored in | |
141 | * the fragged part. | |
142 | */ | |
143 | WARN_ON_ONCE(skb_headroom(skb) != sizeof(*virtio_vsock_hdr(skb))); | |
144 | ||
145 | si = skb_shinfo(skb); | |
146 | ||
147 | for (i = 0; i < si->nr_frags; i++) { | |
148 | skb_frag_t *skb_frag = &si->frags[i]; | |
149 | void *va; | |
150 | ||
151 | /* We will use 'page_to_virt()' for the userspace page | |
152 | * here, because virtio or dma-mapping layers will call | |
153 | * 'virt_to_phys()' later to fill the buffer descriptor. | |
154 | * We don't touch memory at "virtual" address of this page. | |
155 | */ | |
156 | va = page_to_virt(skb_frag->bv_page); | |
157 | sg_init_one(sgs[out_sg], | |
158 | va + skb_frag->bv_offset, | |
159 | skb_frag->bv_len); | |
160 | out_sg++; | |
161 | } | |
0ea9e1d3 AH |
162 | } |
163 | ||
71dc9ec9 | 164 | ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, skb, GFP_KERNEL); |
21bc54fc GG |
165 | /* Usually this means that there is no more space available in |
166 | * the vq | |
167 | */ | |
0ea9e1d3 | 168 | if (ret < 0) { |
71dc9ec9 | 169 | virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb); |
0ea9e1d3 AH |
170 | break; |
171 | } | |
172 | ||
173 | if (reply) { | |
174 | struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX]; | |
175 | int val; | |
176 | ||
177 | val = atomic_dec_return(&vsock->queued_replies); | |
178 | ||
179 | /* Do we now have resources to resume rx processing? */ | |
180 | if (val + 1 == virtqueue_get_vring_size(rx_vq)) | |
181 | restart_rx = true; | |
182 | } | |
183 | ||
184 | added = true; | |
185 | } | |
186 | ||
187 | if (added) | |
188 | virtqueue_kick(vq); | |
189 | ||
b917507e | 190 | out: |
0ea9e1d3 AH |
191 | mutex_unlock(&vsock->tx_lock); |
192 | ||
193 | if (restart_rx) | |
194 | queue_work(virtio_vsock_workqueue, &vsock->rx_work); | |
195 | } | |
196 | ||
197 | static int | |
71dc9ec9 | 198 | virtio_transport_send_pkt(struct sk_buff *skb) |
0ea9e1d3 | 199 | { |
71dc9ec9 | 200 | struct virtio_vsock_hdr *hdr; |
0ea9e1d3 | 201 | struct virtio_vsock *vsock; |
71dc9ec9 BE |
202 | int len = skb->len; |
203 | ||
204 | hdr = virtio_vsock_hdr(skb); | |
0ea9e1d3 | 205 | |
0deab087 SG |
206 | rcu_read_lock(); |
207 | vsock = rcu_dereference(the_virtio_vsock); | |
0ea9e1d3 | 208 | if (!vsock) { |
71dc9ec9 | 209 | kfree_skb(skb); |
0deab087 SG |
210 | len = -ENODEV; |
211 | goto out_rcu; | |
0ea9e1d3 AH |
212 | } |
213 | ||
71dc9ec9 BE |
214 | if (le64_to_cpu(hdr->dst_cid) == vsock->guest_cid) { |
215 | kfree_skb(skb); | |
bf5432b1 | 216 | len = -ENODEV; |
0deab087 SG |
217 | goto out_rcu; |
218 | } | |
b9116823 | 219 | |
71dc9ec9 | 220 | if (virtio_vsock_skb_reply(skb)) |
0ea9e1d3 AH |
221 | atomic_inc(&vsock->queued_replies); |
222 | ||
71dc9ec9 | 223 | virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb); |
0ea9e1d3 | 224 | queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work); |
0deab087 SG |
225 | |
226 | out_rcu: | |
227 | rcu_read_unlock(); | |
0ea9e1d3 AH |
228 | return len; |
229 | } | |
230 | ||
073b4f2c PT |
231 | static int |
232 | virtio_transport_cancel_pkt(struct vsock_sock *vsk) | |
233 | { | |
234 | struct virtio_vsock *vsock; | |
0deab087 | 235 | int cnt = 0, ret; |
073b4f2c | 236 | |
0deab087 SG |
237 | rcu_read_lock(); |
238 | vsock = rcu_dereference(the_virtio_vsock); | |
073b4f2c | 239 | if (!vsock) { |
0deab087 SG |
240 | ret = -ENODEV; |
241 | goto out_rcu; | |
073b4f2c PT |
242 | } |
243 | ||
71dc9ec9 | 244 | cnt = virtio_transport_purge_skbs(vsk, &vsock->send_pkt_queue); |
073b4f2c PT |
245 | |
246 | if (cnt) { | |
247 | struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX]; | |
248 | int new_cnt; | |
249 | ||
250 | new_cnt = atomic_sub_return(cnt, &vsock->queued_replies); | |
251 | if (new_cnt + cnt >= virtqueue_get_vring_size(rx_vq) && | |
252 | new_cnt < virtqueue_get_vring_size(rx_vq)) | |
253 | queue_work(virtio_vsock_workqueue, &vsock->rx_work); | |
254 | } | |
255 | ||
0deab087 SG |
256 | ret = 0; |
257 | ||
258 | out_rcu: | |
259 | rcu_read_unlock(); | |
260 | return ret; | |
073b4f2c PT |
261 | } |
262 | ||
0ea9e1d3 AH |
263 | static void virtio_vsock_rx_fill(struct virtio_vsock *vsock) |
264 | { | |
71dc9ec9 BE |
265 | int total_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE + VIRTIO_VSOCK_SKB_HEADROOM; |
266 | struct scatterlist pkt, *p; | |
0ea9e1d3 | 267 | struct virtqueue *vq; |
71dc9ec9 | 268 | struct sk_buff *skb; |
0ea9e1d3 AH |
269 | int ret; |
270 | ||
271 | vq = vsock->vqs[VSOCK_VQ_RX]; | |
272 | ||
273 | do { | |
71dc9ec9 BE |
274 | skb = virtio_vsock_alloc_skb(total_len, GFP_KERNEL); |
275 | if (!skb) | |
0ea9e1d3 AH |
276 | break; |
277 | ||
71dc9ec9 BE |
278 | memset(skb->head, 0, VIRTIO_VSOCK_SKB_HEADROOM); |
279 | sg_init_one(&pkt, virtio_vsock_hdr(skb), total_len); | |
280 | p = &pkt; | |
281 | ret = virtqueue_add_sgs(vq, &p, 0, 1, skb, GFP_KERNEL); | |
282 | if (ret < 0) { | |
283 | kfree_skb(skb); | |
0ea9e1d3 AH |
284 | break; |
285 | } | |
286 | ||
0ea9e1d3 AH |
287 | vsock->rx_buf_nr++; |
288 | } while (vq->num_free); | |
289 | if (vsock->rx_buf_nr > vsock->rx_buf_max_nr) | |
290 | vsock->rx_buf_max_nr = vsock->rx_buf_nr; | |
291 | virtqueue_kick(vq); | |
292 | } | |
293 | ||
294 | static void virtio_transport_tx_work(struct work_struct *work) | |
295 | { | |
296 | struct virtio_vsock *vsock = | |
297 | container_of(work, struct virtio_vsock, tx_work); | |
298 | struct virtqueue *vq; | |
299 | bool added = false; | |
300 | ||
301 | vq = vsock->vqs[VSOCK_VQ_TX]; | |
302 | mutex_lock(&vsock->tx_lock); | |
b917507e SG |
303 | |
304 | if (!vsock->tx_run) | |
305 | goto out; | |
306 | ||
0ea9e1d3 | 307 | do { |
71dc9ec9 | 308 | struct sk_buff *skb; |
0ea9e1d3 AH |
309 | unsigned int len; |
310 | ||
311 | virtqueue_disable_cb(vq); | |
71dc9ec9 BE |
312 | while ((skb = virtqueue_get_buf(vq, &len)) != NULL) { |
313 | consume_skb(skb); | |
0ea9e1d3 AH |
314 | added = true; |
315 | } | |
316 | } while (!virtqueue_enable_cb(vq)); | |
b917507e SG |
317 | |
318 | out: | |
0ea9e1d3 AH |
319 | mutex_unlock(&vsock->tx_lock); |
320 | ||
321 | if (added) | |
322 | queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work); | |
323 | } | |
324 | ||
325 | /* Is there space left for replies to rx packets? */ | |
326 | static bool virtio_transport_more_replies(struct virtio_vsock *vsock) | |
327 | { | |
328 | struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX]; | |
329 | int val; | |
330 | ||
331 | smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */ | |
332 | val = atomic_read(&vsock->queued_replies); | |
333 | ||
334 | return val < virtqueue_get_vring_size(vq); | |
335 | } | |
336 | ||
0ea9e1d3 AH |
337 | /* event_lock must be held */ |
338 | static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock, | |
339 | struct virtio_vsock_event *event) | |
340 | { | |
341 | struct scatterlist sg; | |
342 | struct virtqueue *vq; | |
343 | ||
344 | vq = vsock->vqs[VSOCK_VQ_EVENT]; | |
345 | ||
346 | sg_init_one(&sg, event, sizeof(*event)); | |
347 | ||
348 | return virtqueue_add_inbuf(vq, &sg, 1, event, GFP_KERNEL); | |
349 | } | |
350 | ||
351 | /* event_lock must be held */ | |
352 | static void virtio_vsock_event_fill(struct virtio_vsock *vsock) | |
353 | { | |
354 | size_t i; | |
355 | ||
356 | for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) { | |
357 | struct virtio_vsock_event *event = &vsock->event_list[i]; | |
358 | ||
359 | virtio_vsock_event_fill_one(vsock, event); | |
360 | } | |
361 | ||
362 | virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]); | |
363 | } | |
364 | ||
365 | static void virtio_vsock_reset_sock(struct sock *sk) | |
366 | { | |
49b0b6ff LM |
367 | /* vmci_transport.c doesn't take sk_lock here either. At least we're |
368 | * under vsock_table_lock so the sock cannot disappear while we're | |
369 | * executing. | |
370 | */ | |
371 | ||
3b4477d2 | 372 | sk->sk_state = TCP_CLOSE; |
0ea9e1d3 | 373 | sk->sk_err = ECONNRESET; |
e3ae2365 | 374 | sk_error_report(sk); |
0ea9e1d3 AH |
375 | } |
376 | ||
377 | static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock) | |
378 | { | |
379 | struct virtio_device *vdev = vsock->vdev; | |
6c7efafd | 380 | __le64 guest_cid; |
0ea9e1d3 AH |
381 | |
382 | vdev->config->get(vdev, offsetof(struct virtio_vsock_config, guest_cid), | |
383 | &guest_cid, sizeof(guest_cid)); | |
384 | vsock->guest_cid = le64_to_cpu(guest_cid); | |
385 | } | |
386 | ||
387 | /* event_lock must be held */ | |
388 | static void virtio_vsock_event_handle(struct virtio_vsock *vsock, | |
389 | struct virtio_vsock_event *event) | |
390 | { | |
391 | switch (le32_to_cpu(event->id)) { | |
392 | case VIRTIO_VSOCK_EVENT_TRANSPORT_RESET: | |
393 | virtio_vsock_update_guest_cid(vsock); | |
8e6ed963 JP |
394 | vsock_for_each_connected_socket(&virtio_transport.transport, |
395 | virtio_vsock_reset_sock); | |
0ea9e1d3 AH |
396 | break; |
397 | } | |
398 | } | |
399 | ||
400 | static void virtio_transport_event_work(struct work_struct *work) | |
401 | { | |
402 | struct virtio_vsock *vsock = | |
403 | container_of(work, struct virtio_vsock, event_work); | |
404 | struct virtqueue *vq; | |
405 | ||
406 | vq = vsock->vqs[VSOCK_VQ_EVENT]; | |
407 | ||
408 | mutex_lock(&vsock->event_lock); | |
409 | ||
b917507e SG |
410 | if (!vsock->event_run) |
411 | goto out; | |
412 | ||
0ea9e1d3 AH |
413 | do { |
414 | struct virtio_vsock_event *event; | |
415 | unsigned int len; | |
416 | ||
417 | virtqueue_disable_cb(vq); | |
418 | while ((event = virtqueue_get_buf(vq, &len)) != NULL) { | |
419 | if (len == sizeof(*event)) | |
420 | virtio_vsock_event_handle(vsock, event); | |
421 | ||
422 | virtio_vsock_event_fill_one(vsock, event); | |
423 | } | |
424 | } while (!virtqueue_enable_cb(vq)); | |
425 | ||
426 | virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]); | |
b917507e | 427 | out: |
0ea9e1d3 AH |
428 | mutex_unlock(&vsock->event_lock); |
429 | } | |
430 | ||
431 | static void virtio_vsock_event_done(struct virtqueue *vq) | |
432 | { | |
433 | struct virtio_vsock *vsock = vq->vdev->priv; | |
434 | ||
435 | if (!vsock) | |
436 | return; | |
437 | queue_work(virtio_vsock_workqueue, &vsock->event_work); | |
438 | } | |
439 | ||
440 | static void virtio_vsock_tx_done(struct virtqueue *vq) | |
441 | { | |
442 | struct virtio_vsock *vsock = vq->vdev->priv; | |
443 | ||
444 | if (!vsock) | |
445 | return; | |
446 | queue_work(virtio_vsock_workqueue, &vsock->tx_work); | |
447 | } | |
448 | ||
449 | static void virtio_vsock_rx_done(struct virtqueue *vq) | |
450 | { | |
451 | struct virtio_vsock *vsock = vq->vdev->priv; | |
452 | ||
453 | if (!vsock) | |
454 | return; | |
455 | queue_work(virtio_vsock_workqueue, &vsock->rx_work); | |
456 | } | |
457 | ||
581512a6 AK |
458 | static bool virtio_transport_can_msgzerocopy(int bufs_num) |
459 | { | |
460 | struct virtio_vsock *vsock; | |
461 | bool res = false; | |
462 | ||
463 | rcu_read_lock(); | |
464 | ||
465 | vsock = rcu_dereference(the_virtio_vsock); | |
466 | if (vsock) { | |
467 | struct virtqueue *vq = vsock->vqs[VSOCK_VQ_TX]; | |
468 | ||
469 | /* Check that tx queue is large enough to keep whole | |
470 | * data to send. This is needed, because when there is | |
471 | * not enough free space in the queue, current skb to | |
472 | * send will be reinserted to the head of tx list of | |
473 | * the socket to retry transmission later, so if skb | |
474 | * is bigger than whole queue, it will be reinserted | |
475 | * again and again, thus blocking other skbs to be sent. | |
476 | * Each page of the user provided buffer will be added | |
477 | * as a single buffer to the tx virtqueue, so compare | |
478 | * number of pages against maximum capacity of the queue. | |
479 | */ | |
480 | if (bufs_num <= vq->num_max) | |
481 | res = true; | |
482 | } | |
483 | ||
484 | rcu_read_unlock(); | |
485 | ||
486 | return res; | |
487 | } | |
488 | ||
e2fcc326 AK |
489 | static bool virtio_transport_msgzerocopy_allow(void) |
490 | { | |
491 | return true; | |
492 | } | |
493 | ||
53efbba1 AK |
494 | static bool virtio_transport_seqpacket_allow(u32 remote_cid); |
495 | ||
0ea9e1d3 AH |
496 | static struct virtio_transport virtio_transport = { |
497 | .transport = { | |
6a2c0962 SG |
498 | .module = THIS_MODULE, |
499 | ||
0ea9e1d3 AH |
500 | .get_local_cid = virtio_transport_get_local_cid, |
501 | ||
502 | .init = virtio_transport_do_socket_init, | |
503 | .destruct = virtio_transport_destruct, | |
504 | .release = virtio_transport_release, | |
505 | .connect = virtio_transport_connect, | |
506 | .shutdown = virtio_transport_shutdown, | |
073b4f2c | 507 | .cancel_pkt = virtio_transport_cancel_pkt, |
0ea9e1d3 AH |
508 | |
509 | .dgram_bind = virtio_transport_dgram_bind, | |
510 | .dgram_dequeue = virtio_transport_dgram_dequeue, | |
511 | .dgram_enqueue = virtio_transport_dgram_enqueue, | |
512 | .dgram_allow = virtio_transport_dgram_allow, | |
513 | ||
514 | .stream_dequeue = virtio_transport_stream_dequeue, | |
515 | .stream_enqueue = virtio_transport_stream_enqueue, | |
516 | .stream_has_data = virtio_transport_stream_has_data, | |
517 | .stream_has_space = virtio_transport_stream_has_space, | |
518 | .stream_rcvhiwat = virtio_transport_stream_rcvhiwat, | |
519 | .stream_is_active = virtio_transport_stream_is_active, | |
520 | .stream_allow = virtio_transport_stream_allow, | |
521 | ||
53efbba1 AK |
522 | .seqpacket_dequeue = virtio_transport_seqpacket_dequeue, |
523 | .seqpacket_enqueue = virtio_transport_seqpacket_enqueue, | |
524 | .seqpacket_allow = virtio_transport_seqpacket_allow, | |
525 | .seqpacket_has_data = virtio_transport_seqpacket_has_data, | |
526 | ||
e2fcc326 AK |
527 | .msgzerocopy_allow = virtio_transport_msgzerocopy_allow, |
528 | ||
0ea9e1d3 AH |
529 | .notify_poll_in = virtio_transport_notify_poll_in, |
530 | .notify_poll_out = virtio_transport_notify_poll_out, | |
531 | .notify_recv_init = virtio_transport_notify_recv_init, | |
532 | .notify_recv_pre_block = virtio_transport_notify_recv_pre_block, | |
533 | .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue, | |
534 | .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue, | |
535 | .notify_send_init = virtio_transport_notify_send_init, | |
536 | .notify_send_pre_block = virtio_transport_notify_send_pre_block, | |
537 | .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, | |
538 | .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, | |
b9f2b0ff | 539 | .notify_buffer_size = virtio_transport_notify_buffer_size, |
634f1a71 BE |
540 | |
541 | .read_skb = virtio_transport_read_skb, | |
0ea9e1d3 AH |
542 | }, |
543 | ||
544 | .send_pkt = virtio_transport_send_pkt, | |
581512a6 | 545 | .can_msgzerocopy = virtio_transport_can_msgzerocopy, |
0ea9e1d3 AH |
546 | }; |
547 | ||
53efbba1 AK |
548 | static bool virtio_transport_seqpacket_allow(u32 remote_cid) |
549 | { | |
550 | struct virtio_vsock *vsock; | |
551 | bool seqpacket_allow; | |
552 | ||
64295f0d | 553 | seqpacket_allow = false; |
53efbba1 AK |
554 | rcu_read_lock(); |
555 | vsock = rcu_dereference(the_virtio_vsock); | |
64295f0d ED |
556 | if (vsock) |
557 | seqpacket_allow = vsock->seqpacket_allow; | |
53efbba1 AK |
558 | rcu_read_unlock(); |
559 | ||
560 | return seqpacket_allow; | |
561 | } | |
562 | ||
4c7246dc SG |
563 | static void virtio_transport_rx_work(struct work_struct *work) |
564 | { | |
565 | struct virtio_vsock *vsock = | |
566 | container_of(work, struct virtio_vsock, rx_work); | |
567 | struct virtqueue *vq; | |
568 | ||
569 | vq = vsock->vqs[VSOCK_VQ_RX]; | |
570 | ||
571 | mutex_lock(&vsock->rx_lock); | |
572 | ||
573 | if (!vsock->rx_run) | |
574 | goto out; | |
575 | ||
576 | do { | |
577 | virtqueue_disable_cb(vq); | |
578 | for (;;) { | |
71dc9ec9 | 579 | struct sk_buff *skb; |
4c7246dc SG |
580 | unsigned int len; |
581 | ||
582 | if (!virtio_transport_more_replies(vsock)) { | |
583 | /* Stop rx until the device processes already | |
584 | * pending replies. Leave rx virtqueue | |
585 | * callbacks disabled. | |
586 | */ | |
587 | goto out; | |
588 | } | |
589 | ||
71dc9ec9 BE |
590 | skb = virtqueue_get_buf(vq, &len); |
591 | if (!skb) | |
4c7246dc | 592 | break; |
4c7246dc SG |
593 | |
594 | vsock->rx_buf_nr--; | |
595 | ||
596 | /* Drop short/long packets */ | |
71dc9ec9 BE |
597 | if (unlikely(len < sizeof(struct virtio_vsock_hdr) || |
598 | len > virtio_vsock_skb_len(skb))) { | |
599 | kfree_skb(skb); | |
4c7246dc SG |
600 | continue; |
601 | } | |
602 | ||
71dc9ec9 BE |
603 | virtio_vsock_skb_rx_put(skb); |
604 | virtio_transport_deliver_tap_pkt(skb); | |
605 | virtio_transport_recv_pkt(&virtio_transport, skb); | |
4c7246dc SG |
606 | } |
607 | } while (!virtqueue_enable_cb(vq)); | |
608 | ||
609 | out: | |
610 | if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2) | |
611 | virtio_vsock_rx_fill(vsock); | |
612 | mutex_unlock(&vsock->rx_lock); | |
613 | } | |
614 | ||
a1032098 | 615 | static int virtio_vsock_vqs_init(struct virtio_vsock *vsock) |
0ea9e1d3 | 616 | { |
a1032098 | 617 | struct virtio_device *vdev = vsock->vdev; |
0ea9e1d3 AH |
618 | static const char * const names[] = { |
619 | "rx", | |
620 | "tx", | |
621 | "event", | |
622 | }; | |
a1032098 SG |
623 | vq_callback_t *callbacks[] = { |
624 | virtio_vsock_rx_done, | |
625 | virtio_vsock_tx_done, | |
626 | virtio_vsock_event_done, | |
627 | }; | |
0ea9e1d3 AH |
628 | int ret; |
629 | ||
a1032098 | 630 | ret = virtio_find_vqs(vdev, VSOCK_VQ_MAX, vsock->vqs, callbacks, names, |
9b2bbdb2 | 631 | NULL); |
0ea9e1d3 | 632 | if (ret < 0) |
a1032098 | 633 | return ret; |
0ea9e1d3 AH |
634 | |
635 | virtio_vsock_update_guest_cid(vsock); | |
636 | ||
88704454 SG |
637 | virtio_device_ready(vdev); |
638 | ||
53b08c49 AM |
639 | return 0; |
640 | } | |
641 | ||
642 | static void virtio_vsock_vqs_start(struct virtio_vsock *vsock) | |
643 | { | |
b917507e SG |
644 | mutex_lock(&vsock->tx_lock); |
645 | vsock->tx_run = true; | |
646 | mutex_unlock(&vsock->tx_lock); | |
647 | ||
0ea9e1d3 AH |
648 | mutex_lock(&vsock->rx_lock); |
649 | virtio_vsock_rx_fill(vsock); | |
b917507e | 650 | vsock->rx_run = true; |
0ea9e1d3 AH |
651 | mutex_unlock(&vsock->rx_lock); |
652 | ||
653 | mutex_lock(&vsock->event_lock); | |
654 | virtio_vsock_event_fill(vsock); | |
b917507e | 655 | vsock->event_run = true; |
0ea9e1d3 AH |
656 | mutex_unlock(&vsock->event_lock); |
657 | ||
53b08c49 AM |
658 | /* virtio_transport_send_pkt() can queue packets once |
659 | * the_virtio_vsock is set, but they won't be processed until | |
660 | * vsock->tx_run is set to true. We queue vsock->send_pkt_work | |
661 | * when initialization finishes to send those packets queued | |
662 | * earlier. | |
663 | * We don't need to queue the other workers (rx, event) because | |
664 | * as long as we don't fill the queues with empty buffers, the | |
665 | * host can't send us any notification. | |
666 | */ | |
667 | queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work); | |
0ea9e1d3 AH |
668 | } |
669 | ||
a1032098 | 670 | static void virtio_vsock_vqs_del(struct virtio_vsock *vsock) |
0ea9e1d3 | 671 | { |
a1032098 | 672 | struct virtio_device *vdev = vsock->vdev; |
71dc9ec9 | 673 | struct sk_buff *skb; |
0ea9e1d3 | 674 | |
a1032098 | 675 | /* Reset all connected sockets when the VQs disappear */ |
8e6ed963 JP |
676 | vsock_for_each_connected_socket(&virtio_transport.transport, |
677 | virtio_vsock_reset_sock); | |
85965487 | 678 | |
b917507e | 679 | /* Stop all work handlers to make sure no one is accessing the device, |
d9679d00 | 680 | * so we can safely call virtio_reset_device(). |
b917507e SG |
681 | */ |
682 | mutex_lock(&vsock->rx_lock); | |
683 | vsock->rx_run = false; | |
684 | mutex_unlock(&vsock->rx_lock); | |
685 | ||
686 | mutex_lock(&vsock->tx_lock); | |
687 | vsock->tx_run = false; | |
688 | mutex_unlock(&vsock->tx_lock); | |
689 | ||
690 | mutex_lock(&vsock->event_lock); | |
691 | vsock->event_run = false; | |
692 | mutex_unlock(&vsock->event_lock); | |
693 | ||
694 | /* Flush all device writes and interrupts, device will not use any | |
695 | * more buffers. | |
696 | */ | |
d9679d00 | 697 | virtio_reset_device(vdev); |
0ea9e1d3 AH |
698 | |
699 | mutex_lock(&vsock->rx_lock); | |
71dc9ec9 BE |
700 | while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX]))) |
701 | kfree_skb(skb); | |
0ea9e1d3 AH |
702 | mutex_unlock(&vsock->rx_lock); |
703 | ||
704 | mutex_lock(&vsock->tx_lock); | |
71dc9ec9 BE |
705 | while ((skb = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX]))) |
706 | kfree_skb(skb); | |
0ea9e1d3 AH |
707 | mutex_unlock(&vsock->tx_lock); |
708 | ||
71dc9ec9 | 709 | virtio_vsock_skb_queue_purge(&vsock->send_pkt_queue); |
0ea9e1d3 | 710 | |
b917507e | 711 | /* Delete virtqueues and flush outstanding callbacks if any */ |
0ea9e1d3 | 712 | vdev->config->del_vqs(vdev); |
a1032098 SG |
713 | } |
714 | ||
715 | static int virtio_vsock_probe(struct virtio_device *vdev) | |
716 | { | |
717 | struct virtio_vsock *vsock = NULL; | |
718 | int ret; | |
64c99d2d | 719 | int i; |
a1032098 SG |
720 | |
721 | ret = mutex_lock_interruptible(&the_virtio_vsock_mutex); | |
722 | if (ret) | |
723 | return ret; | |
724 | ||
725 | /* Only one virtio-vsock device per guest is supported */ | |
726 | if (rcu_dereference_protected(the_virtio_vsock, | |
727 | lockdep_is_held(&the_virtio_vsock_mutex))) { | |
728 | ret = -EBUSY; | |
729 | goto out; | |
730 | } | |
731 | ||
732 | vsock = kzalloc(sizeof(*vsock), GFP_KERNEL); | |
733 | if (!vsock) { | |
734 | ret = -ENOMEM; | |
735 | goto out; | |
736 | } | |
737 | ||
738 | vsock->vdev = vdev; | |
739 | ||
740 | vsock->rx_buf_nr = 0; | |
741 | vsock->rx_buf_max_nr = 0; | |
742 | atomic_set(&vsock->queued_replies, 0); | |
743 | ||
744 | mutex_init(&vsock->tx_lock); | |
745 | mutex_init(&vsock->rx_lock); | |
746 | mutex_init(&vsock->event_lock); | |
71dc9ec9 | 747 | skb_queue_head_init(&vsock->send_pkt_queue); |
a1032098 SG |
748 | INIT_WORK(&vsock->rx_work, virtio_transport_rx_work); |
749 | INIT_WORK(&vsock->tx_work, virtio_transport_tx_work); | |
750 | INIT_WORK(&vsock->event_work, virtio_transport_event_work); | |
751 | INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work); | |
752 | ||
753 | if (virtio_has_feature(vdev, VIRTIO_VSOCK_F_SEQPACKET)) | |
754 | vsock->seqpacket_allow = true; | |
755 | ||
756 | vdev->priv = vsock; | |
757 | ||
758 | ret = virtio_vsock_vqs_init(vsock); | |
759 | if (ret < 0) | |
760 | goto out; | |
761 | ||
64c99d2d AK |
762 | for (i = 0; i < ARRAY_SIZE(vsock->out_sgs); i++) |
763 | vsock->out_sgs[i] = &vsock->out_bufs[i]; | |
764 | ||
a1032098 | 765 | rcu_assign_pointer(the_virtio_vsock, vsock); |
53b08c49 | 766 | virtio_vsock_vqs_start(vsock); |
a1032098 SG |
767 | |
768 | mutex_unlock(&the_virtio_vsock_mutex); | |
769 | ||
770 | return 0; | |
771 | ||
772 | out: | |
773 | kfree(vsock); | |
774 | mutex_unlock(&the_virtio_vsock_mutex); | |
775 | return ret; | |
776 | } | |
777 | ||
778 | static void virtio_vsock_remove(struct virtio_device *vdev) | |
779 | { | |
780 | struct virtio_vsock *vsock = vdev->priv; | |
781 | ||
782 | mutex_lock(&the_virtio_vsock_mutex); | |
783 | ||
784 | vdev->priv = NULL; | |
785 | rcu_assign_pointer(the_virtio_vsock, NULL); | |
786 | synchronize_rcu(); | |
787 | ||
788 | virtio_vsock_vqs_del(vsock); | |
0ea9e1d3 | 789 | |
e226121f SG |
790 | /* Other works can be queued before 'config->del_vqs()', so we flush |
791 | * all works before to free the vsock object to avoid use after free. | |
792 | */ | |
e226121f SG |
793 | flush_work(&vsock->rx_work); |
794 | flush_work(&vsock->tx_work); | |
795 | flush_work(&vsock->event_work); | |
796 | flush_work(&vsock->send_pkt_work); | |
797 | ||
0deab087 SG |
798 | mutex_unlock(&the_virtio_vsock_mutex); |
799 | ||
0ea9e1d3 AH |
800 | kfree(vsock); |
801 | } | |
802 | ||
bd50c5dc SG |
803 | #ifdef CONFIG_PM_SLEEP |
804 | static int virtio_vsock_freeze(struct virtio_device *vdev) | |
805 | { | |
806 | struct virtio_vsock *vsock = vdev->priv; | |
807 | ||
808 | mutex_lock(&the_virtio_vsock_mutex); | |
809 | ||
810 | rcu_assign_pointer(the_virtio_vsock, NULL); | |
811 | synchronize_rcu(); | |
812 | ||
813 | virtio_vsock_vqs_del(vsock); | |
814 | ||
815 | mutex_unlock(&the_virtio_vsock_mutex); | |
816 | ||
817 | return 0; | |
818 | } | |
819 | ||
820 | static int virtio_vsock_restore(struct virtio_device *vdev) | |
821 | { | |
822 | struct virtio_vsock *vsock = vdev->priv; | |
823 | int ret; | |
824 | ||
825 | mutex_lock(&the_virtio_vsock_mutex); | |
826 | ||
827 | /* Only one virtio-vsock device per guest is supported */ | |
828 | if (rcu_dereference_protected(the_virtio_vsock, | |
829 | lockdep_is_held(&the_virtio_vsock_mutex))) { | |
830 | ret = -EBUSY; | |
831 | goto out; | |
832 | } | |
833 | ||
834 | ret = virtio_vsock_vqs_init(vsock); | |
835 | if (ret < 0) | |
836 | goto out; | |
837 | ||
838 | rcu_assign_pointer(the_virtio_vsock, vsock); | |
53b08c49 | 839 | virtio_vsock_vqs_start(vsock); |
bd50c5dc SG |
840 | |
841 | out: | |
842 | mutex_unlock(&the_virtio_vsock_mutex); | |
843 | return ret; | |
844 | } | |
845 | #endif /* CONFIG_PM_SLEEP */ | |
846 | ||
0ea9e1d3 AH |
847 | static struct virtio_device_id id_table[] = { |
848 | { VIRTIO_ID_VSOCK, VIRTIO_DEV_ANY_ID }, | |
849 | { 0 }, | |
850 | }; | |
851 | ||
852 | static unsigned int features[] = { | |
53efbba1 | 853 | VIRTIO_VSOCK_F_SEQPACKET |
0ea9e1d3 AH |
854 | }; |
855 | ||
856 | static struct virtio_driver virtio_vsock_driver = { | |
857 | .feature_table = features, | |
858 | .feature_table_size = ARRAY_SIZE(features), | |
859 | .driver.name = KBUILD_MODNAME, | |
860 | .driver.owner = THIS_MODULE, | |
861 | .id_table = id_table, | |
862 | .probe = virtio_vsock_probe, | |
863 | .remove = virtio_vsock_remove, | |
bd50c5dc SG |
864 | #ifdef CONFIG_PM_SLEEP |
865 | .freeze = virtio_vsock_freeze, | |
866 | .restore = virtio_vsock_restore, | |
867 | #endif | |
0ea9e1d3 AH |
868 | }; |
869 | ||
870 | static int __init virtio_vsock_init(void) | |
871 | { | |
872 | int ret; | |
873 | ||
874 | virtio_vsock_workqueue = alloc_workqueue("virtio_vsock", 0, 0); | |
875 | if (!virtio_vsock_workqueue) | |
876 | return -ENOMEM; | |
22b5c0b6 | 877 | |
c0cfa2d8 SG |
878 | ret = vsock_core_register(&virtio_transport.transport, |
879 | VSOCK_TRANSPORT_F_G2H); | |
0ea9e1d3 | 880 | if (ret) |
22b5c0b6 SG |
881 | goto out_wq; |
882 | ||
ba95e5df | 883 | ret = register_virtio_driver(&virtio_vsock_driver); |
22b5c0b6 | 884 | if (ret) |
ba95e5df | 885 | goto out_vci; |
22b5c0b6 SG |
886 | |
887 | return 0; | |
888 | ||
ba95e5df | 889 | out_vci: |
c0cfa2d8 | 890 | vsock_core_unregister(&virtio_transport.transport); |
22b5c0b6 SG |
891 | out_wq: |
892 | destroy_workqueue(virtio_vsock_workqueue); | |
0ea9e1d3 AH |
893 | return ret; |
894 | } | |
895 | ||
896 | static void __exit virtio_vsock_exit(void) | |
897 | { | |
898 | unregister_virtio_driver(&virtio_vsock_driver); | |
c0cfa2d8 | 899 | vsock_core_unregister(&virtio_transport.transport); |
0ea9e1d3 AH |
900 | destroy_workqueue(virtio_vsock_workqueue); |
901 | } | |
902 | ||
903 | module_init(virtio_vsock_init); | |
904 | module_exit(virtio_vsock_exit); | |
905 | MODULE_LICENSE("GPL v2"); | |
906 | MODULE_AUTHOR("Asias He"); | |
907 | MODULE_DESCRIPTION("virtio transport for vsock"); | |
908 | MODULE_DEVICE_TABLE(virtio, id_table); |