Commit | Line | Data |
---|---|---|
7a338472 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
06a8fc78 AH |
2 | /* |
3 | * common code for virtio vsock | |
4 | * | |
5 | * Copyright (C) 2013-2015 Red Hat, Inc. | |
6 | * Author: Asias He <asias@redhat.com> | |
7 | * Stefan Hajnoczi <stefanha@redhat.com> | |
06a8fc78 AH |
8 | */ |
9 | #include <linux/spinlock.h> | |
10 | #include <linux/module.h> | |
174cd4b1 | 11 | #include <linux/sched/signal.h> |
06a8fc78 AH |
12 | #include <linux/ctype.h> |
13 | #include <linux/list.h> | |
06a8fc78 | 14 | #include <linux/virtio_vsock.h> |
82dfb540 | 15 | #include <uapi/linux/vsockmon.h> |
06a8fc78 AH |
16 | |
17 | #include <net/sock.h> | |
18 | #include <net/af_vsock.h> | |
19 | ||
20 | #define CREATE_TRACE_POINTS | |
21 | #include <trace/events/vsock_virtio_transport_common.h> | |
22 | ||
23 | /* How long to wait for graceful shutdown of a connection */ | |
24 | #define VSOCK_CLOSE_TIMEOUT (8 * HZ) | |
25 | ||
473c7391 SG |
26 | /* Threshold for detecting small packets to copy */ |
27 | #define GOOD_COPY_LEN 128 | |
28 | ||
daabfbca SG |
29 | static const struct virtio_transport * |
30 | virtio_transport_get_ops(struct vsock_sock *vsk) | |
06a8fc78 | 31 | { |
daabfbca | 32 | const struct vsock_transport *t = vsock_core_get_transport(vsk); |
06a8fc78 | 33 | |
4aaf5961 SG |
34 | if (WARN_ON(!t)) |
35 | return NULL; | |
36 | ||
06a8fc78 AH |
37 | return container_of(t, struct virtio_transport, transport); |
38 | } | |
39 | ||
71dc9ec9 BE |
40 | /* Returns a new packet on success, otherwise returns NULL. |
41 | * | |
42 | * If NULL is returned, errp is set to a negative errno. | |
43 | */ | |
44 | static struct sk_buff * | |
45 | virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *info, | |
06a8fc78 AH |
46 | size_t len, |
47 | u32 src_cid, | |
48 | u32 src_port, | |
49 | u32 dst_cid, | |
50 | u32 dst_port) | |
51 | { | |
71dc9ec9 BE |
52 | const size_t skb_len = VIRTIO_VSOCK_SKB_HEADROOM + len; |
53 | struct virtio_vsock_hdr *hdr; | |
54 | struct sk_buff *skb; | |
55 | void *payload; | |
06a8fc78 AH |
56 | int err; |
57 | ||
71dc9ec9 BE |
58 | skb = virtio_vsock_alloc_skb(skb_len, GFP_KERNEL); |
59 | if (!skb) | |
06a8fc78 AH |
60 | return NULL; |
61 | ||
71dc9ec9 BE |
62 | hdr = virtio_vsock_hdr(skb); |
63 | hdr->type = cpu_to_le16(info->type); | |
64 | hdr->op = cpu_to_le16(info->op); | |
65 | hdr->src_cid = cpu_to_le64(src_cid); | |
66 | hdr->dst_cid = cpu_to_le64(dst_cid); | |
67 | hdr->src_port = cpu_to_le32(src_port); | |
68 | hdr->dst_port = cpu_to_le32(dst_port); | |
69 | hdr->flags = cpu_to_le32(info->flags); | |
70 | hdr->len = cpu_to_le32(len); | |
06a8fc78 AH |
71 | |
72 | if (info->msg && len > 0) { | |
71dc9ec9 BE |
73 | payload = skb_put(skb, len); |
74 | err = memcpy_from_msg(payload, info->msg, len); | |
06a8fc78 AH |
75 | if (err) |
76 | goto out; | |
9ac841f5 AK |
77 | |
78 | if (msg_data_left(info->msg) == 0 && | |
8d5ac871 | 79 | info->type == VIRTIO_VSOCK_TYPE_SEQPACKET) { |
71dc9ec9 | 80 | hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM); |
8d5ac871 AK |
81 | |
82 | if (info->msg->msg_flags & MSG_EOR) | |
71dc9ec9 | 83 | hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR); |
8d5ac871 | 84 | } |
06a8fc78 AH |
85 | } |
86 | ||
71dc9ec9 BE |
87 | if (info->reply) |
88 | virtio_vsock_skb_set_reply(skb); | |
89 | ||
06a8fc78 AH |
90 | trace_virtio_transport_alloc_pkt(src_cid, src_port, |
91 | dst_cid, dst_port, | |
92 | len, | |
93 | info->type, | |
94 | info->op, | |
95 | info->flags); | |
96 | ||
71dc9ec9 | 97 | return skb; |
06a8fc78 AH |
98 | |
99 | out: | |
71dc9ec9 | 100 | kfree_skb(skb); |
06a8fc78 AH |
101 | return NULL; |
102 | } | |
06a8fc78 | 103 | |
82dfb540 GG |
104 | /* Packet capture */ |
105 | static struct sk_buff *virtio_transport_build_skb(void *opaque) | |
106 | { | |
71dc9ec9 BE |
107 | struct virtio_vsock_hdr *pkt_hdr; |
108 | struct sk_buff *pkt = opaque; | |
82dfb540 GG |
109 | struct af_vsockmon_hdr *hdr; |
110 | struct sk_buff *skb; | |
6dbd3e66 SG |
111 | size_t payload_len; |
112 | void *payload_buf; | |
82dfb540 | 113 | |
6dbd3e66 SG |
114 | /* A packet could be split to fit the RX buffer, so we can retrieve |
115 | * the payload length from the header and the buffer pointer taking | |
116 | * care of the offset in the original packet. | |
117 | */ | |
71dc9ec9 BE |
118 | pkt_hdr = virtio_vsock_hdr(pkt); |
119 | payload_len = pkt->len; | |
120 | payload_buf = pkt->data; | |
6dbd3e66 | 121 | |
71dc9ec9 | 122 | skb = alloc_skb(sizeof(*hdr) + sizeof(*pkt_hdr) + payload_len, |
82dfb540 GG |
123 | GFP_ATOMIC); |
124 | if (!skb) | |
125 | return NULL; | |
126 | ||
4df864c1 | 127 | hdr = skb_put(skb, sizeof(*hdr)); |
82dfb540 GG |
128 | |
129 | /* pkt->hdr is little-endian so no need to byteswap here */ | |
71dc9ec9 BE |
130 | hdr->src_cid = pkt_hdr->src_cid; |
131 | hdr->src_port = pkt_hdr->src_port; | |
132 | hdr->dst_cid = pkt_hdr->dst_cid; | |
133 | hdr->dst_port = pkt_hdr->dst_port; | |
82dfb540 GG |
134 | |
135 | hdr->transport = cpu_to_le16(AF_VSOCK_TRANSPORT_VIRTIO); | |
71dc9ec9 | 136 | hdr->len = cpu_to_le16(sizeof(*pkt_hdr)); |
82dfb540 GG |
137 | memset(hdr->reserved, 0, sizeof(hdr->reserved)); |
138 | ||
71dc9ec9 | 139 | switch (le16_to_cpu(pkt_hdr->op)) { |
82dfb540 GG |
140 | case VIRTIO_VSOCK_OP_REQUEST: |
141 | case VIRTIO_VSOCK_OP_RESPONSE: | |
142 | hdr->op = cpu_to_le16(AF_VSOCK_OP_CONNECT); | |
143 | break; | |
144 | case VIRTIO_VSOCK_OP_RST: | |
145 | case VIRTIO_VSOCK_OP_SHUTDOWN: | |
146 | hdr->op = cpu_to_le16(AF_VSOCK_OP_DISCONNECT); | |
147 | break; | |
148 | case VIRTIO_VSOCK_OP_RW: | |
149 | hdr->op = cpu_to_le16(AF_VSOCK_OP_PAYLOAD); | |
150 | break; | |
151 | case VIRTIO_VSOCK_OP_CREDIT_UPDATE: | |
152 | case VIRTIO_VSOCK_OP_CREDIT_REQUEST: | |
153 | hdr->op = cpu_to_le16(AF_VSOCK_OP_CONTROL); | |
154 | break; | |
155 | default: | |
156 | hdr->op = cpu_to_le16(AF_VSOCK_OP_UNKNOWN); | |
157 | break; | |
158 | } | |
159 | ||
71dc9ec9 | 160 | skb_put_data(skb, pkt_hdr, sizeof(*pkt_hdr)); |
82dfb540 | 161 | |
6dbd3e66 SG |
162 | if (payload_len) { |
163 | skb_put_data(skb, payload_buf, payload_len); | |
82dfb540 GG |
164 | } |
165 | ||
166 | return skb; | |
167 | } | |
168 | ||
71dc9ec9 | 169 | void virtio_transport_deliver_tap_pkt(struct sk_buff *skb) |
82dfb540 | 170 | { |
71dc9ec9 | 171 | if (virtio_vsock_skb_tap_delivered(skb)) |
a78d1639 SG |
172 | return; |
173 | ||
71dc9ec9 BE |
174 | vsock_deliver_tap(virtio_transport_build_skb, skb); |
175 | virtio_vsock_skb_set_tap_delivered(skb); | |
82dfb540 GG |
176 | } |
177 | EXPORT_SYMBOL_GPL(virtio_transport_deliver_tap_pkt); | |
178 | ||
e4b1ef15 AK |
179 | static u16 virtio_transport_get_type(struct sock *sk) |
180 | { | |
181 | if (sk->sk_type == SOCK_STREAM) | |
182 | return VIRTIO_VSOCK_TYPE_STREAM; | |
183 | else | |
184 | return VIRTIO_VSOCK_TYPE_SEQPACKET; | |
185 | } | |
186 | ||
4aaf5961 SG |
187 | /* This function can only be used on connecting/connected sockets, |
188 | * since a socket assigned to a transport is required. | |
189 | * | |
190 | * Do not use on listener sockets! | |
191 | */ | |
06a8fc78 AH |
192 | static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, |
193 | struct virtio_vsock_pkt_info *info) | |
194 | { | |
195 | u32 src_cid, src_port, dst_cid, dst_port; | |
4aaf5961 | 196 | const struct virtio_transport *t_ops; |
06a8fc78 | 197 | struct virtio_vsock_sock *vvs; |
06a8fc78 | 198 | u32 pkt_len = info->pkt_len; |
71dc9ec9 | 199 | struct sk_buff *skb; |
06a8fc78 | 200 | |
9ac841f5 | 201 | info->type = virtio_transport_get_type(sk_vsock(vsk)); |
b93f8877 | 202 | |
4aaf5961 SG |
203 | t_ops = virtio_transport_get_ops(vsk); |
204 | if (unlikely(!t_ops)) | |
205 | return -EFAULT; | |
206 | ||
207 | src_cid = t_ops->transport.get_local_cid(); | |
06a8fc78 AH |
208 | src_port = vsk->local_addr.svm_port; |
209 | if (!info->remote_cid) { | |
210 | dst_cid = vsk->remote_addr.svm_cid; | |
211 | dst_port = vsk->remote_addr.svm_port; | |
212 | } else { | |
213 | dst_cid = info->remote_cid; | |
214 | dst_port = info->remote_port; | |
215 | } | |
216 | ||
217 | vvs = vsk->trans; | |
218 | ||
219 | /* we can send less than pkt_len bytes */ | |
0038ff35 SG |
220 | if (pkt_len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) |
221 | pkt_len = VIRTIO_VSOCK_MAX_PKT_BUF_SIZE; | |
06a8fc78 AH |
222 | |
223 | /* virtio_transport_get_credit might return less than pkt_len credit */ | |
224 | pkt_len = virtio_transport_get_credit(vvs, pkt_len); | |
225 | ||
226 | /* Do not send zero length OP_RW pkt */ | |
227 | if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW) | |
228 | return pkt_len; | |
229 | ||
71dc9ec9 | 230 | skb = virtio_transport_alloc_skb(info, pkt_len, |
06a8fc78 AH |
231 | src_cid, src_port, |
232 | dst_cid, dst_port); | |
71dc9ec9 | 233 | if (!skb) { |
06a8fc78 AH |
234 | virtio_transport_put_credit(vvs, pkt_len); |
235 | return -ENOMEM; | |
236 | } | |
237 | ||
71dc9ec9 | 238 | virtio_transport_inc_tx_pkt(vvs, skb); |
06a8fc78 | 239 | |
71dc9ec9 | 240 | return t_ops->send_pkt(skb); |
06a8fc78 AH |
241 | } |
242 | ||
ae6fcfbf | 243 | static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs, |
07770616 | 244 | u32 len) |
06a8fc78 | 245 | { |
07770616 | 246 | if (vvs->rx_bytes + len > vvs->buf_alloc) |
ae6fcfbf SG |
247 | return false; |
248 | ||
07770616 | 249 | vvs->rx_bytes += len; |
ae6fcfbf | 250 | return true; |
06a8fc78 AH |
251 | } |
252 | ||
253 | static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, | |
07770616 | 254 | u32 len) |
06a8fc78 | 255 | { |
71dc9ec9 BE |
256 | vvs->rx_bytes -= len; |
257 | vvs->fwd_cnt += len; | |
06a8fc78 AH |
258 | } |
259 | ||
71dc9ec9 | 260 | void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct sk_buff *skb) |
06a8fc78 | 261 | { |
71dc9ec9 BE |
262 | struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); |
263 | ||
9632e9f6 | 264 | spin_lock_bh(&vvs->rx_lock); |
b89d882d | 265 | vvs->last_fwd_cnt = vvs->fwd_cnt; |
71dc9ec9 BE |
266 | hdr->fwd_cnt = cpu_to_le32(vvs->fwd_cnt); |
267 | hdr->buf_alloc = cpu_to_le32(vvs->buf_alloc); | |
9632e9f6 | 268 | spin_unlock_bh(&vvs->rx_lock); |
06a8fc78 AH |
269 | } |
270 | EXPORT_SYMBOL_GPL(virtio_transport_inc_tx_pkt); | |
271 | ||
272 | u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 credit) | |
273 | { | |
274 | u32 ret; | |
275 | ||
276 | spin_lock_bh(&vvs->tx_lock); | |
277 | ret = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); | |
278 | if (ret > credit) | |
279 | ret = credit; | |
280 | vvs->tx_cnt += ret; | |
281 | spin_unlock_bh(&vvs->tx_lock); | |
282 | ||
283 | return ret; | |
284 | } | |
285 | EXPORT_SYMBOL_GPL(virtio_transport_get_credit); | |
286 | ||
287 | void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit) | |
288 | { | |
289 | spin_lock_bh(&vvs->tx_lock); | |
290 | vvs->tx_cnt -= credit; | |
291 | spin_unlock_bh(&vvs->tx_lock); | |
292 | } | |
293 | EXPORT_SYMBOL_GPL(virtio_transport_put_credit); | |
294 | ||
c10844c5 | 295 | static int virtio_transport_send_credit_update(struct vsock_sock *vsk) |
06a8fc78 AH |
296 | { |
297 | struct virtio_vsock_pkt_info info = { | |
298 | .op = VIRTIO_VSOCK_OP_CREDIT_UPDATE, | |
36d277ba | 299 | .vsk = vsk, |
06a8fc78 AH |
300 | }; |
301 | ||
302 | return virtio_transport_send_pkt_info(vsk, &info); | |
303 | } | |
304 | ||
a786ab36 MEVL |
305 | static ssize_t |
306 | virtio_transport_stream_do_peek(struct vsock_sock *vsk, | |
307 | struct msghdr *msg, | |
308 | size_t len) | |
309 | { | |
310 | struct virtio_vsock_sock *vvs = vsk->trans; | |
a786ab36 | 311 | size_t bytes, total = 0, off; |
71dc9ec9 | 312 | struct sk_buff *skb, *tmp; |
a786ab36 MEVL |
313 | int err = -EFAULT; |
314 | ||
315 | spin_lock_bh(&vvs->rx_lock); | |
316 | ||
71dc9ec9 BE |
317 | skb_queue_walk_safe(&vvs->rx_queue, skb, tmp) { |
318 | off = 0; | |
a786ab36 MEVL |
319 | |
320 | if (total == len) | |
321 | break; | |
322 | ||
71dc9ec9 | 323 | while (total < len && off < skb->len) { |
a786ab36 | 324 | bytes = len - total; |
71dc9ec9 BE |
325 | if (bytes > skb->len - off) |
326 | bytes = skb->len - off; | |
a786ab36 MEVL |
327 | |
328 | /* sk_lock is held by caller so no one else can dequeue. | |
329 | * Unlock rx_lock since memcpy_to_msg() may sleep. | |
330 | */ | |
331 | spin_unlock_bh(&vvs->rx_lock); | |
332 | ||
71dc9ec9 | 333 | err = memcpy_to_msg(msg, skb->data + off, bytes); |
a786ab36 MEVL |
334 | if (err) |
335 | goto out; | |
336 | ||
337 | spin_lock_bh(&vvs->rx_lock); | |
338 | ||
339 | total += bytes; | |
340 | off += bytes; | |
341 | } | |
342 | } | |
343 | ||
344 | spin_unlock_bh(&vvs->rx_lock); | |
345 | ||
346 | return total; | |
347 | ||
348 | out: | |
349 | if (total) | |
350 | err = total; | |
351 | return err; | |
352 | } | |
353 | ||
06a8fc78 AH |
354 | static ssize_t |
355 | virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, | |
356 | struct msghdr *msg, | |
357 | size_t len) | |
358 | { | |
359 | struct virtio_vsock_sock *vvs = vsk->trans; | |
06a8fc78 | 360 | size_t bytes, total = 0; |
71dc9ec9 | 361 | struct sk_buff *skb; |
06a8fc78 | 362 | int err = -EFAULT; |
71dc9ec9 | 363 | u32 free_space; |
06a8fc78 AH |
364 | |
365 | spin_lock_bh(&vvs->rx_lock); | |
71dc9ec9 BE |
366 | while (total < len && !skb_queue_empty(&vvs->rx_queue)) { |
367 | skb = __skb_dequeue(&vvs->rx_queue); | |
06a8fc78 AH |
368 | |
369 | bytes = len - total; | |
71dc9ec9 BE |
370 | if (bytes > skb->len) |
371 | bytes = skb->len; | |
06a8fc78 AH |
372 | |
373 | /* sk_lock is held by caller so no one else can dequeue. | |
374 | * Unlock rx_lock since memcpy_to_msg() may sleep. | |
375 | */ | |
376 | spin_unlock_bh(&vvs->rx_lock); | |
377 | ||
71dc9ec9 | 378 | err = memcpy_to_msg(msg, skb->data, bytes); |
06a8fc78 AH |
379 | if (err) |
380 | goto out; | |
381 | ||
382 | spin_lock_bh(&vvs->rx_lock); | |
383 | ||
384 | total += bytes; | |
71dc9ec9 BE |
385 | skb_pull(skb, bytes); |
386 | ||
387 | if (skb->len == 0) { | |
07770616 AK |
388 | u32 pkt_len = le32_to_cpu(virtio_vsock_hdr(skb)->len); |
389 | ||
390 | virtio_transport_dec_rx_pkt(vvs, pkt_len); | |
71dc9ec9 BE |
391 | consume_skb(skb); |
392 | } else { | |
393 | __skb_queue_head(&vvs->rx_queue, skb); | |
06a8fc78 AH |
394 | } |
395 | } | |
b89d882d SG |
396 | |
397 | free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); | |
398 | ||
06a8fc78 AH |
399 | spin_unlock_bh(&vvs->rx_lock); |
400 | ||
f4d7c8e3 MT |
401 | /* To reduce the number of credit update messages, |
402 | * don't update credits as long as lots of space is available. | |
403 | * Note: the limit chosen here is arbitrary. Setting the limit | |
404 | * too high causes extra messages. Too low causes transmitter | |
405 | * stalls. As stalls are in theory more expensive than extra | |
406 | * messages, we set the limit to a high value. TODO: experiment | |
407 | * with different values. | |
b89d882d | 408 | */ |
b93f8877 | 409 | if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) |
c10844c5 | 410 | virtio_transport_send_credit_update(vsk); |
06a8fc78 AH |
411 | |
412 | return total; | |
413 | ||
414 | out: | |
415 | if (total) | |
416 | err = total; | |
417 | return err; | |
418 | } | |
419 | ||
44931195 AK |
420 | static int virtio_transport_seqpacket_do_dequeue(struct vsock_sock *vsk, |
421 | struct msghdr *msg, | |
422 | int flags) | |
423 | { | |
424 | struct virtio_vsock_sock *vvs = vsk->trans; | |
44931195 AK |
425 | int dequeued_len = 0; |
426 | size_t user_buf_len = msg_data_left(msg); | |
44931195 | 427 | bool msg_ready = false; |
71dc9ec9 | 428 | struct sk_buff *skb; |
44931195 AK |
429 | |
430 | spin_lock_bh(&vvs->rx_lock); | |
431 | ||
432 | if (vvs->msg_count == 0) { | |
433 | spin_unlock_bh(&vvs->rx_lock); | |
434 | return 0; | |
435 | } | |
436 | ||
437 | while (!msg_ready) { | |
71dc9ec9 | 438 | struct virtio_vsock_hdr *hdr; |
07770616 | 439 | size_t pkt_len; |
71dc9ec9 BE |
440 | |
441 | skb = __skb_dequeue(&vvs->rx_queue); | |
442 | if (!skb) | |
443 | break; | |
444 | hdr = virtio_vsock_hdr(skb); | |
07770616 | 445 | pkt_len = (size_t)le32_to_cpu(hdr->len); |
44931195 | 446 | |
91aa49a8 | 447 | if (dequeued_len >= 0) { |
44931195 AK |
448 | size_t bytes_to_copy; |
449 | ||
44931195 AK |
450 | bytes_to_copy = min(user_buf_len, pkt_len); |
451 | ||
452 | if (bytes_to_copy) { | |
453 | int err; | |
454 | ||
455 | /* sk_lock is held by caller so no one else can dequeue. | |
456 | * Unlock rx_lock since memcpy_to_msg() may sleep. | |
457 | */ | |
458 | spin_unlock_bh(&vvs->rx_lock); | |
459 | ||
71dc9ec9 | 460 | err = memcpy_to_msg(msg, skb->data, bytes_to_copy); |
44931195 | 461 | if (err) { |
91aa49a8 | 462 | /* Copy of message failed. Rest of |
44931195 AK |
463 | * fragments will be freed without copy. |
464 | */ | |
44931195 AK |
465 | dequeued_len = err; |
466 | } else { | |
467 | user_buf_len -= bytes_to_copy; | |
71dc9ec9 | 468 | skb_pull(skb, bytes_to_copy); |
44931195 AK |
469 | } |
470 | ||
471 | spin_lock_bh(&vvs->rx_lock); | |
472 | } | |
473 | ||
474 | if (dequeued_len >= 0) | |
475 | dequeued_len += pkt_len; | |
476 | } | |
477 | ||
71dc9ec9 | 478 | if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) { |
44931195 AK |
479 | msg_ready = true; |
480 | vvs->msg_count--; | |
8d5ac871 | 481 | |
71dc9ec9 | 482 | if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) |
8d5ac871 | 483 | msg->msg_flags |= MSG_EOR; |
44931195 AK |
484 | } |
485 | ||
07770616 | 486 | virtio_transport_dec_rx_pkt(vvs, pkt_len); |
71dc9ec9 | 487 | kfree_skb(skb); |
44931195 AK |
488 | } |
489 | ||
490 | spin_unlock_bh(&vvs->rx_lock); | |
491 | ||
492 | virtio_transport_send_credit_update(vsk); | |
493 | ||
494 | return dequeued_len; | |
495 | } | |
496 | ||
06a8fc78 AH |
497 | ssize_t |
498 | virtio_transport_stream_dequeue(struct vsock_sock *vsk, | |
499 | struct msghdr *msg, | |
500 | size_t len, int flags) | |
501 | { | |
502 | if (flags & MSG_PEEK) | |
a786ab36 MEVL |
503 | return virtio_transport_stream_do_peek(vsk, msg, len); |
504 | else | |
505 | return virtio_transport_stream_do_dequeue(vsk, msg, len); | |
06a8fc78 AH |
506 | } |
507 | EXPORT_SYMBOL_GPL(virtio_transport_stream_dequeue); | |
508 | ||
44931195 AK |
509 | ssize_t |
510 | virtio_transport_seqpacket_dequeue(struct vsock_sock *vsk, | |
511 | struct msghdr *msg, | |
512 | int flags) | |
513 | { | |
514 | if (flags & MSG_PEEK) | |
515 | return -EOPNOTSUPP; | |
516 | ||
517 | return virtio_transport_seqpacket_do_dequeue(vsk, msg, flags); | |
518 | } | |
519 | EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_dequeue); | |
520 | ||
9ac841f5 AK |
521 | int |
522 | virtio_transport_seqpacket_enqueue(struct vsock_sock *vsk, | |
523 | struct msghdr *msg, | |
524 | size_t len) | |
525 | { | |
526 | struct virtio_vsock_sock *vvs = vsk->trans; | |
527 | ||
528 | spin_lock_bh(&vvs->tx_lock); | |
529 | ||
530 | if (len > vvs->peer_buf_alloc) { | |
531 | spin_unlock_bh(&vvs->tx_lock); | |
532 | return -EMSGSIZE; | |
533 | } | |
534 | ||
535 | spin_unlock_bh(&vvs->tx_lock); | |
536 | ||
537 | return virtio_transport_stream_enqueue(vsk, msg, len); | |
538 | } | |
539 | EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_enqueue); | |
540 | ||
06a8fc78 AH |
541 | int |
542 | virtio_transport_dgram_dequeue(struct vsock_sock *vsk, | |
543 | struct msghdr *msg, | |
544 | size_t len, int flags) | |
545 | { | |
546 | return -EOPNOTSUPP; | |
547 | } | |
548 | EXPORT_SYMBOL_GPL(virtio_transport_dgram_dequeue); | |
549 | ||
550 | s64 virtio_transport_stream_has_data(struct vsock_sock *vsk) | |
551 | { | |
552 | struct virtio_vsock_sock *vvs = vsk->trans; | |
553 | s64 bytes; | |
554 | ||
555 | spin_lock_bh(&vvs->rx_lock); | |
556 | bytes = vvs->rx_bytes; | |
557 | spin_unlock_bh(&vvs->rx_lock); | |
558 | ||
559 | return bytes; | |
560 | } | |
561 | EXPORT_SYMBOL_GPL(virtio_transport_stream_has_data); | |
562 | ||
9ac841f5 AK |
563 | u32 virtio_transport_seqpacket_has_data(struct vsock_sock *vsk) |
564 | { | |
565 | struct virtio_vsock_sock *vvs = vsk->trans; | |
566 | u32 msg_count; | |
567 | ||
568 | spin_lock_bh(&vvs->rx_lock); | |
569 | msg_count = vvs->msg_count; | |
570 | spin_unlock_bh(&vvs->rx_lock); | |
571 | ||
572 | return msg_count; | |
573 | } | |
574 | EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_has_data); | |
575 | ||
06a8fc78 AH |
576 | static s64 virtio_transport_has_space(struct vsock_sock *vsk) |
577 | { | |
578 | struct virtio_vsock_sock *vvs = vsk->trans; | |
579 | s64 bytes; | |
580 | ||
581 | bytes = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); | |
582 | if (bytes < 0) | |
583 | bytes = 0; | |
584 | ||
585 | return bytes; | |
586 | } | |
587 | ||
588 | s64 virtio_transport_stream_has_space(struct vsock_sock *vsk) | |
589 | { | |
590 | struct virtio_vsock_sock *vvs = vsk->trans; | |
591 | s64 bytes; | |
592 | ||
593 | spin_lock_bh(&vvs->tx_lock); | |
594 | bytes = virtio_transport_has_space(vsk); | |
595 | spin_unlock_bh(&vvs->tx_lock); | |
596 | ||
597 | return bytes; | |
598 | } | |
599 | EXPORT_SYMBOL_GPL(virtio_transport_stream_has_space); | |
600 | ||
601 | int virtio_transport_do_socket_init(struct vsock_sock *vsk, | |
602 | struct vsock_sock *psk) | |
603 | { | |
604 | struct virtio_vsock_sock *vvs; | |
605 | ||
606 | vvs = kzalloc(sizeof(*vvs), GFP_KERNEL); | |
607 | if (!vvs) | |
608 | return -ENOMEM; | |
609 | ||
610 | vsk->trans = vvs; | |
611 | vvs->vsk = vsk; | |
c0cfa2d8 | 612 | if (psk && psk->trans) { |
06a8fc78 AH |
613 | struct virtio_vsock_sock *ptrans = psk->trans; |
614 | ||
06a8fc78 | 615 | vvs->peer_buf_alloc = ptrans->peer_buf_alloc; |
06a8fc78 AH |
616 | } |
617 | ||
b9f2b0ff SG |
618 | if (vsk->buffer_size > VIRTIO_VSOCK_MAX_BUF_SIZE) |
619 | vsk->buffer_size = VIRTIO_VSOCK_MAX_BUF_SIZE; | |
620 | ||
621 | vvs->buf_alloc = vsk->buffer_size; | |
06a8fc78 AH |
622 | |
623 | spin_lock_init(&vvs->rx_lock); | |
624 | spin_lock_init(&vvs->tx_lock); | |
71dc9ec9 | 625 | skb_queue_head_init(&vvs->rx_queue); |
06a8fc78 AH |
626 | |
627 | return 0; | |
628 | } | |
629 | EXPORT_SYMBOL_GPL(virtio_transport_do_socket_init); | |
630 | ||
b9f2b0ff SG |
631 | /* sk_lock held by the caller */ |
632 | void virtio_transport_notify_buffer_size(struct vsock_sock *vsk, u64 *val) | |
06a8fc78 AH |
633 | { |
634 | struct virtio_vsock_sock *vvs = vsk->trans; | |
635 | ||
b9f2b0ff SG |
636 | if (*val > VIRTIO_VSOCK_MAX_BUF_SIZE) |
637 | *val = VIRTIO_VSOCK_MAX_BUF_SIZE; | |
06a8fc78 | 638 | |
b9f2b0ff | 639 | vvs->buf_alloc = *val; |
ec3359b6 | 640 | |
c10844c5 | 641 | virtio_transport_send_credit_update(vsk); |
06a8fc78 | 642 | } |
b9f2b0ff | 643 | EXPORT_SYMBOL_GPL(virtio_transport_notify_buffer_size); |
06a8fc78 AH |
644 | |
645 | int | |
646 | virtio_transport_notify_poll_in(struct vsock_sock *vsk, | |
647 | size_t target, | |
648 | bool *data_ready_now) | |
649 | { | |
e7a3266c | 650 | *data_ready_now = vsock_stream_has_data(vsk) >= target; |
06a8fc78 AH |
651 | |
652 | return 0; | |
653 | } | |
654 | EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_in); | |
655 | ||
656 | int | |
657 | virtio_transport_notify_poll_out(struct vsock_sock *vsk, | |
658 | size_t target, | |
659 | bool *space_avail_now) | |
660 | { | |
661 | s64 free_space; | |
662 | ||
663 | free_space = vsock_stream_has_space(vsk); | |
664 | if (free_space > 0) | |
665 | *space_avail_now = true; | |
666 | else if (free_space == 0) | |
667 | *space_avail_now = false; | |
668 | ||
669 | return 0; | |
670 | } | |
671 | EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_out); | |
672 | ||
673 | int virtio_transport_notify_recv_init(struct vsock_sock *vsk, | |
674 | size_t target, struct vsock_transport_recv_notify_data *data) | |
675 | { | |
676 | return 0; | |
677 | } | |
678 | EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_init); | |
679 | ||
680 | int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk, | |
681 | size_t target, struct vsock_transport_recv_notify_data *data) | |
682 | { | |
683 | return 0; | |
684 | } | |
685 | EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_block); | |
686 | ||
687 | int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk, | |
688 | size_t target, struct vsock_transport_recv_notify_data *data) | |
689 | { | |
690 | return 0; | |
691 | } | |
692 | EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_dequeue); | |
693 | ||
694 | int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk, | |
695 | size_t target, ssize_t copied, bool data_read, | |
696 | struct vsock_transport_recv_notify_data *data) | |
697 | { | |
698 | return 0; | |
699 | } | |
700 | EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_post_dequeue); | |
701 | ||
702 | int virtio_transport_notify_send_init(struct vsock_sock *vsk, | |
703 | struct vsock_transport_send_notify_data *data) | |
704 | { | |
705 | return 0; | |
706 | } | |
707 | EXPORT_SYMBOL_GPL(virtio_transport_notify_send_init); | |
708 | ||
709 | int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk, | |
710 | struct vsock_transport_send_notify_data *data) | |
711 | { | |
712 | return 0; | |
713 | } | |
714 | EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_block); | |
715 | ||
716 | int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk, | |
717 | struct vsock_transport_send_notify_data *data) | |
718 | { | |
719 | return 0; | |
720 | } | |
721 | EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_enqueue); | |
722 | ||
723 | int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk, | |
724 | ssize_t written, struct vsock_transport_send_notify_data *data) | |
725 | { | |
726 | return 0; | |
727 | } | |
728 | EXPORT_SYMBOL_GPL(virtio_transport_notify_send_post_enqueue); | |
729 | ||
730 | u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk) | |
731 | { | |
b9f2b0ff | 732 | return vsk->buffer_size; |
06a8fc78 AH |
733 | } |
734 | EXPORT_SYMBOL_GPL(virtio_transport_stream_rcvhiwat); | |
735 | ||
736 | bool virtio_transport_stream_is_active(struct vsock_sock *vsk) | |
737 | { | |
738 | return true; | |
739 | } | |
740 | EXPORT_SYMBOL_GPL(virtio_transport_stream_is_active); | |
741 | ||
742 | bool virtio_transport_stream_allow(u32 cid, u32 port) | |
743 | { | |
744 | return true; | |
745 | } | |
746 | EXPORT_SYMBOL_GPL(virtio_transport_stream_allow); | |
747 | ||
748 | int virtio_transport_dgram_bind(struct vsock_sock *vsk, | |
749 | struct sockaddr_vm *addr) | |
750 | { | |
751 | return -EOPNOTSUPP; | |
752 | } | |
753 | EXPORT_SYMBOL_GPL(virtio_transport_dgram_bind); | |
754 | ||
755 | bool virtio_transport_dgram_allow(u32 cid, u32 port) | |
756 | { | |
757 | return false; | |
758 | } | |
759 | EXPORT_SYMBOL_GPL(virtio_transport_dgram_allow); | |
760 | ||
761 | int virtio_transport_connect(struct vsock_sock *vsk) | |
762 | { | |
763 | struct virtio_vsock_pkt_info info = { | |
764 | .op = VIRTIO_VSOCK_OP_REQUEST, | |
36d277ba | 765 | .vsk = vsk, |
06a8fc78 AH |
766 | }; |
767 | ||
768 | return virtio_transport_send_pkt_info(vsk, &info); | |
769 | } | |
770 | EXPORT_SYMBOL_GPL(virtio_transport_connect); | |
771 | ||
772 | int virtio_transport_shutdown(struct vsock_sock *vsk, int mode) | |
773 | { | |
774 | struct virtio_vsock_pkt_info info = { | |
775 | .op = VIRTIO_VSOCK_OP_SHUTDOWN, | |
06a8fc78 AH |
776 | .flags = (mode & RCV_SHUTDOWN ? |
777 | VIRTIO_VSOCK_SHUTDOWN_RCV : 0) | | |
778 | (mode & SEND_SHUTDOWN ? | |
779 | VIRTIO_VSOCK_SHUTDOWN_SEND : 0), | |
36d277ba | 780 | .vsk = vsk, |
06a8fc78 AH |
781 | }; |
782 | ||
783 | return virtio_transport_send_pkt_info(vsk, &info); | |
784 | } | |
785 | EXPORT_SYMBOL_GPL(virtio_transport_shutdown); | |
786 | ||
787 | int | |
788 | virtio_transport_dgram_enqueue(struct vsock_sock *vsk, | |
789 | struct sockaddr_vm *remote_addr, | |
790 | struct msghdr *msg, | |
791 | size_t dgram_len) | |
792 | { | |
793 | return -EOPNOTSUPP; | |
794 | } | |
795 | EXPORT_SYMBOL_GPL(virtio_transport_dgram_enqueue); | |
796 | ||
797 | ssize_t | |
798 | virtio_transport_stream_enqueue(struct vsock_sock *vsk, | |
799 | struct msghdr *msg, | |
800 | size_t len) | |
801 | { | |
802 | struct virtio_vsock_pkt_info info = { | |
803 | .op = VIRTIO_VSOCK_OP_RW, | |
06a8fc78 AH |
804 | .msg = msg, |
805 | .pkt_len = len, | |
36d277ba | 806 | .vsk = vsk, |
06a8fc78 AH |
807 | }; |
808 | ||
809 | return virtio_transport_send_pkt_info(vsk, &info); | |
810 | } | |
811 | EXPORT_SYMBOL_GPL(virtio_transport_stream_enqueue); | |
812 | ||
813 | void virtio_transport_destruct(struct vsock_sock *vsk) | |
814 | { | |
815 | struct virtio_vsock_sock *vvs = vsk->trans; | |
816 | ||
817 | kfree(vvs); | |
818 | } | |
819 | EXPORT_SYMBOL_GPL(virtio_transport_destruct); | |
820 | ||
821 | static int virtio_transport_reset(struct vsock_sock *vsk, | |
71dc9ec9 | 822 | struct sk_buff *skb) |
06a8fc78 AH |
823 | { |
824 | struct virtio_vsock_pkt_info info = { | |
825 | .op = VIRTIO_VSOCK_OP_RST, | |
71dc9ec9 | 826 | .reply = !!skb, |
36d277ba | 827 | .vsk = vsk, |
06a8fc78 AH |
828 | }; |
829 | ||
830 | /* Send RST only if the original pkt is not a RST pkt */ | |
71dc9ec9 | 831 | if (skb && le16_to_cpu(virtio_vsock_hdr(skb)->op) == VIRTIO_VSOCK_OP_RST) |
06a8fc78 AH |
832 | return 0; |
833 | ||
834 | return virtio_transport_send_pkt_info(vsk, &info); | |
835 | } | |
836 | ||
837 | /* Normally packets are associated with a socket. There may be no socket if an | |
838 | * attempt was made to connect to a socket that does not exist. | |
839 | */ | |
4c7246dc | 840 | static int virtio_transport_reset_no_sock(const struct virtio_transport *t, |
71dc9ec9 | 841 | struct sk_buff *skb) |
06a8fc78 | 842 | { |
71dc9ec9 | 843 | struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); |
06a8fc78 AH |
844 | struct virtio_vsock_pkt_info info = { |
845 | .op = VIRTIO_VSOCK_OP_RST, | |
71dc9ec9 | 846 | .type = le16_to_cpu(hdr->type), |
06a8fc78 AH |
847 | .reply = true, |
848 | }; | |
71dc9ec9 | 849 | struct sk_buff *reply; |
06a8fc78 AH |
850 | |
851 | /* Send RST only if the original pkt is not a RST pkt */ | |
71dc9ec9 | 852 | if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST) |
06a8fc78 AH |
853 | return 0; |
854 | ||
71dc9ec9 BE |
855 | reply = virtio_transport_alloc_skb(&info, 0, |
856 | le64_to_cpu(hdr->dst_cid), | |
857 | le32_to_cpu(hdr->dst_port), | |
858 | le64_to_cpu(hdr->src_cid), | |
859 | le32_to_cpu(hdr->src_port)); | |
4c404ce2 | 860 | if (!reply) |
06a8fc78 AH |
861 | return -ENOMEM; |
862 | ||
4c404ce2 | 863 | if (!t) { |
71dc9ec9 | 864 | kfree_skb(reply); |
4c404ce2 AL |
865 | return -ENOTCONN; |
866 | } | |
867 | ||
868 | return t->send_pkt(reply); | |
06a8fc78 AH |
869 | } |
870 | ||
8432b811 SG |
871 | /* This function should be called with sk_lock held and SOCK_DONE set */ |
872 | static void virtio_transport_remove_sock(struct vsock_sock *vsk) | |
873 | { | |
874 | struct virtio_vsock_sock *vvs = vsk->trans; | |
8432b811 SG |
875 | |
876 | /* We don't need to take rx_lock, as the socket is closing and we are | |
877 | * removing it. | |
878 | */ | |
71dc9ec9 | 879 | __skb_queue_purge(&vvs->rx_queue); |
8432b811 SG |
880 | vsock_remove_sock(vsk); |
881 | } | |
882 | ||
06a8fc78 AH |
883 | static void virtio_transport_wait_close(struct sock *sk, long timeout) |
884 | { | |
885 | if (timeout) { | |
d9dc8b0f WC |
886 | DEFINE_WAIT_FUNC(wait, woken_wake_function); |
887 | ||
888 | add_wait_queue(sk_sleep(sk), &wait); | |
06a8fc78 AH |
889 | |
890 | do { | |
06a8fc78 | 891 | if (sk_wait_event(sk, &timeout, |
d9dc8b0f | 892 | sock_flag(sk, SOCK_DONE), &wait)) |
06a8fc78 AH |
893 | break; |
894 | } while (!signal_pending(current) && timeout); | |
895 | ||
d9dc8b0f | 896 | remove_wait_queue(sk_sleep(sk), &wait); |
06a8fc78 AH |
897 | } |
898 | } | |
899 | ||
900 | static void virtio_transport_do_close(struct vsock_sock *vsk, | |
901 | bool cancel_timeout) | |
902 | { | |
903 | struct sock *sk = sk_vsock(vsk); | |
904 | ||
905 | sock_set_flag(sk, SOCK_DONE); | |
906 | vsk->peer_shutdown = SHUTDOWN_MASK; | |
907 | if (vsock_stream_has_data(vsk) <= 0) | |
3b4477d2 | 908 | sk->sk_state = TCP_CLOSING; |
06a8fc78 AH |
909 | sk->sk_state_change(sk); |
910 | ||
911 | if (vsk->close_work_scheduled && | |
912 | (!cancel_timeout || cancel_delayed_work(&vsk->close_work))) { | |
913 | vsk->close_work_scheduled = false; | |
914 | ||
8432b811 | 915 | virtio_transport_remove_sock(vsk); |
06a8fc78 AH |
916 | |
917 | /* Release refcnt obtained when we scheduled the timeout */ | |
918 | sock_put(sk); | |
919 | } | |
920 | } | |
921 | ||
922 | static void virtio_transport_close_timeout(struct work_struct *work) | |
923 | { | |
924 | struct vsock_sock *vsk = | |
925 | container_of(work, struct vsock_sock, close_work.work); | |
926 | struct sock *sk = sk_vsock(vsk); | |
927 | ||
928 | sock_hold(sk); | |
929 | lock_sock(sk); | |
930 | ||
931 | if (!sock_flag(sk, SOCK_DONE)) { | |
932 | (void)virtio_transport_reset(vsk, NULL); | |
933 | ||
934 | virtio_transport_do_close(vsk, false); | |
935 | } | |
936 | ||
937 | vsk->close_work_scheduled = false; | |
938 | ||
939 | release_sock(sk); | |
940 | sock_put(sk); | |
941 | } | |
942 | ||
943 | /* User context, vsk->sk is locked */ | |
944 | static bool virtio_transport_close(struct vsock_sock *vsk) | |
945 | { | |
946 | struct sock *sk = &vsk->sk; | |
947 | ||
3b4477d2 SH |
948 | if (!(sk->sk_state == TCP_ESTABLISHED || |
949 | sk->sk_state == TCP_CLOSING)) | |
06a8fc78 AH |
950 | return true; |
951 | ||
952 | /* Already received SHUTDOWN from peer, reply with RST */ | |
953 | if ((vsk->peer_shutdown & SHUTDOWN_MASK) == SHUTDOWN_MASK) { | |
954 | (void)virtio_transport_reset(vsk, NULL); | |
955 | return true; | |
956 | } | |
957 | ||
958 | if ((sk->sk_shutdown & SHUTDOWN_MASK) != SHUTDOWN_MASK) | |
959 | (void)virtio_transport_shutdown(vsk, SHUTDOWN_MASK); | |
960 | ||
961 | if (sock_flag(sk, SOCK_LINGER) && !(current->flags & PF_EXITING)) | |
962 | virtio_transport_wait_close(sk, sk->sk_lingertime); | |
963 | ||
964 | if (sock_flag(sk, SOCK_DONE)) { | |
965 | return true; | |
966 | } | |
967 | ||
968 | sock_hold(sk); | |
969 | INIT_DELAYED_WORK(&vsk->close_work, | |
970 | virtio_transport_close_timeout); | |
971 | vsk->close_work_scheduled = true; | |
972 | schedule_delayed_work(&vsk->close_work, VSOCK_CLOSE_TIMEOUT); | |
973 | return false; | |
974 | } | |
975 | ||
976 | void virtio_transport_release(struct vsock_sock *vsk) | |
977 | { | |
978 | struct sock *sk = &vsk->sk; | |
979 | bool remove_sock = true; | |
980 | ||
9ac841f5 | 981 | if (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) |
06a8fc78 | 982 | remove_sock = virtio_transport_close(vsk); |
ac03046e | 983 | |
3fe356d5 SG |
984 | if (remove_sock) { |
985 | sock_set_flag(sk, SOCK_DONE); | |
8432b811 | 986 | virtio_transport_remove_sock(vsk); |
3fe356d5 | 987 | } |
06a8fc78 AH |
988 | } |
989 | EXPORT_SYMBOL_GPL(virtio_transport_release); | |
990 | ||
991 | static int | |
992 | virtio_transport_recv_connecting(struct sock *sk, | |
71dc9ec9 | 993 | struct sk_buff *skb) |
06a8fc78 | 994 | { |
71dc9ec9 | 995 | struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); |
06a8fc78 | 996 | struct vsock_sock *vsk = vsock_sk(sk); |
06a8fc78 | 997 | int skerr; |
71dc9ec9 | 998 | int err; |
06a8fc78 | 999 | |
71dc9ec9 | 1000 | switch (le16_to_cpu(hdr->op)) { |
06a8fc78 | 1001 | case VIRTIO_VSOCK_OP_RESPONSE: |
3b4477d2 | 1002 | sk->sk_state = TCP_ESTABLISHED; |
06a8fc78 AH |
1003 | sk->sk_socket->state = SS_CONNECTED; |
1004 | vsock_insert_connected(vsk); | |
1005 | sk->sk_state_change(sk); | |
1006 | break; | |
1007 | case VIRTIO_VSOCK_OP_INVALID: | |
1008 | break; | |
1009 | case VIRTIO_VSOCK_OP_RST: | |
1010 | skerr = ECONNRESET; | |
1011 | err = 0; | |
1012 | goto destroy; | |
1013 | default: | |
1014 | skerr = EPROTO; | |
1015 | err = -EINVAL; | |
1016 | goto destroy; | |
1017 | } | |
1018 | return 0; | |
1019 | ||
1020 | destroy: | |
71dc9ec9 | 1021 | virtio_transport_reset(vsk, skb); |
3b4477d2 | 1022 | sk->sk_state = TCP_CLOSE; |
06a8fc78 | 1023 | sk->sk_err = skerr; |
e3ae2365 | 1024 | sk_error_report(sk); |
06a8fc78 AH |
1025 | return err; |
1026 | } | |
1027 | ||
473c7391 SG |
1028 | static void |
1029 | virtio_transport_recv_enqueue(struct vsock_sock *vsk, | |
71dc9ec9 | 1030 | struct sk_buff *skb) |
473c7391 SG |
1031 | { |
1032 | struct virtio_vsock_sock *vvs = vsk->trans; | |
ae6fcfbf | 1033 | bool can_enqueue, free_pkt = false; |
71dc9ec9 BE |
1034 | struct virtio_vsock_hdr *hdr; |
1035 | u32 len; | |
473c7391 | 1036 | |
71dc9ec9 BE |
1037 | hdr = virtio_vsock_hdr(skb); |
1038 | len = le32_to_cpu(hdr->len); | |
473c7391 SG |
1039 | |
1040 | spin_lock_bh(&vvs->rx_lock); | |
1041 | ||
07770616 | 1042 | can_enqueue = virtio_transport_inc_rx_pkt(vvs, len); |
ae6fcfbf SG |
1043 | if (!can_enqueue) { |
1044 | free_pkt = true; | |
1045 | goto out; | |
1046 | } | |
473c7391 | 1047 | |
71dc9ec9 | 1048 | if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) |
e4b1ef15 AK |
1049 | vvs->msg_count++; |
1050 | ||
473c7391 SG |
1051 | /* Try to copy small packets into the buffer of last packet queued, |
1052 | * to avoid wasting memory queueing the entire buffer with a small | |
1053 | * payload. | |
1054 | */ | |
71dc9ec9 BE |
1055 | if (len <= GOOD_COPY_LEN && !skb_queue_empty(&vvs->rx_queue)) { |
1056 | struct virtio_vsock_hdr *last_hdr; | |
1057 | struct sk_buff *last_skb; | |
473c7391 | 1058 | |
71dc9ec9 BE |
1059 | last_skb = skb_peek_tail(&vvs->rx_queue); |
1060 | last_hdr = virtio_vsock_hdr(last_skb); | |
473c7391 SG |
1061 | |
1062 | /* If there is space in the last packet queued, we copy the | |
e4b1ef15 | 1063 | * new packet in its buffer. We avoid this if the last packet |
9af8f106 AK |
1064 | * queued has VIRTIO_VSOCK_SEQ_EOM set, because this is |
1065 | * delimiter of SEQPACKET message, so 'pkt' is the first packet | |
1066 | * of a new message. | |
473c7391 | 1067 | */ |
71dc9ec9 BE |
1068 | if (skb->len < skb_tailroom(last_skb) && |
1069 | !(le32_to_cpu(last_hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)) { | |
1070 | memcpy(skb_put(last_skb, skb->len), skb->data, skb->len); | |
473c7391 | 1071 | free_pkt = true; |
71dc9ec9 BE |
1072 | last_hdr->flags |= hdr->flags; |
1073 | last_hdr->len = cpu_to_le32(last_skb->len); | |
473c7391 SG |
1074 | goto out; |
1075 | } | |
1076 | } | |
1077 | ||
71dc9ec9 | 1078 | __skb_queue_tail(&vvs->rx_queue, skb); |
473c7391 SG |
1079 | |
1080 | out: | |
1081 | spin_unlock_bh(&vvs->rx_lock); | |
1082 | if (free_pkt) | |
71dc9ec9 | 1083 | kfree_skb(skb); |
473c7391 SG |
1084 | } |
1085 | ||
06a8fc78 AH |
1086 | static int |
1087 | virtio_transport_recv_connected(struct sock *sk, | |
71dc9ec9 | 1088 | struct sk_buff *skb) |
06a8fc78 | 1089 | { |
71dc9ec9 | 1090 | struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); |
06a8fc78 | 1091 | struct vsock_sock *vsk = vsock_sk(sk); |
06a8fc78 AH |
1092 | int err = 0; |
1093 | ||
71dc9ec9 | 1094 | switch (le16_to_cpu(hdr->op)) { |
06a8fc78 | 1095 | case VIRTIO_VSOCK_OP_RW: |
71dc9ec9 | 1096 | virtio_transport_recv_enqueue(vsk, skb); |
39f1ed33 | 1097 | vsock_data_ready(sk); |
06a8fc78 | 1098 | return err; |
e3ea110d HU |
1099 | case VIRTIO_VSOCK_OP_CREDIT_REQUEST: |
1100 | virtio_transport_send_credit_update(vsk); | |
1101 | break; | |
06a8fc78 AH |
1102 | case VIRTIO_VSOCK_OP_CREDIT_UPDATE: |
1103 | sk->sk_write_space(sk); | |
1104 | break; | |
1105 | case VIRTIO_VSOCK_OP_SHUTDOWN: | |
71dc9ec9 | 1106 | if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_RCV) |
06a8fc78 | 1107 | vsk->peer_shutdown |= RCV_SHUTDOWN; |
71dc9ec9 | 1108 | if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_SEND) |
06a8fc78 AH |
1109 | vsk->peer_shutdown |= SEND_SHUTDOWN; |
1110 | if (vsk->peer_shutdown == SHUTDOWN_MASK && | |
ad8a7220 SG |
1111 | vsock_stream_has_data(vsk) <= 0 && |
1112 | !sock_flag(sk, SOCK_DONE)) { | |
1113 | (void)virtio_transport_reset(vsk, NULL); | |
ad8a7220 | 1114 | virtio_transport_do_close(vsk, true); |
42f5cda5 | 1115 | } |
71dc9ec9 | 1116 | if (le32_to_cpu(virtio_vsock_hdr(skb)->flags)) |
06a8fc78 AH |
1117 | sk->sk_state_change(sk); |
1118 | break; | |
1119 | case VIRTIO_VSOCK_OP_RST: | |
1120 | virtio_transport_do_close(vsk, true); | |
1121 | break; | |
1122 | default: | |
1123 | err = -EINVAL; | |
1124 | break; | |
1125 | } | |
1126 | ||
71dc9ec9 | 1127 | kfree_skb(skb); |
06a8fc78 AH |
1128 | return err; |
1129 | } | |
1130 | ||
1131 | static void | |
1132 | virtio_transport_recv_disconnecting(struct sock *sk, | |
71dc9ec9 | 1133 | struct sk_buff *skb) |
06a8fc78 | 1134 | { |
71dc9ec9 | 1135 | struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); |
06a8fc78 AH |
1136 | struct vsock_sock *vsk = vsock_sk(sk); |
1137 | ||
71dc9ec9 | 1138 | if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST) |
06a8fc78 AH |
1139 | virtio_transport_do_close(vsk, true); |
1140 | } | |
1141 | ||
1142 | static int | |
1143 | virtio_transport_send_response(struct vsock_sock *vsk, | |
71dc9ec9 | 1144 | struct sk_buff *skb) |
06a8fc78 | 1145 | { |
71dc9ec9 | 1146 | struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); |
06a8fc78 AH |
1147 | struct virtio_vsock_pkt_info info = { |
1148 | .op = VIRTIO_VSOCK_OP_RESPONSE, | |
71dc9ec9 BE |
1149 | .remote_cid = le64_to_cpu(hdr->src_cid), |
1150 | .remote_port = le32_to_cpu(hdr->src_port), | |
06a8fc78 | 1151 | .reply = true, |
36d277ba | 1152 | .vsk = vsk, |
06a8fc78 AH |
1153 | }; |
1154 | ||
1155 | return virtio_transport_send_pkt_info(vsk, &info); | |
1156 | } | |
1157 | ||
c0cfa2d8 | 1158 | static bool virtio_transport_space_update(struct sock *sk, |
71dc9ec9 | 1159 | struct sk_buff *skb) |
c0cfa2d8 | 1160 | { |
71dc9ec9 | 1161 | struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); |
c0cfa2d8 SG |
1162 | struct vsock_sock *vsk = vsock_sk(sk); |
1163 | struct virtio_vsock_sock *vvs = vsk->trans; | |
1164 | bool space_available; | |
1165 | ||
1166 | /* Listener sockets are not associated with any transport, so we are | |
1167 | * not able to take the state to see if there is space available in the | |
1168 | * remote peer, but since they are only used to receive requests, we | |
1169 | * can assume that there is always space available in the other peer. | |
1170 | */ | |
1171 | if (!vvs) | |
1172 | return true; | |
1173 | ||
1174 | /* buf_alloc and fwd_cnt is always included in the hdr */ | |
1175 | spin_lock_bh(&vvs->tx_lock); | |
71dc9ec9 BE |
1176 | vvs->peer_buf_alloc = le32_to_cpu(hdr->buf_alloc); |
1177 | vvs->peer_fwd_cnt = le32_to_cpu(hdr->fwd_cnt); | |
c0cfa2d8 SG |
1178 | space_available = virtio_transport_has_space(vsk); |
1179 | spin_unlock_bh(&vvs->tx_lock); | |
1180 | return space_available; | |
1181 | } | |
1182 | ||
06a8fc78 AH |
1183 | /* Handle server socket */ |
1184 | static int | |
71dc9ec9 | 1185 | virtio_transport_recv_listen(struct sock *sk, struct sk_buff *skb, |
c0cfa2d8 | 1186 | struct virtio_transport *t) |
06a8fc78 | 1187 | { |
71dc9ec9 | 1188 | struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); |
06a8fc78 AH |
1189 | struct vsock_sock *vsk = vsock_sk(sk); |
1190 | struct vsock_sock *vchild; | |
1191 | struct sock *child; | |
c0cfa2d8 | 1192 | int ret; |
06a8fc78 | 1193 | |
71dc9ec9 BE |
1194 | if (le16_to_cpu(hdr->op) != VIRTIO_VSOCK_OP_REQUEST) { |
1195 | virtio_transport_reset_no_sock(t, skb); | |
06a8fc78 AH |
1196 | return -EINVAL; |
1197 | } | |
1198 | ||
1199 | if (sk_acceptq_is_full(sk)) { | |
71dc9ec9 | 1200 | virtio_transport_reset_no_sock(t, skb); |
06a8fc78 AH |
1201 | return -ENOMEM; |
1202 | } | |
1203 | ||
b9ca2f5f | 1204 | child = vsock_create_connected(sk); |
06a8fc78 | 1205 | if (!child) { |
71dc9ec9 | 1206 | virtio_transport_reset_no_sock(t, skb); |
06a8fc78 AH |
1207 | return -ENOMEM; |
1208 | } | |
1209 | ||
7976a11b | 1210 | sk_acceptq_added(sk); |
06a8fc78 AH |
1211 | |
1212 | lock_sock_nested(child, SINGLE_DEPTH_NESTING); | |
1213 | ||
3b4477d2 | 1214 | child->sk_state = TCP_ESTABLISHED; |
06a8fc78 AH |
1215 | |
1216 | vchild = vsock_sk(child); | |
71dc9ec9 BE |
1217 | vsock_addr_init(&vchild->local_addr, le64_to_cpu(hdr->dst_cid), |
1218 | le32_to_cpu(hdr->dst_port)); | |
1219 | vsock_addr_init(&vchild->remote_addr, le64_to_cpu(hdr->src_cid), | |
1220 | le32_to_cpu(hdr->src_port)); | |
06a8fc78 | 1221 | |
c0cfa2d8 SG |
1222 | ret = vsock_assign_transport(vchild, vsk); |
1223 | /* Transport assigned (looking at remote_addr) must be the same | |
1224 | * where we received the request. | |
1225 | */ | |
1226 | if (ret || vchild->transport != &t->transport) { | |
1227 | release_sock(child); | |
71dc9ec9 | 1228 | virtio_transport_reset_no_sock(t, skb); |
c0cfa2d8 SG |
1229 | sock_put(child); |
1230 | return ret; | |
1231 | } | |
1232 | ||
71dc9ec9 | 1233 | if (virtio_transport_space_update(child, skb)) |
c0cfa2d8 SG |
1234 | child->sk_write_space(child); |
1235 | ||
06a8fc78 AH |
1236 | vsock_insert_connected(vchild); |
1237 | vsock_enqueue_accept(sk, child); | |
71dc9ec9 | 1238 | virtio_transport_send_response(vchild, skb); |
06a8fc78 AH |
1239 | |
1240 | release_sock(child); | |
1241 | ||
1242 | sk->sk_data_ready(sk); | |
1243 | return 0; | |
1244 | } | |
1245 | ||
e4b1ef15 AK |
1246 | static bool virtio_transport_valid_type(u16 type) |
1247 | { | |
1248 | return (type == VIRTIO_VSOCK_TYPE_STREAM) || | |
1249 | (type == VIRTIO_VSOCK_TYPE_SEQPACKET); | |
1250 | } | |
1251 | ||
06a8fc78 AH |
1252 | /* We are under the virtio-vsock's vsock->rx_lock or vhost-vsock's vq->mutex |
1253 | * lock. | |
1254 | */ | |
4c7246dc | 1255 | void virtio_transport_recv_pkt(struct virtio_transport *t, |
71dc9ec9 | 1256 | struct sk_buff *skb) |
06a8fc78 | 1257 | { |
71dc9ec9 | 1258 | struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); |
06a8fc78 AH |
1259 | struct sockaddr_vm src, dst; |
1260 | struct vsock_sock *vsk; | |
1261 | struct sock *sk; | |
1262 | bool space_available; | |
1263 | ||
71dc9ec9 BE |
1264 | vsock_addr_init(&src, le64_to_cpu(hdr->src_cid), |
1265 | le32_to_cpu(hdr->src_port)); | |
1266 | vsock_addr_init(&dst, le64_to_cpu(hdr->dst_cid), | |
1267 | le32_to_cpu(hdr->dst_port)); | |
06a8fc78 AH |
1268 | |
1269 | trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port, | |
1270 | dst.svm_cid, dst.svm_port, | |
71dc9ec9 BE |
1271 | le32_to_cpu(hdr->len), |
1272 | le16_to_cpu(hdr->type), | |
1273 | le16_to_cpu(hdr->op), | |
1274 | le32_to_cpu(hdr->flags), | |
1275 | le32_to_cpu(hdr->buf_alloc), | |
1276 | le32_to_cpu(hdr->fwd_cnt)); | |
1277 | ||
1278 | if (!virtio_transport_valid_type(le16_to_cpu(hdr->type))) { | |
1279 | (void)virtio_transport_reset_no_sock(t, skb); | |
06a8fc78 AH |
1280 | goto free_pkt; |
1281 | } | |
1282 | ||
1283 | /* The socket must be in connected or bound table | |
1284 | * otherwise send reset back | |
1285 | */ | |
1286 | sk = vsock_find_connected_socket(&src, &dst); | |
1287 | if (!sk) { | |
1288 | sk = vsock_find_bound_socket(&dst); | |
1289 | if (!sk) { | |
71dc9ec9 | 1290 | (void)virtio_transport_reset_no_sock(t, skb); |
06a8fc78 AH |
1291 | goto free_pkt; |
1292 | } | |
1293 | } | |
1294 | ||
71dc9ec9 BE |
1295 | if (virtio_transport_get_type(sk) != le16_to_cpu(hdr->type)) { |
1296 | (void)virtio_transport_reset_no_sock(t, skb); | |
e4b1ef15 AK |
1297 | sock_put(sk); |
1298 | goto free_pkt; | |
1299 | } | |
1300 | ||
06a8fc78 AH |
1301 | vsk = vsock_sk(sk); |
1302 | ||
06a8fc78 AH |
1303 | lock_sock(sk); |
1304 | ||
3fe356d5 SG |
1305 | /* Check if sk has been closed before lock_sock */ |
1306 | if (sock_flag(sk, SOCK_DONE)) { | |
71dc9ec9 | 1307 | (void)virtio_transport_reset_no_sock(t, skb); |
8692cefc JH |
1308 | release_sock(sk); |
1309 | sock_put(sk); | |
1310 | goto free_pkt; | |
1311 | } | |
1312 | ||
71dc9ec9 | 1313 | space_available = virtio_transport_space_update(sk, skb); |
ce7536bc | 1314 | |
06a8fc78 | 1315 | /* Update CID in case it has changed after a transport reset event */ |
1db8f5fc WW |
1316 | if (vsk->local_addr.svm_cid != VMADDR_CID_ANY) |
1317 | vsk->local_addr.svm_cid = dst.svm_cid; | |
06a8fc78 AH |
1318 | |
1319 | if (space_available) | |
1320 | sk->sk_write_space(sk); | |
1321 | ||
1322 | switch (sk->sk_state) { | |
3b4477d2 | 1323 | case TCP_LISTEN: |
71dc9ec9 BE |
1324 | virtio_transport_recv_listen(sk, skb, t); |
1325 | kfree_skb(skb); | |
06a8fc78 | 1326 | break; |
3b4477d2 | 1327 | case TCP_SYN_SENT: |
71dc9ec9 BE |
1328 | virtio_transport_recv_connecting(sk, skb); |
1329 | kfree_skb(skb); | |
06a8fc78 | 1330 | break; |
3b4477d2 | 1331 | case TCP_ESTABLISHED: |
71dc9ec9 | 1332 | virtio_transport_recv_connected(sk, skb); |
06a8fc78 | 1333 | break; |
3b4477d2 | 1334 | case TCP_CLOSING: |
71dc9ec9 BE |
1335 | virtio_transport_recv_disconnecting(sk, skb); |
1336 | kfree_skb(skb); | |
06a8fc78 AH |
1337 | break; |
1338 | default: | |
71dc9ec9 BE |
1339 | (void)virtio_transport_reset_no_sock(t, skb); |
1340 | kfree_skb(skb); | |
06a8fc78 AH |
1341 | break; |
1342 | } | |
c0cfa2d8 | 1343 | |
06a8fc78 AH |
1344 | release_sock(sk); |
1345 | ||
1346 | /* Release refcnt obtained when we fetched this socket out of the | |
1347 | * bound or connected list. | |
1348 | */ | |
1349 | sock_put(sk); | |
1350 | return; | |
1351 | ||
1352 | free_pkt: | |
71dc9ec9 | 1353 | kfree_skb(skb); |
06a8fc78 AH |
1354 | } |
1355 | EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt); | |
1356 | ||
71dc9ec9 BE |
1357 | /* Remove skbs found in a queue that have a vsk that matches. |
1358 | * | |
1359 | * Each skb is freed. | |
1360 | * | |
1361 | * Returns the count of skbs that were reply packets. | |
1362 | */ | |
1363 | int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *queue) | |
06a8fc78 | 1364 | { |
71dc9ec9 BE |
1365 | struct sk_buff_head freeme; |
1366 | struct sk_buff *skb, *tmp; | |
1367 | int cnt = 0; | |
1368 | ||
1369 | skb_queue_head_init(&freeme); | |
1370 | ||
1371 | spin_lock_bh(&queue->lock); | |
1372 | skb_queue_walk_safe(queue, skb, tmp) { | |
1373 | if (vsock_sk(skb->sk) != vsk) | |
1374 | continue; | |
1375 | ||
1376 | __skb_unlink(skb, queue); | |
1377 | __skb_queue_tail(&freeme, skb); | |
1378 | ||
1379 | if (virtio_vsock_skb_reply(skb)) | |
1380 | cnt++; | |
1381 | } | |
1382 | spin_unlock_bh(&queue->lock); | |
1383 | ||
1384 | __skb_queue_purge(&freeme); | |
1385 | ||
1386 | return cnt; | |
06a8fc78 | 1387 | } |
71dc9ec9 | 1388 | EXPORT_SYMBOL_GPL(virtio_transport_purge_skbs); |
06a8fc78 AH |
1389 | |
1390 | MODULE_LICENSE("GPL v2"); | |
1391 | MODULE_AUTHOR("Asias He"); | |
1392 | MODULE_DESCRIPTION("common code for virtio vsock"); |