Commit | Line | Data |
---|---|---|
604326b4 DB |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ | |
3 | ||
4 | #include <linux/skmsg.h> | |
5 | #include <linux/filter.h> | |
6 | #include <linux/bpf.h> | |
7 | #include <linux/init.h> | |
8 | #include <linux/wait.h> | |
9 | ||
10 | #include <net/inet_common.h> | |
0608c69c | 11 | #include <net/tls.h> |
604326b4 | 12 | |
604326b4 | 13 | int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock, |
02c558b2 | 14 | struct msghdr *msg, int len, int flags) |
604326b4 DB |
15 | { |
16 | struct iov_iter *iter = &msg->msg_iter; | |
02c558b2 | 17 | int peek = flags & MSG_PEEK; |
604326b4 | 18 | int i, ret, copied = 0; |
02c558b2 JF |
19 | struct sk_msg *msg_rx; |
20 | ||
21 | msg_rx = list_first_entry_or_null(&psock->ingress_msg, | |
22 | struct sk_msg, list); | |
604326b4 DB |
23 | |
24 | while (copied != len) { | |
25 | struct scatterlist *sge; | |
604326b4 | 26 | |
604326b4 DB |
27 | if (unlikely(!msg_rx)) |
28 | break; | |
29 | ||
30 | i = msg_rx->sg.start; | |
31 | do { | |
32 | struct page *page; | |
33 | int copy; | |
34 | ||
35 | sge = sk_msg_elem(msg_rx, i); | |
36 | copy = sge->length; | |
37 | page = sg_page(sge); | |
38 | if (copied + copy > len) | |
39 | copy = len - copied; | |
40 | ret = copy_page_to_iter(page, sge->offset, copy, iter); | |
41 | if (ret != copy) { | |
42 | msg_rx->sg.start = i; | |
43 | return -EFAULT; | |
44 | } | |
45 | ||
46 | copied += copy; | |
02c558b2 JF |
47 | if (likely(!peek)) { |
48 | sge->offset += copy; | |
49 | sge->length -= copy; | |
50 | sk_mem_uncharge(sk, copy); | |
51 | msg_rx->sg.size -= copy; | |
52 | ||
53 | if (!sge->length) { | |
54 | sk_msg_iter_var_next(i); | |
55 | if (!msg_rx->skb) | |
56 | put_page(page); | |
57 | } | |
58 | } else { | |
59 | sk_msg_iter_var_next(i); | |
604326b4 DB |
60 | } |
61 | ||
62 | if (copied == len) | |
63 | break; | |
64 | } while (i != msg_rx->sg.end); | |
65 | ||
02c558b2 | 66 | if (unlikely(peek)) { |
487082fb | 67 | if (msg_rx == list_last_entry(&psock->ingress_msg, |
68 | struct sk_msg, list)) | |
69 | break; | |
02c558b2 JF |
70 | msg_rx = list_next_entry(msg_rx, list); |
71 | continue; | |
72 | } | |
73 | ||
604326b4 DB |
74 | msg_rx->sg.start = i; |
75 | if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) { | |
76 | list_del(&msg_rx->list); | |
77 | if (msg_rx->skb) | |
78 | consume_skb(msg_rx->skb); | |
79 | kfree(msg_rx); | |
80 | } | |
02c558b2 JF |
81 | msg_rx = list_first_entry_or_null(&psock->ingress_msg, |
82 | struct sk_msg, list); | |
604326b4 DB |
83 | } |
84 | ||
85 | return copied; | |
86 | } | |
87 | EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg); | |
88 | ||
604326b4 DB |
89 | static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, |
90 | struct sk_msg *msg, u32 apply_bytes, int flags) | |
91 | { | |
92 | bool apply = apply_bytes; | |
93 | struct scatterlist *sge; | |
94 | u32 size, copied = 0; | |
95 | struct sk_msg *tmp; | |
96 | int i, ret = 0; | |
97 | ||
98 | tmp = kzalloc(sizeof(*tmp), __GFP_NOWARN | GFP_KERNEL); | |
99 | if (unlikely(!tmp)) | |
100 | return -ENOMEM; | |
101 | ||
102 | lock_sock(sk); | |
103 | tmp->sg.start = msg->sg.start; | |
104 | i = msg->sg.start; | |
105 | do { | |
106 | sge = sk_msg_elem(msg, i); | |
107 | size = (apply && apply_bytes < sge->length) ? | |
108 | apply_bytes : sge->length; | |
109 | if (!sk_wmem_schedule(sk, size)) { | |
110 | if (!copied) | |
111 | ret = -ENOMEM; | |
112 | break; | |
113 | } | |
114 | ||
115 | sk_mem_charge(sk, size); | |
116 | sk_msg_xfer(tmp, msg, i, size); | |
117 | copied += size; | |
118 | if (sge->length) | |
119 | get_page(sk_msg_page(tmp, i)); | |
120 | sk_msg_iter_var_next(i); | |
121 | tmp->sg.end = i; | |
122 | if (apply) { | |
123 | apply_bytes -= size; | |
124 | if (!apply_bytes) | |
125 | break; | |
126 | } | |
127 | } while (i != msg->sg.end); | |
128 | ||
129 | if (!ret) { | |
130 | msg->sg.start = i; | |
604326b4 | 131 | sk_psock_queue_msg(psock, tmp); |
552de910 | 132 | sk_psock_data_ready(sk, psock); |
604326b4 DB |
133 | } else { |
134 | sk_msg_free(sk, tmp); | |
135 | kfree(tmp); | |
136 | } | |
137 | ||
138 | release_sock(sk); | |
139 | return ret; | |
140 | } | |
141 | ||
142 | static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, | |
143 | int flags, bool uncharge) | |
144 | { | |
145 | bool apply = apply_bytes; | |
146 | struct scatterlist *sge; | |
147 | struct page *page; | |
148 | int size, ret = 0; | |
149 | u32 off; | |
150 | ||
151 | while (1) { | |
0608c69c JF |
152 | bool has_tx_ulp; |
153 | ||
604326b4 DB |
154 | sge = sk_msg_elem(msg, msg->sg.start); |
155 | size = (apply && apply_bytes < sge->length) ? | |
156 | apply_bytes : sge->length; | |
157 | off = sge->offset; | |
158 | page = sg_page(sge); | |
159 | ||
160 | tcp_rate_check_app_limited(sk); | |
161 | retry: | |
0608c69c JF |
162 | has_tx_ulp = tls_sw_has_ctx_tx(sk); |
163 | if (has_tx_ulp) { | |
164 | flags |= MSG_SENDPAGE_NOPOLICY; | |
165 | ret = kernel_sendpage_locked(sk, | |
166 | page, off, size, flags); | |
167 | } else { | |
168 | ret = do_tcp_sendpages(sk, page, off, size, flags); | |
169 | } | |
170 | ||
604326b4 DB |
171 | if (ret <= 0) |
172 | return ret; | |
173 | if (apply) | |
174 | apply_bytes -= ret; | |
175 | msg->sg.size -= ret; | |
176 | sge->offset += ret; | |
177 | sge->length -= ret; | |
178 | if (uncharge) | |
179 | sk_mem_uncharge(sk, ret); | |
180 | if (ret != size) { | |
181 | size -= ret; | |
182 | off += ret; | |
183 | goto retry; | |
184 | } | |
185 | if (!sge->length) { | |
186 | put_page(page); | |
187 | sk_msg_iter_next(msg, start); | |
188 | sg_init_table(sge, 1); | |
189 | if (msg->sg.start == msg->sg.end) | |
190 | break; | |
191 | } | |
192 | if (apply && !apply_bytes) | |
193 | break; | |
194 | } | |
195 | ||
196 | return 0; | |
197 | } | |
198 | ||
199 | static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg, | |
200 | u32 apply_bytes, int flags, bool uncharge) | |
201 | { | |
202 | int ret; | |
203 | ||
204 | lock_sock(sk); | |
205 | ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge); | |
206 | release_sock(sk); | |
207 | return ret; | |
208 | } | |
209 | ||
210 | int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, | |
211 | u32 bytes, int flags) | |
212 | { | |
213 | bool ingress = sk_msg_to_ingress(msg); | |
214 | struct sk_psock *psock = sk_psock_get(sk); | |
215 | int ret; | |
216 | ||
217 | if (unlikely(!psock)) { | |
218 | sk_msg_free(sk, msg); | |
219 | return 0; | |
220 | } | |
221 | ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) : | |
222 | tcp_bpf_push_locked(sk, msg, bytes, flags, false); | |
223 | sk_psock_put(sk, psock); | |
224 | return ret; | |
225 | } | |
226 | EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir); | |
227 | ||
a2652798 Y |
228 | #ifdef CONFIG_BPF_STREAM_PARSER |
229 | static bool tcp_bpf_stream_read(const struct sock *sk) | |
230 | { | |
231 | struct sk_psock *psock; | |
232 | bool empty = true; | |
233 | ||
234 | rcu_read_lock(); | |
235 | psock = sk_psock(sk); | |
236 | if (likely(psock)) | |
237 | empty = list_empty(&psock->ingress_msg); | |
238 | rcu_read_unlock(); | |
239 | return !empty; | |
240 | } | |
241 | ||
c0fd336e Y |
242 | static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock, |
243 | int flags, long timeo, int *err) | |
244 | { | |
245 | DEFINE_WAIT_FUNC(wait, woken_wake_function); | |
246 | int ret = 0; | |
247 | ||
2c7269b2 SD |
248 | if (sk->sk_shutdown & RCV_SHUTDOWN) |
249 | return 1; | |
250 | ||
c0fd336e Y |
251 | if (!timeo) |
252 | return ret; | |
253 | ||
254 | add_wait_queue(sk_sleep(sk), &wait); | |
255 | sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); | |
256 | ret = sk_wait_event(sk, &timeo, | |
257 | !list_empty(&psock->ingress_msg) || | |
258 | !skb_queue_empty(&sk->sk_receive_queue), &wait); | |
259 | sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); | |
260 | remove_wait_queue(sk_sleep(sk), &wait); | |
261 | return ret; | |
262 | } | |
263 | ||
264 | static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, | |
265 | int nonblock, int flags, int *addr_len) | |
266 | { | |
267 | struct sk_psock *psock; | |
268 | int copied, ret; | |
269 | ||
18f02ad1 XY |
270 | if (unlikely(flags & MSG_ERRQUEUE)) |
271 | return inet_recv_error(sk, msg, len, addr_len); | |
272 | ||
c0fd336e Y |
273 | psock = sk_psock_get(sk); |
274 | if (unlikely(!psock)) | |
275 | return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len); | |
c0fd336e | 276 | if (!skb_queue_empty(&sk->sk_receive_queue) && |
18f02ad1 XY |
277 | sk_psock_queue_empty(psock)) { |
278 | sk_psock_put(sk, psock); | |
c0fd336e | 279 | return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len); |
18f02ad1 | 280 | } |
c0fd336e Y |
281 | lock_sock(sk); |
282 | msg_bytes_ready: | |
283 | copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags); | |
284 | if (!copied) { | |
285 | int data, err = 0; | |
286 | long timeo; | |
287 | ||
288 | timeo = sock_rcvtimeo(sk, nonblock); | |
289 | data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err); | |
290 | if (data) { | |
291 | if (!sk_psock_queue_empty(psock)) | |
292 | goto msg_bytes_ready; | |
293 | release_sock(sk); | |
294 | sk_psock_put(sk, psock); | |
295 | return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len); | |
296 | } | |
297 | if (err) { | |
298 | ret = err; | |
299 | goto out; | |
300 | } | |
301 | copied = -EAGAIN; | |
302 | } | |
303 | ret = copied; | |
304 | out: | |
305 | release_sock(sk); | |
306 | sk_psock_put(sk, psock); | |
307 | return ret; | |
308 | } | |
309 | ||
604326b4 DB |
310 | static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, |
311 | struct sk_msg *msg, int *copied, int flags) | |
312 | { | |
031097d9 | 313 | bool cork = false, enospc = sk_msg_full(msg); |
604326b4 | 314 | struct sock *sk_redir; |
7246d8ed | 315 | u32 tosend, delta = 0; |
604326b4 DB |
316 | int ret; |
317 | ||
318 | more_data: | |
7246d8ed JF |
319 | if (psock->eval == __SK_NONE) { |
320 | /* Track delta in msg size to add/subtract it on SK_DROP from | |
321 | * returned to user copied size. This ensures user doesn't | |
322 | * get a positive return code with msg_cut_data and SK_DROP | |
323 | * verdict. | |
324 | */ | |
325 | delta = msg->sg.size; | |
604326b4 | 326 | psock->eval = sk_psock_msg_verdict(sk, psock, msg); |
7361d448 | 327 | delta -= msg->sg.size; |
7246d8ed | 328 | } |
604326b4 DB |
329 | |
330 | if (msg->cork_bytes && | |
331 | msg->cork_bytes > msg->sg.size && !enospc) { | |
332 | psock->cork_bytes = msg->cork_bytes - msg->sg.size; | |
333 | if (!psock->cork) { | |
334 | psock->cork = kzalloc(sizeof(*psock->cork), | |
335 | GFP_ATOMIC | __GFP_NOWARN); | |
336 | if (!psock->cork) | |
337 | return -ENOMEM; | |
338 | } | |
339 | memcpy(psock->cork, msg, sizeof(*msg)); | |
340 | return 0; | |
341 | } | |
342 | ||
343 | tosend = msg->sg.size; | |
344 | if (psock->apply_bytes && psock->apply_bytes < tosend) | |
345 | tosend = psock->apply_bytes; | |
346 | ||
347 | switch (psock->eval) { | |
348 | case __SK_PASS: | |
349 | ret = tcp_bpf_push(sk, msg, tosend, flags, true); | |
350 | if (unlikely(ret)) { | |
351 | *copied -= sk_msg_free(sk, msg); | |
352 | break; | |
353 | } | |
354 | sk_msg_apply_bytes(psock, tosend); | |
355 | break; | |
356 | case __SK_REDIRECT: | |
357 | sk_redir = psock->sk_redir; | |
358 | sk_msg_apply_bytes(psock, tosend); | |
359 | if (psock->cork) { | |
360 | cork = true; | |
361 | psock->cork = NULL; | |
362 | } | |
363 | sk_msg_return(sk, msg, tosend); | |
364 | release_sock(sk); | |
365 | ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags); | |
366 | lock_sock(sk); | |
367 | if (unlikely(ret < 0)) { | |
368 | int free = sk_msg_free_nocharge(sk, msg); | |
369 | ||
370 | if (!cork) | |
371 | *copied -= free; | |
372 | } | |
373 | if (cork) { | |
374 | sk_msg_free(sk, msg); | |
375 | kfree(msg); | |
376 | msg = NULL; | |
377 | ret = 0; | |
378 | } | |
379 | break; | |
380 | case __SK_DROP: | |
381 | default: | |
382 | sk_msg_free_partial(sk, msg, tosend); | |
383 | sk_msg_apply_bytes(psock, tosend); | |
7246d8ed | 384 | *copied -= (tosend + delta); |
604326b4 DB |
385 | return -EACCES; |
386 | } | |
387 | ||
388 | if (likely(!ret)) { | |
389 | if (!psock->apply_bytes) { | |
390 | psock->eval = __SK_NONE; | |
391 | if (psock->sk_redir) { | |
392 | sock_put(psock->sk_redir); | |
393 | psock->sk_redir = NULL; | |
394 | } | |
395 | } | |
396 | if (msg && | |
397 | msg->sg.data[msg->sg.start].page_link && | |
398 | msg->sg.data[msg->sg.start].length) | |
399 | goto more_data; | |
400 | } | |
401 | return ret; | |
402 | } | |
403 | ||
404 | static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) | |
405 | { | |
406 | struct sk_msg tmp, *msg_tx = NULL; | |
604326b4 DB |
407 | int copied = 0, err = 0; |
408 | struct sk_psock *psock; | |
409 | long timeo; | |
41477662 JK |
410 | int flags; |
411 | ||
412 | /* Don't let internal do_tcp_sendpages() flags through */ | |
413 | flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED); | |
414 | flags |= MSG_NO_SHARED_FRAGS; | |
604326b4 DB |
415 | |
416 | psock = sk_psock_get(sk); | |
417 | if (unlikely(!psock)) | |
418 | return tcp_sendmsg(sk, msg, size); | |
419 | ||
420 | lock_sock(sk); | |
421 | timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); | |
422 | while (msg_data_left(msg)) { | |
423 | bool enospc = false; | |
424 | u32 copy, osize; | |
425 | ||
426 | if (sk->sk_err) { | |
427 | err = -sk->sk_err; | |
428 | goto out_err; | |
429 | } | |
430 | ||
431 | copy = msg_data_left(msg); | |
432 | if (!sk_stream_memory_free(sk)) | |
433 | goto wait_for_sndbuf; | |
434 | if (psock->cork) { | |
435 | msg_tx = psock->cork; | |
436 | } else { | |
437 | msg_tx = &tmp; | |
438 | sk_msg_init(msg_tx); | |
439 | } | |
440 | ||
441 | osize = msg_tx->sg.size; | |
442 | err = sk_msg_alloc(sk, msg_tx, msg_tx->sg.size + copy, msg_tx->sg.end - 1); | |
443 | if (err) { | |
444 | if (err != -ENOSPC) | |
445 | goto wait_for_memory; | |
446 | enospc = true; | |
447 | copy = msg_tx->sg.size - osize; | |
448 | } | |
449 | ||
450 | err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx, | |
451 | copy); | |
452 | if (err < 0) { | |
453 | sk_msg_trim(sk, msg_tx, osize); | |
454 | goto out_err; | |
455 | } | |
456 | ||
457 | copied += copy; | |
458 | if (psock->cork_bytes) { | |
459 | if (size > psock->cork_bytes) | |
460 | psock->cork_bytes = 0; | |
461 | else | |
462 | psock->cork_bytes -= size; | |
463 | if (psock->cork_bytes && !enospc) | |
464 | goto out_err; | |
465 | /* All cork bytes are accounted, rerun the prog. */ | |
466 | psock->eval = __SK_NONE; | |
467 | psock->cork_bytes = 0; | |
468 | } | |
469 | ||
470 | err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags); | |
471 | if (unlikely(err < 0)) | |
472 | goto out_err; | |
473 | continue; | |
474 | wait_for_sndbuf: | |
475 | set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); | |
476 | wait_for_memory: | |
477 | err = sk_stream_wait_memory(sk, &timeo); | |
478 | if (err) { | |
479 | if (msg_tx && msg_tx != psock->cork) | |
480 | sk_msg_free(sk, msg_tx); | |
481 | goto out_err; | |
482 | } | |
483 | } | |
484 | out_err: | |
485 | if (err < 0) | |
486 | err = sk_stream_error(sk, msg->msg_flags, err); | |
487 | release_sock(sk); | |
488 | sk_psock_put(sk, psock); | |
489 | return copied ? copied : err; | |
490 | } | |
491 | ||
492 | static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset, | |
493 | size_t size, int flags) | |
494 | { | |
495 | struct sk_msg tmp, *msg = NULL; | |
496 | int err = 0, copied = 0; | |
497 | struct sk_psock *psock; | |
498 | bool enospc = false; | |
499 | ||
500 | psock = sk_psock_get(sk); | |
501 | if (unlikely(!psock)) | |
502 | return tcp_sendpage(sk, page, offset, size, flags); | |
503 | ||
504 | lock_sock(sk); | |
505 | if (psock->cork) { | |
506 | msg = psock->cork; | |
507 | } else { | |
508 | msg = &tmp; | |
509 | sk_msg_init(msg); | |
510 | } | |
511 | ||
512 | /* Catch case where ring is full and sendpage is stalled. */ | |
513 | if (unlikely(sk_msg_full(msg))) | |
514 | goto out_err; | |
515 | ||
516 | sk_msg_page_add(msg, page, size, offset); | |
517 | sk_mem_charge(sk, size); | |
518 | copied = size; | |
519 | if (sk_msg_full(msg)) | |
520 | enospc = true; | |
521 | if (psock->cork_bytes) { | |
522 | if (size > psock->cork_bytes) | |
523 | psock->cork_bytes = 0; | |
524 | else | |
525 | psock->cork_bytes -= size; | |
526 | if (psock->cork_bytes && !enospc) | |
527 | goto out_err; | |
528 | /* All cork bytes are accounted, rerun the prog. */ | |
529 | psock->eval = __SK_NONE; | |
530 | psock->cork_bytes = 0; | |
531 | } | |
532 | ||
533 | err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags); | |
534 | out_err: | |
535 | release_sock(sk); | |
536 | sk_psock_put(sk, psock); | |
537 | return copied ? copied : err; | |
538 | } | |
539 | ||
604326b4 DB |
540 | enum { |
541 | TCP_BPF_IPV4, | |
542 | TCP_BPF_IPV6, | |
543 | TCP_BPF_NUM_PROTS, | |
544 | }; | |
545 | ||
546 | enum { | |
547 | TCP_BPF_BASE, | |
548 | TCP_BPF_TX, | |
549 | TCP_BPF_NUM_CFGS, | |
550 | }; | |
551 | ||
552 | static struct proto *tcpv6_prot_saved __read_mostly; | |
553 | static DEFINE_SPINLOCK(tcpv6_prot_lock); | |
554 | static struct proto tcp_bpf_prots[TCP_BPF_NUM_PROTS][TCP_BPF_NUM_CFGS]; | |
555 | ||
556 | static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], | |
557 | struct proto *base) | |
558 | { | |
559 | prot[TCP_BPF_BASE] = *base; | |
f747632b LB |
560 | prot[TCP_BPF_BASE].unhash = sock_map_unhash; |
561 | prot[TCP_BPF_BASE].close = sock_map_close; | |
604326b4 DB |
562 | prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; |
563 | prot[TCP_BPF_BASE].stream_memory_read = tcp_bpf_stream_read; | |
564 | ||
565 | prot[TCP_BPF_TX] = prot[TCP_BPF_BASE]; | |
566 | prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg; | |
567 | prot[TCP_BPF_TX].sendpage = tcp_bpf_sendpage; | |
568 | } | |
569 | ||
570 | static void tcp_bpf_check_v6_needs_rebuild(struct sock *sk, struct proto *ops) | |
571 | { | |
572 | if (sk->sk_family == AF_INET6 && | |
573 | unlikely(ops != smp_load_acquire(&tcpv6_prot_saved))) { | |
574 | spin_lock_bh(&tcpv6_prot_lock); | |
575 | if (likely(ops != tcpv6_prot_saved)) { | |
576 | tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV6], ops); | |
577 | smp_store_release(&tcpv6_prot_saved, ops); | |
578 | } | |
579 | spin_unlock_bh(&tcpv6_prot_lock); | |
580 | } | |
581 | } | |
582 | ||
583 | static int __init tcp_bpf_v4_build_proto(void) | |
584 | { | |
585 | tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot); | |
586 | return 0; | |
587 | } | |
588 | core_initcall(tcp_bpf_v4_build_proto); | |
589 | ||
604326b4 DB |
590 | static int tcp_bpf_assert_proto_ops(struct proto *ops) |
591 | { | |
592 | /* In order to avoid retpoline, we make assumptions when we call | |
593 | * into ops if e.g. a psock is not present. Make sure they are | |
594 | * indeed valid assumptions. | |
595 | */ | |
596 | return ops->recvmsg == tcp_recvmsg && | |
597 | ops->sendmsg == tcp_sendmsg && | |
598 | ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP; | |
599 | } | |
600 | ||
f747632b | 601 | struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock) |
604326b4 | 602 | { |
d19da360 LB |
603 | int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; |
604 | int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE; | |
604326b4 | 605 | |
d19da360 LB |
606 | if (!psock->sk_proto) { |
607 | struct proto *ops = READ_ONCE(sk->sk_prot); | |
604326b4 | 608 | |
d19da360 LB |
609 | if (tcp_bpf_assert_proto_ops(ops)) |
610 | return ERR_PTR(-EINVAL); | |
611 | ||
612 | tcp_bpf_check_v6_needs_rebuild(sk, ops); | |
613 | } | |
614 | ||
615 | return &tcp_bpf_prots[family][config]; | |
604326b4 DB |
616 | } |
617 | ||
e8025155 JS |
618 | /* If a child got cloned from a listening socket that had tcp_bpf |
619 | * protocol callbacks installed, we need to restore the callbacks to | |
620 | * the default ones because the child does not inherit the psock state | |
621 | * that tcp_bpf callbacks expect. | |
622 | */ | |
623 | void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) | |
624 | { | |
625 | int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; | |
626 | struct proto *prot = newsk->sk_prot; | |
627 | ||
628 | if (prot == &tcp_bpf_prots[family][TCP_BPF_BASE]) | |
629 | newsk->sk_prot = sk->sk_prot_creator; | |
630 | } | |
f747632b | 631 | #endif /* CONFIG_BPF_STREAM_PARSER */ |