bpf, sockmap: Fix missing BPF_F_INGRESS flag when using apply_bytes

When redirecting, we use sk_msg_to_ingress() to get the BPF_F_INGRESS
flag from the msg->flags. If apply_bytes is used and it is larger than
the current data being processed, sk_psock_msg_verdict() will not be
called when sendmsg() is called again. At this time, the msg->flags is 0,
and we lost the BPF_F_INGRESS flag.

So we need to save the BPF_F_INGRESS flag in sk_psock and use it when
redirection.

Fixes: 8934ce2fd081 ("bpf: sockmap redirect ingress support")
Signed-off-by: Pengcheng Yang <yangpc@wangsu.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jakub Sitnicki <jakub@cloudflare.com>
Link: https://lore.kernel.org/bpf/1669718441-2654-3-git-send-email-yangpc@wangsu.com

diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h
index 70d6cb94e5802d17f160f8383e7e38cae17dcfa1..84f787416a54d4597aa9a0f032a63d43ecc8aac7 100644 (file)
--- a/include/linux/skmsg.h
+++ b/include/linux/skmsg.h
@@ -82,6 +82,7 @@ struct sk_psock {
        u32                             apply_bytes;
        u32                             cork_bytes;
        u32                             eval;
+       bool                            redir_ingress; /* undefined if sk_redir is null */
        struct sk_msg                   *cork;
        struct sk_psock_progs           progs;
 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)

diff --git a/include/net/tcp.h b/include/net/tcp.h
index 6b814e788f00d2c3a68f8e27e5871bd8fc2dfd59..b87e7381bddf5082056bc86adde6ffc31aa6176f 100644 (file)
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -2319,8 +2319,8 @@ int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore);
 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
 #endif /* CONFIG_BPF_SYSCALL */
 
-int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, u32 bytes,
-                         int flags);
+int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress,
+                         struct sk_msg *msg, u32 bytes, int flags);
 #endif /* CONFIG_NET_SOCK_MSG */
 
 #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG)

diff --git a/net/core/skmsg.c b/net/core/skmsg.c
index e6b9ced3eda82f8af545c88eeeeaa899f54f5f9f..53d0251788aa26c80145dbeee2c2c67afdf40b3b 100644 (file)
--- a/net/core/skmsg.c
+++ b/net/core/skmsg.c
@@ -886,13 +886,16 @@ int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
        ret = sk_psock_map_verd(ret, msg->sk_redir);
        psock->apply_bytes = msg->apply_bytes;
        if (ret == __SK_REDIRECT) {
-               if (psock->sk_redir)
+               if (psock->sk_redir) {
                        sock_put(psock->sk_redir);
-               psock->sk_redir = msg->sk_redir;
-               if (!psock->sk_redir) {
+                       psock->sk_redir = NULL;
+               }
+               if (!msg->sk_redir) {
                        ret = __SK_DROP;
                        goto out;
                }
+               psock->redir_ingress = sk_msg_to_ingress(msg);
+               psock->sk_redir = msg->sk_redir;
                sock_hold(psock->sk_redir);
        }
 out:

diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c
index f3e868f4cd9e7645d0b145c5d03acf41ee849bb9..275c5ca9e04dfb6294ce7911067a579314f94b95 100644 (file)
--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -131,10 +131,9 @@ static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg,
        return ret;
 }
 
-int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg,
-                         u32 bytes, int flags)
+int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress,
+                         struct sk_msg *msg, u32 bytes, int flags)
 {
-       bool ingress = sk_msg_to_ingress(msg);
        struct sk_psock *psock = sk_psock_get(sk);
        int ret;
 
@@ -276,7 +275,7 @@ msg_bytes_ready:
 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,
                                struct sk_msg *msg, int *copied, int flags)
 {
-       bool cork = false, enospc = sk_msg_full(msg);
+       bool cork = false, enospc = sk_msg_full(msg), redir_ingress;
        struct sock *sk_redir;
        u32 tosend, origsize, sent, delta = 0;
        u32 eval;
@@ -322,6 +321,7 @@ more_data:
                sk_msg_apply_bytes(psock, tosend);
                break;
        case __SK_REDIRECT:
+               redir_ingress = psock->redir_ingress;
                sk_redir = psock->sk_redir;
                sk_msg_apply_bytes(psock, tosend);
                if (!psock->apply_bytes) {
@@ -338,7 +338,8 @@ more_data:
                release_sock(sk);
 
                origsize = msg->sg.size;
-               ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags);
+               ret = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress,
+                                           msg, tosend, flags);
                sent = origsize - msg->sg.size;
 
                if (eval == __SK_REDIRECT)

diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 264cf367e26566281851de2ee8e2eb7e5808268c..9ed9786341259ea6dc4ed1a255f35534f9f902bf 100644 (file)
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -792,7 +792,7 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,
        struct sk_psock *psock;
        struct sock *sk_redir;
        struct tls_rec *rec;
-       bool enospc, policy;
+       bool enospc, policy, redir_ingress;
        int err = 0, send;
        u32 delta = 0;
 
@@ -837,6 +837,7 @@ more_data:
                }
                break;
        case __SK_REDIRECT:
+               redir_ingress = psock->redir_ingress;
                sk_redir = psock->sk_redir;
                memcpy(&msg_redir, msg, sizeof(*msg));
                if (msg->apply_bytes < send)
@@ -846,7 +847,8 @@ more_data:
                sk_msg_return_zero(sk, msg, send);
                msg->sg.size -= send;
                release_sock(sk);
-               err = tcp_bpf_sendmsg_redir(sk_redir, &msg_redir, send, flags);
+               err = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress,
+                                           &msg_redir, send, flags);
                lock_sock(sk);
                if (err < 0) {
                        *copied -= sk_msg_free_nocharge(sk, &msg_redir);