selftests/bpf: Wait for the netstamp_needed_key static key to be turned on

After the previous patch that speeded up the test (by avoiding neigh
discovery in IPv6), the BPF CI occasionally hits this error:

rcv tstamp unexpected pkt rcv tstamp: actual 0 == expected 0

The test complains about the cmsg returned from the recvmsg() does not
have the rcv timestamp. Setting skb->tstamp or not is
controlled by a kernel static key "netstamp_needed_key". The static
key is enabled whenever this is at least one sk with the SOCK_TIMESTAMP
set.

The test_redirect_dtime does use setsockopt() to turn on
the SOCK_TIMESTAMP for the reading sk. In the kernel
net_enable_timestamp() has a delay to enable the "netstamp_needed_key"
when CONFIG_JUMP_LABEL is set. This potential delay is the likely reason
for packet missing rcv timestamp occasionally.

This patch is to create udp sockets with SOCK_TIMESTAMP set.
It sends and receives some packets until the received packet
has a rcv timestamp. It currently retries at most 5 times with 1s
in between. This should be enough to wait for the "netstamp_needed_key".
It then holds on to the socket and only closes it at the end of the test.
This guarantees that the test has the "netstamp_needed_key" key turned
on from the beginning.

To simplify the udp sockets setup, they are sending/receiving packets
in the same netns (ns_dst is used) and communicate over the "lo" dev.
Hence, the patch enables the "lo" dev in the ns_dst.

Fixes: c803475fd8dd ("bpf: selftests: test skb->tstamp in redirect_neigh")
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20240120060518.3604920-2-martin.lau@linux.dev

diff --git a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
index 610887157fd856ed59addd7d2494d8de048f1ac0..dbe06aeaa2b27f66cc89d0349259560c3587de76 100644 (file)
--- a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
+++ b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
@@ -291,6 +291,7 @@ static int netns_setup_links_and_routes(struct netns_setup_result *result)
        SYS(fail, "ip addr add " IP4_DST "/32 dev dst");
        SYS(fail, "ip addr add " IP6_DST "/128 dev dst nodad");
        SYS(fail, "ip link set dev dst up");
+       SYS(fail, "ip link set dev lo up");
 
        SYS(fail, "ip route add " IP4_SRC "/32 dev dst scope global");
        SYS(fail, "ip route add " IP4_NET "/16 dev dst scope global");
@@ -468,7 +469,7 @@ static int set_forwarding(bool enable)
        return 0;
 }
 
-static void rcv_tstamp(int fd, const char *expected, size_t s)
+static int __rcv_tstamp(int fd, const char *expected, size_t s, __u64 *tstamp)
 {
        struct __kernel_timespec pkt_ts = {};
        char ctl[CMSG_SPACE(sizeof(pkt_ts))];
@@ -489,7 +490,7 @@ static void rcv_tstamp(int fd, const char *expected, size_t s)
 
        ret = recvmsg(fd, &msg, 0);
        if (!ASSERT_EQ(ret, s, "recvmsg"))
-               return;
+               return -1;
        ASSERT_STRNEQ(data, expected, s, "expected rcv data");
 
        cmsg = CMSG_FIRSTHDR(&msg);
@@ -498,6 +499,12 @@ static void rcv_tstamp(int fd, const char *expected, size_t s)
                memcpy(&pkt_ts, CMSG_DATA(cmsg), sizeof(pkt_ts));
 
        pkt_ns = pkt_ts.tv_sec * NSEC_PER_SEC + pkt_ts.tv_nsec;
+       if (tstamp) {
+               /* caller will check the tstamp itself */
+               *tstamp = pkt_ns;
+               return 0;
+       }
+
        ASSERT_NEQ(pkt_ns, 0, "pkt rcv tstamp");
 
        ret = clock_gettime(CLOCK_REALTIME, &now_ts);
@@ -507,6 +514,60 @@ static void rcv_tstamp(int fd, const char *expected, size_t s)
        if (ASSERT_GE(now_ns, pkt_ns, "check rcv tstamp"))
                ASSERT_LT(now_ns - pkt_ns, 5 * NSEC_PER_SEC,
                          "check rcv tstamp");
+       return 0;
+}
+
+static void rcv_tstamp(int fd, const char *expected, size_t s)
+{
+       __rcv_tstamp(fd, expected, s, NULL);
+}
+
+static int wait_netstamp_needed_key(void)
+{
+       int opt = 1, srv_fd = -1, cli_fd = -1, nretries = 0, err, n;
+       char buf[] = "testing testing";
+       struct nstoken *nstoken;
+       __u64 tstamp = 0;
+
+       nstoken = open_netns(NS_DST);
+       if (!nstoken)
+               return -1;
+
+       srv_fd = start_server(AF_INET6, SOCK_DGRAM, "::1", 0, 0);
+       if (!ASSERT_GE(srv_fd, 0, "start_server"))
+               goto done;
+
+       err = setsockopt(srv_fd, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
+                        &opt, sizeof(opt));
+       if (!ASSERT_OK(err, "setsockopt(SO_TIMESTAMPNS_NEW)"))
+               goto done;
+
+       cli_fd = connect_to_fd(srv_fd, TIMEOUT_MILLIS);
+       if (!ASSERT_GE(cli_fd, 0, "connect_to_fd"))
+               goto done;
+
+again:
+       n = write(cli_fd, buf, sizeof(buf));
+       if (!ASSERT_EQ(n, sizeof(buf), "send to server"))
+               goto done;
+       err = __rcv_tstamp(srv_fd, buf, sizeof(buf), &tstamp);
+       if (!ASSERT_OK(err, "__rcv_tstamp"))
+               goto done;
+       if (!tstamp && nretries++ < 5) {
+               sleep(1);
+               printf("netstamp_needed_key retry#%d\n", nretries);
+               goto again;
+       }
+
+done:
+       if (!tstamp && srv_fd != -1) {
+               close(srv_fd);
+               srv_fd = -1;
+       }
+       if (cli_fd != -1)
+               close(cli_fd);
+       close_netns(nstoken);
+       return srv_fd;
 }
 
 static void snd_tstamp(int fd, char *b, size_t s)
@@ -843,11 +904,20 @@ static void test_tc_redirect_dtime(struct netns_setup_result *setup_result)
 {
        struct test_tc_dtime *skel;
        struct nstoken *nstoken;
-       int err;
+       int hold_tstamp_fd, err;
+
+       /* Hold a sk with the SOCK_TIMESTAMP set to ensure there
+        * is no delay in the kernel net_enable_timestamp().
+        * This ensures the following tests must have
+        * non zero rcv tstamp in the recvmsg().
+        */
+       hold_tstamp_fd = wait_netstamp_needed_key();
+       if (!ASSERT_GE(hold_tstamp_fd, 0, "wait_netstamp_needed_key"))
+               return;
 
        skel = test_tc_dtime__open();
        if (!ASSERT_OK_PTR(skel, "test_tc_dtime__open"))
-               return;
+               goto done;
 
        skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd;
        skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd;
@@ -892,6 +962,7 @@ static void test_tc_redirect_dtime(struct netns_setup_result *setup_result)
 
 done:
        test_tc_dtime__destroy(skel);
+       close(hold_tstamp_fd);
 }
 
 static void test_tc_redirect_neigh_fib(struct netns_setup_result *setup_result)