net: support non paged skb frags

Make skb_frag_page() fail in the case where the frag is not backed
by a page, and fix its relevant callers to handle this case.

Signed-off-by: Mina Almasry <almasrymina@google.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Jakub Kicinski <kuba@kernel.org>
Link: https://patch.msgid.link/20240910171458.219195-8-almasrymina@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 5803eb8a157dfe54c4294f93c528bca03a082726..1019f14583c9ce60830083c37b4fe6f34e910e71 100644 (file)
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -3524,21 +3524,58 @@ static inline void skb_frag_off_copy(skb_frag_t *fragto,
        fragto->offset = fragfrom->offset;
 }
 
+/* Return: true if the skb_frag contains a net_iov. */
+static inline bool skb_frag_is_net_iov(const skb_frag_t *frag)
+{
+       return netmem_is_net_iov(frag->netmem);
+}
+
+/**
+ * skb_frag_net_iov - retrieve the net_iov referred to by fragment
+ * @frag: the fragment
+ *
+ * Return: the &struct net_iov associated with @frag. Returns NULL if this
+ * frag has no associated net_iov.
+ */
+static inline struct net_iov *skb_frag_net_iov(const skb_frag_t *frag)
+{
+       if (!skb_frag_is_net_iov(frag))
+               return NULL;
+
+       return netmem_to_net_iov(frag->netmem);
+}
+
 /**
  * skb_frag_page - retrieve the page referred to by a paged fragment
  * @frag: the paged fragment
  *
- * Returns the &struct page associated with @frag.
+ * Return: the &struct page associated with @frag. Returns NULL if this frag
+ * has no associated page.
  */
 static inline struct page *skb_frag_page(const skb_frag_t *frag)
 {
+       if (skb_frag_is_net_iov(frag))
+               return NULL;
+
        return netmem_to_page(frag->netmem);
 }
 
+/**
+ * skb_frag_netmem - retrieve the netmem referred to by a fragment
+ * @frag: the fragment
+ *
+ * Return: the &netmem_ref associated with @frag.
+ */
+static inline netmem_ref skb_frag_netmem(const skb_frag_t *frag)
+{
+       return frag->netmem;
+}
+
 int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb,
                    unsigned int headroom);
 int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
                         struct bpf_prog *prog);
+
 /**
  * skb_frag_address - gets the address of the data contained in a paged fragment
  * @frag: the paged fragment buffer
@@ -3548,6 +3585,9 @@ int skb_cow_data_for_xdp(struct page_pool *pool, struct sk_buff **pskb,
  */
 static inline void *skb_frag_address(const skb_frag_t *frag)
 {
+       if (!skb_frag_page(frag))
+               return NULL;
+
        return page_address(skb_frag_page(frag)) + skb_frag_off(frag);
 }
 

diff --git a/include/linux/skbuff_ref.h b/include/linux/skbuff_ref.h
index 16c241a234728b0c6448885ad6e1d9be945d44b7..0f3c58007488ab2cde13a7aeacb6d56221e09eaa 100644 (file)
--- a/include/linux/skbuff_ref.h
+++ b/include/linux/skbuff_ref.h
@@ -34,14 +34,13 @@ static inline void skb_frag_ref(struct sk_buff *skb, int f)
 
 bool napi_pp_put_page(netmem_ref netmem);
 
-static inline void
-skb_page_unref(struct page *page, bool recycle)
+static inline void skb_page_unref(netmem_ref netmem, bool recycle)
 {
 #ifdef CONFIG_PAGE_POOL
-       if (recycle && napi_pp_put_page(page_to_netmem(page)))
+       if (recycle && napi_pp_put_page(netmem))
                return;
 #endif
-       put_page(page);
+       put_page(netmem_to_page(netmem));
 }
 
 /**
@@ -54,7 +53,7 @@ skb_page_unref(struct page *page, bool recycle)
  */
 static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle)
 {
-       skb_page_unref(skb_frag_page(frag), recycle);
+       skb_page_unref(skb_frag_netmem(frag), recycle);
 }
 
 /**

diff --git a/net/core/dev.c b/net/core/dev.c
index 7818d7f6295e11485ba019bfddb18a657902f9c2..517b934bdf9c5e4e56ca70effab59a5068dfaf8e 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3434,8 +3434,9 @@ static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
        if (!(dev->features & NETIF_F_HIGHDMA)) {
                for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
                        skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+                       struct page *page = skb_frag_page(frag);
 
-                       if (PageHighMem(skb_frag_page(frag)))
+                       if (page && PageHighMem(page))
                                return 1;
                }
        }

diff --git a/net/core/gro.c b/net/core/gro.c
index 3abad1b567dd9a95d78b37cc5a0e78fdf84b1633..802b4a0624009eca637a52698ed1f19228252f87 100644 (file)
--- a/net/core/gro.c
+++ b/net/core/gro.c
@@ -408,7 +408,8 @@ static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
        pinfo = skb_shinfo(skb);
        frag0 = &pinfo->frags[0];
 
-       if (pinfo->nr_frags && !PageHighMem(skb_frag_page(frag0)) &&
+       if (pinfo->nr_frags && skb_frag_page(frag0) &&
+           !PageHighMem(skb_frag_page(frag0)) &&
            (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
                NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
                NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 8abcc28bb1898aac26d3c8115c69eee411581d81..671c456b93f5384b9558d97bf89d26e131e478c1 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -1371,6 +1371,14 @@ void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt)
                struct page *p;
                u8 *vaddr;
 
+               if (skb_frag_is_net_iov(frag)) {
+                       printk("%sskb frag %d: not readable\n", level, i);
+                       len -= skb_frag_size(frag);
+                       if (!len)
+                               break;
+                       continue;
+               }
+
                skb_frag_foreach_page(frag, skb_frag_off(frag),
                                      skb_frag_size(frag), p, p_off, p_len,
                                      copied) {
@@ -3163,6 +3171,9 @@ static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
        for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
                const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
 
+               if (WARN_ON_ONCE(!skb_frag_page(f)))
+                       return false;
+
                if (__splice_segment(skb_frag_page(f),
                                     skb_frag_off(f), skb_frag_size(f),
                                     offset, len, spd, false, sk, pipe))

diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c
index 47378ca41904bb0ff7d943806960f19b05cc592f..f3281312eb5eba53110e8276184703eab6bc81e8 100644 (file)
--- a/net/ipv4/esp4.c
+++ b/net/ipv4/esp4.c
@@ -115,7 +115,8 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb)
         */
        if (req->src != req->dst)
                for (sg = sg_next(req->src); sg; sg = sg_next(sg))
-                       skb_page_unref(sg_page(sg), skb->pp_recycle);
+                       skb_page_unref(page_to_netmem(sg_page(sg)),
+                                      skb->pp_recycle);
 }
 
 #ifdef CONFIG_INET_ESPINTCP

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index e359a91614451bfc9272fc49fa5acc23d5528959..06dcf89e2cca32de692376f8bfe081e493245599 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -2177,6 +2177,9 @@ static int tcp_zerocopy_receive(struct sock *sk,
                        break;
                }
                page = skb_frag_page(frags);
+               if (WARN_ON_ONCE(!page))
+                       break;
+
                prefetchw(page);
                pages[pages_to_map++] = page;
                length += PAGE_SIZE;

diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c
index 3920e8aa1031ee2ecc0e92276bfb8132bae16656..b2400c226a32582641407e7ed7beecb681d05a9e 100644 (file)
--- a/net/ipv6/esp6.c
+++ b/net/ipv6/esp6.c
@@ -132,7 +132,8 @@ static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb)
         */
        if (req->src != req->dst)
                for (sg = sg_next(req->src); sg; sg = sg_next(sg))
-                       skb_page_unref(sg_page(sg), skb->pp_recycle);
+                       skb_page_unref(page_to_netmem(sg_page(sg)),
+                                      skb->pp_recycle);
 }
 
 #ifdef CONFIG_INET6_ESPINTCP