Merge tag 'dma-mapping-6.10-2024-05-20' of git://git.infradead.org/users/hch/dma...

[linux-2.6-block.git] / net / core / page_pool.c

diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index 8bcc7014a61a006d1cdaffed352c38cca8689334..f4444b4e39e63a4fa0ad24f36db24123068a0152 100644 (file)
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -173,19 +173,29 @@ static void page_pool_producer_unlock(struct page_pool *pool,
                spin_unlock_bh(&pool->ring.producer_lock);
 }
 
+static void page_pool_struct_check(void)
+{
+       CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_users);
+       CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_page);
+       CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_offset);
+       CACHELINE_ASSERT_GROUP_SIZE(struct page_pool, frag, 4 * sizeof(long));
+}
+
 static int page_pool_init(struct page_pool *pool,
                          const struct page_pool_params *params,
                          int cpuid)
 {
        unsigned int ring_qsize = 1024; /* Default */
 
+       page_pool_struct_check();
+
        memcpy(&pool->p, &params->fast, sizeof(pool->p));
        memcpy(&pool->slow, &params->slow, sizeof(pool->slow));
 
        pool->cpuid = cpuid;
 
        /* Validate only known flags were used */
-       if (pool->p.flags & ~(PP_FLAG_ALL))
+       if (pool->slow.flags & ~PP_FLAG_ALL)
                return -EINVAL;
 
        if (pool->p.pool_size)
@@ -199,22 +209,26 @@ static int page_pool_init(struct page_pool *pool,
         * DMA_BIDIRECTIONAL is for allowing page used for DMA sending,
         * which is the XDP_TX use-case.
         */
-       if (pool->p.flags & PP_FLAG_DMA_MAP) {
+       if (pool->slow.flags & PP_FLAG_DMA_MAP) {
                if ((pool->p.dma_dir != DMA_FROM_DEVICE) &&
                    (pool->p.dma_dir != DMA_BIDIRECTIONAL))
                        return -EINVAL;
+
+               pool->dma_map = true;
        }
 
-       if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) {
+       if (pool->slow.flags & PP_FLAG_DMA_SYNC_DEV) {
                /* In order to request DMA-sync-for-device the page
                 * needs to be mapped
                 */
-               if (!(pool->p.flags & PP_FLAG_DMA_MAP))
+               if (!(pool->slow.flags & PP_FLAG_DMA_MAP))
                        return -EINVAL;
 
                if (!pool->p.max_len)
                        return -EINVAL;
 
+               pool->dma_sync = true;
+
                /* pool->p.offset has to be set according to the address
                 * offset used by the DMA engine to start copying rx data
                 */
@@ -223,7 +237,7 @@ static int page_pool_init(struct page_pool *pool,
        pool->has_init_callback = !!pool->slow.init_callback;
 
 #ifdef CONFIG_PAGE_POOL_STATS
-       if (!(pool->p.flags & PP_FLAG_SYSTEM_POOL)) {
+       if (!(pool->slow.flags & PP_FLAG_SYSTEM_POOL)) {
                pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats);
                if (!pool->recycle_stats)
                        return -ENOMEM;
@@ -233,12 +247,13 @@ static int page_pool_init(struct page_pool *pool,
                 * (also percpu) page pool instance.
                 */
                pool->recycle_stats = &pp_system_recycle_stats;
+               pool->system = true;
        }
 #endif
 
        if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) {
 #ifdef CONFIG_PAGE_POOL_STATS
-               if (!(pool->p.flags & PP_FLAG_SYSTEM_POOL))
+               if (!pool->system)
                        free_percpu(pool->recycle_stats);
 #endif
                return -ENOMEM;
@@ -249,7 +264,7 @@ static int page_pool_init(struct page_pool *pool,
        /* Driver calling page_pool_create() also call page_pool_destroy() */
        refcount_set(&pool->user_cnt, 1);
 
-       if (pool->p.flags & PP_FLAG_DMA_MAP)
+       if (pool->dma_map)
                get_device(pool->p.dev);
 
        return 0;
@@ -259,11 +274,11 @@ static void page_pool_uninit(struct page_pool *pool)
 {
        ptr_ring_cleanup(&pool->ring, NULL);
 
-       if (pool->p.flags & PP_FLAG_DMA_MAP)
+       if (pool->dma_map)
                put_device(pool->p.dev);
 
 #ifdef CONFIG_PAGE_POOL_STATS
-       if (!(pool->p.flags & PP_FLAG_SYSTEM_POOL))
+       if (!pool->system)
                free_percpu(pool->recycle_stats);
 #endif
 }
@@ -384,16 +399,26 @@ static struct page *__page_pool_get_cached(struct page_pool *pool)
        return page;
 }
 
-static void page_pool_dma_sync_for_device(const struct page_pool *pool,
-                                         const struct page *page,
-                                         unsigned int dma_sync_size)
+static void __page_pool_dma_sync_for_device(const struct page_pool *pool,
+                                           const struct page *page,
+                                           u32 dma_sync_size)
 {
+#if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC)
        dma_addr_t dma_addr = page_pool_get_dma_addr(page);
 
        dma_sync_size = min(dma_sync_size, pool->p.max_len);
-       dma_sync_single_range_for_device(pool->p.dev, dma_addr,
-                                        pool->p.offset, dma_sync_size,
-                                        pool->p.dma_dir);
+       __dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset,
+                                    dma_sync_size, pool->p.dma_dir);
+#endif
+}
+
+static __always_inline void
+page_pool_dma_sync_for_device(const struct page_pool *pool,
+                             const struct page *page,
+                             u32 dma_sync_size)
+{
+       if (pool->dma_sync && dma_dev_need_sync(pool->p.dev))
+               __page_pool_dma_sync_for_device(pool, page, dma_sync_size);
 }
 
 static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
@@ -415,8 +440,7 @@ static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
        if (page_pool_set_dma_addr(page, dma))
                goto unmap_failed;
 
-       if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
-               page_pool_dma_sync_for_device(pool, page, pool->p.max_len);
+       page_pool_dma_sync_for_device(pool, page, pool->p.max_len);
 
        return true;
 
@@ -461,8 +485,7 @@ static struct page *__page_pool_alloc_page_order(struct page_pool *pool,
        if (unlikely(!page))
                return NULL;
 
-       if ((pool->p.flags & PP_FLAG_DMA_MAP) &&
-           unlikely(!page_pool_dma_map(pool, page))) {
+       if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page))) {
                put_page(page);
                return NULL;
        }
@@ -482,8 +505,8 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
                                                 gfp_t gfp)
 {
        const int bulk = PP_ALLOC_CACHE_REFILL;
-       unsigned int pp_flags = pool->p.flags;
        unsigned int pp_order = pool->p.order;
+       bool dma_map = pool->dma_map;
        struct page *page;
        int i, nr_pages;
 
@@ -508,8 +531,7 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
         */
        for (i = 0; i < nr_pages; i++) {
                page = pool->alloc.cache[i];
-               if ((pp_flags & PP_FLAG_DMA_MAP) &&
-                   unlikely(!page_pool_dma_map(pool, page))) {
+               if (dma_map && unlikely(!page_pool_dma_map(pool, page))) {
                        put_page(page);
                        continue;
                }
@@ -582,7 +604,7 @@ void __page_pool_release_page_dma(struct page_pool *pool, struct page *page)
 {
        dma_addr_t dma;
 
-       if (!(pool->p.flags & PP_FLAG_DMA_MAP))
+       if (!pool->dma_map)
                /* Always account for inflight pages, even if we didn't
                 * map them
                 */
@@ -665,7 +687,7 @@ static bool __page_pool_page_can_be_recycled(const struct page *page)
 }
 
 /* If the page refcnt == 1, this will try to recycle the page.
- * if PP_FLAG_DMA_SYNC_DEV is set, we'll try to sync the DMA area for
+ * If pool->dma_sync is set, we'll try to sync the DMA area for
  * the configured size min(dma_sync_size, pool->max_len).
  * If the page refcnt != 1, then the page will be returned to memory
  * subsystem.
@@ -688,9 +710,7 @@ __page_pool_put_page(struct page_pool *pool, struct page *page,
        if (likely(__page_pool_page_can_be_recycled(page))) {
                /* Read barrier done in page_ref_count / READ_ONCE */
 
-               if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
-                       page_pool_dma_sync_for_device(pool, page,
-                                                     dma_sync_size);
+               page_pool_dma_sync_for_device(pool, page, dma_sync_size);
 
                if (allow_direct && page_pool_recycle_in_cache(page, pool))
                        return NULL;
@@ -829,9 +849,7 @@ static struct page *page_pool_drain_frag(struct page_pool *pool,
                return NULL;
 
        if (__page_pool_page_can_be_recycled(page)) {
-               if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV)
-                       page_pool_dma_sync_for_device(pool, page, -1);
-
+               page_pool_dma_sync_for_device(pool, page, -1);
                return page;
        }