Commit | Line | Data |
---|---|---|
ff7d6b27 JDB |
1 | /* SPDX-License-Identifier: GPL-2.0 |
2 | * | |
3 | * page_pool.h | |
4 | * Author: Jesper Dangaard Brouer <netoptimizer@brouer.com> | |
5 | * Copyright (C) 2016 Red Hat, Inc. | |
6 | */ | |
7 | ||
8 | /** | |
9 | * DOC: page_pool allocator | |
10 | * | |
11 | * This page_pool allocator is optimized for the XDP mode that | |
12 | * uses one-frame-per-page, but have fallbacks that act like the | |
13 | * regular page allocator APIs. | |
14 | * | |
15 | * Basic use involve replacing alloc_pages() calls with the | |
16 | * page_pool_alloc_pages() call. Drivers should likely use | |
17 | * page_pool_dev_alloc_pages() replacing dev_alloc_pages(). | |
18 | * | |
99c07c43 JDB |
19 | * API keeps track of in-flight pages, in-order to let API user know |
20 | * when it is safe to dealloactor page_pool object. Thus, API users | |
21 | * must make sure to call page_pool_release_page() when a page is | |
22 | * "leaving" the page_pool. Or call page_pool_put_page() where | |
23 | * appropiate. For maintaining correct accounting. | |
ff7d6b27 | 24 | * |
99c07c43 JDB |
25 | * API user must only call page_pool_put_page() once on a page, as it |
26 | * will either recycle the page, or in case of elevated refcnt, it | |
27 | * will release the DMA mapping and in-flight state accounting. We | |
28 | * hope to lift this requirement in the future. | |
ff7d6b27 JDB |
29 | */ |
30 | #ifndef _NET_PAGE_POOL_H | |
31 | #define _NET_PAGE_POOL_H | |
32 | ||
33 | #include <linux/mm.h> /* Needed by ptr_ring */ | |
34 | #include <linux/ptr_ring.h> | |
35 | #include <linux/dma-direction.h> | |
36 | ||
37 | #define PP_FLAG_DMA_MAP 1 /* Should page_pool do the DMA map/unmap */ | |
38 | #define PP_FLAG_ALL PP_FLAG_DMA_MAP | |
39 | ||
40 | /* | |
41 | * Fast allocation side cache array/stack | |
42 | * | |
43 | * The cache size and refill watermark is related to the network | |
44 | * use-case. The NAPI budget is 64 packets. After a NAPI poll the RX | |
45 | * ring is usually refilled and the max consumed elements will be 64, | |
46 | * thus a natural max size of objects needed in the cache. | |
47 | * | |
48 | * Keeping room for more objects, is due to XDP_DROP use-case. As | |
49 | * XDP_DROP allows the opportunity to recycle objects directly into | |
50 | * this array, as it shares the same softirq/NAPI protection. If | |
51 | * cache is already full (or partly full) then the XDP_DROP recycles | |
52 | * would have to take a slower code path. | |
53 | */ | |
54 | #define PP_ALLOC_CACHE_SIZE 128 | |
55 | #define PP_ALLOC_CACHE_REFILL 64 | |
56 | struct pp_alloc_cache { | |
57 | u32 count; | |
58 | void *cache[PP_ALLOC_CACHE_SIZE]; | |
59 | }; | |
60 | ||
61 | struct page_pool_params { | |
62 | unsigned int flags; | |
63 | unsigned int order; | |
64 | unsigned int pool_size; | |
65 | int nid; /* Numa node id to allocate from pages from */ | |
66 | struct device *dev; /* device, for DMA pre-mapping purposes */ | |
67 | enum dma_data_direction dma_dir; /* DMA mapping direction */ | |
68 | }; | |
69 | ||
70 | struct page_pool { | |
ff7d6b27 JDB |
71 | struct page_pool_params p; |
72 | ||
99c07c43 JDB |
73 | u32 pages_state_hold_cnt; |
74 | ||
ff7d6b27 JDB |
75 | /* |
76 | * Data structure for allocation side | |
77 | * | |
78 | * Drivers allocation side usually already perform some kind | |
79 | * of resource protection. Piggyback on this protection, and | |
80 | * require driver to protect allocation side. | |
81 | * | |
82 | * For NIC drivers this means, allocate a page_pool per | |
83 | * RX-queue. As the RX-queue is already protected by | |
84 | * Softirq/BH scheduling and napi_schedule. NAPI schedule | |
85 | * guarantee that a single napi_struct will only be scheduled | |
86 | * on a single CPU (see napi_schedule). | |
87 | */ | |
88 | struct pp_alloc_cache alloc ____cacheline_aligned_in_smp; | |
89 | ||
90 | /* Data structure for storing recycled pages. | |
91 | * | |
92 | * Returning/freeing pages is more complicated synchronization | |
93 | * wise, because free's can happen on remote CPUs, with no | |
94 | * association with allocation resource. | |
95 | * | |
96 | * Use ptr_ring, as it separates consumer and producer | |
97 | * effeciently, it a way that doesn't bounce cache-lines. | |
98 | * | |
99 | * TODO: Implement bulk return pages into this structure. | |
100 | */ | |
101 | struct ptr_ring ring; | |
99c07c43 JDB |
102 | |
103 | atomic_t pages_state_release_cnt; | |
1da4bbef IK |
104 | |
105 | /* A page_pool is strictly tied to a single RX-queue being | |
106 | * protected by NAPI, due to above pp_alloc_cache. This | |
107 | * refcnt serves purpose is to simplify drivers error handling. | |
108 | */ | |
109 | refcount_t user_cnt; | |
ff7d6b27 JDB |
110 | }; |
111 | ||
112 | struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); | |
113 | ||
114 | static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool) | |
115 | { | |
116 | gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); | |
117 | ||
118 | return page_pool_alloc_pages(pool, gfp); | |
119 | } | |
120 | ||
bb005f2a IA |
121 | /* get the stored dma direction. A driver might decide to treat this locally and |
122 | * avoid the extra cache line from page_pool to determine the direction | |
123 | */ | |
124 | static | |
125 | inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) | |
126 | { | |
127 | return pool->p.dma_dir; | |
128 | } | |
129 | ||
ff7d6b27 JDB |
130 | struct page_pool *page_pool_create(const struct page_pool_params *params); |
131 | ||
e54cfd7e JDB |
132 | void __page_pool_free(struct page_pool *pool); |
133 | static inline void page_pool_free(struct page_pool *pool) | |
134 | { | |
135 | /* When page_pool isn't compiled-in, net/core/xdp.c doesn't | |
136 | * allow registering MEM_TYPE_PAGE_POOL, but shield linker. | |
137 | */ | |
138 | #ifdef CONFIG_PAGE_POOL | |
139 | __page_pool_free(pool); | |
140 | #endif | |
141 | } | |
142 | ||
1da4bbef IK |
143 | /* Drivers use this instead of page_pool_free */ |
144 | static inline void page_pool_destroy(struct page_pool *pool) | |
145 | { | |
146 | if (!pool) | |
147 | return; | |
148 | ||
149 | page_pool_free(pool); | |
150 | } | |
151 | ||
ff7d6b27 JDB |
152 | /* Never call this directly, use helpers below */ |
153 | void __page_pool_put_page(struct page_pool *pool, | |
154 | struct page *page, bool allow_direct); | |
155 | ||
389ab7f0 JDB |
156 | static inline void page_pool_put_page(struct page_pool *pool, |
157 | struct page *page, bool allow_direct) | |
ff7d6b27 | 158 | { |
57d0a1c1 JDB |
159 | /* When page_pool isn't compiled-in, net/core/xdp.c doesn't |
160 | * allow registering MEM_TYPE_PAGE_POOL, but shield linker. | |
161 | */ | |
162 | #ifdef CONFIG_PAGE_POOL | |
389ab7f0 | 163 | __page_pool_put_page(pool, page, allow_direct); |
57d0a1c1 | 164 | #endif |
ff7d6b27 JDB |
165 | } |
166 | /* Very limited use-cases allow recycle direct */ | |
167 | static inline void page_pool_recycle_direct(struct page_pool *pool, | |
168 | struct page *page) | |
169 | { | |
170 | __page_pool_put_page(pool, page, true); | |
171 | } | |
172 | ||
99c07c43 JDB |
173 | /* API user MUST have disconnected alloc-side (not allowed to call |
174 | * page_pool_alloc_pages()) before calling this. The free-side can | |
175 | * still run concurrently, to handle in-flight packet-pages. | |
176 | * | |
177 | * A request to shutdown can fail (with false) if there are still | |
178 | * in-flight packet-pages. | |
179 | */ | |
180 | bool __page_pool_request_shutdown(struct page_pool *pool); | |
181 | static inline bool page_pool_request_shutdown(struct page_pool *pool) | |
182 | { | |
497ad9f5 JDB |
183 | bool safe_to_remove = false; |
184 | ||
99c07c43 | 185 | #ifdef CONFIG_PAGE_POOL |
497ad9f5 | 186 | safe_to_remove = __page_pool_request_shutdown(pool); |
99c07c43 | 187 | #endif |
497ad9f5 | 188 | return safe_to_remove; |
99c07c43 JDB |
189 | } |
190 | ||
6bf071bf JDB |
191 | /* Disconnects a page (from a page_pool). API users can have a need |
192 | * to disconnect a page (from a page_pool), to allow it to be used as | |
193 | * a regular page (that will eventually be returned to the normal | |
194 | * page-allocator via put_page). | |
195 | */ | |
196 | void page_pool_unmap_page(struct page_pool *pool, struct page *page); | |
197 | static inline void page_pool_release_page(struct page_pool *pool, | |
198 | struct page *page) | |
199 | { | |
200 | #ifdef CONFIG_PAGE_POOL | |
201 | page_pool_unmap_page(pool, page); | |
202 | #endif | |
203 | } | |
204 | ||
0afdeeed IA |
205 | static inline dma_addr_t page_pool_get_dma_addr(struct page *page) |
206 | { | |
207 | return page->dma_addr; | |
208 | } | |
209 | ||
57d0a1c1 JDB |
210 | static inline bool is_page_pool_compiled_in(void) |
211 | { | |
212 | #ifdef CONFIG_PAGE_POOL | |
213 | return true; | |
214 | #else | |
215 | return false; | |
216 | #endif | |
217 | } | |
218 | ||
1da4bbef IK |
219 | static inline void page_pool_get(struct page_pool *pool) |
220 | { | |
221 | refcount_inc(&pool->user_cnt); | |
222 | } | |
223 | ||
224 | static inline bool page_pool_put(struct page_pool *pool) | |
225 | { | |
226 | return refcount_dec_and_test(&pool->user_cnt); | |
227 | } | |
228 | ||
ff7d6b27 | 229 | #endif /* _NET_PAGE_POOL_H */ |