Commit | Line | Data |
---|---|---|
ff7d6b27 JDB |
1 | /* SPDX-License-Identifier: GPL-2.0 |
2 | * | |
3 | * page_pool.h | |
4 | * Author: Jesper Dangaard Brouer <netoptimizer@brouer.com> | |
5 | * Copyright (C) 2016 Red Hat, Inc. | |
6 | */ | |
7 | ||
8 | /** | |
9 | * DOC: page_pool allocator | |
10 | * | |
11 | * This page_pool allocator is optimized for the XDP mode that | |
12 | * uses one-frame-per-page, but have fallbacks that act like the | |
13 | * regular page allocator APIs. | |
14 | * | |
15 | * Basic use involve replacing alloc_pages() calls with the | |
16 | * page_pool_alloc_pages() call. Drivers should likely use | |
17 | * page_pool_dev_alloc_pages() replacing dev_alloc_pages(). | |
18 | * | |
99c07c43 JDB |
19 | * API keeps track of in-flight pages, in-order to let API user know |
20 | * when it is safe to dealloactor page_pool object. Thus, API users | |
21 | * must make sure to call page_pool_release_page() when a page is | |
22 | * "leaving" the page_pool. Or call page_pool_put_page() where | |
23 | * appropiate. For maintaining correct accounting. | |
ff7d6b27 | 24 | * |
99c07c43 JDB |
25 | * API user must only call page_pool_put_page() once on a page, as it |
26 | * will either recycle the page, or in case of elevated refcnt, it | |
27 | * will release the DMA mapping and in-flight state accounting. We | |
28 | * hope to lift this requirement in the future. | |
ff7d6b27 JDB |
29 | */ |
30 | #ifndef _NET_PAGE_POOL_H | |
31 | #define _NET_PAGE_POOL_H | |
32 | ||
33 | #include <linux/mm.h> /* Needed by ptr_ring */ | |
34 | #include <linux/ptr_ring.h> | |
35 | #include <linux/dma-direction.h> | |
36 | ||
e68bc756 LB |
37 | #define PP_FLAG_DMA_MAP BIT(0) /* Should page_pool do the DMA |
38 | * map/unmap | |
39 | */ | |
40 | #define PP_FLAG_DMA_SYNC_DEV BIT(1) /* If set all pages that the driver gets | |
41 | * from page_pool will be | |
42 | * DMA-synced-for-device according to | |
43 | * the length provided by the device | |
44 | * driver. | |
45 | * Please note DMA-sync-for-CPU is still | |
46 | * device driver responsibility | |
47 | */ | |
0e9d2a0a YL |
48 | #define PP_FLAG_PAGE_FRAG BIT(2) /* for page frag feature */ |
49 | #define PP_FLAG_ALL (PP_FLAG_DMA_MAP |\ | |
50 | PP_FLAG_DMA_SYNC_DEV |\ | |
51 | PP_FLAG_PAGE_FRAG) | |
ff7d6b27 JDB |
52 | |
53 | /* | |
54 | * Fast allocation side cache array/stack | |
55 | * | |
56 | * The cache size and refill watermark is related to the network | |
57 | * use-case. The NAPI budget is 64 packets. After a NAPI poll the RX | |
58 | * ring is usually refilled and the max consumed elements will be 64, | |
59 | * thus a natural max size of objects needed in the cache. | |
60 | * | |
61 | * Keeping room for more objects, is due to XDP_DROP use-case. As | |
62 | * XDP_DROP allows the opportunity to recycle objects directly into | |
63 | * this array, as it shares the same softirq/NAPI protection. If | |
64 | * cache is already full (or partly full) then the XDP_DROP recycles | |
65 | * would have to take a slower code path. | |
66 | */ | |
67 | #define PP_ALLOC_CACHE_SIZE 128 | |
68 | #define PP_ALLOC_CACHE_REFILL 64 | |
69 | struct pp_alloc_cache { | |
70 | u32 count; | |
be5dba25 | 71 | struct page *cache[PP_ALLOC_CACHE_SIZE]; |
ff7d6b27 JDB |
72 | }; |
73 | ||
74 | struct page_pool_params { | |
75 | unsigned int flags; | |
76 | unsigned int order; | |
77 | unsigned int pool_size; | |
78 | int nid; /* Numa node id to allocate from pages from */ | |
79 | struct device *dev; /* device, for DMA pre-mapping purposes */ | |
8c48eea3 | 80 | struct napi_struct *napi; /* Sole consumer of pages, otherwise NULL */ |
ff7d6b27 | 81 | enum dma_data_direction dma_dir; /* DMA mapping direction */ |
e68bc756 LB |
82 | unsigned int max_len; /* max DMA sync memory size */ |
83 | unsigned int offset; /* DMA addr offset */ | |
35b2e549 THJ |
84 | void (*init_callback)(struct page *page, void *arg); |
85 | void *init_arg; | |
ff7d6b27 JDB |
86 | }; |
87 | ||
8610037e JD |
88 | #ifdef CONFIG_PAGE_POOL_STATS |
89 | struct page_pool_alloc_stats { | |
90 | u64 fast; /* fast path allocations */ | |
91 | u64 slow; /* slow-path order 0 allocations */ | |
92 | u64 slow_high_order; /* slow-path high order allocations */ | |
93 | u64 empty; /* failed refills due to empty ptr ring, forcing | |
94 | * slow path allocation | |
95 | */ | |
96 | u64 refill; /* allocations via successful refill */ | |
97 | u64 waive; /* failed refills due to numa zone mismatch */ | |
98 | }; | |
ad6fa1e1 JD |
99 | |
100 | struct page_pool_recycle_stats { | |
101 | u64 cached; /* recycling placed page in the cache. */ | |
102 | u64 cache_full; /* cache was full */ | |
103 | u64 ring; /* recycling placed page back into ptr ring */ | |
104 | u64 ring_full; /* page was released from page-pool because | |
105 | * PTR ring was full. | |
106 | */ | |
107 | u64 released_refcnt; /* page released because of elevated | |
108 | * refcnt | |
109 | */ | |
110 | }; | |
6b95e338 JD |
111 | |
112 | /* This struct wraps the above stats structs so users of the | |
113 | * page_pool_get_stats API can pass a single argument when requesting the | |
114 | * stats for the page pool. | |
115 | */ | |
116 | struct page_pool_stats { | |
117 | struct page_pool_alloc_stats alloc_stats; | |
118 | struct page_pool_recycle_stats recycle_stats; | |
119 | }; | |
120 | ||
f3c5264f LB |
121 | int page_pool_ethtool_stats_get_count(void); |
122 | u8 *page_pool_ethtool_stats_get_strings(u8 *data); | |
123 | u64 *page_pool_ethtool_stats_get(u64 *data, void *stats); | |
124 | ||
6b95e338 JD |
125 | /* |
126 | * Drivers that wish to harvest page pool stats and report them to users | |
127 | * (perhaps via ethtool, debugfs, or another mechanism) can allocate a | |
128 | * struct page_pool_stats call page_pool_get_stats to get stats for the specified pool. | |
129 | */ | |
130 | bool page_pool_get_stats(struct page_pool *pool, | |
131 | struct page_pool_stats *stats); | |
f3c5264f LB |
132 | #else |
133 | ||
134 | static inline int page_pool_ethtool_stats_get_count(void) | |
135 | { | |
136 | return 0; | |
137 | } | |
138 | ||
139 | static inline u8 *page_pool_ethtool_stats_get_strings(u8 *data) | |
140 | { | |
141 | return data; | |
142 | } | |
143 | ||
144 | static inline u64 *page_pool_ethtool_stats_get(u64 *data, void *stats) | |
145 | { | |
146 | return data; | |
147 | } | |
148 | ||
8610037e JD |
149 | #endif |
150 | ||
ff7d6b27 | 151 | struct page_pool { |
ff7d6b27 JDB |
152 | struct page_pool_params p; |
153 | ||
c3f812ce JL |
154 | struct delayed_work release_dw; |
155 | void (*disconnect)(void *); | |
156 | unsigned long defer_start; | |
157 | unsigned long defer_warn; | |
158 | ||
159 | u32 pages_state_hold_cnt; | |
53e0961d YL |
160 | unsigned int frag_offset; |
161 | struct page *frag_page; | |
162 | long frag_users; | |
8610037e JD |
163 | |
164 | #ifdef CONFIG_PAGE_POOL_STATS | |
165 | /* these stats are incremented while in softirq context */ | |
166 | struct page_pool_alloc_stats alloc_stats; | |
167 | #endif | |
64693ec7 | 168 | u32 xdp_mem_id; |
99c07c43 | 169 | |
ff7d6b27 JDB |
170 | /* |
171 | * Data structure for allocation side | |
172 | * | |
173 | * Drivers allocation side usually already perform some kind | |
174 | * of resource protection. Piggyback on this protection, and | |
175 | * require driver to protect allocation side. | |
176 | * | |
177 | * For NIC drivers this means, allocate a page_pool per | |
178 | * RX-queue. As the RX-queue is already protected by | |
179 | * Softirq/BH scheduling and napi_schedule. NAPI schedule | |
180 | * guarantee that a single napi_struct will only be scheduled | |
181 | * on a single CPU (see napi_schedule). | |
182 | */ | |
183 | struct pp_alloc_cache alloc ____cacheline_aligned_in_smp; | |
184 | ||
185 | /* Data structure for storing recycled pages. | |
186 | * | |
187 | * Returning/freeing pages is more complicated synchronization | |
188 | * wise, because free's can happen on remote CPUs, with no | |
189 | * association with allocation resource. | |
190 | * | |
191 | * Use ptr_ring, as it separates consumer and producer | |
192 | * effeciently, it a way that doesn't bounce cache-lines. | |
193 | * | |
194 | * TODO: Implement bulk return pages into this structure. | |
195 | */ | |
196 | struct ptr_ring ring; | |
99c07c43 | 197 | |
ad6fa1e1 JD |
198 | #ifdef CONFIG_PAGE_POOL_STATS |
199 | /* recycle stats are per-cpu to avoid locking */ | |
200 | struct page_pool_recycle_stats __percpu *recycle_stats; | |
201 | #endif | |
99c07c43 | 202 | atomic_t pages_state_release_cnt; |
1da4bbef IK |
203 | |
204 | /* A page_pool is strictly tied to a single RX-queue being | |
205 | * protected by NAPI, due to above pp_alloc_cache. This | |
206 | * refcnt serves purpose is to simplify drivers error handling. | |
207 | */ | |
208 | refcount_t user_cnt; | |
7c9e6942 JDB |
209 | |
210 | u64 destroy_cnt; | |
ff7d6b27 JDB |
211 | }; |
212 | ||
213 | struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp); | |
214 | ||
215 | static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool) | |
216 | { | |
217 | gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); | |
218 | ||
219 | return page_pool_alloc_pages(pool, gfp); | |
220 | } | |
221 | ||
53e0961d YL |
222 | struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, |
223 | unsigned int size, gfp_t gfp); | |
224 | ||
225 | static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, | |
226 | unsigned int *offset, | |
227 | unsigned int size) | |
228 | { | |
229 | gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); | |
230 | ||
231 | return page_pool_alloc_frag(pool, offset, size, gfp); | |
232 | } | |
233 | ||
bb005f2a IA |
234 | /* get the stored dma direction. A driver might decide to treat this locally and |
235 | * avoid the extra cache line from page_pool to determine the direction | |
236 | */ | |
237 | static | |
238 | inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool) | |
239 | { | |
240 | return pool->p.dma_dir; | |
241 | } | |
242 | ||
8c48eea3 | 243 | bool page_pool_return_skb_page(struct page *page, bool napi_safe); |
6a5bcd84 | 244 | |
ff7d6b27 JDB |
245 | struct page_pool *page_pool_create(const struct page_pool_params *params); |
246 | ||
64693ec7 THJ |
247 | struct xdp_mem_info; |
248 | ||
e54cfd7e | 249 | #ifdef CONFIG_PAGE_POOL |
dd64b232 | 250 | void page_pool_unlink_napi(struct page_pool *pool); |
c3f812ce | 251 | void page_pool_destroy(struct page_pool *pool); |
64693ec7 THJ |
252 | void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), |
253 | struct xdp_mem_info *mem); | |
458de8a9 | 254 | void page_pool_release_page(struct page_pool *pool, struct page *page); |
78862447 LB |
255 | void page_pool_put_page_bulk(struct page_pool *pool, void **data, |
256 | int count); | |
c3f812ce | 257 | #else |
dd64b232 JK |
258 | static inline void page_pool_unlink_napi(struct page_pool *pool) |
259 | { | |
260 | } | |
261 | ||
1da4bbef IK |
262 | static inline void page_pool_destroy(struct page_pool *pool) |
263 | { | |
c3f812ce | 264 | } |
1da4bbef | 265 | |
c3f812ce | 266 | static inline void page_pool_use_xdp_mem(struct page_pool *pool, |
64693ec7 THJ |
267 | void (*disconnect)(void *), |
268 | struct xdp_mem_info *mem) | |
c3f812ce | 269 | { |
1da4bbef | 270 | } |
458de8a9 IA |
271 | static inline void page_pool_release_page(struct page_pool *pool, |
272 | struct page *page) | |
273 | { | |
274 | } | |
78862447 LB |
275 | |
276 | static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data, | |
277 | int count) | |
278 | { | |
279 | } | |
c3f812ce | 280 | #endif |
1da4bbef | 281 | |
52cc6ffc AD |
282 | void page_pool_put_defragged_page(struct page_pool *pool, struct page *page, |
283 | unsigned int dma_sync_size, | |
284 | bool allow_direct); | |
ff7d6b27 | 285 | |
4d4266e3 IA |
286 | /* pp_frag_count represents the number of writers who can update the page |
287 | * either by updating skb->data or via DMA mappings for the device. | |
288 | * We can't rely on the page refcnt for that as we don't know who might be | |
289 | * holding page references and we can't reliably destroy or sync DMA mappings | |
290 | * of the fragments. | |
291 | * | |
292 | * When pp_frag_count reaches 0 we can either recycle the page if the page | |
293 | * refcnt is 1 or return it back to the memory allocator and destroy any | |
294 | * mappings we have. | |
295 | */ | |
52cc6ffc AD |
296 | static inline void page_pool_fragment_page(struct page *page, long nr) |
297 | { | |
298 | atomic_long_set(&page->pp_frag_count, nr); | |
299 | } | |
300 | ||
301 | static inline long page_pool_defrag_page(struct page *page, long nr) | |
302 | { | |
303 | long ret; | |
304 | ||
305 | /* If nr == pp_frag_count then we have cleared all remaining | |
306 | * references to the page. No need to actually overwrite it, instead | |
307 | * we can leave this to be overwritten by the calling function. | |
308 | * | |
309 | * The main advantage to doing this is that an atomic_read is | |
310 | * generally a much cheaper operation than an atomic update, | |
311 | * especially when dealing with a page that may be partitioned | |
312 | * into only 2 or 3 pieces. | |
313 | */ | |
314 | if (atomic_long_read(&page->pp_frag_count) == nr) | |
315 | return 0; | |
316 | ||
317 | ret = atomic_long_sub_return(nr, &page->pp_frag_count); | |
318 | WARN_ON(ret < 0); | |
319 | return ret; | |
320 | } | |
321 | ||
322 | static inline bool page_pool_is_last_frag(struct page_pool *pool, | |
323 | struct page *page) | |
324 | { | |
325 | /* If fragments aren't enabled or count is 0 we were the last user */ | |
326 | return !(pool->p.flags & PP_FLAG_PAGE_FRAG) || | |
327 | (page_pool_defrag_page(page, 1) == 0); | |
328 | } | |
329 | ||
330 | static inline void page_pool_put_page(struct page_pool *pool, | |
331 | struct page *page, | |
332 | unsigned int dma_sync_size, | |
333 | bool allow_direct) | |
ff7d6b27 | 334 | { |
57d0a1c1 JDB |
335 | /* When page_pool isn't compiled-in, net/core/xdp.c doesn't |
336 | * allow registering MEM_TYPE_PAGE_POOL, but shield linker. | |
337 | */ | |
338 | #ifdef CONFIG_PAGE_POOL | |
52cc6ffc AD |
339 | if (!page_pool_is_last_frag(pool, page)) |
340 | return; | |
341 | ||
342 | page_pool_put_defragged_page(pool, page, dma_sync_size, allow_direct); | |
57d0a1c1 | 343 | #endif |
ff7d6b27 | 344 | } |
458de8a9 | 345 | |
52cc6ffc AD |
346 | /* Same as above but will try to sync the entire area pool->max_len */ |
347 | static inline void page_pool_put_full_page(struct page_pool *pool, | |
348 | struct page *page, bool allow_direct) | |
349 | { | |
350 | page_pool_put_page(pool, page, -1, allow_direct); | |
351 | } | |
352 | ||
458de8a9 | 353 | /* Same as above but the caller must guarantee safe context. e.g NAPI */ |
ff7d6b27 JDB |
354 | static inline void page_pool_recycle_direct(struct page_pool *pool, |
355 | struct page *page) | |
356 | { | |
458de8a9 | 357 | page_pool_put_full_page(pool, page, true); |
6bf071bf JDB |
358 | } |
359 | ||
f915b75b YL |
360 | #define PAGE_POOL_DMA_USE_PP_FRAG_COUNT \ |
361 | (sizeof(dma_addr_t) > sizeof(unsigned long)) | |
362 | ||
0afdeeed IA |
363 | static inline dma_addr_t page_pool_get_dma_addr(struct page *page) |
364 | { | |
f915b75b YL |
365 | dma_addr_t ret = page->dma_addr; |
366 | ||
367 | if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT) | |
368 | ret |= (dma_addr_t)page->dma_addr_upper << 16 << 16; | |
369 | ||
370 | return ret; | |
9ddb3c14 MWO |
371 | } |
372 | ||
373 | static inline void page_pool_set_dma_addr(struct page *page, dma_addr_t addr) | |
374 | { | |
0e9d2a0a | 375 | page->dma_addr = addr; |
f915b75b YL |
376 | if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT) |
377 | page->dma_addr_upper = upper_32_bits(addr); | |
0e9d2a0a YL |
378 | } |
379 | ||
57d0a1c1 JDB |
380 | static inline bool is_page_pool_compiled_in(void) |
381 | { | |
382 | #ifdef CONFIG_PAGE_POOL | |
383 | return true; | |
384 | #else | |
385 | return false; | |
386 | #endif | |
387 | } | |
388 | ||
1da4bbef IK |
389 | static inline bool page_pool_put(struct page_pool *pool) |
390 | { | |
391 | return refcount_dec_and_test(&pool->user_cnt); | |
392 | } | |
393 | ||
bc836748 SM |
394 | /* Caller must provide appropriate safe context, e.g. NAPI. */ |
395 | void page_pool_update_nid(struct page_pool *pool, int new_nid); | |
396 | static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid) | |
397 | { | |
398 | if (unlikely(pool->p.nid != new_nid)) | |
399 | page_pool_update_nid(pool, new_nid); | |
400 | } | |
78862447 LB |
401 | |
402 | static inline void page_pool_ring_lock(struct page_pool *pool) | |
403 | __acquires(&pool->ring.producer_lock) | |
404 | { | |
542bcea4 | 405 | if (in_softirq()) |
78862447 LB |
406 | spin_lock(&pool->ring.producer_lock); |
407 | else | |
408 | spin_lock_bh(&pool->ring.producer_lock); | |
409 | } | |
410 | ||
411 | static inline void page_pool_ring_unlock(struct page_pool *pool) | |
412 | __releases(&pool->ring.producer_lock) | |
413 | { | |
542bcea4 | 414 | if (in_softirq()) |
78862447 LB |
415 | spin_unlock(&pool->ring.producer_lock); |
416 | else | |
417 | spin_unlock_bh(&pool->ring.producer_lock); | |
418 | } | |
419 | ||
ff7d6b27 | 420 | #endif /* _NET_PAGE_POOL_H */ |