Commit | Line | Data |
---|---|---|
04f00ab2 LR |
1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | ||
3 | #ifndef _NET_IPV6_GRO_H | |
4 | #define _NET_IPV6_GRO_H | |
5 | ||
e75ec151 | 6 | #include <linux/indirect_call_wrapper.h> |
4721031c ED |
7 | #include <linux/ip.h> |
8 | #include <linux/ipv6.h> | |
75082e7f | 9 | #include <net/ip6_checksum.h> |
4721031c ED |
10 | #include <linux/skbuff.h> |
11 | #include <net/udp.h> | |
e75ec151 | 12 | |
4721031c | 13 | struct napi_gro_cb { |
7b355b76 RG |
14 | union { |
15 | struct { | |
16 | /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */ | |
17 | void *frag0; | |
4721031c | 18 | |
7b355b76 RG |
19 | /* Length of frag0. */ |
20 | unsigned int frag0_len; | |
21 | }; | |
22 | ||
23 | struct { | |
24 | /* used in skb_gro_receive() slow path */ | |
25 | struct sk_buff *last; | |
26 | ||
27 | /* jiffies when first packet was created/queued */ | |
28 | unsigned long age; | |
29 | }; | |
30 | }; | |
4721031c ED |
31 | |
32 | /* This indicates where we are processing relative to skb->data. */ | |
33 | int data_offset; | |
34 | ||
35 | /* This is non-zero if the packet cannot be merged with the new skb. */ | |
36 | u16 flush; | |
37 | ||
38 | /* Save the IP ID here and check when we get to the transport layer */ | |
39 | u16 flush_id; | |
40 | ||
41 | /* Number of segments aggregated. */ | |
42 | u16 count; | |
43 | ||
172bf009 | 44 | /* Used in ipv6_gro_receive() and foo-over-udp and esp-in-udp */ |
de5a1f3c | 45 | u16 proto; |
4721031c | 46 | |
8467fadc GP |
47 | /* Used in napi_gro_cb::free */ |
48 | #define NAPI_GRO_FREE 1 | |
49 | #define NAPI_GRO_FREE_STOLEN_HEAD 2 | |
de5a1f3c PA |
50 | /* portion of the cb set to zero at every gro iteration */ |
51 | struct_group(zeroed, | |
52 | ||
53 | /* Start offset for remote checksum offload */ | |
54 | u16 gro_remcsum_start; | |
4721031c | 55 | |
de5a1f3c PA |
56 | /* This is non-zero if the packet may be of the same flow. */ |
57 | u8 same_flow:1; | |
4721031c | 58 | |
de5a1f3c PA |
59 | /* Used in tunnel GRO receive */ |
60 | u8 encap_mark:1; | |
4721031c | 61 | |
de5a1f3c PA |
62 | /* GRO checksum is valid */ |
63 | u8 csum_valid:1; | |
4721031c | 64 | |
de5a1f3c PA |
65 | /* Number of checksums via CHECKSUM_UNNECESSARY */ |
66 | u8 csum_cnt:3; | |
4721031c | 67 | |
de5a1f3c PA |
68 | /* Free the skb? */ |
69 | u8 free:2; | |
4721031c | 70 | |
de5a1f3c PA |
71 | /* Used in foo-over-udp, set in udp[46]_gro_receive */ |
72 | u8 is_ipv6:1; | |
4721031c | 73 | |
de5a1f3c PA |
74 | /* Used in GRE, set in fou/gue_gro_receive */ |
75 | u8 is_fou:1; | |
4721031c | 76 | |
de5a1f3c PA |
77 | /* Used to determine if flush_id can be ignored */ |
78 | u8 is_atomic:1; | |
4721031c | 79 | |
de5a1f3c PA |
80 | /* Number of gro_receive callbacks this packet already went through */ |
81 | u8 recursion_counter:4; | |
4721031c | 82 | |
de5a1f3c PA |
83 | /* GRO is done by frag_list pointer chaining. */ |
84 | u8 is_flist:1; | |
85 | ); | |
4721031c ED |
86 | |
87 | /* used to support CHECKSUM_COMPLETE for tunneling protocols */ | |
88 | __wsum csum; | |
4721031c ED |
89 | }; |
90 | ||
91 | #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) | |
92 | ||
93 | #define GRO_RECURSION_LIMIT 15 | |
94 | static inline int gro_recursion_inc_test(struct sk_buff *skb) | |
95 | { | |
96 | return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; | |
97 | } | |
98 | ||
99 | typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *); | |
100 | static inline struct sk_buff *call_gro_receive(gro_receive_t cb, | |
101 | struct list_head *head, | |
102 | struct sk_buff *skb) | |
103 | { | |
104 | if (unlikely(gro_recursion_inc_test(skb))) { | |
105 | NAPI_GRO_CB(skb)->flush |= 1; | |
106 | return NULL; | |
107 | } | |
108 | ||
109 | return cb(head, skb); | |
110 | } | |
111 | ||
112 | typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *, | |
113 | struct sk_buff *); | |
114 | static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb, | |
115 | struct sock *sk, | |
116 | struct list_head *head, | |
117 | struct sk_buff *skb) | |
118 | { | |
119 | if (unlikely(gro_recursion_inc_test(skb))) { | |
120 | NAPI_GRO_CB(skb)->flush |= 1; | |
121 | return NULL; | |
122 | } | |
123 | ||
124 | return cb(sk, head, skb); | |
125 | } | |
126 | ||
127 | static inline unsigned int skb_gro_offset(const struct sk_buff *skb) | |
128 | { | |
129 | return NAPI_GRO_CB(skb)->data_offset; | |
130 | } | |
131 | ||
132 | static inline unsigned int skb_gro_len(const struct sk_buff *skb) | |
133 | { | |
134 | return skb->len - NAPI_GRO_CB(skb)->data_offset; | |
135 | } | |
136 | ||
137 | static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) | |
138 | { | |
139 | NAPI_GRO_CB(skb)->data_offset += len; | |
140 | } | |
141 | ||
142 | static inline void *skb_gro_header_fast(struct sk_buff *skb, | |
143 | unsigned int offset) | |
144 | { | |
145 | return NAPI_GRO_CB(skb)->frag0 + offset; | |
146 | } | |
147 | ||
148 | static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen) | |
149 | { | |
150 | return NAPI_GRO_CB(skb)->frag0_len < hlen; | |
151 | } | |
152 | ||
153 | static inline void skb_gro_frag0_invalidate(struct sk_buff *skb) | |
154 | { | |
155 | NAPI_GRO_CB(skb)->frag0 = NULL; | |
156 | NAPI_GRO_CB(skb)->frag0_len = 0; | |
157 | } | |
158 | ||
159 | static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, | |
160 | unsigned int offset) | |
161 | { | |
162 | if (!pskb_may_pull(skb, hlen)) | |
163 | return NULL; | |
164 | ||
165 | skb_gro_frag0_invalidate(skb); | |
166 | return skb->data + offset; | |
167 | } | |
168 | ||
35ffb665 RG |
169 | static inline void *skb_gro_header(struct sk_buff *skb, |
170 | unsigned int hlen, unsigned int offset) | |
171 | { | |
172 | void *ptr; | |
173 | ||
174 | ptr = skb_gro_header_fast(skb, offset); | |
175 | if (skb_gro_header_hard(skb, hlen)) | |
176 | ptr = skb_gro_header_slow(skb, hlen, offset); | |
177 | return ptr; | |
178 | } | |
179 | ||
4721031c ED |
180 | static inline void *skb_gro_network_header(struct sk_buff *skb) |
181 | { | |
182 | return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + | |
183 | skb_network_offset(skb); | |
184 | } | |
185 | ||
186 | static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto) | |
187 | { | |
188 | const struct iphdr *iph = skb_gro_network_header(skb); | |
189 | ||
190 | return csum_tcpudp_nofold(iph->saddr, iph->daddr, | |
191 | skb_gro_len(skb), proto, 0); | |
192 | } | |
193 | ||
194 | static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, | |
195 | const void *start, unsigned int len) | |
196 | { | |
197 | if (NAPI_GRO_CB(skb)->csum_valid) | |
45cac675 ED |
198 | NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len, |
199 | wsum_negate(NAPI_GRO_CB(skb)->csum))); | |
4721031c ED |
200 | } |
201 | ||
202 | /* GRO checksum functions. These are logical equivalents of the normal | |
203 | * checksum functions (in skbuff.h) except that they operate on the GRO | |
204 | * offsets and fields in sk_buff. | |
205 | */ | |
206 | ||
207 | __sum16 __skb_gro_checksum_complete(struct sk_buff *skb); | |
208 | ||
209 | static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb) | |
210 | { | |
211 | return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb)); | |
212 | } | |
213 | ||
214 | static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb, | |
215 | bool zero_okay, | |
216 | __sum16 check) | |
217 | { | |
218 | return ((skb->ip_summed != CHECKSUM_PARTIAL || | |
219 | skb_checksum_start_offset(skb) < | |
220 | skb_gro_offset(skb)) && | |
221 | !skb_at_gro_remcsum_start(skb) && | |
222 | NAPI_GRO_CB(skb)->csum_cnt == 0 && | |
223 | (!zero_okay || check)); | |
224 | } | |
225 | ||
226 | static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb, | |
227 | __wsum psum) | |
228 | { | |
229 | if (NAPI_GRO_CB(skb)->csum_valid && | |
230 | !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum))) | |
231 | return 0; | |
232 | ||
233 | NAPI_GRO_CB(skb)->csum = psum; | |
234 | ||
235 | return __skb_gro_checksum_complete(skb); | |
236 | } | |
237 | ||
238 | static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb) | |
239 | { | |
240 | if (NAPI_GRO_CB(skb)->csum_cnt > 0) { | |
241 | /* Consume a checksum from CHECKSUM_UNNECESSARY */ | |
242 | NAPI_GRO_CB(skb)->csum_cnt--; | |
243 | } else { | |
244 | /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we | |
245 | * verified a new top level checksum or an encapsulated one | |
246 | * during GRO. This saves work if we fallback to normal path. | |
247 | */ | |
248 | __skb_incr_checksum_unnecessary(skb); | |
249 | } | |
250 | } | |
251 | ||
252 | #define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \ | |
253 | compute_pseudo) \ | |
254 | ({ \ | |
255 | __sum16 __ret = 0; \ | |
256 | if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \ | |
257 | __ret = __skb_gro_checksum_validate_complete(skb, \ | |
258 | compute_pseudo(skb, proto)); \ | |
259 | if (!__ret) \ | |
260 | skb_gro_incr_csum_unnecessary(skb); \ | |
261 | __ret; \ | |
262 | }) | |
263 | ||
264 | #define skb_gro_checksum_validate(skb, proto, compute_pseudo) \ | |
265 | __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo) | |
266 | ||
267 | #define skb_gro_checksum_validate_zero_check(skb, proto, check, \ | |
268 | compute_pseudo) \ | |
269 | __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo) | |
270 | ||
271 | #define skb_gro_checksum_simple_validate(skb) \ | |
272 | __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo) | |
273 | ||
274 | static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb) | |
275 | { | |
276 | return (NAPI_GRO_CB(skb)->csum_cnt == 0 && | |
277 | !NAPI_GRO_CB(skb)->csum_valid); | |
278 | } | |
279 | ||
280 | static inline void __skb_gro_checksum_convert(struct sk_buff *skb, | |
281 | __wsum pseudo) | |
282 | { | |
283 | NAPI_GRO_CB(skb)->csum = ~pseudo; | |
284 | NAPI_GRO_CB(skb)->csum_valid = 1; | |
285 | } | |
286 | ||
287 | #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \ | |
288 | do { \ | |
289 | if (__skb_gro_checksum_convert_check(skb)) \ | |
290 | __skb_gro_checksum_convert(skb, \ | |
291 | compute_pseudo(skb, proto)); \ | |
292 | } while (0) | |
293 | ||
294 | struct gro_remcsum { | |
295 | int offset; | |
296 | __wsum delta; | |
297 | }; | |
298 | ||
299 | static inline void skb_gro_remcsum_init(struct gro_remcsum *grc) | |
300 | { | |
301 | grc->offset = 0; | |
302 | grc->delta = 0; | |
303 | } | |
304 | ||
305 | static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, | |
306 | unsigned int off, size_t hdrlen, | |
307 | int start, int offset, | |
308 | struct gro_remcsum *grc, | |
309 | bool nopartial) | |
310 | { | |
311 | __wsum delta; | |
312 | size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); | |
313 | ||
314 | BUG_ON(!NAPI_GRO_CB(skb)->csum_valid); | |
315 | ||
316 | if (!nopartial) { | |
317 | NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start; | |
318 | return ptr; | |
319 | } | |
320 | ||
35ffb665 RG |
321 | ptr = skb_gro_header(skb, off + plen, off); |
322 | if (!ptr) | |
323 | return NULL; | |
4721031c ED |
324 | |
325 | delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum, | |
326 | start, offset); | |
327 | ||
328 | /* Adjust skb->csum since we changed the packet */ | |
329 | NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta); | |
330 | ||
331 | grc->offset = off + hdrlen + offset; | |
332 | grc->delta = delta; | |
333 | ||
334 | return ptr; | |
335 | } | |
336 | ||
337 | static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, | |
338 | struct gro_remcsum *grc) | |
339 | { | |
340 | void *ptr; | |
341 | size_t plen = grc->offset + sizeof(u16); | |
342 | ||
343 | if (!grc->delta) | |
344 | return; | |
345 | ||
35ffb665 RG |
346 | ptr = skb_gro_header(skb, plen, grc->offset); |
347 | if (!ptr) | |
348 | return; | |
4721031c ED |
349 | |
350 | remcsum_unadjust((__sum16 *)ptr, grc->delta); | |
351 | } | |
352 | ||
353 | #ifdef CONFIG_XFRM_OFFLOAD | |
354 | static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) | |
355 | { | |
356 | if (PTR_ERR(pp) != -EINPROGRESS) | |
357 | NAPI_GRO_CB(skb)->flush |= flush; | |
358 | } | |
359 | static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, | |
360 | struct sk_buff *pp, | |
361 | int flush, | |
362 | struct gro_remcsum *grc) | |
363 | { | |
364 | if (PTR_ERR(pp) != -EINPROGRESS) { | |
365 | NAPI_GRO_CB(skb)->flush |= flush; | |
366 | skb_gro_remcsum_cleanup(skb, grc); | |
367 | skb->remcsum_offload = 0; | |
368 | } | |
369 | } | |
370 | #else | |
371 | static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) | |
372 | { | |
373 | NAPI_GRO_CB(skb)->flush |= flush; | |
374 | } | |
375 | static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, | |
376 | struct sk_buff *pp, | |
377 | int flush, | |
378 | struct gro_remcsum *grc) | |
379 | { | |
380 | NAPI_GRO_CB(skb)->flush |= flush; | |
381 | skb_gro_remcsum_cleanup(skb, grc); | |
382 | skb->remcsum_offload = 0; | |
383 | } | |
384 | #endif | |
e75ec151 | 385 | |
04f00ab2 LR |
386 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, |
387 | struct sk_buff *)); | |
388 | INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int)); | |
389 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, | |
390 | struct sk_buff *)); | |
391 | INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); | |
86af2c82 | 392 | |
4721031c ED |
393 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, |
394 | struct sk_buff *)); | |
395 | INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int)); | |
396 | ||
397 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *, | |
398 | struct sk_buff *)); | |
399 | INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int)); | |
400 | ||
86af2c82 AL |
401 | #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \ |
402 | ({ \ | |
403 | unlikely(gro_recursion_inc_test(skb)) ? \ | |
404 | NAPI_GRO_CB(skb)->flush |= 1, NULL : \ | |
405 | INDIRECT_CALL_INET(cb, f2, f1, head, skb); \ | |
406 | }) | |
407 | ||
4721031c ED |
408 | struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, |
409 | struct udphdr *uh, struct sock *sk); | |
410 | int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); | |
411 | ||
412 | static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) | |
413 | { | |
414 | struct udphdr *uh; | |
415 | unsigned int hlen, off; | |
416 | ||
417 | off = skb_gro_offset(skb); | |
418 | hlen = off + sizeof(*uh); | |
35ffb665 | 419 | uh = skb_gro_header(skb, hlen, off); |
4721031c ED |
420 | |
421 | return uh; | |
422 | } | |
423 | ||
424 | static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) | |
425 | { | |
426 | const struct ipv6hdr *iph = skb_gro_network_header(skb); | |
427 | ||
428 | return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr, | |
429 | skb_gro_len(skb), proto, 0)); | |
430 | } | |
431 | ||
e456a18a ED |
432 | int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); |
433 | ||
587652bb ED |
434 | /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ |
435 | static inline void gro_normal_list(struct napi_struct *napi) | |
436 | { | |
437 | if (!napi->rx_count) | |
438 | return; | |
439 | netif_receive_skb_list_internal(&napi->rx_list); | |
440 | INIT_LIST_HEAD(&napi->rx_list); | |
441 | napi->rx_count = 0; | |
442 | } | |
443 | ||
444 | /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, | |
445 | * pass the whole batch up to the stack. | |
446 | */ | |
447 | static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) | |
448 | { | |
449 | list_add_tail(&skb->list, &napi->rx_list); | |
450 | napi->rx_count += segs; | |
8db24af3 | 451 | if (napi->rx_count >= READ_ONCE(gro_normal_batch)) |
587652bb ED |
452 | gro_normal_list(napi); |
453 | } | |
454 | ||
7938cd15 RG |
455 | /* This function is the alternative of 'inet_iif' and 'inet_sdif' |
456 | * functions in case we can not rely on fields of IPCB. | |
457 | * | |
458 | * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized. | |
459 | * The caller must hold the RCU read lock. | |
460 | */ | |
461 | static inline void inet_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif) | |
462 | { | |
463 | *iif = inet_iif(skb) ?: skb->dev->ifindex; | |
464 | *sdif = 0; | |
465 | ||
466 | #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) | |
467 | if (netif_is_l3_slave(skb->dev)) { | |
468 | struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev); | |
469 | ||
470 | *sdif = *iif; | |
471 | *iif = master ? master->ifindex : 0; | |
472 | } | |
473 | #endif | |
474 | } | |
475 | ||
476 | /* This function is the alternative of 'inet6_iif' and 'inet6_sdif' | |
477 | * functions in case we can not rely on fields of IP6CB. | |
478 | * | |
479 | * The caller must verify skb_valid_dst(skb) is false and skb->dev is initialized. | |
480 | * The caller must hold the RCU read lock. | |
481 | */ | |
482 | static inline void inet6_get_iif_sdif(const struct sk_buff *skb, int *iif, int *sdif) | |
483 | { | |
484 | /* using skb->dev->ifindex because skb_dst(skb) is not initialized */ | |
485 | *iif = skb->dev->ifindex; | |
486 | *sdif = 0; | |
487 | ||
488 | #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) | |
489 | if (netif_is_l3_slave(skb->dev)) { | |
490 | struct net_device *master = netdev_master_upper_dev_get_rcu(skb->dev); | |
491 | ||
492 | *sdif = *iif; | |
493 | *iif = master ? master->ifindex : 0; | |
494 | } | |
495 | #endif | |
496 | } | |
497 | ||
d457a0e3 | 498 | extern struct list_head offload_base; |
587652bb | 499 | |
04f00ab2 | 500 | #endif /* _NET_IPV6_GRO_H */ |