Commit | Line | Data |
---|---|---|
04f00ab2 LR |
1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | ||
3 | #ifndef _NET_IPV6_GRO_H | |
4 | #define _NET_IPV6_GRO_H | |
5 | ||
e75ec151 | 6 | #include <linux/indirect_call_wrapper.h> |
4721031c ED |
7 | #include <linux/ip.h> |
8 | #include <linux/ipv6.h> | |
75082e7f | 9 | #include <net/ip6_checksum.h> |
4721031c ED |
10 | #include <linux/skbuff.h> |
11 | #include <net/udp.h> | |
e75ec151 | 12 | |
4721031c ED |
13 | struct napi_gro_cb { |
14 | /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */ | |
15 | void *frag0; | |
16 | ||
17 | /* Length of frag0. */ | |
18 | unsigned int frag0_len; | |
19 | ||
20 | /* This indicates where we are processing relative to skb->data. */ | |
21 | int data_offset; | |
22 | ||
23 | /* This is non-zero if the packet cannot be merged with the new skb. */ | |
24 | u16 flush; | |
25 | ||
26 | /* Save the IP ID here and check when we get to the transport layer */ | |
27 | u16 flush_id; | |
28 | ||
29 | /* Number of segments aggregated. */ | |
30 | u16 count; | |
31 | ||
de5a1f3c PA |
32 | /* Used in ipv6_gro_receive() and foo-over-udp */ |
33 | u16 proto; | |
4721031c ED |
34 | |
35 | /* jiffies when first packet was created/queued */ | |
36 | unsigned long age; | |
37 | ||
8467fadc GP |
38 | /* Used in napi_gro_cb::free */ |
39 | #define NAPI_GRO_FREE 1 | |
40 | #define NAPI_GRO_FREE_STOLEN_HEAD 2 | |
de5a1f3c PA |
41 | /* portion of the cb set to zero at every gro iteration */ |
42 | struct_group(zeroed, | |
43 | ||
44 | /* Start offset for remote checksum offload */ | |
45 | u16 gro_remcsum_start; | |
4721031c | 46 | |
de5a1f3c PA |
47 | /* This is non-zero if the packet may be of the same flow. */ |
48 | u8 same_flow:1; | |
4721031c | 49 | |
de5a1f3c PA |
50 | /* Used in tunnel GRO receive */ |
51 | u8 encap_mark:1; | |
4721031c | 52 | |
de5a1f3c PA |
53 | /* GRO checksum is valid */ |
54 | u8 csum_valid:1; | |
4721031c | 55 | |
de5a1f3c PA |
56 | /* Number of checksums via CHECKSUM_UNNECESSARY */ |
57 | u8 csum_cnt:3; | |
4721031c | 58 | |
de5a1f3c PA |
59 | /* Free the skb? */ |
60 | u8 free:2; | |
4721031c | 61 | |
de5a1f3c PA |
62 | /* Used in foo-over-udp, set in udp[46]_gro_receive */ |
63 | u8 is_ipv6:1; | |
4721031c | 64 | |
de5a1f3c PA |
65 | /* Used in GRE, set in fou/gue_gro_receive */ |
66 | u8 is_fou:1; | |
4721031c | 67 | |
de5a1f3c PA |
68 | /* Used to determine if flush_id can be ignored */ |
69 | u8 is_atomic:1; | |
4721031c | 70 | |
de5a1f3c PA |
71 | /* Number of gro_receive callbacks this packet already went through */ |
72 | u8 recursion_counter:4; | |
4721031c | 73 | |
de5a1f3c PA |
74 | /* GRO is done by frag_list pointer chaining. */ |
75 | u8 is_flist:1; | |
76 | ); | |
4721031c ED |
77 | |
78 | /* used to support CHECKSUM_COMPLETE for tunneling protocols */ | |
79 | __wsum csum; | |
80 | ||
81 | /* used in skb_gro_receive() slow path */ | |
82 | struct sk_buff *last; | |
83 | }; | |
84 | ||
85 | #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) | |
86 | ||
87 | #define GRO_RECURSION_LIMIT 15 | |
88 | static inline int gro_recursion_inc_test(struct sk_buff *skb) | |
89 | { | |
90 | return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; | |
91 | } | |
92 | ||
93 | typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *); | |
94 | static inline struct sk_buff *call_gro_receive(gro_receive_t cb, | |
95 | struct list_head *head, | |
96 | struct sk_buff *skb) | |
97 | { | |
98 | if (unlikely(gro_recursion_inc_test(skb))) { | |
99 | NAPI_GRO_CB(skb)->flush |= 1; | |
100 | return NULL; | |
101 | } | |
102 | ||
103 | return cb(head, skb); | |
104 | } | |
105 | ||
106 | typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *, | |
107 | struct sk_buff *); | |
108 | static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb, | |
109 | struct sock *sk, | |
110 | struct list_head *head, | |
111 | struct sk_buff *skb) | |
112 | { | |
113 | if (unlikely(gro_recursion_inc_test(skb))) { | |
114 | NAPI_GRO_CB(skb)->flush |= 1; | |
115 | return NULL; | |
116 | } | |
117 | ||
118 | return cb(sk, head, skb); | |
119 | } | |
120 | ||
121 | static inline unsigned int skb_gro_offset(const struct sk_buff *skb) | |
122 | { | |
123 | return NAPI_GRO_CB(skb)->data_offset; | |
124 | } | |
125 | ||
126 | static inline unsigned int skb_gro_len(const struct sk_buff *skb) | |
127 | { | |
128 | return skb->len - NAPI_GRO_CB(skb)->data_offset; | |
129 | } | |
130 | ||
131 | static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) | |
132 | { | |
133 | NAPI_GRO_CB(skb)->data_offset += len; | |
134 | } | |
135 | ||
136 | static inline void *skb_gro_header_fast(struct sk_buff *skb, | |
137 | unsigned int offset) | |
138 | { | |
139 | return NAPI_GRO_CB(skb)->frag0 + offset; | |
140 | } | |
141 | ||
142 | static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen) | |
143 | { | |
144 | return NAPI_GRO_CB(skb)->frag0_len < hlen; | |
145 | } | |
146 | ||
147 | static inline void skb_gro_frag0_invalidate(struct sk_buff *skb) | |
148 | { | |
149 | NAPI_GRO_CB(skb)->frag0 = NULL; | |
150 | NAPI_GRO_CB(skb)->frag0_len = 0; | |
151 | } | |
152 | ||
153 | static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, | |
154 | unsigned int offset) | |
155 | { | |
156 | if (!pskb_may_pull(skb, hlen)) | |
157 | return NULL; | |
158 | ||
159 | skb_gro_frag0_invalidate(skb); | |
160 | return skb->data + offset; | |
161 | } | |
162 | ||
35ffb665 RG |
163 | static inline void *skb_gro_header(struct sk_buff *skb, |
164 | unsigned int hlen, unsigned int offset) | |
165 | { | |
166 | void *ptr; | |
167 | ||
168 | ptr = skb_gro_header_fast(skb, offset); | |
169 | if (skb_gro_header_hard(skb, hlen)) | |
170 | ptr = skb_gro_header_slow(skb, hlen, offset); | |
171 | return ptr; | |
172 | } | |
173 | ||
4721031c ED |
174 | static inline void *skb_gro_network_header(struct sk_buff *skb) |
175 | { | |
176 | return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + | |
177 | skb_network_offset(skb); | |
178 | } | |
179 | ||
180 | static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto) | |
181 | { | |
182 | const struct iphdr *iph = skb_gro_network_header(skb); | |
183 | ||
184 | return csum_tcpudp_nofold(iph->saddr, iph->daddr, | |
185 | skb_gro_len(skb), proto, 0); | |
186 | } | |
187 | ||
188 | static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, | |
189 | const void *start, unsigned int len) | |
190 | { | |
191 | if (NAPI_GRO_CB(skb)->csum_valid) | |
45cac675 ED |
192 | NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len, |
193 | wsum_negate(NAPI_GRO_CB(skb)->csum))); | |
4721031c ED |
194 | } |
195 | ||
196 | /* GRO checksum functions. These are logical equivalents of the normal | |
197 | * checksum functions (in skbuff.h) except that they operate on the GRO | |
198 | * offsets and fields in sk_buff. | |
199 | */ | |
200 | ||
201 | __sum16 __skb_gro_checksum_complete(struct sk_buff *skb); | |
202 | ||
203 | static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb) | |
204 | { | |
205 | return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb)); | |
206 | } | |
207 | ||
208 | static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb, | |
209 | bool zero_okay, | |
210 | __sum16 check) | |
211 | { | |
212 | return ((skb->ip_summed != CHECKSUM_PARTIAL || | |
213 | skb_checksum_start_offset(skb) < | |
214 | skb_gro_offset(skb)) && | |
215 | !skb_at_gro_remcsum_start(skb) && | |
216 | NAPI_GRO_CB(skb)->csum_cnt == 0 && | |
217 | (!zero_okay || check)); | |
218 | } | |
219 | ||
220 | static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb, | |
221 | __wsum psum) | |
222 | { | |
223 | if (NAPI_GRO_CB(skb)->csum_valid && | |
224 | !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum))) | |
225 | return 0; | |
226 | ||
227 | NAPI_GRO_CB(skb)->csum = psum; | |
228 | ||
229 | return __skb_gro_checksum_complete(skb); | |
230 | } | |
231 | ||
232 | static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb) | |
233 | { | |
234 | if (NAPI_GRO_CB(skb)->csum_cnt > 0) { | |
235 | /* Consume a checksum from CHECKSUM_UNNECESSARY */ | |
236 | NAPI_GRO_CB(skb)->csum_cnt--; | |
237 | } else { | |
238 | /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we | |
239 | * verified a new top level checksum or an encapsulated one | |
240 | * during GRO. This saves work if we fallback to normal path. | |
241 | */ | |
242 | __skb_incr_checksum_unnecessary(skb); | |
243 | } | |
244 | } | |
245 | ||
246 | #define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \ | |
247 | compute_pseudo) \ | |
248 | ({ \ | |
249 | __sum16 __ret = 0; \ | |
250 | if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \ | |
251 | __ret = __skb_gro_checksum_validate_complete(skb, \ | |
252 | compute_pseudo(skb, proto)); \ | |
253 | if (!__ret) \ | |
254 | skb_gro_incr_csum_unnecessary(skb); \ | |
255 | __ret; \ | |
256 | }) | |
257 | ||
258 | #define skb_gro_checksum_validate(skb, proto, compute_pseudo) \ | |
259 | __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo) | |
260 | ||
261 | #define skb_gro_checksum_validate_zero_check(skb, proto, check, \ | |
262 | compute_pseudo) \ | |
263 | __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo) | |
264 | ||
265 | #define skb_gro_checksum_simple_validate(skb) \ | |
266 | __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo) | |
267 | ||
268 | static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb) | |
269 | { | |
270 | return (NAPI_GRO_CB(skb)->csum_cnt == 0 && | |
271 | !NAPI_GRO_CB(skb)->csum_valid); | |
272 | } | |
273 | ||
274 | static inline void __skb_gro_checksum_convert(struct sk_buff *skb, | |
275 | __wsum pseudo) | |
276 | { | |
277 | NAPI_GRO_CB(skb)->csum = ~pseudo; | |
278 | NAPI_GRO_CB(skb)->csum_valid = 1; | |
279 | } | |
280 | ||
281 | #define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \ | |
282 | do { \ | |
283 | if (__skb_gro_checksum_convert_check(skb)) \ | |
284 | __skb_gro_checksum_convert(skb, \ | |
285 | compute_pseudo(skb, proto)); \ | |
286 | } while (0) | |
287 | ||
288 | struct gro_remcsum { | |
289 | int offset; | |
290 | __wsum delta; | |
291 | }; | |
292 | ||
293 | static inline void skb_gro_remcsum_init(struct gro_remcsum *grc) | |
294 | { | |
295 | grc->offset = 0; | |
296 | grc->delta = 0; | |
297 | } | |
298 | ||
299 | static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, | |
300 | unsigned int off, size_t hdrlen, | |
301 | int start, int offset, | |
302 | struct gro_remcsum *grc, | |
303 | bool nopartial) | |
304 | { | |
305 | __wsum delta; | |
306 | size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); | |
307 | ||
308 | BUG_ON(!NAPI_GRO_CB(skb)->csum_valid); | |
309 | ||
310 | if (!nopartial) { | |
311 | NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start; | |
312 | return ptr; | |
313 | } | |
314 | ||
35ffb665 RG |
315 | ptr = skb_gro_header(skb, off + plen, off); |
316 | if (!ptr) | |
317 | return NULL; | |
4721031c ED |
318 | |
319 | delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum, | |
320 | start, offset); | |
321 | ||
322 | /* Adjust skb->csum since we changed the packet */ | |
323 | NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta); | |
324 | ||
325 | grc->offset = off + hdrlen + offset; | |
326 | grc->delta = delta; | |
327 | ||
328 | return ptr; | |
329 | } | |
330 | ||
331 | static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, | |
332 | struct gro_remcsum *grc) | |
333 | { | |
334 | void *ptr; | |
335 | size_t plen = grc->offset + sizeof(u16); | |
336 | ||
337 | if (!grc->delta) | |
338 | return; | |
339 | ||
35ffb665 RG |
340 | ptr = skb_gro_header(skb, plen, grc->offset); |
341 | if (!ptr) | |
342 | return; | |
4721031c ED |
343 | |
344 | remcsum_unadjust((__sum16 *)ptr, grc->delta); | |
345 | } | |
346 | ||
347 | #ifdef CONFIG_XFRM_OFFLOAD | |
348 | static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) | |
349 | { | |
350 | if (PTR_ERR(pp) != -EINPROGRESS) | |
351 | NAPI_GRO_CB(skb)->flush |= flush; | |
352 | } | |
353 | static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, | |
354 | struct sk_buff *pp, | |
355 | int flush, | |
356 | struct gro_remcsum *grc) | |
357 | { | |
358 | if (PTR_ERR(pp) != -EINPROGRESS) { | |
359 | NAPI_GRO_CB(skb)->flush |= flush; | |
360 | skb_gro_remcsum_cleanup(skb, grc); | |
361 | skb->remcsum_offload = 0; | |
362 | } | |
363 | } | |
364 | #else | |
365 | static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) | |
366 | { | |
367 | NAPI_GRO_CB(skb)->flush |= flush; | |
368 | } | |
369 | static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, | |
370 | struct sk_buff *pp, | |
371 | int flush, | |
372 | struct gro_remcsum *grc) | |
373 | { | |
374 | NAPI_GRO_CB(skb)->flush |= flush; | |
375 | skb_gro_remcsum_cleanup(skb, grc); | |
376 | skb->remcsum_offload = 0; | |
377 | } | |
378 | #endif | |
e75ec151 | 379 | |
04f00ab2 LR |
380 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, |
381 | struct sk_buff *)); | |
382 | INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int)); | |
383 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, | |
384 | struct sk_buff *)); | |
385 | INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); | |
86af2c82 | 386 | |
4721031c ED |
387 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, |
388 | struct sk_buff *)); | |
389 | INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int)); | |
390 | ||
391 | INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *, | |
392 | struct sk_buff *)); | |
393 | INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int)); | |
394 | ||
86af2c82 AL |
395 | #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \ |
396 | ({ \ | |
397 | unlikely(gro_recursion_inc_test(skb)) ? \ | |
398 | NAPI_GRO_CB(skb)->flush |= 1, NULL : \ | |
399 | INDIRECT_CALL_INET(cb, f2, f1, head, skb); \ | |
400 | }) | |
401 | ||
4721031c ED |
402 | struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, |
403 | struct udphdr *uh, struct sock *sk); | |
404 | int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); | |
405 | ||
406 | static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) | |
407 | { | |
408 | struct udphdr *uh; | |
409 | unsigned int hlen, off; | |
410 | ||
411 | off = skb_gro_offset(skb); | |
412 | hlen = off + sizeof(*uh); | |
35ffb665 | 413 | uh = skb_gro_header(skb, hlen, off); |
4721031c ED |
414 | |
415 | return uh; | |
416 | } | |
417 | ||
418 | static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) | |
419 | { | |
420 | const struct ipv6hdr *iph = skb_gro_network_header(skb); | |
421 | ||
422 | return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr, | |
423 | skb_gro_len(skb), proto, 0)); | |
424 | } | |
425 | ||
e456a18a ED |
426 | int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); |
427 | ||
587652bb ED |
428 | /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ |
429 | static inline void gro_normal_list(struct napi_struct *napi) | |
430 | { | |
431 | if (!napi->rx_count) | |
432 | return; | |
433 | netif_receive_skb_list_internal(&napi->rx_list); | |
434 | INIT_LIST_HEAD(&napi->rx_list); | |
435 | napi->rx_count = 0; | |
436 | } | |
437 | ||
438 | /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, | |
439 | * pass the whole batch up to the stack. | |
440 | */ | |
441 | static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) | |
442 | { | |
443 | list_add_tail(&skb->list, &napi->rx_list); | |
444 | napi->rx_count += segs; | |
8db24af3 | 445 | if (napi->rx_count >= READ_ONCE(gro_normal_batch)) |
587652bb ED |
446 | gro_normal_list(napi); |
447 | } | |
448 | ||
449 | ||
04f00ab2 | 450 | #endif /* _NET_IPV6_GRO_H */ |