projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'macb-next'
[linux-2.6-block.git] / net / core / flow_dissector.c
CommitLineData
fbff949e 1#include <linux/kernel.h>
0744dd00 2#include <linux/skbuff.h>
c452ed70 3#include <linux/export.h>
0744dd00
ED		 4#include <linux/ip.h>
5#include <linux/ipv6.h>
6#include <linux/if_vlan.h>
43e66528 7#include <net/dsa.h>
a38402bc 8#include <net/dst_metadata.h>
0744dd00 9#include <net/ip.h>
ddbe5032 10#include <net/ipv6.h>
ab10dccb
GF		 11#include <net/gre.h>
12#include <net/pptp.h>
f77668dc
DB		 13#include <linux/igmp.h>
14#include <linux/icmp.h>
15#include <linux/sctp.h>
16#include <linux/dccp.h>
0744dd00
ED		 17#include <linux/if_tunnel.h>
18#include <linux/if_pppox.h>
19#include <linux/ppp_defs.h>
06635a35 20#include <linux/stddef.h>
67a900cc 21#include <linux/if_ether.h>
b3baa0fb 22#include <linux/mpls.h>
ac4bb5de 23#include <linux/tcp.h>
1bd758eb 24#include <net/flow_dissector.h>
56193d1b 25#include <scsi/fc/fc_fcoe.h>
0744dd00 26
20a17bf6
DM		 27static void dissector_set_key(struct flow_dissector *flow_dissector,
28 enum flow_dissector_key_id key_id)
fbff949e
JP		 29{
30 flow_dissector->used_keys |= (1 << key_id);
31}
32
fbff949e
JP		 33void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
34 const struct flow_dissector_key *key,
35 unsigned int key_count)
36{
37 unsigned int i;
38
39 memset(flow_dissector, 0, sizeof(*flow_dissector));
40
41 for (i = 0; i < key_count; i++, key++) {
42 /* User should make sure that every key target offset is withing
43 * boundaries of unsigned short.
44 */
45 BUG_ON(key->offset > USHRT_MAX);
20a17bf6
DM		 46 BUG_ON(dissector_uses_key(flow_dissector,
47 key->key_id));
fbff949e 48
20a17bf6 49 dissector_set_key(flow_dissector, key->key_id);
fbff949e
JP		 50 flow_dissector->offset[key->key_id] = key->offset;
51 }
52
42aecaa9
TH		 53 /* Ensure that the dissector always includes control and basic key.
54 * That way we are able to avoid handling lack of these in fast path.
fbff949e 55 */
20a17bf6
DM		 56 BUG_ON(!dissector_uses_key(flow_dissector,
57 FLOW_DISSECTOR_KEY_CONTROL));
58 BUG_ON(!dissector_uses_key(flow_dissector,
59 FLOW_DISSECTOR_KEY_BASIC));
fbff949e
JP		 60}
61EXPORT_SYMBOL(skb_flow_dissector_init);
62
972d3876
SH		 63/**
64 * skb_flow_get_be16 - extract be16 entity
65 * @skb: sk_buff to extract from
66 * @poff: offset to extract at
67 * @data: raw buffer pointer to the packet
68 * @hlen: packet header length
69 *
70 * The function will try to retrieve a be32 entity at
71 * offset poff
72 */
d9584d8c
ED		 73static __be16 skb_flow_get_be16(const struct sk_buff *skb, int poff,
74 void *data, int hlen)
972d3876
SH		 75{
76 __be16 *u, _u;
77
78 u = __skb_header_pointer(skb, poff, sizeof(_u), data, hlen, &_u);
79 if (u)
80 return *u;
81
82 return 0;
83}
84
357afe9c 85/**
6451b3f5
WC		 86 * __skb_flow_get_ports - extract the upper layer ports and return them
87 * @skb: sk_buff to extract the ports from
357afe9c
NA		 88 * @thoff: transport header offset
89 * @ip_proto: protocol for which to get port offset
6451b3f5
WC		 90 * @data: raw buffer pointer to the packet, if NULL use skb->data
91 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
357afe9c
NA		 92 *
93 * The function will try to retrieve the ports at offset thoff + poff where poff
94 * is the protocol port offset returned from proto_ports_offset
95 */
690e36e7
DM		 96__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
97 void *data, int hlen)
357afe9c
NA		 98{
99 int poff = proto_ports_offset(ip_proto);
100
690e36e7
DM		 101 if (!data) {
102 data = skb->data;
103 hlen = skb_headlen(skb);
104 }
105
357afe9c
NA		 106 if (poff >= 0) {
107 __be32 *ports, _ports;
108
690e36e7
DM		 109 ports = __skb_header_pointer(skb, thoff + poff,
110 sizeof(_ports), data, hlen, &_ports);
357afe9c
NA		 111 if (ports)
112 return *ports;
113 }
114
115 return 0;
116}
690e36e7 117EXPORT_SYMBOL(__skb_flow_get_ports);
357afe9c 118
a38402bc
SH		 119static void
120skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type,
121 struct flow_dissector *flow_dissector,
122 void *target_container)
123{
124 struct flow_dissector_key_control *ctrl;
125
126 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL))
127 return;
128
129 ctrl = skb_flow_dissector_target(flow_dissector,
130 FLOW_DISSECTOR_KEY_ENC_CONTROL,
131 target_container);
132 ctrl->addr_type = type;
133}
134
135static void
136__skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
137 struct flow_dissector *flow_dissector,
138 void *target_container)
139{
140 struct ip_tunnel_info *info;
141 struct ip_tunnel_key *key;
142
143 /* A quick check to see if there might be something to do. */
144 if (!dissector_uses_key(flow_dissector,
145 FLOW_DISSECTOR_KEY_ENC_KEYID) &&
146 !dissector_uses_key(flow_dissector,
147 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) &&
148 !dissector_uses_key(flow_dissector,
149 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) &&
150 !dissector_uses_key(flow_dissector,
151 FLOW_DISSECTOR_KEY_ENC_CONTROL) &&
152 !dissector_uses_key(flow_dissector,
153 FLOW_DISSECTOR_KEY_ENC_PORTS))
154 return;
155
156 info = skb_tunnel_info(skb);
157 if (!info)
158 return;
159
160 key = &info->key;
161
162 switch (ip_tunnel_info_af(info)) {
163 case AF_INET:
164 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV4_ADDRS,
165 flow_dissector,
166 target_container);
167 if (dissector_uses_key(flow_dissector,
168 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
169 struct flow_dissector_key_ipv4_addrs *ipv4;
170
171 ipv4 = skb_flow_dissector_target(flow_dissector,
172 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
173 target_container);
174 ipv4->src = key->u.ipv4.src;
175 ipv4->dst = key->u.ipv4.dst;
176 }
177 break;
178 case AF_INET6:
179 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV6_ADDRS,
180 flow_dissector,
181 target_container);
182 if (dissector_uses_key(flow_dissector,
183 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
184 struct flow_dissector_key_ipv6_addrs *ipv6;
185
186 ipv6 = skb_flow_dissector_target(flow_dissector,
187 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
188 target_container);
189 ipv6->src = key->u.ipv6.src;
190 ipv6->dst = key->u.ipv6.dst;
191 }
192 break;
193 }
194
195 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
196 struct flow_dissector_key_keyid *keyid;
197
198 keyid = skb_flow_dissector_target(flow_dissector,
199 FLOW_DISSECTOR_KEY_ENC_KEYID,
200 target_container);
201 keyid->keyid = tunnel_id_to_key32(key->tun_id);
202 }
203
204 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
205 struct flow_dissector_key_ports *tp;
206
207 tp = skb_flow_dissector_target(flow_dissector,
208 FLOW_DISSECTOR_KEY_ENC_PORTS,
209 target_container);
210 tp->src = key->tp_src;
211 tp->dst = key->tp_dst;
212 }
213}
214
4a5d6c8b
JP		 215static enum flow_dissect_ret
216__skb_flow_dissect_mpls(const struct sk_buff *skb,
217 struct flow_dissector *flow_dissector,
218 void *target_container, void *data, int nhoff, int hlen)
219{
220 struct flow_dissector_key_keyid *key_keyid;
221 struct mpls_label *hdr, _hdr[2];
029c1ecb 222 u32 entry, label;
4a5d6c8b
JP		 223
224 if (!dissector_uses_key(flow_dissector,
029c1ecb
BL		 225 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) &&
226 !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS))
4a5d6c8b
JP		 227 return FLOW_DISSECT_RET_OUT_GOOD;
228
229 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
230 hlen, &_hdr);
231 if (!hdr)
232 return FLOW_DISSECT_RET_OUT_BAD;
233
029c1ecb
BL		 234 entry = ntohl(hdr[0].entry);
235 label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT;
236
237 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) {
238 struct flow_dissector_key_mpls *key_mpls;
239
240 key_mpls = skb_flow_dissector_target(flow_dissector,
241 FLOW_DISSECTOR_KEY_MPLS,
242 target_container);
243 key_mpls->mpls_label = label;
244 key_mpls->mpls_ttl = (entry & MPLS_LS_TTL_MASK)
245 >> MPLS_LS_TTL_SHIFT;
246 key_mpls->mpls_tc = (entry & MPLS_LS_TC_MASK)
247 >> MPLS_LS_TC_SHIFT;
248 key_mpls->mpls_bos = (entry & MPLS_LS_S_MASK)
249 >> MPLS_LS_S_SHIFT;
250 }
251
252 if (label == MPLS_LABEL_ENTROPY) {
4a5d6c8b
JP		 253 key_keyid = skb_flow_dissector_target(flow_dissector,
254 FLOW_DISSECTOR_KEY_MPLS_ENTROPY,
255 target_container);
256 key_keyid->keyid = hdr[1].entry & htonl(MPLS_LS_LABEL_MASK);
257 }
258 return FLOW_DISSECT_RET_OUT_GOOD;
259}
260
9bf881ff
JP		 261static enum flow_dissect_ret
262__skb_flow_dissect_arp(const struct sk_buff *skb,
263 struct flow_dissector *flow_dissector,
264 void *target_container, void *data, int nhoff, int hlen)
265{
266 struct flow_dissector_key_arp *key_arp;
267 struct {
268 unsigned char ar_sha[ETH_ALEN];
269 unsigned char ar_sip[4];
270 unsigned char ar_tha[ETH_ALEN];
271 unsigned char ar_tip[4];
272 } *arp_eth, _arp_eth;
273 const struct arphdr *arp;
6f14f443 274 struct arphdr _arp;
9bf881ff
JP		 275
276 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP))
277 return FLOW_DISSECT_RET_OUT_GOOD;
278
279 arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data,
280 hlen, &_arp);
281 if (!arp)
282 return FLOW_DISSECT_RET_OUT_BAD;
283
284 if (arp->ar_hrd != htons(ARPHRD_ETHER) ||
285 arp->ar_pro != htons(ETH_P_IP) ||
286 arp->ar_hln != ETH_ALEN ||
287 arp->ar_pln != 4 ||
288 (arp->ar_op != htons(ARPOP_REPLY) &&
289 arp->ar_op != htons(ARPOP_REQUEST)))
290 return FLOW_DISSECT_RET_OUT_BAD;
291
292 arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp),
293 sizeof(_arp_eth), data,
294 hlen, &_arp_eth);
295 if (!arp_eth)
296 return FLOW_DISSECT_RET_OUT_BAD;
297
298 key_arp = skb_flow_dissector_target(flow_dissector,
299 FLOW_DISSECTOR_KEY_ARP,
300 target_container);
301
302 memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip));
303 memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip));
304
305 /* Only store the lower byte of the opcode;
306 * this covers ARPOP_REPLY and ARPOP_REQUEST.
307 */
308 key_arp->op = ntohs(arp->ar_op) & 0xff;
309
310 ether_addr_copy(key_arp->sha, arp_eth->ar_sha);
311 ether_addr_copy(key_arp->tha, arp_eth->ar_tha);
312
313 return FLOW_DISSECT_RET_OUT_GOOD;
314}
315
7c92de8e
JP		 316static enum flow_dissect_ret
317__skb_flow_dissect_gre(const struct sk_buff *skb,
318 struct flow_dissector_key_control *key_control,
319 struct flow_dissector *flow_dissector,
320 void *target_container, void *data,
321 __be16 *p_proto, int *p_nhoff, int *p_hlen,
322 unsigned int flags)
323{
324 struct flow_dissector_key_keyid *key_keyid;
325 struct gre_base_hdr *hdr, _hdr;
326 int offset = 0;
327 u16 gre_ver;
328
329 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr),
330 data, *p_hlen, &_hdr);
331 if (!hdr)
332 return FLOW_DISSECT_RET_OUT_BAD;
333
334 /* Only look inside GRE without routing */
335 if (hdr->flags & GRE_ROUTING)
336 return FLOW_DISSECT_RET_OUT_GOOD;
337
338 /* Only look inside GRE for version 0 and 1 */
339 gre_ver = ntohs(hdr->flags & GRE_VERSION);
340 if (gre_ver > 1)
341 return FLOW_DISSECT_RET_OUT_GOOD;
342
343 *p_proto = hdr->protocol;
344 if (gre_ver) {
345 /* Version1 must be PPTP, and check the flags */
346 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY)))
347 return FLOW_DISSECT_RET_OUT_GOOD;
348 }
349
350 offset += sizeof(struct gre_base_hdr);
351
352 if (hdr->flags & GRE_CSUM)
353 offset += sizeof(((struct gre_full_hdr *) 0)->csum) +
354 sizeof(((struct gre_full_hdr *) 0)->reserved1);
355
356 if (hdr->flags & GRE_KEY) {
357 const __be32 *keyid;
358 __be32 _keyid;
359
360 keyid = __skb_header_pointer(skb, *p_nhoff + offset,
361 sizeof(_keyid),
362 data, *p_hlen, &_keyid);
363 if (!keyid)
364 return FLOW_DISSECT_RET_OUT_BAD;
365
366 if (dissector_uses_key(flow_dissector,
367 FLOW_DISSECTOR_KEY_GRE_KEYID)) {
368 key_keyid = skb_flow_dissector_target(flow_dissector,
369 FLOW_DISSECTOR_KEY_GRE_KEYID,
370 target_container);
371 if (gre_ver == 0)
372 key_keyid->keyid = *keyid;
373 else
374 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK;
375 }
376 offset += sizeof(((struct gre_full_hdr *) 0)->key);
377 }
378
379 if (hdr->flags & GRE_SEQ)
380 offset += sizeof(((struct pptp_gre_header *) 0)->seq);
381
382 if (gre_ver == 0) {
383 if (*p_proto == htons(ETH_P_TEB)) {
384 const struct ethhdr *eth;
385 struct ethhdr _eth;
386
387 eth = __skb_header_pointer(skb, *p_nhoff + offset,
388 sizeof(_eth),
389 data, *p_hlen, &_eth);
390 if (!eth)
391 return FLOW_DISSECT_RET_OUT_BAD;
392 *p_proto = eth->h_proto;
393 offset += sizeof(*eth);
394
395 /* Cap headers that we access via pointers at the
396 * end of the Ethernet header as our maximum alignment
397 * at that point is only 2 bytes.
398 */
399 if (NET_IP_ALIGN)
400 *p_hlen = *p_nhoff + offset;
401 }
402 } else { /* version 1, must be PPTP */
403 u8 _ppp_hdr[PPP_HDRLEN];
404 u8 *ppp_hdr;
405
406 if (hdr->flags & GRE_ACK)
407 offset += sizeof(((struct pptp_gre_header *) 0)->ack);
408
409 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset,
410 sizeof(_ppp_hdr),
411 data, *p_hlen, _ppp_hdr);
412 if (!ppp_hdr)
413 return FLOW_DISSECT_RET_OUT_BAD;
414
415 switch (PPP_PROTOCOL(ppp_hdr)) {
416 case PPP_IP:
417 *p_proto = htons(ETH_P_IP);
418 break;
419 case PPP_IPV6:
420 *p_proto = htons(ETH_P_IPV6);
421 break;
422 default:
423 /* Could probably catch some more like MPLS */
424 break;
425 }
426
427 offset += PPP_HDRLEN;
428 }
429
430 *p_nhoff += offset;
431 key_control->flags |= FLOW_DIS_ENCAPSULATION;
432 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
433 return FLOW_DISSECT_RET_OUT_GOOD;
434
3a1214e8 435 return FLOW_DISSECT_RET_PROTO_AGAIN;
7c92de8e
JP		 436}
437
ac4bb5de
JP		 438static void
439__skb_flow_dissect_tcp(const struct sk_buff *skb,
440 struct flow_dissector *flow_dissector,
441 void *target_container, void *data, int thoff, int hlen)
442{
443 struct flow_dissector_key_tcp *key_tcp;
444 struct tcphdr *th, _th;
445
446 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP))
447 return;
448
449 th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th);
450 if (!th)
451 return;
452
453 if (unlikely(__tcp_hdrlen(th) < sizeof(_th)))
454 return;
455
456 key_tcp = skb_flow_dissector_target(flow_dissector,
457 FLOW_DISSECTOR_KEY_TCP,
458 target_container);
459 key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF));
460}
461
518d8a2e
OG		 462static void
463__skb_flow_dissect_ipv4(const struct sk_buff *skb,
464 struct flow_dissector *flow_dissector,
465 void *target_container, void *data, const struct iphdr *iph)
466{
467 struct flow_dissector_key_ip *key_ip;
468
469 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
470 return;
471
472 key_ip = skb_flow_dissector_target(flow_dissector,
473 FLOW_DISSECTOR_KEY_IP,
474 target_container);
475 key_ip->tos = iph->tos;
476 key_ip->ttl = iph->ttl;
477}
478
479static void
480__skb_flow_dissect_ipv6(const struct sk_buff *skb,
481 struct flow_dissector *flow_dissector,
482 void *target_container, void *data, const struct ipv6hdr *iph)
483{
484 struct flow_dissector_key_ip *key_ip;
485
486 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
487 return;
488
489 key_ip = skb_flow_dissector_target(flow_dissector,
490 FLOW_DISSECTOR_KEY_IP,
491 target_container);
492 key_ip->tos = ipv6_get_dsfield(iph);
493 key_ip->ttl = iph->hop_limit;
494}
495
1eed4dfb
TH		 496/* Maximum number of protocol headers that can be parsed in
497 * __skb_flow_dissect
498 */
499#define MAX_FLOW_DISSECT_HDRS 15
500
501static bool skb_flow_dissect_allowed(int *num_hdrs)
502{
503 ++*num_hdrs;
504
505 return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS);
506}
507
453a940e
WC		 508/**
509 * __skb_flow_dissect - extract the flow_keys struct and return it
510 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
06635a35
JP		 511 * @flow_dissector: list of keys to dissect
512 * @target_container: target structure to put dissected values into
453a940e
WC		 513 * @data: raw buffer pointer to the packet, if NULL use skb->data
514 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
515 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
516 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
517 *
06635a35
JP		 518 * The function will try to retrieve individual keys into target specified
519 * by flow_dissector from either the skbuff or a raw buffer specified by the
520 * rest parameters.
521 *
522 * Caller must take care of zeroing target container memory.
453a940e 523 */
06635a35
JP		 524bool __skb_flow_dissect(const struct sk_buff *skb,
525 struct flow_dissector *flow_dissector,
526 void *target_container,
cd79a238
TH		 527 void *data, __be16 proto, int nhoff, int hlen,
528 unsigned int flags)
0744dd00 529{
42aecaa9 530 struct flow_dissector_key_control *key_control;
06635a35
JP		 531 struct flow_dissector_key_basic *key_basic;
532 struct flow_dissector_key_addrs *key_addrs;
533 struct flow_dissector_key_ports *key_ports;
972d3876 534 struct flow_dissector_key_icmp *key_icmp;
d34af823 535 struct flow_dissector_key_tags *key_tags;
f6a66927 536 struct flow_dissector_key_vlan *key_vlan;
3a1214e8 537 enum flow_dissect_ret fdret;
d5709f7a 538 bool skip_vlan = false;
1eed4dfb 539 int num_hdrs = 0;
8e690ffd 540 u8 ip_proto = 0;
34fad54c 541 bool ret;
0744dd00 542
690e36e7
DM		 543 if (!data) {
544 data = skb->data;
d5709f7a
HHZ		 545 proto = skb_vlan_tag_present(skb) ?
546 skb->vlan_proto : skb->protocol;
453a940e 547 nhoff = skb_network_offset(skb);
690e36e7 548 hlen = skb_headlen(skb);
2d571645 549#if IS_ENABLED(CONFIG_NET_DSA)
7324157b 550 if (unlikely(skb->dev && netdev_uses_dsa(skb->dev))) {
43e66528
JC		 551 const struct dsa_device_ops *ops;
552 int offset;
553
554 ops = skb->dev->dsa_ptr->tag_ops;
555 if (ops->flow_dissect &&
556 !ops->flow_dissect(skb, &proto, &offset)) {
557 hlen -= offset;
558 nhoff += offset;
559 }
560 }
2d571645 561#endif
690e36e7
DM		 562 }
563
42aecaa9
TH		 564 /* It is ensured by skb_flow_dissector_init() that control key will
565 * be always present.
566 */
567 key_control = skb_flow_dissector_target(flow_dissector,
568 FLOW_DISSECTOR_KEY_CONTROL,
569 target_container);
570
06635a35
JP		 571 /* It is ensured by skb_flow_dissector_init() that basic key will
572 * be always present.
573 */
574 key_basic = skb_flow_dissector_target(flow_dissector,
575 FLOW_DISSECTOR_KEY_BASIC,
576 target_container);
0744dd00 577
a38402bc
SH		 578 __skb_flow_dissect_tunnel_info(skb, flow_dissector,
579 target_container);
580
20a17bf6
DM		 581 if (dissector_uses_key(flow_dissector,
582 FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
67a900cc
JP		 583 struct ethhdr *eth = eth_hdr(skb);
584 struct flow_dissector_key_eth_addrs *key_eth_addrs;
585
586 key_eth_addrs = skb_flow_dissector_target(flow_dissector,
587 FLOW_DISSECTOR_KEY_ETH_ADDRS,
588 target_container);
589 memcpy(key_eth_addrs, &eth->h_dest, sizeof(*key_eth_addrs));
590 }
591
c5ef188e 592proto_again:
3a1214e8
TH		 593 fdret = FLOW_DISSECT_RET_CONTINUE;
594
0744dd00 595 switch (proto) {
2b8837ae 596 case htons(ETH_P_IP): {
0744dd00
ED		 597 const struct iphdr *iph;
598 struct iphdr _iph;
3a1214e8 599
690e36e7 600 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
3a1214e8
TH		 601 if (!iph || iph->ihl < 5) {
602 fdret = FLOW_DISSECT_RET_OUT_BAD;
603 break;
604 }
605
3797d3e8 606 nhoff += iph->ihl * 4;
0744dd00 607
3797d3e8 608 ip_proto = iph->protocol;
3797d3e8 609
918c023f
AD		 610 if (dissector_uses_key(flow_dissector,
611 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
612 key_addrs = skb_flow_dissector_target(flow_dissector,
613 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
614 target_container);
615
616 memcpy(&key_addrs->v4addrs, &iph->saddr,
617 sizeof(key_addrs->v4addrs));
618 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
619 }
807e165d
TH		 620
621 if (ip_is_fragment(iph)) {
4b36993d 622 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
807e165d
TH		 623
624 if (iph->frag_off & htons(IP_OFFSET)) {
3a1214e8
TH		 625 fdret = FLOW_DISSECT_RET_OUT_GOOD;
626 break;
807e165d 627 } else {
4b36993d 628 key_control->flags |= FLOW_DIS_FIRST_FRAG;
3a1214e8
TH		 629 if (!(flags &
630 FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) {
631 fdret = FLOW_DISSECT_RET_OUT_GOOD;
632 break;
633 }
807e165d
TH		 634 }
635 }
636
518d8a2e
OG		 637 __skb_flow_dissect_ipv4(skb, flow_dissector,
638 target_container, data, iph);
639
3a1214e8
TH		 640 if (flags & FLOW_DISSECTOR_F_STOP_AT_L3) {
641 fdret = FLOW_DISSECT_RET_OUT_GOOD;
642 break;
643 }
8306b688 644
0744dd00
ED		 645 break;
646 }
2b8837ae 647 case htons(ETH_P_IPV6): {
0744dd00
ED		 648 const struct ipv6hdr *iph;
649 struct ipv6hdr _iph;
19469a87 650
690e36e7 651 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
3a1214e8
TH		 652 if (!iph) {
653 fdret = FLOW_DISSECT_RET_OUT_BAD;
654 break;
655 }
0744dd00
ED		 656
657 ip_proto = iph->nexthdr;
0744dd00 658 nhoff += sizeof(struct ipv6hdr);
19469a87 659
20a17bf6
DM		 660 if (dissector_uses_key(flow_dissector,
661 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
b3c3106c
AD		 662 key_addrs = skb_flow_dissector_target(flow_dissector,
663 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
664 target_container);
5af7fb6e 665
b3c3106c
AD		 666 memcpy(&key_addrs->v6addrs, &iph->saddr,
667 sizeof(key_addrs->v6addrs));
c3f83241 668 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
b924933c 669 }
87ee9e52 670
461547f3
AD		 671 if ((dissector_uses_key(flow_dissector,
672 FLOW_DISSECTOR_KEY_FLOW_LABEL) ||
673 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) &&
674 ip6_flowlabel(iph)) {
675 __be32 flow_label = ip6_flowlabel(iph);
676
20a17bf6
DM		 677 if (dissector_uses_key(flow_dissector,
678 FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
87ee9e52
TH		 679 key_tags = skb_flow_dissector_target(flow_dissector,
680 FLOW_DISSECTOR_KEY_FLOW_LABEL,
681 target_container);
682 key_tags->flow_label = ntohl(flow_label);
12c227ec 683 }
3a1214e8
TH		 684 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) {
685 fdret = FLOW_DISSECT_RET_OUT_GOOD;
686 break;
687 }
19469a87
TH		 688 }
689
518d8a2e
OG		 690 __skb_flow_dissect_ipv6(skb, flow_dissector,
691 target_container, data, iph);
692
8306b688 693 if (flags & FLOW_DISSECTOR_F_STOP_AT_L3)
3a1214e8 694 fdret = FLOW_DISSECT_RET_OUT_GOOD;
8306b688 695
0744dd00
ED		 696 break;
697 }
2b8837ae
JP		 698 case htons(ETH_P_8021AD):
699 case htons(ETH_P_8021Q): {
0744dd00 700 const struct vlan_hdr *vlan;
bc72f3dd
AB		 701 struct vlan_hdr _vlan;
702 bool vlan_tag_present = skb && skb_vlan_tag_present(skb);
0744dd00 703
bc72f3dd 704 if (vlan_tag_present)
d5709f7a
HHZ		 705 proto = skb->protocol;
706
bc72f3dd 707 if (!vlan_tag_present || eth_type_vlan(skb->protocol)) {
d5709f7a
HHZ		 708 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan),
709 data, hlen, &_vlan);
3a1214e8
TH		 710 if (!vlan) {
711 fdret = FLOW_DISSECT_RET_OUT_BAD;
712 break;
713 }
714
d5709f7a
HHZ		 715 proto = vlan->h_vlan_encapsulated_proto;
716 nhoff += sizeof(*vlan);
3a1214e8
TH		 717 if (skip_vlan) {
718 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
719 break;
720 }
d5709f7a 721 }
0744dd00 722
d5709f7a 723 skip_vlan = true;
20a17bf6 724 if (dissector_uses_key(flow_dissector,
f6a66927
HHZ		 725 FLOW_DISSECTOR_KEY_VLAN)) {
726 key_vlan = skb_flow_dissector_target(flow_dissector,
727 FLOW_DISSECTOR_KEY_VLAN,
d34af823
TH		 728 target_container);
729
bc72f3dd 730 if (vlan_tag_present) {
f6a66927
HHZ		 731 key_vlan->vlan_id = skb_vlan_tag_get_id(skb);
732 key_vlan->vlan_priority =
733 (skb_vlan_tag_get_prio(skb) >> VLAN_PRIO_SHIFT);
734 } else {
735 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) &
d5709f7a 736 VLAN_VID_MASK;
f6a66927
HHZ		 737 key_vlan->vlan_priority =
738 (ntohs(vlan->h_vlan_TCI) &
739 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
740 }
d34af823
TH		 741 }
742
3a1214e8
TH		 743 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
744 break;
0744dd00 745 }
2b8837ae 746 case htons(ETH_P_PPP_SES): {
0744dd00
ED		 747 struct {
748 struct pppoe_hdr hdr;
749 __be16 proto;
750 } *hdr, _hdr;
690e36e7 751 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
3a1214e8
TH		 752 if (!hdr) {
753 fdret = FLOW_DISSECT_RET_OUT_BAD;
754 break;
755 }
756
0744dd00
ED		 757 proto = hdr->proto;
758 nhoff += PPPOE_SES_HLEN;
759 switch (proto) {
2b8837ae 760 case htons(PPP_IP):
3a1214e8
TH		 761 proto = htons(ETH_P_IP);
762 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
763 break;
2b8837ae 764 case htons(PPP_IPV6):
3a1214e8
TH		 765 proto = htons(ETH_P_IPV6);
766 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
767 break;
0744dd00 768 default:
3a1214e8
TH		 769 fdret = FLOW_DISSECT_RET_OUT_BAD;
770 break;
0744dd00 771 }
3a1214e8 772 break;
0744dd00 773 }
08bfc9cb
EH		 774 case htons(ETH_P_TIPC): {
775 struct {
776 __be32 pre[3];
777 __be32 srcnode;
778 } *hdr, _hdr;
779 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
3a1214e8
TH		 780 if (!hdr) {
781 fdret = FLOW_DISSECT_RET_OUT_BAD;
782 break;
783 }
06635a35 784
20a17bf6
DM		 785 if (dissector_uses_key(flow_dissector,
786 FLOW_DISSECTOR_KEY_TIPC_ADDRS)) {
06635a35 787 key_addrs = skb_flow_dissector_target(flow_dissector,
9f249089 788 FLOW_DISSECTOR_KEY_TIPC_ADDRS,
06635a35 789 target_container);
9f249089
TH		 790 key_addrs->tipcaddrs.srcnode = hdr->srcnode;
791 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC_ADDRS;
06635a35 792 }
3a1214e8
TH		 793 fdret = FLOW_DISSECT_RET_OUT_GOOD;
794 break;
08bfc9cb 795 }
b3baa0fb
TH		 796
797 case htons(ETH_P_MPLS_UC):
4a5d6c8b 798 case htons(ETH_P_MPLS_MC):
3a1214e8 799 fdret = __skb_flow_dissect_mpls(skb, flow_dissector,
4a5d6c8b 800 target_container, data,
3a1214e8
TH		 801 nhoff, hlen);
802 break;
56193d1b 803 case htons(ETH_P_FCOE):
3a1214e8
TH		 804 if ((hlen - nhoff) < FCOE_HEADER_LEN) {
805 fdret = FLOW_DISSECT_RET_OUT_BAD;
806 break;
807 }
224516b3
AD		 808
809 nhoff += FCOE_HEADER_LEN;
3a1214e8
TH		 810 fdret = FLOW_DISSECT_RET_OUT_GOOD;
811 break;
55733350
SH		 812
813 case htons(ETH_P_ARP):
9bf881ff 814 case htons(ETH_P_RARP):
3a1214e8 815 fdret = __skb_flow_dissect_arp(skb, flow_dissector,
9bf881ff 816 target_container, data,
3a1214e8
TH		 817 nhoff, hlen);
818 break;
819
820 default:
821 fdret = FLOW_DISSECT_RET_OUT_BAD;
822 break;
823 }
824
825 /* Process result of proto processing */
826 switch (fdret) {
827 case FLOW_DISSECT_RET_OUT_GOOD:
828 goto out_good;
829 case FLOW_DISSECT_RET_PROTO_AGAIN:
1eed4dfb
TH		 830 if (skb_flow_dissect_allowed(&num_hdrs))
831 goto proto_again;
832 goto out_good;
3a1214e8
TH		 833 case FLOW_DISSECT_RET_CONTINUE:
834 case FLOW_DISSECT_RET_IPPROTO_AGAIN:
835 break;
836 case FLOW_DISSECT_RET_OUT_BAD:
0744dd00 837 default:
a6e544b0 838 goto out_bad;
0744dd00
ED		 839 }
840
6a74fcf4 841ip_proto_again:
3a1214e8
TH		 842 fdret = FLOW_DISSECT_RET_CONTINUE;
843
0744dd00 844 switch (ip_proto) {
7c92de8e 845 case IPPROTO_GRE:
3a1214e8 846 fdret = __skb_flow_dissect_gre(skb, key_control, flow_dissector,
7c92de8e 847 target_container, data,
3a1214e8
TH		 848 &proto, &nhoff, &hlen, flags);
849 break;
850
6a74fcf4
TH		 851 case NEXTHDR_HOP:
852 case NEXTHDR_ROUTING:
853 case NEXTHDR_DEST: {
854 u8 _opthdr[2], *opthdr;
855
856 if (proto != htons(ETH_P_IPV6))
857 break;
858
859 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr),
860 data, hlen, &_opthdr);
3a1214e8
TH		 861 if (!opthdr) {
862 fdret = FLOW_DISSECT_RET_OUT_BAD;
863 break;
864 }
6a74fcf4 865
1e98a0f0
ED		 866 ip_proto = opthdr[0];
867 nhoff += (opthdr[1] + 1) << 3;
6a74fcf4 868
3a1214e8
TH		 869 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
870 break;
6a74fcf4 871 }
b840f28b
TH		 872 case NEXTHDR_FRAGMENT: {
873 struct frag_hdr _fh, *fh;
874
875 if (proto != htons(ETH_P_IPV6))
876 break;
877
878 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh),
879 data, hlen, &_fh);
880
3a1214e8
TH		 881 if (!fh) {
882 fdret = FLOW_DISSECT_RET_OUT_BAD;
883 break;
884 }
b840f28b 885
4b36993d 886 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
b840f28b
TH		 887
888 nhoff += sizeof(_fh);
43d2ccb3 889 ip_proto = fh->nexthdr;
b840f28b
TH		 890
891 if (!(fh->frag_off & htons(IP6_OFFSET))) {
4b36993d 892 key_control->flags |= FLOW_DIS_FIRST_FRAG;
3a1214e8
TH		 893 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) {
894 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
895 break;
896 }
b840f28b 897 }
3a1214e8
TH		 898
899 fdret = FLOW_DISSECT_RET_OUT_GOOD;
900 break;
b840f28b 901 }
0744dd00 902 case IPPROTO_IPIP:
fca41895 903 proto = htons(ETH_P_IP);
823b9693 904
4b36993d 905 key_control->flags |= FLOW_DIS_ENCAPSULATION;
3a1214e8
TH		 906 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
907 fdret = FLOW_DISSECT_RET_OUT_GOOD;
908 break;
909 }
910
911 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
912 break;
823b9693 913
b438f940
TH		 914 case IPPROTO_IPV6:
915 proto = htons(ETH_P_IPV6);
823b9693 916
4b36993d 917 key_control->flags |= FLOW_DIS_ENCAPSULATION;
3a1214e8
TH		 918 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
919 fdret = FLOW_DISSECT_RET_OUT_GOOD;
920 break;
921 }
922
923 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
924 break;
925
823b9693 926
b3baa0fb
TH		 927 case IPPROTO_MPLS:
928 proto = htons(ETH_P_MPLS_UC);
3a1214e8
TH		 929 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
930 break;
931
ac4bb5de
JP		 932 case IPPROTO_TCP:
933 __skb_flow_dissect_tcp(skb, flow_dissector, target_container,
934 data, nhoff, hlen);
935 break;
3a1214e8 936
0744dd00
ED		 937 default:
938 break;
939 }
940
20a17bf6
DM		 941 if (dissector_uses_key(flow_dissector,
942 FLOW_DISSECTOR_KEY_PORTS)) {
06635a35
JP		 943 key_ports = skb_flow_dissector_target(flow_dissector,
944 FLOW_DISSECTOR_KEY_PORTS,
945 target_container);
946 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
947 data, hlen);
948 }
5af7fb6e 949
972d3876
SH		 950 if (dissector_uses_key(flow_dissector,
951 FLOW_DISSECTOR_KEY_ICMP)) {
952 key_icmp = skb_flow_dissector_target(flow_dissector,
953 FLOW_DISSECTOR_KEY_ICMP,
954 target_container);
955 key_icmp->icmp = skb_flow_get_be16(skb, nhoff, data, hlen);
956 }
957
3a1214e8
TH		 958 /* Process result of IP proto processing */
959 switch (fdret) {
960 case FLOW_DISSECT_RET_PROTO_AGAIN:
1eed4dfb
TH		 961 if (skb_flow_dissect_allowed(&num_hdrs))
962 goto proto_again;
963 break;
3a1214e8 964 case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1eed4dfb
TH		 965 if (skb_flow_dissect_allowed(&num_hdrs))
966 goto ip_proto_again;
967 break;
3a1214e8
TH		 968 case FLOW_DISSECT_RET_OUT_GOOD:
969 case FLOW_DISSECT_RET_CONTINUE:
970 break;
971 case FLOW_DISSECT_RET_OUT_BAD:
972 default:
973 goto out_bad;
974 }
975
a6e544b0
TH		 976out_good:
977 ret = true;
978
34fad54c
ED		 979 key_control->thoff = (u16)nhoff;
980out:
a6e544b0
TH		 981 key_basic->n_proto = proto;
982 key_basic->ip_proto = ip_proto;
a6e544b0
TH		 983
984 return ret;
34fad54c
ED		 985
986out_bad:
987 ret = false;
988 key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
989 goto out;
0744dd00 990}
690e36e7 991EXPORT_SYMBOL(__skb_flow_dissect);
441d9d32
CW		 992
993static u32 hashrnd __read_mostly;
66415cf8
HFS		 994static __always_inline void __flow_hash_secret_init(void)
995{
996 net_get_random_once(&hashrnd, sizeof(hashrnd));
997}
998
20a17bf6
DM		 999static __always_inline u32 __flow_hash_words(const u32 *words, u32 length,
1000 u32 keyval)
42aecaa9
TH		 1001{
1002 return jhash2(words, length, keyval);
1003}
1004
20a17bf6 1005static inline const u32 *flow_keys_hash_start(const struct flow_keys *flow)
66415cf8 1006{
20a17bf6
DM		 1007 const void *p = flow;
1008
42aecaa9 1009 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % sizeof(u32));
20a17bf6 1010 return (const u32 *)(p + FLOW_KEYS_HASH_OFFSET);
42aecaa9
TH		 1011}
1012
20a17bf6 1013static inline size_t flow_keys_hash_length(const struct flow_keys *flow)
42aecaa9 1014{
c3f83241 1015 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs);
42aecaa9 1016 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
c3f83241
TH		 1017 BUILD_BUG_ON(offsetof(typeof(*flow), addrs) !=
1018 sizeof(*flow) - sizeof(flow->addrs));
1019
1020 switch (flow->control.addr_type) {
1021 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1022 diff -= sizeof(flow->addrs.v4addrs);
1023 break;
1024 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1025 diff -= sizeof(flow->addrs.v6addrs);
1026 break;
9f249089
TH		 1027 case FLOW_DISSECTOR_KEY_TIPC_ADDRS:
1028 diff -= sizeof(flow->addrs.tipcaddrs);
1029 break;
c3f83241
TH		 1030 }
1031 return (sizeof(*flow) - diff) / sizeof(u32);
1032}
1033
1034__be32 flow_get_u32_src(const struct flow_keys *flow)
1035{
1036 switch (flow->control.addr_type) {
1037 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1038 return flow->addrs.v4addrs.src;
1039 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1040 return (__force __be32)ipv6_addr_hash(
1041 &flow->addrs.v6addrs.src);
9f249089
TH		 1042 case FLOW_DISSECTOR_KEY_TIPC_ADDRS:
1043 return flow->addrs.tipcaddrs.srcnode;
c3f83241
TH		 1044 default:
1045 return 0;
1046 }
1047}
1048EXPORT_SYMBOL(flow_get_u32_src);
1049
1050__be32 flow_get_u32_dst(const struct flow_keys *flow)
1051{
1052 switch (flow->control.addr_type) {
1053 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1054 return flow->addrs.v4addrs.dst;
1055 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1056 return (__force __be32)ipv6_addr_hash(
1057 &flow->addrs.v6addrs.dst);
1058 default:
1059 return 0;
1060 }
1061}
1062EXPORT_SYMBOL(flow_get_u32_dst);
1063
1064static inline void __flow_hash_consistentify(struct flow_keys *keys)
1065{
1066 int addr_diff, i;
1067
1068 switch (keys->control.addr_type) {
1069 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1070 addr_diff = (__force u32)keys->addrs.v4addrs.dst -
1071 (__force u32)keys->addrs.v4addrs.src;
1072 if ((addr_diff < 0) ||
1073 (addr_diff == 0 &&
1074 ((__force u16)keys->ports.dst <
1075 (__force u16)keys->ports.src))) {
1076 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst);
1077 swap(keys->ports.src, keys->ports.dst);
1078 }
1079 break;
1080 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1081 addr_diff = memcmp(&keys->addrs.v6addrs.dst,
1082 &keys->addrs.v6addrs.src,
1083 sizeof(keys->addrs.v6addrs.dst));
1084 if ((addr_diff < 0) ||
1085 (addr_diff == 0 &&
1086 ((__force u16)keys->ports.dst <
1087 (__force u16)keys->ports.src))) {
1088 for (i = 0; i < 4; i++)
1089 swap(keys->addrs.v6addrs.src.s6_addr32[i],
1090 keys->addrs.v6addrs.dst.s6_addr32[i]);
1091 swap(keys->ports.src, keys->ports.dst);
1092 }
1093 break;
1094 }
66415cf8
HFS		 1095}
1096
50fb7992 1097static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
5ed20a68
TH		 1098{
1099 u32 hash;
1100
c3f83241 1101 __flow_hash_consistentify(keys);
5ed20a68 1102
20a17bf6 1103 hash = __flow_hash_words(flow_keys_hash_start(keys),
42aecaa9 1104 flow_keys_hash_length(keys), keyval);
5ed20a68
TH		 1105 if (!hash)
1106 hash = 1;
1107
1108 return hash;
1109}
1110
1111u32 flow_hash_from_keys(struct flow_keys *keys)
1112{
50fb7992
TH		 1113 __flow_hash_secret_init();
1114 return __flow_hash_from_keys(keys, hashrnd);
5ed20a68
TH		 1115}
1116EXPORT_SYMBOL(flow_hash_from_keys);
1117
50fb7992
TH		 1118static inline u32 ___skb_get_hash(const struct sk_buff *skb,
1119 struct flow_keys *keys, u32 keyval)
1120{
6db61d79
TH		 1121 skb_flow_dissect_flow_keys(skb, keys,
1122 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
50fb7992
TH		 1123
1124 return __flow_hash_from_keys(keys, keyval);
1125}
1126
2f59e1eb
TH		 1127struct _flow_keys_digest_data {
1128 __be16 n_proto;
1129 u8 ip_proto;
1130 u8 padding;
1131 __be32 ports;
1132 __be32 src;
1133 __be32 dst;
1134};
1135
1136void make_flow_keys_digest(struct flow_keys_digest *digest,
1137 const struct flow_keys *flow)
1138{
1139 struct _flow_keys_digest_data *data =
1140 (struct _flow_keys_digest_data *)digest;
1141
1142 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
1143
1144 memset(digest, 0, sizeof(*digest));
1145
06635a35
JP		 1146 data->n_proto = flow->basic.n_proto;
1147 data->ip_proto = flow->basic.ip_proto;
1148 data->ports = flow->ports.ports;
c3f83241
TH		 1149 data->src = flow->addrs.v4addrs.src;
1150 data->dst = flow->addrs.v4addrs.dst;
2f59e1eb
TH		 1151}
1152EXPORT_SYMBOL(make_flow_keys_digest);
1153
eb70db87
DM		 1154static struct flow_dissector flow_keys_dissector_symmetric __read_mostly;
1155
b917783c 1156u32 __skb_get_hash_symmetric(const struct sk_buff *skb)
eb70db87
DM		 1157{
1158 struct flow_keys keys;
1159
1160 __flow_hash_secret_init();
1161
1162 memset(&keys, 0, sizeof(keys));
1163 __skb_flow_dissect(skb, &flow_keys_dissector_symmetric, &keys,
1164 NULL, 0, 0, 0,
1165 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1166
1167 return __flow_hash_from_keys(&keys, hashrnd);
1168}
1169EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric);
1170
d4fd3275
JP		 1171/**
1172 * __skb_get_hash: calculate a flow hash
1173 * @skb: sk_buff to calculate flow hash from
1174 *
1175 * This function calculates a flow hash based on src/dst addresses
61b905da
TH		 1176 * and src/dst port numbers. Sets hash in skb to non-zero hash value
1177 * on success, zero indicates no valid hash. Also, sets l4_hash in skb
441d9d32
CW		 1178 * if hash is a canonical 4-tuple hash over transport ports.
1179 */
3958afa1 1180void __skb_get_hash(struct sk_buff *skb)
441d9d32
CW		 1181{
1182 struct flow_keys keys;
635c223c 1183 u32 hash;
441d9d32 1184
50fb7992
TH		 1185 __flow_hash_secret_init();
1186
635c223c
GF		 1187 hash = ___skb_get_hash(skb, &keys, hashrnd);
1188
1189 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
441d9d32 1190}
3958afa1 1191EXPORT_SYMBOL(__skb_get_hash);
441d9d32 1192
50fb7992
TH		 1193__u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
1194{
1195 struct flow_keys keys;
1196
1197 return ___skb_get_hash(skb, &keys, perturb);
1198}
1199EXPORT_SYMBOL(skb_get_hash_perturb);
1200
56193d1b
AD		 1201u32 __skb_get_poff(const struct sk_buff *skb, void *data,
1202 const struct flow_keys *keys, int hlen)
f77668dc 1203{
42aecaa9 1204 u32 poff = keys->control.thoff;
f77668dc 1205
43d2ccb3
AD		 1206 /* skip L4 headers for fragments after the first */
1207 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) &&
1208 !(keys->control.flags & FLOW_DIS_FIRST_FRAG))
1209 return poff;
1210
06635a35 1211 switch (keys->basic.ip_proto) {
f77668dc 1212 case IPPROTO_TCP: {
5af7fb6e
AD		 1213 /* access doff as u8 to avoid unaligned access */
1214 const u8 *doff;
1215 u8 _doff;
f77668dc 1216
5af7fb6e
AD		 1217 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
1218 data, hlen, &_doff);
1219 if (!doff)
f77668dc
DB		 1220 return poff;
1221
5af7fb6e 1222 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
f77668dc
DB		 1223 break;
1224 }
1225 case IPPROTO_UDP:
1226 case IPPROTO_UDPLITE:
1227 poff += sizeof(struct udphdr);
1228 break;
1229 /* For the rest, we do not really care about header
1230 * extensions at this point for now.
1231 */
1232 case IPPROTO_ICMP:
1233 poff += sizeof(struct icmphdr);
1234 break;
1235 case IPPROTO_ICMPV6:
1236 poff += sizeof(struct icmp6hdr);
1237 break;
1238 case IPPROTO_IGMP:
1239 poff += sizeof(struct igmphdr);
1240 break;
1241 case IPPROTO_DCCP:
1242 poff += sizeof(struct dccp_hdr);
1243 break;
1244 case IPPROTO_SCTP:
1245 poff += sizeof(struct sctphdr);
1246 break;
1247 }
1248
1249 return poff;
1250}
1251
0db89b8b
JP		 1252/**
1253 * skb_get_poff - get the offset to the payload
1254 * @skb: sk_buff to get the payload offset from
1255 *
1256 * The function will get the offset to the payload as far as it could
1257 * be dissected. The main user is currently BPF, so that we can dynamically
56193d1b
AD		 1258 * truncate packets without needing to push actual payload to the user
1259 * space and can analyze headers only, instead.
1260 */
1261u32 skb_get_poff(const struct sk_buff *skb)
1262{
1263 struct flow_keys keys;
1264
cd79a238 1265 if (!skb_flow_dissect_flow_keys(skb, &keys, 0))
56193d1b
AD		 1266 return 0;
1267
1268 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
1269}
06635a35 1270
20a17bf6 1271__u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys)
a17ace95
DM		 1272{
1273 memset(keys, 0, sizeof(*keys));
1274
1275 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr,
1276 sizeof(keys->addrs.v6addrs.src));
1277 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr,
1278 sizeof(keys->addrs.v6addrs.dst));
1279 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1280 keys->ports.src = fl6->fl6_sport;
1281 keys->ports.dst = fl6->fl6_dport;
1282 keys->keyid.keyid = fl6->fl6_gre_key;
1283 keys->tags.flow_label = (__force u32)fl6->flowlabel;
1284 keys->basic.ip_proto = fl6->flowi6_proto;
1285
1286 return flow_hash_from_keys(keys);
1287}
1288EXPORT_SYMBOL(__get_hash_from_flowi6);
1289
20a17bf6 1290__u32 __get_hash_from_flowi4(const struct flowi4 *fl4, struct flow_keys *keys)
a17ace95
DM		 1291{
1292 memset(keys, 0, sizeof(*keys));
1293
1294 keys->addrs.v4addrs.src = fl4->saddr;
1295 keys->addrs.v4addrs.dst = fl4->daddr;
1296 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1297 keys->ports.src = fl4->fl4_sport;
1298 keys->ports.dst = fl4->fl4_dport;
1299 keys->keyid.keyid = fl4->fl4_gre_key;
1300 keys->basic.ip_proto = fl4->flowi4_proto;
1301
1302 return flow_hash_from_keys(keys);
1303}
1304EXPORT_SYMBOL(__get_hash_from_flowi4);
1305
06635a35 1306static const struct flow_dissector_key flow_keys_dissector_keys[] = {
42aecaa9
TH		 1307 {
1308 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1309 .offset = offsetof(struct flow_keys, control),
1310 },
06635a35
JP		 1311 {
1312 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1313 .offset = offsetof(struct flow_keys, basic),
1314 },
1315 {
1316 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
c3f83241
TH		 1317 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1318 },
1319 {
1320 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1321 .offset = offsetof(struct flow_keys, addrs.v6addrs),
06635a35 1322 },
9f249089
TH		 1323 {
1324 .key_id = FLOW_DISSECTOR_KEY_TIPC_ADDRS,
1325 .offset = offsetof(struct flow_keys, addrs.tipcaddrs),
1326 },
06635a35
JP		 1327 {
1328 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1329 .offset = offsetof(struct flow_keys, ports),
1330 },
d34af823 1331 {
f6a66927
HHZ		 1332 .key_id = FLOW_DISSECTOR_KEY_VLAN,
1333 .offset = offsetof(struct flow_keys, vlan),
d34af823 1334 },
87ee9e52
TH		 1335 {
1336 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL,
1337 .offset = offsetof(struct flow_keys, tags),
1338 },
1fdd512c
TH		 1339 {
1340 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
1341 .offset = offsetof(struct flow_keys, keyid),
1342 },
06635a35
JP		 1343};
1344
eb70db87
DM		 1345static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = {
1346 {
1347 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1348 .offset = offsetof(struct flow_keys, control),
1349 },
1350 {
1351 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1352 .offset = offsetof(struct flow_keys, basic),
1353 },
1354 {
1355 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1356 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1357 },
1358 {
1359 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1360 .offset = offsetof(struct flow_keys, addrs.v6addrs),
1361 },
1362 {
1363 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1364 .offset = offsetof(struct flow_keys, ports),
1365 },
1366};
1367
06635a35 1368static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
42aecaa9
TH		 1369 {
1370 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1371 .offset = offsetof(struct flow_keys, control),
1372 },
06635a35
JP		 1373 {
1374 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1375 .offset = offsetof(struct flow_keys, basic),
1376 },
1377};
1378
1379struct flow_dissector flow_keys_dissector __read_mostly;
1380EXPORT_SYMBOL(flow_keys_dissector);
1381
1382struct flow_dissector flow_keys_buf_dissector __read_mostly;
1383
1384static int __init init_default_flow_dissectors(void)
1385{
1386 skb_flow_dissector_init(&flow_keys_dissector,
1387 flow_keys_dissector_keys,
1388 ARRAY_SIZE(flow_keys_dissector_keys));
eb70db87
DM		 1389 skb_flow_dissector_init(&flow_keys_dissector_symmetric,
1390 flow_keys_dissector_symmetric_keys,
1391 ARRAY_SIZE(flow_keys_dissector_symmetric_keys));
06635a35
JP		 1392 skb_flow_dissector_init(&flow_keys_buf_dissector,
1393 flow_keys_buf_dissector_keys,
1394 ARRAY_SIZE(flow_keys_buf_dissector_keys));
1395 return 0;
1396}
1397
c9b8af13 1398core_initcall(init_default_flow_dissectors);
Linux 6.x block layer and io_uring tree(s)