projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'iwlwifi-next-for-kalle-2021-04-12-v2' of git://git.kernel.org/pub/scm...
[linux-block.git] / net / core / flow_dissector.c
CommitLineData
457c8996 1// SPDX-License-Identifier: GPL-2.0-only
fbff949e 2#include <linux/kernel.h>
0744dd00 3#include <linux/skbuff.h>
c452ed70 4#include <linux/export.h>
0744dd00
ED		 5#include <linux/ip.h>
6#include <linux/ipv6.h>
7#include <linux/if_vlan.h>
43e66528 8#include <net/dsa.h>
a38402bc 9#include <net/dst_metadata.h>
0744dd00 10#include <net/ip.h>
ddbe5032 11#include <net/ipv6.h>
ab10dccb
GF		 12#include <net/gre.h>
13#include <net/pptp.h>
8d6e79d3 14#include <net/tipc.h>
f77668dc
DB		 15#include <linux/igmp.h>
16#include <linux/icmp.h>
17#include <linux/sctp.h>
18#include <linux/dccp.h>
0744dd00
ED		 19#include <linux/if_tunnel.h>
20#include <linux/if_pppox.h>
21#include <linux/ppp_defs.h>
06635a35 22#include <linux/stddef.h>
67a900cc 23#include <linux/if_ether.h>
b3baa0fb 24#include <linux/mpls.h>
ac4bb5de 25#include <linux/tcp.h>
4f1cc51f 26#include <linux/ptp_classify.h>
1bd758eb 27#include <net/flow_dissector.h>
56193d1b 28#include <scsi/fc/fc_fcoe.h>
5b0890a9 29#include <uapi/linux/batadv_packet.h>
d58e468b 30#include <linux/bpf.h>
75a56758
PB		 31#if IS_ENABLED(CONFIG_NF_CONNTRACK)
32#include <net/netfilter/nf_conntrack_core.h>
33#include <net/netfilter/nf_conntrack_labels.h>
34#endif
a3fd7cee 35#include <linux/bpf-netns.h>
d58e468b 36
20a17bf6
DM		 37static void dissector_set_key(struct flow_dissector *flow_dissector,
38 enum flow_dissector_key_id key_id)
fbff949e
JP		 39{
40 flow_dissector->used_keys |= (1 << key_id);
41}
42
fbff949e
JP		 43void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
44 const struct flow_dissector_key *key,
45 unsigned int key_count)
46{
47 unsigned int i;
48
49 memset(flow_dissector, 0, sizeof(*flow_dissector));
50
51 for (i = 0; i < key_count; i++, key++) {
75a5fb0c 52 /* User should make sure that every key target offset is within
fbff949e
JP		 53 * boundaries of unsigned short.
54 */
55 BUG_ON(key->offset > USHRT_MAX);
20a17bf6
DM		 56 BUG_ON(dissector_uses_key(flow_dissector,
57 key->key_id));
fbff949e 58
20a17bf6 59 dissector_set_key(flow_dissector, key->key_id);
fbff949e
JP		 60 flow_dissector->offset[key->key_id] = key->offset;
61 }
62
42aecaa9
TH		 63 /* Ensure that the dissector always includes control and basic key.
64 * That way we are able to avoid handling lack of these in fast path.
fbff949e 65 */
20a17bf6
DM		 66 BUG_ON(!dissector_uses_key(flow_dissector,
67 FLOW_DISSECTOR_KEY_CONTROL));
68 BUG_ON(!dissector_uses_key(flow_dissector,
69 FLOW_DISSECTOR_KEY_BASIC));
fbff949e
JP		 70}
71EXPORT_SYMBOL(skb_flow_dissector_init);
72
b27f7bb5 73#ifdef CONFIG_BPF_SYSCALL
3b701699
JS		 74int flow_dissector_bpf_prog_attach_check(struct net *net,
75 struct bpf_prog *prog)
d58e468b 76{
a3fd7cee 77 enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
a11c397c
SF		 78
79 if (net == &init_net) {
80 /* BPF flow dissector in the root namespace overrides
81 * any per-net-namespace one. When attaching to root,
82 * make sure we don't have any BPF program attached
83 * to the non-root namespaces.
84 */
85 struct net *ns;
86
87 for_each_net(ns) {
719b78a5
JS		 88 if (ns == &init_net)
89 continue;
695c1214 90 if (rcu_access_pointer(ns->bpf.run_array[type]))
171526f6 91 return -EEXIST;
a11c397c
SF		 92 }
93 } else {
94 /* Make sure root flow dissector is not attached
95 * when attaching to the non-root namespace.
96 */
695c1214 97 if (rcu_access_pointer(init_net.bpf.run_array[type]))
171526f6 98 return -EEXIST;
a11c397c
SF		 99 }
100
171526f6
JS		 101 return 0;
102}
b27f7bb5 103#endif /* CONFIG_BPF_SYSCALL */
5cf65922 104
357afe9c 105/**
6451b3f5
WC		 106 * __skb_flow_get_ports - extract the upper layer ports and return them
107 * @skb: sk_buff to extract the ports from
357afe9c
NA		 108 * @thoff: transport header offset
109 * @ip_proto: protocol for which to get port offset
6451b3f5
WC		 110 * @data: raw buffer pointer to the packet, if NULL use skb->data
111 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
357afe9c
NA		 112 *
113 * The function will try to retrieve the ports at offset thoff + poff where poff
114 * is the protocol port offset returned from proto_ports_offset
115 */
690e36e7 116__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
f96533cd 117 const void *data, int hlen)
357afe9c
NA		 118{
119 int poff = proto_ports_offset(ip_proto);
120
690e36e7
DM		 121 if (!data) {
122 data = skb->data;
123 hlen = skb_headlen(skb);
124 }
125
357afe9c
NA		 126 if (poff >= 0) {
127 __be32 *ports, _ports;
128
690e36e7
DM		 129 ports = __skb_header_pointer(skb, thoff + poff,
130 sizeof(_ports), data, hlen, &_ports);
357afe9c
NA		 131 if (ports)
132 return *ports;
133 }
134
135 return 0;
136}
690e36e7 137EXPORT_SYMBOL(__skb_flow_get_ports);
357afe9c 138
5dec597e
MC		 139static bool icmp_has_id(u8 type)
140{
141 switch (type) {
142 case ICMP_ECHO:
143 case ICMP_ECHOREPLY:
144 case ICMP_TIMESTAMP:
145 case ICMP_TIMESTAMPREPLY:
146 case ICMPV6_ECHO_REQUEST:
147 case ICMPV6_ECHO_REPLY:
148 return true;
149 }
150
151 return false;
152}
153
154/**
155 * skb_flow_get_icmp_tci - extract ICMP(6) Type, Code and Identifier fields
156 * @skb: sk_buff to extract from
157 * @key_icmp: struct flow_dissector_key_icmp to fill
158 * @data: raw buffer pointer to the packet
6b3acfc3 159 * @thoff: offset to extract at
5dec597e
MC		 160 * @hlen: packet header length
161 */
162void skb_flow_get_icmp_tci(const struct sk_buff *skb,
163 struct flow_dissector_key_icmp *key_icmp,
f96533cd 164 const void *data, int thoff, int hlen)
5dec597e
MC		 165{
166 struct icmphdr *ih, _ih;
167
168 ih = __skb_header_pointer(skb, thoff, sizeof(_ih), data, hlen, &_ih);
169 if (!ih)
170 return;
171
172 key_icmp->type = ih->type;
173 key_icmp->code = ih->code;
174
175 /* As we use 0 to signal that the Id field is not present,
176 * avoid confusion with packets without such field
177 */
178 if (icmp_has_id(ih->type))
a25f8222 179 key_icmp->id = ih->un.echo.id ? ntohs(ih->un.echo.id) : 1;
5dec597e
MC		 180 else
181 key_icmp->id = 0;
182}
183EXPORT_SYMBOL(skb_flow_get_icmp_tci);
184
185/* If FLOW_DISSECTOR_KEY_ICMP is set, dissect an ICMP packet
186 * using skb_flow_get_icmp_tci().
3b336d6f
MC		 187 */
188static void __skb_flow_dissect_icmp(const struct sk_buff *skb,
189 struct flow_dissector *flow_dissector,
f96533cd
AL		 190 void *target_container, const void *data,
191 int thoff, int hlen)
3b336d6f
MC		 192{
193 struct flow_dissector_key_icmp *key_icmp;
194
195 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ICMP))
196 return;
197
198 key_icmp = skb_flow_dissector_target(flow_dissector,
199 FLOW_DISSECTOR_KEY_ICMP,
200 target_container);
5dec597e
MC		 201
202 skb_flow_get_icmp_tci(skb, key_icmp, data, thoff, hlen);
3b336d6f
MC		 203}
204
82828b88
JP		 205void skb_flow_dissect_meta(const struct sk_buff *skb,
206 struct flow_dissector *flow_dissector,
207 void *target_container)
208{
209 struct flow_dissector_key_meta *meta;
210
211 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_META))
212 return;
213
214 meta = skb_flow_dissector_target(flow_dissector,
215 FLOW_DISSECTOR_KEY_META,
216 target_container);
217 meta->ingress_ifindex = skb->skb_iif;
218}
219EXPORT_SYMBOL(skb_flow_dissect_meta);
220
a38402bc
SH		 221static void
222skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type,
223 struct flow_dissector *flow_dissector,
224 void *target_container)
225{
226 struct flow_dissector_key_control *ctrl;
227
228 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL))
229 return;
230
231 ctrl = skb_flow_dissector_target(flow_dissector,
232 FLOW_DISSECTOR_KEY_ENC_CONTROL,
233 target_container);
234 ctrl->addr_type = type;
235}
236
75a56758
PB		 237void
238skb_flow_dissect_ct(const struct sk_buff *skb,
239 struct flow_dissector *flow_dissector,
7baf2429 240 void *target_container, u16 *ctinfo_map,
241 size_t mapsize, bool post_ct)
75a56758
PB		 242{
243#if IS_ENABLED(CONFIG_NF_CONNTRACK)
244 struct flow_dissector_key_ct *key;
245 enum ip_conntrack_info ctinfo;
246 struct nf_conn_labels *cl;
247 struct nf_conn *ct;
248
249 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_CT))
250 return;
251
252 ct = nf_ct_get(skb, &ctinfo);
7baf2429 253 if (!ct && !post_ct)
75a56758
PB		 254 return;
255
256 key = skb_flow_dissector_target(flow_dissector,
257 FLOW_DISSECTOR_KEY_CT,
258 target_container);
259
7baf2429 260 if (!ct) {
261 key->ct_state = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
262 TCA_FLOWER_KEY_CT_FLAGS_INVALID;
263 return;
264 }
265
75a56758
PB		 266 if (ctinfo < mapsize)
267 key->ct_state = ctinfo_map[ctinfo];
268#if IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)
269 key->ct_zone = ct->zone.id;
270#endif
271#if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
272 key->ct_mark = ct->mark;
273#endif
274
275 cl = nf_ct_labels_find(ct);
276 if (cl)
277 memcpy(key->ct_labels, cl->bits, sizeof(key->ct_labels));
278#endif /* CONFIG_NF_CONNTRACK */
279}
280EXPORT_SYMBOL(skb_flow_dissect_ct);
281
62b32379
SH		 282void
283skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
284 struct flow_dissector *flow_dissector,
285 void *target_container)
a38402bc
SH		 286{
287 struct ip_tunnel_info *info;
288 struct ip_tunnel_key *key;
289
290 /* A quick check to see if there might be something to do. */
291 if (!dissector_uses_key(flow_dissector,
292 FLOW_DISSECTOR_KEY_ENC_KEYID) &&
293 !dissector_uses_key(flow_dissector,
294 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) &&
295 !dissector_uses_key(flow_dissector,
296 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) &&
297 !dissector_uses_key(flow_dissector,
298 FLOW_DISSECTOR_KEY_ENC_CONTROL) &&
299 !dissector_uses_key(flow_dissector,
5544adb9
OG		 300 FLOW_DISSECTOR_KEY_ENC_PORTS) &&
301 !dissector_uses_key(flow_dissector,
92e2c405
SH		 302 FLOW_DISSECTOR_KEY_ENC_IP) &&
303 !dissector_uses_key(flow_dissector,
304 FLOW_DISSECTOR_KEY_ENC_OPTS))
a38402bc
SH		 305 return;
306
307 info = skb_tunnel_info(skb);
308 if (!info)
309 return;
310
311 key = &info->key;
312
313 switch (ip_tunnel_info_af(info)) {
314 case AF_INET:
315 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV4_ADDRS,
316 flow_dissector,
317 target_container);
318 if (dissector_uses_key(flow_dissector,
319 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
320 struct flow_dissector_key_ipv4_addrs *ipv4;
321
322 ipv4 = skb_flow_dissector_target(flow_dissector,
323 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
324 target_container);
325 ipv4->src = key->u.ipv4.src;
326 ipv4->dst = key->u.ipv4.dst;
327 }
328 break;
329 case AF_INET6:
330 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV6_ADDRS,
331 flow_dissector,
332 target_container);
333 if (dissector_uses_key(flow_dissector,
334 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
335 struct flow_dissector_key_ipv6_addrs *ipv6;
336
337 ipv6 = skb_flow_dissector_target(flow_dissector,
338 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
339 target_container);
340 ipv6->src = key->u.ipv6.src;
341 ipv6->dst = key->u.ipv6.dst;
342 }
343 break;
344 }
345
346 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
347 struct flow_dissector_key_keyid *keyid;
348
349 keyid = skb_flow_dissector_target(flow_dissector,
350 FLOW_DISSECTOR_KEY_ENC_KEYID,
351 target_container);
352 keyid->keyid = tunnel_id_to_key32(key->tun_id);
353 }
354
355 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
356 struct flow_dissector_key_ports *tp;
357
358 tp = skb_flow_dissector_target(flow_dissector,
359 FLOW_DISSECTOR_KEY_ENC_PORTS,
360 target_container);
361 tp->src = key->tp_src;
362 tp->dst = key->tp_dst;
363 }
5544adb9
OG		 364
365 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_IP)) {
366 struct flow_dissector_key_ip *ip;
367
368 ip = skb_flow_dissector_target(flow_dissector,
369 FLOW_DISSECTOR_KEY_ENC_IP,
370 target_container);
371 ip->tos = key->tos;
372 ip->ttl = key->ttl;
373 }
92e2c405
SH		 374
375 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_OPTS)) {
376 struct flow_dissector_key_enc_opts *enc_opt;
377
378 enc_opt = skb_flow_dissector_target(flow_dissector,
379 FLOW_DISSECTOR_KEY_ENC_OPTS,
380 target_container);
381
382 if (info->options_len) {
383 enc_opt->len = info->options_len;
384 ip_tunnel_info_opts_get(enc_opt->data, info);
385 enc_opt->dst_opt_type = info->key.tun_flags &
386 TUNNEL_OPTIONS_PRESENT;
387 }
388 }
a38402bc 389}
62b32379 390EXPORT_SYMBOL(skb_flow_dissect_tunnel_info);
a38402bc 391
0cb09aff
AL		 392void skb_flow_dissect_hash(const struct sk_buff *skb,
393 struct flow_dissector *flow_dissector,
394 void *target_container)
395{
396 struct flow_dissector_key_hash *key;
397
398 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_HASH))
399 return;
400
401 key = skb_flow_dissector_target(flow_dissector,
402 FLOW_DISSECTOR_KEY_HASH,
403 target_container);
404
405 key->hash = skb_get_hash_raw(skb);
406}
407EXPORT_SYMBOL(skb_flow_dissect_hash);
408
4a5d6c8b
JP		 409static enum flow_dissect_ret
410__skb_flow_dissect_mpls(const struct sk_buff *skb,
411 struct flow_dissector *flow_dissector,
f96533cd
AL		 412 void *target_container, const void *data, int nhoff,
413 int hlen, int lse_index, bool *entropy_label)
4a5d6c8b 414{
58cff782
GN		 415 struct mpls_label *hdr, _hdr;
416 u32 entry, label, bos;
4a5d6c8b
JP		 417
418 if (!dissector_uses_key(flow_dissector,
029c1ecb
BL		 419 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) &&
420 !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS))
4a5d6c8b
JP		 421 return FLOW_DISSECT_RET_OUT_GOOD;
422
58cff782
GN		 423 if (lse_index >= FLOW_DIS_MPLS_MAX)
424 return FLOW_DISSECT_RET_OUT_GOOD;
425
4a5d6c8b
JP		 426 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
427 hlen, &_hdr);
428 if (!hdr)
429 return FLOW_DISSECT_RET_OUT_BAD;
430
58cff782 431 entry = ntohl(hdr->entry);
029c1ecb 432 label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT;
58cff782 433 bos = (entry & MPLS_LS_S_MASK) >> MPLS_LS_S_SHIFT;
029c1ecb
BL		 434
435 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) {
436 struct flow_dissector_key_mpls *key_mpls;
58cff782 437 struct flow_dissector_mpls_lse *lse;
029c1ecb
BL		 438
439 key_mpls = skb_flow_dissector_target(flow_dissector,
440 FLOW_DISSECTOR_KEY_MPLS,
441 target_container);
58cff782
GN		 442 lse = &key_mpls->ls[lse_index];
443
444 lse->mpls_ttl = (entry & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
445 lse->mpls_bos = bos;
446 lse->mpls_tc = (entry & MPLS_LS_TC_MASK) >> MPLS_LS_TC_SHIFT;
447 lse->mpls_label = label;
448 dissector_set_mpls_lse(key_mpls, lse_index);
029c1ecb
BL		 449 }
450
58cff782
GN		 451 if (*entropy_label &&
452 dissector_uses_key(flow_dissector,
453 FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) {
454 struct flow_dissector_key_keyid *key_keyid;
455
4a5d6c8b
JP		 456 key_keyid = skb_flow_dissector_target(flow_dissector,
457 FLOW_DISSECTOR_KEY_MPLS_ENTROPY,
458 target_container);
58cff782 459 key_keyid->keyid = cpu_to_be32(label);
4a5d6c8b 460 }
58cff782
GN		 461
462 *entropy_label = label == MPLS_LABEL_ENTROPY;
463
464 return bos ? FLOW_DISSECT_RET_OUT_GOOD : FLOW_DISSECT_RET_PROTO_AGAIN;
4a5d6c8b
JP		 465}
466
9bf881ff
JP		 467static enum flow_dissect_ret
468__skb_flow_dissect_arp(const struct sk_buff *skb,
469 struct flow_dissector *flow_dissector,
f96533cd
AL		 470 void *target_container, const void *data,
471 int nhoff, int hlen)
9bf881ff
JP		 472{
473 struct flow_dissector_key_arp *key_arp;
474 struct {
475 unsigned char ar_sha[ETH_ALEN];
476 unsigned char ar_sip[4];
477 unsigned char ar_tha[ETH_ALEN];
478 unsigned char ar_tip[4];
479 } *arp_eth, _arp_eth;
480 const struct arphdr *arp;
6f14f443 481 struct arphdr _arp;
9bf881ff
JP		 482
483 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP))
484 return FLOW_DISSECT_RET_OUT_GOOD;
485
486 arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data,
487 hlen, &_arp);
488 if (!arp)
489 return FLOW_DISSECT_RET_OUT_BAD;
490
491 if (arp->ar_hrd != htons(ARPHRD_ETHER) ||
492 arp->ar_pro != htons(ETH_P_IP) ||
493 arp->ar_hln != ETH_ALEN ||
494 arp->ar_pln != 4 ||
495 (arp->ar_op != htons(ARPOP_REPLY) &&
496 arp->ar_op != htons(ARPOP_REQUEST)))
497 return FLOW_DISSECT_RET_OUT_BAD;
498
499 arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp),
500 sizeof(_arp_eth), data,
501 hlen, &_arp_eth);
502 if (!arp_eth)
503 return FLOW_DISSECT_RET_OUT_BAD;
504
505 key_arp = skb_flow_dissector_target(flow_dissector,
506 FLOW_DISSECTOR_KEY_ARP,
507 target_container);
508
509 memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip));
510 memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip));
511
512 /* Only store the lower byte of the opcode;
513 * this covers ARPOP_REPLY and ARPOP_REQUEST.
514 */
515 key_arp->op = ntohs(arp->ar_op) & 0xff;
516
517 ether_addr_copy(key_arp->sha, arp_eth->ar_sha);
518 ether_addr_copy(key_arp->tha, arp_eth->ar_tha);
519
520 return FLOW_DISSECT_RET_OUT_GOOD;
521}
522
7c92de8e
JP		 523static enum flow_dissect_ret
524__skb_flow_dissect_gre(const struct sk_buff *skb,
525 struct flow_dissector_key_control *key_control,
526 struct flow_dissector *flow_dissector,
f96533cd 527 void *target_container, const void *data,
7c92de8e
JP		 528 __be16 *p_proto, int *p_nhoff, int *p_hlen,
529 unsigned int flags)
530{
531 struct flow_dissector_key_keyid *key_keyid;
532 struct gre_base_hdr *hdr, _hdr;
533 int offset = 0;
534 u16 gre_ver;
535
536 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr),
537 data, *p_hlen, &_hdr);
538 if (!hdr)
539 return FLOW_DISSECT_RET_OUT_BAD;
540
541 /* Only look inside GRE without routing */
542 if (hdr->flags & GRE_ROUTING)
543 return FLOW_DISSECT_RET_OUT_GOOD;
544
545 /* Only look inside GRE for version 0 and 1 */
546 gre_ver = ntohs(hdr->flags & GRE_VERSION);
547 if (gre_ver > 1)
548 return FLOW_DISSECT_RET_OUT_GOOD;
549
550 *p_proto = hdr->protocol;
551 if (gre_ver) {
552 /* Version1 must be PPTP, and check the flags */
553 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY)))
554 return FLOW_DISSECT_RET_OUT_GOOD;
555 }
556
557 offset += sizeof(struct gre_base_hdr);
558
559 if (hdr->flags & GRE_CSUM)
c593642c
PB		 560 offset += sizeof_field(struct gre_full_hdr, csum) +
561 sizeof_field(struct gre_full_hdr, reserved1);
7c92de8e
JP		 562
563 if (hdr->flags & GRE_KEY) {
564 const __be32 *keyid;
565 __be32 _keyid;
566
567 keyid = __skb_header_pointer(skb, *p_nhoff + offset,
568 sizeof(_keyid),
569 data, *p_hlen, &_keyid);
570 if (!keyid)
571 return FLOW_DISSECT_RET_OUT_BAD;
572
573 if (dissector_uses_key(flow_dissector,
574 FLOW_DISSECTOR_KEY_GRE_KEYID)) {
575 key_keyid = skb_flow_dissector_target(flow_dissector,
576 FLOW_DISSECTOR_KEY_GRE_KEYID,
577 target_container);
578 if (gre_ver == 0)
579 key_keyid->keyid = *keyid;
580 else
581 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK;
582 }
c593642c 583 offset += sizeof_field(struct gre_full_hdr, key);
7c92de8e
JP		 584 }
585
586 if (hdr->flags & GRE_SEQ)
c593642c 587 offset += sizeof_field(struct pptp_gre_header, seq);
7c92de8e
JP		 588
589 if (gre_ver == 0) {
590 if (*p_proto == htons(ETH_P_TEB)) {
591 const struct ethhdr *eth;
592 struct ethhdr _eth;
593
594 eth = __skb_header_pointer(skb, *p_nhoff + offset,
595 sizeof(_eth),
596 data, *p_hlen, &_eth);
597 if (!eth)
598 return FLOW_DISSECT_RET_OUT_BAD;
599 *p_proto = eth->h_proto;
600 offset += sizeof(*eth);
601
602 /* Cap headers that we access via pointers at the
603 * end of the Ethernet header as our maximum alignment
604 * at that point is only 2 bytes.
605 */
606 if (NET_IP_ALIGN)
607 *p_hlen = *p_nhoff + offset;
608 }
609 } else { /* version 1, must be PPTP */
610 u8 _ppp_hdr[PPP_HDRLEN];
611 u8 *ppp_hdr;
612
613 if (hdr->flags & GRE_ACK)
c593642c 614 offset += sizeof_field(struct pptp_gre_header, ack);
7c92de8e
JP		 615
616 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset,
617 sizeof(_ppp_hdr),
618 data, *p_hlen, _ppp_hdr);
619 if (!ppp_hdr)
620 return FLOW_DISSECT_RET_OUT_BAD;
621
622 switch (PPP_PROTOCOL(ppp_hdr)) {
623 case PPP_IP:
624 *p_proto = htons(ETH_P_IP);
625 break;
626 case PPP_IPV6:
627 *p_proto = htons(ETH_P_IPV6);
628 break;
629 default:
630 /* Could probably catch some more like MPLS */
631 break;
632 }
633
634 offset += PPP_HDRLEN;
635 }
636
637 *p_nhoff += offset;
638 key_control->flags |= FLOW_DIS_ENCAPSULATION;
639 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
640 return FLOW_DISSECT_RET_OUT_GOOD;
641
3a1214e8 642 return FLOW_DISSECT_RET_PROTO_AGAIN;
7c92de8e
JP		 643}
644
5b0890a9
SE		 645/**
646 * __skb_flow_dissect_batadv() - dissect batman-adv header
647 * @skb: sk_buff to with the batman-adv header
648 * @key_control: flow dissectors control key
649 * @data: raw buffer pointer to the packet, if NULL use skb->data
650 * @p_proto: pointer used to update the protocol to process next
651 * @p_nhoff: pointer used to update inner network header offset
652 * @hlen: packet header length
653 * @flags: any combination of FLOW_DISSECTOR_F_*
654 *
655 * ETH_P_BATMAN packets are tried to be dissected. Only
656 * &struct batadv_unicast packets are actually processed because they contain an
657 * inner ethernet header and are usually followed by actual network header. This
658 * allows the flow dissector to continue processing the packet.
659 *
660 * Return: FLOW_DISSECT_RET_PROTO_AGAIN when &struct batadv_unicast was found,
661 * FLOW_DISSECT_RET_OUT_GOOD when dissector should stop after encapsulation,
662 * otherwise FLOW_DISSECT_RET_OUT_BAD
663 */
664static enum flow_dissect_ret
665__skb_flow_dissect_batadv(const struct sk_buff *skb,
666 struct flow_dissector_key_control *key_control,
f96533cd
AL		 667 const void *data, __be16 *p_proto, int *p_nhoff,
668 int hlen, unsigned int flags)
5b0890a9
SE		 669{
670 struct {
671 struct batadv_unicast_packet batadv_unicast;
672 struct ethhdr eth;
673 } *hdr, _hdr;
674
675 hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), data, hlen,
676 &_hdr);
677 if (!hdr)
678 return FLOW_DISSECT_RET_OUT_BAD;
679
680 if (hdr->batadv_unicast.version != BATADV_COMPAT_VERSION)
681 return FLOW_DISSECT_RET_OUT_BAD;
682
683 if (hdr->batadv_unicast.packet_type != BATADV_UNICAST)
684 return FLOW_DISSECT_RET_OUT_BAD;
685
686 *p_proto = hdr->eth.h_proto;
687 *p_nhoff += sizeof(*hdr);
688
689 key_control->flags |= FLOW_DIS_ENCAPSULATION;
690 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
691 return FLOW_DISSECT_RET_OUT_GOOD;
692
693 return FLOW_DISSECT_RET_PROTO_AGAIN;
694}
695
ac4bb5de
JP		 696static void
697__skb_flow_dissect_tcp(const struct sk_buff *skb,
698 struct flow_dissector *flow_dissector,
f96533cd
AL		 699 void *target_container, const void *data,
700 int thoff, int hlen)
ac4bb5de
JP		 701{
702 struct flow_dissector_key_tcp *key_tcp;
703 struct tcphdr *th, _th;
704
705 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP))
706 return;
707
708 th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th);
709 if (!th)
710 return;
711
712 if (unlikely(__tcp_hdrlen(th) < sizeof(_th)))
713 return;
714
715 key_tcp = skb_flow_dissector_target(flow_dissector,
716 FLOW_DISSECTOR_KEY_TCP,
717 target_container);
718 key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF));
719}
720
8ffb055b
YK		 721static void
722__skb_flow_dissect_ports(const struct sk_buff *skb,
723 struct flow_dissector *flow_dissector,
f96533cd
AL		 724 void *target_container, const void *data,
725 int nhoff, u8 ip_proto, int hlen)
8ffb055b
YK		 726{
727 enum flow_dissector_key_id dissector_ports = FLOW_DISSECTOR_KEY_MAX;
728 struct flow_dissector_key_ports *key_ports;
729
730 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS))
731 dissector_ports = FLOW_DISSECTOR_KEY_PORTS;
732 else if (dissector_uses_key(flow_dissector,
733 FLOW_DISSECTOR_KEY_PORTS_RANGE))
734 dissector_ports = FLOW_DISSECTOR_KEY_PORTS_RANGE;
735
736 if (dissector_ports == FLOW_DISSECTOR_KEY_MAX)
737 return;
738
739 key_ports = skb_flow_dissector_target(flow_dissector,
740 dissector_ports,
741 target_container);
742 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
743 data, hlen);
744}
745
518d8a2e
OG		 746static void
747__skb_flow_dissect_ipv4(const struct sk_buff *skb,
748 struct flow_dissector *flow_dissector,
f96533cd
AL		 749 void *target_container, const void *data,
750 const struct iphdr *iph)
518d8a2e
OG		 751{
752 struct flow_dissector_key_ip *key_ip;
753
754 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
755 return;
756
757 key_ip = skb_flow_dissector_target(flow_dissector,
758 FLOW_DISSECTOR_KEY_IP,
759 target_container);
760 key_ip->tos = iph->tos;
761 key_ip->ttl = iph->ttl;
762}
763
764static void
765__skb_flow_dissect_ipv6(const struct sk_buff *skb,
766 struct flow_dissector *flow_dissector,
f96533cd
AL		 767 void *target_container, const void *data,
768 const struct ipv6hdr *iph)
518d8a2e
OG		 769{
770 struct flow_dissector_key_ip *key_ip;
771
772 if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
773 return;
774
775 key_ip = skb_flow_dissector_target(flow_dissector,
776 FLOW_DISSECTOR_KEY_IP,
777 target_container);
778 key_ip->tos = ipv6_get_dsfield(iph);
779 key_ip->ttl = iph->hop_limit;
780}
781
1eed4dfb
TH		 782/* Maximum number of protocol headers that can be parsed in
783 * __skb_flow_dissect
784 */
785#define MAX_FLOW_DISSECT_HDRS 15
786
787static bool skb_flow_dissect_allowed(int *num_hdrs)
788{
789 ++*num_hdrs;
790
791 return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS);
792}
793
d58e468b
PP		 794static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys,
795 struct flow_dissector *flow_dissector,
796 void *target_container)
797{
59fb9b62 798 struct flow_dissector_key_ports *key_ports = NULL;
d58e468b
PP		 799 struct flow_dissector_key_control *key_control;
800 struct flow_dissector_key_basic *key_basic;
801 struct flow_dissector_key_addrs *key_addrs;
71c99e32 802 struct flow_dissector_key_tags *key_tags;
d58e468b
PP		 803
804 key_control = skb_flow_dissector_target(flow_dissector,
805 FLOW_DISSECTOR_KEY_CONTROL,
806 target_container);
807 key_control->thoff = flow_keys->thoff;
808 if (flow_keys->is_frag)
809 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
810 if (flow_keys->is_first_frag)
811 key_control->flags |= FLOW_DIS_FIRST_FRAG;
812 if (flow_keys->is_encap)
813 key_control->flags |= FLOW_DIS_ENCAPSULATION;
814
815 key_basic = skb_flow_dissector_target(flow_dissector,
816 FLOW_DISSECTOR_KEY_BASIC,
817 target_container);
818 key_basic->n_proto = flow_keys->n_proto;
819 key_basic->ip_proto = flow_keys->ip_proto;
820
821 if (flow_keys->addr_proto == ETH_P_IP &&
822 dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
823 key_addrs = skb_flow_dissector_target(flow_dissector,
824 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
825 target_container);
826 key_addrs->v4addrs.src = flow_keys->ipv4_src;
827 key_addrs->v4addrs.dst = flow_keys->ipv4_dst;
828 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
829 } else if (flow_keys->addr_proto == ETH_P_IPV6 &&
830 dissector_uses_key(flow_dissector,
831 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
832 key_addrs = skb_flow_dissector_target(flow_dissector,
833 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
834 target_container);
835 memcpy(&key_addrs->v6addrs, &flow_keys->ipv6_src,
836 sizeof(key_addrs->v6addrs));
837 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
838 }
839
59fb9b62 840 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS))
d58e468b
PP		 841 key_ports = skb_flow_dissector_target(flow_dissector,
842 FLOW_DISSECTOR_KEY_PORTS,
843 target_container);
59fb9b62
YK		 844 else if (dissector_uses_key(flow_dissector,
845 FLOW_DISSECTOR_KEY_PORTS_RANGE))
846 key_ports = skb_flow_dissector_target(flow_dissector,
847 FLOW_DISSECTOR_KEY_PORTS_RANGE,
848 target_container);
849
850 if (key_ports) {
d58e468b
PP		 851 key_ports->src = flow_keys->sport;
852 key_ports->dst = flow_keys->dport;
853 }
71c99e32
SF		 854
855 if (dissector_uses_key(flow_dissector,
856 FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
857 key_tags = skb_flow_dissector_target(flow_dissector,
858 FLOW_DISSECTOR_KEY_FLOW_LABEL,
859 target_container);
860 key_tags->flow_label = ntohl(flow_keys->flow_label);
861 }
d58e468b
PP		 862}
863
089b19a9 864bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
086f9568 865 __be16 proto, int nhoff, int hlen, unsigned int flags)
089b19a9
SF		 866{
867 struct bpf_flow_keys *flow_keys = ctx->flow_keys;
868 u32 result;
c8aa7038
SF		 869
870 /* Pass parameters to the BPF program */
871 memset(flow_keys, 0, sizeof(*flow_keys));
089b19a9
SF		 872 flow_keys->n_proto = proto;
873 flow_keys->nhoff = nhoff;
c8aa7038
SF		 874 flow_keys->thoff = flow_keys->nhoff;
875
086f9568
SF		 876 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG !=
877 (int)FLOW_DISSECTOR_F_PARSE_1ST_FRAG);
878 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL !=
879 (int)FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
880 BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP !=
881 (int)FLOW_DISSECTOR_F_STOP_AT_ENCAP);
882 flow_keys->flags = flags;
883
3d9f773c 884 result = bpf_prog_run_pin_on_cpu(prog, ctx);
c8aa7038 885
089b19a9 886 flow_keys->nhoff = clamp_t(u16, flow_keys->nhoff, nhoff, hlen);
c8aa7038 887 flow_keys->thoff = clamp_t(u16, flow_keys->thoff,
089b19a9 888 flow_keys->nhoff, hlen);
c8aa7038
SF		 889
890 return result == BPF_OK;
891}
892
453a940e
WC		 893/**
894 * __skb_flow_dissect - extract the flow_keys struct and return it
3cbf4ffb 895 * @net: associated network namespace, derived from @skb if NULL
453a940e 896 * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
06635a35
JP		 897 * @flow_dissector: list of keys to dissect
898 * @target_container: target structure to put dissected values into
453a940e
WC		 899 * @data: raw buffer pointer to the packet, if NULL use skb->data
900 * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
901 * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
902 * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
d79b3baf 903 * @flags: flags that control the dissection process, e.g.
1cc26450 904 * FLOW_DISSECTOR_F_STOP_AT_ENCAP.
453a940e 905 *
06635a35
JP		 906 * The function will try to retrieve individual keys into target specified
907 * by flow_dissector from either the skbuff or a raw buffer specified by the
908 * rest parameters.
909 *
910 * Caller must take care of zeroing target container memory.
453a940e 911 */
3cbf4ffb
SF		 912bool __skb_flow_dissect(const struct net *net,
913 const struct sk_buff *skb,
06635a35 914 struct flow_dissector *flow_dissector,
f96533cd
AL		 915 void *target_container, const void *data,
916 __be16 proto, int nhoff, int hlen, unsigned int flags)
0744dd00 917{
42aecaa9 918 struct flow_dissector_key_control *key_control;
06635a35
JP		 919 struct flow_dissector_key_basic *key_basic;
920 struct flow_dissector_key_addrs *key_addrs;
d34af823 921 struct flow_dissector_key_tags *key_tags;
f6a66927 922 struct flow_dissector_key_vlan *key_vlan;
3a1214e8 923 enum flow_dissect_ret fdret;
24c590e3 924 enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX;
58cff782
GN		 925 bool mpls_el = false;
926 int mpls_lse = 0;
1eed4dfb 927 int num_hdrs = 0;
8e690ffd 928 u8 ip_proto = 0;
34fad54c 929 bool ret;
0744dd00 930
690e36e7
DM		 931 if (!data) {
932 data = skb->data;
d5709f7a
HHZ		 933 proto = skb_vlan_tag_present(skb) ?
934 skb->vlan_proto : skb->protocol;
453a940e 935 nhoff = skb_network_offset(skb);
690e36e7 936 hlen = skb_headlen(skb);
2d571645 937#if IS_ENABLED(CONFIG_NET_DSA)
8bef0af0
AL		 938 if (unlikely(skb->dev && netdev_uses_dsa(skb->dev) &&
939 proto == htons(ETH_P_XDSA))) {
43e66528 940 const struct dsa_device_ops *ops;
8bef0af0 941 int offset = 0;
43e66528
JC		 942
943 ops = skb->dev->dsa_ptr->tag_ops;
54fec335
VO		 944 /* Tail taggers don't break flow dissection */
945 if (!ops->tail_tag) {
946 if (ops->flow_dissect)
947 ops->flow_dissect(skb, &proto, &offset);
948 else
949 dsa_tag_generic_flow_dissect(skb,
950 &proto,
951 &offset);
43e66528
JC		 952 hlen -= offset;
953 nhoff += offset;
954 }
955 }
2d571645 956#endif
690e36e7
DM		 957 }
958
42aecaa9
TH		 959 /* It is ensured by skb_flow_dissector_init() that control key will
960 * be always present.
961 */
962 key_control = skb_flow_dissector_target(flow_dissector,
963 FLOW_DISSECTOR_KEY_CONTROL,
964 target_container);
965
06635a35
JP		 966 /* It is ensured by skb_flow_dissector_init() that basic key will
967 * be always present.
968 */
969 key_basic = skb_flow_dissector_target(flow_dissector,
970 FLOW_DISSECTOR_KEY_BASIC,
971 target_container);
0744dd00 972
d0e13a14 973 if (skb) {
3cbf4ffb
SF		 974 if (!net) {
975 if (skb->dev)
976 net = dev_net(skb->dev);
977 else if (skb->sk)
978 net = sock_net(skb->sk);
3cbf4ffb 979 }
9b52e3f2 980 }
c8aa7038 981
9b52e3f2
SF		 982 WARN_ON_ONCE(!net);
983 if (net) {
a3fd7cee 984 enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
695c1214 985 struct bpf_prog_array *run_array;
a3fd7cee 986
9b52e3f2 987 rcu_read_lock();
695c1214
JS		 988 run_array = rcu_dereference(init_net.bpf.run_array[type]);
989 if (!run_array)
990 run_array = rcu_dereference(net->bpf.run_array[type]);
a11c397c 991
695c1214 992 if (run_array) {
9b52e3f2
SF		 993 struct bpf_flow_keys flow_keys;
994 struct bpf_flow_dissector ctx = {
995 .flow_keys = &flow_keys,
996 .data = data,
997 .data_end = data + hlen,
998 };
999 __be16 n_proto = proto;
695c1214 1000 struct bpf_prog *prog;
9b52e3f2
SF		 1001
1002 if (skb) {
1003 ctx.skb = skb;
1004 /* we can't use 'proto' in the skb case
1005 * because it might be set to skb->vlan_proto
1006 * which has been pulled from the data
1007 */
1008 n_proto = skb->protocol;
1009 }
1010
695c1214
JS		 1011 prog = READ_ONCE(run_array->items[0].prog);
1012 ret = bpf_flow_dissect(prog, &ctx, n_proto, nhoff,
086f9568 1013 hlen, flags);
c8aa7038
SF		 1014 __skb_flow_bpf_to_target(&flow_keys, flow_dissector,
1015 target_container);
1016 rcu_read_unlock();
1017 return ret;
1018 }
d58e468b 1019 rcu_read_unlock();
d58e468b 1020 }
d58e468b 1021
20a17bf6
DM		 1022 if (dissector_uses_key(flow_dissector,
1023 FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
67a900cc
JP		 1024 struct ethhdr *eth = eth_hdr(skb);
1025 struct flow_dissector_key_eth_addrs *key_eth_addrs;
1026
1027 key_eth_addrs = skb_flow_dissector_target(flow_dissector,
1028 FLOW_DISSECTOR_KEY_ETH_ADDRS,
1029 target_container);
1030 memcpy(key_eth_addrs, &eth->h_dest, sizeof(*key_eth_addrs));
1031 }
1032
c5ef188e 1033proto_again:
3a1214e8
TH		 1034 fdret = FLOW_DISSECT_RET_CONTINUE;
1035
0744dd00 1036 switch (proto) {
2b8837ae 1037 case htons(ETH_P_IP): {
0744dd00
ED		 1038 const struct iphdr *iph;
1039 struct iphdr _iph;
3a1214e8 1040
690e36e7 1041 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
3a1214e8
TH		 1042 if (!iph || iph->ihl < 5) {
1043 fdret = FLOW_DISSECT_RET_OUT_BAD;
1044 break;
1045 }
1046
3797d3e8 1047 nhoff += iph->ihl * 4;
0744dd00 1048
3797d3e8 1049 ip_proto = iph->protocol;
3797d3e8 1050
918c023f
AD		 1051 if (dissector_uses_key(flow_dissector,
1052 FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
1053 key_addrs = skb_flow_dissector_target(flow_dissector,
1054 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1055 target_container);
1056
1057 memcpy(&key_addrs->v4addrs, &iph->saddr,
1058 sizeof(key_addrs->v4addrs));
1059 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1060 }
807e165d 1061
d2126838
DC		 1062 __skb_flow_dissect_ipv4(skb, flow_dissector,
1063 target_container, data, iph);
1064
807e165d 1065 if (ip_is_fragment(iph)) {
4b36993d 1066 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
807e165d
TH		 1067
1068 if (iph->frag_off & htons(IP_OFFSET)) {
3a1214e8
TH		 1069 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1070 break;
807e165d 1071 } else {
4b36993d 1072 key_control->flags |= FLOW_DIS_FIRST_FRAG;
3a1214e8
TH		 1073 if (!(flags &
1074 FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) {
1075 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1076 break;
1077 }
807e165d
TH		 1078 }
1079 }
1080
0744dd00
ED		 1081 break;
1082 }
2b8837ae 1083 case htons(ETH_P_IPV6): {
0744dd00
ED		 1084 const struct ipv6hdr *iph;
1085 struct ipv6hdr _iph;
19469a87 1086
690e36e7 1087 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
3a1214e8
TH		 1088 if (!iph) {
1089 fdret = FLOW_DISSECT_RET_OUT_BAD;
1090 break;
1091 }
0744dd00
ED		 1092
1093 ip_proto = iph->nexthdr;
0744dd00 1094 nhoff += sizeof(struct ipv6hdr);
19469a87 1095
20a17bf6
DM		 1096 if (dissector_uses_key(flow_dissector,
1097 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
b3c3106c
AD		 1098 key_addrs = skb_flow_dissector_target(flow_dissector,
1099 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1100 target_container);
5af7fb6e 1101
b3c3106c
AD		 1102 memcpy(&key_addrs->v6addrs, &iph->saddr,
1103 sizeof(key_addrs->v6addrs));
c3f83241 1104 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
b924933c 1105 }
87ee9e52 1106
461547f3
AD		 1107 if ((dissector_uses_key(flow_dissector,
1108 FLOW_DISSECTOR_KEY_FLOW_LABEL) ||
1109 (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) &&
1110 ip6_flowlabel(iph)) {
1111 __be32 flow_label = ip6_flowlabel(iph);
1112
20a17bf6
DM		 1113 if (dissector_uses_key(flow_dissector,
1114 FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
87ee9e52
TH		 1115 key_tags = skb_flow_dissector_target(flow_dissector,
1116 FLOW_DISSECTOR_KEY_FLOW_LABEL,
1117 target_container);
1118 key_tags->flow_label = ntohl(flow_label);
12c227ec 1119 }
3a1214e8
TH		 1120 if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) {
1121 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1122 break;
1123 }
19469a87
TH		 1124 }
1125
518d8a2e
OG		 1126 __skb_flow_dissect_ipv6(skb, flow_dissector,
1127 target_container, data, iph);
1128
0744dd00
ED		 1129 break;
1130 }
2b8837ae
JP		 1131 case htons(ETH_P_8021AD):
1132 case htons(ETH_P_8021Q): {
24c590e3 1133 const struct vlan_hdr *vlan = NULL;
bc72f3dd 1134 struct vlan_hdr _vlan;
2064c3d4 1135 __be16 saved_vlan_tpid = proto;
0744dd00 1136
24c590e3
JL		 1137 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX &&
1138 skb && skb_vlan_tag_present(skb)) {
d5709f7a 1139 proto = skb->protocol;
24c590e3 1140 } else {
d5709f7a
HHZ		 1141 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan),
1142 data, hlen, &_vlan);
3a1214e8
TH		 1143 if (!vlan) {
1144 fdret = FLOW_DISSECT_RET_OUT_BAD;
1145 break;
1146 }
1147
d5709f7a
HHZ		 1148 proto = vlan->h_vlan_encapsulated_proto;
1149 nhoff += sizeof(*vlan);
d5709f7a 1150 }
0744dd00 1151
24c590e3
JL		 1152 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX) {
1153 dissector_vlan = FLOW_DISSECTOR_KEY_VLAN;
1154 } else if (dissector_vlan == FLOW_DISSECTOR_KEY_VLAN) {
1155 dissector_vlan = FLOW_DISSECTOR_KEY_CVLAN;
1156 } else {
1157 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1158 break;
1159 }
1160
1161 if (dissector_uses_key(flow_dissector, dissector_vlan)) {
f6a66927 1162 key_vlan = skb_flow_dissector_target(flow_dissector,
24c590e3 1163 dissector_vlan,
d34af823
TH		 1164 target_container);
1165
24c590e3 1166 if (!vlan) {
f6a66927 1167 key_vlan->vlan_id = skb_vlan_tag_get_id(skb);
9b319148 1168 key_vlan->vlan_priority = skb_vlan_tag_get_prio(skb);
f6a66927
HHZ		 1169 } else {
1170 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) &
d5709f7a 1171 VLAN_VID_MASK;
f6a66927
HHZ		 1172 key_vlan->vlan_priority =
1173 (ntohs(vlan->h_vlan_TCI) &
1174 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1175 }
2064c3d4 1176 key_vlan->vlan_tpid = saved_vlan_tpid;
d34af823
TH		 1177 }
1178
3a1214e8
TH		 1179 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1180 break;
0744dd00 1181 }
2b8837ae 1182 case htons(ETH_P_PPP_SES): {
0744dd00
ED		 1183 struct {
1184 struct pppoe_hdr hdr;
1185 __be16 proto;
1186 } *hdr, _hdr;
690e36e7 1187 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
3a1214e8
TH		 1188 if (!hdr) {
1189 fdret = FLOW_DISSECT_RET_OUT_BAD;
1190 break;
1191 }
1192
0744dd00
ED		 1193 proto = hdr->proto;
1194 nhoff += PPPOE_SES_HLEN;
1195 switch (proto) {
2b8837ae 1196 case htons(PPP_IP):
3a1214e8
TH		 1197 proto = htons(ETH_P_IP);
1198 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1199 break;
2b8837ae 1200 case htons(PPP_IPV6):
3a1214e8
TH		 1201 proto = htons(ETH_P_IPV6);
1202 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1203 break;
0744dd00 1204 default:
3a1214e8
TH		 1205 fdret = FLOW_DISSECT_RET_OUT_BAD;
1206 break;
0744dd00 1207 }
3a1214e8 1208 break;
0744dd00 1209 }
08bfc9cb 1210 case htons(ETH_P_TIPC): {
8d6e79d3
JM		 1211 struct tipc_basic_hdr *hdr, _hdr;
1212
1213 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr),
1214 data, hlen, &_hdr);
3a1214e8
TH		 1215 if (!hdr) {
1216 fdret = FLOW_DISSECT_RET_OUT_BAD;
1217 break;
1218 }
06635a35 1219
20a17bf6 1220 if (dissector_uses_key(flow_dissector,
8d6e79d3 1221 FLOW_DISSECTOR_KEY_TIPC)) {
06635a35 1222 key_addrs = skb_flow_dissector_target(flow_dissector,
8d6e79d3 1223 FLOW_DISSECTOR_KEY_TIPC,
06635a35 1224 target_container);
8d6e79d3
JM		 1225 key_addrs->tipckey.key = tipc_hdr_rps_key(hdr);
1226 key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC;
06635a35 1227 }
3a1214e8
TH		 1228 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1229 break;
08bfc9cb 1230 }
b3baa0fb
TH		 1231
1232 case htons(ETH_P_MPLS_UC):
4a5d6c8b 1233 case htons(ETH_P_MPLS_MC):
3a1214e8 1234 fdret = __skb_flow_dissect_mpls(skb, flow_dissector,
4a5d6c8b 1235 target_container, data,
58cff782
GN		 1236 nhoff, hlen, mpls_lse,
1237 &mpls_el);
1238 nhoff += sizeof(struct mpls_label);
1239 mpls_lse++;
3a1214e8 1240 break;
56193d1b 1241 case htons(ETH_P_FCOE):
3a1214e8
TH		 1242 if ((hlen - nhoff) < FCOE_HEADER_LEN) {
1243 fdret = FLOW_DISSECT_RET_OUT_BAD;
1244 break;
1245 }
224516b3
AD		 1246
1247 nhoff += FCOE_HEADER_LEN;
3a1214e8
TH		 1248 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1249 break;
55733350
SH		 1250
1251 case htons(ETH_P_ARP):
9bf881ff 1252 case htons(ETH_P_RARP):
3a1214e8 1253 fdret = __skb_flow_dissect_arp(skb, flow_dissector,
9bf881ff 1254 target_container, data,
3a1214e8
TH		 1255 nhoff, hlen);
1256 break;
1257
5b0890a9
SE		 1258 case htons(ETH_P_BATMAN):
1259 fdret = __skb_flow_dissect_batadv(skb, key_control, data,
1260 &proto, &nhoff, hlen, flags);
1261 break;
1262
4f1cc51f
EBE		 1263 case htons(ETH_P_1588): {
1264 struct ptp_header *hdr, _hdr;
1265
1266 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
1267 hlen, &_hdr);
1268 if (!hdr) {
1269 fdret = FLOW_DISSECT_RET_OUT_BAD;
1270 break;
1271 }
1272
1273 nhoff += ntohs(hdr->message_length);
1274 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1275 break;
1276 }
1277
3a1214e8
TH		 1278 default:
1279 fdret = FLOW_DISSECT_RET_OUT_BAD;
1280 break;
1281 }
1282
1283 /* Process result of proto processing */
1284 switch (fdret) {
1285 case FLOW_DISSECT_RET_OUT_GOOD:
1286 goto out_good;
1287 case FLOW_DISSECT_RET_PROTO_AGAIN:
1eed4dfb
TH		 1288 if (skb_flow_dissect_allowed(&num_hdrs))
1289 goto proto_again;
1290 goto out_good;
3a1214e8
TH		 1291 case FLOW_DISSECT_RET_CONTINUE:
1292 case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1293 break;
1294 case FLOW_DISSECT_RET_OUT_BAD:
0744dd00 1295 default:
a6e544b0 1296 goto out_bad;
0744dd00
ED		 1297 }
1298
6a74fcf4 1299ip_proto_again:
3a1214e8
TH		 1300 fdret = FLOW_DISSECT_RET_CONTINUE;
1301
0744dd00 1302 switch (ip_proto) {
7c92de8e 1303 case IPPROTO_GRE:
3a1214e8 1304 fdret = __skb_flow_dissect_gre(skb, key_control, flow_dissector,
7c92de8e 1305 target_container, data,
3a1214e8
TH		 1306 &proto, &nhoff, &hlen, flags);
1307 break;
1308
6a74fcf4
TH		 1309 case NEXTHDR_HOP:
1310 case NEXTHDR_ROUTING:
1311 case NEXTHDR_DEST: {
1312 u8 _opthdr[2], *opthdr;
1313
1314 if (proto != htons(ETH_P_IPV6))
1315 break;
1316
1317 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr),
1318 data, hlen, &_opthdr);
3a1214e8
TH		 1319 if (!opthdr) {
1320 fdret = FLOW_DISSECT_RET_OUT_BAD;
1321 break;
1322 }
6a74fcf4 1323
1e98a0f0
ED		 1324 ip_proto = opthdr[0];
1325 nhoff += (opthdr[1] + 1) << 3;
6a74fcf4 1326
3a1214e8
TH		 1327 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1328 break;
6a74fcf4 1329 }
b840f28b
TH		 1330 case NEXTHDR_FRAGMENT: {
1331 struct frag_hdr _fh, *fh;
1332
1333 if (proto != htons(ETH_P_IPV6))
1334 break;
1335
1336 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh),
1337 data, hlen, &_fh);
1338
3a1214e8
TH		 1339 if (!fh) {
1340 fdret = FLOW_DISSECT_RET_OUT_BAD;
1341 break;
1342 }
b840f28b 1343
4b36993d 1344 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
b840f28b
TH		 1345
1346 nhoff += sizeof(_fh);
43d2ccb3 1347 ip_proto = fh->nexthdr;
b840f28b
TH		 1348
1349 if (!(fh->frag_off & htons(IP6_OFFSET))) {
4b36993d 1350 key_control->flags |= FLOW_DIS_FIRST_FRAG;
3a1214e8
TH		 1351 if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) {
1352 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1353 break;
1354 }
b840f28b 1355 }
3a1214e8
TH		 1356
1357 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1358 break;
b840f28b 1359 }
0744dd00 1360 case IPPROTO_IPIP:
fca41895 1361 proto = htons(ETH_P_IP);
823b9693 1362
4b36993d 1363 key_control->flags |= FLOW_DIS_ENCAPSULATION;
3a1214e8
TH		 1364 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1365 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1366 break;
1367 }
1368
1369 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1370 break;
823b9693 1371
b438f940
TH		 1372 case IPPROTO_IPV6:
1373 proto = htons(ETH_P_IPV6);
823b9693 1374
4b36993d 1375 key_control->flags |= FLOW_DIS_ENCAPSULATION;
3a1214e8
TH		 1376 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1377 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1378 break;
1379 }
1380
1381 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1382 break;
1383
823b9693 1384
b3baa0fb
TH		 1385 case IPPROTO_MPLS:
1386 proto = htons(ETH_P_MPLS_UC);
3a1214e8
TH		 1387 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1388 break;
1389
ac4bb5de
JP		 1390 case IPPROTO_TCP:
1391 __skb_flow_dissect_tcp(skb, flow_dissector, target_container,
1392 data, nhoff, hlen);
1393 break;
3a1214e8 1394
3b336d6f
MC		 1395 case IPPROTO_ICMP:
1396 case IPPROTO_ICMPV6:
1397 __skb_flow_dissect_icmp(skb, flow_dissector, target_container,
1398 data, nhoff, hlen);
1399 break;
1400
0744dd00
ED		 1401 default:
1402 break;
1403 }
1404
8ffb055b
YK		 1405 if (!(key_control->flags & FLOW_DIS_IS_FRAGMENT))
1406 __skb_flow_dissect_ports(skb, flow_dissector, target_container,
1407 data, nhoff, ip_proto, hlen);
5af7fb6e 1408
3a1214e8
TH		 1409 /* Process result of IP proto processing */
1410 switch (fdret) {
1411 case FLOW_DISSECT_RET_PROTO_AGAIN:
1eed4dfb
TH		 1412 if (skb_flow_dissect_allowed(&num_hdrs))
1413 goto proto_again;
1414 break;
3a1214e8 1415 case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1eed4dfb
TH		 1416 if (skb_flow_dissect_allowed(&num_hdrs))
1417 goto ip_proto_again;
1418 break;
3a1214e8
TH		 1419 case FLOW_DISSECT_RET_OUT_GOOD:
1420 case FLOW_DISSECT_RET_CONTINUE:
1421 break;
1422 case FLOW_DISSECT_RET_OUT_BAD:
1423 default:
1424 goto out_bad;
1425 }
1426
a6e544b0
TH		 1427out_good:
1428 ret = true;
1429
34fad54c 1430out:
d0c081b4 1431 key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
a6e544b0
TH		 1432 key_basic->n_proto = proto;
1433 key_basic->ip_proto = ip_proto;
a6e544b0
TH		 1434
1435 return ret;
34fad54c
ED		 1436
1437out_bad:
1438 ret = false;
34fad54c 1439 goto out;
0744dd00 1440}
690e36e7 1441EXPORT_SYMBOL(__skb_flow_dissect);
441d9d32 1442
55667441 1443static siphash_key_t hashrnd __read_mostly;
66415cf8
HFS		 1444static __always_inline void __flow_hash_secret_init(void)
1445{
1446 net_get_random_once(&hashrnd, sizeof(hashrnd));
1447}
1448
55667441 1449static const void *flow_keys_hash_start(const struct flow_keys *flow)
42aecaa9 1450{
55667441
ED		 1451 BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % SIPHASH_ALIGNMENT);
1452 return &flow->FLOW_KEYS_HASH_START_FIELD;
42aecaa9
TH		 1453}
1454
20a17bf6 1455static inline size_t flow_keys_hash_length(const struct flow_keys *flow)
42aecaa9 1456{
c3f83241 1457 size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs);
d31e9558 1458
42aecaa9 1459 BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
c3f83241
TH		 1460
1461 switch (flow->control.addr_type) {
1462 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1463 diff -= sizeof(flow->addrs.v4addrs);
1464 break;
1465 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1466 diff -= sizeof(flow->addrs.v6addrs);
1467 break;
8d6e79d3
JM		 1468 case FLOW_DISSECTOR_KEY_TIPC:
1469 diff -= sizeof(flow->addrs.tipckey);
9f249089 1470 break;
c3f83241 1471 }
55667441 1472 return sizeof(*flow) - diff;
c3f83241
TH		 1473}
1474
1475__be32 flow_get_u32_src(const struct flow_keys *flow)
1476{
1477 switch (flow->control.addr_type) {
1478 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1479 return flow->addrs.v4addrs.src;
1480 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1481 return (__force __be32)ipv6_addr_hash(
1482 &flow->addrs.v6addrs.src);
8d6e79d3
JM		 1483 case FLOW_DISSECTOR_KEY_TIPC:
1484 return flow->addrs.tipckey.key;
c3f83241
TH		 1485 default:
1486 return 0;
1487 }
1488}
1489EXPORT_SYMBOL(flow_get_u32_src);
1490
1491__be32 flow_get_u32_dst(const struct flow_keys *flow)
1492{
1493 switch (flow->control.addr_type) {
1494 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1495 return flow->addrs.v4addrs.dst;
1496 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1497 return (__force __be32)ipv6_addr_hash(
1498 &flow->addrs.v6addrs.dst);
1499 default:
1500 return 0;
1501 }
1502}
1503EXPORT_SYMBOL(flow_get_u32_dst);
1504
98298e6c
MC		 1505/* Sort the source and destination IP (and the ports if the IP are the same),
1506 * to have consistent hash within the two directions
1507 */
c3f83241
TH		 1508static inline void __flow_hash_consistentify(struct flow_keys *keys)
1509{
1510 int addr_diff, i;
1511
1512 switch (keys->control.addr_type) {
1513 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1514 addr_diff = (__force u32)keys->addrs.v4addrs.dst -
1515 (__force u32)keys->addrs.v4addrs.src;
1516 if ((addr_diff < 0) ||
1517 (addr_diff == 0 &&
1518 ((__force u16)keys->ports.dst <
1519 (__force u16)keys->ports.src))) {
1520 swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst);
1521 swap(keys->ports.src, keys->ports.dst);
1522 }
1523 break;
1524 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1525 addr_diff = memcmp(&keys->addrs.v6addrs.dst,
1526 &keys->addrs.v6addrs.src,
1527 sizeof(keys->addrs.v6addrs.dst));
1528 if ((addr_diff < 0) ||
1529 (addr_diff == 0 &&
1530 ((__force u16)keys->ports.dst <
1531 (__force u16)keys->ports.src))) {
1532 for (i = 0; i < 4; i++)
1533 swap(keys->addrs.v6addrs.src.s6_addr32[i],
1534 keys->addrs.v6addrs.dst.s6_addr32[i]);
1535 swap(keys->ports.src, keys->ports.dst);
1536 }
1537 break;
1538 }
66415cf8
HFS		 1539}
1540
55667441
ED		 1541static inline u32 __flow_hash_from_keys(struct flow_keys *keys,
1542 const siphash_key_t *keyval)
5ed20a68
TH		 1543{
1544 u32 hash;
1545
c3f83241 1546 __flow_hash_consistentify(keys);
5ed20a68 1547
55667441
ED		 1548 hash = siphash(flow_keys_hash_start(keys),
1549 flow_keys_hash_length(keys), keyval);
5ed20a68
TH		 1550 if (!hash)
1551 hash = 1;
1552
1553 return hash;
1554}
1555
1556u32 flow_hash_from_keys(struct flow_keys *keys)
1557{
50fb7992 1558 __flow_hash_secret_init();
55667441 1559 return __flow_hash_from_keys(keys, &hashrnd);
5ed20a68
TH		 1560}
1561EXPORT_SYMBOL(flow_hash_from_keys);
1562
50fb7992 1563static inline u32 ___skb_get_hash(const struct sk_buff *skb,
55667441
ED		 1564 struct flow_keys *keys,
1565 const siphash_key_t *keyval)
50fb7992 1566{
6db61d79
TH		 1567 skb_flow_dissect_flow_keys(skb, keys,
1568 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
50fb7992
TH		 1569
1570 return __flow_hash_from_keys(keys, keyval);
1571}
1572
2f59e1eb
TH		 1573struct _flow_keys_digest_data {
1574 __be16 n_proto;
1575 u8 ip_proto;
1576 u8 padding;
1577 __be32 ports;
1578 __be32 src;
1579 __be32 dst;
1580};
1581
1582void make_flow_keys_digest(struct flow_keys_digest *digest,
1583 const struct flow_keys *flow)
1584{
1585 struct _flow_keys_digest_data *data =
1586 (struct _flow_keys_digest_data *)digest;
1587
1588 BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
1589
1590 memset(digest, 0, sizeof(*digest));
1591
06635a35
JP		 1592 data->n_proto = flow->basic.n_proto;
1593 data->ip_proto = flow->basic.ip_proto;
1594 data->ports = flow->ports.ports;
c3f83241
TH		 1595 data->src = flow->addrs.v4addrs.src;
1596 data->dst = flow->addrs.v4addrs.dst;
2f59e1eb
TH		 1597}
1598EXPORT_SYMBOL(make_flow_keys_digest);
1599
eb70db87
DM		 1600static struct flow_dissector flow_keys_dissector_symmetric __read_mostly;
1601
b917783c 1602u32 __skb_get_hash_symmetric(const struct sk_buff *skb)
eb70db87
DM		 1603{
1604 struct flow_keys keys;
1605
1606 __flow_hash_secret_init();
1607
1608 memset(&keys, 0, sizeof(keys));
3cbf4ffb
SF		 1609 __skb_flow_dissect(NULL, skb, &flow_keys_dissector_symmetric,
1610 &keys, NULL, 0, 0, 0,
eb70db87
DM		 1611 FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1612
55667441 1613 return __flow_hash_from_keys(&keys, &hashrnd);
eb70db87
DM		 1614}
1615EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric);
1616
d4fd3275
JP		 1617/**
1618 * __skb_get_hash: calculate a flow hash
1619 * @skb: sk_buff to calculate flow hash from
1620 *
1621 * This function calculates a flow hash based on src/dst addresses
61b905da
TH		 1622 * and src/dst port numbers. Sets hash in skb to non-zero hash value
1623 * on success, zero indicates no valid hash. Also, sets l4_hash in skb
441d9d32
CW		 1624 * if hash is a canonical 4-tuple hash over transport ports.
1625 */
3958afa1 1626void __skb_get_hash(struct sk_buff *skb)
441d9d32
CW		 1627{
1628 struct flow_keys keys;
635c223c 1629 u32 hash;
441d9d32 1630
50fb7992
TH		 1631 __flow_hash_secret_init();
1632
55667441 1633 hash = ___skb_get_hash(skb, &keys, &hashrnd);
635c223c
GF		 1634
1635 __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
441d9d32 1636}
3958afa1 1637EXPORT_SYMBOL(__skb_get_hash);
441d9d32 1638
55667441
ED		 1639__u32 skb_get_hash_perturb(const struct sk_buff *skb,
1640 const siphash_key_t *perturb)
50fb7992
TH		 1641{
1642 struct flow_keys keys;
1643
1644 return ___skb_get_hash(skb, &keys, perturb);
1645}
1646EXPORT_SYMBOL(skb_get_hash_perturb);
1647
f96533cd 1648u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
72a338bc 1649 const struct flow_keys_basic *keys, int hlen)
f77668dc 1650{
42aecaa9 1651 u32 poff = keys->control.thoff;
f77668dc 1652
43d2ccb3
AD		 1653 /* skip L4 headers for fragments after the first */
1654 if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) &&
1655 !(keys->control.flags & FLOW_DIS_FIRST_FRAG))
1656 return poff;
1657
06635a35 1658 switch (keys->basic.ip_proto) {
f77668dc 1659 case IPPROTO_TCP: {
5af7fb6e
AD		 1660 /* access doff as u8 to avoid unaligned access */
1661 const u8 *doff;
1662 u8 _doff;
f77668dc 1663
5af7fb6e
AD		 1664 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
1665 data, hlen, &_doff);
1666 if (!doff)
f77668dc
DB		 1667 return poff;
1668
5af7fb6e 1669 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
f77668dc
DB		 1670 break;
1671 }
1672 case IPPROTO_UDP:
1673 case IPPROTO_UDPLITE:
1674 poff += sizeof(struct udphdr);
1675 break;
1676 /* For the rest, we do not really care about header
1677 * extensions at this point for now.
1678 */
1679 case IPPROTO_ICMP:
1680 poff += sizeof(struct icmphdr);
1681 break;
1682 case IPPROTO_ICMPV6:
1683 poff += sizeof(struct icmp6hdr);
1684 break;
1685 case IPPROTO_IGMP:
1686 poff += sizeof(struct igmphdr);
1687 break;
1688 case IPPROTO_DCCP:
1689 poff += sizeof(struct dccp_hdr);
1690 break;
1691 case IPPROTO_SCTP:
1692 poff += sizeof(struct sctphdr);
1693 break;
1694 }
1695
1696 return poff;
1697}
1698
0db89b8b
JP		 1699/**
1700 * skb_get_poff - get the offset to the payload
1701 * @skb: sk_buff to get the payload offset from
1702 *
1703 * The function will get the offset to the payload as far as it could
1704 * be dissected. The main user is currently BPF, so that we can dynamically
56193d1b
AD		 1705 * truncate packets without needing to push actual payload to the user
1706 * space and can analyze headers only, instead.
1707 */
1708u32 skb_get_poff(const struct sk_buff *skb)
1709{
72a338bc 1710 struct flow_keys_basic keys;
56193d1b 1711
3cbf4ffb
SF		 1712 if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
1713 NULL, 0, 0, 0, 0))
56193d1b
AD		 1714 return 0;
1715
1716 return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
1717}
06635a35 1718
20a17bf6 1719__u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys)
a17ace95
DM		 1720{
1721 memset(keys, 0, sizeof(*keys));
1722
1723 memcpy(&keys->addrs.v6addrs.src, &fl6->saddr,
1724 sizeof(keys->addrs.v6addrs.src));
1725 memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr,
1726 sizeof(keys->addrs.v6addrs.dst));
1727 keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1728 keys->ports.src = fl6->fl6_sport;
1729 keys->ports.dst = fl6->fl6_dport;
1730 keys->keyid.keyid = fl6->fl6_gre_key;
fa1be7e0 1731 keys->tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
a17ace95
DM		 1732 keys->basic.ip_proto = fl6->flowi6_proto;
1733
1734 return flow_hash_from_keys(keys);
1735}
1736EXPORT_SYMBOL(__get_hash_from_flowi6);
1737
06635a35 1738static const struct flow_dissector_key flow_keys_dissector_keys[] = {
42aecaa9
TH		 1739 {
1740 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1741 .offset = offsetof(struct flow_keys, control),
1742 },
06635a35
JP		 1743 {
1744 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1745 .offset = offsetof(struct flow_keys, basic),
1746 },
1747 {
1748 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
c3f83241
TH		 1749 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1750 },
1751 {
1752 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1753 .offset = offsetof(struct flow_keys, addrs.v6addrs),
06635a35 1754 },
9f249089 1755 {
8d6e79d3
JM		 1756 .key_id = FLOW_DISSECTOR_KEY_TIPC,
1757 .offset = offsetof(struct flow_keys, addrs.tipckey),
9f249089 1758 },
06635a35
JP		 1759 {
1760 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1761 .offset = offsetof(struct flow_keys, ports),
1762 },
d34af823 1763 {
f6a66927
HHZ		 1764 .key_id = FLOW_DISSECTOR_KEY_VLAN,
1765 .offset = offsetof(struct flow_keys, vlan),
d34af823 1766 },
87ee9e52
TH		 1767 {
1768 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL,
1769 .offset = offsetof(struct flow_keys, tags),
1770 },
1fdd512c
TH		 1771 {
1772 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
1773 .offset = offsetof(struct flow_keys, keyid),
1774 },
06635a35
JP		 1775};
1776
eb70db87
DM		 1777static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = {
1778 {
1779 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1780 .offset = offsetof(struct flow_keys, control),
1781 },
1782 {
1783 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1784 .offset = offsetof(struct flow_keys, basic),
1785 },
1786 {
1787 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1788 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1789 },
1790 {
1791 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1792 .offset = offsetof(struct flow_keys, addrs.v6addrs),
1793 },
1794 {
1795 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1796 .offset = offsetof(struct flow_keys, ports),
1797 },
1798};
1799
72a338bc 1800static const struct flow_dissector_key flow_keys_basic_dissector_keys[] = {
42aecaa9
TH		 1801 {
1802 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1803 .offset = offsetof(struct flow_keys, control),
1804 },
06635a35
JP		 1805 {
1806 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1807 .offset = offsetof(struct flow_keys, basic),
1808 },
1809};
1810
1811struct flow_dissector flow_keys_dissector __read_mostly;
1812EXPORT_SYMBOL(flow_keys_dissector);
1813
72a338bc
PA		 1814struct flow_dissector flow_keys_basic_dissector __read_mostly;
1815EXPORT_SYMBOL(flow_keys_basic_dissector);
06635a35
JP		 1816
1817static int __init init_default_flow_dissectors(void)
1818{
1819 skb_flow_dissector_init(&flow_keys_dissector,
1820 flow_keys_dissector_keys,
1821 ARRAY_SIZE(flow_keys_dissector_keys));
eb70db87
DM		 1822 skb_flow_dissector_init(&flow_keys_dissector_symmetric,
1823 flow_keys_dissector_symmetric_keys,
1824 ARRAY_SIZE(flow_keys_dissector_symmetric_keys));
72a338bc
PA		 1825 skb_flow_dissector_init(&flow_keys_basic_dissector,
1826 flow_keys_basic_dissector_keys,
1827 ARRAY_SIZE(flow_keys_basic_dissector_keys));
b27f7bb5 1828 return 0;
5cf65922 1829}
c9b8af13 1830core_initcall(init_default_flow_dissectors);
Linux 6.x block layer and io_uring tree(s)