net/core/flow_dissector.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/kernel.h>
   3 #include <linux/skbuff.h>
   4 #include <linux/export.h>
   5 #include <linux/ip.h>
   6 #include <linux/ipv6.h>
   7 #include <linux/if_vlan.h>
   8 #include <linux/filter.h>
   9 #include <net/dsa.h>
  10 #include <net/dst_metadata.h>
  11 #include <net/ip.h>
  12 #include <net/ipv6.h>
  13 #include <net/gre.h>
  14 #include <net/pptp.h>
  15 #include <net/tipc.h>
  16 #include <linux/igmp.h>
  17 #include <linux/icmp.h>
  18 #include <linux/sctp.h>
  19 #include <linux/dccp.h>
  20 #include <linux/if_tunnel.h>
  21 #include <linux/if_pppox.h>
  22 #include <linux/ppp_defs.h>
  23 #include <linux/stddef.h>
  24 #include <linux/if_ether.h>
  25 #include <linux/if_hsr.h>
  26 #include <linux/mpls.h>
  27 #include <linux/tcp.h>
  28 #include <linux/ptp_classify.h>
  29 #include <net/flow_dissector.h>
  30 #include <net/pkt_cls.h>
  31 #include <scsi/fc/fc_fcoe.h>
  32 #include <uapi/linux/batadv_packet.h>
  33 #include <linux/bpf.h>
  34 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
  35 #include <net/netfilter/nf_conntrack_core.h>
  36 #include <net/netfilter/nf_conntrack_labels.h>
  37 #endif
  38 #include <linux/bpf-netns.h>
  39
  40 static void dissector_set_key(struct flow_dissector *flow_dissector,
  41                               enum flow_dissector_key_id key_id)
  42 {
  43         flow_dissector->used_keys |= (1 << key_id);
  44 }
  45
  46 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
  47                              const struct flow_dissector_key *key,
  48                              unsigned int key_count)
  49 {
  50         unsigned int i;
  51
  52         memset(flow_dissector, 0, sizeof(*flow_dissector));
  53
  54         for (i = 0; i < key_count; i++, key++) {
  55                 /* User should make sure that every key target offset is within
  56                  * boundaries of unsigned short.
  57                  */
  58                 BUG_ON(key->offset > USHRT_MAX);
  59                 BUG_ON(dissector_uses_key(flow_dissector,
  60                                           key->key_id));
  61
  62                 dissector_set_key(flow_dissector, key->key_id);
  63                 flow_dissector->offset[key->key_id] = key->offset;
  64         }
  65
  66         /* Ensure that the dissector always includes control and basic key.
  67          * That way we are able to avoid handling lack of these in fast path.
  68          */
  69         BUG_ON(!dissector_uses_key(flow_dissector,
  70                                    FLOW_DISSECTOR_KEY_CONTROL));
  71         BUG_ON(!dissector_uses_key(flow_dissector,
  72                                    FLOW_DISSECTOR_KEY_BASIC));
  73 }
  74 EXPORT_SYMBOL(skb_flow_dissector_init);
  75
  76 #ifdef CONFIG_BPF_SYSCALL
  77 int flow_dissector_bpf_prog_attach_check(struct net *net,
  78                                          struct bpf_prog *prog)
  79 {
  80         enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
  81
  82         if (net == &init_net) {
  83                 /* BPF flow dissector in the root namespace overrides
  84                  * any per-net-namespace one. When attaching to root,
  85                  * make sure we don't have any BPF program attached
  86                  * to the non-root namespaces.
  87                  */
  88                 struct net *ns;
  89
  90                 for_each_net(ns) {
  91                         if (ns == &init_net)
  92                                 continue;
  93                         if (rcu_access_pointer(ns->bpf.run_array[type]))
  94                                 return -EEXIST;
  95                 }
  96         } else {
  97                 /* Make sure root flow dissector is not attached
  98                  * when attaching to the non-root namespace.
  99                  */
 100                 if (rcu_access_pointer(init_net.bpf.run_array[type]))
 101                         return -EEXIST;
 102         }
 103
 104         return 0;
 105 }
 106 #endif /* CONFIG_BPF_SYSCALL */
 107
 108 /**
 109  * __skb_flow_get_ports - extract the upper layer ports and return them
 110  * @skb: sk_buff to extract the ports from
 111  * @thoff: transport header offset
 112  * @ip_proto: protocol for which to get port offset
 113  * @data: raw buffer pointer to the packet, if NULL use skb->data
 114  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 115  *
 116  * The function will try to retrieve the ports at offset thoff + poff where poff
 117  * is the protocol port offset returned from proto_ports_offset
 118  */
 119 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
 120                             const void *data, int hlen)
 121 {
 122         int poff = proto_ports_offset(ip_proto);
 123
 124         if (!data) {
 125                 data = skb->data;
 126                 hlen = skb_headlen(skb);
 127         }
 128
 129         if (poff >= 0) {
 130                 __be32 *ports, _ports;
 131
 132                 ports = __skb_header_pointer(skb, thoff + poff,
 133                                              sizeof(_ports), data, hlen, &_ports);
 134                 if (ports)
 135                         return *ports;
 136         }
 137
 138         return 0;
 139 }
 140 EXPORT_SYMBOL(__skb_flow_get_ports);
 141
 142 static bool icmp_has_id(u8 type)
 143 {
 144         switch (type) {
 145         case ICMP_ECHO:
 146         case ICMP_ECHOREPLY:
 147         case ICMP_TIMESTAMP:
 148         case ICMP_TIMESTAMPREPLY:
 149         case ICMPV6_ECHO_REQUEST:
 150         case ICMPV6_ECHO_REPLY:
 151                 return true;
 152         }
 153
 154         return false;
 155 }
 156
 157 /**
 158  * skb_flow_get_icmp_tci - extract ICMP(6) Type, Code and Identifier fields
 159  * @skb: sk_buff to extract from
 160  * @key_icmp: struct flow_dissector_key_icmp to fill
 161  * @data: raw buffer pointer to the packet
 162  * @thoff: offset to extract at
 163  * @hlen: packet header length
 164  */
 165 void skb_flow_get_icmp_tci(const struct sk_buff *skb,
 166                            struct flow_dissector_key_icmp *key_icmp,
 167                            const void *data, int thoff, int hlen)
 168 {
 169         struct icmphdr *ih, _ih;
 170
 171         ih = __skb_header_pointer(skb, thoff, sizeof(_ih), data, hlen, &_ih);
 172         if (!ih)
 173                 return;
 174
 175         key_icmp->type = ih->type;
 176         key_icmp->code = ih->code;
 177
 178         /* As we use 0 to signal that the Id field is not present,
 179          * avoid confusion with packets without such field
 180          */
 181         if (icmp_has_id(ih->type))
 182                 key_icmp->id = ih->un.echo.id ? ntohs(ih->un.echo.id) : 1;
 183         else
 184                 key_icmp->id = 0;
 185 }
 186 EXPORT_SYMBOL(skb_flow_get_icmp_tci);
 187
 188 /* If FLOW_DISSECTOR_KEY_ICMP is set, dissect an ICMP packet
 189  * using skb_flow_get_icmp_tci().
 190  */
 191 static void __skb_flow_dissect_icmp(const struct sk_buff *skb,
 192                                     struct flow_dissector *flow_dissector,
 193                                     void *target_container, const void *data,
 194                                     int thoff, int hlen)
 195 {
 196         struct flow_dissector_key_icmp *key_icmp;
 197
 198         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ICMP))
 199                 return;
 200
 201         key_icmp = skb_flow_dissector_target(flow_dissector,
 202                                              FLOW_DISSECTOR_KEY_ICMP,
 203                                              target_container);
 204
 205         skb_flow_get_icmp_tci(skb, key_icmp, data, thoff, hlen);
 206 }
 207
 208 static void __skb_flow_dissect_l2tpv3(const struct sk_buff *skb,
 209                                       struct flow_dissector *flow_dissector,
 210                                       void *target_container, const void *data,
 211                                       int nhoff, int hlen)
 212 {
 213         struct flow_dissector_key_l2tpv3 *key_l2tpv3;
 214         struct {
 215                 __be32 session_id;
 216         } *hdr, _hdr;
 217
 218         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_L2TPV3))
 219                 return;
 220
 221         hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 222         if (!hdr)
 223                 return;
 224
 225         key_l2tpv3 = skb_flow_dissector_target(flow_dissector,
 226                                                FLOW_DISSECTOR_KEY_L2TPV3,
 227                                                target_container);
 228
 229         key_l2tpv3->session_id = hdr->session_id;
 230 }
 231
 232 void skb_flow_dissect_meta(const struct sk_buff *skb,
 233                            struct flow_dissector *flow_dissector,
 234                            void *target_container)
 235 {
 236         struct flow_dissector_key_meta *meta;
 237
 238         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_META))
 239                 return;
 240
 241         meta = skb_flow_dissector_target(flow_dissector,
 242                                          FLOW_DISSECTOR_KEY_META,
 243                                          target_container);
 244         meta->ingress_ifindex = skb->skb_iif;
 245 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
 246         if (tc_skb_ext_tc_enabled()) {
 247                 struct tc_skb_ext *ext;
 248
 249                 ext = skb_ext_find(skb, TC_SKB_EXT);
 250                 if (ext)
 251                         meta->l2_miss = ext->l2_miss;
 252         }
 253 #endif
 254 }
 255 EXPORT_SYMBOL(skb_flow_dissect_meta);
 256
 257 static void
 258 skb_flow_dissect_set_enc_addr_type(enum flow_dissector_key_id type,
 259                                    struct flow_dissector *flow_dissector,
 260                                    void *target_container)
 261 {
 262         struct flow_dissector_key_control *ctrl;
 263
 264         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL))
 265                 return;
 266
 267         ctrl = skb_flow_dissector_target(flow_dissector,
 268                                          FLOW_DISSECTOR_KEY_ENC_CONTROL,
 269                                          target_container);
 270         ctrl->addr_type = type;
 271 }
 272
 273 void
 274 skb_flow_dissect_ct(const struct sk_buff *skb,
 275                     struct flow_dissector *flow_dissector,
 276                     void *target_container, u16 *ctinfo_map,
 277                     size_t mapsize, bool post_ct, u16 zone)
 278 {
 279 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
 280         struct flow_dissector_key_ct *key;
 281         enum ip_conntrack_info ctinfo;
 282         struct nf_conn_labels *cl;
 283         struct nf_conn *ct;
 284
 285         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_CT))
 286                 return;
 287
 288         ct = nf_ct_get(skb, &ctinfo);
 289         if (!ct && !post_ct)
 290                 return;
 291
 292         key = skb_flow_dissector_target(flow_dissector,
 293                                         FLOW_DISSECTOR_KEY_CT,
 294                                         target_container);
 295
 296         if (!ct) {
 297                 key->ct_state = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
 298                                 TCA_FLOWER_KEY_CT_FLAGS_INVALID;
 299                 key->ct_zone = zone;
 300                 return;
 301         }
 302
 303         if (ctinfo < mapsize)
 304                 key->ct_state = ctinfo_map[ctinfo];
 305 #if IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)
 306         key->ct_zone = ct->zone.id;
 307 #endif
 308 #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
 309         key->ct_mark = READ_ONCE(ct->mark);
 310 #endif
 311
 312         cl = nf_ct_labels_find(ct);
 313         if (cl)
 314                 memcpy(key->ct_labels, cl->bits, sizeof(key->ct_labels));
 315 #endif /* CONFIG_NF_CONNTRACK */
 316 }
 317 EXPORT_SYMBOL(skb_flow_dissect_ct);
 318
 319 void
 320 skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
 321                              struct flow_dissector *flow_dissector,
 322                              void *target_container)
 323 {
 324         struct ip_tunnel_info *info;
 325         struct ip_tunnel_key *key;
 326
 327         /* A quick check to see if there might be something to do. */
 328         if (!dissector_uses_key(flow_dissector,
 329                                 FLOW_DISSECTOR_KEY_ENC_KEYID) &&
 330             !dissector_uses_key(flow_dissector,
 331                                 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) &&
 332             !dissector_uses_key(flow_dissector,
 333                                 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) &&
 334             !dissector_uses_key(flow_dissector,
 335                                 FLOW_DISSECTOR_KEY_ENC_CONTROL) &&
 336             !dissector_uses_key(flow_dissector,
 337                                 FLOW_DISSECTOR_KEY_ENC_PORTS) &&
 338             !dissector_uses_key(flow_dissector,
 339                                 FLOW_DISSECTOR_KEY_ENC_IP) &&
 340             !dissector_uses_key(flow_dissector,
 341                                 FLOW_DISSECTOR_KEY_ENC_OPTS))
 342                 return;
 343
 344         info = skb_tunnel_info(skb);
 345         if (!info)
 346                 return;
 347
 348         key = &info->key;
 349
 350         switch (ip_tunnel_info_af(info)) {
 351         case AF_INET:
 352                 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 353                                                    flow_dissector,
 354                                                    target_container);
 355                 if (dissector_uses_key(flow_dissector,
 356                                        FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
 357                         struct flow_dissector_key_ipv4_addrs *ipv4;
 358
 359                         ipv4 = skb_flow_dissector_target(flow_dissector,
 360                                                          FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
 361                                                          target_container);
 362                         ipv4->src = key->u.ipv4.src;
 363                         ipv4->dst = key->u.ipv4.dst;
 364                 }
 365                 break;
 366         case AF_INET6:
 367                 skb_flow_dissect_set_enc_addr_type(FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 368                                                    flow_dissector,
 369                                                    target_container);
 370                 if (dissector_uses_key(flow_dissector,
 371                                        FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
 372                         struct flow_dissector_key_ipv6_addrs *ipv6;
 373
 374                         ipv6 = skb_flow_dissector_target(flow_dissector,
 375                                                          FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
 376                                                          target_container);
 377                         ipv6->src = key->u.ipv6.src;
 378                         ipv6->dst = key->u.ipv6.dst;
 379                 }
 380                 break;
 381         }
 382
 383         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
 384                 struct flow_dissector_key_keyid *keyid;
 385
 386                 keyid = skb_flow_dissector_target(flow_dissector,
 387                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
 388                                                   target_container);
 389                 keyid->keyid = tunnel_id_to_key32(key->tun_id);
 390         }
 391
 392         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
 393                 struct flow_dissector_key_ports *tp;
 394
 395                 tp = skb_flow_dissector_target(flow_dissector,
 396                                                FLOW_DISSECTOR_KEY_ENC_PORTS,
 397                                                target_container);
 398                 tp->src = key->tp_src;
 399                 tp->dst = key->tp_dst;
 400         }
 401
 402         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_IP)) {
 403                 struct flow_dissector_key_ip *ip;
 404
 405                 ip = skb_flow_dissector_target(flow_dissector,
 406                                                FLOW_DISSECTOR_KEY_ENC_IP,
 407                                                target_container);
 408                 ip->tos = key->tos;
 409                 ip->ttl = key->ttl;
 410         }
 411
 412         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_OPTS)) {
 413                 struct flow_dissector_key_enc_opts *enc_opt;
 414
 415                 enc_opt = skb_flow_dissector_target(flow_dissector,
 416                                                     FLOW_DISSECTOR_KEY_ENC_OPTS,
 417                                                     target_container);
 418
 419                 if (info->options_len) {
 420                         enc_opt->len = info->options_len;
 421                         ip_tunnel_info_opts_get(enc_opt->data, info);
 422                         enc_opt->dst_opt_type = info->key.tun_flags &
 423                                                 TUNNEL_OPTIONS_PRESENT;
 424                 }
 425         }
 426 }
 427 EXPORT_SYMBOL(skb_flow_dissect_tunnel_info);
 428
 429 void skb_flow_dissect_hash(const struct sk_buff *skb,
 430                            struct flow_dissector *flow_dissector,
 431                            void *target_container)
 432 {
 433         struct flow_dissector_key_hash *key;
 434
 435         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_HASH))
 436                 return;
 437
 438         key = skb_flow_dissector_target(flow_dissector,
 439                                         FLOW_DISSECTOR_KEY_HASH,
 440                                         target_container);
 441
 442         key->hash = skb_get_hash_raw(skb);
 443 }
 444 EXPORT_SYMBOL(skb_flow_dissect_hash);
 445
 446 static enum flow_dissect_ret
 447 __skb_flow_dissect_mpls(const struct sk_buff *skb,
 448                         struct flow_dissector *flow_dissector,
 449                         void *target_container, const void *data, int nhoff,
 450                         int hlen, int lse_index, bool *entropy_label)
 451 {
 452         struct mpls_label *hdr, _hdr;
 453         u32 entry, label, bos;
 454
 455         if (!dissector_uses_key(flow_dissector,
 456                                 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) &&
 457             !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS))
 458                 return FLOW_DISSECT_RET_OUT_GOOD;
 459
 460         if (lse_index >= FLOW_DIS_MPLS_MAX)
 461                 return FLOW_DISSECT_RET_OUT_GOOD;
 462
 463         hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
 464                                    hlen, &_hdr);
 465         if (!hdr)
 466                 return FLOW_DISSECT_RET_OUT_BAD;
 467
 468         entry = ntohl(hdr->entry);
 469         label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT;
 470         bos = (entry & MPLS_LS_S_MASK) >> MPLS_LS_S_SHIFT;
 471
 472         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) {
 473                 struct flow_dissector_key_mpls *key_mpls;
 474                 struct flow_dissector_mpls_lse *lse;
 475
 476                 key_mpls = skb_flow_dissector_target(flow_dissector,
 477                                                      FLOW_DISSECTOR_KEY_MPLS,
 478                                                      target_container);
 479                 lse = &key_mpls->ls[lse_index];
 480
 481                 lse->mpls_ttl = (entry & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
 482                 lse->mpls_bos = bos;
 483                 lse->mpls_tc = (entry & MPLS_LS_TC_MASK) >> MPLS_LS_TC_SHIFT;
 484                 lse->mpls_label = label;
 485                 dissector_set_mpls_lse(key_mpls, lse_index);
 486         }
 487
 488         if (*entropy_label &&
 489             dissector_uses_key(flow_dissector,
 490                                FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) {
 491                 struct flow_dissector_key_keyid *key_keyid;
 492
 493                 key_keyid = skb_flow_dissector_target(flow_dissector,
 494                                                       FLOW_DISSECTOR_KEY_MPLS_ENTROPY,
 495                                                       target_container);
 496                 key_keyid->keyid = cpu_to_be32(label);
 497         }
 498
 499         *entropy_label = label == MPLS_LABEL_ENTROPY;
 500
 501         return bos ? FLOW_DISSECT_RET_OUT_GOOD : FLOW_DISSECT_RET_PROTO_AGAIN;
 502 }
 503
 504 static enum flow_dissect_ret
 505 __skb_flow_dissect_arp(const struct sk_buff *skb,
 506                        struct flow_dissector *flow_dissector,
 507                        void *target_container, const void *data,
 508                        int nhoff, int hlen)
 509 {
 510         struct flow_dissector_key_arp *key_arp;
 511         struct {
 512                 unsigned char ar_sha[ETH_ALEN];
 513                 unsigned char ar_sip[4];
 514                 unsigned char ar_tha[ETH_ALEN];
 515                 unsigned char ar_tip[4];
 516         } *arp_eth, _arp_eth;
 517         const struct arphdr *arp;
 518         struct arphdr _arp;
 519
 520         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP))
 521                 return FLOW_DISSECT_RET_OUT_GOOD;
 522
 523         arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data,
 524                                    hlen, &_arp);
 525         if (!arp)
 526                 return FLOW_DISSECT_RET_OUT_BAD;
 527
 528         if (arp->ar_hrd != htons(ARPHRD_ETHER) ||
 529             arp->ar_pro != htons(ETH_P_IP) ||
 530             arp->ar_hln != ETH_ALEN ||
 531             arp->ar_pln != 4 ||
 532             (arp->ar_op != htons(ARPOP_REPLY) &&
 533              arp->ar_op != htons(ARPOP_REQUEST)))
 534                 return FLOW_DISSECT_RET_OUT_BAD;
 535
 536         arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp),
 537                                        sizeof(_arp_eth), data,
 538                                        hlen, &_arp_eth);
 539         if (!arp_eth)
 540                 return FLOW_DISSECT_RET_OUT_BAD;
 541
 542         key_arp = skb_flow_dissector_target(flow_dissector,
 543                                             FLOW_DISSECTOR_KEY_ARP,
 544                                             target_container);
 545
 546         memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip));
 547         memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip));
 548
 549         /* Only store the lower byte of the opcode;
 550          * this covers ARPOP_REPLY and ARPOP_REQUEST.
 551          */
 552         key_arp->op = ntohs(arp->ar_op) & 0xff;
 553
 554         ether_addr_copy(key_arp->sha, arp_eth->ar_sha);
 555         ether_addr_copy(key_arp->tha, arp_eth->ar_tha);
 556
 557         return FLOW_DISSECT_RET_OUT_GOOD;
 558 }
 559
 560 static enum flow_dissect_ret
 561 __skb_flow_dissect_gre(const struct sk_buff *skb,
 562                        struct flow_dissector_key_control *key_control,
 563                        struct flow_dissector *flow_dissector,
 564                        void *target_container, const void *data,
 565                        __be16 *p_proto, int *p_nhoff, int *p_hlen,
 566                        unsigned int flags)
 567 {
 568         struct flow_dissector_key_keyid *key_keyid;
 569         struct gre_base_hdr *hdr, _hdr;
 570         int offset = 0;
 571         u16 gre_ver;
 572
 573         hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr),
 574                                    data, *p_hlen, &_hdr);
 575         if (!hdr)
 576                 return FLOW_DISSECT_RET_OUT_BAD;
 577
 578         /* Only look inside GRE without routing */
 579         if (hdr->flags & GRE_ROUTING)
 580                 return FLOW_DISSECT_RET_OUT_GOOD;
 581
 582         /* Only look inside GRE for version 0 and 1 */
 583         gre_ver = ntohs(hdr->flags & GRE_VERSION);
 584         if (gre_ver > 1)
 585                 return FLOW_DISSECT_RET_OUT_GOOD;
 586
 587         *p_proto = hdr->protocol;
 588         if (gre_ver) {
 589                 /* Version1 must be PPTP, and check the flags */
 590                 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY)))
 591                         return FLOW_DISSECT_RET_OUT_GOOD;
 592         }
 593
 594         offset += sizeof(struct gre_base_hdr);
 595
 596         if (hdr->flags & GRE_CSUM)
 597                 offset += sizeof_field(struct gre_full_hdr, csum) +
 598                           sizeof_field(struct gre_full_hdr, reserved1);
 599
 600         if (hdr->flags & GRE_KEY) {
 601                 const __be32 *keyid;
 602                 __be32 _keyid;
 603
 604                 keyid = __skb_header_pointer(skb, *p_nhoff + offset,
 605                                              sizeof(_keyid),
 606                                              data, *p_hlen, &_keyid);
 607                 if (!keyid)
 608                         return FLOW_DISSECT_RET_OUT_BAD;
 609
 610                 if (dissector_uses_key(flow_dissector,
 611                                        FLOW_DISSECTOR_KEY_GRE_KEYID)) {
 612                         key_keyid = skb_flow_dissector_target(flow_dissector,
 613                                                               FLOW_DISSECTOR_KEY_GRE_KEYID,
 614                                                               target_container);
 615                         if (gre_ver == 0)
 616                                 key_keyid->keyid = *keyid;
 617                         else
 618                                 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK;
 619                 }
 620                 offset += sizeof_field(struct gre_full_hdr, key);
 621         }
 622
 623         if (hdr->flags & GRE_SEQ)
 624                 offset += sizeof_field(struct pptp_gre_header, seq);
 625
 626         if (gre_ver == 0) {
 627                 if (*p_proto == htons(ETH_P_TEB)) {
 628                         const struct ethhdr *eth;
 629                         struct ethhdr _eth;
 630
 631                         eth = __skb_header_pointer(skb, *p_nhoff + offset,
 632                                                    sizeof(_eth),
 633                                                    data, *p_hlen, &_eth);
 634                         if (!eth)
 635                                 return FLOW_DISSECT_RET_OUT_BAD;
 636                         *p_proto = eth->h_proto;
 637                         offset += sizeof(*eth);
 638
 639                         /* Cap headers that we access via pointers at the
 640                          * end of the Ethernet header as our maximum alignment
 641                          * at that point is only 2 bytes.
 642                          */
 643                         if (NET_IP_ALIGN)
 644                                 *p_hlen = *p_nhoff + offset;
 645                 }
 646         } else { /* version 1, must be PPTP */
 647                 u8 _ppp_hdr[PPP_HDRLEN];
 648                 u8 *ppp_hdr;
 649
 650                 if (hdr->flags & GRE_ACK)
 651                         offset += sizeof_field(struct pptp_gre_header, ack);
 652
 653                 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset,
 654                                                sizeof(_ppp_hdr),
 655                                                data, *p_hlen, _ppp_hdr);
 656                 if (!ppp_hdr)
 657                         return FLOW_DISSECT_RET_OUT_BAD;
 658
 659                 switch (PPP_PROTOCOL(ppp_hdr)) {
 660                 case PPP_IP:
 661                         *p_proto = htons(ETH_P_IP);
 662                         break;
 663                 case PPP_IPV6:
 664                         *p_proto = htons(ETH_P_IPV6);
 665                         break;
 666                 default:
 667                         /* Could probably catch some more like MPLS */
 668                         break;
 669                 }
 670
 671                 offset += PPP_HDRLEN;
 672         }
 673
 674         *p_nhoff += offset;
 675         key_control->flags |= FLOW_DIS_ENCAPSULATION;
 676         if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
 677                 return FLOW_DISSECT_RET_OUT_GOOD;
 678
 679         return FLOW_DISSECT_RET_PROTO_AGAIN;
 680 }
 681
 682 /**
 683  * __skb_flow_dissect_batadv() - dissect batman-adv header
 684  * @skb: sk_buff to with the batman-adv header
 685  * @key_control: flow dissectors control key
 686  * @data: raw buffer pointer to the packet, if NULL use skb->data
 687  * @p_proto: pointer used to update the protocol to process next
 688  * @p_nhoff: pointer used to update inner network header offset
 689  * @hlen: packet header length
 690  * @flags: any combination of FLOW_DISSECTOR_F_*
 691  *
 692  * ETH_P_BATMAN packets are tried to be dissected. Only
 693  * &struct batadv_unicast packets are actually processed because they contain an
 694  * inner ethernet header and are usually followed by actual network header. This
 695  * allows the flow dissector to continue processing the packet.
 696  *
 697  * Return: FLOW_DISSECT_RET_PROTO_AGAIN when &struct batadv_unicast was found,
 698  *  FLOW_DISSECT_RET_OUT_GOOD when dissector should stop after encapsulation,
 699  *  otherwise FLOW_DISSECT_RET_OUT_BAD
 700  */
 701 static enum flow_dissect_ret
 702 __skb_flow_dissect_batadv(const struct sk_buff *skb,
 703                           struct flow_dissector_key_control *key_control,
 704                           const void *data, __be16 *p_proto, int *p_nhoff,
 705                           int hlen, unsigned int flags)
 706 {
 707         struct {
 708                 struct batadv_unicast_packet batadv_unicast;
 709                 struct ethhdr eth;
 710         } *hdr, _hdr;
 711
 712         hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), data, hlen,
 713                                    &_hdr);
 714         if (!hdr)
 715                 return FLOW_DISSECT_RET_OUT_BAD;
 716
 717         if (hdr->batadv_unicast.version != BATADV_COMPAT_VERSION)
 718                 return FLOW_DISSECT_RET_OUT_BAD;
 719
 720         if (hdr->batadv_unicast.packet_type != BATADV_UNICAST)
 721                 return FLOW_DISSECT_RET_OUT_BAD;
 722
 723         *p_proto = hdr->eth.h_proto;
 724         *p_nhoff += sizeof(*hdr);
 725
 726         key_control->flags |= FLOW_DIS_ENCAPSULATION;
 727         if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
 728                 return FLOW_DISSECT_RET_OUT_GOOD;
 729
 730         return FLOW_DISSECT_RET_PROTO_AGAIN;
 731 }
 732
 733 static void
 734 __skb_flow_dissect_tcp(const struct sk_buff *skb,
 735                        struct flow_dissector *flow_dissector,
 736                        void *target_container, const void *data,
 737                        int thoff, int hlen)
 738 {
 739         struct flow_dissector_key_tcp *key_tcp;
 740         struct tcphdr *th, _th;
 741
 742         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP))
 743                 return;
 744
 745         th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th);
 746         if (!th)
 747                 return;
 748
 749         if (unlikely(__tcp_hdrlen(th) < sizeof(_th)))
 750                 return;
 751
 752         key_tcp = skb_flow_dissector_target(flow_dissector,
 753                                             FLOW_DISSECTOR_KEY_TCP,
 754                                             target_container);
 755         key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF));
 756 }
 757
 758 static void
 759 __skb_flow_dissect_ports(const struct sk_buff *skb,
 760                          struct flow_dissector *flow_dissector,
 761                          void *target_container, const void *data,
 762                          int nhoff, u8 ip_proto, int hlen)
 763 {
 764         enum flow_dissector_key_id dissector_ports = FLOW_DISSECTOR_KEY_MAX;
 765         struct flow_dissector_key_ports *key_ports;
 766
 767         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS))
 768                 dissector_ports = FLOW_DISSECTOR_KEY_PORTS;
 769         else if (dissector_uses_key(flow_dissector,
 770                                     FLOW_DISSECTOR_KEY_PORTS_RANGE))
 771                 dissector_ports = FLOW_DISSECTOR_KEY_PORTS_RANGE;
 772
 773         if (dissector_ports == FLOW_DISSECTOR_KEY_MAX)
 774                 return;
 775
 776         key_ports = skb_flow_dissector_target(flow_dissector,
 777                                               dissector_ports,
 778                                               target_container);
 779         key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
 780                                                 data, hlen);
 781 }
 782
 783 static void
 784 __skb_flow_dissect_ipv4(const struct sk_buff *skb,
 785                         struct flow_dissector *flow_dissector,
 786                         void *target_container, const void *data,
 787                         const struct iphdr *iph)
 788 {
 789         struct flow_dissector_key_ip *key_ip;
 790
 791         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
 792                 return;
 793
 794         key_ip = skb_flow_dissector_target(flow_dissector,
 795                                            FLOW_DISSECTOR_KEY_IP,
 796                                            target_container);
 797         key_ip->tos = iph->tos;
 798         key_ip->ttl = iph->ttl;
 799 }
 800
 801 static void
 802 __skb_flow_dissect_ipv6(const struct sk_buff *skb,
 803                         struct flow_dissector *flow_dissector,
 804                         void *target_container, const void *data,
 805                         const struct ipv6hdr *iph)
 806 {
 807         struct flow_dissector_key_ip *key_ip;
 808
 809         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
 810                 return;
 811
 812         key_ip = skb_flow_dissector_target(flow_dissector,
 813                                            FLOW_DISSECTOR_KEY_IP,
 814                                            target_container);
 815         key_ip->tos = ipv6_get_dsfield(iph);
 816         key_ip->ttl = iph->hop_limit;
 817 }
 818
 819 /* Maximum number of protocol headers that can be parsed in
 820  * __skb_flow_dissect
 821  */
 822 #define MAX_FLOW_DISSECT_HDRS   15
 823
 824 static bool skb_flow_dissect_allowed(int *num_hdrs)
 825 {
 826         ++*num_hdrs;
 827
 828         return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS);
 829 }
 830
 831 static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys,
 832                                      struct flow_dissector *flow_dissector,
 833                                      void *target_container)
 834 {
 835         struct flow_dissector_key_ports *key_ports = NULL;
 836         struct flow_dissector_key_control *key_control;
 837         struct flow_dissector_key_basic *key_basic;
 838         struct flow_dissector_key_addrs *key_addrs;
 839         struct flow_dissector_key_tags *key_tags;
 840
 841         key_control = skb_flow_dissector_target(flow_dissector,
 842                                                 FLOW_DISSECTOR_KEY_CONTROL,
 843                                                 target_container);
 844         key_control->thoff = flow_keys->thoff;
 845         if (flow_keys->is_frag)
 846                 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
 847         if (flow_keys->is_first_frag)
 848                 key_control->flags |= FLOW_DIS_FIRST_FRAG;
 849         if (flow_keys->is_encap)
 850                 key_control->flags |= FLOW_DIS_ENCAPSULATION;
 851
 852         key_basic = skb_flow_dissector_target(flow_dissector,
 853                                               FLOW_DISSECTOR_KEY_BASIC,
 854                                               target_container);
 855         key_basic->n_proto = flow_keys->n_proto;
 856         key_basic->ip_proto = flow_keys->ip_proto;
 857
 858         if (flow_keys->addr_proto == ETH_P_IP &&
 859             dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
 860                 key_addrs = skb_flow_dissector_target(flow_dissector,
 861                                                       FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 862                                                       target_container);
 863                 key_addrs->v4addrs.src = flow_keys->ipv4_src;
 864                 key_addrs->v4addrs.dst = flow_keys->ipv4_dst;
 865                 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
 866         } else if (flow_keys->addr_proto == ETH_P_IPV6 &&
 867                    dissector_uses_key(flow_dissector,
 868                                       FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 869                 key_addrs = skb_flow_dissector_target(flow_dissector,
 870                                                       FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 871                                                       target_container);
 872                 memcpy(&key_addrs->v6addrs.src, &flow_keys->ipv6_src,
 873                        sizeof(key_addrs->v6addrs.src));
 874                 memcpy(&key_addrs->v6addrs.dst, &flow_keys->ipv6_dst,
 875                        sizeof(key_addrs->v6addrs.dst));
 876                 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
 877         }
 878
 879         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS))
 880                 key_ports = skb_flow_dissector_target(flow_dissector,
 881                                                       FLOW_DISSECTOR_KEY_PORTS,
 882                                                       target_container);
 883         else if (dissector_uses_key(flow_dissector,
 884                                     FLOW_DISSECTOR_KEY_PORTS_RANGE))
 885                 key_ports = skb_flow_dissector_target(flow_dissector,
 886                                                       FLOW_DISSECTOR_KEY_PORTS_RANGE,
 887                                                       target_container);
 888
 889         if (key_ports) {
 890                 key_ports->src = flow_keys->sport;
 891                 key_ports->dst = flow_keys->dport;
 892         }
 893
 894         if (dissector_uses_key(flow_dissector,
 895                                FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
 896                 key_tags = skb_flow_dissector_target(flow_dissector,
 897                                                      FLOW_DISSECTOR_KEY_FLOW_LABEL,
 898                                                      target_container);
 899                 key_tags->flow_label = ntohl(flow_keys->flow_label);
 900         }
 901 }
 902
 903 u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
 904                      __be16 proto, int nhoff, int hlen, unsigned int flags)
 905 {
 906         struct bpf_flow_keys *flow_keys = ctx->flow_keys;
 907         u32 result;
 908
 909         /* Pass parameters to the BPF program */
 910         memset(flow_keys, 0, sizeof(*flow_keys));
 911         flow_keys->n_proto = proto;
 912         flow_keys->nhoff = nhoff;
 913         flow_keys->thoff = flow_keys->nhoff;
 914
 915         BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG !=
 916                      (int)FLOW_DISSECTOR_F_PARSE_1ST_FRAG);
 917         BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL !=
 918                      (int)FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
 919         BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP !=
 920                      (int)FLOW_DISSECTOR_F_STOP_AT_ENCAP);
 921         flow_keys->flags = flags;
 922
 923         result = bpf_prog_run_pin_on_cpu(prog, ctx);
 924
 925         flow_keys->nhoff = clamp_t(u16, flow_keys->nhoff, nhoff, hlen);
 926         flow_keys->thoff = clamp_t(u16, flow_keys->thoff,
 927                                    flow_keys->nhoff, hlen);
 928
 929         return result;
 930 }
 931
 932 static bool is_pppoe_ses_hdr_valid(const struct pppoe_hdr *hdr)
 933 {
 934         return hdr->ver == 1 && hdr->type == 1 && hdr->code == 0;
 935 }
 936
 937 /**
 938  * __skb_flow_dissect - extract the flow_keys struct and return it
 939  * @net: associated network namespace, derived from @skb if NULL
 940  * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
 941  * @flow_dissector: list of keys to dissect
 942  * @target_container: target structure to put dissected values into
 943  * @data: raw buffer pointer to the packet, if NULL use skb->data
 944  * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
 945  * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
 946  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 947  * @flags: flags that control the dissection process, e.g.
 948  *         FLOW_DISSECTOR_F_STOP_AT_ENCAP.
 949  *
 950  * The function will try to retrieve individual keys into target specified
 951  * by flow_dissector from either the skbuff or a raw buffer specified by the
 952  * rest parameters.
 953  *
 954  * Caller must take care of zeroing target container memory.
 955  */
 956 bool __skb_flow_dissect(const struct net *net,
 957                         const struct sk_buff *skb,
 958                         struct flow_dissector *flow_dissector,
 959                         void *target_container, const void *data,
 960                         __be16 proto, int nhoff, int hlen, unsigned int flags)
 961 {
 962         struct flow_dissector_key_control *key_control;
 963         struct flow_dissector_key_basic *key_basic;
 964         struct flow_dissector_key_addrs *key_addrs;
 965         struct flow_dissector_key_tags *key_tags;
 966         struct flow_dissector_key_vlan *key_vlan;
 967         enum flow_dissect_ret fdret;
 968         enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX;
 969         bool mpls_el = false;
 970         int mpls_lse = 0;
 971         int num_hdrs = 0;
 972         u8 ip_proto = 0;
 973         bool ret;
 974
 975         if (!data) {
 976                 data = skb->data;
 977                 proto = skb_vlan_tag_present(skb) ?
 978                          skb->vlan_proto : skb->protocol;
 979                 nhoff = skb_network_offset(skb);
 980                 hlen = skb_headlen(skb);
 981 #if IS_ENABLED(CONFIG_NET_DSA)
 982                 if (unlikely(skb->dev && netdev_uses_dsa(skb->dev) &&
 983                              proto == htons(ETH_P_XDSA))) {
 984                         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 985                         const struct dsa_device_ops *ops;
 986                         int offset = 0;
 987
 988                         ops = skb->dev->dsa_ptr->tag_ops;
 989                         /* Only DSA header taggers break flow dissection */
 990                         if (ops->needed_headroom &&
 991                             (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)) {
 992                                 if (ops->flow_dissect)
 993                                         ops->flow_dissect(skb, &proto, &offset);
 994                                 else
 995                                         dsa_tag_generic_flow_dissect(skb,
 996                                                                      &proto,
 997                                                                      &offset);
 998                                 hlen -= offset;
 999                                 nhoff += offset;
1000                         }
1001                 }
1002 #endif
1003         }
1004
1005         /* It is ensured by skb_flow_dissector_init() that control key will
1006          * be always present.
1007          */
1008         key_control = skb_flow_dissector_target(flow_dissector,
1009                                                 FLOW_DISSECTOR_KEY_CONTROL,
1010                                                 target_container);
1011
1012         /* It is ensured by skb_flow_dissector_init() that basic key will
1013          * be always present.
1014          */
1015         key_basic = skb_flow_dissector_target(flow_dissector,
1016                                               FLOW_DISSECTOR_KEY_BASIC,
1017                                               target_container);
1018
1019         if (skb) {
1020                 if (!net) {
1021                         if (skb->dev)
1022                                 net = dev_net(skb->dev);
1023                         else if (skb->sk)
1024                                 net = sock_net(skb->sk);
1025                 }
1026         }
1027
1028         WARN_ON_ONCE(!net);
1029         if (net) {
1030                 enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
1031                 struct bpf_prog_array *run_array;
1032
1033                 rcu_read_lock();
1034                 run_array = rcu_dereference(init_net.bpf.run_array[type]);
1035                 if (!run_array)
1036                         run_array = rcu_dereference(net->bpf.run_array[type]);
1037
1038                 if (run_array) {
1039                         struct bpf_flow_keys flow_keys;
1040                         struct bpf_flow_dissector ctx = {
1041                                 .flow_keys = &flow_keys,
1042                                 .data = data,
1043                                 .data_end = data + hlen,
1044                         };
1045                         __be16 n_proto = proto;
1046                         struct bpf_prog *prog;
1047                         u32 result;
1048
1049                         if (skb) {
1050                                 ctx.skb = skb;
1051                                 /* we can't use 'proto' in the skb case
1052                                  * because it might be set to skb->vlan_proto
1053                                  * which has been pulled from the data
1054                                  */
1055                                 n_proto = skb->protocol;
1056                         }
1057
1058                         prog = READ_ONCE(run_array->items[0].prog);
1059                         result = bpf_flow_dissect(prog, &ctx, n_proto, nhoff,
1060                                                   hlen, flags);
1061                         if (result == BPF_FLOW_DISSECTOR_CONTINUE)
1062                                 goto dissect_continue;
1063                         __skb_flow_bpf_to_target(&flow_keys, flow_dissector,
1064                                                  target_container);
1065                         rcu_read_unlock();
1066                         return result == BPF_OK;
1067                 }
1068 dissect_continue:
1069                 rcu_read_unlock();
1070         }
1071
1072         if (dissector_uses_key(flow_dissector,
1073                                FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
1074                 struct ethhdr *eth = eth_hdr(skb);
1075                 struct flow_dissector_key_eth_addrs *key_eth_addrs;
1076
1077                 key_eth_addrs = skb_flow_dissector_target(flow_dissector,
1078                                                           FLOW_DISSECTOR_KEY_ETH_ADDRS,
1079                                                           target_container);
1080                 memcpy(key_eth_addrs, eth, sizeof(*key_eth_addrs));
1081         }
1082
1083         if (dissector_uses_key(flow_dissector,
1084                                FLOW_DISSECTOR_KEY_NUM_OF_VLANS)) {
1085                 struct flow_dissector_key_num_of_vlans *key_num_of_vlans;
1086
1087                 key_num_of_vlans = skb_flow_dissector_target(flow_dissector,
1088                                                              FLOW_DISSECTOR_KEY_NUM_OF_VLANS,
1089                                                              target_container);
1090                 key_num_of_vlans->num_of_vlans = 0;
1091         }
1092
1093 proto_again:
1094         fdret = FLOW_DISSECT_RET_CONTINUE;
1095
1096         switch (proto) {
1097         case htons(ETH_P_IP): {
1098                 const struct iphdr *iph;
1099                 struct iphdr _iph;
1100
1101                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
1102                 if (!iph || iph->ihl < 5) {
1103                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1104                         break;
1105                 }
1106
1107                 nhoff += iph->ihl * 4;
1108
1109                 ip_proto = iph->protocol;
1110
1111                 if (dissector_uses_key(flow_dissector,
1112                                        FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
1113                         key_addrs = skb_flow_dissector_target(flow_dissector,
1114                                                               FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1115                                                               target_container);
1116
1117                         memcpy(&key_addrs->v4addrs.src, &iph->saddr,
1118                                sizeof(key_addrs->v4addrs.src));
1119                         memcpy(&key_addrs->v4addrs.dst, &iph->daddr,
1120                                sizeof(key_addrs->v4addrs.dst));
1121                         key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1122                 }
1123
1124                 __skb_flow_dissect_ipv4(skb, flow_dissector,
1125                                         target_container, data, iph);
1126
1127                 if (ip_is_fragment(iph)) {
1128                         key_control->flags |= FLOW_DIS_IS_FRAGMENT;
1129
1130                         if (iph->frag_off & htons(IP_OFFSET)) {
1131                                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1132                                 break;
1133                         } else {
1134                                 key_control->flags |= FLOW_DIS_FIRST_FRAG;
1135                                 if (!(flags &
1136                                       FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) {
1137                                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1138                                         break;
1139                                 }
1140                         }
1141                 }
1142
1143                 break;
1144         }
1145         case htons(ETH_P_IPV6): {
1146                 const struct ipv6hdr *iph;
1147                 struct ipv6hdr _iph;
1148
1149                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
1150                 if (!iph) {
1151                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1152                         break;
1153                 }
1154
1155                 ip_proto = iph->nexthdr;
1156                 nhoff += sizeof(struct ipv6hdr);
1157
1158                 if (dissector_uses_key(flow_dissector,
1159                                        FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
1160                         key_addrs = skb_flow_dissector_target(flow_dissector,
1161                                                               FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1162                                                               target_container);
1163
1164                         memcpy(&key_addrs->v6addrs.src, &iph->saddr,
1165                                sizeof(key_addrs->v6addrs.src));
1166                         memcpy(&key_addrs->v6addrs.dst, &iph->daddr,
1167                                sizeof(key_addrs->v6addrs.dst));
1168                         key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1169                 }
1170
1171                 if ((dissector_uses_key(flow_dissector,
1172                                         FLOW_DISSECTOR_KEY_FLOW_LABEL) ||
1173                      (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) &&
1174                     ip6_flowlabel(iph)) {
1175                         __be32 flow_label = ip6_flowlabel(iph);
1176
1177                         if (dissector_uses_key(flow_dissector,
1178                                                FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
1179                                 key_tags = skb_flow_dissector_target(flow_dissector,
1180                                                                      FLOW_DISSECTOR_KEY_FLOW_LABEL,
1181                                                                      target_container);
1182                                 key_tags->flow_label = ntohl(flow_label);
1183                         }
1184                         if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) {
1185                                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1186                                 break;
1187                         }
1188                 }
1189
1190                 __skb_flow_dissect_ipv6(skb, flow_dissector,
1191                                         target_container, data, iph);
1192
1193                 break;
1194         }
1195         case htons(ETH_P_8021AD):
1196         case htons(ETH_P_8021Q): {
1197                 const struct vlan_hdr *vlan = NULL;
1198                 struct vlan_hdr _vlan;
1199                 __be16 saved_vlan_tpid = proto;
1200
1201                 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX &&
1202                     skb && skb_vlan_tag_present(skb)) {
1203                         proto = skb->protocol;
1204                 } else {
1205                         vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan),
1206                                                     data, hlen, &_vlan);
1207                         if (!vlan) {
1208                                 fdret = FLOW_DISSECT_RET_OUT_BAD;
1209                                 break;
1210                         }
1211
1212                         proto = vlan->h_vlan_encapsulated_proto;
1213                         nhoff += sizeof(*vlan);
1214                 }
1215
1216                 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_NUM_OF_VLANS) &&
1217                     !(key_control->flags & FLOW_DIS_ENCAPSULATION)) {
1218                         struct flow_dissector_key_num_of_vlans *key_nvs;
1219
1220                         key_nvs = skb_flow_dissector_target(flow_dissector,
1221                                                             FLOW_DISSECTOR_KEY_NUM_OF_VLANS,
1222                                                             target_container);
1223                         key_nvs->num_of_vlans++;
1224                 }
1225
1226                 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX) {
1227                         dissector_vlan = FLOW_DISSECTOR_KEY_VLAN;
1228                 } else if (dissector_vlan == FLOW_DISSECTOR_KEY_VLAN) {
1229                         dissector_vlan = FLOW_DISSECTOR_KEY_CVLAN;
1230                 } else {
1231                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1232                         break;
1233                 }
1234
1235                 if (dissector_uses_key(flow_dissector, dissector_vlan)) {
1236                         key_vlan = skb_flow_dissector_target(flow_dissector,
1237                                                              dissector_vlan,
1238                                                              target_container);
1239
1240                         if (!vlan) {
1241                                 key_vlan->vlan_id = skb_vlan_tag_get_id(skb);
1242                                 key_vlan->vlan_priority = skb_vlan_tag_get_prio(skb);
1243                         } else {
1244                                 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) &
1245                                         VLAN_VID_MASK;
1246                                 key_vlan->vlan_priority =
1247                                         (ntohs(vlan->h_vlan_TCI) &
1248                                          VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1249                         }
1250                         key_vlan->vlan_tpid = saved_vlan_tpid;
1251                         key_vlan->vlan_eth_type = proto;
1252                 }
1253
1254                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1255                 break;
1256         }
1257         case htons(ETH_P_PPP_SES): {
1258                 struct {
1259                         struct pppoe_hdr hdr;
1260                         __be16 proto;
1261                 } *hdr, _hdr;
1262                 u16 ppp_proto;
1263
1264                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
1265                 if (!hdr) {
1266                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1267                         break;
1268                 }
1269
1270                 if (!is_pppoe_ses_hdr_valid(&hdr->hdr)) {
1271                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1272                         break;
1273                 }
1274
1275                 /* least significant bit of the most significant octet
1276                  * indicates if protocol field was compressed
1277                  */
1278                 ppp_proto = ntohs(hdr->proto);
1279                 if (ppp_proto & 0x0100) {
1280                         ppp_proto = ppp_proto >> 8;
1281                         nhoff += PPPOE_SES_HLEN - 1;
1282                 } else {
1283                         nhoff += PPPOE_SES_HLEN;
1284                 }
1285
1286                 if (ppp_proto == PPP_IP) {
1287                         proto = htons(ETH_P_IP);
1288                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1289                 } else if (ppp_proto == PPP_IPV6) {
1290                         proto = htons(ETH_P_IPV6);
1291                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1292                 } else if (ppp_proto == PPP_MPLS_UC) {
1293                         proto = htons(ETH_P_MPLS_UC);
1294                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1295                 } else if (ppp_proto == PPP_MPLS_MC) {
1296                         proto = htons(ETH_P_MPLS_MC);
1297                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1298                 } else if (ppp_proto_is_valid(ppp_proto)) {
1299                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1300                 } else {
1301                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1302                         break;
1303                 }
1304
1305                 if (dissector_uses_key(flow_dissector,
1306                                        FLOW_DISSECTOR_KEY_PPPOE)) {
1307                         struct flow_dissector_key_pppoe *key_pppoe;
1308
1309                         key_pppoe = skb_flow_dissector_target(flow_dissector,
1310                                                               FLOW_DISSECTOR_KEY_PPPOE,
1311                                                               target_container);
1312                         key_pppoe->session_id = hdr->hdr.sid;
1313                         key_pppoe->ppp_proto = htons(ppp_proto);
1314                         key_pppoe->type = htons(ETH_P_PPP_SES);
1315                 }
1316                 break;
1317         }
1318         case htons(ETH_P_TIPC): {
1319                 struct tipc_basic_hdr *hdr, _hdr;
1320
1321                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr),
1322                                            data, hlen, &_hdr);
1323                 if (!hdr) {
1324                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1325                         break;
1326                 }
1327
1328                 if (dissector_uses_key(flow_dissector,
1329                                        FLOW_DISSECTOR_KEY_TIPC)) {
1330                         key_addrs = skb_flow_dissector_target(flow_dissector,
1331                                                               FLOW_DISSECTOR_KEY_TIPC,
1332                                                               target_container);
1333                         key_addrs->tipckey.key = tipc_hdr_rps_key(hdr);
1334                         key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC;
1335                 }
1336                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1337                 break;
1338         }
1339
1340         case htons(ETH_P_MPLS_UC):
1341         case htons(ETH_P_MPLS_MC):
1342                 fdret = __skb_flow_dissect_mpls(skb, flow_dissector,
1343                                                 target_container, data,
1344                                                 nhoff, hlen, mpls_lse,
1345                                                 &mpls_el);
1346                 nhoff += sizeof(struct mpls_label);
1347                 mpls_lse++;
1348                 break;
1349         case htons(ETH_P_FCOE):
1350                 if ((hlen - nhoff) < FCOE_HEADER_LEN) {
1351                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1352                         break;
1353                 }
1354
1355                 nhoff += FCOE_HEADER_LEN;
1356                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1357                 break;
1358
1359         case htons(ETH_P_ARP):
1360         case htons(ETH_P_RARP):
1361                 fdret = __skb_flow_dissect_arp(skb, flow_dissector,
1362                                                target_container, data,
1363                                                nhoff, hlen);
1364                 break;
1365
1366         case htons(ETH_P_BATMAN):
1367                 fdret = __skb_flow_dissect_batadv(skb, key_control, data,
1368                                                   &proto, &nhoff, hlen, flags);
1369                 break;
1370
1371         case htons(ETH_P_1588): {
1372                 struct ptp_header *hdr, _hdr;
1373
1374                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
1375                                            hlen, &_hdr);
1376                 if (!hdr) {
1377                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1378                         break;
1379                 }
1380
1381                 nhoff += ntohs(hdr->message_length);
1382                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1383                 break;
1384         }
1385
1386         case htons(ETH_P_PRP):
1387         case htons(ETH_P_HSR): {
1388                 struct hsr_tag *hdr, _hdr;
1389
1390                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen,
1391                                            &_hdr);
1392                 if (!hdr) {
1393                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1394                         break;
1395                 }
1396
1397                 proto = hdr->encap_proto;
1398                 nhoff += HSR_HLEN;
1399                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1400                 break;
1401         }
1402
1403         default:
1404                 fdret = FLOW_DISSECT_RET_OUT_BAD;
1405                 break;
1406         }
1407
1408         /* Process result of proto processing */
1409         switch (fdret) {
1410         case FLOW_DISSECT_RET_OUT_GOOD:
1411                 goto out_good;
1412         case FLOW_DISSECT_RET_PROTO_AGAIN:
1413                 if (skb_flow_dissect_allowed(&num_hdrs))
1414                         goto proto_again;
1415                 goto out_good;
1416         case FLOW_DISSECT_RET_CONTINUE:
1417         case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1418                 break;
1419         case FLOW_DISSECT_RET_OUT_BAD:
1420         default:
1421                 goto out_bad;
1422         }
1423
1424 ip_proto_again:
1425         fdret = FLOW_DISSECT_RET_CONTINUE;
1426
1427         switch (ip_proto) {
1428         case IPPROTO_GRE:
1429                 if (flags & FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP) {
1430                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1431                         break;
1432                 }
1433
1434                 fdret = __skb_flow_dissect_gre(skb, key_control, flow_dissector,
1435                                                target_container, data,
1436                                                &proto, &nhoff, &hlen, flags);
1437                 break;
1438
1439         case NEXTHDR_HOP:
1440         case NEXTHDR_ROUTING:
1441         case NEXTHDR_DEST: {
1442                 u8 _opthdr[2], *opthdr;
1443
1444                 if (proto != htons(ETH_P_IPV6))
1445                         break;
1446
1447                 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr),
1448                                               data, hlen, &_opthdr);
1449                 if (!opthdr) {
1450                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1451                         break;
1452                 }
1453
1454                 ip_proto = opthdr[0];
1455                 nhoff += (opthdr[1] + 1) << 3;
1456
1457                 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1458                 break;
1459         }
1460         case NEXTHDR_FRAGMENT: {
1461                 struct frag_hdr _fh, *fh;
1462
1463                 if (proto != htons(ETH_P_IPV6))
1464                         break;
1465
1466                 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh),
1467                                           data, hlen, &_fh);
1468
1469                 if (!fh) {
1470                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1471                         break;
1472                 }
1473
1474                 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
1475
1476                 nhoff += sizeof(_fh);
1477                 ip_proto = fh->nexthdr;
1478
1479                 if (!(fh->frag_off & htons(IP6_OFFSET))) {
1480                         key_control->flags |= FLOW_DIS_FIRST_FRAG;
1481                         if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) {
1482                                 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1483                                 break;
1484                         }
1485                 }
1486
1487                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1488                 break;
1489         }
1490         case IPPROTO_IPIP:
1491                 if (flags & FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP) {
1492                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1493                         break;
1494                 }
1495
1496                 proto = htons(ETH_P_IP);
1497
1498                 key_control->flags |= FLOW_DIS_ENCAPSULATION;
1499                 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1500                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1501                         break;
1502                 }
1503
1504                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1505                 break;
1506
1507         case IPPROTO_IPV6:
1508                 if (flags & FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP) {
1509                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1510                         break;
1511                 }
1512
1513                 proto = htons(ETH_P_IPV6);
1514
1515                 key_control->flags |= FLOW_DIS_ENCAPSULATION;
1516                 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1517                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1518                         break;
1519                 }
1520
1521                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1522                 break;
1523
1524
1525         case IPPROTO_MPLS:
1526                 proto = htons(ETH_P_MPLS_UC);
1527                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1528                 break;
1529
1530         case IPPROTO_TCP:
1531                 __skb_flow_dissect_tcp(skb, flow_dissector, target_container,
1532                                        data, nhoff, hlen);
1533                 break;
1534
1535         case IPPROTO_ICMP:
1536         case IPPROTO_ICMPV6:
1537                 __skb_flow_dissect_icmp(skb, flow_dissector, target_container,
1538                                         data, nhoff, hlen);
1539                 break;
1540         case IPPROTO_L2TP:
1541                 __skb_flow_dissect_l2tpv3(skb, flow_dissector, target_container,
1542                                           data, nhoff, hlen);
1543                 break;
1544
1545         default:
1546                 break;
1547         }
1548
1549         if (!(key_control->flags & FLOW_DIS_IS_FRAGMENT))
1550                 __skb_flow_dissect_ports(skb, flow_dissector, target_container,
1551                                          data, nhoff, ip_proto, hlen);
1552
1553         /* Process result of IP proto processing */
1554         switch (fdret) {
1555         case FLOW_DISSECT_RET_PROTO_AGAIN:
1556                 if (skb_flow_dissect_allowed(&num_hdrs))
1557                         goto proto_again;
1558                 break;
1559         case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1560                 if (skb_flow_dissect_allowed(&num_hdrs))
1561                         goto ip_proto_again;
1562                 break;
1563         case FLOW_DISSECT_RET_OUT_GOOD:
1564         case FLOW_DISSECT_RET_CONTINUE:
1565                 break;
1566         case FLOW_DISSECT_RET_OUT_BAD:
1567         default:
1568                 goto out_bad;
1569         }
1570
1571 out_good:
1572         ret = true;
1573
1574 out:
1575         key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
1576         key_basic->n_proto = proto;
1577         key_basic->ip_proto = ip_proto;
1578
1579         return ret;
1580
1581 out_bad:
1582         ret = false;
1583         goto out;
1584 }
1585 EXPORT_SYMBOL(__skb_flow_dissect);
1586
1587 static siphash_aligned_key_t hashrnd;
1588 static __always_inline void __flow_hash_secret_init(void)
1589 {
1590         net_get_random_once(&hashrnd, sizeof(hashrnd));
1591 }
1592
1593 static const void *flow_keys_hash_start(const struct flow_keys *flow)
1594 {
1595         BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % SIPHASH_ALIGNMENT);
1596         return &flow->FLOW_KEYS_HASH_START_FIELD;
1597 }
1598
1599 static inline size_t flow_keys_hash_length(const struct flow_keys *flow)
1600 {
1601         size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs);
1602
1603         BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
1604
1605         switch (flow->control.addr_type) {
1606         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1607                 diff -= sizeof(flow->addrs.v4addrs);
1608                 break;
1609         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1610                 diff -= sizeof(flow->addrs.v6addrs);
1611                 break;
1612         case FLOW_DISSECTOR_KEY_TIPC:
1613                 diff -= sizeof(flow->addrs.tipckey);
1614                 break;
1615         }
1616         return sizeof(*flow) - diff;
1617 }
1618
1619 __be32 flow_get_u32_src(const struct flow_keys *flow)
1620 {
1621         switch (flow->control.addr_type) {
1622         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1623                 return flow->addrs.v4addrs.src;
1624         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1625                 return (__force __be32)ipv6_addr_hash(
1626                         &flow->addrs.v6addrs.src);
1627         case FLOW_DISSECTOR_KEY_TIPC:
1628                 return flow->addrs.tipckey.key;
1629         default:
1630                 return 0;
1631         }
1632 }
1633 EXPORT_SYMBOL(flow_get_u32_src);
1634
1635 __be32 flow_get_u32_dst(const struct flow_keys *flow)
1636 {
1637         switch (flow->control.addr_type) {
1638         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1639                 return flow->addrs.v4addrs.dst;
1640         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1641                 return (__force __be32)ipv6_addr_hash(
1642                         &flow->addrs.v6addrs.dst);
1643         default:
1644                 return 0;
1645         }
1646 }
1647 EXPORT_SYMBOL(flow_get_u32_dst);
1648
1649 /* Sort the source and destination IP and the ports,
1650  * to have consistent hash within the two directions
1651  */
1652 static inline void __flow_hash_consistentify(struct flow_keys *keys)
1653 {
1654         int addr_diff, i;
1655
1656         switch (keys->control.addr_type) {
1657         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1658                 if ((__force u32)keys->addrs.v4addrs.dst <
1659                     (__force u32)keys->addrs.v4addrs.src)
1660                         swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst);
1661
1662                 if ((__force u16)keys->ports.dst <
1663                     (__force u16)keys->ports.src) {
1664                         swap(keys->ports.src, keys->ports.dst);
1665                 }
1666                 break;
1667         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1668                 addr_diff = memcmp(&keys->addrs.v6addrs.dst,
1669                                    &keys->addrs.v6addrs.src,
1670                                    sizeof(keys->addrs.v6addrs.dst));
1671                 if (addr_diff < 0) {
1672                         for (i = 0; i < 4; i++)
1673                                 swap(keys->addrs.v6addrs.src.s6_addr32[i],
1674                                      keys->addrs.v6addrs.dst.s6_addr32[i]);
1675                 }
1676                 if ((__force u16)keys->ports.dst <
1677                     (__force u16)keys->ports.src) {
1678                         swap(keys->ports.src, keys->ports.dst);
1679                 }
1680                 break;
1681         }
1682 }
1683
1684 static inline u32 __flow_hash_from_keys(struct flow_keys *keys,
1685                                         const siphash_key_t *keyval)
1686 {
1687         u32 hash;
1688
1689         __flow_hash_consistentify(keys);
1690
1691         hash = siphash(flow_keys_hash_start(keys),
1692                        flow_keys_hash_length(keys), keyval);
1693         if (!hash)
1694                 hash = 1;
1695
1696         return hash;
1697 }
1698
1699 u32 flow_hash_from_keys(struct flow_keys *keys)
1700 {
1701         __flow_hash_secret_init();
1702         return __flow_hash_from_keys(keys, &hashrnd);
1703 }
1704 EXPORT_SYMBOL(flow_hash_from_keys);
1705
1706 static inline u32 ___skb_get_hash(const struct sk_buff *skb,
1707                                   struct flow_keys *keys,
1708                                   const siphash_key_t *keyval)
1709 {
1710         skb_flow_dissect_flow_keys(skb, keys,
1711                                    FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1712
1713         return __flow_hash_from_keys(keys, keyval);
1714 }
1715
1716 struct _flow_keys_digest_data {
1717         __be16  n_proto;
1718         u8      ip_proto;
1719         u8      padding;
1720         __be32  ports;
1721         __be32  src;
1722         __be32  dst;
1723 };
1724
1725 void make_flow_keys_digest(struct flow_keys_digest *digest,
1726                            const struct flow_keys *flow)
1727 {
1728         struct _flow_keys_digest_data *data =
1729             (struct _flow_keys_digest_data *)digest;
1730
1731         BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
1732
1733         memset(digest, 0, sizeof(*digest));
1734
1735         data->n_proto = flow->basic.n_proto;
1736         data->ip_proto = flow->basic.ip_proto;
1737         data->ports = flow->ports.ports;
1738         data->src = flow->addrs.v4addrs.src;
1739         data->dst = flow->addrs.v4addrs.dst;
1740 }
1741 EXPORT_SYMBOL(make_flow_keys_digest);
1742
1743 static struct flow_dissector flow_keys_dissector_symmetric __read_mostly;
1744
1745 u32 __skb_get_hash_symmetric(const struct sk_buff *skb)
1746 {
1747         struct flow_keys keys;
1748
1749         __flow_hash_secret_init();
1750
1751         memset(&keys, 0, sizeof(keys));
1752         __skb_flow_dissect(NULL, skb, &flow_keys_dissector_symmetric,
1753                            &keys, NULL, 0, 0, 0,
1754                            FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1755
1756         return __flow_hash_from_keys(&keys, &hashrnd);
1757 }
1758 EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric);
1759
1760 /**
1761  * __skb_get_hash: calculate a flow hash
1762  * @skb: sk_buff to calculate flow hash from
1763  *
1764  * This function calculates a flow hash based on src/dst addresses
1765  * and src/dst port numbers.  Sets hash in skb to non-zero hash value
1766  * on success, zero indicates no valid hash.  Also, sets l4_hash in skb
1767  * if hash is a canonical 4-tuple hash over transport ports.
1768  */
1769 void __skb_get_hash(struct sk_buff *skb)
1770 {
1771         struct flow_keys keys;
1772         u32 hash;
1773
1774         __flow_hash_secret_init();
1775
1776         hash = ___skb_get_hash(skb, &keys, &hashrnd);
1777
1778         __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1779 }
1780 EXPORT_SYMBOL(__skb_get_hash);
1781
1782 __u32 skb_get_hash_perturb(const struct sk_buff *skb,
1783                            const siphash_key_t *perturb)
1784 {
1785         struct flow_keys keys;
1786
1787         return ___skb_get_hash(skb, &keys, perturb);
1788 }
1789 EXPORT_SYMBOL(skb_get_hash_perturb);
1790
1791 u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
1792                    const struct flow_keys_basic *keys, int hlen)
1793 {
1794         u32 poff = keys->control.thoff;
1795
1796         /* skip L4 headers for fragments after the first */
1797         if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) &&
1798             !(keys->control.flags & FLOW_DIS_FIRST_FRAG))
1799                 return poff;
1800
1801         switch (keys->basic.ip_proto) {
1802         case IPPROTO_TCP: {
1803                 /* access doff as u8 to avoid unaligned access */
1804                 const u8 *doff;
1805                 u8 _doff;
1806
1807                 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
1808                                             data, hlen, &_doff);
1809                 if (!doff)
1810                         return poff;
1811
1812                 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
1813                 break;
1814         }
1815         case IPPROTO_UDP:
1816         case IPPROTO_UDPLITE:
1817                 poff += sizeof(struct udphdr);
1818                 break;
1819         /* For the rest, we do not really care about header
1820          * extensions at this point for now.
1821          */
1822         case IPPROTO_ICMP:
1823                 poff += sizeof(struct icmphdr);
1824                 break;
1825         case IPPROTO_ICMPV6:
1826                 poff += sizeof(struct icmp6hdr);
1827                 break;
1828         case IPPROTO_IGMP:
1829                 poff += sizeof(struct igmphdr);
1830                 break;
1831         case IPPROTO_DCCP:
1832                 poff += sizeof(struct dccp_hdr);
1833                 break;
1834         case IPPROTO_SCTP:
1835                 poff += sizeof(struct sctphdr);
1836                 break;
1837         }
1838
1839         return poff;
1840 }
1841
1842 /**
1843  * skb_get_poff - get the offset to the payload
1844  * @skb: sk_buff to get the payload offset from
1845  *
1846  * The function will get the offset to the payload as far as it could
1847  * be dissected.  The main user is currently BPF, so that we can dynamically
1848  * truncate packets without needing to push actual payload to the user
1849  * space and can analyze headers only, instead.
1850  */
1851 u32 skb_get_poff(const struct sk_buff *skb)
1852 {
1853         struct flow_keys_basic keys;
1854
1855         if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
1856                                               NULL, 0, 0, 0, 0))
1857                 return 0;
1858
1859         return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
1860 }
1861
1862 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys)
1863 {
1864         memset(keys, 0, sizeof(*keys));
1865
1866         memcpy(&keys->addrs.v6addrs.src, &fl6->saddr,
1867             sizeof(keys->addrs.v6addrs.src));
1868         memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr,
1869             sizeof(keys->addrs.v6addrs.dst));
1870         keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1871         keys->ports.src = fl6->fl6_sport;
1872         keys->ports.dst = fl6->fl6_dport;
1873         keys->keyid.keyid = fl6->fl6_gre_key;
1874         keys->tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
1875         keys->basic.ip_proto = fl6->flowi6_proto;
1876
1877         return flow_hash_from_keys(keys);
1878 }
1879 EXPORT_SYMBOL(__get_hash_from_flowi6);
1880
1881 static const struct flow_dissector_key flow_keys_dissector_keys[] = {
1882         {
1883                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1884                 .offset = offsetof(struct flow_keys, control),
1885         },
1886         {
1887                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1888                 .offset = offsetof(struct flow_keys, basic),
1889         },
1890         {
1891                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1892                 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1893         },
1894         {
1895                 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1896                 .offset = offsetof(struct flow_keys, addrs.v6addrs),
1897         },
1898         {
1899                 .key_id = FLOW_DISSECTOR_KEY_TIPC,
1900                 .offset = offsetof(struct flow_keys, addrs.tipckey),
1901         },
1902         {
1903                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1904                 .offset = offsetof(struct flow_keys, ports),
1905         },
1906         {
1907                 .key_id = FLOW_DISSECTOR_KEY_VLAN,
1908                 .offset = offsetof(struct flow_keys, vlan),
1909         },
1910         {
1911                 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL,
1912                 .offset = offsetof(struct flow_keys, tags),
1913         },
1914         {
1915                 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
1916                 .offset = offsetof(struct flow_keys, keyid),
1917         },
1918 };
1919
1920 static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = {
1921         {
1922                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1923                 .offset = offsetof(struct flow_keys, control),
1924         },
1925         {
1926                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1927                 .offset = offsetof(struct flow_keys, basic),
1928         },
1929         {
1930                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1931                 .offset = offsetof(struct flow_keys, addrs.v4addrs),
1932         },
1933         {
1934                 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1935                 .offset = offsetof(struct flow_keys, addrs.v6addrs),
1936         },
1937         {
1938                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
1939                 .offset = offsetof(struct flow_keys, ports),
1940         },
1941 };
1942
1943 static const struct flow_dissector_key flow_keys_basic_dissector_keys[] = {
1944         {
1945                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
1946                 .offset = offsetof(struct flow_keys, control),
1947         },
1948         {
1949                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
1950                 .offset = offsetof(struct flow_keys, basic),
1951         },
1952 };
1953
1954 struct flow_dissector flow_keys_dissector __read_mostly;
1955 EXPORT_SYMBOL(flow_keys_dissector);
1956
1957 struct flow_dissector flow_keys_basic_dissector __read_mostly;
1958 EXPORT_SYMBOL(flow_keys_basic_dissector);
1959
1960 static int __init init_default_flow_dissectors(void)
1961 {
1962         skb_flow_dissector_init(&flow_keys_dissector,
1963                                 flow_keys_dissector_keys,
1964                                 ARRAY_SIZE(flow_keys_dissector_keys));
1965         skb_flow_dissector_init(&flow_keys_dissector_symmetric,
1966                                 flow_keys_dissector_symmetric_keys,
1967                                 ARRAY_SIZE(flow_keys_dissector_symmetric_keys));
1968         skb_flow_dissector_init(&flow_keys_basic_dissector,
1969                                 flow_keys_basic_dissector_keys,
1970                                 ARRAY_SIZE(flow_keys_basic_dissector_keys));
1971         return 0;
1972 }
1973 core_initcall(init_default_flow_dissectors);