net/core/flow_dissector.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/kernel.h>
   3 #include <linux/skbuff.h>
   4 #include <linux/export.h>
   5 #include <linux/ip.h>
   6 #include <linux/ipv6.h>
   7 #include <linux/if_vlan.h>
   8 #include <linux/filter.h>
   9 #include <net/dsa.h>
  10 #include <net/dst_metadata.h>
  11 #include <net/ip.h>
  12 #include <net/ipv6.h>
  13 #include <net/gre.h>
  14 #include <net/pptp.h>
  15 #include <net/tipc.h>
  16 #include <linux/igmp.h>
  17 #include <linux/icmp.h>
  18 #include <linux/sctp.h>
  19 #include <linux/dccp.h>
  20 #include <linux/if_tunnel.h>
  21 #include <linux/if_pppox.h>
  22 #include <linux/ppp_defs.h>
  23 #include <linux/stddef.h>
  24 #include <linux/if_ether.h>
  25 #include <linux/if_hsr.h>
  26 #include <linux/mpls.h>
  27 #include <linux/tcp.h>
  28 #include <linux/ptp_classify.h>
  29 #include <net/flow_dissector.h>
  30 #include <net/pkt_cls.h>
  31 #include <scsi/fc/fc_fcoe.h>
  32 #include <uapi/linux/batadv_packet.h>
  33 #include <linux/bpf.h>
  34 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
  35 #include <net/netfilter/nf_conntrack_core.h>
  36 #include <net/netfilter/nf_conntrack_labels.h>
  37 #endif
  38 #include <linux/bpf-netns.h>
  39
  40 static void dissector_set_key(struct flow_dissector *flow_dissector,
  41                               enum flow_dissector_key_id key_id)
  42 {
  43         flow_dissector->used_keys |= (1ULL << key_id);
  44 }
  45
  46 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
  47                              const struct flow_dissector_key *key,
  48                              unsigned int key_count)
  49 {
  50         unsigned int i;
  51
  52         memset(flow_dissector, 0, sizeof(*flow_dissector));
  53
  54         for (i = 0; i < key_count; i++, key++) {
  55                 /* User should make sure that every key target offset is within
  56                  * boundaries of unsigned short.
  57                  */
  58                 BUG_ON(key->offset > USHRT_MAX);
  59                 BUG_ON(dissector_uses_key(flow_dissector,
  60                                           key->key_id));
  61
  62                 dissector_set_key(flow_dissector, key->key_id);
  63                 flow_dissector->offset[key->key_id] = key->offset;
  64         }
  65
  66         /* Ensure that the dissector always includes control and basic key.
  67          * That way we are able to avoid handling lack of these in fast path.
  68          */
  69         BUG_ON(!dissector_uses_key(flow_dissector,
  70                                    FLOW_DISSECTOR_KEY_CONTROL));
  71         BUG_ON(!dissector_uses_key(flow_dissector,
  72                                    FLOW_DISSECTOR_KEY_BASIC));
  73 }
  74 EXPORT_SYMBOL(skb_flow_dissector_init);
  75
  76 #ifdef CONFIG_BPF_SYSCALL
  77 int flow_dissector_bpf_prog_attach_check(struct net *net,
  78                                          struct bpf_prog *prog)
  79 {
  80         enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
  81
  82         if (net == &init_net) {
  83                 /* BPF flow dissector in the root namespace overrides
  84                  * any per-net-namespace one. When attaching to root,
  85                  * make sure we don't have any BPF program attached
  86                  * to the non-root namespaces.
  87                  */
  88                 struct net *ns;
  89
  90                 for_each_net(ns) {
  91                         if (ns == &init_net)
  92                                 continue;
  93                         if (rcu_access_pointer(ns->bpf.run_array[type]))
  94                                 return -EEXIST;
  95                 }
  96         } else {
  97                 /* Make sure root flow dissector is not attached
  98                  * when attaching to the non-root namespace.
  99                  */
 100                 if (rcu_access_pointer(init_net.bpf.run_array[type]))
 101                         return -EEXIST;
 102         }
 103
 104         return 0;
 105 }
 106 #endif /* CONFIG_BPF_SYSCALL */
 107
 108 /**
 109  * skb_flow_get_ports - extract the upper layer ports and return them
 110  * @skb: sk_buff to extract the ports from
 111  * @thoff: transport header offset
 112  * @ip_proto: protocol for which to get port offset
 113  * @data: raw buffer pointer to the packet, if NULL use skb->data
 114  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 115  *
 116  * The function will try to retrieve the ports at offset thoff + poff where poff
 117  * is the protocol port offset returned from proto_ports_offset
 118  */
 119 __be32 skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
 120                           const void *data, int hlen)
 121 {
 122         int poff = proto_ports_offset(ip_proto);
 123
 124         if (!data) {
 125                 data = skb->data;
 126                 hlen = skb_headlen(skb);
 127         }
 128
 129         if (poff >= 0) {
 130                 __be32 *ports, _ports;
 131
 132                 ports = __skb_header_pointer(skb, thoff + poff,
 133                                              sizeof(_ports), data, hlen, &_ports);
 134                 if (ports)
 135                         return *ports;
 136         }
 137
 138         return 0;
 139 }
 140 EXPORT_SYMBOL(skb_flow_get_ports);
 141
 142 static bool icmp_has_id(u8 type)
 143 {
 144         switch (type) {
 145         case ICMP_ECHO:
 146         case ICMP_ECHOREPLY:
 147         case ICMP_TIMESTAMP:
 148         case ICMP_TIMESTAMPREPLY:
 149         case ICMPV6_ECHO_REQUEST:
 150         case ICMPV6_ECHO_REPLY:
 151                 return true;
 152         }
 153
 154         return false;
 155 }
 156
 157 /**
 158  * skb_flow_get_icmp_tci - extract ICMP(6) Type, Code and Identifier fields
 159  * @skb: sk_buff to extract from
 160  * @key_icmp: struct flow_dissector_key_icmp to fill
 161  * @data: raw buffer pointer to the packet
 162  * @thoff: offset to extract at
 163  * @hlen: packet header length
 164  */
 165 void skb_flow_get_icmp_tci(const struct sk_buff *skb,
 166                            struct flow_dissector_key_icmp *key_icmp,
 167                            const void *data, int thoff, int hlen)
 168 {
 169         struct icmphdr *ih, _ih;
 170
 171         ih = __skb_header_pointer(skb, thoff, sizeof(_ih), data, hlen, &_ih);
 172         if (!ih)
 173                 return;
 174
 175         key_icmp->type = ih->type;
 176         key_icmp->code = ih->code;
 177
 178         /* As we use 0 to signal that the Id field is not present,
 179          * avoid confusion with packets without such field
 180          */
 181         if (icmp_has_id(ih->type))
 182                 key_icmp->id = ih->un.echo.id ? ntohs(ih->un.echo.id) : 1;
 183         else
 184                 key_icmp->id = 0;
 185 }
 186 EXPORT_SYMBOL(skb_flow_get_icmp_tci);
 187
 188 /* If FLOW_DISSECTOR_KEY_ICMP is set, dissect an ICMP packet
 189  * using skb_flow_get_icmp_tci().
 190  */
 191 static void __skb_flow_dissect_icmp(const struct sk_buff *skb,
 192                                     struct flow_dissector *flow_dissector,
 193                                     void *target_container, const void *data,
 194                                     int thoff, int hlen)
 195 {
 196         struct flow_dissector_key_icmp *key_icmp;
 197
 198         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ICMP))
 199                 return;
 200
 201         key_icmp = skb_flow_dissector_target(flow_dissector,
 202                                              FLOW_DISSECTOR_KEY_ICMP,
 203                                              target_container);
 204
 205         skb_flow_get_icmp_tci(skb, key_icmp, data, thoff, hlen);
 206 }
 207
 208 static void __skb_flow_dissect_ah(const struct sk_buff *skb,
 209                                   struct flow_dissector *flow_dissector,
 210                                   void *target_container, const void *data,
 211                                   int nhoff, int hlen)
 212 {
 213         struct flow_dissector_key_ipsec *key_ah;
 214         struct ip_auth_hdr _hdr, *hdr;
 215
 216         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPSEC))
 217                 return;
 218
 219         hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 220         if (!hdr)
 221                 return;
 222
 223         key_ah = skb_flow_dissector_target(flow_dissector,
 224                                            FLOW_DISSECTOR_KEY_IPSEC,
 225                                            target_container);
 226
 227         key_ah->spi = hdr->spi;
 228 }
 229
 230 static void __skb_flow_dissect_esp(const struct sk_buff *skb,
 231                                    struct flow_dissector *flow_dissector,
 232                                    void *target_container, const void *data,
 233                                    int nhoff, int hlen)
 234 {
 235         struct flow_dissector_key_ipsec *key_esp;
 236         struct ip_esp_hdr _hdr, *hdr;
 237
 238         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPSEC))
 239                 return;
 240
 241         hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 242         if (!hdr)
 243                 return;
 244
 245         key_esp = skb_flow_dissector_target(flow_dissector,
 246                                             FLOW_DISSECTOR_KEY_IPSEC,
 247                                             target_container);
 248
 249         key_esp->spi = hdr->spi;
 250 }
 251
 252 static void __skb_flow_dissect_l2tpv3(const struct sk_buff *skb,
 253                                       struct flow_dissector *flow_dissector,
 254                                       void *target_container, const void *data,
 255                                       int nhoff, int hlen)
 256 {
 257         struct flow_dissector_key_l2tpv3 *key_l2tpv3;
 258         struct {
 259                 __be32 session_id;
 260         } *hdr, _hdr;
 261
 262         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_L2TPV3))
 263                 return;
 264
 265         hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 266         if (!hdr)
 267                 return;
 268
 269         key_l2tpv3 = skb_flow_dissector_target(flow_dissector,
 270                                                FLOW_DISSECTOR_KEY_L2TPV3,
 271                                                target_container);
 272
 273         key_l2tpv3->session_id = hdr->session_id;
 274 }
 275
 276 void skb_flow_dissect_meta(const struct sk_buff *skb,
 277                            struct flow_dissector *flow_dissector,
 278                            void *target_container)
 279 {
 280         struct flow_dissector_key_meta *meta;
 281
 282         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_META))
 283                 return;
 284
 285         meta = skb_flow_dissector_target(flow_dissector,
 286                                          FLOW_DISSECTOR_KEY_META,
 287                                          target_container);
 288         meta->ingress_ifindex = skb->skb_iif;
 289 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
 290         if (tc_skb_ext_tc_enabled()) {
 291                 struct tc_skb_ext *ext;
 292
 293                 ext = skb_ext_find(skb, TC_SKB_EXT);
 294                 if (ext)
 295                         meta->l2_miss = ext->l2_miss;
 296         }
 297 #endif
 298 }
 299 EXPORT_SYMBOL(skb_flow_dissect_meta);
 300
 301 static void
 302 skb_flow_dissect_set_enc_control(enum flow_dissector_key_id type,
 303                                  u32 ctrl_flags,
 304                                  struct flow_dissector *flow_dissector,
 305                                  void *target_container)
 306 {
 307         struct flow_dissector_key_control *ctrl;
 308
 309         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL))
 310                 return;
 311
 312         ctrl = skb_flow_dissector_target(flow_dissector,
 313                                          FLOW_DISSECTOR_KEY_ENC_CONTROL,
 314                                          target_container);
 315         ctrl->addr_type = type;
 316         ctrl->flags = ctrl_flags;
 317 }
 318
 319 void
 320 skb_flow_dissect_ct(const struct sk_buff *skb,
 321                     struct flow_dissector *flow_dissector,
 322                     void *target_container, u16 *ctinfo_map,
 323                     size_t mapsize, bool post_ct, u16 zone)
 324 {
 325 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
 326         struct flow_dissector_key_ct *key;
 327         enum ip_conntrack_info ctinfo;
 328         struct nf_conn_labels *cl;
 329         struct nf_conn *ct;
 330
 331         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_CT))
 332                 return;
 333
 334         ct = nf_ct_get(skb, &ctinfo);
 335         if (!ct && !post_ct)
 336                 return;
 337
 338         key = skb_flow_dissector_target(flow_dissector,
 339                                         FLOW_DISSECTOR_KEY_CT,
 340                                         target_container);
 341
 342         if (!ct) {
 343                 key->ct_state = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
 344                                 TCA_FLOWER_KEY_CT_FLAGS_INVALID;
 345                 key->ct_zone = zone;
 346                 return;
 347         }
 348
 349         if (ctinfo < mapsize)
 350                 key->ct_state = ctinfo_map[ctinfo];
 351 #if IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)
 352         key->ct_zone = ct->zone.id;
 353 #endif
 354 #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
 355         key->ct_mark = READ_ONCE(ct->mark);
 356 #endif
 357
 358         cl = nf_ct_labels_find(ct);
 359         if (cl)
 360                 memcpy(key->ct_labels, cl->bits, sizeof(key->ct_labels));
 361 #endif /* CONFIG_NF_CONNTRACK */
 362 }
 363 EXPORT_SYMBOL(skb_flow_dissect_ct);
 364
 365 void
 366 skb_flow_dissect_tunnel_info(const struct sk_buff *skb,
 367                              struct flow_dissector *flow_dissector,
 368                              void *target_container)
 369 {
 370         struct ip_tunnel_info *info;
 371         struct ip_tunnel_key *key;
 372         u32 ctrl_flags = 0;
 373
 374         /* A quick check to see if there might be something to do. */
 375         if (!dissector_uses_key(flow_dissector,
 376                                 FLOW_DISSECTOR_KEY_ENC_KEYID) &&
 377             !dissector_uses_key(flow_dissector,
 378                                 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) &&
 379             !dissector_uses_key(flow_dissector,
 380                                 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) &&
 381             !dissector_uses_key(flow_dissector,
 382                                 FLOW_DISSECTOR_KEY_ENC_CONTROL) &&
 383             !dissector_uses_key(flow_dissector,
 384                                 FLOW_DISSECTOR_KEY_ENC_PORTS) &&
 385             !dissector_uses_key(flow_dissector,
 386                                 FLOW_DISSECTOR_KEY_ENC_IP) &&
 387             !dissector_uses_key(flow_dissector,
 388                                 FLOW_DISSECTOR_KEY_ENC_OPTS))
 389                 return;
 390
 391         info = skb_tunnel_info(skb);
 392         if (!info)
 393                 return;
 394
 395         key = &info->key;
 396
 397         if (test_bit(IP_TUNNEL_CSUM_BIT, key->tun_flags))
 398                 ctrl_flags |= FLOW_DIS_F_TUNNEL_CSUM;
 399         if (test_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, key->tun_flags))
 400                 ctrl_flags |= FLOW_DIS_F_TUNNEL_DONT_FRAGMENT;
 401         if (test_bit(IP_TUNNEL_OAM_BIT, key->tun_flags))
 402                 ctrl_flags |= FLOW_DIS_F_TUNNEL_OAM;
 403         if (test_bit(IP_TUNNEL_CRIT_OPT_BIT, key->tun_flags))
 404                 ctrl_flags |= FLOW_DIS_F_TUNNEL_CRIT_OPT;
 405
 406         switch (ip_tunnel_info_af(info)) {
 407         case AF_INET:
 408                 skb_flow_dissect_set_enc_control(FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 409                                                  ctrl_flags, flow_dissector,
 410                                                  target_container);
 411                 if (dissector_uses_key(flow_dissector,
 412                                        FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
 413                         struct flow_dissector_key_ipv4_addrs *ipv4;
 414
 415                         ipv4 = skb_flow_dissector_target(flow_dissector,
 416                                                          FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
 417                                                          target_container);
 418                         ipv4->src = key->u.ipv4.src;
 419                         ipv4->dst = key->u.ipv4.dst;
 420                 }
 421                 break;
 422         case AF_INET6:
 423                 skb_flow_dissect_set_enc_control(FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 424                                                  ctrl_flags, flow_dissector,
 425                                                  target_container);
 426                 if (dissector_uses_key(flow_dissector,
 427                                        FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
 428                         struct flow_dissector_key_ipv6_addrs *ipv6;
 429
 430                         ipv6 = skb_flow_dissector_target(flow_dissector,
 431                                                          FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
 432                                                          target_container);
 433                         ipv6->src = key->u.ipv6.src;
 434                         ipv6->dst = key->u.ipv6.dst;
 435                 }
 436                 break;
 437         default:
 438                 skb_flow_dissect_set_enc_control(0, ctrl_flags, flow_dissector,
 439                                                  target_container);
 440                 break;
 441         }
 442
 443         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
 444                 struct flow_dissector_key_keyid *keyid;
 445
 446                 keyid = skb_flow_dissector_target(flow_dissector,
 447                                                   FLOW_DISSECTOR_KEY_ENC_KEYID,
 448                                                   target_container);
 449                 keyid->keyid = tunnel_id_to_key32(key->tun_id);
 450         }
 451
 452         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
 453                 struct flow_dissector_key_ports *tp;
 454
 455                 tp = skb_flow_dissector_target(flow_dissector,
 456                                                FLOW_DISSECTOR_KEY_ENC_PORTS,
 457                                                target_container);
 458                 tp->src = key->tp_src;
 459                 tp->dst = key->tp_dst;
 460         }
 461
 462         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_IP)) {
 463                 struct flow_dissector_key_ip *ip;
 464
 465                 ip = skb_flow_dissector_target(flow_dissector,
 466                                                FLOW_DISSECTOR_KEY_ENC_IP,
 467                                                target_container);
 468                 ip->tos = key->tos;
 469                 ip->ttl = key->ttl;
 470         }
 471
 472         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ENC_OPTS)) {
 473                 struct flow_dissector_key_enc_opts *enc_opt;
 474                 IP_TUNNEL_DECLARE_FLAGS(flags) = { };
 475                 u32 val;
 476
 477                 enc_opt = skb_flow_dissector_target(flow_dissector,
 478                                                     FLOW_DISSECTOR_KEY_ENC_OPTS,
 479                                                     target_container);
 480
 481                 if (!info->options_len)
 482                         return;
 483
 484                 enc_opt->len = info->options_len;
 485                 ip_tunnel_info_opts_get(enc_opt->data, info);
 486
 487                 ip_tunnel_set_options_present(flags);
 488                 ip_tunnel_flags_and(flags, info->key.tun_flags, flags);
 489
 490                 val = find_next_bit(flags, __IP_TUNNEL_FLAG_NUM,
 491                                     IP_TUNNEL_GENEVE_OPT_BIT);
 492                 enc_opt->dst_opt_type = val < __IP_TUNNEL_FLAG_NUM ? val : 0;
 493         }
 494 }
 495 EXPORT_SYMBOL(skb_flow_dissect_tunnel_info);
 496
 497 void skb_flow_dissect_hash(const struct sk_buff *skb,
 498                            struct flow_dissector *flow_dissector,
 499                            void *target_container)
 500 {
 501         struct flow_dissector_key_hash *key;
 502
 503         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_HASH))
 504                 return;
 505
 506         key = skb_flow_dissector_target(flow_dissector,
 507                                         FLOW_DISSECTOR_KEY_HASH,
 508                                         target_container);
 509
 510         key->hash = skb_get_hash_raw(skb);
 511 }
 512 EXPORT_SYMBOL(skb_flow_dissect_hash);
 513
 514 static enum flow_dissect_ret
 515 __skb_flow_dissect_mpls(const struct sk_buff *skb,
 516                         struct flow_dissector *flow_dissector,
 517                         void *target_container, const void *data, int nhoff,
 518                         int hlen, int lse_index, bool *entropy_label)
 519 {
 520         struct mpls_label *hdr, _hdr;
 521         u32 entry, label, bos;
 522
 523         if (!dissector_uses_key(flow_dissector,
 524                                 FLOW_DISSECTOR_KEY_MPLS_ENTROPY) &&
 525             !dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS))
 526                 return FLOW_DISSECT_RET_OUT_GOOD;
 527
 528         if (lse_index >= FLOW_DIS_MPLS_MAX)
 529                 return FLOW_DISSECT_RET_OUT_GOOD;
 530
 531         hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
 532                                    hlen, &_hdr);
 533         if (!hdr)
 534                 return FLOW_DISSECT_RET_OUT_BAD;
 535
 536         entry = ntohl(hdr->entry);
 537         label = (entry & MPLS_LS_LABEL_MASK) >> MPLS_LS_LABEL_SHIFT;
 538         bos = (entry & MPLS_LS_S_MASK) >> MPLS_LS_S_SHIFT;
 539
 540         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_MPLS)) {
 541                 struct flow_dissector_key_mpls *key_mpls;
 542                 struct flow_dissector_mpls_lse *lse;
 543
 544                 key_mpls = skb_flow_dissector_target(flow_dissector,
 545                                                      FLOW_DISSECTOR_KEY_MPLS,
 546                                                      target_container);
 547                 lse = &key_mpls->ls[lse_index];
 548
 549                 lse->mpls_ttl = (entry & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
 550                 lse->mpls_bos = bos;
 551                 lse->mpls_tc = (entry & MPLS_LS_TC_MASK) >> MPLS_LS_TC_SHIFT;
 552                 lse->mpls_label = label;
 553                 dissector_set_mpls_lse(key_mpls, lse_index);
 554         }
 555
 556         if (*entropy_label &&
 557             dissector_uses_key(flow_dissector,
 558                                FLOW_DISSECTOR_KEY_MPLS_ENTROPY)) {
 559                 struct flow_dissector_key_keyid *key_keyid;
 560
 561                 key_keyid = skb_flow_dissector_target(flow_dissector,
 562                                                       FLOW_DISSECTOR_KEY_MPLS_ENTROPY,
 563                                                       target_container);
 564                 key_keyid->keyid = cpu_to_be32(label);
 565         }
 566
 567         *entropy_label = label == MPLS_LABEL_ENTROPY;
 568
 569         return bos ? FLOW_DISSECT_RET_OUT_GOOD : FLOW_DISSECT_RET_PROTO_AGAIN;
 570 }
 571
 572 static enum flow_dissect_ret
 573 __skb_flow_dissect_arp(const struct sk_buff *skb,
 574                        struct flow_dissector *flow_dissector,
 575                        void *target_container, const void *data,
 576                        int nhoff, int hlen)
 577 {
 578         struct flow_dissector_key_arp *key_arp;
 579         struct {
 580                 unsigned char ar_sha[ETH_ALEN];
 581                 unsigned char ar_sip[4];
 582                 unsigned char ar_tha[ETH_ALEN];
 583                 unsigned char ar_tip[4];
 584         } *arp_eth, _arp_eth;
 585         const struct arphdr *arp;
 586         struct arphdr _arp;
 587
 588         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ARP))
 589                 return FLOW_DISSECT_RET_OUT_GOOD;
 590
 591         arp = __skb_header_pointer(skb, nhoff, sizeof(_arp), data,
 592                                    hlen, &_arp);
 593         if (!arp)
 594                 return FLOW_DISSECT_RET_OUT_BAD;
 595
 596         if (arp->ar_hrd != htons(ARPHRD_ETHER) ||
 597             arp->ar_pro != htons(ETH_P_IP) ||
 598             arp->ar_hln != ETH_ALEN ||
 599             arp->ar_pln != 4 ||
 600             (arp->ar_op != htons(ARPOP_REPLY) &&
 601              arp->ar_op != htons(ARPOP_REQUEST)))
 602                 return FLOW_DISSECT_RET_OUT_BAD;
 603
 604         arp_eth = __skb_header_pointer(skb, nhoff + sizeof(_arp),
 605                                        sizeof(_arp_eth), data,
 606                                        hlen, &_arp_eth);
 607         if (!arp_eth)
 608                 return FLOW_DISSECT_RET_OUT_BAD;
 609
 610         key_arp = skb_flow_dissector_target(flow_dissector,
 611                                             FLOW_DISSECTOR_KEY_ARP,
 612                                             target_container);
 613
 614         memcpy(&key_arp->sip, arp_eth->ar_sip, sizeof(key_arp->sip));
 615         memcpy(&key_arp->tip, arp_eth->ar_tip, sizeof(key_arp->tip));
 616
 617         /* Only store the lower byte of the opcode;
 618          * this covers ARPOP_REPLY and ARPOP_REQUEST.
 619          */
 620         key_arp->op = ntohs(arp->ar_op) & 0xff;
 621
 622         ether_addr_copy(key_arp->sha, arp_eth->ar_sha);
 623         ether_addr_copy(key_arp->tha, arp_eth->ar_tha);
 624
 625         return FLOW_DISSECT_RET_OUT_GOOD;
 626 }
 627
 628 static enum flow_dissect_ret
 629 __skb_flow_dissect_cfm(const struct sk_buff *skb,
 630                        struct flow_dissector *flow_dissector,
 631                        void *target_container, const void *data,
 632                        int nhoff, int hlen)
 633 {
 634         struct flow_dissector_key_cfm *key, *hdr, _hdr;
 635
 636         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_CFM))
 637                 return FLOW_DISSECT_RET_OUT_GOOD;
 638
 639         hdr = __skb_header_pointer(skb, nhoff, sizeof(*key), data, hlen, &_hdr);
 640         if (!hdr)
 641                 return FLOW_DISSECT_RET_OUT_BAD;
 642
 643         key = skb_flow_dissector_target(flow_dissector, FLOW_DISSECTOR_KEY_CFM,
 644                                         target_container);
 645
 646         key->mdl_ver = hdr->mdl_ver;
 647         key->opcode = hdr->opcode;
 648
 649         return FLOW_DISSECT_RET_OUT_GOOD;
 650 }
 651
 652 static enum flow_dissect_ret
 653 __skb_flow_dissect_gre(const struct sk_buff *skb,
 654                        struct flow_dissector_key_control *key_control,
 655                        struct flow_dissector *flow_dissector,
 656                        void *target_container, const void *data,
 657                        __be16 *p_proto, int *p_nhoff, int *p_hlen,
 658                        unsigned int flags)
 659 {
 660         struct flow_dissector_key_keyid *key_keyid;
 661         struct gre_base_hdr *hdr, _hdr;
 662         int offset = 0;
 663         u16 gre_ver;
 664
 665         hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr),
 666                                    data, *p_hlen, &_hdr);
 667         if (!hdr)
 668                 return FLOW_DISSECT_RET_OUT_BAD;
 669
 670         /* Only look inside GRE without routing */
 671         if (hdr->flags & GRE_ROUTING)
 672                 return FLOW_DISSECT_RET_OUT_GOOD;
 673
 674         /* Only look inside GRE for version 0 and 1 */
 675         gre_ver = ntohs(hdr->flags & GRE_VERSION);
 676         if (gre_ver > 1)
 677                 return FLOW_DISSECT_RET_OUT_GOOD;
 678
 679         *p_proto = hdr->protocol;
 680         if (gre_ver) {
 681                 /* Version1 must be PPTP, and check the flags */
 682                 if (!(*p_proto == GRE_PROTO_PPP && (hdr->flags & GRE_KEY)))
 683                         return FLOW_DISSECT_RET_OUT_GOOD;
 684         }
 685
 686         offset += sizeof(struct gre_base_hdr);
 687
 688         if (hdr->flags & GRE_CSUM)
 689                 offset += sizeof_field(struct gre_full_hdr, csum) +
 690                           sizeof_field(struct gre_full_hdr, reserved1);
 691
 692         if (hdr->flags & GRE_KEY) {
 693                 const __be32 *keyid;
 694                 __be32 _keyid;
 695
 696                 keyid = __skb_header_pointer(skb, *p_nhoff + offset,
 697                                              sizeof(_keyid),
 698                                              data, *p_hlen, &_keyid);
 699                 if (!keyid)
 700                         return FLOW_DISSECT_RET_OUT_BAD;
 701
 702                 if (dissector_uses_key(flow_dissector,
 703                                        FLOW_DISSECTOR_KEY_GRE_KEYID)) {
 704                         key_keyid = skb_flow_dissector_target(flow_dissector,
 705                                                               FLOW_DISSECTOR_KEY_GRE_KEYID,
 706                                                               target_container);
 707                         if (gre_ver == 0)
 708                                 key_keyid->keyid = *keyid;
 709                         else
 710                                 key_keyid->keyid = *keyid & GRE_PPTP_KEY_MASK;
 711                 }
 712                 offset += sizeof_field(struct gre_full_hdr, key);
 713         }
 714
 715         if (hdr->flags & GRE_SEQ)
 716                 offset += sizeof_field(struct pptp_gre_header, seq);
 717
 718         if (gre_ver == 0) {
 719                 if (*p_proto == htons(ETH_P_TEB)) {
 720                         const struct ethhdr *eth;
 721                         struct ethhdr _eth;
 722
 723                         eth = __skb_header_pointer(skb, *p_nhoff + offset,
 724                                                    sizeof(_eth),
 725                                                    data, *p_hlen, &_eth);
 726                         if (!eth)
 727                                 return FLOW_DISSECT_RET_OUT_BAD;
 728                         *p_proto = eth->h_proto;
 729                         offset += sizeof(*eth);
 730
 731                         /* Cap headers that we access via pointers at the
 732                          * end of the Ethernet header as our maximum alignment
 733                          * at that point is only 2 bytes.
 734                          */
 735                         if (NET_IP_ALIGN)
 736                                 *p_hlen = *p_nhoff + offset;
 737                 }
 738         } else { /* version 1, must be PPTP */
 739                 u8 _ppp_hdr[PPP_HDRLEN];
 740                 u8 *ppp_hdr;
 741
 742                 if (hdr->flags & GRE_ACK)
 743                         offset += sizeof_field(struct pptp_gre_header, ack);
 744
 745                 ppp_hdr = __skb_header_pointer(skb, *p_nhoff + offset,
 746                                                sizeof(_ppp_hdr),
 747                                                data, *p_hlen, _ppp_hdr);
 748                 if (!ppp_hdr)
 749                         return FLOW_DISSECT_RET_OUT_BAD;
 750
 751                 switch (PPP_PROTOCOL(ppp_hdr)) {
 752                 case PPP_IP:
 753                         *p_proto = htons(ETH_P_IP);
 754                         break;
 755                 case PPP_IPV6:
 756                         *p_proto = htons(ETH_P_IPV6);
 757                         break;
 758                 default:
 759                         /* Could probably catch some more like MPLS */
 760                         break;
 761                 }
 762
 763                 offset += PPP_HDRLEN;
 764         }
 765
 766         *p_nhoff += offset;
 767         key_control->flags |= FLOW_DIS_ENCAPSULATION;
 768         if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
 769                 return FLOW_DISSECT_RET_OUT_GOOD;
 770
 771         return FLOW_DISSECT_RET_PROTO_AGAIN;
 772 }
 773
 774 /**
 775  * __skb_flow_dissect_batadv() - dissect batman-adv header
 776  * @skb: sk_buff to with the batman-adv header
 777  * @key_control: flow dissectors control key
 778  * @data: raw buffer pointer to the packet, if NULL use skb->data
 779  * @p_proto: pointer used to update the protocol to process next
 780  * @p_nhoff: pointer used to update inner network header offset
 781  * @hlen: packet header length
 782  * @flags: any combination of FLOW_DISSECTOR_F_*
 783  *
 784  * ETH_P_BATMAN packets are tried to be dissected. Only
 785  * &struct batadv_unicast packets are actually processed because they contain an
 786  * inner ethernet header and are usually followed by actual network header. This
 787  * allows the flow dissector to continue processing the packet.
 788  *
 789  * Return: FLOW_DISSECT_RET_PROTO_AGAIN when &struct batadv_unicast was found,
 790  *  FLOW_DISSECT_RET_OUT_GOOD when dissector should stop after encapsulation,
 791  *  otherwise FLOW_DISSECT_RET_OUT_BAD
 792  */
 793 static enum flow_dissect_ret
 794 __skb_flow_dissect_batadv(const struct sk_buff *skb,
 795                           struct flow_dissector_key_control *key_control,
 796                           const void *data, __be16 *p_proto, int *p_nhoff,
 797                           int hlen, unsigned int flags)
 798 {
 799         struct {
 800                 struct batadv_unicast_packet batadv_unicast;
 801                 struct ethhdr eth;
 802         } *hdr, _hdr;
 803
 804         hdr = __skb_header_pointer(skb, *p_nhoff, sizeof(_hdr), data, hlen,
 805                                    &_hdr);
 806         if (!hdr)
 807                 return FLOW_DISSECT_RET_OUT_BAD;
 808
 809         if (hdr->batadv_unicast.version != BATADV_COMPAT_VERSION)
 810                 return FLOW_DISSECT_RET_OUT_BAD;
 811
 812         if (hdr->batadv_unicast.packet_type != BATADV_UNICAST)
 813                 return FLOW_DISSECT_RET_OUT_BAD;
 814
 815         *p_proto = hdr->eth.h_proto;
 816         *p_nhoff += sizeof(*hdr);
 817
 818         key_control->flags |= FLOW_DIS_ENCAPSULATION;
 819         if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP)
 820                 return FLOW_DISSECT_RET_OUT_GOOD;
 821
 822         return FLOW_DISSECT_RET_PROTO_AGAIN;
 823 }
 824
 825 static void
 826 __skb_flow_dissect_tcp(const struct sk_buff *skb,
 827                        struct flow_dissector *flow_dissector,
 828                        void *target_container, const void *data,
 829                        int thoff, int hlen)
 830 {
 831         struct flow_dissector_key_tcp *key_tcp;
 832         struct tcphdr *th, _th;
 833
 834         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_TCP))
 835                 return;
 836
 837         th = __skb_header_pointer(skb, thoff, sizeof(_th), data, hlen, &_th);
 838         if (!th)
 839                 return;
 840
 841         if (unlikely(__tcp_hdrlen(th) < sizeof(_th)))
 842                 return;
 843
 844         key_tcp = skb_flow_dissector_target(flow_dissector,
 845                                             FLOW_DISSECTOR_KEY_TCP,
 846                                             target_container);
 847         key_tcp->flags = (*(__be16 *) &tcp_flag_word(th) & htons(0x0FFF));
 848 }
 849
 850 static void
 851 __skb_flow_dissect_ports(const struct sk_buff *skb,
 852                          struct flow_dissector *flow_dissector,
 853                          void *target_container, const void *data,
 854                          int nhoff, u8 ip_proto, int hlen)
 855 {
 856         struct flow_dissector_key_ports_range *key_ports_range = NULL;
 857         struct flow_dissector_key_ports *key_ports = NULL;
 858         __be32 ports;
 859
 860         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS))
 861                 key_ports = skb_flow_dissector_target(flow_dissector,
 862                                                       FLOW_DISSECTOR_KEY_PORTS,
 863                                                       target_container);
 864
 865         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS_RANGE))
 866                 key_ports_range = skb_flow_dissector_target(flow_dissector,
 867                                                             FLOW_DISSECTOR_KEY_PORTS_RANGE,
 868                                                             target_container);
 869
 870         if (!key_ports && !key_ports_range)
 871                 return;
 872
 873         ports = skb_flow_get_ports(skb, nhoff, ip_proto, data, hlen);
 874
 875         if (key_ports)
 876                 key_ports->ports = ports;
 877
 878         if (key_ports_range)
 879                 key_ports_range->tp.ports = ports;
 880 }
 881
 882 static void
 883 __skb_flow_dissect_ipv4(const struct sk_buff *skb,
 884                         struct flow_dissector *flow_dissector,
 885                         void *target_container, const void *data,
 886                         const struct iphdr *iph)
 887 {
 888         struct flow_dissector_key_ip *key_ip;
 889
 890         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
 891                 return;
 892
 893         key_ip = skb_flow_dissector_target(flow_dissector,
 894                                            FLOW_DISSECTOR_KEY_IP,
 895                                            target_container);
 896         key_ip->tos = iph->tos;
 897         key_ip->ttl = iph->ttl;
 898 }
 899
 900 static void
 901 __skb_flow_dissect_ipv6(const struct sk_buff *skb,
 902                         struct flow_dissector *flow_dissector,
 903                         void *target_container, const void *data,
 904                         const struct ipv6hdr *iph)
 905 {
 906         struct flow_dissector_key_ip *key_ip;
 907
 908         if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IP))
 909                 return;
 910
 911         key_ip = skb_flow_dissector_target(flow_dissector,
 912                                            FLOW_DISSECTOR_KEY_IP,
 913                                            target_container);
 914         key_ip->tos = ipv6_get_dsfield(iph);
 915         key_ip->ttl = iph->hop_limit;
 916 }
 917
 918 /* Maximum number of protocol headers that can be parsed in
 919  * __skb_flow_dissect
 920  */
 921 #define MAX_FLOW_DISSECT_HDRS   15
 922
 923 static bool skb_flow_dissect_allowed(int *num_hdrs)
 924 {
 925         ++*num_hdrs;
 926
 927         return (*num_hdrs <= MAX_FLOW_DISSECT_HDRS);
 928 }
 929
 930 static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys,
 931                                      struct flow_dissector *flow_dissector,
 932                                      void *target_container)
 933 {
 934         struct flow_dissector_key_ports_range *key_ports_range = NULL;
 935         struct flow_dissector_key_ports *key_ports = NULL;
 936         struct flow_dissector_key_control *key_control;
 937         struct flow_dissector_key_basic *key_basic;
 938         struct flow_dissector_key_addrs *key_addrs;
 939         struct flow_dissector_key_tags *key_tags;
 940
 941         key_control = skb_flow_dissector_target(flow_dissector,
 942                                                 FLOW_DISSECTOR_KEY_CONTROL,
 943                                                 target_container);
 944         key_control->thoff = flow_keys->thoff;
 945         if (flow_keys->is_frag)
 946                 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
 947         if (flow_keys->is_first_frag)
 948                 key_control->flags |= FLOW_DIS_FIRST_FRAG;
 949         if (flow_keys->is_encap)
 950                 key_control->flags |= FLOW_DIS_ENCAPSULATION;
 951
 952         key_basic = skb_flow_dissector_target(flow_dissector,
 953                                               FLOW_DISSECTOR_KEY_BASIC,
 954                                               target_container);
 955         key_basic->n_proto = flow_keys->n_proto;
 956         key_basic->ip_proto = flow_keys->ip_proto;
 957
 958         if (flow_keys->addr_proto == ETH_P_IP &&
 959             dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
 960                 key_addrs = skb_flow_dissector_target(flow_dissector,
 961                                                       FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 962                                                       target_container);
 963                 key_addrs->v4addrs.src = flow_keys->ipv4_src;
 964                 key_addrs->v4addrs.dst = flow_keys->ipv4_dst;
 965                 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
 966         } else if (flow_keys->addr_proto == ETH_P_IPV6 &&
 967                    dissector_uses_key(flow_dissector,
 968                                       FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 969                 key_addrs = skb_flow_dissector_target(flow_dissector,
 970                                                       FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 971                                                       target_container);
 972                 memcpy(&key_addrs->v6addrs.src, &flow_keys->ipv6_src,
 973                        sizeof(key_addrs->v6addrs.src));
 974                 memcpy(&key_addrs->v6addrs.dst, &flow_keys->ipv6_dst,
 975                        sizeof(key_addrs->v6addrs.dst));
 976                 key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
 977         }
 978
 979         if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS)) {
 980                 key_ports = skb_flow_dissector_target(flow_dissector,
 981                                                       FLOW_DISSECTOR_KEY_PORTS,
 982                                                       target_container);
 983                 key_ports->src = flow_keys->sport;
 984                 key_ports->dst = flow_keys->dport;
 985         }
 986         if (dissector_uses_key(flow_dissector,
 987                                FLOW_DISSECTOR_KEY_PORTS_RANGE)) {
 988                 key_ports_range = skb_flow_dissector_target(flow_dissector,
 989                                                             FLOW_DISSECTOR_KEY_PORTS_RANGE,
 990                                                             target_container);
 991                 key_ports_range->tp.src = flow_keys->sport;
 992                 key_ports_range->tp.dst = flow_keys->dport;
 993         }
 994
 995         if (dissector_uses_key(flow_dissector,
 996                                FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
 997                 key_tags = skb_flow_dissector_target(flow_dissector,
 998                                                      FLOW_DISSECTOR_KEY_FLOW_LABEL,
 999                                                      target_container);
1000                 key_tags->flow_label = ntohl(flow_keys->flow_label);
1001         }
1002 }
1003
1004 u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
1005                      __be16 proto, int nhoff, int hlen, unsigned int flags)
1006 {
1007         struct bpf_flow_keys *flow_keys = ctx->flow_keys;
1008         u32 result;
1009
1010         /* Pass parameters to the BPF program */
1011         memset(flow_keys, 0, sizeof(*flow_keys));
1012         flow_keys->n_proto = proto;
1013         flow_keys->nhoff = nhoff;
1014         flow_keys->thoff = flow_keys->nhoff;
1015
1016         BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG !=
1017                      (int)FLOW_DISSECTOR_F_PARSE_1ST_FRAG);
1018         BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL !=
1019                      (int)FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1020         BUILD_BUG_ON((int)BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP !=
1021                      (int)FLOW_DISSECTOR_F_STOP_AT_ENCAP);
1022         flow_keys->flags = flags;
1023
1024         result = bpf_prog_run_pin_on_cpu(prog, ctx);
1025
1026         flow_keys->nhoff = clamp_t(u16, flow_keys->nhoff, nhoff, hlen);
1027         flow_keys->thoff = clamp_t(u16, flow_keys->thoff,
1028                                    flow_keys->nhoff, hlen);
1029
1030         return result;
1031 }
1032
1033 static bool is_pppoe_ses_hdr_valid(const struct pppoe_hdr *hdr)
1034 {
1035         return hdr->ver == 1 && hdr->type == 1 && hdr->code == 0;
1036 }
1037
1038 /**
1039  * __skb_flow_dissect - extract the flow_keys struct and return it
1040  * @net: associated network namespace, derived from @skb if NULL
1041  * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
1042  * @flow_dissector: list of keys to dissect
1043  * @target_container: target structure to put dissected values into
1044  * @data: raw buffer pointer to the packet, if NULL use skb->data
1045  * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
1046  * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
1047  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
1048  * @flags: flags that control the dissection process, e.g.
1049  *         FLOW_DISSECTOR_F_STOP_AT_ENCAP.
1050  *
1051  * The function will try to retrieve individual keys into target specified
1052  * by flow_dissector from either the skbuff or a raw buffer specified by the
1053  * rest parameters.
1054  *
1055  * Caller must take care of zeroing target container memory.
1056  */
1057 bool __skb_flow_dissect(const struct net *net,
1058                         const struct sk_buff *skb,
1059                         struct flow_dissector *flow_dissector,
1060                         void *target_container, const void *data,
1061                         __be16 proto, int nhoff, int hlen, unsigned int flags)
1062 {
1063         struct flow_dissector_key_control *key_control;
1064         struct flow_dissector_key_basic *key_basic;
1065         struct flow_dissector_key_addrs *key_addrs;
1066         struct flow_dissector_key_tags *key_tags;
1067         struct flow_dissector_key_vlan *key_vlan;
1068         enum flow_dissect_ret fdret;
1069         enum flow_dissector_key_id dissector_vlan = FLOW_DISSECTOR_KEY_MAX;
1070         bool mpls_el = false;
1071         int mpls_lse = 0;
1072         int num_hdrs = 0;
1073         u8 ip_proto = 0;
1074         bool ret;
1075
1076         if (!data) {
1077                 data = skb->data;
1078                 proto = skb_vlan_tag_present(skb) ?
1079                          skb->vlan_proto : skb->protocol;
1080                 nhoff = skb_network_offset(skb);
1081                 hlen = skb_headlen(skb);
1082 #if IS_ENABLED(CONFIG_NET_DSA)
1083                 if (unlikely(skb->dev && netdev_uses_dsa(skb->dev) &&
1084                              proto == htons(ETH_P_XDSA))) {
1085                         struct metadata_dst *md_dst = skb_metadata_dst(skb);
1086                         const struct dsa_device_ops *ops;
1087                         int offset = 0;
1088
1089                         ops = skb->dev->dsa_ptr->tag_ops;
1090                         /* Only DSA header taggers break flow dissection */
1091                         if (ops->needed_headroom &&
1092                             (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)) {
1093                                 if (ops->flow_dissect)
1094                                         ops->flow_dissect(skb, &proto, &offset);
1095                                 else
1096                                         dsa_tag_generic_flow_dissect(skb,
1097                                                                      &proto,
1098                                                                      &offset);
1099                                 hlen -= offset;
1100                                 nhoff += offset;
1101                         }
1102                 }
1103 #endif
1104         }
1105
1106         /* It is ensured by skb_flow_dissector_init() that control key will
1107          * be always present.
1108          */
1109         key_control = skb_flow_dissector_target(flow_dissector,
1110                                                 FLOW_DISSECTOR_KEY_CONTROL,
1111                                                 target_container);
1112
1113         /* It is ensured by skb_flow_dissector_init() that basic key will
1114          * be always present.
1115          */
1116         key_basic = skb_flow_dissector_target(flow_dissector,
1117                                               FLOW_DISSECTOR_KEY_BASIC,
1118                                               target_container);
1119
1120         rcu_read_lock();
1121
1122         if (skb) {
1123                 if (!net) {
1124                         if (skb->dev)
1125                                 net = dev_net_rcu(skb->dev);
1126                         else if (skb->sk)
1127                                 net = sock_net(skb->sk);
1128                 }
1129         }
1130
1131         DEBUG_NET_WARN_ON_ONCE(!net);
1132         if (net) {
1133                 enum netns_bpf_attach_type type = NETNS_BPF_FLOW_DISSECTOR;
1134                 struct bpf_prog_array *run_array;
1135
1136                 run_array = rcu_dereference(init_net.bpf.run_array[type]);
1137                 if (!run_array)
1138                         run_array = rcu_dereference(net->bpf.run_array[type]);
1139
1140                 if (run_array) {
1141                         struct bpf_flow_keys flow_keys;
1142                         struct bpf_flow_dissector ctx = {
1143                                 .flow_keys = &flow_keys,
1144                                 .data = data,
1145                                 .data_end = data + hlen,
1146                         };
1147                         __be16 n_proto = proto;
1148                         struct bpf_prog *prog;
1149                         u32 result;
1150
1151                         if (skb) {
1152                                 ctx.skb = skb;
1153                                 /* we can't use 'proto' in the skb case
1154                                  * because it might be set to skb->vlan_proto
1155                                  * which has been pulled from the data
1156                                  */
1157                                 n_proto = skb->protocol;
1158                         }
1159
1160                         prog = READ_ONCE(run_array->items[0].prog);
1161                         result = bpf_flow_dissect(prog, &ctx, n_proto, nhoff,
1162                                                   hlen, flags);
1163                         if (result != BPF_FLOW_DISSECTOR_CONTINUE) {
1164                                 __skb_flow_bpf_to_target(&flow_keys, flow_dissector,
1165                                                          target_container);
1166                                 rcu_read_unlock();
1167                                 return result == BPF_OK;
1168                         }
1169                 }
1170         }
1171
1172         rcu_read_unlock();
1173
1174         if (dissector_uses_key(flow_dissector,
1175                                FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
1176                 struct ethhdr *eth = eth_hdr(skb);
1177                 struct flow_dissector_key_eth_addrs *key_eth_addrs;
1178
1179                 key_eth_addrs = skb_flow_dissector_target(flow_dissector,
1180                                                           FLOW_DISSECTOR_KEY_ETH_ADDRS,
1181                                                           target_container);
1182                 memcpy(key_eth_addrs, eth, sizeof(*key_eth_addrs));
1183         }
1184
1185         if (dissector_uses_key(flow_dissector,
1186                                FLOW_DISSECTOR_KEY_NUM_OF_VLANS)) {
1187                 struct flow_dissector_key_num_of_vlans *key_num_of_vlans;
1188
1189                 key_num_of_vlans = skb_flow_dissector_target(flow_dissector,
1190                                                              FLOW_DISSECTOR_KEY_NUM_OF_VLANS,
1191                                                              target_container);
1192                 key_num_of_vlans->num_of_vlans = 0;
1193         }
1194
1195 proto_again:
1196         fdret = FLOW_DISSECT_RET_CONTINUE;
1197
1198         switch (proto) {
1199         case htons(ETH_P_IP): {
1200                 const struct iphdr *iph;
1201                 struct iphdr _iph;
1202
1203                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
1204                 if (!iph || iph->ihl < 5) {
1205                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1206                         break;
1207                 }
1208
1209                 nhoff += iph->ihl * 4;
1210
1211                 ip_proto = iph->protocol;
1212
1213                 if (dissector_uses_key(flow_dissector,
1214                                        FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
1215                         key_addrs = skb_flow_dissector_target(flow_dissector,
1216                                                               FLOW_DISSECTOR_KEY_IPV4_ADDRS,
1217                                                               target_container);
1218
1219                         memcpy(&key_addrs->v4addrs.src, &iph->saddr,
1220                                sizeof(key_addrs->v4addrs.src));
1221                         memcpy(&key_addrs->v4addrs.dst, &iph->daddr,
1222                                sizeof(key_addrs->v4addrs.dst));
1223                         key_control->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1224                 }
1225
1226                 __skb_flow_dissect_ipv4(skb, flow_dissector,
1227                                         target_container, data, iph);
1228
1229                 if (ip_is_fragment(iph)) {
1230                         key_control->flags |= FLOW_DIS_IS_FRAGMENT;
1231
1232                         if (iph->frag_off & htons(IP_OFFSET)) {
1233                                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1234                                 break;
1235                         } else {
1236                                 key_control->flags |= FLOW_DIS_FIRST_FRAG;
1237                                 if (!(flags &
1238                                       FLOW_DISSECTOR_F_PARSE_1ST_FRAG)) {
1239                                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1240                                         break;
1241                                 }
1242                         }
1243                 }
1244
1245                 break;
1246         }
1247         case htons(ETH_P_IPV6): {
1248                 const struct ipv6hdr *iph;
1249                 struct ipv6hdr _iph;
1250
1251                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
1252                 if (!iph) {
1253                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1254                         break;
1255                 }
1256
1257                 ip_proto = iph->nexthdr;
1258                 nhoff += sizeof(struct ipv6hdr);
1259
1260                 if (dissector_uses_key(flow_dissector,
1261                                        FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
1262                         key_addrs = skb_flow_dissector_target(flow_dissector,
1263                                                               FLOW_DISSECTOR_KEY_IPV6_ADDRS,
1264                                                               target_container);
1265
1266                         memcpy(&key_addrs->v6addrs.src, &iph->saddr,
1267                                sizeof(key_addrs->v6addrs.src));
1268                         memcpy(&key_addrs->v6addrs.dst, &iph->daddr,
1269                                sizeof(key_addrs->v6addrs.dst));
1270                         key_control->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1271                 }
1272
1273                 if ((dissector_uses_key(flow_dissector,
1274                                         FLOW_DISSECTOR_KEY_FLOW_LABEL) ||
1275                      (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL)) &&
1276                     ip6_flowlabel(iph)) {
1277                         __be32 flow_label = ip6_flowlabel(iph);
1278
1279                         if (dissector_uses_key(flow_dissector,
1280                                                FLOW_DISSECTOR_KEY_FLOW_LABEL)) {
1281                                 key_tags = skb_flow_dissector_target(flow_dissector,
1282                                                                      FLOW_DISSECTOR_KEY_FLOW_LABEL,
1283                                                                      target_container);
1284                                 key_tags->flow_label = ntohl(flow_label);
1285                         }
1286                         if (flags & FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL) {
1287                                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1288                                 break;
1289                         }
1290                 }
1291
1292                 __skb_flow_dissect_ipv6(skb, flow_dissector,
1293                                         target_container, data, iph);
1294
1295                 break;
1296         }
1297         case htons(ETH_P_8021AD):
1298         case htons(ETH_P_8021Q): {
1299                 const struct vlan_hdr *vlan = NULL;
1300                 struct vlan_hdr _vlan;
1301                 __be16 saved_vlan_tpid = proto;
1302
1303                 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX &&
1304                     skb && skb_vlan_tag_present(skb)) {
1305                         proto = skb->protocol;
1306                 } else {
1307                         vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan),
1308                                                     data, hlen, &_vlan);
1309                         if (!vlan) {
1310                                 fdret = FLOW_DISSECT_RET_OUT_BAD;
1311                                 break;
1312                         }
1313
1314                         proto = vlan->h_vlan_encapsulated_proto;
1315                         nhoff += sizeof(*vlan);
1316                 }
1317
1318                 if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_NUM_OF_VLANS) &&
1319                     !(key_control->flags & FLOW_DIS_ENCAPSULATION)) {
1320                         struct flow_dissector_key_num_of_vlans *key_nvs;
1321
1322                         key_nvs = skb_flow_dissector_target(flow_dissector,
1323                                                             FLOW_DISSECTOR_KEY_NUM_OF_VLANS,
1324                                                             target_container);
1325                         key_nvs->num_of_vlans++;
1326                 }
1327
1328                 if (dissector_vlan == FLOW_DISSECTOR_KEY_MAX) {
1329                         dissector_vlan = FLOW_DISSECTOR_KEY_VLAN;
1330                 } else if (dissector_vlan == FLOW_DISSECTOR_KEY_VLAN) {
1331                         dissector_vlan = FLOW_DISSECTOR_KEY_CVLAN;
1332                 } else {
1333                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1334                         break;
1335                 }
1336
1337                 if (dissector_uses_key(flow_dissector, dissector_vlan)) {
1338                         key_vlan = skb_flow_dissector_target(flow_dissector,
1339                                                              dissector_vlan,
1340                                                              target_container);
1341
1342                         if (!vlan) {
1343                                 key_vlan->vlan_id = skb_vlan_tag_get_id(skb);
1344                                 key_vlan->vlan_priority = skb_vlan_tag_get_prio(skb);
1345                         } else {
1346                                 key_vlan->vlan_id = ntohs(vlan->h_vlan_TCI) &
1347                                         VLAN_VID_MASK;
1348                                 key_vlan->vlan_priority =
1349                                         (ntohs(vlan->h_vlan_TCI) &
1350                                          VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1351                         }
1352                         key_vlan->vlan_tpid = saved_vlan_tpid;
1353                         key_vlan->vlan_eth_type = proto;
1354                 }
1355
1356                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1357                 break;
1358         }
1359         case htons(ETH_P_PPP_SES): {
1360                 struct {
1361                         struct pppoe_hdr hdr;
1362                         __be16 proto;
1363                 } *hdr, _hdr;
1364                 u16 ppp_proto;
1365
1366                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
1367                 if (!hdr) {
1368                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1369                         break;
1370                 }
1371
1372                 if (!is_pppoe_ses_hdr_valid(&hdr->hdr)) {
1373                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1374                         break;
1375                 }
1376
1377                 /* least significant bit of the most significant octet
1378                  * indicates if protocol field was compressed
1379                  */
1380                 ppp_proto = ntohs(hdr->proto);
1381                 if (ppp_proto & 0x0100) {
1382                         ppp_proto = ppp_proto >> 8;
1383                         nhoff += PPPOE_SES_HLEN - 1;
1384                 } else {
1385                         nhoff += PPPOE_SES_HLEN;
1386                 }
1387
1388                 if (ppp_proto == PPP_IP) {
1389                         proto = htons(ETH_P_IP);
1390                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1391                 } else if (ppp_proto == PPP_IPV6) {
1392                         proto = htons(ETH_P_IPV6);
1393                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1394                 } else if (ppp_proto == PPP_MPLS_UC) {
1395                         proto = htons(ETH_P_MPLS_UC);
1396                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1397                 } else if (ppp_proto == PPP_MPLS_MC) {
1398                         proto = htons(ETH_P_MPLS_MC);
1399                         fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1400                 } else if (ppp_proto_is_valid(ppp_proto)) {
1401                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1402                 } else {
1403                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1404                         break;
1405                 }
1406
1407                 if (dissector_uses_key(flow_dissector,
1408                                        FLOW_DISSECTOR_KEY_PPPOE)) {
1409                         struct flow_dissector_key_pppoe *key_pppoe;
1410
1411                         key_pppoe = skb_flow_dissector_target(flow_dissector,
1412                                                               FLOW_DISSECTOR_KEY_PPPOE,
1413                                                               target_container);
1414                         key_pppoe->session_id = hdr->hdr.sid;
1415                         key_pppoe->ppp_proto = htons(ppp_proto);
1416                         key_pppoe->type = htons(ETH_P_PPP_SES);
1417                 }
1418                 break;
1419         }
1420         case htons(ETH_P_TIPC): {
1421                 struct tipc_basic_hdr *hdr, _hdr;
1422
1423                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr),
1424                                            data, hlen, &_hdr);
1425                 if (!hdr) {
1426                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1427                         break;
1428                 }
1429
1430                 if (dissector_uses_key(flow_dissector,
1431                                        FLOW_DISSECTOR_KEY_TIPC)) {
1432                         key_addrs = skb_flow_dissector_target(flow_dissector,
1433                                                               FLOW_DISSECTOR_KEY_TIPC,
1434                                                               target_container);
1435                         key_addrs->tipckey.key = tipc_hdr_rps_key(hdr);
1436                         key_control->addr_type = FLOW_DISSECTOR_KEY_TIPC;
1437                 }
1438                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1439                 break;
1440         }
1441
1442         case htons(ETH_P_MPLS_UC):
1443         case htons(ETH_P_MPLS_MC):
1444                 fdret = __skb_flow_dissect_mpls(skb, flow_dissector,
1445                                                 target_container, data,
1446                                                 nhoff, hlen, mpls_lse,
1447                                                 &mpls_el);
1448                 nhoff += sizeof(struct mpls_label);
1449                 mpls_lse++;
1450                 break;
1451         case htons(ETH_P_FCOE):
1452                 if ((hlen - nhoff) < FCOE_HEADER_LEN) {
1453                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1454                         break;
1455                 }
1456
1457                 nhoff += FCOE_HEADER_LEN;
1458                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1459                 break;
1460
1461         case htons(ETH_P_ARP):
1462         case htons(ETH_P_RARP):
1463                 fdret = __skb_flow_dissect_arp(skb, flow_dissector,
1464                                                target_container, data,
1465                                                nhoff, hlen);
1466                 break;
1467
1468         case htons(ETH_P_BATMAN):
1469                 fdret = __skb_flow_dissect_batadv(skb, key_control, data,
1470                                                   &proto, &nhoff, hlen, flags);
1471                 break;
1472
1473         case htons(ETH_P_1588): {
1474                 struct ptp_header *hdr, _hdr;
1475
1476                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data,
1477                                            hlen, &_hdr);
1478                 if (!hdr) {
1479                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1480                         break;
1481                 }
1482
1483                 nhoff += sizeof(struct ptp_header);
1484                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1485                 break;
1486         }
1487
1488         case htons(ETH_P_PRP):
1489         case htons(ETH_P_HSR): {
1490                 struct hsr_tag *hdr, _hdr;
1491
1492                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen,
1493                                            &_hdr);
1494                 if (!hdr) {
1495                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1496                         break;
1497                 }
1498
1499                 proto = hdr->encap_proto;
1500                 nhoff += HSR_HLEN;
1501                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1502                 break;
1503         }
1504
1505         case htons(ETH_P_CFM):
1506                 fdret = __skb_flow_dissect_cfm(skb, flow_dissector,
1507                                                target_container, data,
1508                                                nhoff, hlen);
1509                 break;
1510
1511         default:
1512                 fdret = FLOW_DISSECT_RET_OUT_BAD;
1513                 break;
1514         }
1515
1516         /* Process result of proto processing */
1517         switch (fdret) {
1518         case FLOW_DISSECT_RET_OUT_GOOD:
1519                 goto out_good;
1520         case FLOW_DISSECT_RET_PROTO_AGAIN:
1521                 if (skb_flow_dissect_allowed(&num_hdrs))
1522                         goto proto_again;
1523                 goto out_good;
1524         case FLOW_DISSECT_RET_CONTINUE:
1525         case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1526                 break;
1527         case FLOW_DISSECT_RET_OUT_BAD:
1528         default:
1529                 goto out_bad;
1530         }
1531
1532 ip_proto_again:
1533         fdret = FLOW_DISSECT_RET_CONTINUE;
1534
1535         switch (ip_proto) {
1536         case IPPROTO_GRE:
1537                 if (flags & FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP) {
1538                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1539                         break;
1540                 }
1541
1542                 fdret = __skb_flow_dissect_gre(skb, key_control, flow_dissector,
1543                                                target_container, data,
1544                                                &proto, &nhoff, &hlen, flags);
1545                 break;
1546
1547         case NEXTHDR_HOP:
1548         case NEXTHDR_ROUTING:
1549         case NEXTHDR_DEST: {
1550                 u8 _opthdr[2], *opthdr;
1551
1552                 if (proto != htons(ETH_P_IPV6))
1553                         break;
1554
1555                 opthdr = __skb_header_pointer(skb, nhoff, sizeof(_opthdr),
1556                                               data, hlen, &_opthdr);
1557                 if (!opthdr) {
1558                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1559                         break;
1560                 }
1561
1562                 ip_proto = opthdr[0];
1563                 nhoff += (opthdr[1] + 1) << 3;
1564
1565                 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1566                 break;
1567         }
1568         case NEXTHDR_FRAGMENT: {
1569                 struct frag_hdr _fh, *fh;
1570
1571                 if (proto != htons(ETH_P_IPV6))
1572                         break;
1573
1574                 fh = __skb_header_pointer(skb, nhoff, sizeof(_fh),
1575                                           data, hlen, &_fh);
1576
1577                 if (!fh) {
1578                         fdret = FLOW_DISSECT_RET_OUT_BAD;
1579                         break;
1580                 }
1581
1582                 key_control->flags |= FLOW_DIS_IS_FRAGMENT;
1583
1584                 nhoff += sizeof(_fh);
1585                 ip_proto = fh->nexthdr;
1586
1587                 if (!(fh->frag_off & htons(IP6_OFFSET))) {
1588                         key_control->flags |= FLOW_DIS_FIRST_FRAG;
1589                         if (flags & FLOW_DISSECTOR_F_PARSE_1ST_FRAG) {
1590                                 fdret = FLOW_DISSECT_RET_IPPROTO_AGAIN;
1591                                 break;
1592                         }
1593                 }
1594
1595                 fdret = FLOW_DISSECT_RET_OUT_GOOD;
1596                 break;
1597         }
1598         case IPPROTO_IPIP:
1599                 if (flags & FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP) {
1600                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1601                         break;
1602                 }
1603
1604                 proto = htons(ETH_P_IP);
1605
1606                 key_control->flags |= FLOW_DIS_ENCAPSULATION;
1607                 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1608                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1609                         break;
1610                 }
1611
1612                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1613                 break;
1614
1615         case IPPROTO_IPV6:
1616                 if (flags & FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP) {
1617                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1618                         break;
1619                 }
1620
1621                 proto = htons(ETH_P_IPV6);
1622
1623                 key_control->flags |= FLOW_DIS_ENCAPSULATION;
1624                 if (flags & FLOW_DISSECTOR_F_STOP_AT_ENCAP) {
1625                         fdret = FLOW_DISSECT_RET_OUT_GOOD;
1626                         break;
1627                 }
1628
1629                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1630                 break;
1631
1632
1633         case IPPROTO_MPLS:
1634                 proto = htons(ETH_P_MPLS_UC);
1635                 fdret = FLOW_DISSECT_RET_PROTO_AGAIN;
1636                 break;
1637
1638         case IPPROTO_TCP:
1639                 __skb_flow_dissect_tcp(skb, flow_dissector, target_container,
1640                                        data, nhoff, hlen);
1641                 break;
1642
1643         case IPPROTO_ICMP:
1644         case IPPROTO_ICMPV6:
1645                 __skb_flow_dissect_icmp(skb, flow_dissector, target_container,
1646                                         data, nhoff, hlen);
1647                 break;
1648         case IPPROTO_L2TP:
1649                 __skb_flow_dissect_l2tpv3(skb, flow_dissector, target_container,
1650                                           data, nhoff, hlen);
1651                 break;
1652         case IPPROTO_ESP:
1653                 __skb_flow_dissect_esp(skb, flow_dissector, target_container,
1654                                        data, nhoff, hlen);
1655                 break;
1656         case IPPROTO_AH:
1657                 __skb_flow_dissect_ah(skb, flow_dissector, target_container,
1658                                       data, nhoff, hlen);
1659                 break;
1660         default:
1661                 break;
1662         }
1663
1664         if (!(key_control->flags & FLOW_DIS_IS_FRAGMENT))
1665                 __skb_flow_dissect_ports(skb, flow_dissector, target_container,
1666                                          data, nhoff, ip_proto, hlen);
1667
1668         /* Process result of IP proto processing */
1669         switch (fdret) {
1670         case FLOW_DISSECT_RET_PROTO_AGAIN:
1671                 if (skb_flow_dissect_allowed(&num_hdrs))
1672                         goto proto_again;
1673                 break;
1674         case FLOW_DISSECT_RET_IPPROTO_AGAIN:
1675                 if (skb_flow_dissect_allowed(&num_hdrs))
1676                         goto ip_proto_again;
1677                 break;
1678         case FLOW_DISSECT_RET_OUT_GOOD:
1679         case FLOW_DISSECT_RET_CONTINUE:
1680                 break;
1681         case FLOW_DISSECT_RET_OUT_BAD:
1682         default:
1683                 goto out_bad;
1684         }
1685
1686 out_good:
1687         ret = true;
1688
1689 out:
1690         key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
1691         key_basic->n_proto = proto;
1692         key_basic->ip_proto = ip_proto;
1693
1694         return ret;
1695
1696 out_bad:
1697         ret = false;
1698         goto out;
1699 }
1700 EXPORT_SYMBOL(__skb_flow_dissect);
1701
1702 static siphash_aligned_key_t hashrnd;
1703 static __always_inline void __flow_hash_secret_init(void)
1704 {
1705         net_get_random_once(&hashrnd, sizeof(hashrnd));
1706 }
1707
1708 static const void *flow_keys_hash_start(const struct flow_keys *flow)
1709 {
1710         BUILD_BUG_ON(FLOW_KEYS_HASH_OFFSET % SIPHASH_ALIGNMENT);
1711         return &flow->FLOW_KEYS_HASH_START_FIELD;
1712 }
1713
1714 static inline size_t flow_keys_hash_length(const struct flow_keys *flow)
1715 {
1716         size_t diff = FLOW_KEYS_HASH_OFFSET + sizeof(flow->addrs);
1717
1718         BUILD_BUG_ON((sizeof(*flow) - FLOW_KEYS_HASH_OFFSET) % sizeof(u32));
1719
1720         switch (flow->control.addr_type) {
1721         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1722                 diff -= sizeof(flow->addrs.v4addrs);
1723                 break;
1724         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1725                 diff -= sizeof(flow->addrs.v6addrs);
1726                 break;
1727         case FLOW_DISSECTOR_KEY_TIPC:
1728                 diff -= sizeof(flow->addrs.tipckey);
1729                 break;
1730         }
1731         return sizeof(*flow) - diff;
1732 }
1733
1734 __be32 flow_get_u32_src(const struct flow_keys *flow)
1735 {
1736         switch (flow->control.addr_type) {
1737         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1738                 return flow->addrs.v4addrs.src;
1739         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1740                 return (__force __be32)ipv6_addr_hash(
1741                         &flow->addrs.v6addrs.src);
1742         case FLOW_DISSECTOR_KEY_TIPC:
1743                 return flow->addrs.tipckey.key;
1744         default:
1745                 return 0;
1746         }
1747 }
1748 EXPORT_SYMBOL(flow_get_u32_src);
1749
1750 __be32 flow_get_u32_dst(const struct flow_keys *flow)
1751 {
1752         switch (flow->control.addr_type) {
1753         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1754                 return flow->addrs.v4addrs.dst;
1755         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1756                 return (__force __be32)ipv6_addr_hash(
1757                         &flow->addrs.v6addrs.dst);
1758         default:
1759                 return 0;
1760         }
1761 }
1762 EXPORT_SYMBOL(flow_get_u32_dst);
1763
1764 /* Sort the source and destination IP and the ports,
1765  * to have consistent hash within the two directions
1766  */
1767 static inline void __flow_hash_consistentify(struct flow_keys *keys)
1768 {
1769         int addr_diff, i;
1770
1771         switch (keys->control.addr_type) {
1772         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
1773                 if ((__force u32)keys->addrs.v4addrs.dst <
1774                     (__force u32)keys->addrs.v4addrs.src)
1775                         swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst);
1776
1777                 if ((__force u16)keys->ports.dst <
1778                     (__force u16)keys->ports.src) {
1779                         swap(keys->ports.src, keys->ports.dst);
1780                 }
1781                 break;
1782         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
1783                 addr_diff = memcmp(&keys->addrs.v6addrs.dst,
1784                                    &keys->addrs.v6addrs.src,
1785                                    sizeof(keys->addrs.v6addrs.dst));
1786                 if (addr_diff < 0) {
1787                         for (i = 0; i < 4; i++)
1788                                 swap(keys->addrs.v6addrs.src.s6_addr32[i],
1789                                      keys->addrs.v6addrs.dst.s6_addr32[i]);
1790                 }
1791                 if ((__force u16)keys->ports.dst <
1792                     (__force u16)keys->ports.src) {
1793                         swap(keys->ports.src, keys->ports.dst);
1794                 }
1795                 break;
1796         }
1797 }
1798
1799 static inline u32 __flow_hash_from_keys(struct flow_keys *keys,
1800                                         const siphash_key_t *keyval)
1801 {
1802         u32 hash;
1803
1804         __flow_hash_consistentify(keys);
1805
1806         hash = siphash(flow_keys_hash_start(keys),
1807                        flow_keys_hash_length(keys), keyval);
1808         if (!hash)
1809                 hash = 1;
1810
1811         return hash;
1812 }
1813
1814 u32 flow_hash_from_keys(struct flow_keys *keys)
1815 {
1816         __flow_hash_secret_init();
1817         return __flow_hash_from_keys(keys, &hashrnd);
1818 }
1819 EXPORT_SYMBOL(flow_hash_from_keys);
1820
1821 u32 flow_hash_from_keys_seed(struct flow_keys *keys,
1822                              const siphash_key_t *keyval)
1823 {
1824         return __flow_hash_from_keys(keys, keyval);
1825 }
1826 EXPORT_SYMBOL(flow_hash_from_keys_seed);
1827
1828 static inline u32 ___skb_get_hash(const struct sk_buff *skb,
1829                                   struct flow_keys *keys,
1830                                   const siphash_key_t *keyval)
1831 {
1832         skb_flow_dissect_flow_keys(skb, keys,
1833                                    FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1834
1835         return __flow_hash_from_keys(keys, keyval);
1836 }
1837
1838 struct _flow_keys_digest_data {
1839         __be16  n_proto;
1840         u8      ip_proto;
1841         u8      padding;
1842         __be32  ports;
1843         __be32  src;
1844         __be32  dst;
1845 };
1846
1847 void make_flow_keys_digest(struct flow_keys_digest *digest,
1848                            const struct flow_keys *flow)
1849 {
1850         struct _flow_keys_digest_data *data =
1851             (struct _flow_keys_digest_data *)digest;
1852
1853         BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
1854
1855         memset(digest, 0, sizeof(*digest));
1856
1857         data->n_proto = flow->basic.n_proto;
1858         data->ip_proto = flow->basic.ip_proto;
1859         data->ports = flow->ports.ports;
1860         data->src = flow->addrs.v4addrs.src;
1861         data->dst = flow->addrs.v4addrs.dst;
1862 }
1863 EXPORT_SYMBOL(make_flow_keys_digest);
1864
1865 static struct flow_dissector flow_keys_dissector_symmetric __read_mostly;
1866
1867 u32 __skb_get_hash_symmetric_net(const struct net *net, const struct sk_buff *skb)
1868 {
1869         struct flow_keys keys;
1870
1871         __flow_hash_secret_init();
1872
1873         memset(&keys, 0, sizeof(keys));
1874         __skb_flow_dissect(net, skb, &flow_keys_dissector_symmetric,
1875                            &keys, NULL, 0, 0, 0, 0);
1876
1877         return __flow_hash_from_keys(&keys, &hashrnd);
1878 }
1879 EXPORT_SYMBOL_GPL(__skb_get_hash_symmetric_net);
1880
1881 /**
1882  * __skb_get_hash_net: calculate a flow hash
1883  * @net: associated network namespace, derived from @skb if NULL
1884  * @skb: sk_buff to calculate flow hash from
1885  *
1886  * This function calculates a flow hash based on src/dst addresses
1887  * and src/dst port numbers.  Sets hash in skb to non-zero hash value
1888  * on success, zero indicates no valid hash.  Also, sets l4_hash in skb
1889  * if hash is a canonical 4-tuple hash over transport ports.
1890  */
1891 void __skb_get_hash_net(const struct net *net, struct sk_buff *skb)
1892 {
1893         struct flow_keys keys;
1894         u32 hash;
1895
1896         memset(&keys, 0, sizeof(keys));
1897
1898         __skb_flow_dissect(net, skb, &flow_keys_dissector,
1899                            &keys, NULL, 0, 0, 0,
1900                            FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL);
1901
1902         __flow_hash_secret_init();
1903
1904         hash = __flow_hash_from_keys(&keys, &hashrnd);
1905
1906         __skb_set_sw_hash(skb, hash, flow_keys_have_l4(&keys));
1907 }
1908 EXPORT_SYMBOL(__skb_get_hash_net);
1909
1910 __u32 skb_get_hash_perturb(const struct sk_buff *skb,
1911                            const siphash_key_t *perturb)
1912 {
1913         struct flow_keys keys;
1914
1915         return ___skb_get_hash(skb, &keys, perturb);
1916 }
1917 EXPORT_SYMBOL(skb_get_hash_perturb);
1918
1919 u32 __skb_get_poff(const struct sk_buff *skb, const void *data,
1920                    const struct flow_keys_basic *keys, int hlen)
1921 {
1922         u32 poff = keys->control.thoff;
1923
1924         /* skip L4 headers for fragments after the first */
1925         if ((keys->control.flags & FLOW_DIS_IS_FRAGMENT) &&
1926             !(keys->control.flags & FLOW_DIS_FIRST_FRAG))
1927                 return poff;
1928
1929         switch (keys->basic.ip_proto) {
1930         case IPPROTO_TCP: {
1931                 /* access doff as u8 to avoid unaligned access */
1932                 const u8 *doff;
1933                 u8 _doff;
1934
1935                 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
1936                                             data, hlen, &_doff);
1937                 if (!doff)
1938                         return poff;
1939
1940                 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
1941                 break;
1942         }
1943         case IPPROTO_UDP:
1944         case IPPROTO_UDPLITE:
1945                 poff += sizeof(struct udphdr);
1946                 break;
1947         /* For the rest, we do not really care about header
1948          * extensions at this point for now.
1949          */
1950         case IPPROTO_ICMP:
1951                 poff += sizeof(struct icmphdr);
1952                 break;
1953         case IPPROTO_ICMPV6:
1954                 poff += sizeof(struct icmp6hdr);
1955                 break;
1956         case IPPROTO_IGMP:
1957                 poff += sizeof(struct igmphdr);
1958                 break;
1959         case IPPROTO_DCCP:
1960                 poff += sizeof(struct dccp_hdr);
1961                 break;
1962         case IPPROTO_SCTP:
1963                 poff += sizeof(struct sctphdr);
1964                 break;
1965         }
1966
1967         return poff;
1968 }
1969
1970 /**
1971  * skb_get_poff - get the offset to the payload
1972  * @skb: sk_buff to get the payload offset from
1973  *
1974  * The function will get the offset to the payload as far as it could
1975  * be dissected.  The main user is currently BPF, so that we can dynamically
1976  * truncate packets without needing to push actual payload to the user
1977  * space and can analyze headers only, instead.
1978  */
1979 u32 skb_get_poff(const struct sk_buff *skb)
1980 {
1981         struct flow_keys_basic keys;
1982
1983         if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys,
1984                                               NULL, 0, 0, 0, 0))
1985                 return 0;
1986
1987         return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
1988 }
1989
1990 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys)
1991 {
1992         memset(keys, 0, sizeof(*keys));
1993
1994         memcpy(&keys->addrs.v6addrs.src, &fl6->saddr,
1995             sizeof(keys->addrs.v6addrs.src));
1996         memcpy(&keys->addrs.v6addrs.dst, &fl6->daddr,
1997             sizeof(keys->addrs.v6addrs.dst));
1998         keys->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1999         keys->ports.src = fl6->fl6_sport;
2000         keys->ports.dst = fl6->fl6_dport;
2001         keys->keyid.keyid = fl6->fl6_gre_key;
2002         keys->tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
2003         keys->basic.ip_proto = fl6->flowi6_proto;
2004
2005         return flow_hash_from_keys(keys);
2006 }
2007 EXPORT_SYMBOL(__get_hash_from_flowi6);
2008
2009 static const struct flow_dissector_key flow_keys_dissector_keys[] = {
2010         {
2011                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
2012                 .offset = offsetof(struct flow_keys, control),
2013         },
2014         {
2015                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
2016                 .offset = offsetof(struct flow_keys, basic),
2017         },
2018         {
2019                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
2020                 .offset = offsetof(struct flow_keys, addrs.v4addrs),
2021         },
2022         {
2023                 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
2024                 .offset = offsetof(struct flow_keys, addrs.v6addrs),
2025         },
2026         {
2027                 .key_id = FLOW_DISSECTOR_KEY_TIPC,
2028                 .offset = offsetof(struct flow_keys, addrs.tipckey),
2029         },
2030         {
2031                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
2032                 .offset = offsetof(struct flow_keys, ports),
2033         },
2034         {
2035                 .key_id = FLOW_DISSECTOR_KEY_VLAN,
2036                 .offset = offsetof(struct flow_keys, vlan),
2037         },
2038         {
2039                 .key_id = FLOW_DISSECTOR_KEY_FLOW_LABEL,
2040                 .offset = offsetof(struct flow_keys, tags),
2041         },
2042         {
2043                 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
2044                 .offset = offsetof(struct flow_keys, keyid),
2045         },
2046 };
2047
2048 static const struct flow_dissector_key flow_keys_dissector_symmetric_keys[] = {
2049         {
2050                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
2051                 .offset = offsetof(struct flow_keys, control),
2052         },
2053         {
2054                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
2055                 .offset = offsetof(struct flow_keys, basic),
2056         },
2057         {
2058                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
2059                 .offset = offsetof(struct flow_keys, addrs.v4addrs),
2060         },
2061         {
2062                 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
2063                 .offset = offsetof(struct flow_keys, addrs.v6addrs),
2064         },
2065         {
2066                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
2067                 .offset = offsetof(struct flow_keys, ports),
2068         },
2069 };
2070
2071 static const struct flow_dissector_key flow_keys_basic_dissector_keys[] = {
2072         {
2073                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
2074                 .offset = offsetof(struct flow_keys, control),
2075         },
2076         {
2077                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
2078                 .offset = offsetof(struct flow_keys, basic),
2079         },
2080 };
2081
2082 struct flow_dissector flow_keys_dissector __read_mostly;
2083 EXPORT_SYMBOL(flow_keys_dissector);
2084
2085 struct flow_dissector flow_keys_basic_dissector __read_mostly;
2086 EXPORT_SYMBOL(flow_keys_basic_dissector);
2087
2088 static int __init init_default_flow_dissectors(void)
2089 {
2090         skb_flow_dissector_init(&flow_keys_dissector,
2091                                 flow_keys_dissector_keys,
2092                                 ARRAY_SIZE(flow_keys_dissector_keys));
2093         skb_flow_dissector_init(&flow_keys_dissector_symmetric,
2094                                 flow_keys_dissector_symmetric_keys,
2095                                 ARRAY_SIZE(flow_keys_dissector_symmetric_keys));
2096         skb_flow_dissector_init(&flow_keys_basic_dissector,
2097                                 flow_keys_basic_dissector_keys,
2098                                 ARRAY_SIZE(flow_keys_basic_dissector_keys));
2099         return 0;
2100 }
2101 core_initcall(init_default_flow_dissectors);