1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2018 Mellanox Technologies. */
9 static int get_route_and_out_devs(struct mlx5e_priv *priv,
10 struct net_device *dev,
11 struct net_device **route_dev,
12 struct net_device **out_dev)
14 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
15 struct net_device *uplink_dev, *uplink_upper;
18 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
19 uplink_upper = netdev_master_upper_dev_get(uplink_dev);
20 dst_is_lag_dev = (uplink_upper &&
21 netif_is_lag_master(uplink_upper) &&
22 dev == uplink_upper &&
23 mlx5_lag_is_sriov(priv->mdev));
25 /* if the egress device isn't on the same HW e-switch or
26 * it's a LAG device, use the uplink
28 if (!switchdev_port_same_parent_id(priv->netdev, dev) ||
30 *route_dev = uplink_dev;
31 *out_dev = *route_dev;
34 if (is_vlan_dev(*route_dev))
35 *out_dev = uplink_dev;
36 else if (mlx5e_eswitch_rep(dev))
37 *out_dev = *route_dev;
45 static int mlx5e_route_lookup_ipv4(struct mlx5e_priv *priv,
46 struct net_device *mirred_dev,
47 struct net_device **out_dev,
48 struct net_device **route_dev,
50 struct neighbour **out_n,
54 struct neighbour *n = NULL;
56 #if IS_ENABLED(CONFIG_INET)
59 rt = ip_route_output_key(dev_net(mirred_dev), fl4);
60 ret = PTR_ERR_OR_ZERO(rt);
67 ret = get_route_and_out_devs(priv, rt->dst.dev, route_dev, out_dev);
72 *out_ttl = ip4_dst_hoplimit(&rt->dst);
73 n = dst_neigh_lookup(&rt->dst, &fl4->daddr);
82 static const char *mlx5e_netdev_kind(struct net_device *dev)
84 if (dev->rtnl_link_ops)
85 return dev->rtnl_link_ops->kind;
90 static int mlx5e_route_lookup_ipv6(struct mlx5e_priv *priv,
91 struct net_device *mirred_dev,
92 struct net_device **out_dev,
93 struct net_device **route_dev,
95 struct neighbour **out_n,
98 struct neighbour *n = NULL;
99 struct dst_entry *dst;
101 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
104 ret = ipv6_stub->ipv6_dst_lookup(dev_net(mirred_dev), NULL, &dst,
110 *out_ttl = ip6_dst_hoplimit(dst);
112 ret = get_route_and_out_devs(priv, dst->dev, route_dev, out_dev);
119 n = dst_neigh_lookup(dst, &fl6->daddr);
128 static int mlx5e_gen_vxlan_header(char buf[], struct ip_tunnel_key *tun_key)
130 __be32 tun_id = tunnel_id_to_key32(tun_key->tun_id);
131 struct udphdr *udp = (struct udphdr *)(buf);
132 struct vxlanhdr *vxh = (struct vxlanhdr *)
133 ((char *)udp + sizeof(struct udphdr));
135 udp->dest = tun_key->tp_dst;
136 vxh->vx_flags = VXLAN_HF_VNI;
137 vxh->vx_vni = vxlan_vni_field(tun_id);
142 static int mlx5e_gen_gre_header(char buf[], struct ip_tunnel_key *tun_key)
144 __be32 tun_id = tunnel_id_to_key32(tun_key->tun_id);
146 struct gre_base_hdr *greh = (struct gre_base_hdr *)(buf);
148 /* the HW does not calculate GRE csum or sequences */
149 if (tun_key->tun_flags & (TUNNEL_CSUM | TUNNEL_SEQ))
152 greh->protocol = htons(ETH_P_TEB);
155 hdr_len = gre_calc_hlen(tun_key->tun_flags);
156 greh->flags = gre_tnl_flags_to_gre_flags(tun_key->tun_flags);
157 if (tun_key->tun_flags & TUNNEL_KEY) {
158 __be32 *ptr = (__be32 *)(((u8 *)greh) + hdr_len - 4);
166 static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto,
167 struct mlx5e_encap_entry *e)
170 struct ip_tunnel_key *key = &e->tun_info.key;
172 if (e->tunnel_type == MLX5E_TC_TUNNEL_TYPE_VXLAN) {
173 *ip_proto = IPPROTO_UDP;
174 err = mlx5e_gen_vxlan_header(buf, key);
175 } else if (e->tunnel_type == MLX5E_TC_TUNNEL_TYPE_GRETAP) {
176 *ip_proto = IPPROTO_GRE;
177 err = mlx5e_gen_gre_header(buf, key);
179 pr_warn("mlx5: Cannot generate tunnel header for tunnel type (%d)\n"
187 static char *gen_eth_tnl_hdr(char *buf, struct net_device *dev,
188 struct mlx5e_encap_entry *e,
191 struct ethhdr *eth = (struct ethhdr *)buf;
194 ether_addr_copy(eth->h_dest, e->h_dest);
195 ether_addr_copy(eth->h_source, dev->dev_addr);
196 if (is_vlan_dev(dev)) {
197 struct vlan_hdr *vlan = (struct vlan_hdr *)
198 ((char *)eth + ETH_HLEN);
199 ip = (char *)vlan + VLAN_HLEN;
200 eth->h_proto = vlan_dev_vlan_proto(dev);
201 vlan->h_vlan_TCI = htons(vlan_dev_vlan_id(dev));
202 vlan->h_vlan_encapsulated_proto = htons(proto);
204 eth->h_proto = htons(proto);
205 ip = (char *)eth + ETH_HLEN;
211 int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv,
212 struct net_device *mirred_dev,
213 struct mlx5e_encap_entry *e)
215 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
216 struct ip_tunnel_key *tun_key = &e->tun_info.key;
217 struct net_device *out_dev, *route_dev;
218 struct neighbour *n = NULL;
219 struct flowi4 fl4 = {};
226 /* add the IP fields */
227 fl4.flowi4_tos = tun_key->tos;
228 fl4.daddr = tun_key->u.ipv4.dst;
229 fl4.saddr = tun_key->u.ipv4.src;
232 err = mlx5e_route_lookup_ipv4(priv, mirred_dev, &out_dev, &route_dev,
238 (is_vlan_dev(route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
239 sizeof(struct iphdr) +
242 if (max_encap_size < ipv4_encap_size) {
243 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
244 ipv4_encap_size, max_encap_size);
248 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
252 /* used by mlx5e_detach_encap to lookup a neigh hash table
253 * entry in the neigh hash table when a user deletes a rule
255 e->m_neigh.dev = n->dev;
256 e->m_neigh.family = n->ops->family;
257 memcpy(&e->m_neigh.dst_ip, n->primary_key, n->tbl->key_len);
258 e->out_dev = out_dev;
259 e->route_dev = route_dev;
261 /* It's important to add the neigh to the hash table before checking
262 * the neigh validity state. So if we'll get a notification, in case the
263 * neigh changes it's validity state, we would find the relevant neigh
266 err = mlx5e_rep_encap_entry_attach(netdev_priv(out_dev), e);
270 read_lock_bh(&n->lock);
271 nud_state = n->nud_state;
272 ether_addr_copy(e->h_dest, n->ha);
273 read_unlock_bh(&n->lock);
275 /* add ethernet header */
276 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, route_dev, e,
280 ip->tos = tun_key->tos;
284 ip->daddr = fl4.daddr;
285 ip->saddr = fl4.saddr;
287 /* add tunneling protocol header */
288 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
291 goto destroy_neigh_entry;
293 e->encap_size = ipv4_encap_size;
294 e->encap_header = encap_header;
296 if (!(nud_state & NUD_VALID)) {
297 neigh_event_send(n, NULL);
302 err = mlx5_packet_reformat_alloc(priv->mdev,
304 ipv4_encap_size, encap_header,
305 MLX5_FLOW_NAMESPACE_FDB,
308 goto destroy_neigh_entry;
310 e->flags |= MLX5_ENCAP_ENTRY_VALID;
311 mlx5e_rep_queue_neigh_stats_work(netdev_priv(out_dev));
316 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
325 int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv,
326 struct net_device *mirred_dev,
327 struct mlx5e_encap_entry *e)
329 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
330 struct ip_tunnel_key *tun_key = &e->tun_info.key;
331 struct net_device *out_dev, *route_dev;
332 struct neighbour *n = NULL;
333 struct flowi6 fl6 = {};
334 struct ipv6hdr *ip6h;
342 fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label);
343 fl6.daddr = tun_key->u.ipv6.dst;
344 fl6.saddr = tun_key->u.ipv6.src;
346 err = mlx5e_route_lookup_ipv6(priv, mirred_dev, &out_dev, &route_dev,
352 (is_vlan_dev(route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
353 sizeof(struct ipv6hdr) +
356 if (max_encap_size < ipv6_encap_size) {
357 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
358 ipv6_encap_size, max_encap_size);
362 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
366 /* used by mlx5e_detach_encap to lookup a neigh hash table
367 * entry in the neigh hash table when a user deletes a rule
369 e->m_neigh.dev = n->dev;
370 e->m_neigh.family = n->ops->family;
371 memcpy(&e->m_neigh.dst_ip, n->primary_key, n->tbl->key_len);
372 e->out_dev = out_dev;
373 e->route_dev = route_dev;
375 /* It's importent to add the neigh to the hash table before checking
376 * the neigh validity state. So if we'll get a notification, in case the
377 * neigh changes it's validity state, we would find the relevant neigh
380 err = mlx5e_rep_encap_entry_attach(netdev_priv(out_dev), e);
384 read_lock_bh(&n->lock);
385 nud_state = n->nud_state;
386 ether_addr_copy(e->h_dest, n->ha);
387 read_unlock_bh(&n->lock);
389 /* add ethernet header */
390 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, route_dev, e,
394 ip6_flow_hdr(ip6h, tun_key->tos, 0);
395 /* the HW fills up ipv6 payload len */
396 ip6h->hop_limit = ttl;
397 ip6h->daddr = fl6.daddr;
398 ip6h->saddr = fl6.saddr;
400 /* add tunneling protocol header */
401 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
404 goto destroy_neigh_entry;
406 e->encap_size = ipv6_encap_size;
407 e->encap_header = encap_header;
409 if (!(nud_state & NUD_VALID)) {
410 neigh_event_send(n, NULL);
415 err = mlx5_packet_reformat_alloc(priv->mdev,
417 ipv6_encap_size, encap_header,
418 MLX5_FLOW_NAMESPACE_FDB,
421 goto destroy_neigh_entry;
423 e->flags |= MLX5_ENCAP_ENTRY_VALID;
424 mlx5e_rep_queue_neigh_stats_work(netdev_priv(out_dev));
429 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
438 int mlx5e_tc_tun_get_type(struct net_device *tunnel_dev)
440 if (netif_is_vxlan(tunnel_dev))
441 return MLX5E_TC_TUNNEL_TYPE_VXLAN;
442 else if (netif_is_gretap(tunnel_dev) ||
443 netif_is_ip6gretap(tunnel_dev))
444 return MLX5E_TC_TUNNEL_TYPE_GRETAP;
446 return MLX5E_TC_TUNNEL_TYPE_UNKNOWN;
449 bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv,
450 struct net_device *netdev)
452 int tunnel_type = mlx5e_tc_tun_get_type(netdev);
454 if (tunnel_type == MLX5E_TC_TUNNEL_TYPE_VXLAN &&
455 MLX5_CAP_ESW(priv->mdev, vxlan_encap_decap))
457 else if (tunnel_type == MLX5E_TC_TUNNEL_TYPE_GRETAP &&
458 MLX5_CAP_ESW(priv->mdev, nvgre_encap_decap))
464 int mlx5e_tc_tun_init_encap_attr(struct net_device *tunnel_dev,
465 struct mlx5e_priv *priv,
466 struct mlx5e_encap_entry *e,
467 struct netlink_ext_ack *extack)
469 e->tunnel_type = mlx5e_tc_tun_get_type(tunnel_dev);
471 if (e->tunnel_type == MLX5E_TC_TUNNEL_TYPE_VXLAN) {
472 int dst_port = be16_to_cpu(e->tun_info.key.tp_dst);
474 if (!mlx5_vxlan_lookup_port(priv->mdev->vxlan, dst_port)) {
475 NL_SET_ERR_MSG_MOD(extack,
476 "vxlan udp dport was not registered with the HW");
477 netdev_warn(priv->netdev,
478 "%d isn't an offloaded vxlan udp dport\n",
482 e->reformat_type = MLX5_REFORMAT_TYPE_L2_TO_VXLAN;
483 e->tunnel_hlen = VXLAN_HLEN;
484 } else if (e->tunnel_type == MLX5E_TC_TUNNEL_TYPE_GRETAP) {
485 e->reformat_type = MLX5_REFORMAT_TYPE_L2_TO_NVGRE;
486 e->tunnel_hlen = gre_calc_hlen(e->tun_info.key.tun_flags);
488 e->reformat_type = -1;
495 static int mlx5e_tc_tun_parse_vxlan(struct mlx5e_priv *priv,
496 struct mlx5_flow_spec *spec,
497 struct tc_cls_flower_offload *f,
501 struct netlink_ext_ack *extack = f->common.extack;
502 struct flow_dissector_key_ports *key =
503 skb_flow_dissector_target(f->dissector,
504 FLOW_DISSECTOR_KEY_ENC_PORTS,
506 struct flow_dissector_key_ports *mask =
507 skb_flow_dissector_target(f->dissector,
508 FLOW_DISSECTOR_KEY_ENC_PORTS,
510 void *misc_c = MLX5_ADDR_OF(fte_match_param,
511 spec->match_criteria,
513 void *misc_v = MLX5_ADDR_OF(fte_match_param,
517 /* Full udp dst port must be given */
518 if (!dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS) ||
519 memchr_inv(&mask->dst, 0xff, sizeof(mask->dst))) {
520 NL_SET_ERR_MSG_MOD(extack,
521 "VXLAN decap filter must include enc_dst_port condition");
522 netdev_warn(priv->netdev,
523 "VXLAN decap filter must include enc_dst_port condition\n");
527 /* udp dst port must be knonwn as a VXLAN port */
528 if (!mlx5_vxlan_lookup_port(priv->mdev->vxlan, be16_to_cpu(key->dst))) {
529 NL_SET_ERR_MSG_MOD(extack,
530 "Matched UDP port is not registered as a VXLAN port");
531 netdev_warn(priv->netdev,
532 "UDP port %d is not registered as a VXLAN port\n",
533 be16_to_cpu(key->dst));
537 /* dst UDP port is valid here */
538 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
539 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
541 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport, ntohs(mask->dst));
542 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport, ntohs(key->dst));
544 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport, ntohs(mask->src));
545 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport, ntohs(key->src));
548 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
549 struct flow_dissector_key_keyid *key =
550 skb_flow_dissector_target(f->dissector,
551 FLOW_DISSECTOR_KEY_ENC_KEYID,
553 struct flow_dissector_key_keyid *mask =
554 skb_flow_dissector_target(f->dissector,
555 FLOW_DISSECTOR_KEY_ENC_KEYID,
557 MLX5_SET(fte_match_set_misc, misc_c, vxlan_vni,
558 be32_to_cpu(mask->keyid));
559 MLX5_SET(fte_match_set_misc, misc_v, vxlan_vni,
560 be32_to_cpu(key->keyid));
565 static int mlx5e_tc_tun_parse_gretap(struct mlx5e_priv *priv,
566 struct mlx5_flow_spec *spec,
567 struct tc_cls_flower_offload *f,
568 void *outer_headers_c,
569 void *outer_headers_v)
571 void *misc_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
573 void *misc_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
576 if (!MLX5_CAP_ESW(priv->mdev, nvgre_encap_decap)) {
577 NL_SET_ERR_MSG_MOD(f->common.extack,
578 "GRE HW offloading is not supported");
579 netdev_warn(priv->netdev, "GRE HW offloading is not supported\n");
583 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
584 MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
585 ip_protocol, IPPROTO_GRE);
588 MLX5_SET_TO_ONES(fte_match_set_misc, misc_c, gre_protocol);
589 MLX5_SET(fte_match_set_misc, misc_v, gre_protocol, ETH_P_TEB);
592 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
593 struct flow_dissector_key_keyid *mask = NULL;
594 struct flow_dissector_key_keyid *key = NULL;
596 mask = skb_flow_dissector_target(f->dissector,
597 FLOW_DISSECTOR_KEY_ENC_KEYID,
599 MLX5_SET(fte_match_set_misc, misc_c,
600 gre_key.key, be32_to_cpu(mask->keyid));
602 key = skb_flow_dissector_target(f->dissector,
603 FLOW_DISSECTOR_KEY_ENC_KEYID,
605 MLX5_SET(fte_match_set_misc, misc_v,
606 gre_key.key, be32_to_cpu(key->keyid));
612 int mlx5e_tc_tun_parse(struct net_device *filter_dev,
613 struct mlx5e_priv *priv,
614 struct mlx5_flow_spec *spec,
615 struct tc_cls_flower_offload *f,
617 void *headers_v, u8 *match_level)
622 tunnel_type = mlx5e_tc_tun_get_type(filter_dev);
623 if (tunnel_type == MLX5E_TC_TUNNEL_TYPE_VXLAN) {
624 *match_level = MLX5_MATCH_L4;
625 err = mlx5e_tc_tun_parse_vxlan(priv, spec, f,
626 headers_c, headers_v);
627 } else if (tunnel_type == MLX5E_TC_TUNNEL_TYPE_GRETAP) {
628 *match_level = MLX5_MATCH_L3;
629 err = mlx5e_tc_tun_parse_gretap(priv, spec, f,
630 headers_c, headers_v);
632 netdev_warn(priv->netdev,
633 "decapsulation offload is not supported for %s net device (%d)\n",
634 mlx5e_netdev_kind(filter_dev), tunnel_type);