1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
4 #include <linux/skbuff.h>
5 #include <net/devlink.h>
6 #include <net/pkt_cls.h>
10 #include "../nfpcore/nfp_cpp.h"
11 #include "../nfpcore/nfp_nsp.h"
12 #include "../nfp_app.h"
13 #include "../nfp_main.h"
14 #include "../nfp_net.h"
15 #include "../nfp_port.h"
17 #define NFP_FLOWER_SUPPORTED_TCPFLAGS \
18 (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST | \
19 TCPHDR_PSH | TCPHDR_URG)
21 #define NFP_FLOWER_SUPPORTED_CTLFLAGS \
22 (FLOW_DIS_IS_FRAGMENT | \
25 #define NFP_FLOWER_WHITELIST_DISSECTOR \
26 (BIT(FLOW_DISSECTOR_KEY_CONTROL) | \
27 BIT(FLOW_DISSECTOR_KEY_BASIC) | \
28 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | \
29 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | \
30 BIT(FLOW_DISSECTOR_KEY_TCP) | \
31 BIT(FLOW_DISSECTOR_KEY_PORTS) | \
32 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | \
33 BIT(FLOW_DISSECTOR_KEY_VLAN) | \
34 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | \
35 BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \
36 BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \
37 BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
38 BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | \
39 BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) | \
40 BIT(FLOW_DISSECTOR_KEY_ENC_IP) | \
41 BIT(FLOW_DISSECTOR_KEY_MPLS) | \
42 BIT(FLOW_DISSECTOR_KEY_IP))
44 #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR \
45 (BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
46 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | \
47 BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \
48 BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | \
49 BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) | \
50 BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | \
51 BIT(FLOW_DISSECTOR_KEY_ENC_IP))
53 #define NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R \
54 (BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL) | \
55 BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) | \
56 BIT(FLOW_DISSECTOR_KEY_ENC_PORTS))
58 #define NFP_FLOWER_MERGE_FIELDS \
59 (NFP_FLOWER_LAYER_PORT | \
60 NFP_FLOWER_LAYER_MAC | \
61 NFP_FLOWER_LAYER_TP | \
62 NFP_FLOWER_LAYER_IPV4 | \
63 NFP_FLOWER_LAYER_IPV6)
65 struct nfp_flower_merge_check {
69 struct nfp_flower_mac_mpls l2;
70 struct nfp_flower_tp_ports l4;
72 struct nfp_flower_ipv4 ipv4;
73 struct nfp_flower_ipv6 ipv6;
76 unsigned long vals[8];
81 nfp_flower_xmit_flow(struct nfp_app *app, struct nfp_fl_payload *nfp_flow,
84 u32 meta_len, key_len, mask_len, act_len, tot_len;
88 meta_len = sizeof(struct nfp_fl_rule_metadata);
89 key_len = nfp_flow->meta.key_len;
90 mask_len = nfp_flow->meta.mask_len;
91 act_len = nfp_flow->meta.act_len;
93 tot_len = meta_len + key_len + mask_len + act_len;
95 /* Convert to long words as firmware expects
96 * lengths in units of NFP_FL_LW_SIZ.
98 nfp_flow->meta.key_len >>= NFP_FL_LW_SIZ;
99 nfp_flow->meta.mask_len >>= NFP_FL_LW_SIZ;
100 nfp_flow->meta.act_len >>= NFP_FL_LW_SIZ;
102 skb = nfp_flower_cmsg_alloc(app, tot_len, mtype, GFP_KERNEL);
106 msg = nfp_flower_cmsg_get_data(skb);
107 memcpy(msg, &nfp_flow->meta, meta_len);
108 memcpy(&msg[meta_len], nfp_flow->unmasked_data, key_len);
109 memcpy(&msg[meta_len + key_len], nfp_flow->mask_data, mask_len);
110 memcpy(&msg[meta_len + key_len + mask_len],
111 nfp_flow->action_data, act_len);
113 /* Convert back to bytes as software expects
114 * lengths in units of bytes.
116 nfp_flow->meta.key_len <<= NFP_FL_LW_SIZ;
117 nfp_flow->meta.mask_len <<= NFP_FL_LW_SIZ;
118 nfp_flow->meta.act_len <<= NFP_FL_LW_SIZ;
120 nfp_ctrl_tx(app->ctrl, skb);
125 static bool nfp_flower_check_higher_than_mac(struct tc_cls_flower_offload *f)
127 struct flow_rule *rule = tc_cls_flower_offload_flow_rule(f);
129 return flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS) ||
130 flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS) ||
131 flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS) ||
132 flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ICMP);
136 nfp_flower_calc_opt_layer(struct flow_match_enc_opts *enc_opts,
137 u32 *key_layer_two, int *key_size)
139 if (enc_opts->key->len > NFP_FL_MAX_GENEVE_OPT_KEY)
142 if (enc_opts->key->len > 0) {
143 *key_layer_two |= NFP_FLOWER_LAYER2_GENEVE_OP;
144 *key_size += sizeof(struct nfp_flower_geneve_options);
151 nfp_flower_calculate_key_layers(struct nfp_app *app,
152 struct net_device *netdev,
153 struct nfp_fl_key_ls *ret_key_ls,
154 struct tc_cls_flower_offload *flow,
155 enum nfp_flower_tun_type *tun_type)
157 struct flow_rule *rule = tc_cls_flower_offload_flow_rule(flow);
158 struct flow_dissector *dissector = rule->match.dissector;
159 struct flow_match_basic basic = { NULL, NULL};
160 struct nfp_flower_priv *priv = app->priv;
166 if (dissector->used_keys & ~NFP_FLOWER_WHITELIST_DISSECTOR)
169 /* If any tun dissector is used then the required set must be used. */
170 if (dissector->used_keys & NFP_FLOWER_WHITELIST_TUN_DISSECTOR &&
171 (dissector->used_keys & NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R)
172 != NFP_FLOWER_WHITELIST_TUN_DISSECTOR_R)
176 key_layer = NFP_FLOWER_LAYER_PORT;
177 key_size = sizeof(struct nfp_flower_meta_tci) +
178 sizeof(struct nfp_flower_in_port);
180 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS) ||
181 flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_MPLS)) {
182 key_layer |= NFP_FLOWER_LAYER_MAC;
183 key_size += sizeof(struct nfp_flower_mac_mpls);
186 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
187 struct flow_match_vlan vlan;
189 flow_rule_match_vlan(rule, &vlan);
190 if (!(priv->flower_ext_feats & NFP_FL_FEATS_VLAN_PCP) &&
191 vlan.key->vlan_priority)
195 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
196 struct flow_match_enc_opts enc_op = { NULL, NULL };
197 struct flow_match_ipv4_addrs ipv4_addrs;
198 struct flow_match_control enc_ctl;
199 struct flow_match_ports enc_ports;
201 flow_rule_match_enc_control(rule, &enc_ctl);
203 if (enc_ctl.mask->addr_type != 0xffff ||
204 enc_ctl.key->addr_type != FLOW_DISSECTOR_KEY_IPV4_ADDRS)
207 /* These fields are already verified as used. */
208 flow_rule_match_enc_ipv4_addrs(rule, &ipv4_addrs);
209 if (ipv4_addrs.mask->dst != cpu_to_be32(~0))
212 flow_rule_match_enc_ports(rule, &enc_ports);
213 if (enc_ports.mask->dst != cpu_to_be16(~0))
216 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_OPTS))
217 flow_rule_match_enc_opts(rule, &enc_op);
219 switch (enc_ports.key->dst) {
220 case htons(IANA_VXLAN_UDP_PORT):
221 *tun_type = NFP_FL_TUNNEL_VXLAN;
222 key_layer |= NFP_FLOWER_LAYER_VXLAN;
223 key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
228 case htons(GENEVE_UDP_PORT):
229 if (!(priv->flower_ext_feats & NFP_FL_FEATS_GENEVE))
231 *tun_type = NFP_FL_TUNNEL_GENEVE;
232 key_layer |= NFP_FLOWER_LAYER_EXT_META;
233 key_size += sizeof(struct nfp_flower_ext_meta);
234 key_layer_two |= NFP_FLOWER_LAYER2_GENEVE;
235 key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
239 if (!(priv->flower_ext_feats & NFP_FL_FEATS_GENEVE_OPT))
241 err = nfp_flower_calc_opt_layer(&enc_op, &key_layer_two,
250 /* Ensure the ingress netdev matches the expected tun type. */
251 if (!nfp_fl_netdev_is_tunnel_type(netdev, *tun_type))
255 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC))
256 flow_rule_match_basic(rule, &basic);
258 if (basic.mask && basic.mask->n_proto) {
259 /* Ethernet type is present in the key. */
260 switch (basic.key->n_proto) {
261 case cpu_to_be16(ETH_P_IP):
262 key_layer |= NFP_FLOWER_LAYER_IPV4;
263 key_size += sizeof(struct nfp_flower_ipv4);
266 case cpu_to_be16(ETH_P_IPV6):
267 key_layer |= NFP_FLOWER_LAYER_IPV6;
268 key_size += sizeof(struct nfp_flower_ipv6);
271 /* Currently we do not offload ARP
272 * because we rely on it to get to the host.
274 case cpu_to_be16(ETH_P_ARP):
277 case cpu_to_be16(ETH_P_MPLS_UC):
278 case cpu_to_be16(ETH_P_MPLS_MC):
279 if (!(key_layer & NFP_FLOWER_LAYER_MAC)) {
280 key_layer |= NFP_FLOWER_LAYER_MAC;
281 key_size += sizeof(struct nfp_flower_mac_mpls);
285 /* Will be included in layer 2. */
286 case cpu_to_be16(ETH_P_8021Q):
290 /* Other ethtype - we need check the masks for the
291 * remainder of the key to ensure we can offload.
293 if (nfp_flower_check_higher_than_mac(flow))
299 if (basic.mask && basic.mask->ip_proto) {
300 /* Ethernet type is present in the key. */
301 switch (basic.key->ip_proto) {
307 key_layer |= NFP_FLOWER_LAYER_TP;
308 key_size += sizeof(struct nfp_flower_tp_ports);
311 /* Other ip proto - we need check the masks for the
312 * remainder of the key to ensure we can offload.
318 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
319 struct flow_match_tcp tcp;
322 flow_rule_match_tcp(rule, &tcp);
323 tcp_flags = be16_to_cpu(tcp.key->flags);
325 if (tcp_flags & ~NFP_FLOWER_SUPPORTED_TCPFLAGS)
328 /* We only support PSH and URG flags when either
329 * FIN, SYN or RST is present as well.
331 if ((tcp_flags & (TCPHDR_PSH | TCPHDR_URG)) &&
332 !(tcp_flags & (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST)))
335 /* We need to store TCP flags in the either the IPv4 or IPv6 key
336 * space, thus we need to ensure we include a IPv4/IPv6 key
337 * layer if we have not done so already.
342 if (!(key_layer & NFP_FLOWER_LAYER_IPV4) &&
343 !(key_layer & NFP_FLOWER_LAYER_IPV6)) {
344 switch (basic.key->n_proto) {
345 case cpu_to_be16(ETH_P_IP):
346 key_layer |= NFP_FLOWER_LAYER_IPV4;
347 key_size += sizeof(struct nfp_flower_ipv4);
350 case cpu_to_be16(ETH_P_IPV6):
351 key_layer |= NFP_FLOWER_LAYER_IPV6;
352 key_size += sizeof(struct nfp_flower_ipv6);
361 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
362 struct flow_match_control ctl;
364 flow_rule_match_control(rule, &ctl);
365 if (ctl.key->flags & ~NFP_FLOWER_SUPPORTED_CTLFLAGS)
369 ret_key_ls->key_layer = key_layer;
370 ret_key_ls->key_layer_two = key_layer_two;
371 ret_key_ls->key_size = key_size;
376 static struct nfp_fl_payload *
377 nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer)
379 struct nfp_fl_payload *flow_pay;
381 flow_pay = kmalloc(sizeof(*flow_pay), GFP_KERNEL);
385 flow_pay->meta.key_len = key_layer->key_size;
386 flow_pay->unmasked_data = kmalloc(key_layer->key_size, GFP_KERNEL);
387 if (!flow_pay->unmasked_data)
390 flow_pay->meta.mask_len = key_layer->key_size;
391 flow_pay->mask_data = kmalloc(key_layer->key_size, GFP_KERNEL);
392 if (!flow_pay->mask_data)
393 goto err_free_unmasked;
395 flow_pay->action_data = kmalloc(NFP_FL_MAX_A_SIZ, GFP_KERNEL);
396 if (!flow_pay->action_data)
399 flow_pay->nfp_tun_ipv4_addr = 0;
400 flow_pay->meta.flags = 0;
401 INIT_LIST_HEAD(&flow_pay->linked_flows);
402 flow_pay->in_hw = false;
407 kfree(flow_pay->mask_data);
409 kfree(flow_pay->unmasked_data);
416 nfp_flower_update_merge_with_actions(struct nfp_fl_payload *flow,
417 struct nfp_flower_merge_check *merge,
418 u8 *last_act_id, int *act_out)
420 struct nfp_fl_set_ipv6_tc_hl_fl *ipv6_tc_hl_fl;
421 struct nfp_fl_set_ip4_ttl_tos *ipv4_ttl_tos;
422 struct nfp_fl_set_ip4_addrs *ipv4_add;
423 struct nfp_fl_set_ipv6_addr *ipv6_add;
424 struct nfp_fl_push_vlan *push_vlan;
425 struct nfp_fl_set_tport *tport;
426 struct nfp_fl_set_eth *eth;
427 struct nfp_fl_act_head *a;
428 unsigned int act_off = 0;
433 while (act_off < flow->meta.act_len) {
434 a = (struct nfp_fl_act_head *)&flow->action_data[act_off];
438 case NFP_FL_ACTION_OPCODE_OUTPUT:
442 case NFP_FL_ACTION_OPCODE_PUSH_VLAN:
443 push_vlan = (struct nfp_fl_push_vlan *)a;
444 if (push_vlan->vlan_tci)
445 merge->tci = cpu_to_be16(0xffff);
447 case NFP_FL_ACTION_OPCODE_POP_VLAN:
448 merge->tci = cpu_to_be16(0);
450 case NFP_FL_ACTION_OPCODE_SET_IPV4_TUNNEL:
451 /* New tunnel header means l2 to l4 can be matched. */
452 eth_broadcast_addr(&merge->l2.mac_dst[0]);
453 eth_broadcast_addr(&merge->l2.mac_src[0]);
454 memset(&merge->l4, 0xff,
455 sizeof(struct nfp_flower_tp_ports));
456 memset(&merge->ipv4, 0xff,
457 sizeof(struct nfp_flower_ipv4));
459 case NFP_FL_ACTION_OPCODE_SET_ETHERNET:
460 eth = (struct nfp_fl_set_eth *)a;
461 for (i = 0; i < ETH_ALEN; i++)
462 merge->l2.mac_dst[i] |= eth->eth_addr_mask[i];
463 for (i = 0; i < ETH_ALEN; i++)
464 merge->l2.mac_src[i] |=
465 eth->eth_addr_mask[ETH_ALEN + i];
467 case NFP_FL_ACTION_OPCODE_SET_IPV4_ADDRS:
468 ipv4_add = (struct nfp_fl_set_ip4_addrs *)a;
469 merge->ipv4.ipv4_src |= ipv4_add->ipv4_src_mask;
470 merge->ipv4.ipv4_dst |= ipv4_add->ipv4_dst_mask;
472 case NFP_FL_ACTION_OPCODE_SET_IPV4_TTL_TOS:
473 ipv4_ttl_tos = (struct nfp_fl_set_ip4_ttl_tos *)a;
474 merge->ipv4.ip_ext.ttl |= ipv4_ttl_tos->ipv4_ttl_mask;
475 merge->ipv4.ip_ext.tos |= ipv4_ttl_tos->ipv4_tos_mask;
477 case NFP_FL_ACTION_OPCODE_SET_IPV6_SRC:
478 ipv6_add = (struct nfp_fl_set_ipv6_addr *)a;
479 for (i = 0; i < 4; i++)
480 merge->ipv6.ipv6_src.in6_u.u6_addr32[i] |=
481 ipv6_add->ipv6[i].mask;
483 case NFP_FL_ACTION_OPCODE_SET_IPV6_DST:
484 ipv6_add = (struct nfp_fl_set_ipv6_addr *)a;
485 for (i = 0; i < 4; i++)
486 merge->ipv6.ipv6_dst.in6_u.u6_addr32[i] |=
487 ipv6_add->ipv6[i].mask;
489 case NFP_FL_ACTION_OPCODE_SET_IPV6_TC_HL_FL:
490 ipv6_tc_hl_fl = (struct nfp_fl_set_ipv6_tc_hl_fl *)a;
491 merge->ipv6.ip_ext.ttl |=
492 ipv6_tc_hl_fl->ipv6_hop_limit_mask;
493 merge->ipv6.ip_ext.tos |= ipv6_tc_hl_fl->ipv6_tc_mask;
494 merge->ipv6.ipv6_flow_label_exthdr |=
495 ipv6_tc_hl_fl->ipv6_label_mask;
497 case NFP_FL_ACTION_OPCODE_SET_UDP:
498 case NFP_FL_ACTION_OPCODE_SET_TCP:
499 tport = (struct nfp_fl_set_tport *)a;
500 ports = (u8 *)&merge->l4.port_src;
501 for (i = 0; i < 4; i++)
502 ports[i] |= tport->tp_port_mask[i];
504 case NFP_FL_ACTION_OPCODE_PRE_TUNNEL:
505 case NFP_FL_ACTION_OPCODE_PRE_LAG:
506 case NFP_FL_ACTION_OPCODE_PUSH_GENEVE:
512 act_off += a->len_lw << NFP_FL_LW_SIZ;
516 *last_act_id = act_id;
522 nfp_flower_populate_merge_match(struct nfp_fl_payload *flow,
523 struct nfp_flower_merge_check *merge,
526 struct nfp_flower_meta_tci *meta_tci;
527 u8 *mask = flow->mask_data;
528 u8 key_layer, match_size;
530 memset(merge, 0, sizeof(struct nfp_flower_merge_check));
532 meta_tci = (struct nfp_flower_meta_tci *)mask;
533 key_layer = meta_tci->nfp_flow_key_layer;
535 if (key_layer & ~NFP_FLOWER_MERGE_FIELDS && !extra_fields)
538 merge->tci = meta_tci->tci;
539 mask += sizeof(struct nfp_flower_meta_tci);
541 if (key_layer & NFP_FLOWER_LAYER_EXT_META)
542 mask += sizeof(struct nfp_flower_ext_meta);
544 mask += sizeof(struct nfp_flower_in_port);
546 if (key_layer & NFP_FLOWER_LAYER_MAC) {
547 match_size = sizeof(struct nfp_flower_mac_mpls);
548 memcpy(&merge->l2, mask, match_size);
552 if (key_layer & NFP_FLOWER_LAYER_TP) {
553 match_size = sizeof(struct nfp_flower_tp_ports);
554 memcpy(&merge->l4, mask, match_size);
558 if (key_layer & NFP_FLOWER_LAYER_IPV4) {
559 match_size = sizeof(struct nfp_flower_ipv4);
560 memcpy(&merge->ipv4, mask, match_size);
563 if (key_layer & NFP_FLOWER_LAYER_IPV6) {
564 match_size = sizeof(struct nfp_flower_ipv6);
565 memcpy(&merge->ipv6, mask, match_size);
572 nfp_flower_can_merge(struct nfp_fl_payload *sub_flow1,
573 struct nfp_fl_payload *sub_flow2)
575 /* Two flows can be merged if sub_flow2 only matches on bits that are
576 * either matched by sub_flow1 or set by a sub_flow1 action. This
577 * ensures that every packet that hits sub_flow1 and recirculates is
578 * guaranteed to hit sub_flow2.
580 struct nfp_flower_merge_check sub_flow1_merge, sub_flow2_merge;
581 int err, act_out = 0;
584 err = nfp_flower_populate_merge_match(sub_flow1, &sub_flow1_merge,
589 err = nfp_flower_populate_merge_match(sub_flow2, &sub_flow2_merge,
594 err = nfp_flower_update_merge_with_actions(sub_flow1, &sub_flow1_merge,
595 &last_act_id, &act_out);
599 /* Must only be 1 output action and it must be the last in sequence. */
600 if (act_out != 1 || last_act_id != NFP_FL_ACTION_OPCODE_OUTPUT)
603 /* Reject merge if sub_flow2 matches on something that is not matched
604 * on or set in an action by sub_flow1.
606 err = bitmap_andnot(sub_flow2_merge.vals, sub_flow2_merge.vals,
607 sub_flow1_merge.vals,
608 sizeof(struct nfp_flower_merge_check) * 8);
616 nfp_flower_copy_pre_actions(char *act_dst, char *act_src, int len,
619 unsigned int act_off = 0, act_len;
620 struct nfp_fl_act_head *a;
623 while (act_off < len) {
624 a = (struct nfp_fl_act_head *)&act_src[act_off];
625 act_len = a->len_lw << NFP_FL_LW_SIZ;
629 case NFP_FL_ACTION_OPCODE_PRE_TUNNEL:
632 case NFP_FL_ACTION_OPCODE_PRE_LAG:
633 memcpy(act_dst + act_off, act_src + act_off, act_len);
645 static int nfp_fl_verify_post_tun_acts(char *acts, int len)
647 struct nfp_fl_act_head *a;
648 unsigned int act_off = 0;
650 while (act_off < len) {
651 a = (struct nfp_fl_act_head *)&acts[act_off];
652 if (a->jump_id != NFP_FL_ACTION_OPCODE_OUTPUT)
655 act_off += a->len_lw << NFP_FL_LW_SIZ;
662 nfp_flower_merge_action(struct nfp_fl_payload *sub_flow1,
663 struct nfp_fl_payload *sub_flow2,
664 struct nfp_fl_payload *merge_flow)
666 unsigned int sub1_act_len, sub2_act_len, pre_off1, pre_off2;
667 bool tunnel_act = false;
671 /* The last action of sub_flow1 must be output - do not merge this. */
672 sub1_act_len = sub_flow1->meta.act_len - sizeof(struct nfp_fl_output);
673 sub2_act_len = sub_flow2->meta.act_len;
678 if (sub1_act_len + sub2_act_len > NFP_FL_MAX_A_SIZ)
681 /* A shortcut can only be applied if there is a single action. */
683 merge_flow->meta.shortcut = cpu_to_be32(NFP_FL_SC_ACT_NULL);
685 merge_flow->meta.shortcut = sub_flow2->meta.shortcut;
687 merge_flow->meta.act_len = sub1_act_len + sub2_act_len;
688 merge_act = merge_flow->action_data;
690 /* Copy any pre-actions to the start of merge flow action list. */
691 pre_off1 = nfp_flower_copy_pre_actions(merge_act,
692 sub_flow1->action_data,
693 sub1_act_len, &tunnel_act);
694 merge_act += pre_off1;
695 sub1_act_len -= pre_off1;
696 pre_off2 = nfp_flower_copy_pre_actions(merge_act,
697 sub_flow2->action_data,
699 merge_act += pre_off2;
700 sub2_act_len -= pre_off2;
702 /* FW does a tunnel push when egressing, therefore, if sub_flow 1 pushes
703 * a tunnel, sub_flow 2 can only have output actions for a valid merge.
706 char *post_tun_acts = &sub_flow2->action_data[pre_off2];
708 err = nfp_fl_verify_post_tun_acts(post_tun_acts, sub2_act_len);
713 /* Copy remaining actions from sub_flows 1 and 2. */
714 memcpy(merge_act, sub_flow1->action_data + pre_off1, sub1_act_len);
715 merge_act += sub1_act_len;
716 memcpy(merge_act, sub_flow2->action_data + pre_off2, sub2_act_len);
721 /* Flow link code should only be accessed under RTNL. */
722 static void nfp_flower_unlink_flow(struct nfp_fl_payload_link *link)
724 list_del(&link->merge_flow.list);
725 list_del(&link->sub_flow.list);
729 static void nfp_flower_unlink_flows(struct nfp_fl_payload *merge_flow,
730 struct nfp_fl_payload *sub_flow)
732 struct nfp_fl_payload_link *link;
734 list_for_each_entry(link, &merge_flow->linked_flows, merge_flow.list)
735 if (link->sub_flow.flow == sub_flow) {
736 nfp_flower_unlink_flow(link);
741 static int nfp_flower_link_flows(struct nfp_fl_payload *merge_flow,
742 struct nfp_fl_payload *sub_flow)
744 struct nfp_fl_payload_link *link;
746 link = kmalloc(sizeof(*link), GFP_KERNEL);
750 link->merge_flow.flow = merge_flow;
751 list_add_tail(&link->merge_flow.list, &merge_flow->linked_flows);
752 link->sub_flow.flow = sub_flow;
753 list_add_tail(&link->sub_flow.list, &sub_flow->linked_flows);
759 * nfp_flower_merge_offloaded_flows() - Merge 2 existing flows to single flow.
760 * @app: Pointer to the APP handle
761 * @sub_flow1: Initial flow matched to produce merge hint
762 * @sub_flow2: Post recirculation flow matched in merge hint
764 * Combines 2 flows (if valid) to a single flow, removing the initial from hw
765 * and offloading the new, merged flow.
767 * Return: negative value on error, 0 in success.
769 int nfp_flower_merge_offloaded_flows(struct nfp_app *app,
770 struct nfp_fl_payload *sub_flow1,
771 struct nfp_fl_payload *sub_flow2)
773 struct tc_cls_flower_offload merge_tc_off;
774 struct nfp_flower_priv *priv = app->priv;
775 struct nfp_fl_payload *merge_flow;
776 struct nfp_fl_key_ls merge_key_ls;
781 if (sub_flow1 == sub_flow2 ||
782 nfp_flower_is_merge_flow(sub_flow1) ||
783 nfp_flower_is_merge_flow(sub_flow2))
786 err = nfp_flower_can_merge(sub_flow1, sub_flow2);
790 merge_key_ls.key_size = sub_flow1->meta.key_len;
792 merge_flow = nfp_flower_allocate_new(&merge_key_ls);
796 merge_flow->tc_flower_cookie = (unsigned long)merge_flow;
797 merge_flow->ingress_dev = sub_flow1->ingress_dev;
799 memcpy(merge_flow->unmasked_data, sub_flow1->unmasked_data,
800 sub_flow1->meta.key_len);
801 memcpy(merge_flow->mask_data, sub_flow1->mask_data,
802 sub_flow1->meta.mask_len);
804 err = nfp_flower_merge_action(sub_flow1, sub_flow2, merge_flow);
806 goto err_destroy_merge_flow;
808 err = nfp_flower_link_flows(merge_flow, sub_flow1);
810 goto err_destroy_merge_flow;
812 err = nfp_flower_link_flows(merge_flow, sub_flow2);
814 goto err_unlink_sub_flow1;
816 merge_tc_off.cookie = merge_flow->tc_flower_cookie;
817 err = nfp_compile_flow_metadata(app, &merge_tc_off, merge_flow,
818 merge_flow->ingress_dev);
820 goto err_unlink_sub_flow2;
822 err = rhashtable_insert_fast(&priv->flow_table, &merge_flow->fl_node,
823 nfp_flower_table_params);
825 goto err_release_metadata;
827 err = nfp_flower_xmit_flow(app, merge_flow,
828 NFP_FLOWER_CMSG_TYPE_FLOW_MOD);
830 goto err_remove_rhash;
832 merge_flow->in_hw = true;
833 sub_flow1->in_hw = false;
838 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
839 &merge_flow->fl_node,
840 nfp_flower_table_params));
841 err_release_metadata:
842 nfp_modify_flow_metadata(app, merge_flow);
843 err_unlink_sub_flow2:
844 nfp_flower_unlink_flows(merge_flow, sub_flow2);
845 err_unlink_sub_flow1:
846 nfp_flower_unlink_flows(merge_flow, sub_flow1);
847 err_destroy_merge_flow:
848 kfree(merge_flow->action_data);
849 kfree(merge_flow->mask_data);
850 kfree(merge_flow->unmasked_data);
856 * nfp_flower_add_offload() - Adds a new flow to hardware.
857 * @app: Pointer to the APP handle
858 * @netdev: netdev structure.
859 * @flow: TC flower classifier offload structure.
861 * Adds a new flow to the repeated hash structure and action payload.
863 * Return: negative value on error, 0 if configured successfully.
866 nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
867 struct tc_cls_flower_offload *flow)
869 enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE;
870 struct nfp_flower_priv *priv = app->priv;
871 struct nfp_fl_payload *flow_pay;
872 struct nfp_fl_key_ls *key_layer;
873 struct nfp_port *port = NULL;
876 if (nfp_netdev_is_nfp_repr(netdev))
877 port = nfp_port_from_netdev(netdev);
879 key_layer = kmalloc(sizeof(*key_layer), GFP_KERNEL);
883 err = nfp_flower_calculate_key_layers(app, netdev, key_layer, flow,
886 goto err_free_key_ls;
888 flow_pay = nfp_flower_allocate_new(key_layer);
891 goto err_free_key_ls;
894 err = nfp_flower_compile_flow_match(app, flow, key_layer, netdev,
897 goto err_destroy_flow;
899 err = nfp_flower_compile_action(app, flow, netdev, flow_pay);
901 goto err_destroy_flow;
903 err = nfp_compile_flow_metadata(app, flow, flow_pay, netdev);
905 goto err_destroy_flow;
907 flow_pay->tc_flower_cookie = flow->cookie;
908 err = rhashtable_insert_fast(&priv->flow_table, &flow_pay->fl_node,
909 nfp_flower_table_params);
911 goto err_release_metadata;
913 err = nfp_flower_xmit_flow(app, flow_pay,
914 NFP_FLOWER_CMSG_TYPE_FLOW_ADD);
916 goto err_remove_rhash;
919 port->tc_offload_cnt++;
921 flow_pay->in_hw = true;
923 /* Deallocate flow payload when flower rule has been destroyed. */
929 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
931 nfp_flower_table_params));
932 err_release_metadata:
933 nfp_modify_flow_metadata(app, flow_pay);
935 kfree(flow_pay->action_data);
936 kfree(flow_pay->mask_data);
937 kfree(flow_pay->unmasked_data);
945 nfp_flower_remove_merge_flow(struct nfp_app *app,
946 struct nfp_fl_payload *del_sub_flow,
947 struct nfp_fl_payload *merge_flow)
949 struct nfp_flower_priv *priv = app->priv;
950 struct nfp_fl_payload_link *link, *temp;
951 struct nfp_fl_payload *origin;
955 link = list_first_entry(&merge_flow->linked_flows,
956 struct nfp_fl_payload_link, merge_flow.list);
957 origin = link->sub_flow.flow;
959 /* Re-add rule the merge had overwritten if it has not been deleted. */
960 if (origin != del_sub_flow)
963 err = nfp_modify_flow_metadata(app, merge_flow);
965 nfp_flower_cmsg_warn(app, "Metadata fail for merge flow delete.\n");
970 err = nfp_flower_xmit_flow(app, merge_flow,
971 NFP_FLOWER_CMSG_TYPE_FLOW_DEL);
973 nfp_flower_cmsg_warn(app, "Failed to delete merged flow.\n");
977 __nfp_modify_flow_metadata(priv, origin);
978 err = nfp_flower_xmit_flow(app, origin,
979 NFP_FLOWER_CMSG_TYPE_FLOW_MOD);
981 nfp_flower_cmsg_warn(app, "Failed to revert merge flow.\n");
982 origin->in_hw = true;
986 /* Clean any links connected with the merged flow. */
987 list_for_each_entry_safe(link, temp, &merge_flow->linked_flows,
989 nfp_flower_unlink_flow(link);
991 kfree(merge_flow->action_data);
992 kfree(merge_flow->mask_data);
993 kfree(merge_flow->unmasked_data);
994 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
995 &merge_flow->fl_node,
996 nfp_flower_table_params));
997 kfree_rcu(merge_flow, rcu);
1001 nfp_flower_del_linked_merge_flows(struct nfp_app *app,
1002 struct nfp_fl_payload *sub_flow)
1004 struct nfp_fl_payload_link *link, *temp;
1006 /* Remove any merge flow formed from the deleted sub_flow. */
1007 list_for_each_entry_safe(link, temp, &sub_flow->linked_flows,
1009 nfp_flower_remove_merge_flow(app, sub_flow,
1010 link->merge_flow.flow);
1014 * nfp_flower_del_offload() - Removes a flow from hardware.
1015 * @app: Pointer to the APP handle
1016 * @netdev: netdev structure.
1017 * @flow: TC flower classifier offload structure
1019 * Removes a flow from the repeated hash structure and clears the
1020 * action payload. Any flows merged from this are also deleted.
1022 * Return: negative value on error, 0 if removed successfully.
1025 nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev,
1026 struct tc_cls_flower_offload *flow)
1028 struct nfp_flower_priv *priv = app->priv;
1029 struct nfp_fl_payload *nfp_flow;
1030 struct nfp_port *port = NULL;
1033 if (nfp_netdev_is_nfp_repr(netdev))
1034 port = nfp_port_from_netdev(netdev);
1036 nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, netdev);
1040 err = nfp_modify_flow_metadata(app, nfp_flow);
1042 goto err_free_merge_flow;
1044 if (nfp_flow->nfp_tun_ipv4_addr)
1045 nfp_tunnel_del_ipv4_off(app, nfp_flow->nfp_tun_ipv4_addr);
1047 if (!nfp_flow->in_hw) {
1049 goto err_free_merge_flow;
1052 err = nfp_flower_xmit_flow(app, nfp_flow,
1053 NFP_FLOWER_CMSG_TYPE_FLOW_DEL);
1054 /* Fall through on error. */
1056 err_free_merge_flow:
1057 nfp_flower_del_linked_merge_flows(app, nfp_flow);
1059 port->tc_offload_cnt--;
1060 kfree(nfp_flow->action_data);
1061 kfree(nfp_flow->mask_data);
1062 kfree(nfp_flow->unmasked_data);
1063 WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table,
1065 nfp_flower_table_params));
1066 kfree_rcu(nfp_flow, rcu);
1071 __nfp_flower_update_merge_stats(struct nfp_app *app,
1072 struct nfp_fl_payload *merge_flow)
1074 struct nfp_flower_priv *priv = app->priv;
1075 struct nfp_fl_payload_link *link;
1076 struct nfp_fl_payload *sub_flow;
1077 u64 pkts, bytes, used;
1080 ctx_id = be32_to_cpu(merge_flow->meta.host_ctx_id);
1081 pkts = priv->stats[ctx_id].pkts;
1082 /* Do not cycle subflows if no stats to distribute. */
1085 bytes = priv->stats[ctx_id].bytes;
1086 used = priv->stats[ctx_id].used;
1088 /* Reset stats for the merge flow. */
1089 priv->stats[ctx_id].pkts = 0;
1090 priv->stats[ctx_id].bytes = 0;
1092 /* The merge flow has received stats updates from firmware.
1093 * Distribute these stats to all subflows that form the merge.
1094 * The stats will collected from TC via the subflows.
1096 list_for_each_entry(link, &merge_flow->linked_flows, merge_flow.list) {
1097 sub_flow = link->sub_flow.flow;
1098 ctx_id = be32_to_cpu(sub_flow->meta.host_ctx_id);
1099 priv->stats[ctx_id].pkts += pkts;
1100 priv->stats[ctx_id].bytes += bytes;
1101 max_t(u64, priv->stats[ctx_id].used, used);
1106 nfp_flower_update_merge_stats(struct nfp_app *app,
1107 struct nfp_fl_payload *sub_flow)
1109 struct nfp_fl_payload_link *link;
1111 /* Get merge flows that the subflow forms to distribute their stats. */
1112 list_for_each_entry(link, &sub_flow->linked_flows, sub_flow.list)
1113 __nfp_flower_update_merge_stats(app, link->merge_flow.flow);
1117 * nfp_flower_get_stats() - Populates flow stats obtained from hardware.
1118 * @app: Pointer to the APP handle
1119 * @netdev: Netdev structure.
1120 * @flow: TC flower classifier offload structure
1122 * Populates a flow statistics structure which which corresponds to a
1125 * Return: negative value on error, 0 if stats populated successfully.
1128 nfp_flower_get_stats(struct nfp_app *app, struct net_device *netdev,
1129 struct tc_cls_flower_offload *flow)
1131 struct nfp_flower_priv *priv = app->priv;
1132 struct nfp_fl_payload *nfp_flow;
1135 nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, netdev);
1139 ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id);
1141 spin_lock_bh(&priv->stats_lock);
1142 /* If request is for a sub_flow, update stats from merged flows. */
1143 if (!list_empty(&nfp_flow->linked_flows))
1144 nfp_flower_update_merge_stats(app, nfp_flow);
1146 flow_stats_update(&flow->stats, priv->stats[ctx_id].bytes,
1147 priv->stats[ctx_id].pkts, priv->stats[ctx_id].used);
1149 priv->stats[ctx_id].pkts = 0;
1150 priv->stats[ctx_id].bytes = 0;
1151 spin_unlock_bh(&priv->stats_lock);
1157 nfp_flower_repr_offload(struct nfp_app *app, struct net_device *netdev,
1158 struct tc_cls_flower_offload *flower)
1160 if (!eth_proto_is_802_3(flower->common.protocol))
1163 switch (flower->command) {
1164 case TC_CLSFLOWER_REPLACE:
1165 return nfp_flower_add_offload(app, netdev, flower);
1166 case TC_CLSFLOWER_DESTROY:
1167 return nfp_flower_del_offload(app, netdev, flower);
1168 case TC_CLSFLOWER_STATS:
1169 return nfp_flower_get_stats(app, netdev, flower);
1175 static int nfp_flower_setup_tc_block_cb(enum tc_setup_type type,
1176 void *type_data, void *cb_priv)
1178 struct nfp_repr *repr = cb_priv;
1180 if (!tc_cls_can_offload_and_chain0(repr->netdev, type_data))
1184 case TC_SETUP_CLSFLOWER:
1185 return nfp_flower_repr_offload(repr->app, repr->netdev,
1192 static int nfp_flower_setup_tc_block(struct net_device *netdev,
1193 struct tc_block_offload *f)
1195 struct nfp_repr *repr = netdev_priv(netdev);
1197 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
1200 switch (f->command) {
1202 return tcf_block_cb_register(f->block,
1203 nfp_flower_setup_tc_block_cb,
1204 repr, repr, f->extack);
1205 case TC_BLOCK_UNBIND:
1206 tcf_block_cb_unregister(f->block,
1207 nfp_flower_setup_tc_block_cb,
1215 int nfp_flower_setup_tc(struct nfp_app *app, struct net_device *netdev,
1216 enum tc_setup_type type, void *type_data)
1219 case TC_SETUP_BLOCK:
1220 return nfp_flower_setup_tc_block(netdev, type_data);
1226 struct nfp_flower_indr_block_cb_priv {
1227 struct net_device *netdev;
1228 struct nfp_app *app;
1229 struct list_head list;
1232 static struct nfp_flower_indr_block_cb_priv *
1233 nfp_flower_indr_block_cb_priv_lookup(struct nfp_app *app,
1234 struct net_device *netdev)
1236 struct nfp_flower_indr_block_cb_priv *cb_priv;
1237 struct nfp_flower_priv *priv = app->priv;
1239 /* All callback list access should be protected by RTNL. */
1242 list_for_each_entry(cb_priv, &priv->indr_block_cb_priv, list)
1243 if (cb_priv->netdev == netdev)
1249 static int nfp_flower_setup_indr_block_cb(enum tc_setup_type type,
1250 void *type_data, void *cb_priv)
1252 struct nfp_flower_indr_block_cb_priv *priv = cb_priv;
1253 struct tc_cls_flower_offload *flower = type_data;
1255 if (flower->common.chain_index)
1259 case TC_SETUP_CLSFLOWER:
1260 return nfp_flower_repr_offload(priv->app, priv->netdev,
1268 nfp_flower_setup_indr_tc_block(struct net_device *netdev, struct nfp_app *app,
1269 struct tc_block_offload *f)
1271 struct nfp_flower_indr_block_cb_priv *cb_priv;
1272 struct nfp_flower_priv *priv = app->priv;
1275 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1276 !(f->binder_type == TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS &&
1277 nfp_flower_internal_port_can_offload(app, netdev)))
1280 switch (f->command) {
1282 cb_priv = kmalloc(sizeof(*cb_priv), GFP_KERNEL);
1286 cb_priv->netdev = netdev;
1288 list_add(&cb_priv->list, &priv->indr_block_cb_priv);
1290 err = tcf_block_cb_register(f->block,
1291 nfp_flower_setup_indr_block_cb,
1292 cb_priv, cb_priv, f->extack);
1294 list_del(&cb_priv->list);
1299 case TC_BLOCK_UNBIND:
1300 cb_priv = nfp_flower_indr_block_cb_priv_lookup(app, netdev);
1304 tcf_block_cb_unregister(f->block,
1305 nfp_flower_setup_indr_block_cb,
1307 list_del(&cb_priv->list);
1318 nfp_flower_indr_setup_tc_cb(struct net_device *netdev, void *cb_priv,
1319 enum tc_setup_type type, void *type_data)
1322 case TC_SETUP_BLOCK:
1323 return nfp_flower_setup_indr_tc_block(netdev, cb_priv,
1330 int nfp_flower_reg_indir_block_handler(struct nfp_app *app,
1331 struct net_device *netdev,
1332 unsigned long event)
1336 if (!nfp_fl_is_netdev_to_offload(netdev))
1339 if (event == NETDEV_REGISTER) {
1340 err = __tc_indr_block_cb_register(netdev, app,
1341 nfp_flower_indr_setup_tc_cb,
1344 nfp_flower_cmsg_warn(app,
1345 "Indirect block reg failed - %s\n",
1347 } else if (event == NETDEV_UNREGISTER) {
1348 __tc_indr_block_cb_unregister(netdev,
1349 nfp_flower_indr_setup_tc_cb, app);