Commit | Line | Data |
---|---|---|
2ae7408f SP |
1 | /* Broadcom NetXtreme-C/E network driver. |
2 | * | |
3 | * Copyright (c) 2017 Broadcom Limited | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License as published by | |
7 | * the Free Software Foundation. | |
8 | */ | |
9 | ||
10 | #include <linux/netdevice.h> | |
11 | #include <linux/inetdevice.h> | |
12 | #include <linux/if_vlan.h> | |
13 | #include <net/flow_dissector.h> | |
14 | #include <net/pkt_cls.h> | |
15 | #include <net/tc_act/tc_gact.h> | |
16 | #include <net/tc_act/tc_skbedit.h> | |
17 | #include <net/tc_act/tc_mirred.h> | |
18 | #include <net/tc_act/tc_vlan.h> | |
8c95f773 | 19 | #include <net/tc_act/tc_tunnel_key.h> |
2ae7408f SP |
20 | |
21 | #include "bnxt_hsi.h" | |
22 | #include "bnxt.h" | |
23 | #include "bnxt_sriov.h" | |
24 | #include "bnxt_tc.h" | |
25 | #include "bnxt_vfr.h" | |
26 | ||
2ae7408f SP |
27 | #define BNXT_FID_INVALID 0xffff |
28 | #define VLAN_TCI(vid, prio) ((vid) | ((prio) << VLAN_PRIO_SHIFT)) | |
29 | ||
e32d4e60 VD |
30 | #define is_vlan_pcp_wildcarded(vlan_tci_mask) \ |
31 | ((ntohs(vlan_tci_mask) & VLAN_PRIO_MASK) == 0x0000) | |
32 | #define is_vlan_pcp_exactmatch(vlan_tci_mask) \ | |
33 | ((ntohs(vlan_tci_mask) & VLAN_PRIO_MASK) == VLAN_PRIO_MASK) | |
34 | #define is_vlan_pcp_zero(vlan_tci) \ | |
35 | ((ntohs(vlan_tci) & VLAN_PRIO_MASK) == 0x0000) | |
36 | #define is_vid_exactmatch(vlan_tci_mask) \ | |
37 | ((ntohs(vlan_tci_mask) & VLAN_VID_MASK) == VLAN_VID_MASK) | |
38 | ||
2ae7408f SP |
39 | /* Return the dst fid of the func for flow forwarding |
40 | * For PFs: src_fid is the fid of the PF | |
41 | * For VF-reps: src_fid the fid of the VF | |
42 | */ | |
43 | static u16 bnxt_flow_get_dst_fid(struct bnxt *pf_bp, struct net_device *dev) | |
44 | { | |
45 | struct bnxt *bp; | |
46 | ||
47 | /* check if dev belongs to the same switch */ | |
48 | if (!switchdev_port_same_parent_id(pf_bp->dev, dev)) { | |
49 | netdev_info(pf_bp->dev, "dev(ifindex=%d) not on same switch", | |
50 | dev->ifindex); | |
51 | return BNXT_FID_INVALID; | |
52 | } | |
53 | ||
54 | /* Is dev a VF-rep? */ | |
dd4ea1da | 55 | if (bnxt_dev_is_vf_rep(dev)) |
2ae7408f SP |
56 | return bnxt_vf_rep_get_fid(dev); |
57 | ||
58 | bp = netdev_priv(dev); | |
59 | return bp->pf.fw_fid; | |
60 | } | |
61 | ||
62 | static int bnxt_tc_parse_redir(struct bnxt *bp, | |
63 | struct bnxt_tc_actions *actions, | |
64 | const struct tc_action *tc_act) | |
65 | { | |
9f8a739e | 66 | struct net_device *dev = tcf_mirred_dev(tc_act); |
2ae7408f | 67 | |
2ae7408f | 68 | if (!dev) { |
9f8a739e | 69 | netdev_info(bp->dev, "no dev in mirred action"); |
2ae7408f SP |
70 | return -EINVAL; |
71 | } | |
72 | ||
2ae7408f | 73 | actions->flags |= BNXT_TC_ACTION_FLAG_FWD; |
2ae7408f SP |
74 | actions->dst_dev = dev; |
75 | return 0; | |
76 | } | |
77 | ||
78 | static void bnxt_tc_parse_vlan(struct bnxt *bp, | |
79 | struct bnxt_tc_actions *actions, | |
80 | const struct tc_action *tc_act) | |
81 | { | |
82 | if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_POP) { | |
83 | actions->flags |= BNXT_TC_ACTION_FLAG_POP_VLAN; | |
84 | } else if (tcf_vlan_action(tc_act) == TCA_VLAN_ACT_PUSH) { | |
85 | actions->flags |= BNXT_TC_ACTION_FLAG_PUSH_VLAN; | |
86 | actions->push_vlan_tci = htons(tcf_vlan_push_vid(tc_act)); | |
87 | actions->push_vlan_tpid = tcf_vlan_push_proto(tc_act); | |
88 | } | |
89 | } | |
90 | ||
8c95f773 SP |
91 | static int bnxt_tc_parse_tunnel_set(struct bnxt *bp, |
92 | struct bnxt_tc_actions *actions, | |
93 | const struct tc_action *tc_act) | |
94 | { | |
95 | struct ip_tunnel_info *tun_info = tcf_tunnel_info(tc_act); | |
96 | struct ip_tunnel_key *tun_key = &tun_info->key; | |
97 | ||
98 | if (ip_tunnel_info_af(tun_info) != AF_INET) { | |
99 | netdev_info(bp->dev, "only IPv4 tunnel-encap is supported"); | |
100 | return -EOPNOTSUPP; | |
101 | } | |
102 | ||
103 | actions->tun_encap_key = *tun_key; | |
104 | actions->flags |= BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP; | |
105 | return 0; | |
106 | } | |
107 | ||
2ae7408f SP |
108 | static int bnxt_tc_parse_actions(struct bnxt *bp, |
109 | struct bnxt_tc_actions *actions, | |
110 | struct tcf_exts *tc_exts) | |
111 | { | |
112 | const struct tc_action *tc_act; | |
113 | LIST_HEAD(tc_actions); | |
114 | int rc; | |
115 | ||
116 | if (!tcf_exts_has_actions(tc_exts)) { | |
117 | netdev_info(bp->dev, "no actions"); | |
118 | return -EINVAL; | |
119 | } | |
120 | ||
121 | tcf_exts_to_list(tc_exts, &tc_actions); | |
122 | list_for_each_entry(tc_act, &tc_actions, list) { | |
123 | /* Drop action */ | |
124 | if (is_tcf_gact_shot(tc_act)) { | |
125 | actions->flags |= BNXT_TC_ACTION_FLAG_DROP; | |
126 | return 0; /* don't bother with other actions */ | |
127 | } | |
128 | ||
129 | /* Redirect action */ | |
130 | if (is_tcf_mirred_egress_redirect(tc_act)) { | |
131 | rc = bnxt_tc_parse_redir(bp, actions, tc_act); | |
132 | if (rc) | |
133 | return rc; | |
134 | continue; | |
135 | } | |
136 | ||
137 | /* Push/pop VLAN */ | |
138 | if (is_tcf_vlan(tc_act)) { | |
139 | bnxt_tc_parse_vlan(bp, actions, tc_act); | |
140 | continue; | |
141 | } | |
8c95f773 SP |
142 | |
143 | /* Tunnel encap */ | |
144 | if (is_tcf_tunnel_set(tc_act)) { | |
145 | rc = bnxt_tc_parse_tunnel_set(bp, actions, tc_act); | |
146 | if (rc) | |
147 | return rc; | |
148 | continue; | |
149 | } | |
150 | ||
151 | /* Tunnel decap */ | |
152 | if (is_tcf_tunnel_release(tc_act)) { | |
153 | actions->flags |= BNXT_TC_ACTION_FLAG_TUNNEL_DECAP; | |
154 | continue; | |
155 | } | |
2ae7408f SP |
156 | } |
157 | ||
e9ecc731 SP |
158 | if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) { |
159 | if (actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) { | |
160 | /* dst_fid is PF's fid */ | |
161 | actions->dst_fid = bp->pf.fw_fid; | |
162 | } else { | |
163 | /* find the FID from dst_dev */ | |
164 | actions->dst_fid = | |
165 | bnxt_flow_get_dst_fid(bp, actions->dst_dev); | |
166 | if (actions->dst_fid == BNXT_FID_INVALID) | |
167 | return -EINVAL; | |
168 | } | |
8c95f773 SP |
169 | } |
170 | ||
92425c40 | 171 | return 0; |
2ae7408f SP |
172 | } |
173 | ||
174 | #define GET_KEY(flow_cmd, key_type) \ | |
175 | skb_flow_dissector_target((flow_cmd)->dissector, key_type,\ | |
176 | (flow_cmd)->key) | |
177 | #define GET_MASK(flow_cmd, key_type) \ | |
178 | skb_flow_dissector_target((flow_cmd)->dissector, key_type,\ | |
179 | (flow_cmd)->mask) | |
180 | ||
181 | static int bnxt_tc_parse_flow(struct bnxt *bp, | |
182 | struct tc_cls_flower_offload *tc_flow_cmd, | |
183 | struct bnxt_tc_flow *flow) | |
184 | { | |
185 | struct flow_dissector *dissector = tc_flow_cmd->dissector; | |
186 | u16 addr_type = 0; | |
187 | ||
188 | /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */ | |
189 | if ((dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) == 0 || | |
190 | (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) == 0) { | |
191 | netdev_info(bp->dev, "cannot form TC key: used_keys = 0x%x", | |
192 | dissector->used_keys); | |
193 | return -EOPNOTSUPP; | |
194 | } | |
195 | ||
196 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_CONTROL)) { | |
197 | struct flow_dissector_key_control *key = | |
198 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_CONTROL); | |
199 | ||
200 | addr_type = key->addr_type; | |
201 | } | |
202 | ||
203 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC)) { | |
204 | struct flow_dissector_key_basic *key = | |
205 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC); | |
206 | struct flow_dissector_key_basic *mask = | |
207 | GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC); | |
208 | ||
209 | flow->l2_key.ether_type = key->n_proto; | |
210 | flow->l2_mask.ether_type = mask->n_proto; | |
211 | ||
212 | if (key->n_proto == htons(ETH_P_IP) || | |
213 | key->n_proto == htons(ETH_P_IPV6)) { | |
214 | flow->l4_key.ip_proto = key->ip_proto; | |
215 | flow->l4_mask.ip_proto = mask->ip_proto; | |
216 | } | |
217 | } | |
218 | ||
219 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { | |
220 | struct flow_dissector_key_eth_addrs *key = | |
221 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS); | |
222 | struct flow_dissector_key_eth_addrs *mask = | |
223 | GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS); | |
224 | ||
225 | flow->flags |= BNXT_TC_FLOW_FLAGS_ETH_ADDRS; | |
226 | ether_addr_copy(flow->l2_key.dmac, key->dst); | |
227 | ether_addr_copy(flow->l2_mask.dmac, mask->dst); | |
228 | ether_addr_copy(flow->l2_key.smac, key->src); | |
229 | ether_addr_copy(flow->l2_mask.smac, mask->src); | |
230 | } | |
231 | ||
232 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN)) { | |
233 | struct flow_dissector_key_vlan *key = | |
234 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN); | |
235 | struct flow_dissector_key_vlan *mask = | |
236 | GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN); | |
237 | ||
238 | flow->l2_key.inner_vlan_tci = | |
239 | cpu_to_be16(VLAN_TCI(key->vlan_id, key->vlan_priority)); | |
240 | flow->l2_mask.inner_vlan_tci = | |
241 | cpu_to_be16((VLAN_TCI(mask->vlan_id, mask->vlan_priority))); | |
242 | flow->l2_key.inner_vlan_tpid = htons(ETH_P_8021Q); | |
243 | flow->l2_mask.inner_vlan_tpid = htons(0xffff); | |
244 | flow->l2_key.num_vlans = 1; | |
245 | } | |
246 | ||
247 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { | |
248 | struct flow_dissector_key_ipv4_addrs *key = | |
249 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS); | |
250 | struct flow_dissector_key_ipv4_addrs *mask = | |
251 | GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS); | |
252 | ||
253 | flow->flags |= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS; | |
254 | flow->l3_key.ipv4.daddr.s_addr = key->dst; | |
255 | flow->l3_mask.ipv4.daddr.s_addr = mask->dst; | |
256 | flow->l3_key.ipv4.saddr.s_addr = key->src; | |
257 | flow->l3_mask.ipv4.saddr.s_addr = mask->src; | |
258 | } else if (dissector_uses_key(dissector, | |
259 | FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { | |
260 | struct flow_dissector_key_ipv6_addrs *key = | |
261 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS); | |
262 | struct flow_dissector_key_ipv6_addrs *mask = | |
263 | GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS); | |
264 | ||
265 | flow->flags |= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS; | |
266 | flow->l3_key.ipv6.daddr = key->dst; | |
267 | flow->l3_mask.ipv6.daddr = mask->dst; | |
268 | flow->l3_key.ipv6.saddr = key->src; | |
269 | flow->l3_mask.ipv6.saddr = mask->src; | |
270 | } | |
271 | ||
272 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_PORTS)) { | |
273 | struct flow_dissector_key_ports *key = | |
274 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS); | |
275 | struct flow_dissector_key_ports *mask = | |
276 | GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS); | |
277 | ||
278 | flow->flags |= BNXT_TC_FLOW_FLAGS_PORTS; | |
279 | flow->l4_key.ports.dport = key->dst; | |
280 | flow->l4_mask.ports.dport = mask->dst; | |
281 | flow->l4_key.ports.sport = key->src; | |
282 | flow->l4_mask.ports.sport = mask->src; | |
283 | } | |
284 | ||
285 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ICMP)) { | |
286 | struct flow_dissector_key_icmp *key = | |
287 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP); | |
288 | struct flow_dissector_key_icmp *mask = | |
289 | GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP); | |
290 | ||
291 | flow->flags |= BNXT_TC_FLOW_FLAGS_ICMP; | |
292 | flow->l4_key.icmp.type = key->type; | |
293 | flow->l4_key.icmp.code = key->code; | |
294 | flow->l4_mask.icmp.type = mask->type; | |
295 | flow->l4_mask.icmp.code = mask->code; | |
296 | } | |
297 | ||
8c95f773 SP |
298 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) { |
299 | struct flow_dissector_key_control *key = | |
300 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_CONTROL); | |
301 | ||
302 | addr_type = key->addr_type; | |
303 | } | |
304 | ||
305 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { | |
306 | struct flow_dissector_key_ipv4_addrs *key = | |
307 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS); | |
308 | struct flow_dissector_key_ipv4_addrs *mask = | |
309 | GET_MASK(tc_flow_cmd, | |
310 | FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS); | |
311 | ||
312 | flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS; | |
313 | flow->tun_key.u.ipv4.dst = key->dst; | |
314 | flow->tun_mask.u.ipv4.dst = mask->dst; | |
315 | flow->tun_key.u.ipv4.src = key->src; | |
316 | flow->tun_mask.u.ipv4.src = mask->src; | |
317 | } else if (dissector_uses_key(dissector, | |
318 | FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { | |
319 | return -EOPNOTSUPP; | |
320 | } | |
321 | ||
322 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) { | |
323 | struct flow_dissector_key_keyid *key = | |
324 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_KEYID); | |
325 | struct flow_dissector_key_keyid *mask = | |
326 | GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_KEYID); | |
327 | ||
328 | flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_ID; | |
329 | flow->tun_key.tun_id = key32_to_tunnel_id(key->keyid); | |
330 | flow->tun_mask.tun_id = key32_to_tunnel_id(mask->keyid); | |
331 | } | |
332 | ||
333 | if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) { | |
334 | struct flow_dissector_key_ports *key = | |
335 | GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_PORTS); | |
336 | struct flow_dissector_key_ports *mask = | |
337 | GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_PORTS); | |
338 | ||
339 | flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_PORTS; | |
340 | flow->tun_key.tp_dst = key->dst; | |
341 | flow->tun_mask.tp_dst = mask->dst; | |
342 | flow->tun_key.tp_src = key->src; | |
343 | flow->tun_mask.tp_src = mask->src; | |
344 | } | |
345 | ||
2ae7408f SP |
346 | return bnxt_tc_parse_actions(bp, &flow->actions, tc_flow_cmd->exts); |
347 | } | |
348 | ||
349 | static int bnxt_hwrm_cfa_flow_free(struct bnxt *bp, __le16 flow_handle) | |
350 | { | |
db1d36a2 SP |
351 | struct hwrm_cfa_flow_free_input req = { 0 }; |
352 | int rc; | |
353 | ||
354 | bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_FREE, -1, -1); | |
355 | req.flow_handle = flow_handle; | |
356 | ||
357 | rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); | |
358 | if (rc) | |
359 | netdev_info(bp->dev, "Error: %s: flow_handle=0x%x rc=%d", | |
360 | __func__, flow_handle, rc); | |
6ae777ea VD |
361 | |
362 | if (rc) | |
363 | rc = -EIO; | |
db1d36a2 SP |
364 | return rc; |
365 | } | |
366 | ||
367 | static int ipv6_mask_len(struct in6_addr *mask) | |
368 | { | |
369 | int mask_len = 0, i; | |
370 | ||
371 | for (i = 0; i < 4; i++) | |
372 | mask_len += inet_mask_len(mask->s6_addr32[i]); | |
373 | ||
374 | return mask_len; | |
375 | } | |
376 | ||
377 | static bool is_wildcard(void *mask, int len) | |
378 | { | |
379 | const u8 *p = mask; | |
380 | int i; | |
381 | ||
382 | for (i = 0; i < len; i++) { | |
383 | if (p[i] != 0) | |
384 | return false; | |
385 | } | |
386 | return true; | |
2ae7408f SP |
387 | } |
388 | ||
e85a9be9 AG |
389 | static bool is_exactmatch(void *mask, int len) |
390 | { | |
391 | const u8 *p = mask; | |
392 | int i; | |
393 | ||
394 | for (i = 0; i < len; i++) | |
395 | if (p[i] != 0xff) | |
396 | return false; | |
397 | ||
398 | return true; | |
399 | } | |
400 | ||
e32d4e60 VD |
401 | static bool is_vlan_tci_allowed(__be16 vlan_tci_mask, |
402 | __be16 vlan_tci) | |
403 | { | |
404 | /* VLAN priority must be either exactly zero or fully wildcarded and | |
405 | * VLAN id must be exact match. | |
406 | */ | |
407 | if (is_vid_exactmatch(vlan_tci_mask) && | |
408 | ((is_vlan_pcp_exactmatch(vlan_tci_mask) && | |
409 | is_vlan_pcp_zero(vlan_tci)) || | |
410 | is_vlan_pcp_wildcarded(vlan_tci_mask))) | |
411 | return true; | |
412 | ||
413 | return false; | |
414 | } | |
415 | ||
e85a9be9 AG |
416 | static bool bits_set(void *key, int len) |
417 | { | |
418 | const u8 *p = key; | |
419 | int i; | |
420 | ||
421 | for (i = 0; i < len; i++) | |
422 | if (p[i] != 0) | |
423 | return true; | |
424 | ||
425 | return false; | |
426 | } | |
427 | ||
2ae7408f | 428 | static int bnxt_hwrm_cfa_flow_alloc(struct bnxt *bp, struct bnxt_tc_flow *flow, |
8c95f773 SP |
429 | __le16 ref_flow_handle, |
430 | __le32 tunnel_handle, __le16 *flow_handle) | |
2ae7408f | 431 | { |
db1d36a2 SP |
432 | struct hwrm_cfa_flow_alloc_output *resp = bp->hwrm_cmd_resp_addr; |
433 | struct bnxt_tc_actions *actions = &flow->actions; | |
434 | struct bnxt_tc_l3_key *l3_mask = &flow->l3_mask; | |
435 | struct bnxt_tc_l3_key *l3_key = &flow->l3_key; | |
436 | struct hwrm_cfa_flow_alloc_input req = { 0 }; | |
437 | u16 flow_flags = 0, action_flags = 0; | |
438 | int rc; | |
439 | ||
440 | bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_ALLOC, -1, -1); | |
441 | ||
442 | req.src_fid = cpu_to_le16(flow->src_fid); | |
443 | req.ref_flow_handle = ref_flow_handle; | |
8c95f773 SP |
444 | |
445 | if (actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP || | |
446 | actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) { | |
447 | req.tunnel_handle = tunnel_handle; | |
448 | flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_TUNNEL; | |
449 | action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL; | |
450 | } | |
451 | ||
db1d36a2 SP |
452 | req.ethertype = flow->l2_key.ether_type; |
453 | req.ip_proto = flow->l4_key.ip_proto; | |
454 | ||
455 | if (flow->flags & BNXT_TC_FLOW_FLAGS_ETH_ADDRS) { | |
456 | memcpy(req.dmac, flow->l2_key.dmac, ETH_ALEN); | |
457 | memcpy(req.smac, flow->l2_key.smac, ETH_ALEN); | |
458 | } | |
459 | ||
460 | if (flow->l2_key.num_vlans > 0) { | |
461 | flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE; | |
462 | /* FW expects the inner_vlan_tci value to be set | |
463 | * in outer_vlan_tci when num_vlans is 1 (which is | |
464 | * always the case in TC.) | |
465 | */ | |
466 | req.outer_vlan_tci = flow->l2_key.inner_vlan_tci; | |
467 | } | |
468 | ||
469 | /* If all IP and L4 fields are wildcarded then this is an L2 flow */ | |
7deea450 | 470 | if (is_wildcard(l3_mask, sizeof(*l3_mask)) && |
db1d36a2 SP |
471 | is_wildcard(&flow->l4_mask, sizeof(flow->l4_mask))) { |
472 | flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2; | |
473 | } else { | |
474 | flow_flags |= flow->l2_key.ether_type == htons(ETH_P_IP) ? | |
475 | CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4 : | |
476 | CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6; | |
477 | ||
478 | if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV4_ADDRS) { | |
479 | req.ip_dst[0] = l3_key->ipv4.daddr.s_addr; | |
480 | req.ip_dst_mask_len = | |
481 | inet_mask_len(l3_mask->ipv4.daddr.s_addr); | |
482 | req.ip_src[0] = l3_key->ipv4.saddr.s_addr; | |
483 | req.ip_src_mask_len = | |
484 | inet_mask_len(l3_mask->ipv4.saddr.s_addr); | |
485 | } else if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV6_ADDRS) { | |
486 | memcpy(req.ip_dst, l3_key->ipv6.daddr.s6_addr32, | |
487 | sizeof(req.ip_dst)); | |
488 | req.ip_dst_mask_len = | |
489 | ipv6_mask_len(&l3_mask->ipv6.daddr); | |
490 | memcpy(req.ip_src, l3_key->ipv6.saddr.s6_addr32, | |
491 | sizeof(req.ip_src)); | |
492 | req.ip_src_mask_len = | |
493 | ipv6_mask_len(&l3_mask->ipv6.saddr); | |
494 | } | |
495 | } | |
496 | ||
497 | if (flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) { | |
498 | req.l4_src_port = flow->l4_key.ports.sport; | |
499 | req.l4_src_port_mask = flow->l4_mask.ports.sport; | |
500 | req.l4_dst_port = flow->l4_key.ports.dport; | |
501 | req.l4_dst_port_mask = flow->l4_mask.ports.dport; | |
502 | } else if (flow->flags & BNXT_TC_FLOW_FLAGS_ICMP) { | |
503 | /* l4 ports serve as type/code when ip_proto is ICMP */ | |
504 | req.l4_src_port = htons(flow->l4_key.icmp.type); | |
505 | req.l4_src_port_mask = htons(flow->l4_mask.icmp.type); | |
506 | req.l4_dst_port = htons(flow->l4_key.icmp.code); | |
507 | req.l4_dst_port_mask = htons(flow->l4_mask.icmp.code); | |
508 | } | |
509 | req.flags = cpu_to_le16(flow_flags); | |
510 | ||
511 | if (actions->flags & BNXT_TC_ACTION_FLAG_DROP) { | |
512 | action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP; | |
513 | } else { | |
514 | if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) { | |
515 | action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD; | |
516 | req.dst_fid = cpu_to_le16(actions->dst_fid); | |
517 | } | |
518 | if (actions->flags & BNXT_TC_ACTION_FLAG_PUSH_VLAN) { | |
519 | action_flags |= | |
520 | CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE; | |
521 | req.l2_rewrite_vlan_tpid = actions->push_vlan_tpid; | |
522 | req.l2_rewrite_vlan_tci = actions->push_vlan_tci; | |
523 | memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN); | |
524 | memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN); | |
525 | } | |
526 | if (actions->flags & BNXT_TC_ACTION_FLAG_POP_VLAN) { | |
527 | action_flags |= | |
528 | CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE; | |
529 | /* Rewrite config with tpid = 0 implies vlan pop */ | |
530 | req.l2_rewrite_vlan_tpid = 0; | |
531 | memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN); | |
532 | memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN); | |
533 | } | |
534 | } | |
535 | req.action_flags = cpu_to_le16(action_flags); | |
536 | ||
537 | mutex_lock(&bp->hwrm_cmd_lock); | |
db1d36a2 SP |
538 | rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); |
539 | if (!rc) | |
540 | *flow_handle = resp->flow_handle; | |
db1d36a2 SP |
541 | mutex_unlock(&bp->hwrm_cmd_lock); |
542 | ||
6ae777ea VD |
543 | if (rc == HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR) |
544 | rc = -ENOSPC; | |
545 | else if (rc) | |
546 | rc = -EIO; | |
db1d36a2 | 547 | return rc; |
2ae7408f SP |
548 | } |
549 | ||
8c95f773 SP |
550 | static int hwrm_cfa_decap_filter_alloc(struct bnxt *bp, |
551 | struct bnxt_tc_flow *flow, | |
552 | struct bnxt_tc_l2_key *l2_info, | |
553 | __le32 ref_decap_handle, | |
554 | __le32 *decap_filter_handle) | |
555 | { | |
f484f678 SP |
556 | struct hwrm_cfa_decap_filter_alloc_output *resp = |
557 | bp->hwrm_cmd_resp_addr; | |
558 | struct hwrm_cfa_decap_filter_alloc_input req = { 0 }; | |
559 | struct ip_tunnel_key *tun_key = &flow->tun_key; | |
560 | u32 enables = 0; | |
561 | int rc; | |
562 | ||
563 | bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_DECAP_FILTER_ALLOC, -1, -1); | |
564 | ||
565 | req.flags = cpu_to_le32(CFA_DECAP_FILTER_ALLOC_REQ_FLAGS_OVS_TUNNEL); | |
566 | enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE | | |
567 | CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL; | |
568 | req.tunnel_type = CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN; | |
569 | req.ip_protocol = CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP; | |
570 | ||
571 | if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_ID) { | |
572 | enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_ID; | |
573 | /* tunnel_id is wrongly defined in hsi defn. as __le32 */ | |
574 | req.tunnel_id = tunnel_id_to_key32(tun_key->tun_id); | |
575 | } | |
576 | ||
577 | if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS) { | |
c8fb7b82 | 578 | enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR; |
f484f678 | 579 | ether_addr_copy(req.dst_macaddr, l2_info->dmac); |
f484f678 SP |
580 | } |
581 | if (l2_info->num_vlans) { | |
582 | enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_IVLAN_VID; | |
583 | req.t_ivlan_vid = l2_info->inner_vlan_tci; | |
584 | } | |
585 | ||
586 | enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE; | |
587 | req.ethertype = htons(ETH_P_IP); | |
588 | ||
589 | if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS) { | |
590 | enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | | |
591 | CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | | |
592 | CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE; | |
593 | req.ip_addr_type = CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; | |
594 | req.dst_ipaddr[0] = tun_key->u.ipv4.dst; | |
595 | req.src_ipaddr[0] = tun_key->u.ipv4.src; | |
596 | } | |
597 | ||
598 | if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_PORTS) { | |
599 | enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_PORT; | |
600 | req.dst_port = tun_key->tp_dst; | |
601 | } | |
602 | ||
603 | /* Eventhough the decap_handle returned by hwrm_cfa_decap_filter_alloc | |
604 | * is defined as __le32, l2_ctxt_ref_id is defined in HSI as __le16. | |
605 | */ | |
606 | req.l2_ctxt_ref_id = (__force __le16)ref_decap_handle; | |
607 | req.enables = cpu_to_le32(enables); | |
608 | ||
609 | mutex_lock(&bp->hwrm_cmd_lock); | |
610 | rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); | |
611 | if (!rc) | |
612 | *decap_filter_handle = resp->decap_filter_id; | |
613 | else | |
614 | netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc); | |
615 | mutex_unlock(&bp->hwrm_cmd_lock); | |
616 | ||
6ae777ea VD |
617 | if (rc) |
618 | rc = -EIO; | |
f484f678 | 619 | return rc; |
8c95f773 SP |
620 | } |
621 | ||
622 | static int hwrm_cfa_decap_filter_free(struct bnxt *bp, | |
623 | __le32 decap_filter_handle) | |
624 | { | |
f484f678 SP |
625 | struct hwrm_cfa_decap_filter_free_input req = { 0 }; |
626 | int rc; | |
627 | ||
628 | bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_DECAP_FILTER_FREE, -1, -1); | |
629 | req.decap_filter_id = decap_filter_handle; | |
630 | ||
631 | rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); | |
632 | if (rc) | |
633 | netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc); | |
6ae777ea VD |
634 | |
635 | if (rc) | |
636 | rc = -EIO; | |
f484f678 | 637 | return rc; |
8c95f773 SP |
638 | } |
639 | ||
640 | static int hwrm_cfa_encap_record_alloc(struct bnxt *bp, | |
641 | struct ip_tunnel_key *encap_key, | |
642 | struct bnxt_tc_l2_key *l2_info, | |
643 | __le32 *encap_record_handle) | |
644 | { | |
f484f678 SP |
645 | struct hwrm_cfa_encap_record_alloc_output *resp = |
646 | bp->hwrm_cmd_resp_addr; | |
647 | struct hwrm_cfa_encap_record_alloc_input req = { 0 }; | |
648 | struct hwrm_cfa_encap_data_vxlan *encap = | |
649 | (struct hwrm_cfa_encap_data_vxlan *)&req.encap_data; | |
650 | struct hwrm_vxlan_ipv4_hdr *encap_ipv4 = | |
651 | (struct hwrm_vxlan_ipv4_hdr *)encap->l3; | |
652 | int rc; | |
653 | ||
654 | bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ENCAP_RECORD_ALLOC, -1, -1); | |
655 | ||
656 | req.encap_type = CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN; | |
657 | ||
658 | ether_addr_copy(encap->dst_mac_addr, l2_info->dmac); | |
659 | ether_addr_copy(encap->src_mac_addr, l2_info->smac); | |
660 | if (l2_info->num_vlans) { | |
661 | encap->num_vlan_tags = l2_info->num_vlans; | |
662 | encap->ovlan_tci = l2_info->inner_vlan_tci; | |
663 | encap->ovlan_tpid = l2_info->inner_vlan_tpid; | |
664 | } | |
665 | ||
666 | encap_ipv4->ver_hlen = 4 << VXLAN_IPV4_HDR_VER_HLEN_VERSION_SFT; | |
667 | encap_ipv4->ver_hlen |= 5 << VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_SFT; | |
668 | encap_ipv4->ttl = encap_key->ttl; | |
669 | ||
670 | encap_ipv4->dest_ip_addr = encap_key->u.ipv4.dst; | |
671 | encap_ipv4->src_ip_addr = encap_key->u.ipv4.src; | |
672 | encap_ipv4->protocol = IPPROTO_UDP; | |
673 | ||
674 | encap->dst_port = encap_key->tp_dst; | |
675 | encap->vni = tunnel_id_to_key32(encap_key->tun_id); | |
676 | ||
677 | mutex_lock(&bp->hwrm_cmd_lock); | |
678 | rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); | |
679 | if (!rc) | |
680 | *encap_record_handle = resp->encap_record_id; | |
681 | else | |
682 | netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc); | |
683 | mutex_unlock(&bp->hwrm_cmd_lock); | |
684 | ||
6ae777ea VD |
685 | if (rc) |
686 | rc = -EIO; | |
f484f678 | 687 | return rc; |
8c95f773 SP |
688 | } |
689 | ||
690 | static int hwrm_cfa_encap_record_free(struct bnxt *bp, | |
691 | __le32 encap_record_handle) | |
692 | { | |
f484f678 SP |
693 | struct hwrm_cfa_encap_record_free_input req = { 0 }; |
694 | int rc; | |
695 | ||
696 | bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ENCAP_RECORD_FREE, -1, -1); | |
697 | req.encap_record_id = encap_record_handle; | |
698 | ||
699 | rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); | |
700 | if (rc) | |
701 | netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc); | |
6ae777ea VD |
702 | |
703 | if (rc) | |
704 | rc = -EIO; | |
f484f678 | 705 | return rc; |
8c95f773 SP |
706 | } |
707 | ||
2ae7408f SP |
708 | static int bnxt_tc_put_l2_node(struct bnxt *bp, |
709 | struct bnxt_tc_flow_node *flow_node) | |
710 | { | |
711 | struct bnxt_tc_l2_node *l2_node = flow_node->l2_node; | |
cd66358e | 712 | struct bnxt_tc_info *tc_info = bp->tc_info; |
2ae7408f SP |
713 | int rc; |
714 | ||
715 | /* remove flow_node from the L2 shared flow list */ | |
716 | list_del(&flow_node->l2_list_node); | |
717 | if (--l2_node->refcount == 0) { | |
718 | rc = rhashtable_remove_fast(&tc_info->l2_table, &l2_node->node, | |
719 | tc_info->l2_ht_params); | |
720 | if (rc) | |
721 | netdev_err(bp->dev, | |
722 | "Error: %s: rhashtable_remove_fast: %d", | |
723 | __func__, rc); | |
724 | kfree_rcu(l2_node, rcu); | |
725 | } | |
726 | return 0; | |
727 | } | |
728 | ||
729 | static struct bnxt_tc_l2_node * | |
730 | bnxt_tc_get_l2_node(struct bnxt *bp, struct rhashtable *l2_table, | |
731 | struct rhashtable_params ht_params, | |
732 | struct bnxt_tc_l2_key *l2_key) | |
733 | { | |
734 | struct bnxt_tc_l2_node *l2_node; | |
735 | int rc; | |
736 | ||
737 | l2_node = rhashtable_lookup_fast(l2_table, l2_key, ht_params); | |
738 | if (!l2_node) { | |
739 | l2_node = kzalloc(sizeof(*l2_node), GFP_KERNEL); | |
740 | if (!l2_node) { | |
741 | rc = -ENOMEM; | |
742 | return NULL; | |
743 | } | |
744 | ||
745 | l2_node->key = *l2_key; | |
746 | rc = rhashtable_insert_fast(l2_table, &l2_node->node, | |
747 | ht_params); | |
748 | if (rc) { | |
8c95f773 | 749 | kfree_rcu(l2_node, rcu); |
2ae7408f SP |
750 | netdev_err(bp->dev, |
751 | "Error: %s: rhashtable_insert_fast: %d", | |
752 | __func__, rc); | |
753 | return NULL; | |
754 | } | |
755 | INIT_LIST_HEAD(&l2_node->common_l2_flows); | |
756 | } | |
757 | return l2_node; | |
758 | } | |
759 | ||
760 | /* Get the ref_flow_handle for a flow by checking if there are any other | |
761 | * flows that share the same L2 key as this flow. | |
762 | */ | |
763 | static int | |
764 | bnxt_tc_get_ref_flow_handle(struct bnxt *bp, struct bnxt_tc_flow *flow, | |
765 | struct bnxt_tc_flow_node *flow_node, | |
766 | __le16 *ref_flow_handle) | |
767 | { | |
cd66358e | 768 | struct bnxt_tc_info *tc_info = bp->tc_info; |
2ae7408f SP |
769 | struct bnxt_tc_flow_node *ref_flow_node; |
770 | struct bnxt_tc_l2_node *l2_node; | |
771 | ||
772 | l2_node = bnxt_tc_get_l2_node(bp, &tc_info->l2_table, | |
773 | tc_info->l2_ht_params, | |
774 | &flow->l2_key); | |
775 | if (!l2_node) | |
776 | return -1; | |
777 | ||
778 | /* If any other flow is using this l2_node, use it's flow_handle | |
779 | * as the ref_flow_handle | |
780 | */ | |
781 | if (l2_node->refcount > 0) { | |
782 | ref_flow_node = list_first_entry(&l2_node->common_l2_flows, | |
783 | struct bnxt_tc_flow_node, | |
784 | l2_list_node); | |
785 | *ref_flow_handle = ref_flow_node->flow_handle; | |
786 | } else { | |
787 | *ref_flow_handle = cpu_to_le16(0xffff); | |
788 | } | |
789 | ||
790 | /* Insert the l2_node into the flow_node so that subsequent flows | |
791 | * with a matching l2 key can use the flow_handle of this flow | |
792 | * as their ref_flow_handle | |
793 | */ | |
794 | flow_node->l2_node = l2_node; | |
795 | list_add(&flow_node->l2_list_node, &l2_node->common_l2_flows); | |
796 | l2_node->refcount++; | |
797 | return 0; | |
798 | } | |
799 | ||
800 | /* After the flow parsing is done, this routine is used for checking | |
801 | * if there are any aspects of the flow that prevent it from being | |
802 | * offloaded. | |
803 | */ | |
804 | static bool bnxt_tc_can_offload(struct bnxt *bp, struct bnxt_tc_flow *flow) | |
805 | { | |
806 | /* If L4 ports are specified then ip_proto must be TCP or UDP */ | |
807 | if ((flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) && | |
808 | (flow->l4_key.ip_proto != IPPROTO_TCP && | |
809 | flow->l4_key.ip_proto != IPPROTO_UDP)) { | |
810 | netdev_info(bp->dev, "Cannot offload non-TCP/UDP (%d) ports", | |
811 | flow->l4_key.ip_proto); | |
812 | return false; | |
813 | } | |
814 | ||
e85a9be9 AG |
815 | /* Currently source/dest MAC cannot be partial wildcard */ |
816 | if (bits_set(&flow->l2_key.smac, sizeof(flow->l2_key.smac)) && | |
817 | !is_exactmatch(flow->l2_mask.smac, sizeof(flow->l2_mask.smac))) { | |
818 | netdev_info(bp->dev, "Wildcard match unsupported for Source MAC\n"); | |
819 | return false; | |
820 | } | |
821 | if (bits_set(&flow->l2_key.dmac, sizeof(flow->l2_key.dmac)) && | |
822 | !is_exactmatch(&flow->l2_mask.dmac, sizeof(flow->l2_mask.dmac))) { | |
823 | netdev_info(bp->dev, "Wildcard match unsupported for Dest MAC\n"); | |
824 | return false; | |
825 | } | |
826 | ||
827 | /* Currently VLAN fields cannot be partial wildcard */ | |
828 | if (bits_set(&flow->l2_key.inner_vlan_tci, | |
829 | sizeof(flow->l2_key.inner_vlan_tci)) && | |
e32d4e60 VD |
830 | !is_vlan_tci_allowed(flow->l2_mask.inner_vlan_tci, |
831 | flow->l2_key.inner_vlan_tci)) { | |
832 | netdev_info(bp->dev, "Unsupported VLAN TCI\n"); | |
e85a9be9 AG |
833 | return false; |
834 | } | |
835 | if (bits_set(&flow->l2_key.inner_vlan_tpid, | |
836 | sizeof(flow->l2_key.inner_vlan_tpid)) && | |
837 | !is_exactmatch(&flow->l2_mask.inner_vlan_tpid, | |
838 | sizeof(flow->l2_mask.inner_vlan_tpid))) { | |
839 | netdev_info(bp->dev, "Wildcard match unsupported for VLAN TPID\n"); | |
840 | return false; | |
841 | } | |
842 | ||
843 | /* Currently Ethertype must be set */ | |
844 | if (!is_exactmatch(&flow->l2_mask.ether_type, | |
845 | sizeof(flow->l2_mask.ether_type))) { | |
846 | netdev_info(bp->dev, "Wildcard match unsupported for Ethertype\n"); | |
847 | return false; | |
848 | } | |
849 | ||
2ae7408f SP |
850 | return true; |
851 | } | |
852 | ||
8c95f773 SP |
853 | /* Returns the final refcount of the node on success |
854 | * or a -ve error code on failure | |
855 | */ | |
856 | static int bnxt_tc_put_tunnel_node(struct bnxt *bp, | |
857 | struct rhashtable *tunnel_table, | |
858 | struct rhashtable_params *ht_params, | |
859 | struct bnxt_tc_tunnel_node *tunnel_node) | |
860 | { | |
861 | int rc; | |
862 | ||
863 | if (--tunnel_node->refcount == 0) { | |
864 | rc = rhashtable_remove_fast(tunnel_table, &tunnel_node->node, | |
865 | *ht_params); | |
866 | if (rc) { | |
867 | netdev_err(bp->dev, "rhashtable_remove_fast rc=%d", rc); | |
868 | rc = -1; | |
869 | } | |
870 | kfree_rcu(tunnel_node, rcu); | |
871 | return rc; | |
872 | } else { | |
873 | return tunnel_node->refcount; | |
874 | } | |
875 | } | |
876 | ||
877 | /* Get (or add) either encap or decap tunnel node from/to the supplied | |
878 | * hash table. | |
879 | */ | |
880 | static struct bnxt_tc_tunnel_node * | |
881 | bnxt_tc_get_tunnel_node(struct bnxt *bp, struct rhashtable *tunnel_table, | |
882 | struct rhashtable_params *ht_params, | |
883 | struct ip_tunnel_key *tun_key) | |
884 | { | |
885 | struct bnxt_tc_tunnel_node *tunnel_node; | |
886 | int rc; | |
887 | ||
888 | tunnel_node = rhashtable_lookup_fast(tunnel_table, tun_key, *ht_params); | |
889 | if (!tunnel_node) { | |
890 | tunnel_node = kzalloc(sizeof(*tunnel_node), GFP_KERNEL); | |
891 | if (!tunnel_node) { | |
892 | rc = -ENOMEM; | |
893 | goto err; | |
894 | } | |
895 | ||
896 | tunnel_node->key = *tun_key; | |
897 | tunnel_node->tunnel_handle = INVALID_TUNNEL_HANDLE; | |
898 | rc = rhashtable_insert_fast(tunnel_table, &tunnel_node->node, | |
899 | *ht_params); | |
900 | if (rc) { | |
901 | kfree_rcu(tunnel_node, rcu); | |
902 | goto err; | |
903 | } | |
904 | } | |
905 | tunnel_node->refcount++; | |
906 | return tunnel_node; | |
907 | err: | |
908 | netdev_info(bp->dev, "error rc=%d", rc); | |
909 | return NULL; | |
910 | } | |
911 | ||
912 | static int bnxt_tc_get_ref_decap_handle(struct bnxt *bp, | |
913 | struct bnxt_tc_flow *flow, | |
914 | struct bnxt_tc_l2_key *l2_key, | |
915 | struct bnxt_tc_flow_node *flow_node, | |
916 | __le32 *ref_decap_handle) | |
917 | { | |
cd66358e | 918 | struct bnxt_tc_info *tc_info = bp->tc_info; |
8c95f773 SP |
919 | struct bnxt_tc_flow_node *ref_flow_node; |
920 | struct bnxt_tc_l2_node *decap_l2_node; | |
921 | ||
922 | decap_l2_node = bnxt_tc_get_l2_node(bp, &tc_info->decap_l2_table, | |
923 | tc_info->decap_l2_ht_params, | |
924 | l2_key); | |
925 | if (!decap_l2_node) | |
926 | return -1; | |
927 | ||
928 | /* If any other flow is using this decap_l2_node, use it's decap_handle | |
929 | * as the ref_decap_handle | |
930 | */ | |
931 | if (decap_l2_node->refcount > 0) { | |
932 | ref_flow_node = | |
933 | list_first_entry(&decap_l2_node->common_l2_flows, | |
934 | struct bnxt_tc_flow_node, | |
935 | decap_l2_list_node); | |
936 | *ref_decap_handle = ref_flow_node->decap_node->tunnel_handle; | |
937 | } else { | |
938 | *ref_decap_handle = INVALID_TUNNEL_HANDLE; | |
939 | } | |
940 | ||
941 | /* Insert the l2_node into the flow_node so that subsequent flows | |
942 | * with a matching decap l2 key can use the decap_filter_handle of | |
943 | * this flow as their ref_decap_handle | |
944 | */ | |
945 | flow_node->decap_l2_node = decap_l2_node; | |
946 | list_add(&flow_node->decap_l2_list_node, | |
947 | &decap_l2_node->common_l2_flows); | |
948 | decap_l2_node->refcount++; | |
949 | return 0; | |
950 | } | |
951 | ||
952 | static void bnxt_tc_put_decap_l2_node(struct bnxt *bp, | |
953 | struct bnxt_tc_flow_node *flow_node) | |
954 | { | |
955 | struct bnxt_tc_l2_node *decap_l2_node = flow_node->decap_l2_node; | |
cd66358e | 956 | struct bnxt_tc_info *tc_info = bp->tc_info; |
8c95f773 SP |
957 | int rc; |
958 | ||
959 | /* remove flow_node from the decap L2 sharing flow list */ | |
960 | list_del(&flow_node->decap_l2_list_node); | |
961 | if (--decap_l2_node->refcount == 0) { | |
962 | rc = rhashtable_remove_fast(&tc_info->decap_l2_table, | |
963 | &decap_l2_node->node, | |
964 | tc_info->decap_l2_ht_params); | |
965 | if (rc) | |
966 | netdev_err(bp->dev, "rhashtable_remove_fast rc=%d", rc); | |
967 | kfree_rcu(decap_l2_node, rcu); | |
968 | } | |
969 | } | |
970 | ||
971 | static void bnxt_tc_put_decap_handle(struct bnxt *bp, | |
972 | struct bnxt_tc_flow_node *flow_node) | |
973 | { | |
974 | __le32 decap_handle = flow_node->decap_node->tunnel_handle; | |
cd66358e | 975 | struct bnxt_tc_info *tc_info = bp->tc_info; |
8c95f773 SP |
976 | int rc; |
977 | ||
978 | if (flow_node->decap_l2_node) | |
979 | bnxt_tc_put_decap_l2_node(bp, flow_node); | |
980 | ||
981 | rc = bnxt_tc_put_tunnel_node(bp, &tc_info->decap_table, | |
982 | &tc_info->decap_ht_params, | |
983 | flow_node->decap_node); | |
984 | if (!rc && decap_handle != INVALID_TUNNEL_HANDLE) | |
985 | hwrm_cfa_decap_filter_free(bp, decap_handle); | |
986 | } | |
987 | ||
988 | static int bnxt_tc_resolve_tunnel_hdrs(struct bnxt *bp, | |
989 | struct ip_tunnel_key *tun_key, | |
e9ecc731 | 990 | struct bnxt_tc_l2_key *l2_info) |
8c95f773 | 991 | { |
952c5719 | 992 | #ifdef CONFIG_INET |
e9ecc731 | 993 | struct net_device *real_dst_dev = bp->dev; |
8c95f773 SP |
994 | struct flowi4 flow = { {0} }; |
995 | struct net_device *dst_dev; | |
996 | struct neighbour *nbr; | |
997 | struct rtable *rt; | |
998 | int rc; | |
999 | ||
1000 | flow.flowi4_proto = IPPROTO_UDP; | |
1001 | flow.fl4_dport = tun_key->tp_dst; | |
1002 | flow.daddr = tun_key->u.ipv4.dst; | |
1003 | ||
1004 | rt = ip_route_output_key(dev_net(real_dst_dev), &flow); | |
1005 | if (IS_ERR(rt)) { | |
1006 | netdev_info(bp->dev, "no route to %pI4b", &flow.daddr); | |
1007 | return -EOPNOTSUPP; | |
1008 | } | |
1009 | ||
1010 | /* The route must either point to the real_dst_dev or a dst_dev that | |
1011 | * uses the real_dst_dev. | |
1012 | */ | |
1013 | dst_dev = rt->dst.dev; | |
1014 | if (is_vlan_dev(dst_dev)) { | |
952c5719 | 1015 | #if IS_ENABLED(CONFIG_VLAN_8021Q) |
8c95f773 SP |
1016 | struct vlan_dev_priv *vlan = vlan_dev_priv(dst_dev); |
1017 | ||
1018 | if (vlan->real_dev != real_dst_dev) { | |
1019 | netdev_info(bp->dev, | |
1020 | "dst_dev(%s) doesn't use PF-if(%s)", | |
1021 | netdev_name(dst_dev), | |
1022 | netdev_name(real_dst_dev)); | |
1023 | rc = -EOPNOTSUPP; | |
1024 | goto put_rt; | |
1025 | } | |
1026 | l2_info->inner_vlan_tci = htons(vlan->vlan_id); | |
1027 | l2_info->inner_vlan_tpid = vlan->vlan_proto; | |
1028 | l2_info->num_vlans = 1; | |
952c5719 | 1029 | #endif |
8c95f773 SP |
1030 | } else if (dst_dev != real_dst_dev) { |
1031 | netdev_info(bp->dev, | |
1032 | "dst_dev(%s) for %pI4b is not PF-if(%s)", | |
1033 | netdev_name(dst_dev), &flow.daddr, | |
1034 | netdev_name(real_dst_dev)); | |
1035 | rc = -EOPNOTSUPP; | |
1036 | goto put_rt; | |
1037 | } | |
1038 | ||
1039 | nbr = dst_neigh_lookup(&rt->dst, &flow.daddr); | |
1040 | if (!nbr) { | |
1041 | netdev_info(bp->dev, "can't lookup neighbor for %pI4b", | |
1042 | &flow.daddr); | |
1043 | rc = -EOPNOTSUPP; | |
1044 | goto put_rt; | |
1045 | } | |
1046 | ||
1047 | tun_key->u.ipv4.src = flow.saddr; | |
1048 | tun_key->ttl = ip4_dst_hoplimit(&rt->dst); | |
1049 | neigh_ha_snapshot(l2_info->dmac, nbr, dst_dev); | |
1050 | ether_addr_copy(l2_info->smac, dst_dev->dev_addr); | |
1051 | neigh_release(nbr); | |
1052 | ip_rt_put(rt); | |
1053 | ||
1054 | return 0; | |
1055 | put_rt: | |
1056 | ip_rt_put(rt); | |
1057 | return rc; | |
952c5719 MC |
1058 | #else |
1059 | return -EOPNOTSUPP; | |
1060 | #endif | |
8c95f773 SP |
1061 | } |
1062 | ||
1063 | static int bnxt_tc_get_decap_handle(struct bnxt *bp, struct bnxt_tc_flow *flow, | |
1064 | struct bnxt_tc_flow_node *flow_node, | |
1065 | __le32 *decap_filter_handle) | |
1066 | { | |
1067 | struct ip_tunnel_key *decap_key = &flow->tun_key; | |
cd66358e | 1068 | struct bnxt_tc_info *tc_info = bp->tc_info; |
8c95f773 SP |
1069 | struct bnxt_tc_l2_key l2_info = { {0} }; |
1070 | struct bnxt_tc_tunnel_node *decap_node; | |
1071 | struct ip_tunnel_key tun_key = { 0 }; | |
1072 | struct bnxt_tc_l2_key *decap_l2_info; | |
1073 | __le32 ref_decap_handle; | |
1074 | int rc; | |
1075 | ||
1076 | /* Check if there's another flow using the same tunnel decap. | |
1077 | * If not, add this tunnel to the table and resolve the other | |
479ca3bf SB |
1078 | * tunnel header fileds. Ignore src_port in the tunnel_key, |
1079 | * since it is not required for decap filters. | |
8c95f773 | 1080 | */ |
479ca3bf | 1081 | decap_key->tp_src = 0; |
8c95f773 SP |
1082 | decap_node = bnxt_tc_get_tunnel_node(bp, &tc_info->decap_table, |
1083 | &tc_info->decap_ht_params, | |
1084 | decap_key); | |
1085 | if (!decap_node) | |
1086 | return -ENOMEM; | |
1087 | ||
1088 | flow_node->decap_node = decap_node; | |
1089 | ||
1090 | if (decap_node->tunnel_handle != INVALID_TUNNEL_HANDLE) | |
1091 | goto done; | |
1092 | ||
1093 | /* Resolve the L2 fields for tunnel decap | |
1094 | * Resolve the route for remote vtep (saddr) of the decap key | |
1095 | * Find it's next-hop mac addrs | |
1096 | */ | |
1097 | tun_key.u.ipv4.dst = flow->tun_key.u.ipv4.src; | |
1098 | tun_key.tp_dst = flow->tun_key.tp_dst; | |
e9ecc731 | 1099 | rc = bnxt_tc_resolve_tunnel_hdrs(bp, &tun_key, &l2_info); |
8c95f773 SP |
1100 | if (rc) |
1101 | goto put_decap; | |
1102 | ||
8c95f773 | 1103 | decap_l2_info = &decap_node->l2_info; |
c8fb7b82 | 1104 | /* decap smac is wildcarded */ |
8c95f773 | 1105 | ether_addr_copy(decap_l2_info->dmac, l2_info.smac); |
8c95f773 SP |
1106 | if (l2_info.num_vlans) { |
1107 | decap_l2_info->num_vlans = l2_info.num_vlans; | |
1108 | decap_l2_info->inner_vlan_tpid = l2_info.inner_vlan_tpid; | |
1109 | decap_l2_info->inner_vlan_tci = l2_info.inner_vlan_tci; | |
1110 | } | |
1111 | flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS; | |
1112 | ||
1113 | /* For getting a decap_filter_handle we first need to check if | |
1114 | * there are any other decap flows that share the same tunnel L2 | |
1115 | * key and if so, pass that flow's decap_filter_handle as the | |
1116 | * ref_decap_handle for this flow. | |
1117 | */ | |
1118 | rc = bnxt_tc_get_ref_decap_handle(bp, flow, decap_l2_info, flow_node, | |
1119 | &ref_decap_handle); | |
1120 | if (rc) | |
1121 | goto put_decap; | |
1122 | ||
1123 | /* Issue the hwrm cmd to allocate a decap filter handle */ | |
1124 | rc = hwrm_cfa_decap_filter_alloc(bp, flow, decap_l2_info, | |
1125 | ref_decap_handle, | |
1126 | &decap_node->tunnel_handle); | |
1127 | if (rc) | |
1128 | goto put_decap_l2; | |
1129 | ||
1130 | done: | |
1131 | *decap_filter_handle = decap_node->tunnel_handle; | |
1132 | return 0; | |
1133 | ||
1134 | put_decap_l2: | |
1135 | bnxt_tc_put_decap_l2_node(bp, flow_node); | |
1136 | put_decap: | |
1137 | bnxt_tc_put_tunnel_node(bp, &tc_info->decap_table, | |
1138 | &tc_info->decap_ht_params, | |
1139 | flow_node->decap_node); | |
1140 | return rc; | |
1141 | } | |
1142 | ||
1143 | static void bnxt_tc_put_encap_handle(struct bnxt *bp, | |
1144 | struct bnxt_tc_tunnel_node *encap_node) | |
1145 | { | |
1146 | __le32 encap_handle = encap_node->tunnel_handle; | |
cd66358e | 1147 | struct bnxt_tc_info *tc_info = bp->tc_info; |
8c95f773 SP |
1148 | int rc; |
1149 | ||
1150 | rc = bnxt_tc_put_tunnel_node(bp, &tc_info->encap_table, | |
1151 | &tc_info->encap_ht_params, encap_node); | |
1152 | if (!rc && encap_handle != INVALID_TUNNEL_HANDLE) | |
1153 | hwrm_cfa_encap_record_free(bp, encap_handle); | |
1154 | } | |
1155 | ||
1156 | /* Lookup the tunnel encap table and check if there's an encap_handle | |
1157 | * alloc'd already. | |
1158 | * If not, query L2 info via a route lookup and issue an encap_record_alloc | |
1159 | * cmd to FW. | |
1160 | */ | |
1161 | static int bnxt_tc_get_encap_handle(struct bnxt *bp, struct bnxt_tc_flow *flow, | |
1162 | struct bnxt_tc_flow_node *flow_node, | |
1163 | __le32 *encap_handle) | |
1164 | { | |
1165 | struct ip_tunnel_key *encap_key = &flow->actions.tun_encap_key; | |
cd66358e | 1166 | struct bnxt_tc_info *tc_info = bp->tc_info; |
8c95f773 SP |
1167 | struct bnxt_tc_tunnel_node *encap_node; |
1168 | int rc; | |
1169 | ||
1170 | /* Check if there's another flow using the same tunnel encap. | |
1171 | * If not, add this tunnel to the table and resolve the other | |
1172 | * tunnel header fileds | |
1173 | */ | |
1174 | encap_node = bnxt_tc_get_tunnel_node(bp, &tc_info->encap_table, | |
1175 | &tc_info->encap_ht_params, | |
1176 | encap_key); | |
1177 | if (!encap_node) | |
1178 | return -ENOMEM; | |
1179 | ||
1180 | flow_node->encap_node = encap_node; | |
1181 | ||
1182 | if (encap_node->tunnel_handle != INVALID_TUNNEL_HANDLE) | |
1183 | goto done; | |
1184 | ||
e9ecc731 | 1185 | rc = bnxt_tc_resolve_tunnel_hdrs(bp, encap_key, &encap_node->l2_info); |
8c95f773 SP |
1186 | if (rc) |
1187 | goto put_encap; | |
1188 | ||
1189 | /* Allocate a new tunnel encap record */ | |
1190 | rc = hwrm_cfa_encap_record_alloc(bp, encap_key, &encap_node->l2_info, | |
1191 | &encap_node->tunnel_handle); | |
1192 | if (rc) | |
1193 | goto put_encap; | |
1194 | ||
1195 | done: | |
1196 | *encap_handle = encap_node->tunnel_handle; | |
1197 | return 0; | |
1198 | ||
1199 | put_encap: | |
1200 | bnxt_tc_put_tunnel_node(bp, &tc_info->encap_table, | |
1201 | &tc_info->encap_ht_params, encap_node); | |
1202 | return rc; | |
1203 | } | |
1204 | ||
1205 | static void bnxt_tc_put_tunnel_handle(struct bnxt *bp, | |
1206 | struct bnxt_tc_flow *flow, | |
1207 | struct bnxt_tc_flow_node *flow_node) | |
1208 | { | |
1209 | if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP) | |
1210 | bnxt_tc_put_decap_handle(bp, flow_node); | |
1211 | else if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) | |
1212 | bnxt_tc_put_encap_handle(bp, flow_node->encap_node); | |
1213 | } | |
1214 | ||
1215 | static int bnxt_tc_get_tunnel_handle(struct bnxt *bp, | |
1216 | struct bnxt_tc_flow *flow, | |
1217 | struct bnxt_tc_flow_node *flow_node, | |
1218 | __le32 *tunnel_handle) | |
1219 | { | |
1220 | if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP) | |
1221 | return bnxt_tc_get_decap_handle(bp, flow, flow_node, | |
1222 | tunnel_handle); | |
1223 | else if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) | |
1224 | return bnxt_tc_get_encap_handle(bp, flow, flow_node, | |
1225 | tunnel_handle); | |
1226 | else | |
1227 | return 0; | |
1228 | } | |
2ae7408f SP |
1229 | static int __bnxt_tc_del_flow(struct bnxt *bp, |
1230 | struct bnxt_tc_flow_node *flow_node) | |
1231 | { | |
cd66358e | 1232 | struct bnxt_tc_info *tc_info = bp->tc_info; |
2ae7408f SP |
1233 | int rc; |
1234 | ||
1235 | /* send HWRM cmd to free the flow-id */ | |
1236 | bnxt_hwrm_cfa_flow_free(bp, flow_node->flow_handle); | |
1237 | ||
1238 | mutex_lock(&tc_info->lock); | |
1239 | ||
8c95f773 SP |
1240 | /* release references to any tunnel encap/decap nodes */ |
1241 | bnxt_tc_put_tunnel_handle(bp, &flow_node->flow, flow_node); | |
1242 | ||
2ae7408f SP |
1243 | /* release reference to l2 node */ |
1244 | bnxt_tc_put_l2_node(bp, flow_node); | |
1245 | ||
1246 | mutex_unlock(&tc_info->lock); | |
1247 | ||
1248 | rc = rhashtable_remove_fast(&tc_info->flow_table, &flow_node->node, | |
1249 | tc_info->flow_ht_params); | |
1250 | if (rc) | |
1251 | netdev_err(bp->dev, "Error: %s: rhashtable_remove_fast rc=%d", | |
1252 | __func__, rc); | |
1253 | ||
1254 | kfree_rcu(flow_node, rcu); | |
1255 | return 0; | |
1256 | } | |
1257 | ||
e9ecc731 SP |
1258 | static void bnxt_tc_set_src_fid(struct bnxt *bp, struct bnxt_tc_flow *flow, |
1259 | u16 src_fid) | |
1260 | { | |
1261 | if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP) | |
1262 | flow->src_fid = bp->pf.fw_fid; | |
1263 | else | |
1264 | flow->src_fid = src_fid; | |
1265 | } | |
1266 | ||
2ae7408f SP |
1267 | /* Add a new flow or replace an existing flow. |
1268 | * Notes on locking: | |
1269 | * There are essentially two critical sections here. | |
1270 | * 1. while adding a new flow | |
1271 | * a) lookup l2-key | |
1272 | * b) issue HWRM cmd and get flow_handle | |
1273 | * c) link l2-key with flow | |
1274 | * 2. while deleting a flow | |
1275 | * a) unlinking l2-key from flow | |
1276 | * A lock is needed to protect these two critical sections. | |
1277 | * | |
1278 | * The hash-tables are already protected by the rhashtable API. | |
1279 | */ | |
1280 | static int bnxt_tc_add_flow(struct bnxt *bp, u16 src_fid, | |
1281 | struct tc_cls_flower_offload *tc_flow_cmd) | |
1282 | { | |
1283 | struct bnxt_tc_flow_node *new_node, *old_node; | |
cd66358e | 1284 | struct bnxt_tc_info *tc_info = bp->tc_info; |
2ae7408f | 1285 | struct bnxt_tc_flow *flow; |
8c95f773 | 1286 | __le32 tunnel_handle = 0; |
2ae7408f SP |
1287 | __le16 ref_flow_handle; |
1288 | int rc; | |
1289 | ||
1290 | /* allocate memory for the new flow and it's node */ | |
1291 | new_node = kzalloc(sizeof(*new_node), GFP_KERNEL); | |
1292 | if (!new_node) { | |
1293 | rc = -ENOMEM; | |
1294 | goto done; | |
1295 | } | |
1296 | new_node->cookie = tc_flow_cmd->cookie; | |
1297 | flow = &new_node->flow; | |
1298 | ||
1299 | rc = bnxt_tc_parse_flow(bp, tc_flow_cmd, flow); | |
1300 | if (rc) | |
1301 | goto free_node; | |
e9ecc731 SP |
1302 | |
1303 | bnxt_tc_set_src_fid(bp, flow, src_fid); | |
2ae7408f SP |
1304 | |
1305 | if (!bnxt_tc_can_offload(bp, flow)) { | |
1306 | rc = -ENOSPC; | |
1307 | goto free_node; | |
1308 | } | |
1309 | ||
1310 | /* If a flow exists with the same cookie, delete it */ | |
1311 | old_node = rhashtable_lookup_fast(&tc_info->flow_table, | |
1312 | &tc_flow_cmd->cookie, | |
1313 | tc_info->flow_ht_params); | |
1314 | if (old_node) | |
1315 | __bnxt_tc_del_flow(bp, old_node); | |
1316 | ||
1317 | /* Check if the L2 part of the flow has been offloaded already. | |
1318 | * If so, bump up it's refcnt and get it's reference handle. | |
1319 | */ | |
1320 | mutex_lock(&tc_info->lock); | |
1321 | rc = bnxt_tc_get_ref_flow_handle(bp, flow, new_node, &ref_flow_handle); | |
1322 | if (rc) | |
1323 | goto unlock; | |
1324 | ||
8c95f773 SP |
1325 | /* If the flow involves tunnel encap/decap, get tunnel_handle */ |
1326 | rc = bnxt_tc_get_tunnel_handle(bp, flow, new_node, &tunnel_handle); | |
1327 | if (rc) | |
1328 | goto put_l2; | |
1329 | ||
2ae7408f SP |
1330 | /* send HWRM cmd to alloc the flow */ |
1331 | rc = bnxt_hwrm_cfa_flow_alloc(bp, flow, ref_flow_handle, | |
8c95f773 | 1332 | tunnel_handle, &new_node->flow_handle); |
2ae7408f | 1333 | if (rc) |
8c95f773 | 1334 | goto put_tunnel; |
2ae7408f | 1335 | |
5a84acbe SP |
1336 | flow->lastused = jiffies; |
1337 | spin_lock_init(&flow->stats_lock); | |
2ae7408f SP |
1338 | /* add new flow to flow-table */ |
1339 | rc = rhashtable_insert_fast(&tc_info->flow_table, &new_node->node, | |
1340 | tc_info->flow_ht_params); | |
1341 | if (rc) | |
1342 | goto hwrm_flow_free; | |
1343 | ||
1344 | mutex_unlock(&tc_info->lock); | |
1345 | return 0; | |
1346 | ||
1347 | hwrm_flow_free: | |
1348 | bnxt_hwrm_cfa_flow_free(bp, new_node->flow_handle); | |
8c95f773 SP |
1349 | put_tunnel: |
1350 | bnxt_tc_put_tunnel_handle(bp, flow, new_node); | |
2ae7408f SP |
1351 | put_l2: |
1352 | bnxt_tc_put_l2_node(bp, new_node); | |
1353 | unlock: | |
1354 | mutex_unlock(&tc_info->lock); | |
1355 | free_node: | |
8c95f773 | 1356 | kfree_rcu(new_node, rcu); |
2ae7408f SP |
1357 | done: |
1358 | netdev_err(bp->dev, "Error: %s: cookie=0x%lx error=%d", | |
1359 | __func__, tc_flow_cmd->cookie, rc); | |
1360 | return rc; | |
1361 | } | |
1362 | ||
1363 | static int bnxt_tc_del_flow(struct bnxt *bp, | |
1364 | struct tc_cls_flower_offload *tc_flow_cmd) | |
1365 | { | |
cd66358e | 1366 | struct bnxt_tc_info *tc_info = bp->tc_info; |
2ae7408f SP |
1367 | struct bnxt_tc_flow_node *flow_node; |
1368 | ||
1369 | flow_node = rhashtable_lookup_fast(&tc_info->flow_table, | |
1370 | &tc_flow_cmd->cookie, | |
1371 | tc_info->flow_ht_params); | |
b9ecc340 | 1372 | if (!flow_node) |
2ae7408f | 1373 | return -EINVAL; |
2ae7408f SP |
1374 | |
1375 | return __bnxt_tc_del_flow(bp, flow_node); | |
1376 | } | |
1377 | ||
1378 | static int bnxt_tc_get_flow_stats(struct bnxt *bp, | |
1379 | struct tc_cls_flower_offload *tc_flow_cmd) | |
1380 | { | |
5a84acbe | 1381 | struct bnxt_tc_flow_stats stats, *curr_stats, *prev_stats; |
cd66358e | 1382 | struct bnxt_tc_info *tc_info = bp->tc_info; |
d7bc7305 | 1383 | struct bnxt_tc_flow_node *flow_node; |
5a84acbe SP |
1384 | struct bnxt_tc_flow *flow; |
1385 | unsigned long lastused; | |
d7bc7305 SP |
1386 | |
1387 | flow_node = rhashtable_lookup_fast(&tc_info->flow_table, | |
1388 | &tc_flow_cmd->cookie, | |
1389 | tc_info->flow_ht_params); | |
b9ecc340 | 1390 | if (!flow_node) |
d7bc7305 | 1391 | return -1; |
d7bc7305 | 1392 | |
5a84acbe SP |
1393 | flow = &flow_node->flow; |
1394 | curr_stats = &flow->stats; | |
1395 | prev_stats = &flow->prev_stats; | |
1396 | ||
1397 | spin_lock(&flow->stats_lock); | |
1398 | stats.packets = curr_stats->packets - prev_stats->packets; | |
1399 | stats.bytes = curr_stats->bytes - prev_stats->bytes; | |
1400 | *prev_stats = *curr_stats; | |
1401 | lastused = flow->lastused; | |
1402 | spin_unlock(&flow->stats_lock); | |
1403 | ||
1404 | tcf_exts_stats_update(tc_flow_cmd->exts, stats.bytes, stats.packets, | |
1405 | lastused); | |
1406 | return 0; | |
1407 | } | |
1408 | ||
1409 | static int | |
1410 | bnxt_hwrm_cfa_flow_stats_get(struct bnxt *bp, int num_flows, | |
1411 | struct bnxt_tc_stats_batch stats_batch[]) | |
1412 | { | |
1413 | struct hwrm_cfa_flow_stats_output *resp = bp->hwrm_cmd_resp_addr; | |
1414 | struct hwrm_cfa_flow_stats_input req = { 0 }; | |
1415 | __le16 *req_flow_handles = &req.flow_handle_0; | |
1416 | int rc, i; | |
1417 | ||
1418 | bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_STATS, -1, -1); | |
1419 | req.num_flows = cpu_to_le16(num_flows); | |
1420 | for (i = 0; i < num_flows; i++) { | |
1421 | struct bnxt_tc_flow_node *flow_node = stats_batch[i].flow_node; | |
1422 | ||
1423 | req_flow_handles[i] = flow_node->flow_handle; | |
1424 | } | |
1425 | ||
1426 | mutex_lock(&bp->hwrm_cmd_lock); | |
1427 | rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); | |
1428 | if (!rc) { | |
1429 | __le64 *resp_packets = &resp->packet_0; | |
1430 | __le64 *resp_bytes = &resp->byte_0; | |
1431 | ||
1432 | for (i = 0; i < num_flows; i++) { | |
1433 | stats_batch[i].hw_stats.packets = | |
1434 | le64_to_cpu(resp_packets[i]); | |
1435 | stats_batch[i].hw_stats.bytes = | |
1436 | le64_to_cpu(resp_bytes[i]); | |
1437 | } | |
1438 | } else { | |
1439 | netdev_info(bp->dev, "error rc=%d", rc); | |
1440 | } | |
5a84acbe | 1441 | mutex_unlock(&bp->hwrm_cmd_lock); |
6ae777ea VD |
1442 | |
1443 | if (rc) | |
1444 | rc = -EIO; | |
5a84acbe SP |
1445 | return rc; |
1446 | } | |
1447 | ||
1448 | /* Add val to accum while handling a possible wraparound | |
1449 | * of val. Eventhough val is of type u64, its actual width | |
1450 | * is denoted by mask and will wrap-around beyond that width. | |
1451 | */ | |
1452 | static void accumulate_val(u64 *accum, u64 val, u64 mask) | |
1453 | { | |
1454 | #define low_bits(x, mask) ((x) & (mask)) | |
1455 | #define high_bits(x, mask) ((x) & ~(mask)) | |
1456 | bool wrapped = val < low_bits(*accum, mask); | |
1457 | ||
1458 | *accum = high_bits(*accum, mask) + val; | |
1459 | if (wrapped) | |
1460 | *accum += (mask + 1); | |
1461 | } | |
1462 | ||
1463 | /* The HW counters' width is much less than 64bits. | |
1464 | * Handle possible wrap-around while updating the stat counters | |
1465 | */ | |
1466 | static void bnxt_flow_stats_accum(struct bnxt_tc_info *tc_info, | |
1467 | struct bnxt_tc_flow_stats *acc_stats, | |
1468 | struct bnxt_tc_flow_stats *hw_stats) | |
1469 | { | |
1470 | accumulate_val(&acc_stats->bytes, hw_stats->bytes, tc_info->bytes_mask); | |
1471 | accumulate_val(&acc_stats->packets, hw_stats->packets, | |
1472 | tc_info->packets_mask); | |
1473 | } | |
1474 | ||
1475 | static int | |
1476 | bnxt_tc_flow_stats_batch_update(struct bnxt *bp, int num_flows, | |
1477 | struct bnxt_tc_stats_batch stats_batch[]) | |
1478 | { | |
cd66358e | 1479 | struct bnxt_tc_info *tc_info = bp->tc_info; |
5a84acbe SP |
1480 | int rc, i; |
1481 | ||
1482 | rc = bnxt_hwrm_cfa_flow_stats_get(bp, num_flows, stats_batch); | |
d7bc7305 SP |
1483 | if (rc) |
1484 | return rc; | |
1485 | ||
5a84acbe SP |
1486 | for (i = 0; i < num_flows; i++) { |
1487 | struct bnxt_tc_flow_node *flow_node = stats_batch[i].flow_node; | |
1488 | struct bnxt_tc_flow *flow = &flow_node->flow; | |
1489 | ||
1490 | spin_lock(&flow->stats_lock); | |
1491 | bnxt_flow_stats_accum(tc_info, &flow->stats, | |
1492 | &stats_batch[i].hw_stats); | |
1493 | if (flow->stats.packets != flow->prev_stats.packets) | |
1494 | flow->lastused = jiffies; | |
1495 | spin_unlock(&flow->stats_lock); | |
1496 | } | |
1497 | ||
2ae7408f SP |
1498 | return 0; |
1499 | } | |
1500 | ||
5a84acbe SP |
1501 | static int |
1502 | bnxt_tc_flow_stats_batch_prep(struct bnxt *bp, | |
1503 | struct bnxt_tc_stats_batch stats_batch[], | |
1504 | int *num_flows) | |
1505 | { | |
cd66358e | 1506 | struct bnxt_tc_info *tc_info = bp->tc_info; |
5a84acbe SP |
1507 | struct rhashtable_iter *iter = &tc_info->iter; |
1508 | void *flow_node; | |
1509 | int rc, i; | |
1510 | ||
97a6ec4a | 1511 | rhashtable_walk_start(iter); |
5a84acbe SP |
1512 | |
1513 | rc = 0; | |
1514 | for (i = 0; i < BNXT_FLOW_STATS_BATCH_MAX; i++) { | |
1515 | flow_node = rhashtable_walk_next(iter); | |
1516 | if (IS_ERR(flow_node)) { | |
1517 | i = 0; | |
1518 | if (PTR_ERR(flow_node) == -EAGAIN) { | |
1519 | continue; | |
1520 | } else { | |
1521 | rc = PTR_ERR(flow_node); | |
1522 | goto done; | |
1523 | } | |
1524 | } | |
1525 | ||
1526 | /* No more flows */ | |
1527 | if (!flow_node) | |
1528 | goto done; | |
1529 | ||
1530 | stats_batch[i].flow_node = flow_node; | |
1531 | } | |
1532 | done: | |
1533 | rhashtable_walk_stop(iter); | |
1534 | *num_flows = i; | |
1535 | return rc; | |
1536 | } | |
1537 | ||
1538 | void bnxt_tc_flow_stats_work(struct bnxt *bp) | |
1539 | { | |
cd66358e | 1540 | struct bnxt_tc_info *tc_info = bp->tc_info; |
5a84acbe SP |
1541 | int num_flows, rc; |
1542 | ||
1543 | num_flows = atomic_read(&tc_info->flow_table.nelems); | |
1544 | if (!num_flows) | |
1545 | return; | |
1546 | ||
1547 | rhashtable_walk_enter(&tc_info->flow_table, &tc_info->iter); | |
1548 | ||
1549 | for (;;) { | |
1550 | rc = bnxt_tc_flow_stats_batch_prep(bp, tc_info->stats_batch, | |
1551 | &num_flows); | |
1552 | if (rc) { | |
1553 | if (rc == -EAGAIN) | |
1554 | continue; | |
1555 | break; | |
1556 | } | |
1557 | ||
1558 | if (!num_flows) | |
1559 | break; | |
1560 | ||
1561 | bnxt_tc_flow_stats_batch_update(bp, num_flows, | |
1562 | tc_info->stats_batch); | |
1563 | } | |
1564 | ||
1565 | rhashtable_walk_exit(&tc_info->iter); | |
1566 | } | |
1567 | ||
2ae7408f SP |
1568 | int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, |
1569 | struct tc_cls_flower_offload *cls_flower) | |
1570 | { | |
1571 | int rc = 0; | |
1572 | ||
1573 | switch (cls_flower->command) { | |
1574 | case TC_CLSFLOWER_REPLACE: | |
1575 | rc = bnxt_tc_add_flow(bp, src_fid, cls_flower); | |
1576 | break; | |
1577 | ||
1578 | case TC_CLSFLOWER_DESTROY: | |
1579 | rc = bnxt_tc_del_flow(bp, cls_flower); | |
1580 | break; | |
1581 | ||
1582 | case TC_CLSFLOWER_STATS: | |
1583 | rc = bnxt_tc_get_flow_stats(bp, cls_flower); | |
1584 | break; | |
1585 | } | |
1586 | return rc; | |
1587 | } | |
1588 | ||
1589 | static const struct rhashtable_params bnxt_tc_flow_ht_params = { | |
1590 | .head_offset = offsetof(struct bnxt_tc_flow_node, node), | |
1591 | .key_offset = offsetof(struct bnxt_tc_flow_node, cookie), | |
1592 | .key_len = sizeof(((struct bnxt_tc_flow_node *)0)->cookie), | |
1593 | .automatic_shrinking = true | |
1594 | }; | |
1595 | ||
1596 | static const struct rhashtable_params bnxt_tc_l2_ht_params = { | |
1597 | .head_offset = offsetof(struct bnxt_tc_l2_node, node), | |
1598 | .key_offset = offsetof(struct bnxt_tc_l2_node, key), | |
1599 | .key_len = BNXT_TC_L2_KEY_LEN, | |
1600 | .automatic_shrinking = true | |
1601 | }; | |
1602 | ||
8c95f773 SP |
1603 | static const struct rhashtable_params bnxt_tc_decap_l2_ht_params = { |
1604 | .head_offset = offsetof(struct bnxt_tc_l2_node, node), | |
1605 | .key_offset = offsetof(struct bnxt_tc_l2_node, key), | |
1606 | .key_len = BNXT_TC_L2_KEY_LEN, | |
1607 | .automatic_shrinking = true | |
1608 | }; | |
1609 | ||
1610 | static const struct rhashtable_params bnxt_tc_tunnel_ht_params = { | |
1611 | .head_offset = offsetof(struct bnxt_tc_tunnel_node, node), | |
1612 | .key_offset = offsetof(struct bnxt_tc_tunnel_node, key), | |
1613 | .key_len = sizeof(struct ip_tunnel_key), | |
1614 | .automatic_shrinking = true | |
1615 | }; | |
1616 | ||
2ae7408f SP |
1617 | /* convert counter width in bits to a mask */ |
1618 | #define mask(width) ((u64)~0 >> (64 - (width))) | |
1619 | ||
1620 | int bnxt_init_tc(struct bnxt *bp) | |
1621 | { | |
cd66358e | 1622 | struct bnxt_tc_info *tc_info; |
2ae7408f SP |
1623 | int rc; |
1624 | ||
8c95f773 | 1625 | if (bp->hwrm_spec_code < 0x10803) { |
2ae7408f SP |
1626 | netdev_warn(bp->dev, |
1627 | "Firmware does not support TC flower offload.\n"); | |
1628 | return -ENOTSUPP; | |
1629 | } | |
cd66358e SP |
1630 | |
1631 | tc_info = kzalloc(sizeof(*tc_info), GFP_KERNEL); | |
1632 | if (!tc_info) | |
1633 | return -ENOMEM; | |
2ae7408f SP |
1634 | mutex_init(&tc_info->lock); |
1635 | ||
1636 | /* Counter widths are programmed by FW */ | |
1637 | tc_info->bytes_mask = mask(36); | |
1638 | tc_info->packets_mask = mask(28); | |
1639 | ||
1640 | tc_info->flow_ht_params = bnxt_tc_flow_ht_params; | |
1641 | rc = rhashtable_init(&tc_info->flow_table, &tc_info->flow_ht_params); | |
1642 | if (rc) | |
cd66358e | 1643 | goto free_tc_info; |
2ae7408f SP |
1644 | |
1645 | tc_info->l2_ht_params = bnxt_tc_l2_ht_params; | |
1646 | rc = rhashtable_init(&tc_info->l2_table, &tc_info->l2_ht_params); | |
1647 | if (rc) | |
1648 | goto destroy_flow_table; | |
1649 | ||
8c95f773 SP |
1650 | tc_info->decap_l2_ht_params = bnxt_tc_decap_l2_ht_params; |
1651 | rc = rhashtable_init(&tc_info->decap_l2_table, | |
1652 | &tc_info->decap_l2_ht_params); | |
1653 | if (rc) | |
1654 | goto destroy_l2_table; | |
1655 | ||
1656 | tc_info->decap_ht_params = bnxt_tc_tunnel_ht_params; | |
1657 | rc = rhashtable_init(&tc_info->decap_table, | |
1658 | &tc_info->decap_ht_params); | |
1659 | if (rc) | |
1660 | goto destroy_decap_l2_table; | |
1661 | ||
1662 | tc_info->encap_ht_params = bnxt_tc_tunnel_ht_params; | |
1663 | rc = rhashtable_init(&tc_info->encap_table, | |
1664 | &tc_info->encap_ht_params); | |
1665 | if (rc) | |
1666 | goto destroy_decap_table; | |
1667 | ||
2ae7408f SP |
1668 | tc_info->enabled = true; |
1669 | bp->dev->hw_features |= NETIF_F_HW_TC; | |
1670 | bp->dev->features |= NETIF_F_HW_TC; | |
cd66358e | 1671 | bp->tc_info = tc_info; |
2ae7408f SP |
1672 | return 0; |
1673 | ||
8c95f773 SP |
1674 | destroy_decap_table: |
1675 | rhashtable_destroy(&tc_info->decap_table); | |
1676 | destroy_decap_l2_table: | |
1677 | rhashtable_destroy(&tc_info->decap_l2_table); | |
1678 | destroy_l2_table: | |
1679 | rhashtable_destroy(&tc_info->l2_table); | |
2ae7408f SP |
1680 | destroy_flow_table: |
1681 | rhashtable_destroy(&tc_info->flow_table); | |
cd66358e SP |
1682 | free_tc_info: |
1683 | kfree(tc_info); | |
2ae7408f SP |
1684 | return rc; |
1685 | } | |
1686 | ||
1687 | void bnxt_shutdown_tc(struct bnxt *bp) | |
1688 | { | |
cd66358e | 1689 | struct bnxt_tc_info *tc_info = bp->tc_info; |
2ae7408f | 1690 | |
cd66358e | 1691 | if (!bnxt_tc_flower_enabled(bp)) |
2ae7408f SP |
1692 | return; |
1693 | ||
1694 | rhashtable_destroy(&tc_info->flow_table); | |
1695 | rhashtable_destroy(&tc_info->l2_table); | |
8c95f773 SP |
1696 | rhashtable_destroy(&tc_info->decap_l2_table); |
1697 | rhashtable_destroy(&tc_info->decap_table); | |
1698 | rhashtable_destroy(&tc_info->encap_table); | |
cd66358e SP |
1699 | kfree(tc_info); |
1700 | bp->tc_info = NULL; | |
2ae7408f | 1701 | } |