projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'for-5.4/apple' into for-linus
[linux-2.6-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_fs_ethtool.c
CommitLineData
6dc6071c
MG		 1/*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/mlx5/fs.h>
34#include "en.h"
db05815b
MM		 35#include "en/params.h"
36#include "en/xsk/umem.h"
6dc6071c
MG		 37
38struct mlx5e_ethtool_rule {
39 struct list_head list;
40 struct ethtool_rx_flow_spec flow_spec;
74491de9 41 struct mlx5_flow_handle *rule;
6dc6071c
MG		 42 struct mlx5e_ethtool_table *eth_ft;
43};
44
45static void put_flow_table(struct mlx5e_ethtool_table *eth_ft)
46{
47 if (!--eth_ft->num_rules) {
48 mlx5_destroy_flow_table(eth_ft->ft);
49 eth_ft->ft = NULL;
50 }
51}
52
1174fce8
MG		 53#define MLX5E_ETHTOOL_L3_L4_PRIO 0
54#define MLX5E_ETHTOOL_L2_PRIO (MLX5E_ETHTOOL_L3_L4_PRIO + ETHTOOL_NUM_L3_L4_FTS)
6dc6071c
MG		 55#define MLX5E_ETHTOOL_NUM_ENTRIES 64000
56#define MLX5E_ETHTOOL_NUM_GROUPS 10
57static struct mlx5e_ethtool_table *get_flow_table(struct mlx5e_priv *priv,
58 struct ethtool_rx_flow_spec *fs,
59 int num_tuples)
60{
61 struct mlx5e_ethtool_table *eth_ft;
62 struct mlx5_flow_namespace *ns;
63 struct mlx5_flow_table *ft;
64 int max_tuples;
65 int table_size;
66 int prio;
67
68 switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) {
1174fce8
MG		 69 case TCP_V4_FLOW:
70 case UDP_V4_FLOW:
ca7deb02
SM		 71 case TCP_V6_FLOW:
72 case UDP_V6_FLOW:
1174fce8
MG		 73 max_tuples = ETHTOOL_NUM_L3_L4_FTS;
74 prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples);
75 eth_ft = &priv->fs.ethtool.l3_l4_ft[prio];
76 break;
77 case IP_USER_FLOW:
ca7deb02 78 case IPV6_USER_FLOW:
1174fce8
MG		 79 max_tuples = ETHTOOL_NUM_L3_L4_FTS;
80 prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples);
81 eth_ft = &priv->fs.ethtool.l3_l4_ft[prio];
82 break;
6dc6071c
MG		 83 case ETHER_FLOW:
84 max_tuples = ETHTOOL_NUM_L2_FTS;
85 prio = max_tuples - num_tuples;
86 eth_ft = &priv->fs.ethtool.l2_ft[prio];
87 prio += MLX5E_ETHTOOL_L2_PRIO;
88 break;
89 default:
90 return ERR_PTR(-EINVAL);
91 }
92
93 eth_ft->num_rules++;
94 if (eth_ft->ft)
95 return eth_ft;
96
97 ns = mlx5_get_flow_namespace(priv->mdev,
98 MLX5_FLOW_NAMESPACE_ETHTOOL);
99 if (!ns)
9eb78923 100 return ERR_PTR(-EOPNOTSUPP);
6dc6071c
MG		 101
102 table_size = min_t(u32, BIT(MLX5_CAP_FLOWTABLE(priv->mdev,
103 flow_table_properties_nic_receive.log_max_ft_size)),
104 MLX5E_ETHTOOL_NUM_ENTRIES);
105 ft = mlx5_create_auto_grouped_flow_table(ns, prio,
106 table_size,
c9f1b073 107 MLX5E_ETHTOOL_NUM_GROUPS, 0, 0);
6dc6071c
MG		 108 if (IS_ERR(ft))
109 return (void *)ft;
110
111 eth_ft->ft = ft;
112 return eth_ft;
113}
114
115static void mask_spec(u8 *mask, u8 *val, size_t size)
116{
117 unsigned int i;
118
119 for (i = 0; i < size; i++, mask++, val++)
120 *((u8 *)val) = *((u8 *)mask) & *((u8 *)val);
121}
122
142644f8
SM		 123#define MLX5E_FTE_SET(header_p, fld, v) \
124 MLX5_SET(fte_match_set_lyr_2_4, header_p, fld, v)
125
126#define MLX5E_FTE_ADDR_OF(header_p, fld) \
127 MLX5_ADDR_OF(fte_match_set_lyr_2_4, header_p, fld)
128
129static void
130set_ip4(void *headers_c, void *headers_v, __be32 ip4src_m,
131 __be32 ip4src_v, __be32 ip4dst_m, __be32 ip4dst_v)
1174fce8
MG		 132{
133 if (ip4src_m) {
142644f8 134 memcpy(MLX5E_FTE_ADDR_OF(headers_v, src_ipv4_src_ipv6.ipv4_layout.ipv4),
1174fce8 135 &ip4src_v, sizeof(ip4src_v));
3a95e0cc
OG		 136 memcpy(MLX5E_FTE_ADDR_OF(headers_c, src_ipv4_src_ipv6.ipv4_layout.ipv4),
137 &ip4src_m, sizeof(ip4src_m));
1174fce8
MG		 138 }
139 if (ip4dst_m) {
142644f8 140 memcpy(MLX5E_FTE_ADDR_OF(headers_v, dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
1174fce8 141 &ip4dst_v, sizeof(ip4dst_v));
3a95e0cc
OG		 142 memcpy(MLX5E_FTE_ADDR_OF(headers_c, dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
143 &ip4dst_m, sizeof(ip4dst_m));
1174fce8 144 }
142644f8
SM		 145
146 MLX5E_FTE_SET(headers_c, ethertype, 0xffff);
147 MLX5E_FTE_SET(headers_v, ethertype, ETH_P_IP);
148}
149
ca7deb02
SM		 150static void
151set_ip6(void *headers_c, void *headers_v, __be32 ip6src_m[4],
152 __be32 ip6src_v[4], __be32 ip6dst_m[4], __be32 ip6dst_v[4])
153{
154 u8 ip6_sz = MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6);
155
156 if (!ipv6_addr_any((struct in6_addr *)ip6src_m)) {
157 memcpy(MLX5E_FTE_ADDR_OF(headers_v, src_ipv4_src_ipv6.ipv6_layout.ipv6),
158 ip6src_v, ip6_sz);
159 memcpy(MLX5E_FTE_ADDR_OF(headers_c, src_ipv4_src_ipv6.ipv6_layout.ipv6),
160 ip6src_m, ip6_sz);
161 }
162 if (!ipv6_addr_any((struct in6_addr *)ip6dst_m)) {
163 memcpy(MLX5E_FTE_ADDR_OF(headers_v, dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
164 ip6dst_v, ip6_sz);
165 memcpy(MLX5E_FTE_ADDR_OF(headers_c, dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
166 ip6dst_m, ip6_sz);
167 }
168
169 MLX5E_FTE_SET(headers_c, ethertype, 0xffff);
170 MLX5E_FTE_SET(headers_v, ethertype, ETH_P_IPV6);
171}
172
142644f8
SM		 173static void
174set_tcp(void *headers_c, void *headers_v, __be16 psrc_m, __be16 psrc_v,
175 __be16 pdst_m, __be16 pdst_v)
176{
177 if (psrc_m) {
3a95e0cc 178 MLX5E_FTE_SET(headers_c, tcp_sport, ntohs(psrc_m));
142644f8
SM		 179 MLX5E_FTE_SET(headers_v, tcp_sport, ntohs(psrc_v));
180 }
181 if (pdst_m) {
3a95e0cc 182 MLX5E_FTE_SET(headers_c, tcp_dport, ntohs(pdst_m));
142644f8
SM		 183 MLX5E_FTE_SET(headers_v, tcp_dport, ntohs(pdst_v));
184 }
185
186 MLX5E_FTE_SET(headers_c, ip_protocol, 0xffff);
187 MLX5E_FTE_SET(headers_v, ip_protocol, IPPROTO_TCP);
188}
189
190static void
191set_udp(void *headers_c, void *headers_v, __be16 psrc_m, __be16 psrc_v,
192 __be16 pdst_m, __be16 pdst_v)
193{
194 if (psrc_m) {
3a95e0cc 195 MLX5E_FTE_SET(headers_c, udp_sport, ntohs(psrc_m));
fc433829 196 MLX5E_FTE_SET(headers_v, udp_sport, ntohs(psrc_v));
142644f8
SM		 197 }
198
199 if (pdst_m) {
3a95e0cc 200 MLX5E_FTE_SET(headers_c, udp_dport, ntohs(pdst_m));
142644f8
SM		 201 MLX5E_FTE_SET(headers_v, udp_dport, ntohs(pdst_v));
202 }
203
204 MLX5E_FTE_SET(headers_c, ip_protocol, 0xffff);
205 MLX5E_FTE_SET(headers_v, ip_protocol, IPPROTO_UDP);
206}
207
208static void
209parse_tcp4(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
210{
211 struct ethtool_tcpip4_spec *l4_mask = &fs->m_u.tcp_ip4_spec;
212 struct ethtool_tcpip4_spec *l4_val = &fs->h_u.tcp_ip4_spec;
213
214 set_ip4(headers_c, headers_v, l4_mask->ip4src, l4_val->ip4src,
215 l4_mask->ip4dst, l4_val->ip4dst);
216
217 set_tcp(headers_c, headers_v, l4_mask->psrc, l4_val->psrc,
218 l4_mask->pdst, l4_val->pdst);
219}
220
221static void
222parse_udp4(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
223{
224 struct ethtool_tcpip4_spec *l4_mask = &fs->m_u.udp_ip4_spec;
225 struct ethtool_tcpip4_spec *l4_val = &fs->h_u.udp_ip4_spec;
226
227 set_ip4(headers_c, headers_v, l4_mask->ip4src, l4_val->ip4src,
228 l4_mask->ip4dst, l4_val->ip4dst);
229
230 set_udp(headers_c, headers_v, l4_mask->psrc, l4_val->psrc,
231 l4_mask->pdst, l4_val->pdst);
232}
233
234static void
235parse_ip4(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
236{
237 struct ethtool_usrip4_spec *l3_mask = &fs->m_u.usr_ip4_spec;
238 struct ethtool_usrip4_spec *l3_val = &fs->h_u.usr_ip4_spec;
239
240 set_ip4(headers_c, headers_v, l3_mask->ip4src, l3_val->ip4src,
241 l3_mask->ip4dst, l3_val->ip4dst);
974ce34a
SM		 242
243 if (l3_mask->proto) {
244 MLX5E_FTE_SET(headers_c, ip_protocol, l3_mask->proto);
245 MLX5E_FTE_SET(headers_v, ip_protocol, l3_val->proto);
246 }
142644f8
SM		 247}
248
ca7deb02
SM		 249static void
250parse_ip6(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
251{
252 struct ethtool_usrip6_spec *l3_mask = &fs->m_u.usr_ip6_spec;
253 struct ethtool_usrip6_spec *l3_val = &fs->h_u.usr_ip6_spec;
254
255 set_ip6(headers_c, headers_v, l3_mask->ip6src,
256 l3_val->ip6src, l3_mask->ip6dst, l3_val->ip6dst);
974ce34a
SM		 257
258 if (l3_mask->l4_proto) {
259 MLX5E_FTE_SET(headers_c, ip_protocol, l3_mask->l4_proto);
260 MLX5E_FTE_SET(headers_v, ip_protocol, l3_val->l4_proto);
261 }
ca7deb02
SM		 262}
263
264static void
265parse_tcp6(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
266{
267 struct ethtool_tcpip6_spec *l4_mask = &fs->m_u.tcp_ip6_spec;
268 struct ethtool_tcpip6_spec *l4_val = &fs->h_u.tcp_ip6_spec;
269
270 set_ip6(headers_c, headers_v, l4_mask->ip6src,
271 l4_val->ip6src, l4_mask->ip6dst, l4_val->ip6dst);
272
273 set_tcp(headers_c, headers_v, l4_mask->psrc, l4_val->psrc,
274 l4_mask->pdst, l4_val->pdst);
275}
276
277static void
278parse_udp6(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
279{
280 struct ethtool_tcpip6_spec *l4_mask = &fs->m_u.udp_ip6_spec;
281 struct ethtool_tcpip6_spec *l4_val = &fs->h_u.udp_ip6_spec;
282
283 set_ip6(headers_c, headers_v, l4_mask->ip6src,
284 l4_val->ip6src, l4_mask->ip6dst, l4_val->ip6dst);
285
286 set_udp(headers_c, headers_v, l4_mask->psrc, l4_val->psrc,
287 l4_mask->pdst, l4_val->pdst);
288}
289
142644f8
SM		 290static void
291parse_ether(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
292{
293 struct ethhdr *eth_mask = &fs->m_u.ether_spec;
294 struct ethhdr *eth_val = &fs->h_u.ether_spec;
295
296 mask_spec((u8 *)eth_mask, (u8 *)eth_val, sizeof(*eth_mask));
297 ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_c, smac_47_16), eth_mask->h_source);
298 ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_v, smac_47_16), eth_val->h_source);
299 ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_c, dmac_47_16), eth_mask->h_dest);
300 ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_v, dmac_47_16), eth_val->h_dest);
301 MLX5E_FTE_SET(headers_c, ethertype, ntohs(eth_mask->h_proto));
302 MLX5E_FTE_SET(headers_v, ethertype, ntohs(eth_val->h_proto));
303}
304
305static void
306set_cvlan(void *headers_c, void *headers_v, __be16 vlan_tci)
307{
308 MLX5E_FTE_SET(headers_c, cvlan_tag, 1);
309 MLX5E_FTE_SET(headers_v, cvlan_tag, 1);
310 MLX5E_FTE_SET(headers_c, first_vid, 0xfff);
311 MLX5E_FTE_SET(headers_v, first_vid, ntohs(vlan_tci));
312}
313
314static void
315set_dmac(void *headers_c, void *headers_v,
316 unsigned char m_dest[ETH_ALEN], unsigned char v_dest[ETH_ALEN])
317{
318 ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_c, dmac_47_16), m_dest);
319 ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_v, dmac_47_16), v_dest);
1174fce8
MG		 320}
321
6dc6071c
MG		 322static int set_flow_attrs(u32 *match_c, u32 *match_v,
323 struct ethtool_rx_flow_spec *fs)
324{
325 void *outer_headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
326 outer_headers);
327 void *outer_headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
328 outer_headers);
329 u32 flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT);
6dc6071c
MG		 330
331 switch (flow_type) {
1174fce8 332 case TCP_V4_FLOW:
142644f8 333 parse_tcp4(outer_headers_c, outer_headers_v, fs);
1174fce8
MG		 334 break;
335 case UDP_V4_FLOW:
142644f8 336 parse_udp4(outer_headers_c, outer_headers_v, fs);
1174fce8
MG		 337 break;
338 case IP_USER_FLOW:
142644f8 339 parse_ip4(outer_headers_c, outer_headers_v, fs);
1174fce8 340 break;
ca7deb02
SM		 341 case TCP_V6_FLOW:
342 parse_tcp6(outer_headers_c, outer_headers_v, fs);
343 break;
344 case UDP_V6_FLOW:
345 parse_udp6(outer_headers_c, outer_headers_v, fs);
346 break;
347 case IPV6_USER_FLOW:
348 parse_ip6(outer_headers_c, outer_headers_v, fs);
349 break;
6dc6071c 350 case ETHER_FLOW:
142644f8 351 parse_ether(outer_headers_c, outer_headers_v, fs);
6dc6071c
MG		 352 break;
353 default:
354 return -EINVAL;
355 }
356
357 if ((fs->flow_type & FLOW_EXT) &&
142644f8
SM		 358 (fs->m_ext.vlan_tci & cpu_to_be16(VLAN_VID_MASK)))
359 set_cvlan(outer_headers_c, outer_headers_v, fs->h_ext.vlan_tci);
360
1174fce8
MG		 361 if (fs->flow_type & FLOW_MAC_EXT &&
362 !is_zero_ether_addr(fs->m_ext.h_dest)) {
077b1e80 363 mask_spec(fs->m_ext.h_dest, fs->h_ext.h_dest, ETH_ALEN);
142644f8
SM		 364 set_dmac(outer_headers_c, outer_headers_v, fs->m_ext.h_dest,
365 fs->h_ext.h_dest);
1174fce8 366 }
6dc6071c
MG		 367
368 return 0;
369}
370
371static void add_rule_to_list(struct mlx5e_priv *priv,
372 struct mlx5e_ethtool_rule *rule)
373{
374 struct mlx5e_ethtool_rule *iter;
375 struct list_head *head = &priv->fs.ethtool.rules;
376
377 list_for_each_entry(iter, &priv->fs.ethtool.rules, list) {
378 if (iter->flow_spec.location > rule->flow_spec.location)
379 break;
380 head = &iter->list;
381 }
382 priv->fs.ethtool.tot_num_rules++;
383 list_add(&rule->list, head);
384}
385
386static bool outer_header_zero(u32 *match_criteria)
387{
0242f4a0 388 int size = MLX5_FLD_SZ_BYTES(fte_match_param, outer_headers);
6dc6071c
MG		 389 char *outer_headers_c = MLX5_ADDR_OF(fte_match_param, match_criteria,
390 outer_headers);
391
392 return outer_headers_c[0] == 0 && !memcmp(outer_headers_c,
393 outer_headers_c + 1,
394 size - 1);
395}
396
74491de9
MB		 397static struct mlx5_flow_handle *
398add_ethtool_flow_rule(struct mlx5e_priv *priv,
399 struct mlx5_flow_table *ft,
400 struct ethtool_rx_flow_spec *fs)
6dc6071c
MG		 401{
402 struct mlx5_flow_destination *dst = NULL;
66958ed9 403 struct mlx5_flow_act flow_act = {0};
6dc6071c 404 struct mlx5_flow_spec *spec;
74491de9 405 struct mlx5_flow_handle *rule;
6dc6071c 406 int err = 0;
6dc6071c 407
1b9a07ee 408 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
6dc6071c
MG		 409 if (!spec)
410 return ERR_PTR(-ENOMEM);
411 err = set_flow_attrs(spec->match_criteria, spec->match_value,
412 fs);
413 if (err)
414 goto free;
415
416 if (fs->ring_cookie == RX_CLS_FLOW_DISC) {
66958ed9 417 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
6dc6071c 418 } else {
db05815b
MM		 419 struct mlx5e_params *params = &priv->channels.params;
420 enum mlx5e_rq_group group;
421 struct mlx5e_tir *tir;
422 u16 ix;
423
424 mlx5e_qid_get_ch_and_group(params, fs->ring_cookie, &ix, &group);
425 tir = group == MLX5E_RQ_GROUP_XSK ? priv->xsk_tir : priv->direct_tir;
426
6dc6071c
MG		 427 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
428 if (!dst) {
429 err = -ENOMEM;
430 goto free;
431 }
432
433 dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
db05815b 434 dst->tir_num = tir[ix].tirn;
66958ed9 435 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
6dc6071c
MG		 436 }
437
438 spec->match_criteria_enable = (!outer_header_zero(spec->match_criteria));
bb0ee7dc 439 spec->flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
bcec601f 440 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dst, dst ? 1 : 0);
6dc6071c
MG		 441 if (IS_ERR(rule)) {
442 err = PTR_ERR(rule);
443 netdev_err(priv->netdev, "%s: failed to add ethtool steering rule: %d\n",
444 __func__, err);
445 goto free;
446 }
447free:
448 kvfree(spec);
449 kfree(dst);
450 return err ? ERR_PTR(err) : rule;
451}
452
453static void del_ethtool_rule(struct mlx5e_priv *priv,
454 struct mlx5e_ethtool_rule *eth_rule)
455{
456 if (eth_rule->rule)
74491de9 457 mlx5_del_flow_rules(eth_rule->rule);
6dc6071c
MG		 458 list_del(&eth_rule->list);
459 priv->fs.ethtool.tot_num_rules--;
460 put_flow_table(eth_rule->eth_ft);
461 kfree(eth_rule);
462}
463
464static struct mlx5e_ethtool_rule *find_ethtool_rule(struct mlx5e_priv *priv,
465 int location)
466{
467 struct mlx5e_ethtool_rule *iter;
468
469 list_for_each_entry(iter, &priv->fs.ethtool.rules, list) {
470 if (iter->flow_spec.location == location)
471 return iter;
472 }
473 return NULL;
474}
475
476static struct mlx5e_ethtool_rule *get_ethtool_rule(struct mlx5e_priv *priv,
477 int location)
478{
479 struct mlx5e_ethtool_rule *eth_rule;
480
481 eth_rule = find_ethtool_rule(priv, location);
482 if (eth_rule)
483 del_ethtool_rule(priv, eth_rule);
484
485 eth_rule = kzalloc(sizeof(*eth_rule), GFP_KERNEL);
486 if (!eth_rule)
487 return ERR_PTR(-ENOMEM);
488
489 add_rule_to_list(priv, eth_rule);
490 return eth_rule;
491}
492
493#define MAX_NUM_OF_ETHTOOL_RULES BIT(10)
1174fce8
MG		 494
495#define all_ones(field) (field == (__force typeof(field))-1)
496#define all_zeros_or_all_ones(field) \
497 ((field) == 0 || (field) == (__force typeof(field))-1)
498
b29c61da
SM		 499static int validate_ethter(struct ethtool_rx_flow_spec *fs)
500{
501 struct ethhdr *eth_mask = &fs->m_u.ether_spec;
502 int ntuples = 0;
503
504 if (!is_zero_ether_addr(eth_mask->h_dest))
505 ntuples++;
506 if (!is_zero_ether_addr(eth_mask->h_source))
507 ntuples++;
508 if (eth_mask->h_proto)
509 ntuples++;
510 return ntuples;
511}
512
513static int validate_tcpudp4(struct ethtool_rx_flow_spec *fs)
514{
515 struct ethtool_tcpip4_spec *l4_mask = &fs->m_u.tcp_ip4_spec;
516 int ntuples = 0;
517
518 if (l4_mask->tos)
519 return -EINVAL;
520
3a95e0cc 521 if (l4_mask->ip4src)
b29c61da 522 ntuples++;
3a95e0cc 523 if (l4_mask->ip4dst)
b29c61da 524 ntuples++;
3a95e0cc 525 if (l4_mask->psrc)
b29c61da 526 ntuples++;
3a95e0cc 527 if (l4_mask->pdst)
b29c61da 528 ntuples++;
b29c61da
SM		 529 /* Flow is TCP/UDP */
530 return ++ntuples;
531}
532
533static int validate_ip4(struct ethtool_rx_flow_spec *fs)
534{
535 struct ethtool_usrip4_spec *l3_mask = &fs->m_u.usr_ip4_spec;
536 int ntuples = 0;
537
974ce34a 538 if (l3_mask->l4_4_bytes || l3_mask->tos ||
b29c61da
SM		 539 fs->h_u.usr_ip4_spec.ip_ver != ETH_RX_NFC_IP4)
540 return -EINVAL;
3a95e0cc 541 if (l3_mask->ip4src)
b29c61da 542 ntuples++;
3a95e0cc 543 if (l3_mask->ip4dst)
b29c61da 544 ntuples++;
974ce34a
SM		 545 if (l3_mask->proto)
546 ntuples++;
b29c61da
SM		 547 /* Flow is IPv4 */
548 return ++ntuples;
549}
550
ca7deb02
SM		 551static int validate_ip6(struct ethtool_rx_flow_spec *fs)
552{
553 struct ethtool_usrip6_spec *l3_mask = &fs->m_u.usr_ip6_spec;
554 int ntuples = 0;
555
974ce34a 556 if (l3_mask->l4_4_bytes || l3_mask->tclass)
ca7deb02
SM		 557 return -EINVAL;
558 if (!ipv6_addr_any((struct in6_addr *)l3_mask->ip6src))
559 ntuples++;
560
561 if (!ipv6_addr_any((struct in6_addr *)l3_mask->ip6dst))
562 ntuples++;
974ce34a
SM		 563 if (l3_mask->l4_proto)
564 ntuples++;
ca7deb02
SM		 565 /* Flow is IPv6 */
566 return ++ntuples;
567}
568
569static int validate_tcpudp6(struct ethtool_rx_flow_spec *fs)
570{
571 struct ethtool_tcpip6_spec *l4_mask = &fs->m_u.tcp_ip6_spec;
572 int ntuples = 0;
573
574 if (l4_mask->tclass)
575 return -EINVAL;
576
577 if (!ipv6_addr_any((struct in6_addr *)l4_mask->ip6src))
578 ntuples++;
579
580 if (!ipv6_addr_any((struct in6_addr *)l4_mask->ip6dst))
581 ntuples++;
582
3a95e0cc 583 if (l4_mask->psrc)
ca7deb02 584 ntuples++;
3a95e0cc 585 if (l4_mask->pdst)
ca7deb02 586 ntuples++;
ca7deb02
SM		 587 /* Flow is TCP/UDP */
588 return ++ntuples;
589}
590
b29c61da
SM		 591static int validate_vlan(struct ethtool_rx_flow_spec *fs)
592{
593 if (fs->m_ext.vlan_etype ||
594 fs->m_ext.vlan_tci != cpu_to_be16(VLAN_VID_MASK))
595 return -EINVAL;
596
597 if (fs->m_ext.vlan_tci &&
598 (be16_to_cpu(fs->h_ext.vlan_tci) >= VLAN_N_VID))
599 return -EINVAL;
600
601 return 1;
602}
603
6dc6071c
MG		 604static int validate_flow(struct mlx5e_priv *priv,
605 struct ethtool_rx_flow_spec *fs)
606{
6dc6071c 607 int num_tuples = 0;
b29c61da 608 int ret = 0;
6dc6071c
MG		 609
610 if (fs->location >= MAX_NUM_OF_ETHTOOL_RULES)
b29c61da 611 return -ENOSPC;
6dc6071c 612
db05815b 613 if (fs->ring_cookie != RX_CLS_FLOW_DISC)
694826e3
TT		 614 if (!mlx5e_qid_validate(priv->profile, &priv->channels.params,
615 fs->ring_cookie))
db05815b 616 return -EINVAL;
6dc6071c
MG		 617
618 switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) {
619 case ETHER_FLOW:
b29c61da 620 num_tuples += validate_ethter(fs);
6dc6071c 621 break;
1174fce8
MG		 622 case TCP_V4_FLOW:
623 case UDP_V4_FLOW:
b29c61da
SM		 624 ret = validate_tcpudp4(fs);
625 if (ret < 0)
626 return ret;
627 num_tuples += ret;
1174fce8
MG		 628 break;
629 case IP_USER_FLOW:
b29c61da
SM		 630 ret = validate_ip4(fs);
631 if (ret < 0)
632 return ret;
633 num_tuples += ret;
1174fce8 634 break;
ca7deb02
SM		 635 case TCP_V6_FLOW:
636 case UDP_V6_FLOW:
637 ret = validate_tcpudp6(fs);
638 if (ret < 0)
639 return ret;
640 num_tuples += ret;
641 break;
642 case IPV6_USER_FLOW:
643 ret = validate_ip6(fs);
644 if (ret < 0)
645 return ret;
646 num_tuples += ret;
647 break;
6dc6071c 648 default:
b29c61da 649 return -ENOTSUPP;
6dc6071c
MG		 650 }
651 if ((fs->flow_type & FLOW_EXT)) {
b29c61da
SM		 652 ret = validate_vlan(fs);
653 if (ret < 0)
654 return ret;
655 num_tuples += ret;
6dc6071c
MG		 656 }
657
1174fce8
MG		 658 if (fs->flow_type & FLOW_MAC_EXT &&
659 !is_zero_ether_addr(fs->m_ext.h_dest))
660 num_tuples++;
661
6dc6071c
MG		 662 return num_tuples;
663}
664
cff2b1e3
SM		 665static int
666mlx5e_ethtool_flow_replace(struct mlx5e_priv *priv,
667 struct ethtool_rx_flow_spec *fs)
6dc6071c
MG		 668{
669 struct mlx5e_ethtool_table *eth_ft;
670 struct mlx5e_ethtool_rule *eth_rule;
74491de9 671 struct mlx5_flow_handle *rule;
6dc6071c
MG		 672 int num_tuples;
673 int err;
674
675 num_tuples = validate_flow(priv, fs);
676 if (num_tuples <= 0) {
b29c61da
SM		 677 netdev_warn(priv->netdev, "%s: flow is not valid %d\n",
678 __func__, num_tuples);
679 return num_tuples;
6dc6071c
MG		 680 }
681
682 eth_ft = get_flow_table(priv, fs, num_tuples);
683 if (IS_ERR(eth_ft))
684 return PTR_ERR(eth_ft);
685
686 eth_rule = get_ethtool_rule(priv, fs->location);
687 if (IS_ERR(eth_rule)) {
688 put_flow_table(eth_ft);
689 return PTR_ERR(eth_rule);
690 }
691
692 eth_rule->flow_spec = *fs;
693 eth_rule->eth_ft = eth_ft;
694 if (!eth_ft->ft) {
695 err = -EINVAL;
696 goto del_ethtool_rule;
697 }
698 rule = add_ethtool_flow_rule(priv, eth_ft->ft, fs);
699 if (IS_ERR(rule)) {
700 err = PTR_ERR(rule);
701 goto del_ethtool_rule;
702 }
703
704 eth_rule->rule = rule;
705
706 return 0;
707
708del_ethtool_rule:
709 del_ethtool_rule(priv, eth_rule);
710
711 return err;
712}
713
cff2b1e3
SM		 714static int
715mlx5e_ethtool_flow_remove(struct mlx5e_priv *priv, int location)
6dc6071c
MG		 716{
717 struct mlx5e_ethtool_rule *eth_rule;
718 int err = 0;
719
720 if (location >= MAX_NUM_OF_ETHTOOL_RULES)
721 return -ENOSPC;
722
723 eth_rule = find_ethtool_rule(priv, location);
724 if (!eth_rule) {
725 err = -ENOENT;
726 goto out;
727 }
728
729 del_ethtool_rule(priv, eth_rule);
730out:
731 return err;
732}
733
cff2b1e3
SM		 734static int
735mlx5e_ethtool_get_flow(struct mlx5e_priv *priv,
736 struct ethtool_rxnfc *info, int location)
f913a72a
MG		 737{
738 struct mlx5e_ethtool_rule *eth_rule;
739
740 if (location < 0 || location >= MAX_NUM_OF_ETHTOOL_RULES)
741 return -EINVAL;
742
743 list_for_each_entry(eth_rule, &priv->fs.ethtool.rules, list) {
744 if (eth_rule->flow_spec.location == location) {
745 info->fs = eth_rule->flow_spec;
746 return 0;
747 }
748 }
749
750 return -ENOENT;
751}
752
cff2b1e3
SM		 753static int
754mlx5e_ethtool_get_all_flows(struct mlx5e_priv *priv,
755 struct ethtool_rxnfc *info, u32 *rule_locs)
f913a72a
MG		 756{
757 int location = 0;
758 int idx = 0;
759 int err = 0;
760
5e82c9e4 761 info->data = MAX_NUM_OF_ETHTOOL_RULES;
f913a72a
MG		 762 while ((!err || err == -ENOENT) && idx < info->rule_cnt) {
763 err = mlx5e_ethtool_get_flow(priv, info, location);
764 if (!err)
765 rule_locs[idx++] = location;
766 location++;
767 }
768 return err;
769}
770
6dc6071c
MG		 771void mlx5e_ethtool_cleanup_steering(struct mlx5e_priv *priv)
772{
773 struct mlx5e_ethtool_rule *iter;
774 struct mlx5e_ethtool_rule *temp;
775
776 list_for_each_entry_safe(iter, temp, &priv->fs.ethtool.rules, list)
777 del_ethtool_rule(priv, iter);
778}
779
780void mlx5e_ethtool_init_steering(struct mlx5e_priv *priv)
781{
782 INIT_LIST_HEAD(&priv->fs.ethtool.rules);
783}
cff2b1e3 784
756c4160
AL		 785static enum mlx5e_traffic_types flow_type_to_traffic_type(u32 flow_type)
786{
787 switch (flow_type) {
788 case TCP_V4_FLOW:
789 return MLX5E_TT_IPV4_TCP;
790 case TCP_V6_FLOW:
791 return MLX5E_TT_IPV6_TCP;
792 case UDP_V4_FLOW:
793 return MLX5E_TT_IPV4_UDP;
794 case UDP_V6_FLOW:
795 return MLX5E_TT_IPV6_UDP;
796 case AH_V4_FLOW:
797 return MLX5E_TT_IPV4_IPSEC_AH;
798 case AH_V6_FLOW:
799 return MLX5E_TT_IPV6_IPSEC_AH;
800 case ESP_V4_FLOW:
801 return MLX5E_TT_IPV4_IPSEC_ESP;
802 case ESP_V6_FLOW:
803 return MLX5E_TT_IPV6_IPSEC_ESP;
804 case IPV4_FLOW:
805 return MLX5E_TT_IPV4;
806 case IPV6_FLOW:
807 return MLX5E_TT_IPV6;
808 default:
809 return MLX5E_NUM_INDIR_TIRS;
810 }
811}
812
813static int mlx5e_set_rss_hash_opt(struct mlx5e_priv *priv,
814 struct ethtool_rxnfc *nfc)
815{
816 int inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
817 enum mlx5e_traffic_types tt;
818 u8 rx_hash_field = 0;
819 void *in;
820
821 tt = flow_type_to_traffic_type(nfc->flow_type);
822 if (tt == MLX5E_NUM_INDIR_TIRS)
823 return -EINVAL;
824
825 /* RSS does not support anything other than hashing to queues
826 * on src IP, dest IP, TCP/UDP src port and TCP/UDP dest
827 * port.
828 */
829 if (nfc->flow_type != TCP_V4_FLOW &&
830 nfc->flow_type != TCP_V6_FLOW &&
831 nfc->flow_type != UDP_V4_FLOW &&
832 nfc->flow_type != UDP_V6_FLOW)
833 return -EOPNOTSUPP;
834
835 if (nfc->data & ~(RXH_IP_SRC | RXH_IP_DST |
836 RXH_L4_B_0_1 | RXH_L4_B_2_3))
837 return -EOPNOTSUPP;
838
839 if (nfc->data & RXH_IP_SRC)
840 rx_hash_field |= MLX5_HASH_FIELD_SEL_SRC_IP;
841 if (nfc->data & RXH_IP_DST)
842 rx_hash_field |= MLX5_HASH_FIELD_SEL_DST_IP;
843 if (nfc->data & RXH_L4_B_0_1)
844 rx_hash_field |= MLX5_HASH_FIELD_SEL_L4_SPORT;
845 if (nfc->data & RXH_L4_B_2_3)
846 rx_hash_field |= MLX5_HASH_FIELD_SEL_L4_DPORT;
847
848 in = kvzalloc(inlen, GFP_KERNEL);
849 if (!in)
850 return -ENOMEM;
851
852 mutex_lock(&priv->state_lock);
853
854 if (rx_hash_field == priv->rss_params.rx_hash_fields[tt])
855 goto out;
856
857 priv->rss_params.rx_hash_fields[tt] = rx_hash_field;
858 mlx5e_modify_tirs_hash(priv, in, inlen);
859
860out:
861 mutex_unlock(&priv->state_lock);
862 kvfree(in);
863 return 0;
864}
865
866static int mlx5e_get_rss_hash_opt(struct mlx5e_priv *priv,
867 struct ethtool_rxnfc *nfc)
868{
869 enum mlx5e_traffic_types tt;
870 u32 hash_field = 0;
871
872 tt = flow_type_to_traffic_type(nfc->flow_type);
873 if (tt == MLX5E_NUM_INDIR_TIRS)
874 return -EINVAL;
875
876 hash_field = priv->rss_params.rx_hash_fields[tt];
877 nfc->data = 0;
878
879 if (hash_field & MLX5_HASH_FIELD_SEL_SRC_IP)
880 nfc->data |= RXH_IP_SRC;
881 if (hash_field & MLX5_HASH_FIELD_SEL_DST_IP)
882 nfc->data |= RXH_IP_DST;
883 if (hash_field & MLX5_HASH_FIELD_SEL_L4_SPORT)
884 nfc->data |= RXH_L4_B_0_1;
885 if (hash_field & MLX5_HASH_FIELD_SEL_L4_DPORT)
886 nfc->data |= RXH_L4_B_2_3;
887
888 return 0;
889}
890
cff2b1e3
SM		 891int mlx5e_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
892{
893 int err = 0;
894 struct mlx5e_priv *priv = netdev_priv(dev);
895
896 switch (cmd->cmd) {
897 case ETHTOOL_SRXCLSRLINS:
898 err = mlx5e_ethtool_flow_replace(priv, &cmd->fs);
899 break;
900 case ETHTOOL_SRXCLSRLDEL:
901 err = mlx5e_ethtool_flow_remove(priv, cmd->fs.location);
902 break;
756c4160
AL		 903 case ETHTOOL_SRXFH:
904 err = mlx5e_set_rss_hash_opt(priv, cmd);
905 break;
cff2b1e3
SM		 906 default:
907 err = -EOPNOTSUPP;
908 break;
909 }
910
911 return err;
912}
913
914int mlx5e_get_rxnfc(struct net_device *dev,
915 struct ethtool_rxnfc *info, u32 *rule_locs)
916{
917 struct mlx5e_priv *priv = netdev_priv(dev);
918 int err = 0;
919
920 switch (info->cmd) {
921 case ETHTOOL_GRXRINGS:
922 info->data = priv->channels.params.num_channels;
923 break;
924 case ETHTOOL_GRXCLSRLCNT:
925 info->rule_cnt = priv->fs.ethtool.tot_num_rules;
926 break;
927 case ETHTOOL_GRXCLSRULE:
928 err = mlx5e_ethtool_get_flow(priv, info, info->fs.location);
929 break;
930 case ETHTOOL_GRXCLSRLALL:
931 err = mlx5e_ethtool_get_all_flows(priv, info, rule_locs);
932 break;
756c4160
AL		 933 case ETHTOOL_GRXFH:
934 err = mlx5e_get_rss_hash_opt(priv, info);
935 break;
cff2b1e3
SM		 936 default:
937 err = -EOPNOTSUPP;
938 break;
939 }
940
941 return err;
942}
943
Linux 6.x block layer and io_uring tree(s)