projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/mlx5e: Refactor reg_c1 usage
[linux-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / eswitch_offloads.c
CommitLineData
69697b6e
OG		 1/*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/etherdevice.h>
133dcfc5 34#include <linux/idr.h>
69697b6e
OG		 35#include <linux/mlx5/driver.h>
36#include <linux/mlx5/mlx5_ifc.h>
37#include <linux/mlx5/vport.h>
38#include <linux/mlx5/fs.h>
39#include "mlx5_core.h"
40#include "eswitch.h"
34ca6535 41#include "esw/indir_table.h"
ea651a86 42#include "esw/acl/ofld.h"
a508728a 43#include "esw/indir_table.h"
80f09dfc 44#include "rdma.h"
e52c2802
PB		 45#include "en.h"
46#include "fs_core.h"
ac004b83 47#include "lib/devcom.h"
a3888f33 48#include "lib/eq.h"
ae430332 49#include "lib/fs_chains.h"
c620b772 50#include "en_tc.h"
69697b6e 51
cd7e4186
BW		 52/* There are two match-all miss flows, one for unicast dst mac and
53 * one for multicast.
54 */
55#define MLX5_ESW_MISS_FLOWS (2)
c9b99abc
BW		 56#define UPLINK_REP_INDEX 0
57
96e32687
EC		 58/* Per vport tables */
59
60#define MLX5_ESW_VPORT_TABLE_SIZE 128
61
62/* This struct is used as a key to the hash table and we need it to be packed
63 * so hash result is consistent
64 */
65struct mlx5_vport_key {
66 u32 chain;
67 u16 prio;
68 u16 vport;
69 u16 vhca_id;
70} __packed;
71
c620b772
AL		 72struct mlx5_vport_tbl_attr {
73 u16 chain;
74 u16 prio;
75 u16 vport;
76};
77
96e32687
EC		 78struct mlx5_vport_table {
79 struct hlist_node hlist;
80 struct mlx5_flow_table *fdb;
81 u32 num_rules;
82 struct mlx5_vport_key key;
83};
84
87dac697
JL		 85#define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
86
96e32687
EC		 87static struct mlx5_flow_table *
88esw_vport_tbl_create(struct mlx5_eswitch *esw, struct mlx5_flow_namespace *ns)
89{
90 struct mlx5_flow_table_attr ft_attr = {};
91 struct mlx5_flow_table *fdb;
92
87dac697 93 ft_attr.autogroup.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS;
96e32687
EC		 94 ft_attr.max_fte = MLX5_ESW_VPORT_TABLE_SIZE;
95 ft_attr.prio = FDB_PER_VPORT;
96 fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
97 if (IS_ERR(fdb)) {
98 esw_warn(esw->dev, "Failed to create per vport FDB Table err %ld\n",
99 PTR_ERR(fdb));
100 }
101
102 return fdb;
103}
104
105static u32 flow_attr_to_vport_key(struct mlx5_eswitch *esw,
c620b772 106 struct mlx5_vport_tbl_attr *attr,
96e32687
EC		 107 struct mlx5_vport_key *key)
108{
c620b772 109 key->vport = attr->vport;
96e32687
EC		 110 key->chain = attr->chain;
111 key->prio = attr->prio;
112 key->vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
113 return jhash(key, sizeof(*key), 0);
114}
115
116/* caller must hold vports.lock */
117static struct mlx5_vport_table *
118esw_vport_tbl_lookup(struct mlx5_eswitch *esw, struct mlx5_vport_key *skey, u32 key)
119{
120 struct mlx5_vport_table *e;
121
122 hash_for_each_possible(esw->fdb_table.offloads.vports.table, e, hlist, key)
123 if (!memcmp(&e->key, skey, sizeof(*skey)))
124 return e;
125
126 return NULL;
127}
128
129static void
c620b772 130esw_vport_tbl_put(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr)
96e32687
EC		 131{
132 struct mlx5_vport_table *e;
133 struct mlx5_vport_key key;
134 u32 hkey;
135
136 mutex_lock(&esw->fdb_table.offloads.vports.lock);
137 hkey = flow_attr_to_vport_key(esw, attr, &key);
138 e = esw_vport_tbl_lookup(esw, &key, hkey);
139 if (!e || --e->num_rules)
140 goto out;
141
142 hash_del(&e->hlist);
143 mlx5_destroy_flow_table(e->fdb);
144 kfree(e);
145out:
146 mutex_unlock(&esw->fdb_table.offloads.vports.lock);
147}
148
149static struct mlx5_flow_table *
c620b772 150esw_vport_tbl_get(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr)
96e32687
EC		 151{
152 struct mlx5_core_dev *dev = esw->dev;
153 struct mlx5_flow_namespace *ns;
154 struct mlx5_flow_table *fdb;
155 struct mlx5_vport_table *e;
156 struct mlx5_vport_key skey;
157 u32 hkey;
158
159 mutex_lock(&esw->fdb_table.offloads.vports.lock);
160 hkey = flow_attr_to_vport_key(esw, attr, &skey);
161 e = esw_vport_tbl_lookup(esw, &skey, hkey);
162 if (e) {
163 e->num_rules++;
164 goto out;
165 }
166
167 e = kzalloc(sizeof(*e), GFP_KERNEL);
168 if (!e) {
169 fdb = ERR_PTR(-ENOMEM);
170 goto err_alloc;
171 }
172
173 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
174 if (!ns) {
175 esw_warn(dev, "Failed to get FDB namespace\n");
176 fdb = ERR_PTR(-ENOENT);
177 goto err_ns;
178 }
179
180 fdb = esw_vport_tbl_create(esw, ns);
181 if (IS_ERR(fdb))
182 goto err_ns;
183
184 e->fdb = fdb;
185 e->num_rules = 1;
186 e->key = skey;
187 hash_add(esw->fdb_table.offloads.vports.table, &e->hlist, hkey);
188out:
189 mutex_unlock(&esw->fdb_table.offloads.vports.lock);
190 return e->fdb;
191
192err_ns:
193 kfree(e);
194err_alloc:
195 mutex_unlock(&esw->fdb_table.offloads.vports.lock);
196 return fdb;
197}
198
199int mlx5_esw_vport_tbl_get(struct mlx5_eswitch *esw)
200{
c620b772 201 struct mlx5_vport_tbl_attr attr;
96e32687
EC		 202 struct mlx5_flow_table *fdb;
203 struct mlx5_vport *vport;
204 int i;
205
c620b772 206 attr.chain = 0;
96e32687 207 attr.prio = 1;
96e32687 208 mlx5_esw_for_all_vports(esw, i, vport) {
c620b772 209 attr.vport = vport->vport;
96e32687 210 fdb = esw_vport_tbl_get(esw, &attr);
d9fb932f 211 if (IS_ERR(fdb))
96e32687
EC		 212 goto out;
213 }
214 return 0;
215
216out:
217 mlx5_esw_vport_tbl_put(esw);
218 return PTR_ERR(fdb);
219}
220
221void mlx5_esw_vport_tbl_put(struct mlx5_eswitch *esw)
222{
c620b772 223 struct mlx5_vport_tbl_attr attr;
96e32687
EC		 224 struct mlx5_vport *vport;
225 int i;
226
c620b772 227 attr.chain = 0;
96e32687 228 attr.prio = 1;
96e32687 229 mlx5_esw_for_all_vports(esw, i, vport) {
c620b772 230 attr.vport = vport->vport;
96e32687
EC		 231 esw_vport_tbl_put(esw, &attr);
232 }
233}
234
235/* End: Per vport tables */
236
879c8f84
BW		 237static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
238 u16 vport_num)
239{
02f3afd9 240 int idx = mlx5_eswitch_vport_num_to_index(esw, vport_num);
879c8f84
BW		 241
242 WARN_ON(idx > esw->total_vports - 1);
243 return &esw->offloads.vport_reps[idx];
244}
245
6f7bbad1
JL		 246static void
247mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
248 struct mlx5_flow_spec *spec,
249 struct mlx5_esw_flow_attr *attr)
250{
251 if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) &&
036e19b9
HI		 252 attr && attr->in_rep)
253 spec->flow_context.flow_source =
254 attr->in_rep->vport == MLX5_VPORT_UPLINK ?
255 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
256 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
6f7bbad1 257}
b7826076 258
c01cfd0f
JL		 259static void
260mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
261 struct mlx5_flow_spec *spec,
a508728a 262 struct mlx5_flow_attr *attr,
b055ecf5
MB		 263 struct mlx5_eswitch *src_esw,
264 u16 vport)
c01cfd0f
JL		 265{
266 void *misc2;
267 void *misc;
268
269 /* Use metadata matching because vport is not represented by single
270 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
271 */
272 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
a508728a
VB		 273 if (mlx5_esw_indir_table_decap_vport(attr))
274 vport = mlx5_esw_indir_table_decap_vport(attr);
c01cfd0f
JL		 275 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
276 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
b055ecf5
MB		 277 mlx5_eswitch_get_vport_metadata_for_match(src_esw,
278 vport));
c01cfd0f
JL		 279
280 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
0f0d3827
PB		 281 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
282 mlx5_eswitch_get_vport_metadata_mask());
c01cfd0f
JL		 283
284 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
c01cfd0f
JL		 285 } else {
286 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
b055ecf5 287 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
c01cfd0f
JL		 288
289 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
290 MLX5_SET(fte_match_set_misc, misc,
291 source_eswitch_owner_vhca_id,
b055ecf5 292 MLX5_CAP_GEN(src_esw->dev, vhca_id));
c01cfd0f
JL		 293
294 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
295 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
296 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
297 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
298 source_eswitch_owner_vhca_id);
299
300 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
301 }
c01cfd0f
JL		 302}
303
a508728a
VB		 304static int
305esw_setup_decap_indir(struct mlx5_eswitch *esw,
306 struct mlx5_flow_attr *attr,
307 struct mlx5_flow_spec *spec)
308{
309 struct mlx5_flow_table *ft;
310
311 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE))
312 return -EOPNOTSUPP;
313
314 ft = mlx5_esw_indir_table_get(esw, attr, spec,
315 mlx5_esw_indir_table_decap_vport(attr), true);
316 return PTR_ERR_OR_ZERO(ft);
317}
318
9e51c0a6 319static void
a508728a
VB		 320esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
321 struct mlx5_flow_attr *attr)
322{
323 if (mlx5_esw_indir_table_decap_vport(attr))
324 mlx5_esw_indir_table_put(esw, attr,
325 mlx5_esw_indir_table_decap_vport(attr),
326 true);
327}
328
329static int
9e51c0a6
VB		 330esw_setup_ft_dest(struct mlx5_flow_destination *dest,
331 struct mlx5_flow_act *flow_act,
a508728a 332 struct mlx5_eswitch *esw,
9e51c0a6 333 struct mlx5_flow_attr *attr,
a508728a 334 struct mlx5_flow_spec *spec,
9e51c0a6
VB		 335 int i)
336{
337 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
338 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
339 dest[i].ft = attr->dest_ft;
a508728a
VB		 340
341 if (mlx5_esw_indir_table_decap_vport(attr))
342 return esw_setup_decap_indir(esw, attr, spec);
343 return 0;
9e51c0a6
VB		 344}
345
346static void
347esw_setup_slow_path_dest(struct mlx5_flow_destination *dest,
348 struct mlx5_flow_act *flow_act,
349 struct mlx5_fs_chains *chains,
350 int i)
351{
352 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
353 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
354 dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
355}
356
357static int
358esw_setup_chain_dest(struct mlx5_flow_destination *dest,
359 struct mlx5_flow_act *flow_act,
360 struct mlx5_fs_chains *chains,
361 u32 chain, u32 prio, u32 level,
362 int i)
363{
364 struct mlx5_flow_table *ft;
365
366 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
367 ft = mlx5_chains_get_table(chains, chain, prio, level);
368 if (IS_ERR(ft))
369 return PTR_ERR(ft);
370
371 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
372 dest[i].ft = ft;
373 return 0;
374}
375
10742efc
VB		 376static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
377 int from, int to)
378{
379 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
380 struct mlx5_fs_chains *chains = esw_chains(esw);
381 int i;
382
383 for (i = from; i < to; i++)
384 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
385 mlx5_chains_put_table(chains, 0, 1, 0);
a508728a
VB		 386 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
387 esw_attr->dests[i].mdev))
388 mlx5_esw_indir_table_put(esw, attr, esw_attr->dests[i].rep->vport,
389 false);
10742efc
VB		 390}
391
392static bool
393esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
394{
395 int i;
396
397 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
398 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
399 return true;
400 return false;
401}
402
403static int
404esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
405 struct mlx5_flow_act *flow_act,
406 struct mlx5_eswitch *esw,
407 struct mlx5_fs_chains *chains,
408 struct mlx5_flow_attr *attr,
409 int *i)
410{
411 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
412 int j, err;
413
414 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE))
415 return -EOPNOTSUPP;
416
417 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
418 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i);
419 if (err)
420 goto err_setup_chain;
421 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
422 flow_act->pkt_reformat = esw_attr->dests[j].pkt_reformat;
423 }
424 return 0;
425
426err_setup_chain:
427 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
428 return err;
429}
430
431static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
432 struct mlx5_flow_attr *attr)
433{
434 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
435
436 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
437}
438
a508728a
VB		 439static bool
440esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
441{
442 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
443 int i;
444
445 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
446 if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
447 esw_attr->dests[i].mdev))
448 return true;
449 return false;
450}
451
452static int
453esw_setup_indir_table(struct mlx5_flow_destination *dest,
454 struct mlx5_flow_act *flow_act,
455 struct mlx5_eswitch *esw,
456 struct mlx5_flow_attr *attr,
457 struct mlx5_flow_spec *spec,
458 bool ignore_flow_lvl,
459 int *i)
460{
461 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
462 int j, err;
463
464 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE))
465 return -EOPNOTSUPP;
466
467 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
468 if (ignore_flow_lvl)
469 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
470 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
471
472 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, spec,
473 esw_attr->dests[j].rep->vport, false);
474 if (IS_ERR(dest[*i].ft)) {
475 err = PTR_ERR(dest[*i].ft);
476 goto err_indir_tbl_get;
477 }
478 }
479
480 if (mlx5_esw_indir_table_decap_vport(attr)) {
481 err = esw_setup_decap_indir(esw, attr, spec);
482 if (err)
483 goto err_indir_tbl_get;
484 }
485
486 return 0;
487
488err_indir_tbl_get:
489 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
490 return err;
491}
492
493static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
494{
495 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
496
497 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
498 esw_cleanup_decap_indir(esw, attr);
499}
500
9e51c0a6
VB		 501static void
502esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
503{
504 mlx5_chains_put_table(chains, chain, prio, level);
505}
506
507static void
508esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
509 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
510 int attr_idx, int dest_idx, bool pkt_reformat)
511{
512 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
513 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport;
514 dest[dest_idx].vport.vhca_id =
515 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
516 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
517 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
518 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP) {
519 if (pkt_reformat) {
520 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
521 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
522 }
523 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
524 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
525 }
526}
527
528static int
529esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
530 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
531 int i)
532{
533 int j;
534
535 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
536 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true);
537 return i;
538}
539
540static int
541esw_setup_dests(struct mlx5_flow_destination *dest,
542 struct mlx5_flow_act *flow_act,
543 struct mlx5_eswitch *esw,
544 struct mlx5_flow_attr *attr,
10742efc 545 struct mlx5_flow_spec *spec,
9e51c0a6
VB		 546 int *i)
547{
548 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
549 struct mlx5_fs_chains *chains = esw_chains(esw);
550 int err = 0;
551
10742efc
VB		 552 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
553 MLX5_CAP_GEN(esw_attr->in_mdev, reg_c_preserve) &&
554 mlx5_eswitch_vport_match_metadata_enabled(esw))
555 attr->flags |= MLX5_ESW_ATTR_FLAG_SRC_REWRITE;
556
9e51c0a6 557 if (attr->dest_ft) {
a508728a 558 esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i);
9e51c0a6
VB		 559 (*i)++;
560 } else if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) {
561 esw_setup_slow_path_dest(dest, flow_act, chains, *i);
562 (*i)++;
563 } else if (attr->dest_chain) {
564 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
565 1, 0, *i);
566 (*i)++;
a508728a
VB		 567 } else if (esw_is_indir_table(esw, attr)) {
568 err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i);
10742efc
VB		 569 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
570 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
9e51c0a6
VB		 571 } else {
572 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
573 }
574
575 return err;
576}
577
578static void
579esw_cleanup_dests(struct mlx5_eswitch *esw,
580 struct mlx5_flow_attr *attr)
581{
10742efc 582 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
9e51c0a6
VB		 583 struct mlx5_fs_chains *chains = esw_chains(esw);
584
a508728a
VB		 585 if (attr->dest_ft) {
586 esw_cleanup_decap_indir(esw, attr);
587 } else if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)) {
10742efc
VB		 588 if (attr->dest_chain)
589 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0);
a508728a
VB		 590 else if (esw_is_indir_table(esw, attr))
591 esw_cleanup_indir_table(esw, attr);
10742efc
VB		 592 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
593 esw_cleanup_chain_src_port_rewrite(esw, attr);
594 }
9e51c0a6
VB		 595}
596
74491de9 597struct mlx5_flow_handle *
3d80d1a2
OG		 598mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
599 struct mlx5_flow_spec *spec,
c620b772 600 struct mlx5_flow_attr *attr)
3d80d1a2 601{
592d3651 602 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {};
42f7ad67 603 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
c620b772 604 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
ae430332 605 struct mlx5_fs_chains *chains = esw_chains(esw);
c620b772
AL		 606 bool split = !!(esw_attr->split_count);
607 struct mlx5_vport_tbl_attr fwd_attr;
74491de9 608 struct mlx5_flow_handle *rule;
e52c2802 609 struct mlx5_flow_table *fdb;
9e51c0a6 610 int i = 0;
3d80d1a2 611
f6455de0 612 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
3d80d1a2
OG		 613 return ERR_PTR(-EOPNOTSUPP);
614
6acfbf38
OG		 615 flow_act.action = attr->action;
616 /* if per flow vlan pop/push is emulated, don't set that into the firmware */
cc495188 617 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
6acfbf38
OG		 618 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH |
619 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
620 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
c620b772
AL		 621 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
622 flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
623 flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
cc495188 624 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
c620b772
AL		 625 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
626 flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
627 flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
cc495188 628 }
6acfbf38 629 }
776b12b6 630
10742efc
VB		 631 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr);
632
66958ed9 633 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
9e51c0a6
VB		 634 int err;
635
10742efc 636 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i);
9e51c0a6
VB		 637 if (err) {
638 rule = ERR_PTR(err);
639 goto err_create_goto_table;
56e858df 640 }
e37a79e5 641 }
14e6b038 642
c620b772
AL		 643 if (esw_attr->decap_pkt_reformat)
644 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
14e6b038 645
66958ed9 646 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
e37a79e5 647 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
171c7625 648 dest[i].counter_id = mlx5_fc_id(attr->counter);
e37a79e5 649 i++;
3d80d1a2
OG		 650 }
651
93b3586e 652 if (attr->outer_match_level != MLX5_MATCH_NONE)
6363651d 653 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
93b3586e
HN		 654 if (attr->inner_match_level != MLX5_MATCH_NONE)
655 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
3d80d1a2 656
aa24670e 657 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
2b688ea5 658 flow_act.modify_hdr = attr->modify_hdr;
d7e75a32 659
96e32687 660 if (split) {
c620b772
AL		 661 fwd_attr.chain = attr->chain;
662 fwd_attr.prio = attr->prio;
663 fwd_attr.vport = esw_attr->in_rep->vport;
664
665 fdb = esw_vport_tbl_get(esw, &fwd_attr);
96e32687 666 } else {
d18296ff 667 if (attr->chain || attr->prio)
ae430332
AL		 668 fdb = mlx5_chains_get_table(chains, attr->chain,
669 attr->prio, 0);
d18296ff 670 else
c620b772 671 fdb = attr->ft;
6fb0701a
PB		 672
673 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_NO_IN_PORT))
a508728a 674 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
b055ecf5
MB		 675 esw_attr->in_mdev->priv.eswitch,
676 esw_attr->in_rep->vport);
96e32687 677 }
e52c2802
PB		 678 if (IS_ERR(fdb)) {
679 rule = ERR_CAST(fdb);
680 goto err_esw_get;
681 }
682
84be2fda 683 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec))
c620b772 684 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr,
10caabda 685 &flow_act, dest, i);
84be2fda 686 else
10caabda 687 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i);
3d80d1a2 688 if (IS_ERR(rule))
e52c2802 689 goto err_add_rule;
375f51e2 690 else
525e84be 691 atomic64_inc(&esw->offloads.num_flows);
3d80d1a2 692
e52c2802
PB		 693 return rule;
694
695err_add_rule:
96e32687 696 if (split)
c620b772 697 esw_vport_tbl_put(esw, &fwd_attr);
d18296ff 698 else if (attr->chain || attr->prio)
ae430332 699 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
e52c2802 700err_esw_get:
9e51c0a6 701 esw_cleanup_dests(esw, attr);
e52c2802 702err_create_goto_table:
aa0cbbae 703 return rule;
3d80d1a2
OG		 704}
705
e4ad91f2
CM		 706struct mlx5_flow_handle *
707mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
708 struct mlx5_flow_spec *spec,
c620b772 709 struct mlx5_flow_attr *attr)
e4ad91f2
CM		 710{
711 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {};
42f7ad67 712 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
c620b772 713 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
ae430332 714 struct mlx5_fs_chains *chains = esw_chains(esw);
c620b772 715 struct mlx5_vport_tbl_attr fwd_attr;
e52c2802
PB		 716 struct mlx5_flow_table *fast_fdb;
717 struct mlx5_flow_table *fwd_fdb;
e4ad91f2 718 struct mlx5_flow_handle *rule;
10742efc 719 int i, err = 0;
e4ad91f2 720
ae430332 721 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0);
e52c2802
PB		 722 if (IS_ERR(fast_fdb)) {
723 rule = ERR_CAST(fast_fdb);
724 goto err_get_fast;
725 }
726
c620b772
AL		 727 fwd_attr.chain = attr->chain;
728 fwd_attr.prio = attr->prio;
729 fwd_attr.vport = esw_attr->in_rep->vport;
730 fwd_fdb = esw_vport_tbl_get(esw, &fwd_attr);
e52c2802
PB		 731 if (IS_ERR(fwd_fdb)) {
732 rule = ERR_CAST(fwd_fdb);
733 goto err_get_fwd;
734 }
735
e4ad91f2 736 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
10742efc 737 for (i = 0; i < esw_attr->split_count; i++) {
a508728a
VB		 738 if (esw_is_indir_table(esw, attr))
739 err = esw_setup_indir_table(dest, &flow_act, esw, attr, spec, false, &i);
740 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
10742efc
VB		 741 err = esw_setup_chain_src_port_rewrite(dest, &flow_act, esw, chains, attr,
742 &i);
743 else
744 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false);
745
746 if (err) {
747 rule = ERR_PTR(err);
748 goto err_chain_src_rewrite;
749 }
750 }
e4ad91f2 751 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
873d2f12 752 dest[i].ft = fwd_fdb;
e4ad91f2
CM		 753 i++;
754
a508728a 755 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
b055ecf5
MB		 756 esw_attr->in_mdev->priv.eswitch,
757 esw_attr->in_rep->vport);
e4ad91f2 758
93b3586e 759 if (attr->outer_match_level != MLX5_MATCH_NONE)
c01cfd0f 760 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
e4ad91f2 761
278d51f2 762 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
e52c2802 763 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
e4ad91f2 764
10742efc
VB		 765 if (IS_ERR(rule)) {
766 i = esw_attr->split_count;
767 goto err_chain_src_rewrite;
768 }
e4ad91f2 769
525e84be 770 atomic64_inc(&esw->offloads.num_flows);
e52c2802
PB		 771
772 return rule;
10742efc
VB		 773err_chain_src_rewrite:
774 esw_put_dest_tables_loop(esw, attr, 0, i);
c620b772 775 esw_vport_tbl_put(esw, &fwd_attr);
e52c2802 776err_get_fwd:
ae430332 777 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
e52c2802 778err_get_fast:
e4ad91f2
CM		 779 return rule;
780}
781
e52c2802
PB		 782static void
783__mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
784 struct mlx5_flow_handle *rule,
c620b772 785 struct mlx5_flow_attr *attr,
e52c2802
PB		 786 bool fwd_rule)
787{
c620b772 788 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
ae430332 789 struct mlx5_fs_chains *chains = esw_chains(esw);
c620b772
AL		 790 bool split = (esw_attr->split_count > 0);
791 struct mlx5_vport_tbl_attr fwd_attr;
10caabda 792 int i;
e52c2802
PB		 793
794 mlx5_del_flow_rules(rule);
10caabda 795
84be2fda 796 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)) {
d8a2034f
EC		 797 /* unref the term table */
798 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
c620b772
AL		 799 if (esw_attr->dests[i].termtbl)
800 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl);
d8a2034f 801 }
10caabda
OS		 802 }
803
525e84be 804 atomic64_dec(&esw->offloads.num_flows);
e52c2802 805
c620b772
AL		 806 if (fwd_rule || split) {
807 fwd_attr.chain = attr->chain;
808 fwd_attr.prio = attr->prio;
809 fwd_attr.vport = esw_attr->in_rep->vport;
810 }
811
e52c2802 812 if (fwd_rule) {
c620b772 813 esw_vport_tbl_put(esw, &fwd_attr);
ae430332 814 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
10742efc 815 esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count);
e52c2802 816 } else {
96e32687 817 if (split)
c620b772 818 esw_vport_tbl_put(esw, &fwd_attr);
d18296ff 819 else if (attr->chain || attr->prio)
ae430332 820 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
9e51c0a6 821 esw_cleanup_dests(esw, attr);
e52c2802
PB		 822 }
823}
824
d85cdccb
OG		 825void
826mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
827 struct mlx5_flow_handle *rule,
c620b772 828 struct mlx5_flow_attr *attr)
d85cdccb 829{
e52c2802 830 __mlx5_eswitch_del_rule(esw, rule, attr, false);
d85cdccb
OG		 831}
832
48265006
OG		 833void
834mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
835 struct mlx5_flow_handle *rule,
c620b772 836 struct mlx5_flow_attr *attr)
48265006 837{
e52c2802 838 __mlx5_eswitch_del_rule(esw, rule, attr, true);
48265006
OG		 839}
840
f5f82476
OG		 841static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val)
842{
843 struct mlx5_eswitch_rep *rep;
411ec9e0 844 int i, err = 0;
f5f82476
OG		 845
846 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none");
411ec9e0 847 mlx5_esw_for_each_host_func_rep(esw, i, rep, esw->esw_funcs.num_vfs) {
8693115a 848 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
f5f82476
OG		 849 continue;
850
851 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val);
852 if (err)
853 goto out;
854 }
855
856out:
857 return err;
858}
859
860static struct mlx5_eswitch_rep *
861esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop)
862{
863 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL;
864
865 in_rep = attr->in_rep;
df65a573 866 out_rep = attr->dests[0].rep;
f5f82476
OG		 867
868 if (push)
869 vport = in_rep;
870 else if (pop)
871 vport = out_rep;
872 else
873 vport = in_rep;
874
875 return vport;
876}
877
878static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr,
879 bool push, bool pop, bool fwd)
880{
881 struct mlx5_eswitch_rep *in_rep, *out_rep;
882
883 if ((push || pop) && !fwd)
884 goto out_notsupp;
885
886 in_rep = attr->in_rep;
df65a573 887 out_rep = attr->dests[0].rep;
f5f82476 888
b05af6aa 889 if (push && in_rep->vport == MLX5_VPORT_UPLINK)
f5f82476
OG		 890 goto out_notsupp;
891
b05af6aa 892 if (pop && out_rep->vport == MLX5_VPORT_UPLINK)
f5f82476
OG		 893 goto out_notsupp;
894
895 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */
896 if (!push && !pop && fwd)
b05af6aa 897 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK)
f5f82476
OG		 898 goto out_notsupp;
899
900 /* protects against (1) setting rules with different vlans to push and
901 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0)
902 */
1482bd3d 903 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0]))
f5f82476
OG		 904 goto out_notsupp;
905
906 return 0;
907
908out_notsupp:
9eb78923 909 return -EOPNOTSUPP;
f5f82476
OG		 910}
911
912int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
c620b772 913 struct mlx5_flow_attr *attr)
f5f82476
OG		 914{
915 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
c620b772 916 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
f5f82476
OG		 917 struct mlx5_eswitch_rep *vport = NULL;
918 bool push, pop, fwd;
919 int err = 0;
920
6acfbf38 921 /* nop if we're on the vlan push/pop non emulation mode */
cc495188 922 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
6acfbf38
OG		 923 return 0;
924
f5f82476
OG		 925 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
926 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
e52c2802
PB		 927 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) &&
928 !attr->dest_chain);
f5f82476 929
0e18134f
VB		 930 mutex_lock(&esw->state_lock);
931
c620b772 932 err = esw_add_vlan_action_check(esw_attr, push, pop, fwd);
f5f82476 933 if (err)
0e18134f 934 goto unlock;
f5f82476 935
39ac237c 936 attr->flags &= ~MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
f5f82476 937
c620b772 938 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
f5f82476
OG		 939
940 if (!push && !pop && fwd) {
941 /* tracks VF --> wire rules without vlan push action */
c620b772 942 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) {
f5f82476 943 vport->vlan_refcount++;
39ac237c 944 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
f5f82476
OG		 945 }
946
0e18134f 947 goto unlock;
f5f82476
OG		 948 }
949
950 if (!push && !pop)
0e18134f 951 goto unlock;
f5f82476
OG		 952
953 if (!(offloads->vlan_push_pop_refcount)) {
954 /* it's the 1st vlan rule, apply global vlan pop policy */
955 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP);
956 if (err)
957 goto out;
958 }
959 offloads->vlan_push_pop_refcount++;
960
961 if (push) {
962 if (vport->vlan_refcount)
963 goto skip_set_push;
964
c620b772
AL		 965 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0],
966 0, SET_VLAN_INSERT | SET_VLAN_STRIP);
f5f82476
OG		 967 if (err)
968 goto out;
c620b772 969 vport->vlan = esw_attr->vlan_vid[0];
f5f82476
OG		 970skip_set_push:
971 vport->vlan_refcount++;
972 }
973out:
974 if (!err)
39ac237c 975 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
0e18134f
VB		 976unlock:
977 mutex_unlock(&esw->state_lock);
f5f82476
OG		 978 return err;
979}
980
981int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw,
c620b772 982 struct mlx5_flow_attr *attr)
f5f82476
OG		 983{
984 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
c620b772 985 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
f5f82476
OG		 986 struct mlx5_eswitch_rep *vport = NULL;
987 bool push, pop, fwd;
988 int err = 0;
989
6acfbf38 990 /* nop if we're on the vlan push/pop non emulation mode */
cc495188 991 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
6acfbf38
OG		 992 return 0;
993
39ac237c 994 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_VLAN_HANDLED))
f5f82476
OG		 995 return 0;
996
997 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
998 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
999 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST);
1000
0e18134f
VB		 1001 mutex_lock(&esw->state_lock);
1002
c620b772 1003 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
f5f82476
OG		 1004
1005 if (!push && !pop && fwd) {
1006 /* tracks VF --> wire rules without vlan push action */
c620b772 1007 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK)
f5f82476
OG		 1008 vport->vlan_refcount--;
1009
0e18134f 1010 goto out;
f5f82476
OG		 1011 }
1012
1013 if (push) {
1014 vport->vlan_refcount--;
1015 if (vport->vlan_refcount)
1016 goto skip_unset_push;
1017
1018 vport->vlan = 0;
1019 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport,
1020 0, 0, SET_VLAN_STRIP);
1021 if (err)
1022 goto out;
1023 }
1024
1025skip_unset_push:
1026 offloads->vlan_push_pop_refcount--;
1027 if (offloads->vlan_push_pop_refcount)
0e18134f 1028 goto out;
f5f82476
OG		 1029
1030 /* no more vlan rules, stop global vlan pop policy */
1031 err = esw_set_global_vlan_pop(esw, 0);
1032
1033out:
0e18134f 1034 mutex_unlock(&esw->state_lock);
f5f82476
OG		 1035 return err;
1036}
1037
f7a68945 1038struct mlx5_flow_handle *
02f3afd9
PP		 1039mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, u16 vport,
1040 u32 sqn)
ab22be9b 1041{
66958ed9 1042 struct mlx5_flow_act flow_act = {0};
4c5009c5 1043 struct mlx5_flow_destination dest = {};
74491de9 1044 struct mlx5_flow_handle *flow_rule;
c5bb1730 1045 struct mlx5_flow_spec *spec;
ab22be9b
OG		 1046 void *misc;
1047
1b9a07ee 1048 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
c5bb1730 1049 if (!spec) {
ab22be9b
OG		 1050 flow_rule = ERR_PTR(-ENOMEM);
1051 goto out;
1052 }
1053
c5bb1730 1054 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
ab22be9b 1055 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
a1b3839a
BW		 1056 /* source vport is the esw manager */
1057 MLX5_SET(fte_match_set_misc, misc, source_port, esw->manager_vport);
ab22be9b 1058
c5bb1730 1059 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
ab22be9b
OG		 1060 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
1061 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1062
c5bb1730 1063 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
ab22be9b 1064 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
b17f7fc1 1065 dest.vport.num = vport;
66958ed9 1066 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
ab22be9b 1067
39ac237c
PB		 1068 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1069 spec, &flow_act, &dest, 1);
ab22be9b
OG		 1070 if (IS_ERR(flow_rule))
1071 esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule));
1072out:
c5bb1730 1073 kvfree(spec);
ab22be9b
OG		 1074 return flow_rule;
1075}
57cbd893 1076EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
ab22be9b 1077
159fe639
MB		 1078void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1079{
1080 mlx5_del_flow_rules(rule);
1081}
1082
5b7cb745
PB		 1083static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1084{
1085 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1086 MLX5_FDB_TO_VPORT_REG_C_1;
1087}
1088
332bd3a5 1089static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
c1286050
JL		 1090{
1091 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
e08a6832
LR		 1092 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1093 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
5b7cb745 1094 u8 curr, wanted;
c1286050
JL		 1095 int err;
1096
5b7cb745
PB		 1097 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1098 !mlx5_eswitch_vport_match_metadata_enabled(esw))
332bd3a5 1099 return 0;
c1286050 1100
e08a6832
LR		 1101 MLX5_SET(query_esw_vport_context_in, in, opcode,
1102 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1103 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
c1286050
JL		 1104 if (err)
1105 return err;
1106
5b7cb745
PB		 1107 curr = MLX5_GET(query_esw_vport_context_out, out,
1108 esw_vport_context.fdb_to_vport_reg_c_id);
1109 wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1110 if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1111 wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
c1286050 1112
332bd3a5 1113 if (enable)
5b7cb745 1114 curr |= wanted;
332bd3a5 1115 else
5b7cb745 1116 curr &= ~wanted;
c1286050 1117
e08a6832 1118 MLX5_SET(modify_esw_vport_context_in, min,
5b7cb745 1119 esw_vport_context.fdb_to_vport_reg_c_id, curr);
e08a6832 1120 MLX5_SET(modify_esw_vport_context_in, min,
c1286050
JL		 1121 field_select.fdb_to_vport_reg_c_id, 1);
1122
e08a6832 1123 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min);
5b7cb745
PB		 1124 if (!err) {
1125 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1126 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1127 else
1128 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1129 }
1130
1131 return err;
c1286050
JL		 1132}
1133
a5641cb5
JL		 1134static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1135 struct mlx5_core_dev *peer_dev,
ac004b83
RD		 1136 struct mlx5_flow_spec *spec,
1137 struct mlx5_flow_destination *dest)
1138{
a5641cb5 1139 void *misc;
ac004b83 1140
a5641cb5
JL		 1141 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1142 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1143 misc_parameters_2);
0f0d3827
PB		 1144 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1145 mlx5_eswitch_get_vport_metadata_mask());
ac004b83 1146
a5641cb5
JL		 1147 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1148 } else {
1149 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1150 misc_parameters);
ac004b83 1151
a5641cb5
JL		 1152 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1153 MLX5_CAP_GEN(peer_dev, vhca_id));
1154
1155 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1156
1157 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1158 misc_parameters);
1159 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1160 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1161 source_eswitch_owner_vhca_id);
1162 }
ac004b83
RD		 1163
1164 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
a1b3839a 1165 dest->vport.num = peer_dev->priv.eswitch->manager_vport;
ac004b83 1166 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
04de7dda 1167 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
ac004b83
RD		 1168}
1169
a5641cb5
JL		 1170static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1171 struct mlx5_eswitch *peer_esw,
1172 struct mlx5_flow_spec *spec,
1173 u16 vport)
1174{
1175 void *misc;
1176
1177 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1178 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1179 misc_parameters_2);
1180 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1181 mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1182 vport));
1183 } else {
1184 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1185 misc_parameters);
1186 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1187 }
1188}
1189
ac004b83
RD		 1190static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1191 struct mlx5_core_dev *peer_dev)
1192{
1193 struct mlx5_flow_destination dest = {};
1194 struct mlx5_flow_act flow_act = {0};
1195 struct mlx5_flow_handle **flows;
1196 struct mlx5_flow_handle *flow;
1197 struct mlx5_flow_spec *spec;
1198 /* total vports is the same for both e-switches */
1199 int nvports = esw->total_vports;
1200 void *misc;
1201 int err, i;
1202
1203 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1204 if (!spec)
1205 return -ENOMEM;
1206
a5641cb5 1207 peer_miss_rules_setup(esw, peer_dev, spec, &dest);
ac004b83
RD		 1208
1209 flows = kvzalloc(nvports * sizeof(*flows), GFP_KERNEL);
1210 if (!flows) {
1211 err = -ENOMEM;
1212 goto alloc_flows_err;
1213 }
1214
1215 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1216 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1217 misc_parameters);
1218
81cd229c 1219 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
a5641cb5
JL		 1220 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1221 spec, MLX5_VPORT_PF);
1222
81cd229c
BW		 1223 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1224 spec, &flow_act, &dest, 1);
1225 if (IS_ERR(flow)) {
1226 err = PTR_ERR(flow);
1227 goto add_pf_flow_err;
1228 }
1229 flows[MLX5_VPORT_PF] = flow;
1230 }
1231
1232 if (mlx5_ecpf_vport_exists(esw->dev)) {
1233 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1234 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1235 spec, &flow_act, &dest, 1);
1236 if (IS_ERR(flow)) {
1237 err = PTR_ERR(flow);
1238 goto add_ecpf_flow_err;
1239 }
1240 flows[mlx5_eswitch_ecpf_idx(esw)] = flow;
1241 }
1242
786ef904 1243 mlx5_esw_for_each_vf_vport_num(esw, i, mlx5_core_max_vfs(esw->dev)) {
a5641cb5
JL		 1244 esw_set_peer_miss_rule_source_port(esw,
1245 peer_dev->priv.eswitch,
1246 spec, i);
1247
ac004b83
RD		 1248 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1249 spec, &flow_act, &dest, 1);
1250 if (IS_ERR(flow)) {
1251 err = PTR_ERR(flow);
81cd229c 1252 goto add_vf_flow_err;
ac004b83
RD		 1253 }
1254 flows[i] = flow;
1255 }
1256
1257 esw->fdb_table.offloads.peer_miss_rules = flows;
1258
1259 kvfree(spec);
1260 return 0;
1261
81cd229c 1262add_vf_flow_err:
879c8f84 1263 nvports = --i;
786ef904 1264 mlx5_esw_for_each_vf_vport_num_reverse(esw, i, nvports)
ac004b83 1265 mlx5_del_flow_rules(flows[i]);
81cd229c
BW		 1266
1267 if (mlx5_ecpf_vport_exists(esw->dev))
1268 mlx5_del_flow_rules(flows[mlx5_eswitch_ecpf_idx(esw)]);
1269add_ecpf_flow_err:
1270 if (mlx5_core_is_ecpf_esw_manager(esw->dev))
1271 mlx5_del_flow_rules(flows[MLX5_VPORT_PF]);
1272add_pf_flow_err:
1273 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
ac004b83
RD		 1274 kvfree(flows);
1275alloc_flows_err:
1276 kvfree(spec);
1277 return err;
1278}
1279
1280static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw)
1281{
1282 struct mlx5_flow_handle **flows;
1283 int i;
1284
1285 flows = esw->fdb_table.offloads.peer_miss_rules;
1286
786ef904
PP		 1287 mlx5_esw_for_each_vf_vport_num_reverse(esw, i,
1288 mlx5_core_max_vfs(esw->dev))
ac004b83
RD		 1289 mlx5_del_flow_rules(flows[i]);
1290
81cd229c
BW		 1291 if (mlx5_ecpf_vport_exists(esw->dev))
1292 mlx5_del_flow_rules(flows[mlx5_eswitch_ecpf_idx(esw)]);
1293
1294 if (mlx5_core_is_ecpf_esw_manager(esw->dev))
1295 mlx5_del_flow_rules(flows[MLX5_VPORT_PF]);
1296
ac004b83
RD		 1297 kvfree(flows);
1298}
1299
3aa33572
OG		 1300static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1301{
66958ed9 1302 struct mlx5_flow_act flow_act = {0};
4c5009c5 1303 struct mlx5_flow_destination dest = {};
74491de9 1304 struct mlx5_flow_handle *flow_rule = NULL;
c5bb1730 1305 struct mlx5_flow_spec *spec;
f80be543
MB		 1306 void *headers_c;
1307 void *headers_v;
3aa33572 1308 int err = 0;
f80be543
MB		 1309 u8 *dmac_c;
1310 u8 *dmac_v;
3aa33572 1311
1b9a07ee 1312 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
c5bb1730 1313 if (!spec) {
3aa33572
OG		 1314 err = -ENOMEM;
1315 goto out;
1316 }
1317
f80be543
MB		 1318 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1319 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1320 outer_headers);
1321 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1322 outer_headers.dmac_47_16);
1323 dmac_c[0] = 0x01;
1324
3aa33572 1325 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
a1b3839a 1326 dest.vport.num = esw->manager_vport;
66958ed9 1327 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
3aa33572 1328
39ac237c
PB		 1329 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1330 spec, &flow_act, &dest, 1);
3aa33572
OG		 1331 if (IS_ERR(flow_rule)) {
1332 err = PTR_ERR(flow_rule);
f80be543 1333 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
3aa33572
OG		 1334 goto out;
1335 }
1336
f80be543
MB		 1337 esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1338
1339 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1340 outer_headers);
1341 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1342 outer_headers.dmac_47_16);
1343 dmac_v[0] = 0x01;
39ac237c
PB		 1344 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1345 spec, &flow_act, &dest, 1);
f80be543
MB		 1346 if (IS_ERR(flow_rule)) {
1347 err = PTR_ERR(flow_rule);
1348 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1349 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1350 goto out;
1351 }
1352
1353 esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1354
3aa33572 1355out:
c5bb1730 1356 kvfree(spec);
3aa33572
OG		 1357 return err;
1358}
1359
11b717d6
PB		 1360struct mlx5_flow_handle *
1361esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1362{
1363 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1364 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1365 struct mlx5_flow_context *flow_context;
1366 struct mlx5_flow_handle *flow_rule;
1367 struct mlx5_flow_destination dest;
1368 struct mlx5_flow_spec *spec;
1369 void *misc;
1370
60acc105
PB		 1371 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1372 return ERR_PTR(-EOPNOTSUPP);
1373
11b717d6
PB		 1374 spec = kzalloc(sizeof(*spec), GFP_KERNEL);
1375 if (!spec)
1376 return ERR_PTR(-ENOMEM);
1377
1378 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1379 misc_parameters_2);
1380 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1381 ESW_CHAIN_TAG_METADATA_MASK);
1382 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1383 misc_parameters_2);
1384 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1385 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
6724e66b
PB		 1386 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1387 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1388 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
11b717d6
PB		 1389
1390 flow_context = &spec->flow_context;
1391 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1392 flow_context->flow_tag = tag;
1393 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1394 dest.ft = esw->offloads.ft_offloads;
1395
1396 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1397 kfree(spec);
1398
1399 if (IS_ERR(flow_rule))
1400 esw_warn(esw->dev,
1401 "Failed to create restore rule for tag: %d, err(%d)\n",
1402 tag, (int)PTR_ERR(flow_rule));
1403
1404 return flow_rule;
1405}
1406
1407u32
1408esw_get_max_restore_tag(struct mlx5_eswitch *esw)
1409{
1410 return ESW_CHAIN_TAG_METADATA_MASK;
1411}
1412
1967ce6e 1413#define MAX_PF_SQ 256
cd3d07e7 1414#define MAX_SQ_NVPORTS 32
1967ce6e 1415
a5641cb5
JL		 1416static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1417 u32 *flow_group_in)
1418{
1419 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1420 flow_group_in,
1421 match_criteria);
1422
1423 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1424 MLX5_SET(create_flow_group_in, flow_group_in,
1425 match_criteria_enable,
1426 MLX5_MATCH_MISC_PARAMETERS_2);
1427
0f0d3827
PB		 1428 MLX5_SET(fte_match_param, match_criteria,
1429 misc_parameters_2.metadata_reg_c_0,
1430 mlx5_eswitch_get_vport_metadata_mask());
a5641cb5
JL		 1431 } else {
1432 MLX5_SET(create_flow_group_in, flow_group_in,
1433 match_criteria_enable,
1434 MLX5_MATCH_MISC_PARAMETERS);
1435
1436 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1437 misc_parameters.source_port);
1438 }
1439}
1440
ae430332
AL		 1441#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
1442#define fdb_modify_header_fwd_to_table_supported(esw) \
1443 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
1444static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1445{
1446 struct mlx5_core_dev *dev = esw->dev;
1447
1448 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1449 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1450
1451 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1452 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1453 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1454 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1455 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1456 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1457 esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1458 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1459 /* Disabled when ttl workaround is needed, e.g
1460 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1461 */
1462 esw_warn(dev,
1463 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1464 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1465 } else {
1466 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1467 esw_info(dev, "Supported tc chains and prios offload\n");
1468 }
1469
1470 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1471 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1472}
1473
1474static int
1475esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1476{
1477 struct mlx5_core_dev *dev = esw->dev;
1478 struct mlx5_flow_table *nf_ft, *ft;
1479 struct mlx5_chains_attr attr = {};
1480 struct mlx5_fs_chains *chains;
1481 u32 fdb_max;
1482 int err;
1483
1484 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
1485
1486 esw_init_chains_offload_flags(esw, &attr.flags);
1487 attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1488 attr.max_ft_sz = fdb_max;
1489 attr.max_grp_num = esw->params.large_group_num;
1490 attr.default_ft = miss_fdb;
1491 attr.max_restore_tag = esw_get_max_restore_tag(esw);
1492
1493 chains = mlx5_chains_create(dev, &attr);
1494 if (IS_ERR(chains)) {
1495 err = PTR_ERR(chains);
1496 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1497 return err;
1498 }
1499
1500 esw->fdb_table.offloads.esw_chains_priv = chains;
1501
1502 /* Create tc_end_ft which is the always created ft chain */
1503 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains),
1504 1, 0);
1505 if (IS_ERR(nf_ft)) {
1506 err = PTR_ERR(nf_ft);
1507 goto nf_ft_err;
1508 }
1509
1510 /* Always open the root for fast path */
1511 ft = mlx5_chains_get_table(chains, 0, 1, 0);
1512 if (IS_ERR(ft)) {
1513 err = PTR_ERR(ft);
1514 goto level_0_err;
1515 }
1516
1517 /* Open level 1 for split fdb rules now if prios isn't supported */
1518 if (!mlx5_chains_prios_supported(chains)) {
1519 err = mlx5_esw_vport_tbl_get(esw);
1520 if (err)
1521 goto level_1_err;
1522 }
1523
1524 mlx5_chains_set_end_ft(chains, nf_ft);
1525
1526 return 0;
1527
1528level_1_err:
1529 mlx5_chains_put_table(chains, 0, 1, 0);
1530level_0_err:
1531 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1532nf_ft_err:
1533 mlx5_chains_destroy(chains);
1534 esw->fdb_table.offloads.esw_chains_priv = NULL;
1535
1536 return err;
1537}
1538
1539static void
1540esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1541{
1542 if (!mlx5_chains_prios_supported(chains))
1543 mlx5_esw_vport_tbl_put(esw);
1544 mlx5_chains_put_table(chains, 0, 1, 0);
1545 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1546 mlx5_chains_destroy(chains);
1547}
1548
1549#else /* CONFIG_MLX5_CLS_ACT */
1550
1551static int
1552esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1553{ return 0; }
1554
1555static void
1556esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1557{}
1558
1559#endif
1560
0da3c12d 1561static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1967ce6e
OG		 1562{
1563 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1564 struct mlx5_flow_table_attr ft_attr = {};
1565 struct mlx5_core_dev *dev = esw->dev;
1566 struct mlx5_flow_namespace *root_ns;
1567 struct mlx5_flow_table *fdb = NULL;
39ac237c
PB		 1568 u32 flags = 0, *flow_group_in;
1569 int table_size, ix, err = 0;
1967ce6e
OG		 1570 struct mlx5_flow_group *g;
1571 void *match_criteria;
f80be543 1572 u8 *dmac;
1967ce6e
OG		 1573
1574 esw_debug(esw->dev, "Create offloads FDB Tables\n");
39ac237c 1575
1b9a07ee 1576 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1967ce6e
OG		 1577 if (!flow_group_in)
1578 return -ENOMEM;
1579
1580 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
1581 if (!root_ns) {
1582 esw_warn(dev, "Failed to get FDB flow namespace\n");
1583 err = -EOPNOTSUPP;
1584 goto ns_err;
1585 }
8463daf1
MG		 1586 esw->fdb_table.offloads.ns = root_ns;
1587 err = mlx5_flow_namespace_set_mode(root_ns,
1588 esw->dev->priv.steering->mode);
1589 if (err) {
1590 esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1591 goto ns_err;
1592 }
1967ce6e 1593
0da3c12d 1594 table_size = esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
cd7e4186 1595 MLX5_ESW_MISS_FLOWS + esw->total_vports;
b3ba5149 1596
e52c2802
PB		 1597 /* create the slow path fdb with encap set, so further table instances
1598 * can be created at run time while VFs are probed if the FW allows that.
1599 */
1600 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1601 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1602 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1603
1604 ft_attr.flags = flags;
b3ba5149
ES		 1605 ft_attr.max_fte = table_size;
1606 ft_attr.prio = FDB_SLOW_PATH;
1607
1608 fdb = mlx5_create_flow_table(root_ns, &ft_attr);
1033665e
OG		 1609 if (IS_ERR(fdb)) {
1610 err = PTR_ERR(fdb);
1611 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1612 goto slow_fdb_err;
1613 }
52fff327 1614 esw->fdb_table.offloads.slow_fdb = fdb;
1033665e 1615
ae430332 1616 err = esw_chains_create(esw, fdb);
39ac237c 1617 if (err) {
ae430332 1618 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
39ac237c 1619 goto fdb_chains_err;
e52c2802
PB		 1620 }
1621
69697b6e 1622 /* create send-to-vport group */
69697b6e
OG		 1623 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1624 MLX5_MATCH_MISC_PARAMETERS);
1625
1626 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1627
1628 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1629 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port);
1630
0da3c12d 1631 ix = esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ;
69697b6e
OG		 1632 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1633 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1);
1634
1635 g = mlx5_create_flow_group(fdb, flow_group_in);
1636 if (IS_ERR(g)) {
1637 err = PTR_ERR(g);
1638 esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1639 goto send_vport_err;
1640 }
1641 esw->fdb_table.offloads.send_to_vport_grp = g;
1642
6cec0229
MD		 1643 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1644 /* create peer esw miss group */
1645 memset(flow_group_in, 0, inlen);
ac004b83 1646
6cec0229 1647 esw_set_flow_group_source_port(esw, flow_group_in);
a5641cb5 1648
6cec0229
MD		 1649 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1650 match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1651 flow_group_in,
1652 match_criteria);
ac004b83 1653
6cec0229
MD		 1654 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1655 misc_parameters.source_eswitch_owner_vhca_id);
a5641cb5 1656
6cec0229
MD		 1657 MLX5_SET(create_flow_group_in, flow_group_in,
1658 source_eswitch_owner_vhca_id_valid, 1);
1659 }
ac004b83 1660
6cec0229
MD		 1661 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
1662 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1663 ix + esw->total_vports - 1);
1664 ix += esw->total_vports;
ac004b83 1665
6cec0229
MD		 1666 g = mlx5_create_flow_group(fdb, flow_group_in);
1667 if (IS_ERR(g)) {
1668 err = PTR_ERR(g);
1669 esw_warn(dev, "Failed to create peer miss flow group err(%d)\n", err);
1670 goto peer_miss_err;
1671 }
1672 esw->fdb_table.offloads.peer_miss_grp = g;
ac004b83 1673 }
ac004b83 1674
69697b6e
OG		 1675 /* create miss group */
1676 memset(flow_group_in, 0, inlen);
f80be543
MB		 1677 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1678 MLX5_MATCH_OUTER_HEADERS);
1679 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1680 match_criteria);
1681 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1682 outer_headers.dmac_47_16);
1683 dmac[0] = 0x01;
69697b6e
OG		 1684
1685 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
cd7e4186
BW		 1686 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1687 ix + MLX5_ESW_MISS_FLOWS);
69697b6e
OG		 1688
1689 g = mlx5_create_flow_group(fdb, flow_group_in);
1690 if (IS_ERR(g)) {
1691 err = PTR_ERR(g);
1692 esw_warn(dev, "Failed to create miss flow group err(%d)\n", err);
1693 goto miss_err;
1694 }
1695 esw->fdb_table.offloads.miss_grp = g;
1696
3aa33572
OG		 1697 err = esw_add_fdb_miss_rule(esw);
1698 if (err)
1699 goto miss_rule_err;
1700
c88a026e 1701 kvfree(flow_group_in);
69697b6e
OG		 1702 return 0;
1703
3aa33572
OG		 1704miss_rule_err:
1705 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
69697b6e 1706miss_err:
6cec0229
MD		 1707 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1708 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
ac004b83 1709peer_miss_err:
69697b6e
OG		 1710 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1711send_vport_err:
ae430332 1712 esw_chains_destroy(esw, esw_chains(esw));
39ac237c 1713fdb_chains_err:
52fff327 1714 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1033665e 1715slow_fdb_err:
8463daf1
MG		 1716 /* Holds true only as long as DMFS is the default */
1717 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS);
69697b6e
OG		 1718ns_err:
1719 kvfree(flow_group_in);
1720 return err;
1721}
1722
1967ce6e 1723static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
69697b6e 1724{
e52c2802 1725 if (!esw->fdb_table.offloads.slow_fdb)
69697b6e
OG		 1726 return;
1727
1967ce6e 1728 esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
f80be543
MB		 1729 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
1730 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
69697b6e 1731 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
6cec0229
MD		 1732 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1733 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
69697b6e
OG		 1734 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1735
ae430332
AL		 1736 esw_chains_destroy(esw, esw_chains(esw));
1737
52fff327 1738 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
8463daf1
MG		 1739 /* Holds true only as long as DMFS is the default */
1740 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1741 MLX5_FLOW_STEERING_MODE_DMFS);
69697b6e 1742}
c116c6ee 1743
8d6bd3c3 1744static int esw_create_offloads_table(struct mlx5_eswitch *esw)
c116c6ee 1745{
b3ba5149 1746 struct mlx5_flow_table_attr ft_attr = {};
c116c6ee 1747 struct mlx5_core_dev *dev = esw->dev;
b3ba5149
ES		 1748 struct mlx5_flow_table *ft_offloads;
1749 struct mlx5_flow_namespace *ns;
c116c6ee
OG		 1750 int err = 0;
1751
1752 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1753 if (!ns) {
1754 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
eff596da 1755 return -EOPNOTSUPP;
c116c6ee
OG		 1756 }
1757
8d6bd3c3 1758 ft_attr.max_fte = esw->total_vports + MLX5_ESW_MISS_FLOWS;
11b717d6 1759 ft_attr.prio = 1;
b3ba5149
ES		 1760
1761 ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
c116c6ee
OG		 1762 if (IS_ERR(ft_offloads)) {
1763 err = PTR_ERR(ft_offloads);
1764 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
1765 return err;
1766 }
1767
1768 esw->offloads.ft_offloads = ft_offloads;
1769 return 0;
1770}
1771
1772static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
1773{
1774 struct mlx5_esw_offload *offloads = &esw->offloads;
1775
1776 mlx5_destroy_flow_table(offloads->ft_offloads);
1777}
fed9ce22 1778
8d6bd3c3 1779static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
fed9ce22
OG		 1780{
1781 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1782 struct mlx5_flow_group *g;
fed9ce22 1783 u32 *flow_group_in;
8d6bd3c3 1784 int nvports;
fed9ce22 1785 int err = 0;
fed9ce22 1786
8d6bd3c3 1787 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1b9a07ee 1788 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
fed9ce22
OG		 1789 if (!flow_group_in)
1790 return -ENOMEM;
1791
1792 /* create vport rx group */
a5641cb5 1793 esw_set_flow_group_source_port(esw, flow_group_in);
fed9ce22
OG		 1794
1795 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1796 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
1797
1798 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
1799
1800 if (IS_ERR(g)) {
1801 err = PTR_ERR(g);
1802 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
1803 goto out;
1804 }
1805
1806 esw->offloads.vport_rx_group = g;
1807out:
e574978a 1808 kvfree(flow_group_in);
fed9ce22
OG		 1809 return err;
1810}
1811
1812static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
1813{
1814 mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
1815}
1816
74491de9 1817struct mlx5_flow_handle *
02f3afd9 1818mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
c966f7d5 1819 struct mlx5_flow_destination *dest)
fed9ce22 1820{
66958ed9 1821 struct mlx5_flow_act flow_act = {0};
74491de9 1822 struct mlx5_flow_handle *flow_rule;
c5bb1730 1823 struct mlx5_flow_spec *spec;
fed9ce22
OG		 1824 void *misc;
1825
1b9a07ee 1826 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
c5bb1730 1827 if (!spec) {
fed9ce22
OG		 1828 flow_rule = ERR_PTR(-ENOMEM);
1829 goto out;
1830 }
1831
a5641cb5
JL		 1832 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1833 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
1834 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1835 mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
fed9ce22 1836
a5641cb5 1837 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
0f0d3827
PB		 1838 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1839 mlx5_eswitch_get_vport_metadata_mask());
fed9ce22 1840
a5641cb5
JL		 1841 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1842 } else {
1843 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
1844 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1845
1846 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
1847 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1848
1849 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1850 }
fed9ce22 1851
66958ed9 1852 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
74491de9 1853 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec,
c966f7d5 1854 &flow_act, dest, 1);
fed9ce22
OG		 1855 if (IS_ERR(flow_rule)) {
1856 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
1857 goto out;
1858 }
1859
1860out:
c5bb1730 1861 kvfree(spec);
fed9ce22
OG		 1862 return flow_rule;
1863}
feae9087 1864
bf3347c4 1865
cc617ced
PP		 1866static int mlx5_eswitch_inline_mode_get(const struct mlx5_eswitch *esw, u8 *mode)
1867{
1868 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
1869 struct mlx5_core_dev *dev = esw->dev;
1870 int vport;
1871
1872 if (!MLX5_CAP_GEN(dev, vport_group_manager))
1873 return -EOPNOTSUPP;
1874
1875 if (esw->mode == MLX5_ESWITCH_NONE)
1876 return -EOPNOTSUPP;
1877
1878 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
1879 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
1880 mlx5_mode = MLX5_INLINE_MODE_NONE;
1881 goto out;
1882 case MLX5_CAP_INLINE_MODE_L2:
1883 mlx5_mode = MLX5_INLINE_MODE_L2;
1884 goto out;
1885 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
1886 goto query_vports;
1887 }
1888
1889query_vports:
1890 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode);
1891 mlx5_esw_for_each_host_func_vport(esw, vport, esw->esw_funcs.num_vfs) {
1892 mlx5_query_nic_vport_min_inline(dev, vport, &mlx5_mode);
1893 if (prev_mlx5_mode != mlx5_mode)
1894 return -EINVAL;
1895 prev_mlx5_mode = mlx5_mode;
1896 }
1897
1898out:
1899 *mode = mlx5_mode;
1900 return 0;
e08a6832 1901}
bf3347c4 1902
11b717d6
PB		 1903static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
1904{
1905 struct mlx5_esw_offload *offloads = &esw->offloads;
1906
60acc105
PB		 1907 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1908 return;
1909
6724e66b 1910 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id);
11b717d6
PB		 1911 mlx5_destroy_flow_group(offloads->restore_group);
1912 mlx5_destroy_flow_table(offloads->ft_offloads_restore);
1913}
1914
1915static int esw_create_restore_table(struct mlx5_eswitch *esw)
1916{
d65dbedf 1917 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
11b717d6
PB		 1918 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1919 struct mlx5_flow_table_attr ft_attr = {};
1920 struct mlx5_core_dev *dev = esw->dev;
1921 struct mlx5_flow_namespace *ns;
6724e66b 1922 struct mlx5_modify_hdr *mod_hdr;
11b717d6
PB		 1923 void *match_criteria, *misc;
1924 struct mlx5_flow_table *ft;
1925 struct mlx5_flow_group *g;
1926 u32 *flow_group_in;
1927 int err = 0;
1928
60acc105
PB		 1929 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1930 return 0;
1931
11b717d6
PB		 1932 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1933 if (!ns) {
1934 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
1935 return -EOPNOTSUPP;
1936 }
1937
1938 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1939 if (!flow_group_in) {
1940 err = -ENOMEM;
1941 goto out_free;
1942 }
1943
1944 ft_attr.max_fte = 1 << ESW_CHAIN_TAG_METADATA_BITS;
1945 ft = mlx5_create_flow_table(ns, &ft_attr);
1946 if (IS_ERR(ft)) {
1947 err = PTR_ERR(ft);
1948 esw_warn(esw->dev, "Failed to create restore table, err %d\n",
1949 err);
1950 goto out_free;
1951 }
1952
11b717d6
PB		 1953 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1954 match_criteria);
1955 misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
1956 misc_parameters_2);
1957
1958 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1959 ESW_CHAIN_TAG_METADATA_MASK);
1960 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1961 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1962 ft_attr.max_fte - 1);
1963 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1964 MLX5_MATCH_MISC_PARAMETERS_2);
1965 g = mlx5_create_flow_group(ft, flow_group_in);
1966 if (IS_ERR(g)) {
1967 err = PTR_ERR(g);
1968 esw_warn(dev, "Failed to create restore flow group, err: %d\n",
1969 err);
1970 goto err_group;
1971 }
1972
6724e66b
PB		 1973 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
1974 MLX5_SET(copy_action_in, modact, src_field,
1975 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
1976 MLX5_SET(copy_action_in, modact, dst_field,
1977 MLX5_ACTION_IN_FIELD_METADATA_REG_B);
1978 mod_hdr = mlx5_modify_header_alloc(esw->dev,
1979 MLX5_FLOW_NAMESPACE_KERNEL, 1,
1980 modact);
1981 if (IS_ERR(mod_hdr)) {
e9864539 1982 err = PTR_ERR(mod_hdr);
6724e66b
PB		 1983 esw_warn(dev, "Failed to create restore mod header, err: %d\n",
1984 err);
6724e66b
PB		 1985 goto err_mod_hdr;
1986 }
1987
11b717d6
PB		 1988 esw->offloads.ft_offloads_restore = ft;
1989 esw->offloads.restore_group = g;
6724e66b 1990 esw->offloads.restore_copy_hdr_id = mod_hdr;
11b717d6 1991
c8508713
RD		 1992 kvfree(flow_group_in);
1993
11b717d6
PB		 1994 return 0;
1995
6724e66b
PB		 1996err_mod_hdr:
1997 mlx5_destroy_flow_group(g);
11b717d6
PB		 1998err_group:
1999 mlx5_destroy_flow_table(ft);
2000out_free:
2001 kvfree(flow_group_in);
2002
2003 return err;
cc617ced
PP		 2004}
2005
db7ff19e
EB		 2006static int esw_offloads_start(struct mlx5_eswitch *esw,
2007 struct netlink_ext_ack *extack)
c930a3ad 2008{
062f4bf4 2009 int err, err1;
c930a3ad 2010
8e0aa4bc
PP		 2011 mlx5_eswitch_disable_locked(esw, false);
2012 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS,
2013 esw->dev->priv.sriov.num_vfs);
6c419ba8 2014 if (err) {
8c98ee77
EB		 2015 NL_SET_ERR_MSG_MOD(extack,
2016 "Failed setting eswitch to offloads");
8e0aa4bc
PP		 2017 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY,
2018 MLX5_ESWITCH_IGNORE_NUM_VFS);
8c98ee77
EB		 2019 if (err1) {
2020 NL_SET_ERR_MSG_MOD(extack,
2021 "Failed setting eswitch back to legacy");
2022 }
6c419ba8 2023 }
bffaa916
RD		 2024 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2025 if (mlx5_eswitch_inline_mode_get(esw,
bffaa916
RD		 2026 &esw->offloads.inline_mode)) {
2027 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
8c98ee77
EB		 2028 NL_SET_ERR_MSG_MOD(extack,
2029 "Inline mode is different between vports");
bffaa916
RD		 2030 }
2031 }
c930a3ad
OG		 2032 return err;
2033}
2034
e8d31c4d
MB		 2035void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2036{
2037 kfree(esw->offloads.vport_reps);
2038}
2039
2040int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2041{
2752b823 2042 int total_vports = esw->total_vports;
e8d31c4d 2043 struct mlx5_eswitch_rep *rep;
d6518db2 2044 int vport_index;
ef2e4094 2045 u8 rep_type;
e8d31c4d 2046
2aca1787 2047 esw->offloads.vport_reps = kcalloc(total_vports,
e8d31c4d
MB		 2048 sizeof(struct mlx5_eswitch_rep),
2049 GFP_KERNEL);
2050 if (!esw->offloads.vport_reps)
2051 return -ENOMEM;
2052
d6518db2
BW		 2053 mlx5_esw_for_all_reps(esw, vport_index, rep) {
2054 rep->vport = mlx5_eswitch_index_to_vport_num(esw, vport_index);
2f69e591 2055 rep->vport_index = vport_index;
f121e0ea
BW		 2056
2057 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
8693115a 2058 atomic_set(&rep->rep_data[rep_type].state,
6f4e0219 2059 REP_UNREGISTERED);
e8d31c4d
MB		 2060 }
2061
e8d31c4d
MB		 2062 return 0;
2063}
2064
c9b99abc
BW		 2065static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2066 struct mlx5_eswitch_rep *rep, u8 rep_type)
2067{
8693115a 2068 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
6f4e0219 2069 REP_LOADED, REP_REGISTERED) == REP_LOADED)
8693115a 2070 esw->offloads.rep_ops[rep_type]->unload(rep);
c9b99abc
BW		 2071}
2072
d7f33a45
VP		 2073static void __unload_reps_sf_vport(struct mlx5_eswitch *esw, u8 rep_type)
2074{
2075 struct mlx5_eswitch_rep *rep;
2076 int i;
2077
2078 mlx5_esw_for_each_sf_rep(esw, i, rep)
2079 __esw_offloads_unload_rep(esw, rep, rep_type);
2080}
2081
4110fc59 2082static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
6ed1803a
MB		 2083{
2084 struct mlx5_eswitch_rep *rep;
4110fc59
BW		 2085 int i;
2086
d7f33a45
VP		 2087 __unload_reps_sf_vport(esw, rep_type);
2088
4110fc59
BW		 2089 mlx5_esw_for_each_vf_rep_reverse(esw, i, rep, esw->esw_funcs.num_vfs)
2090 __esw_offloads_unload_rep(esw, rep, rep_type);
c9b99abc 2091
81cd229c
BW		 2092 if (mlx5_ecpf_vport_exists(esw->dev)) {
2093 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF);
2094 __esw_offloads_unload_rep(esw, rep, rep_type);
2095 }
2096
2097 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
2098 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF);
2099 __esw_offloads_unload_rep(esw, rep, rep_type);
2100 }
2101
879c8f84 2102 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
c9b99abc 2103 __esw_offloads_unload_rep(esw, rep, rep_type);
6ed1803a
MB		 2104}
2105
d970812b 2106int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
a4b97ab4 2107{
c2d7712c
BW		 2108 struct mlx5_eswitch_rep *rep;
2109 int rep_type;
a4b97ab4
MB		 2110 int err;
2111
c2d7712c
BW		 2112 rep = mlx5_eswitch_get_rep(esw, vport_num);
2113 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2114 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2115 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) {
2116 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2117 if (err)
2118 goto err_reps;
2119 }
2120
2121 return 0;
a4b97ab4
MB		 2122
2123err_reps:
c2d7712c
BW		 2124 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2125 for (--rep_type; rep_type >= 0; rep_type--)
2126 __esw_offloads_unload_rep(esw, rep, rep_type);
6ed1803a
MB		 2127 return err;
2128}
2129
d970812b 2130void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
c2d7712c
BW		 2131{
2132 struct mlx5_eswitch_rep *rep;
2133 int rep_type;
2134
c2d7712c
BW		 2135 rep = mlx5_eswitch_get_rep(esw, vport_num);
2136 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2137 __esw_offloads_unload_rep(esw, rep, rep_type);
2138}
2139
38679b5a
PP		 2140int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num)
2141{
2142 int err;
2143
2144 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2145 return 0;
2146
c7eddc60
PP		 2147 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num);
2148 if (err)
2149 return err;
2150
38679b5a 2151 err = mlx5_esw_offloads_rep_load(esw, vport_num);
c7eddc60
PP		 2152 if (err)
2153 goto load_err;
2154 return err;
2155
2156load_err:
2157 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
38679b5a
PP		 2158 return err;
2159}
2160
2161void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num)
2162{
2163 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2164 return;
2165
2166 mlx5_esw_offloads_rep_unload(esw, vport_num);
c7eddc60 2167 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
38679b5a
PP		 2168}
2169
ac004b83
RD		 2170#define ESW_OFFLOADS_DEVCOM_PAIR (0)
2171#define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2172
2173static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
2174 struct mlx5_eswitch *peer_esw)
2175{
2176 int err;
2177
2178 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev);
2179 if (err)
2180 return err;
2181
2182 return 0;
2183}
2184
2185static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw)
2186{
d956873f 2187#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
04de7dda 2188 mlx5e_tc_clean_fdb_peer_flows(esw);
d956873f 2189#endif
ac004b83
RD		 2190 esw_del_fdb_peer_miss_rules(esw);
2191}
2192
8463daf1
MG		 2193static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
2194 struct mlx5_eswitch *peer_esw,
2195 bool pair)
2196{
2197 struct mlx5_flow_root_namespace *peer_ns;
2198 struct mlx5_flow_root_namespace *ns;
2199 int err;
2200
2201 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
2202 ns = esw->dev->priv.steering->fdb_root_ns;
2203
2204 if (pair) {
2205 err = mlx5_flow_namespace_set_peer(ns, peer_ns);
2206 if (err)
2207 return err;
2208
e53e6655 2209 err = mlx5_flow_namespace_set_peer(peer_ns, ns);
8463daf1
MG		 2210 if (err) {
2211 mlx5_flow_namespace_set_peer(ns, NULL);
2212 return err;
2213 }
2214 } else {
2215 mlx5_flow_namespace_set_peer(ns, NULL);
2216 mlx5_flow_namespace_set_peer(peer_ns, NULL);
2217 }
2218
2219 return 0;
2220}
2221
ac004b83
RD		 2222static int mlx5_esw_offloads_devcom_event(int event,
2223 void *my_data,
2224 void *event_data)
2225{
2226 struct mlx5_eswitch *esw = my_data;
ac004b83 2227 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
8463daf1 2228 struct mlx5_eswitch *peer_esw = event_data;
ac004b83
RD		 2229 int err;
2230
2231 switch (event) {
2232 case ESW_OFFLOADS_DEVCOM_PAIR:
a5641cb5
JL		 2233 if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
2234 mlx5_eswitch_vport_match_metadata_enabled(peer_esw))
2235 break;
2236
8463daf1 2237 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true);
ac004b83
RD		 2238 if (err)
2239 goto err_out;
8463daf1
MG		 2240 err = mlx5_esw_offloads_pair(esw, peer_esw);
2241 if (err)
2242 goto err_peer;
ac004b83
RD		 2243
2244 err = mlx5_esw_offloads_pair(peer_esw, esw);
2245 if (err)
2246 goto err_pair;
2247
2248 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true);
2249 break;
2250
2251 case ESW_OFFLOADS_DEVCOM_UNPAIR:
2252 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS))
2253 break;
2254
2255 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false);
2256 mlx5_esw_offloads_unpair(peer_esw);
2257 mlx5_esw_offloads_unpair(esw);
8463daf1 2258 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
ac004b83
RD		 2259 break;
2260 }
2261
2262 return 0;
2263
2264err_pair:
2265 mlx5_esw_offloads_unpair(esw);
8463daf1
MG		 2266err_peer:
2267 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
ac004b83
RD		 2268err_out:
2269 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
2270 event, err);
2271 return err;
2272}
2273
2274static void esw_offloads_devcom_init(struct mlx5_eswitch *esw)
2275{
2276 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2277
04de7dda
RD		 2278 INIT_LIST_HEAD(&esw->offloads.peer_flows);
2279 mutex_init(&esw->offloads.peer_mutex);
2280
ac004b83
RD		 2281 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2282 return;
2283
2284 mlx5_devcom_register_component(devcom,
2285 MLX5_DEVCOM_ESW_OFFLOADS,
2286 mlx5_esw_offloads_devcom_event,
2287 esw);
2288
2289 mlx5_devcom_send_event(devcom,
2290 MLX5_DEVCOM_ESW_OFFLOADS,
2291 ESW_OFFLOADS_DEVCOM_PAIR, esw);
2292}
2293
2294static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
2295{
2296 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2297
2298 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2299 return;
2300
2301 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS,
2302 ESW_OFFLOADS_DEVCOM_UNPAIR, esw);
2303
2304 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS);
2305}
2306
92ab1eb3
JL		 2307static bool
2308esw_check_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
2309{
2310 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
2311 return false;
2312
2313 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
2314 MLX5_FDB_TO_VPORT_REG_C_0))
2315 return false;
2316
2317 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
2318 return false;
2319
2320 if (mlx5_core_is_ecpf_esw_manager(esw->dev) ||
2321 mlx5_ecpf_vport_exists(esw->dev))
2322 return false;
2323
2324 return true;
2325}
2326
133dcfc5
VP		 2327u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
2328{
7cd7becd 2329 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
2330 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 1;
2331 u32 pf_num;
133dcfc5
VP		 2332 int id;
2333
7cd7becd 2334 /* Only 4 bits of pf_num */
2335 pf_num = PCI_FUNC(esw->dev->pdev->devfn);
2336 if (pf_num > max_pf_num)
2337 return 0;
133dcfc5 2338
7cd7becd 2339 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
2340 /* Use only non-zero vport_id (1-4095) for all PF's */
2341 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 1, vport_end_ida, GFP_KERNEL);
2342 if (id < 0)
2343 return 0;
2344 id = (pf_num << ESW_VPORT_BITS) | id;
2345 return id;
133dcfc5
VP		 2346}
2347
2348void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
2349{
7cd7becd 2350 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
2351
2352 /* Metadata contains only 12 bits of actual ida id */
2353 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask);
133dcfc5
VP		 2354}
2355
2356static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
2357 struct mlx5_vport *vport)
2358{
133dcfc5
VP		 2359 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
2360 vport->metadata = vport->default_metadata;
2361 return vport->metadata ? 0 : -ENOSPC;
2362}
2363
2364static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
2365 struct mlx5_vport *vport)
2366{
406493a5 2367 if (!vport->default_metadata)
133dcfc5
VP		 2368 return;
2369
2370 WARN_ON(vport->metadata != vport->default_metadata);
2371 mlx5_esw_match_metadata_free(esw, vport->default_metadata);
2372}
2373
fc99c3d6
VP		 2374static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
2375{
2376 struct mlx5_vport *vport;
2377 int i;
2378
2379 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
2380 return;
2381
2382 mlx5_esw_for_all_vports_reverse(esw, i, vport)
2383 esw_offloads_vport_metadata_cleanup(esw, vport);
2384}
2385
2386static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
2387{
2388 struct mlx5_vport *vport;
2389 int err;
2390 int i;
2391
2392 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
2393 return 0;
2394
2395 mlx5_esw_for_all_vports(esw, i, vport) {
2396 err = esw_offloads_vport_metadata_setup(esw, vport);
2397 if (err)
2398 goto metadata_err;
2399 }
2400
2401 return 0;
2402
2403metadata_err:
2404 esw_offloads_metadata_uninit(esw);
2405 return err;
2406}
2407
748da30b 2408int
89a0f1fb
PP		 2409esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
2410 struct mlx5_vport *vport)
7445cfb1 2411{
7445cfb1
JL		 2412 int err;
2413
07bab950 2414 err = esw_acl_ingress_ofld_setup(esw, vport);
89a0f1fb 2415 if (err)
fc99c3d6 2416 return err;
7445cfb1 2417
2c40db2f
PP		 2418 err = esw_acl_egress_ofld_setup(esw, vport);
2419 if (err)
2420 goto egress_err;
07bab950
VP		 2421
2422 return 0;
2423
2424egress_err:
2425 esw_acl_ingress_ofld_cleanup(esw, vport);
89a0f1fb
PP		 2426 return err;
2427}
18486737 2428
748da30b 2429void
89a0f1fb
PP		 2430esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
2431 struct mlx5_vport *vport)
2432{
ea651a86 2433 esw_acl_egress_ofld_cleanup(vport);
07bab950 2434 esw_acl_ingress_ofld_cleanup(esw, vport);
89a0f1fb 2435}
7445cfb1 2436
748da30b 2437static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
7445cfb1
JL		 2438{
2439 struct mlx5_vport *vport;
18486737 2440
748da30b 2441 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
4e9a9ef7 2442 return esw_vport_create_offloads_acl_tables(esw, vport);
18486737
EB		 2443}
2444
748da30b 2445static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
18486737 2446{
786ef904 2447 struct mlx5_vport *vport;
7445cfb1 2448
748da30b
VP		 2449 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
2450 esw_vport_destroy_offloads_acl_tables(esw, vport);
18486737
EB		 2451}
2452
062f4bf4 2453static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
6ed1803a 2454{
34ca6535 2455 struct mlx5_esw_indir_table *indir;
6ed1803a
MB		 2456 int err;
2457
5c1d260e 2458 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
f8d1edda
PP		 2459 mutex_init(&esw->fdb_table.offloads.vports.lock);
2460 hash_init(esw->fdb_table.offloads.vports.table);
e52c2802 2461
34ca6535
VB		 2462 indir = mlx5_esw_indir_table_init();
2463 if (IS_ERR(indir)) {
2464 err = PTR_ERR(indir);
2465 goto create_indir_err;
2466 }
2467 esw->fdb_table.offloads.indir = indir;
2468
748da30b 2469 err = esw_create_uplink_offloads_acl_tables(esw);
7445cfb1 2470 if (err)
f8d1edda 2471 goto create_acl_err;
18486737 2472
8d6bd3c3 2473 err = esw_create_offloads_table(esw);
c930a3ad 2474 if (err)
11b717d6 2475 goto create_offloads_err;
c930a3ad 2476
11b717d6 2477 err = esw_create_restore_table(esw);
c930a3ad 2478 if (err)
11b717d6
PB		 2479 goto create_restore_err;
2480
0da3c12d 2481 err = esw_create_offloads_fdb_tables(esw);
11b717d6
PB		 2482 if (err)
2483 goto create_fdb_err;
c930a3ad 2484
8d6bd3c3 2485 err = esw_create_vport_rx_group(esw);
c930a3ad
OG		 2486 if (err)
2487 goto create_fg_err;
2488
2489 return 0;
2490
2491create_fg_err:
1967ce6e 2492 esw_destroy_offloads_fdb_tables(esw);
7445cfb1 2493create_fdb_err:
11b717d6
PB		 2494 esw_destroy_restore_table(esw);
2495create_restore_err:
2496 esw_destroy_offloads_table(esw);
2497create_offloads_err:
748da30b 2498 esw_destroy_uplink_offloads_acl_tables(esw);
f8d1edda 2499create_acl_err:
34ca6535
VB		 2500 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
2501create_indir_err:
f8d1edda 2502 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
c930a3ad
OG		 2503 return err;
2504}
2505
eca8cc38
BW		 2506static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
2507{
2508 esw_destroy_vport_rx_group(esw);
eca8cc38 2509 esw_destroy_offloads_fdb_tables(esw);
11b717d6
PB		 2510 esw_destroy_restore_table(esw);
2511 esw_destroy_offloads_table(esw);
748da30b 2512 esw_destroy_uplink_offloads_acl_tables(esw);
34ca6535 2513 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
f8d1edda 2514 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
eca8cc38
BW		 2515}
2516
7e736f9a
PP		 2517static void
2518esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
a3888f33 2519{
5ccf2770 2520 bool host_pf_disabled;
7e736f9a 2521 u16 new_num_vfs;
a3888f33 2522
7e736f9a
PP		 2523 new_num_vfs = MLX5_GET(query_esw_functions_out, out,
2524 host_params_context.host_num_of_vfs);
5ccf2770
BW		 2525 host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
2526 host_params_context.host_pf_disabled);
a3888f33 2527
7e736f9a
PP		 2528 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
2529 return;
a3888f33
BW		 2530
2531 /* Number of VFs can only change from "0 to x" or "x to 0". */
cd56f929 2532 if (esw->esw_funcs.num_vfs > 0) {
23bb50cf 2533 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
a3888f33 2534 } else {
7e736f9a 2535 int err;
a3888f33 2536
23bb50cf
BW		 2537 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
2538 MLX5_VPORT_UC_ADDR_CHANGE);
a3888f33 2539 if (err)
7e736f9a 2540 return;
a3888f33 2541 }
7e736f9a 2542 esw->esw_funcs.num_vfs = new_num_vfs;
a3888f33
BW		 2543}
2544
7e736f9a 2545static void esw_functions_changed_event_handler(struct work_struct *work)
ac35dcd6 2546{
7e736f9a
PP		 2547 struct mlx5_host_work *host_work;
2548 struct mlx5_eswitch *esw;
dd28087c 2549 const u32 *out;
ac35dcd6 2550
7e736f9a
PP		 2551 host_work = container_of(work, struct mlx5_host_work, work);
2552 esw = host_work->esw;
a3888f33 2553
dd28087c
PP		 2554 out = mlx5_esw_query_functions(esw->dev);
2555 if (IS_ERR(out))
7e736f9a 2556 goto out;
a3888f33 2557
7e736f9a 2558 esw_vfs_changed_event_handler(esw, out);
dd28087c 2559 kvfree(out);
a3888f33 2560out:
ac35dcd6
VP		 2561 kfree(host_work);
2562}
2563
16fff98a 2564int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
a3888f33 2565{
cd56f929 2566 struct mlx5_esw_functions *esw_funcs;
a3888f33 2567 struct mlx5_host_work *host_work;
a3888f33
BW		 2568 struct mlx5_eswitch *esw;
2569
2570 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
2571 if (!host_work)
2572 return NOTIFY_DONE;
2573
cd56f929
VP		 2574 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
2575 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
a3888f33
BW		 2576
2577 host_work->esw = esw;
2578
062f4bf4 2579 INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
a3888f33
BW		 2580 queue_work(esw->work_queue, &host_work->work);
2581
2582 return NOTIFY_OK;
2583}
2584
a53cf949
PP		 2585static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
2586{
2587 const u32 *query_host_out;
2588
2589 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
2590 return 0;
2591
2592 query_host_out = mlx5_esw_query_functions(esw->dev);
2593 if (IS_ERR(query_host_out))
2594 return PTR_ERR(query_host_out);
2595
2596 /* Mark non local controller with non zero controller number. */
2597 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
2598 host_params_context.host_number);
2599 kvfree(query_host_out);
2600 return 0;
2601}
2602
5896b972 2603int esw_offloads_enable(struct mlx5_eswitch *esw)
eca8cc38 2604{
3b83b6c2
DL		 2605 struct mlx5_vport *vport;
2606 int err, i;
eca8cc38 2607
9a64144d
MG		 2608 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat) &&
2609 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, decap))
2610 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_BASIC;
2611 else
2612 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE;
2613
2bb72e7e 2614 mutex_init(&esw->offloads.termtbl_mutex);
8463daf1 2615 mlx5_rdma_enable_roce(esw->dev);
eca8cc38 2616
a53cf949
PP		 2617 err = mlx5_esw_host_number_init(esw);
2618 if (err)
cd1ef966 2619 goto err_metadata;
a53cf949 2620
cd1ef966 2621 if (esw_check_vport_match_metadata_supported(esw))
4e9a9ef7
VP		 2622 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2623
fc99c3d6
VP		 2624 err = esw_offloads_metadata_init(esw);
2625 if (err)
2626 goto err_metadata;
2627
332bd3a5
PP		 2628 err = esw_set_passing_vport_metadata(esw, true);
2629 if (err)
2630 goto err_vport_metadata;
c1286050 2631
7983a675
PB		 2632 err = esw_offloads_steering_init(esw);
2633 if (err)
2634 goto err_steering_init;
2635
3b83b6c2
DL		 2636 /* Representor will control the vport link state */
2637 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
2638 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
2639
c2d7712c
BW		 2640 /* Uplink vport rep must load first. */
2641 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK);
925a6acc 2642 if (err)
c2d7712c 2643 goto err_uplink;
c1286050 2644
c2d7712c 2645 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE);
eca8cc38 2646 if (err)
c2d7712c 2647 goto err_vports;
eca8cc38
BW		 2648
2649 esw_offloads_devcom_init(esw);
a3888f33 2650
eca8cc38
BW		 2651 return 0;
2652
925a6acc 2653err_vports:
c2d7712c
BW		 2654 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
2655err_uplink:
7983a675 2656 esw_offloads_steering_cleanup(esw);
79949985
PP		 2657err_steering_init:
2658 esw_set_passing_vport_metadata(esw, false);
7983a675 2659err_vport_metadata:
fc99c3d6
VP		 2660 esw_offloads_metadata_uninit(esw);
2661err_metadata:
4e9a9ef7 2662 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
8463daf1 2663 mlx5_rdma_disable_roce(esw->dev);
2bb72e7e 2664 mutex_destroy(&esw->offloads.termtbl_mutex);
eca8cc38
BW		 2665 return err;
2666}
2667
db7ff19e
EB		 2668static int esw_offloads_stop(struct mlx5_eswitch *esw,
2669 struct netlink_ext_ack *extack)
c930a3ad 2670{
062f4bf4 2671 int err, err1;
c930a3ad 2672
8e0aa4bc
PP		 2673 mlx5_eswitch_disable_locked(esw, false);
2674 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY,
2675 MLX5_ESWITCH_IGNORE_NUM_VFS);
6c419ba8 2676 if (err) {
8c98ee77 2677 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
8e0aa4bc
PP		 2678 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS,
2679 MLX5_ESWITCH_IGNORE_NUM_VFS);
8c98ee77
EB		 2680 if (err1) {
2681 NL_SET_ERR_MSG_MOD(extack,
2682 "Failed setting eswitch back to offloads");
2683 }
6c419ba8 2684 }
c930a3ad
OG		 2685
2686 return err;
2687}
2688
5896b972 2689void esw_offloads_disable(struct mlx5_eswitch *esw)
c930a3ad 2690{
ac004b83 2691 esw_offloads_devcom_cleanup(esw);
5896b972 2692 mlx5_eswitch_disable_pf_vf_vports(esw);
c2d7712c 2693 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
332bd3a5 2694 esw_set_passing_vport_metadata(esw, false);
eca8cc38 2695 esw_offloads_steering_cleanup(esw);
fc99c3d6 2696 esw_offloads_metadata_uninit(esw);
4e9a9ef7 2697 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
8463daf1 2698 mlx5_rdma_disable_roce(esw->dev);
2bb72e7e 2699 mutex_destroy(&esw->offloads.termtbl_mutex);
9a64144d 2700 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE;
c930a3ad
OG		 2701}
2702
ef78618b 2703static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
c930a3ad
OG		 2704{
2705 switch (mode) {
2706 case DEVLINK_ESWITCH_MODE_LEGACY:
f6455de0 2707 *mlx5_mode = MLX5_ESWITCH_LEGACY;
c930a3ad
OG		 2708 break;
2709 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
f6455de0 2710 *mlx5_mode = MLX5_ESWITCH_OFFLOADS;
c930a3ad
OG		 2711 break;
2712 default:
2713 return -EINVAL;
2714 }
2715
2716 return 0;
2717}
2718
ef78618b
OG		 2719static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
2720{
2721 switch (mlx5_mode) {
f6455de0 2722 case MLX5_ESWITCH_LEGACY:
ef78618b
OG		 2723 *mode = DEVLINK_ESWITCH_MODE_LEGACY;
2724 break;
f6455de0 2725 case MLX5_ESWITCH_OFFLOADS:
ef78618b
OG		 2726 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
2727 break;
2728 default:
2729 return -EINVAL;
2730 }
2731
2732 return 0;
2733}
2734
bffaa916
RD		 2735static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
2736{
2737 switch (mode) {
2738 case DEVLINK_ESWITCH_INLINE_MODE_NONE:
2739 *mlx5_mode = MLX5_INLINE_MODE_NONE;
2740 break;
2741 case DEVLINK_ESWITCH_INLINE_MODE_LINK:
2742 *mlx5_mode = MLX5_INLINE_MODE_L2;
2743 break;
2744 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
2745 *mlx5_mode = MLX5_INLINE_MODE_IP;
2746 break;
2747 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
2748 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
2749 break;
2750 default:
2751 return -EINVAL;
2752 }
2753
2754 return 0;
2755}
2756
2757static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
2758{
2759 switch (mlx5_mode) {
2760 case MLX5_INLINE_MODE_NONE:
2761 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
2762 break;
2763 case MLX5_INLINE_MODE_L2:
2764 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
2765 break;
2766 case MLX5_INLINE_MODE_IP:
2767 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
2768 break;
2769 case MLX5_INLINE_MODE_TCP_UDP:
2770 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
2771 break;
2772 default:
2773 return -EINVAL;
2774 }
2775
2776 return 0;
2777}
2778
ae24432c
PP		 2779static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw)
2780{
2781 /* devlink commands in NONE eswitch mode are currently supported only
2782 * on ECPF.
2783 */
2784 return (esw->mode == MLX5_ESWITCH_NONE &&
2785 !mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0;
2786}
2787
db7ff19e
EB		 2788int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
2789 struct netlink_ext_ack *extack)
9d1cef19 2790{
9d1cef19 2791 u16 cur_mlx5_mode, mlx5_mode = 0;
bd939753 2792 struct mlx5_eswitch *esw;
ea2128fd 2793 int err = 0;
9d1cef19 2794
bd939753
PP		 2795 esw = mlx5_devlink_eswitch_get(devlink);
2796 if (IS_ERR(esw))
2797 return PTR_ERR(esw);
9d1cef19 2798
ef78618b 2799 if (esw_mode_from_devlink(mode, &mlx5_mode))
c930a3ad
OG		 2800 return -EINVAL;
2801
8e0aa4bc 2802 mutex_lock(&esw->mode_lock);
8e0aa4bc 2803 cur_mlx5_mode = esw->mode;
c930a3ad 2804 if (cur_mlx5_mode == mlx5_mode)
8e0aa4bc 2805 goto unlock;
c930a3ad
OG		 2806
2807 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV)
8e0aa4bc 2808 err = esw_offloads_start(esw, extack);
c930a3ad 2809 else if (mode == DEVLINK_ESWITCH_MODE_LEGACY)
8e0aa4bc 2810 err = esw_offloads_stop(esw, extack);
c930a3ad 2811 else
8e0aa4bc
PP		 2812 err = -EINVAL;
2813
2814unlock:
2815 mutex_unlock(&esw->mode_lock);
2816 return err;
feae9087
OG		 2817}
2818
2819int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
2820{
bd939753 2821 struct mlx5_eswitch *esw;
9d1cef19 2822 int err;
c930a3ad 2823
bd939753
PP		 2824 esw = mlx5_devlink_eswitch_get(devlink);
2825 if (IS_ERR(esw))
2826 return PTR_ERR(esw);
c930a3ad 2827
8e0aa4bc 2828 mutex_lock(&esw->mode_lock);
bd939753 2829 err = eswitch_devlink_esw_mode_check(esw);
ae24432c 2830 if (err)
8e0aa4bc 2831 goto unlock;
ae24432c 2832
8e0aa4bc
PP		 2833 err = esw_mode_to_devlink(esw->mode, mode);
2834unlock:
2835 mutex_unlock(&esw->mode_lock);
2836 return err;
feae9087 2837}
127ea380 2838
db7ff19e
EB		 2839int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
2840 struct netlink_ext_ack *extack)
bffaa916
RD		 2841{
2842 struct mlx5_core_dev *dev = devlink_priv(devlink);
db68cc56 2843 int err, vport, num_vport;
bd939753 2844 struct mlx5_eswitch *esw;
bffaa916
RD		 2845 u8 mlx5_mode;
2846
bd939753
PP		 2847 esw = mlx5_devlink_eswitch_get(devlink);
2848 if (IS_ERR(esw))
2849 return PTR_ERR(esw);
bffaa916 2850
8e0aa4bc 2851 mutex_lock(&esw->mode_lock);
ae24432c
PP		 2852 err = eswitch_devlink_esw_mode_check(esw);
2853 if (err)
8e0aa4bc 2854 goto out;
ae24432c 2855
c415f704
OG		 2856 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2857 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2858 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE)
8e0aa4bc 2859 goto out;
c8b838d1 2860 fallthrough;
c415f704 2861 case MLX5_CAP_INLINE_MODE_L2:
8c98ee77 2862 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
8e0aa4bc
PP		 2863 err = -EOPNOTSUPP;
2864 goto out;
c415f704
OG		 2865 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2866 break;
2867 }
bffaa916 2868
525e84be 2869 if (atomic64_read(&esw->offloads.num_flows) > 0) {
8c98ee77
EB		 2870 NL_SET_ERR_MSG_MOD(extack,
2871 "Can't set inline mode when flows are configured");
8e0aa4bc
PP		 2872 err = -EOPNOTSUPP;
2873 goto out;
375f51e2
RD		 2874 }
2875
bffaa916
RD		 2876 err = esw_inline_mode_from_devlink(mode, &mlx5_mode);
2877 if (err)
2878 goto out;
2879
411ec9e0 2880 mlx5_esw_for_each_host_func_vport(esw, vport, esw->esw_funcs.num_vfs) {
bffaa916
RD		 2881 err = mlx5_modify_nic_vport_min_inline(dev, vport, mlx5_mode);
2882 if (err) {
8c98ee77
EB		 2883 NL_SET_ERR_MSG_MOD(extack,
2884 "Failed to set min inline on vport");
bffaa916
RD		 2885 goto revert_inline_mode;
2886 }
2887 }
2888
2889 esw->offloads.inline_mode = mlx5_mode;
8e0aa4bc 2890 mutex_unlock(&esw->mode_lock);
bffaa916
RD		 2891 return 0;
2892
2893revert_inline_mode:
db68cc56 2894 num_vport = --vport;
411ec9e0 2895 mlx5_esw_for_each_host_func_vport_reverse(esw, vport, num_vport)
bffaa916
RD		 2896 mlx5_modify_nic_vport_min_inline(dev,
2897 vport,
2898 esw->offloads.inline_mode);
2899out:
8e0aa4bc 2900 mutex_unlock(&esw->mode_lock);
bffaa916
RD		 2901 return err;
2902}
2903
2904int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
2905{
bd939753 2906 struct mlx5_eswitch *esw;
9d1cef19 2907 int err;
bffaa916 2908
bd939753
PP		 2909 esw = mlx5_devlink_eswitch_get(devlink);
2910 if (IS_ERR(esw))
2911 return PTR_ERR(esw);
bffaa916 2912
8e0aa4bc 2913 mutex_lock(&esw->mode_lock);
ae24432c
PP		 2914 err = eswitch_devlink_esw_mode_check(esw);
2915 if (err)
8e0aa4bc 2916 goto unlock;
ae24432c 2917
8e0aa4bc
PP		 2918 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
2919unlock:
2920 mutex_unlock(&esw->mode_lock);
2921 return err;
bffaa916
RD		 2922}
2923
98fdbea5
LR		 2924int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
2925 enum devlink_eswitch_encap_mode encap,
db7ff19e 2926 struct netlink_ext_ack *extack)
7768d197
RD		 2927{
2928 struct mlx5_core_dev *dev = devlink_priv(devlink);
bd939753 2929 struct mlx5_eswitch *esw;
7768d197
RD		 2930 int err;
2931
bd939753
PP		 2932 esw = mlx5_devlink_eswitch_get(devlink);
2933 if (IS_ERR(esw))
2934 return PTR_ERR(esw);
7768d197 2935
8e0aa4bc 2936 mutex_lock(&esw->mode_lock);
ae24432c
PP		 2937 err = eswitch_devlink_esw_mode_check(esw);
2938 if (err)
8e0aa4bc 2939 goto unlock;
ae24432c 2940
7768d197 2941 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
60786f09 2942 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
8e0aa4bc
PP		 2943 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
2944 err = -EOPNOTSUPP;
2945 goto unlock;
2946 }
7768d197 2947
8e0aa4bc
PP		 2948 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
2949 err = -EOPNOTSUPP;
2950 goto unlock;
2951 }
7768d197 2952
f6455de0 2953 if (esw->mode == MLX5_ESWITCH_LEGACY) {
7768d197 2954 esw->offloads.encap = encap;
8e0aa4bc 2955 goto unlock;
7768d197
RD		 2956 }
2957
2958 if (esw->offloads.encap == encap)
8e0aa4bc 2959 goto unlock;
7768d197 2960
525e84be 2961 if (atomic64_read(&esw->offloads.num_flows) > 0) {
8c98ee77
EB		 2962 NL_SET_ERR_MSG_MOD(extack,
2963 "Can't set encapsulation when flows are configured");
8e0aa4bc
PP		 2964 err = -EOPNOTSUPP;
2965 goto unlock;
7768d197
RD		 2966 }
2967
e52c2802 2968 esw_destroy_offloads_fdb_tables(esw);
7768d197
RD		 2969
2970 esw->offloads.encap = encap;
e52c2802 2971
0da3c12d 2972 err = esw_create_offloads_fdb_tables(esw);
e52c2802 2973
7768d197 2974 if (err) {
8c98ee77
EB		 2975 NL_SET_ERR_MSG_MOD(extack,
2976 "Failed re-creating fast FDB table");
7768d197 2977 esw->offloads.encap = !encap;
0da3c12d 2978 (void)esw_create_offloads_fdb_tables(esw);
7768d197 2979 }
e52c2802 2980
8e0aa4bc
PP		 2981unlock:
2982 mutex_unlock(&esw->mode_lock);
7768d197
RD		 2983 return err;
2984}
2985
98fdbea5
LR		 2986int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
2987 enum devlink_eswitch_encap_mode *encap)
7768d197 2988{
bd939753 2989 struct mlx5_eswitch *esw;
9d1cef19 2990 int err;
7768d197 2991
bd939753
PP		 2992 esw = mlx5_devlink_eswitch_get(devlink);
2993 if (IS_ERR(esw))
2994 return PTR_ERR(esw);
2995
7768d197 2996
8e0aa4bc 2997 mutex_lock(&esw->mode_lock);
ae24432c
PP		 2998 err = eswitch_devlink_esw_mode_check(esw);
2999 if (err)
8e0aa4bc 3000 goto unlock;
ae24432c 3001
7768d197 3002 *encap = esw->offloads.encap;
8e0aa4bc
PP		 3003unlock:
3004 mutex_unlock(&esw->mode_lock);
7768d197
RD		 3005 return 0;
3006}
3007
c2d7712c
BW		 3008static bool
3009mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
3010{
3011 /* Currently, only ECPF based device has representor for host PF. */
3012 if (vport_num == MLX5_VPORT_PF &&
3013 !mlx5_core_is_ecpf_esw_manager(esw->dev))
3014 return false;
3015
3016 if (vport_num == MLX5_VPORT_ECPF &&
3017 !mlx5_ecpf_vport_exists(esw->dev))
3018 return false;
3019
3020 return true;
3021}
3022
f8e8fa02 3023void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
8693115a 3024 const struct mlx5_eswitch_rep_ops *ops,
f8e8fa02 3025 u8 rep_type)
127ea380 3026{
8693115a 3027 struct mlx5_eswitch_rep_data *rep_data;
f8e8fa02
BW		 3028 struct mlx5_eswitch_rep *rep;
3029 int i;
9deb2241 3030
8693115a 3031 esw->offloads.rep_ops[rep_type] = ops;
f8e8fa02 3032 mlx5_esw_for_all_reps(esw, i, rep) {
c2d7712c
BW		 3033 if (likely(mlx5_eswitch_vport_has_rep(esw, i))) {
3034 rep_data = &rep->rep_data[rep_type];
3035 atomic_set(&rep_data->state, REP_REGISTERED);
3036 }
f8e8fa02 3037 }
127ea380 3038}
f8e8fa02 3039EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
127ea380 3040
f8e8fa02 3041void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
127ea380 3042{
cb67b832 3043 struct mlx5_eswitch_rep *rep;
f8e8fa02 3044 int i;
cb67b832 3045
f6455de0 3046 if (esw->mode == MLX5_ESWITCH_OFFLOADS)
062f4bf4 3047 __unload_reps_all_vport(esw, rep_type);
127ea380 3048
f8e8fa02 3049 mlx5_esw_for_all_reps(esw, i, rep)
8693115a 3050 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
127ea380 3051}
f8e8fa02 3052EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
726293f1 3053
a4b97ab4 3054void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
726293f1 3055{
726293f1
HHZ		 3056 struct mlx5_eswitch_rep *rep;
3057
879c8f84 3058 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
8693115a 3059 return rep->rep_data[rep_type].priv;
726293f1 3060}
22215908
MB		 3061
3062void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
02f3afd9 3063 u16 vport,
22215908
MB		 3064 u8 rep_type)
3065{
22215908
MB		 3066 struct mlx5_eswitch_rep *rep;
3067
879c8f84 3068 rep = mlx5_eswitch_get_rep(esw, vport);
22215908 3069
8693115a
PP		 3070 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
3071 esw->offloads.rep_ops[rep_type]->get_proto_dev)
3072 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
22215908
MB		 3073 return NULL;
3074}
57cbd893 3075EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
22215908
MB		 3076
3077void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
3078{
879c8f84 3079 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
22215908 3080}
57cbd893
MB		 3081EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
3082
3083struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
02f3afd9 3084 u16 vport)
57cbd893 3085{
879c8f84 3086 return mlx5_eswitch_get_rep(esw, vport);
57cbd893
MB		 3087}
3088EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
91d6291c
PP		 3089
3090bool mlx5_eswitch_is_vf_vport(const struct mlx5_eswitch *esw, u16 vport_num)
3091{
3092 return vport_num >= MLX5_VPORT_FIRST_VF &&
3093 vport_num <= esw->dev->priv.sriov.max_vfs;
3094}
7445cfb1 3095
5b7cb745
PB		 3096bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
3097{
3098 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
3099}
3100EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
3101
7445cfb1
JL		 3102bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
3103{
3104 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
3105}
3106EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
3107
0f0d3827 3108u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
7445cfb1
JL		 3109 u16 vport_num)
3110{
133dcfc5 3111 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
0f0d3827 3112
133dcfc5
VP		 3113 if (WARN_ON_ONCE(IS_ERR(vport)))
3114 return 0;
0f0d3827 3115
133dcfc5 3116 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
7445cfb1
JL		 3117}
3118EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
d970812b
PP		 3119
3120int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_port *dl_port,
3121 u16 vport_num, u32 sfnum)
3122{
3123 int err;
3124
3125 err = mlx5_esw_vport_enable(esw, vport_num, MLX5_VPORT_UC_ADDR_CHANGE);
3126 if (err)
3127 return err;
3128
3129 err = mlx5_esw_devlink_sf_port_register(esw, dl_port, vport_num, sfnum);
3130 if (err)
3131 goto devlink_err;
3132
3133 err = mlx5_esw_offloads_rep_load(esw, vport_num);
3134 if (err)
3135 goto rep_err;
3136 return 0;
3137
3138rep_err:
3139 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3140devlink_err:
3141 mlx5_esw_vport_disable(esw, vport_num);
3142 return err;
3143}
3144
3145void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num)
3146{
3147 mlx5_esw_offloads_rep_unload(esw, vport_num);
3148 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3149 mlx5_esw_vport_disable(esw, vport_num);
3150}
84ae9c1f
VB		 3151
3152static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id)
3153{
3154 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
3155 void *query_ctx;
3156 void *hca_caps;
3157 int err;
3158
3159 *vhca_id = 0;
3160 if (mlx5_esw_is_manager_vport(esw, vport_num) ||
3161 !MLX5_CAP_GEN(esw->dev, vhca_resource_manager))
3162 return -EPERM;
3163
3164 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
3165 if (!query_ctx)
3166 return -ENOMEM;
3167
3168 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx);
3169 if (err)
3170 goto out_free;
3171
3172 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
3173 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
3174
3175out_free:
3176 kfree(query_ctx);
3177 return err;
3178}
3179
3180int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num)
3181{
3182 u16 *old_entry, *vhca_map_entry, vhca_id;
3183 int err;
3184
3185 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3186 if (err) {
3187 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
3188 vport_num, err);
3189 return err;
3190 }
3191
3192 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL);
3193 if (!vhca_map_entry)
3194 return -ENOMEM;
3195
3196 *vhca_map_entry = vport_num;
3197 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL);
3198 if (xa_is_err(old_entry)) {
3199 kfree(vhca_map_entry);
3200 return xa_err(old_entry);
3201 }
3202 kfree(old_entry);
3203 return 0;
3204}
3205
3206void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num)
3207{
3208 u16 *vhca_map_entry, vhca_id;
3209 int err;
3210
3211 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3212 if (err)
3213 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
3214 vport_num, err);
3215
3216 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id);
3217 kfree(vhca_map_entry);
3218}
3219
3220int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
3221{
3222 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id);
3223
3224 if (!res)
3225 return -ENOENT;
3226
3227 *vport_num = *res;
3228 return 0;
3229}
10742efc
VB		 3230
3231u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
3232 u16 vport_num)
3233{
3234 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
3235
3236 if (WARN_ON_ONCE(IS_ERR(vport)))
3237 return 0;
3238
3239 return vport->metadata;
3240}
3241EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
Linux 6.x block layer and io_uring tree(s)